Fix requirements for enzyme 0.3

Add the .ass subtitle extension

.gitignore

@@ -20,19 +20,24 @@ pip-log.txt
 .coverage
 .tox
 
-#Translations
+# Translations
 *.mo
 
-#Mr Developer
+# Mr Developer
 .mr.developer.cfg
 
-#Pydev
+# Pydev
 .project
 .pydevproject
 .settings
 
-#Rope
+# Rope
 .ropeproject
 
-#Sphinx
+# Sphinx
 docs/_build
+
+# Subliminal unittests
+tests/*.srt
+tests/*_files
+tests/*_cache

NEWS.rst

@@ -1,6 +1,48 @@
 News
 ====
 
+0.6.4
+-----
+**release date:** 2013-05-19
+
+* Fix requirements due to enzyme 0.3
+
+0.6.3
+-----
+**release date:** 2013-01-17
+
+* Fix requirements due to requests 1.0
+
+0.6.2
+-----
+**release date:** 2012-09-15
+
+* Fix BierDopje
+* Fix Addic7ed
+* Fix SubsWiki
+* Fix missing enzyme import
+* Add Catalan and Galician languages to Addic7ed
+* Add possible services in help message of the CLI
+* Allow existing filenames to be passed without the ./ prefix
+
+0.6.1
+-----
+**release date:** 2012-06-24
+
+* Fix subtitle release name in BierDopje
+* Fix subtitles being downloaded multiple times
+* Add Chinese support to TvSubtitles
+* Fix encoding issues
+* Fix single download subtitles without the force option
+* Add Spanish (Latin America) exception to Addic7ed
+* Fix group_by_video when a list entry has None as subtitles
+* Add support for Galician language in Subtitulos
+* Add an integrity check after subtitles download for Addic7ed
+* Add error handling for if not strict in Language
+* Fix TheSubDB hash method to return None if the file is too small
+* Fix guessit.Language in Video.scan
+* Fix language detection of subtitles
+
 0.6.0
 -----
 **release date:** 2012-06-16

README.rst

@@ -17,7 +17,6 @@ Multiple subtitles services are available:
 * Addic7ed
 * BierDopje
 * OpenSubtitles
-* Podnapisi
 * SubsWiki
 * Subtitulos
 * TheSubDB

docs/index.rst

@@ -21,7 +21,6 @@ Multiple subtitles services are available:
 * Addic7ed
 * BierDopje
 * OpenSubtitles
-* Podnapisi
 * SubsWiki
 * Subtitulos
 * TheSubDB

requirements.txt

@@ -1,5 +1,5 @@
 beautifulsoup4>=4.0
 guessit>=0.4.1
-requests
-enzyme>=0.1
+requests<1.0
+enzyme<0.3
 html5lib

scripts/subliminal

@@ -28,7 +28,7 @@ import sys
 def main():
     parser = argparse.ArgumentParser(description='Subtitles, faster than your thoughts')
     parser.add_argument('-l', '--language', action='append', dest='languages', help='wanted language (ISO 639-1)', metavar='LG')
-    parser.add_argument('-s', '--service', action='append', dest='services', help='service to use', metavar='NAME')
+    parser.add_argument('-s', '--service', action='append', dest='services', help='service to use (%s)' % ', '.join(subliminal.core.filter_services(subliminal.SERVICES)), metavar='NAME')
     parser.add_argument('-m', '--multi', action='store_true', help='download multiple subtitle languages')
     parser.add_argument('-f', '--force', action='store_true', help='replace existing subtitle file')
     parser.add_argument('-w', '--workers', action='store', help='use N threads (default: %(default)s)', metavar='N', type=int, default=4)

setup.py

@@ -24,7 +24,7 @@ from setuptools import setup, find_packages
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
-required = ['beautifulsoup4 >= 4.0', 'guessit >= 0.4.1', 'requests', 'enzyme >= 0.1', 'html5lib']
+required = ['beautifulsoup4 >= 4.0', 'guessit >= 0.4.1', 'requests < 1.0', 'enzyme < 0.3', 'html5lib']
 if sys.hexversion < 0x20700f0:
     required.append('argparse >= 1.1')
 

subliminal/api.py

@@ -81,6 +81,11 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
     :return: downloaded subtitles
     :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
 
+    .. note::
+
+        If you use ``multi=True``, :data:`~subliminal.core.LANGUAGE_INDEX` has to be the first item of the ``order`` list
+        or you might get unexpected results.
+
     """
     services = services or SERVICES
     languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
@@ -92,7 +97,7 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
         subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
     results = []
     service_instances = {}
-    tasks = create_download_tasks(subtitles_by_video, multi)
+    tasks = create_download_tasks(subtitles_by_video, languages, multi)
     for task in tasks:
         try:
             result = consume_task(task, service_instances)

subliminal/async.py

@@ -134,7 +134,7 @@ class Pool(object):
         subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
         for video, subtitles in subtitles_by_video.iteritems():
             subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
-        tasks = create_download_tasks(subtitles_by_video, multi)
+        tasks = create_download_tasks(subtitles_by_video, languages, multi)
         for task in tasks:
             self.tasks.put(task)
         self.join()

subliminal/core.py

@@ -20,6 +20,7 @@ from .services import ServiceConfig
 from .tasks import DownloadTask, ListTask
 from .utils import get_keywords
 from .videos import Episode, Movie, scan
+from .language import Language
 from collections import defaultdict
 from itertools import groupby
 import bs4
@@ -66,7 +67,7 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
             if not wanted_languages:
                 logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
                 continue
-        if not force and not multi and None in detected_languages:
+        if not force and not multi and Language('Undetermined') in detected_languages:
             logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
             continue
         logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
@@ -81,13 +82,13 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
     return tasks
 
 
-def create_download_tasks(subtitles_by_video, multi):
+def create_download_tasks(subtitles_by_video, languages, multi):
     """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
 
-    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
+    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results with ordered subtitles
     :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
-    :param order: preferred order for subtitles sorting
-    :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+    :param languages: languages in preferred order
+    :type languages: :class:`~subliminal.language.language_list`
     :param bool multi: download multiple languages for the same video
     :return: the created tasks
     :rtype: list of :class:`~subliminal.tasks.DownloadTask`
@@ -102,7 +103,7 @@ def create_download_tasks(subtitles_by_video, multi):
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
             continue
-        for _, by_language in groupby(subtitles, lambda s: s.language):
+        for _, by_language in groupby(subtitles, lambda s: languages.index(s.language)):
             task = DownloadTask(video, list(by_language))
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
@@ -157,6 +158,7 @@ def matching_confidence(video, subtitle):
     guess = guessit.guess_file_info(subtitle.release, 'autodetect')
     video_keywords = get_keywords(video.guess)
     subtitle_keywords = get_keywords(guess) | subtitle.keywords
+    logger.debug(u'Video keywords %r - Subtitle keywords %r' % (video_keywords, subtitle_keywords))
     replacement = {'keywords': len(video_keywords & subtitle_keywords)}
     if isinstance(video, Episode):
         replacement.update({'series': 0, 'season': 0, 'episode': 0})
@@ -179,8 +181,11 @@ def matching_confidence(video, subtitle):
             if 'year' in guess and guess['year'] == video.year:
                 replacement['year'] = 1
     else:
-        return 0
+        logger.debug(u'Not able to compute confidence for %r' % video)
+        return 0.0
+    logger.debug(u'Found %r' % replacement)
     confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
+    logger.info(u'Computed confidence %.4f for %r and %r' % (confidence, video, subtitle))
     return confidence
 
 
@@ -248,7 +253,7 @@ def group_by_video(list_results):
     """
     result = defaultdict(list)
     for video, subtitles in list_results:
-        result[video] += subtitles
+        result[video] += subtitles or []
     return result
 
 

subliminal/infos.py

@@ -15,4 +15,4 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.6.0'
+__version__ = '0.6.4'

subliminal/language.py

@@ -17,6 +17,10 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .utils import to_unicode
 import re
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 COUNTRIES = [('AF', 'AFG', '004', u'Afghanistan'),
@@ -838,15 +842,26 @@ class Language(object):
         if isinstance(country, Country):
             self.country = country
         elif isinstance(country, basestring):
-            self.country = Country(country, countries)
+            try:
+                self.country = Country(country, countries)
+            except ValueError:
+                logger.warning(u'Country %s could not be identified' % country)
+                if strict:
+                    raise
 
         # Language + Country format
         #TODO: Improve this part
-        for regexp in [r.match(language) for r in self.with_country_regexps]:
-            if regexp:
-                language = regexp.group(1)
-                self.country = Country(regexp.group(2), countries)
-                break
+        if country is None:
+            for regexp in [r.match(language) for r in self.with_country_regexps]:
+                if regexp:
+                    language = regexp.group(1)
+                    try:
+                        self.country = Country(regexp.group(2), countries)
+                    except ValueError:
+                        logger.warning(u'Country %s could not be identified' % country)
+                        if strict:
+                            raise
+                    break
 
         # Try to find the language
         language = to_unicode(language.strip().lower())

subliminal/services/__init__.py

@@ -160,6 +160,7 @@ class ServiceBase(object):
     def download(self, subtitle):
         """Download a subtitle"""
         self.download_file(subtitle.link, subtitle.path)
+        return subtitle
 
     @classmethod
     def check_validity(cls, video, languages):
@@ -188,17 +189,17 @@ class ServiceBase(object):
         :param string filepath: destination path
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
             with open(filepath, 'wb') as f:
                 f.write(r.content)
         except Exception as e:
-            logger.error(u'Download %s failed: %s' % (url, e))
+            logger.error(u'Download failed: %s' % e)
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
-        logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+        logger.debug(u'Download finished')
 
     def download_zip_file(self, url, filepath):
         """Attempt to download a zip file and extract any subtitle file from it, if any.
@@ -208,7 +209,7 @@ class ServiceBase(object):
         :param string filepath: destination path for the subtitle
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             zippath = filepath + '.zip'
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
@@ -218,17 +219,15 @@ class ServiceBase(object):
                 # TODO: could check if maybe we already have a text file and
                 # download it directly
                 raise DownloadFailedError('Downloaded file is not a zip file')
-            zipsub = zipfile.ZipFile(zippath)
-            for subfile in zipsub.namelist():
-                if os.path.splitext(subfile)[1] in EXTENSIONS:
-                    open(filepath, 'w').write(zipsub.open(subfile).read())
-                    break
-            else:
-                logger.debug(u'No subtitles found in zip file')
-                raise DownloadFailedError('No subtitles found in zip file')
+            with zipfile.ZipFile(zippath) as zipsub:
+                for subfile in zipsub.namelist():
+                    if os.path.splitext(subfile)[1] in EXTENSIONS:
+                        with open(filepath, 'w') as f:
+                            f.write(zipsub.open(subfile).read())
+                        break
+                else:
+                    raise DownloadFailedError('No subtitles found in zip file')
             os.remove(zippath)
-            logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
-            return
         except Exception as e:
             logger.error(u'Download %s failed: %s' % (url, e))
             if os.path.exists(zippath):
@@ -236,6 +235,7 @@ class ServiceBase(object):
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
 
 
 class ServiceConfig(object):

subliminal/services/addic7ed.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2012 Olivier Leveau <olifozzy@gmail.com>
+# Copyright 2012 Antoine Bertin <diaoulael@gmail.com>
 #
 # This file is part of subliminal.
 #
@@ -17,108 +18,46 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
 from ..cache import cachedmethod
+from ..exceptions import DownloadFailedError
 from ..language import Language, language_set
 from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords
+from ..utils import get_keywords, split_keyword
 from ..videos import Episode
 from bs4 import BeautifulSoup
 import logging
+import os
 import re
 
 
 logger = logging.getLogger(__name__)
 
 
-def match(pattern, string):
-    try:
-        return re.search(pattern, string).group(1)
-    except AttributeError:
-        logger.debug(u'Could not match %r on %r' % (pattern, string))
-        return None
-
-
-def matches(pattern, string):
-    try:
-        return re.search(pattern, string).group(1, 2)
-    except AttributeError:
-        logger.debug(u'Could not match %r on %r' % (pattern, string))
-        return None
-
-
 class Addic7ed(ServiceBase):
     server_url = 'http://www.addic7ed.com'
     api_based = False
     #TODO: Complete this
-    languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
-                              'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
-    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre')}
+    languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'gl', 'he', 'hr', 'hu',
+                              'it', 'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
+    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre'),
+                    'Spanish (Latin America)': Language('spa'), 'Galego': Language('glg'),
+                    u'Català': Language('cat')}
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
 
     @cachedmethod
-    def get_likely_series_id(self, name):
+    def get_series_id(self, name):
+        """Get the show page and cache every show found in it"""
         r = self.session.get('%s/shows.php' % self.server_url)
         soup = BeautifulSoup(r.content, self.required_features)
-        for elem in soup.find_all('h3'):
-            show_name = elem.a.text.lower()
-            show_id = int(match('show/([0-9]+)', elem.a['href']))
-            # we could just return the id of the queried show, but as we
-            # already downloaded the whole page we might as well fill in the
-            # information for all the shows
-            self.cache_for(self.get_likely_series_id, args=(show_name,), result=show_id)
-        return self.cached_value(self.get_likely_series_id, args=(name,))
-
-    @cachedmethod
-    def get_episode_url(self, series_id, season, number):
-        """Get the Addic7ed id for the given episode. Raises KeyError if none
-        could be found
-
-        """
-        # download the page of the show, contains ids for all episodes all seasons
-        r = self.session.get('%s/show/%d' % (self.server_url, series_id))
-        soup = BeautifulSoup(r.content, self.required_features)
-        form = soup.find('form', attrs={'name': 'multidl'})
-        for table in form.find_all('table'):
-            for row in table.find_all('tr'):
-                cell = row.find('td', 'MultiDldS')
-                if not cell:
-                    continue
-                m = matches('/serie/.+/([0-9]+)/([0-9]+)/', cell.a['href'])
-                if not m:
-                    continue
-                episode_url = cell.a['href']
-                season_number = int(m[0])
-                episode_number = int(m[1])
-                # we could just return the url of the queried episode, but as we
-                # already downloaded the whole page we might as well fill in the
-                # information for all the episodes of the show
-                self.cache_for(self.get_episode_url, args=(series_id, season_number, episode_number), result=episode_url)
-        # raises KeyError if not found
-        return self.cached_value(self.get_episode_url, args=(series_id, season, number))
-
-    # Do not cache this method in order to always check for the most recent
-    # subtitles
-    def get_sub_urls(self, episode_url):
-        suburls = []
-        r = self.session.get('%s/%s' % (self.server_url, episode_url))
-        epsoup = BeautifulSoup(r.content, self.required_features)
-        for releaseTable in epsoup.find_all('table', 'tabel95'):
-            releaseRow = releaseTable.find('td', 'NewsTitle')
-            if not releaseRow:
+        for html_series in soup.select('h3 > a'):
+            series_name = html_series.text.lower()
+            match = re.search('show/([0-9]+)', html_series['href'])
+            if match is None:
                 continue
-            release = releaseRow.text.strip()
-            for row in releaseTable.find_all('tr'):
-                link = row.find('a', 'buttonDownload')
-                if not link:
-                    continue
-                if 'href' not in link.attrs or not (link['href'].startswith('/original') or link['href'].startswith('/updated')):
-                    continue
-                suburl = link['href']
-                lang = self.get_language(row.find('td', 'language').text.strip())
-                result = {'suburl': suburl, 'language': lang, 'release': release}
-                suburls.append(result)
-        return suburls
+            series_id = int(match.group(1))
+            self.cache_for(self.get_series_id, args=(series_name,), result=series_id)
+        return self.cached_value(self.get_series_id, args=(name,))
 
     def list_checked(self, video, languages):
         return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
@@ -127,28 +66,55 @@ class Addic7ed(ServiceBase):
         logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
         self.init_cache()
         try:
-            sid = self.get_likely_series_id(series.lower())
+            series_id = self.get_series_id(series.lower())
         except KeyError:
             logger.debug(u'Could not find series id for %s' % series)
             return []
-        try:
-            ep_url = self.get_episode_url(sid, season, episode)
-        except KeyError:
-            logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
-            return []
-        suburls = self.get_sub_urls(ep_url)
-        # filter the subtitles with our queried languages
+        r = self.session.get('%s/show/%d&season=%d' % (self.server_url, series_id, season))
+        soup = BeautifulSoup(r.content, self.required_features)
         subtitles = []
-        for suburl in suburls:
-            language = suburl['language']
-            if language not in languages:
+        for row in soup('tr', {'class': 'epeven completed'}):
+            cells = row('td')
+            if int(cells[0].text.strip()) != season or int(cells[1].text.strip()) != episode:
                 continue
-            path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/%s' % (self.server_url, suburl['suburl']),
-                                      keywords=[suburl['release']])
+            if cells[6].text.strip():
+                logger.debug(u'Skipping hearing impaired')
+                continue
+            sub_status = cells[5].text.strip()
+            if sub_status != 'Completed':
+                logger.debug(u'Wrong subtitle status %s' % sub_status)
+                continue
+            sub_language = self.get_language(cells[3].text.strip())
+            if sub_language not in languages:
+                logger.debug(u'Language %r not in wanted languages %r' % (sub_language, languages))
+                continue
+            sub_keywords = split_keyword(cells[4].text.strip().lower())
+            #TODO: Maybe allow empty keywords here? (same in Subtitulos)
+            if not keywords & sub_keywords:
+                logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
+                continue
+            sub_link = '%s/%s' % (self.server_url, cells[9].a['href'])
+            sub_path = get_subtitle_path(filepath, sub_language, self.config.multi)
+            subtitle = ResultSubtitle(sub_path, sub_language, self.__class__.__name__.lower(), sub_link, keywords=sub_keywords)
             subtitles.append(subtitle)
         return subtitles
 
+    def download(self, subtitle):
+        logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path))
+        try:
+            r = self.session.get(subtitle.link, headers={'Referer': subtitle.link, 'User-Agent': self.user_agent})
+            soup = BeautifulSoup(r.content, self.required_features)
+            if soup.title is not None and u'Addic7ed.com' in soup.title.text.strip():
+                raise DownloadFailedError('Download limit exceeded')
+            with open(subtitle.path, 'wb') as f:
+                f.write(r.content)
+        except Exception as e:
+            logger.error(u'Download failed: %s' % e)
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
+        return subtitle
+
 
 Service = Addic7ed

subliminal/services/bierdopje.py

@@ -19,7 +19,7 @@ from . import ServiceBase
 from ..cache import cachedmethod
 from ..exceptions import ServiceError
 from ..language import language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
 from ..utils import to_unicode
 from ..videos import Episode
 from bs4 import BeautifulSoup
@@ -36,6 +36,7 @@ logger = logging.getLogger(__name__)
 
 class BierDopje(ServiceBase):
     server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
+    user_agent = 'Subliminal/0.6'
     api_based = True
     languages = language_set(['eng', 'dut'])
     videos = [Episode]
@@ -87,8 +88,11 @@ class BierDopje(ServiceBase):
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
             for result in soup.results('result'):
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
-                                          release=to_unicode(result.filename.contents[0]))
+                release = to_unicode(result.filename.contents[0])
+                if not release.endswith(tuple(EXTENSIONS)):
+                    release += '.srt'
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result.downloadlink.contents[0],
+                                          release=release)
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/opensubtitles.py

@@ -120,7 +120,7 @@ class OpenSubtitles(ServiceBase):
             language = self.get_language(result['SubLanguageID'])
             path = get_subtitle_path(filepath, language, self.config.multi)
             confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['SubDownloadLink'],
                                       release=to_unicode(result['SubFileName']), confidence=confidence)
             subtitles.append(subtitle)
         return subtitles

subliminal/services/podnapisi.py

@@ -79,7 +79,7 @@ class Podnapisi(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
                                       release=to_unicode(result['release']), confidence=result['weight'])
             subtitles.append(subtitle)
         if not subtitles:

subliminal/services/subswiki.py

@@ -19,11 +19,10 @@ from . import ServiceBase
 from ..exceptions import ServiceError
 from ..language import language_set, Language
 from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords, split_keyword
 from ..videos import Episode, Movie
 from bs4 import BeautifulSoup
-from subliminal.utils import get_keywords, split_keyword
 import logging
-import re
 import urllib
 
 
@@ -40,7 +39,6 @@ class SubsWiki(ServiceBase):
     language_code = 'name'
     videos = [Episode, Movie]
     require_video = False
-    release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
     required_features = ['permissive']
 
     def list_checked(self, video, languages):
@@ -78,22 +76,22 @@ class SubsWiki(ServiceBase):
         soup = BeautifulSoup(r.content, self.required_features)
         subtitles = []
         for sub in soup('td', {'class': 'NewsTitle'}):
-            sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
+            sub_keywords = split_keyword(sub.b.string.lower())
             if not keywords & sub_keywords:
                 logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
                 continue
-            for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
+            for html_language in sub.parent.parent.find_all('td', {'class': 'language'}):
                 language = self.get_language(html_language.string.strip())
                 if language not in languages:
                     logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
                     continue
-                html_status = html_language.findNextSibling('td')
-                status = html_status.find('strong').string.strip()
-                if status != 'Completed':
+                html_status = html_language.find_next_sibling('td')
+                status = html_status.strong.string.strip()
+                if status != 'Completado':
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s%s' % (self.server_url, html_status.find_next('td').find('a')['href']))
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/subtitulos.py

@@ -18,9 +18,9 @@
 from . import ServiceBase
 from ..language import language_set, Language
 from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords, split_keyword
 from ..videos import Episode
 from bs4 import BeautifulSoup
-from subliminal.utils import get_keywords, split_keyword
 import logging
 import re
 import unicodedata
@@ -36,7 +36,7 @@ class Subtitulos(ServiceBase):
     languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
     language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
                     u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
-                    u'English (UK)': Language('eng-GB')}
+                    u'English (UK)': Language('eng-GB'), 'Galego': Language('glg')}
     language_code = 'name'
     videos = [Episode]
     require_video = False
@@ -79,7 +79,8 @@ class Subtitulos(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'],
+                                          keywords=sub_keywords)
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/tvsubtitles.py

@@ -44,7 +44,8 @@ class TvSubtitles(ServiceBase):
                               'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
                               'zh', 'pt-br'])
     #TODO: Find more exceptions
-    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr')}
+    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr'),
+                    'cn': Language('chi')}
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
@@ -128,14 +129,14 @@ class TvSubtitles(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/download-%d.html' % (self.server_url, subid['subid']),
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/download-%d.html' % (self.server_url, subid['subid']),
                                       keywords=[subid['rip'], subid['release']])
             subtitles.append(subtitle)
         return subtitles
 
     def download(self, subtitle):
         self.download_zip_file(subtitle.link, subtitle.path)
+        return subtitle
 
 
 Service = TvSubtitles

subliminal/subtitles.py

@@ -16,13 +16,14 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .language import Language
+from .utils import to_unicode
 import os.path
 
 
 __all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
 
 #: Subtitles extensions
-EXTENSIONS = ['.srt', '.sub', '.txt']
+EXTENSIONS = ['.srt', '.sub', '.txt', '.ass']
 
 
 class Subtitle(object):
@@ -34,6 +35,8 @@ class Subtitle(object):
 
     """
     def __init__(self, path, language):
+        if not isinstance(language, Language):
+            raise TypeError('%r is not an instance of Language')
         self.path = path
         self.language = language
 
@@ -44,6 +47,15 @@ class Subtitle(object):
             return os.path.exists(self.path)
         return False
 
+    def __unicode__(self):
+        return to_unicode(self.path)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __repr__(self):
+        return '%s(%s, %s)' % (self.__class__.__name__, self, self.language)
+
 
 class EmbeddedSubtitle(Subtitle):
     """Subtitle embedded in a container
@@ -60,7 +72,7 @@ class EmbeddedSubtitle(Subtitle):
 
     @classmethod
     def from_enzyme(cls, path, subtitle):
-        language = Language(subtitle.language) or None
+        language = Language(subtitle.language, strict=False)
         return cls(path, language, subtitle.trackno)
 
 
@@ -69,15 +81,14 @@ class ExternalSubtitle(Subtitle):
     @classmethod
     def from_path(cls, path):
         """Create an :class:`ExternalSubtitle` from path"""
-        extension = ''
+        extension = None
         for e in EXTENSIONS:
             if path.endswith(e):
                 extension = e
                 break
-        if not extension:
+        if extension is None:
             raise ValueError('Not a supported subtitle extension')
-        language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
-        language = Language(language) or None
+        language = Language(os.path.splitext(path[:len(path) - len(extension)])[1][1:], strict=False)
         return cls(path, language)
 
 
@@ -94,13 +105,13 @@ class ResultSubtitle(ExternalSubtitle):
     :param set keywords: keywords that describe the subtitle
 
     """
-    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
+    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=None):
         super(ResultSubtitle, self).__init__(path, language)
         self.service = service
         self.link = link
         self.release = release
         self.confidence = confidence
-        self.keywords = keywords
+        self.keywords = keywords or set()
 
     @property
     def single(self):
@@ -110,12 +121,12 @@ class ResultSubtitle(ExternalSubtitle):
         :rtype: bool
 
         """
-        extension = os.path.splitext(self.path)[0]
-        language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
-        return Language(language) == Language('und')
+        return self.language == Language('Undetermined')
 
     def __repr__(self):
-        return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
+        if not self.release:
+            return 'ResultSubtitle(%s, %s, %s, %.2f)' % (self.path, self.language, self.service, self.confidence)
+        return 'ResultSubtitle(%s, %s, %s, %.2f, release=%s)' % (self.path, self.language, self.service, self.confidence, self.release.encode('ascii', 'ignore'))
 
 
 def get_subtitle_path(video_path, language, multi):

subliminal/utils.py

@@ -61,4 +61,9 @@ def to_unicode(data):
         raise ValueError('Basestring expected')
     if isinstance(data, unicode):
         return data
+    for encoding in ('utf-8', 'latin-1'):
+        try:
+            return unicode(data, encoding)
+        except UnicodeDecodeError:
+            pass
     return unicode(data, 'utf-8', 'replace')

subliminal/videos.py

@@ -16,7 +16,9 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import subtitles
-import enzyme
+from .language import Language
+from .utils import to_unicode
+import enzyme.core
 import guessit
 import hashlib
 import logging
@@ -129,26 +131,28 @@ class Video(object):
             logger.debug(u'Failed parsing %s with enzyme' % self.path)
         if isinstance(video_infos, enzyme.core.AVContainer):
             results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
-
         # cannot use glob here because it chokes if there are any square
         # brackets inside the filename, so we have to use basic string
         # startswith/endswith comparisons
         folder, basename = os.path.split(basepath)
+        if folder == '':
+            folder = '.'
         existing = [f for f in os.listdir(folder) if f.startswith(basename)]
         for path in existing:
             for ext in subtitles.EXTENSIONS:
                 if path.endswith(ext):
-                    possible_lang = path[len(basename) + 1:-len(ext)]
-                    if possible_lang == '':
-                        results.append(subtitles.ExternalSubtitle(path, None))
-                    else:
-                        lang = guessit.Language(possible_lang)
-                        if lang:
-                            results.append(subtitles.ExternalSubtitle(path, lang))
+                    language = Language(path[len(basename) + 1:-len(ext)], strict=False)
+                    results.append(subtitles.ExternalSubtitle(path, language))
         return results
 
+    def __unicode__(self):
+        return to_unicode(self.path or self.release)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
     def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.release)
+        return '%s(%s)' % (self.__class__.__name__, self)
 
     def __hash__(self):
         return hash(self.path or self.release)
@@ -212,8 +216,6 @@ def scan(entry, max_depth=3, scan_filter=None, depth=0):
     """
     if depth > max_depth and max_depth != 0:  # we do not want to search the whole file system except if max_depth = 0
         return []
-    if depth == 0:
-        entry = os.path.abspath(entry)
     if os.path.isdir(entry):  # a dir? recurse
         logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
         result = []
@@ -273,6 +275,8 @@ def hash_thesubdb(path):
 
     """
     readsize = 64 * 1024
+    if os.path.getsize(path) < readsize:
+        return None
     with open(path, 'rb') as f:
         data = f.read(readsize)
         f.seek(-readsize, os.SEEK_END)

tests/__init__.py

@@ -15,11 +15,11 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from . import test_language, test_services, test_subliminal
+from . import test_language, test_services, test_subliminal, test_videos
 import unittest
 
 
-suite = unittest.TestSuite([test_language.suite(), test_services.suite(), test_subliminal.suite()])
+suite = unittest.TestSuite([test_language.suite(), test_services.suite(), test_subliminal.suite(), test_videos.suite()])
 
 
 if __name__ == '__main__':

tests/test_services.py

@@ -29,7 +29,6 @@ from subliminal.services.subtitulos import Subtitulos
 from subliminal.services.thesubdb import TheSubDB
 from subliminal.services.tvsubtitles import TvSubtitles
 import os
-import sys
 import unittest
 try:
     import cPickle as pickle
@@ -51,7 +50,6 @@ class ServiceTestCase(unittest.TestCase):
         # Setting config to None allows to delete the object, which will in turn save the cache
         self.config = None
 
-    @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows")
     def test_query_series(self):
         with self.service(self.config) as service:
             results = service.query(service, self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
@@ -138,7 +136,7 @@ class Addic7edTestCase(ServiceTestCase):
         # FIXME: this is the size of the first subtitle that appears on the page
         # which is the original one, not the most updated one. We should make
         # sure the Addic7ed service picks up the most recent one instead
-        self.episode_subfilesizes = [33538, 33643]
+        self.episode_subfilesizes = [33469]
         self.episode_keywords = set(['asap', 'hdtv'])
         self.series = 'The Big Bang Theory'
         self.wrong_series = 'No Existent Show Name'
@@ -174,7 +172,7 @@ class BierDopjeTestCase(ServiceTestCase):
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
         self.episode_path = u'The Big Bang Theory/Season 05/S05E06 - The Rhinitis Revelation - HD TV.mkv'
         self.episode_sublanguage = 'en'
-        self.episode_subfilesizes = [33469]
+        self.episode_subfilesizes = [28982]
         self.movie_path = u'Inception (2010)/Inception - 1080p.mkv'
         self.movie_sublanguage = 'en'
         self.movie_subfilesizes = []
@@ -246,7 +244,7 @@ class OpenSubtitlesTestCase(ServiceTestCase):
         self.fake_file = u'/tmp/fake_file'
         self.episode_path = existing_video
         self.episode_sublanguage = 'en'
-        self.episode_subfilesizes = [33585, 33547, 33563, 33601]
+        self.episode_subfilesizes = [30374, 30358, 33585, 33547, 33563, 33601, 31413, 31429]
         self.movie = 'Inception'
         self.imdbid = '1375666'
         self.wrong_imdbid = '9999999'
@@ -500,4 +498,4 @@ def suite():
 
 
 if __name__ == '__main__':
-    unittest.TextTestRunner(verbosity=2).run(suite())
+    unittest.TextTestRunner().run(suite())

tests/test_subliminal.py

@@ -20,37 +20,74 @@ from subliminal import Pool, list_subtitles, download_subtitles
 import os
 import time
 import unittest
+import requests
+import tarfile
+import StringIO
 
 
-cache_dir = u'/tmp/sublicache'
-if not os.path.exists(cache_dir):
-    os.mkdir(cache_dir)
-existing_video = u'/something/The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
+test_dir = 'test_subliminal_files'
+cache_dir = 'test_subliminal_cache'
+test_video = 'The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_subliminal_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+    if not os.path.exists(cache_dir):
+        os.mkdir(cache_dir)
 
 
 class ApiTestCase(unittest.TestCase):
     def test_list_subtitles(self):
-        results = list_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+        results = list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) > 0)
 
     def test_download_subtitles(self):
-        results = download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) == 1)
         for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
+            self.assertTrue(video.release == test_video)
             self.assertTrue(len(subtitles) == 1)
+            for subtitle in subtitles:
+                self.assertTrue(subtitle.path == os.path.splitext(os.path.basename(test_video))[0] + '.srt')
+                self.assertTrue(os.path.exists(subtitle.path))
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_multi(self):
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir, multi=True)
+        self.assertTrue(len(results) == 1)
+        for video, subtitles in results.iteritems():
+            self.assertTrue(video.release == test_video)
+            self.assertTrue(len(subtitles) == 2)
             for subtitle in subtitles:
                 self.assertTrue(os.path.exists(subtitle.path))
                 os.remove(subtitle.path)
 
-    def test_download_multi_subtitles(self):
-        results = download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3, multi=True)
-        self.assertTrue(len(results) == 1)
-        for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
-            self.assertTrue(len(subtitles) == 2)
+    def test_download_subtitles_multi_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_languages(self):
+        results = download_subtitles('Dexter/Season 04/S04E08 - Road Kill - 720p BluRay.mkv', languages=['en'],
+                                     cache_dir=cache_dir, multi=True, force=False, services=['subtitulos', 'tvsubtitles'])
+        self.assertTrue(len(results) == 1)
+        for _, subtitles in results.iteritems():
+            self.assertTrue(len(subtitles) == 1)
             for subtitle in subtitles:
-                self.assertTrue(os.path.exists(subtitle.path))
                 os.remove(subtitle.path)
 
 
@@ -71,17 +108,18 @@ class AsyncTestCase(unittest.TestCase):
 
     def test_list_subtitles(self):
         with Pool(4) as p:
-            results = p.list_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+            results = p.list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) > 0)
 
     def test_download_subtitles(self):
         with Pool(4) as p:
-            results = p.download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+            results = p.download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) == 1)
         for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
+            self.assertTrue(video.release == test_video)
             self.assertTrue(len(subtitles) == 1)
             for subtitle in subtitles:
+                self.assertTrue(subtitle.path == os.path.splitext(os.path.basename(test_video))[0] + '.srt')
                 self.assertTrue(os.path.exists(subtitle.path))
                 os.remove(subtitle.path)
 

tests/test_videos.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from subliminal.subtitles import EmbeddedSubtitle, ExternalSubtitle
+from subliminal.videos import scan
+from subliminal.language import Language
+import StringIO
+import os
+import requests
+import tarfile
+import unittest
+
+
+test_dir = 'test_videos_files'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_videos_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+
+
+class ScanTestCase(unittest.TestCase):
+    def test_basic(self):
+        results = scan(test_dir)
+        self.assertTrue(len(results) == 1)
+        self.assertTrue(isinstance(results[0], tuple))
+        self.assertTrue(len(results[0]) == 2)
+
+    def test_embedded_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, EmbeddedSubtitle)]
+        self.assertTrue(len(results) == 8)
+        for l in ('fre', 'eng', 'ita', 'spa', 'hun', 'ger', 'jpn', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+    def test_external_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, ExternalSubtitle)]
+        self.assertTrue(len(results) == 3)
+        for l in ('fre', 'eng', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ScanTestCase))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite())