Merge branch 'develop'

Matching confidence could not be computed because of the
missing extension

.gitignore

@@ -20,19 +20,24 @@ pip-log.txt
 .coverage
 .tox
 
-#Translations
+# Translations
 *.mo
 
-#Mr Developer
+# Mr Developer
 .mr.developer.cfg
 
-#Pydev
+# Pydev
 .project
 .pydevproject
 .settings
 
-#Rope
+# Rope
 .ropeproject
 
-#Sphinx
+# Sphinx
 docs/_build
+
+# Subliminal unittests
+tests/*.srt
+tests/*_files
+tests/*_cache

NEWS.rst

@@ -1,6 +1,24 @@
 News
 ====
 
+0.6.1
+-----
+**release date:** 2012-06-24
+
+* Fix subtitle release name in BierDopje
+* Fix subtitles being downloaded multiple times
+* Add Chinese support to TvSubtitles
+* Fix encoding issues
+* Fix single download subtitles without the force option
+* Add Spanish (Latin America) exception to Addic7ed
+* Fix group_by_video when a list entry has None as subtitles
+* Add support for Galician language in Subtitulos
+* Add an integrity check after subtitles download for Addic7ed
+* Add error handling for if not strict in Language
+* Fix TheSubDB hash method to return None if the file is too small
+* Fix guessit.Language in Video.scan
+* Fix language detection of subtitles
+
 0.6.0
 -----
 **release date:** 2012-06-16

README.rst

@@ -17,7 +17,6 @@ Multiple subtitles services are available:
 * Addic7ed
 * BierDopje
 * OpenSubtitles
-* Podnapisi
 * SubsWiki
 * Subtitulos
 * TheSubDB

docs/index.rst

@@ -21,7 +21,6 @@ Multiple subtitles services are available:
 * Addic7ed
 * BierDopje
 * OpenSubtitles
-* Podnapisi
 * SubsWiki
 * Subtitulos
 * TheSubDB

subliminal/api.py

@@ -81,6 +81,11 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
     :return: downloaded subtitles
     :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
 
+    .. note::
+
+        If you use ``multi=True``, :data:`~subliminal.core.LANGUAGE_INDEX` has to be the first item of the ``order`` list
+        or you might get unexpected results.
+
     """
     services = services or SERVICES
     languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
@@ -92,7 +97,7 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
         subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
     results = []
     service_instances = {}
-    tasks = create_download_tasks(subtitles_by_video, multi)
+    tasks = create_download_tasks(subtitles_by_video, languages, multi)
     for task in tasks:
         try:
             result = consume_task(task, service_instances)

subliminal/async.py

@@ -134,7 +134,7 @@ class Pool(object):
         subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
         for video, subtitles in subtitles_by_video.iteritems():
             subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
-        tasks = create_download_tasks(subtitles_by_video, multi)
+        tasks = create_download_tasks(subtitles_by_video, languages, multi)
         for task in tasks:
             self.tasks.put(task)
         self.join()

subliminal/core.py

@@ -20,6 +20,7 @@ from .services import ServiceConfig
 from .tasks import DownloadTask, ListTask
 from .utils import get_keywords
 from .videos import Episode, Movie, scan
+from .language import Language
 from collections import defaultdict
 from itertools import groupby
 import bs4
@@ -66,7 +67,7 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
             if not wanted_languages:
                 logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
                 continue
-        if not force and not multi and None in detected_languages:
+        if not force and not multi and Language('Undetermined') in detected_languages:
             logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
             continue
         logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
@@ -81,13 +82,13 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
     return tasks
 
 
-def create_download_tasks(subtitles_by_video, multi):
+def create_download_tasks(subtitles_by_video, languages, multi):
     """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
 
-    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
+    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results with ordered subtitles
     :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
-    :param order: preferred order for subtitles sorting
-    :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+    :param languages: languages in preferred order
+    :type languages: :class:`~subliminal.language.language_list`
     :param bool multi: download multiple languages for the same video
     :return: the created tasks
     :rtype: list of :class:`~subliminal.tasks.DownloadTask`
@@ -102,7 +103,7 @@ def create_download_tasks(subtitles_by_video, multi):
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
             continue
-        for _, by_language in groupby(subtitles, lambda s: s.language):
+        for _, by_language in groupby(subtitles, lambda s: languages.index(s.language)):
             task = DownloadTask(video, list(by_language))
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
@@ -157,6 +158,7 @@ def matching_confidence(video, subtitle):
     guess = guessit.guess_file_info(subtitle.release, 'autodetect')
     video_keywords = get_keywords(video.guess)
     subtitle_keywords = get_keywords(guess) | subtitle.keywords
+    logger.debug(u'Video keywords %r - Subtitle keywords %r' % (video_keywords, subtitle_keywords))
     replacement = {'keywords': len(video_keywords & subtitle_keywords)}
     if isinstance(video, Episode):
         replacement.update({'series': 0, 'season': 0, 'episode': 0})
@@ -179,8 +181,11 @@ def matching_confidence(video, subtitle):
             if 'year' in guess and guess['year'] == video.year:
                 replacement['year'] = 1
     else:
-        return 0
+        logger.debug(u'Not able to compute confidence for %r' % video)
+        return 0.0
+    logger.debug(u'Found %r' % replacement)
     confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
+    logger.info(u'Computed confidence %.4f for %r and %r' % (confidence, video, subtitle))
     return confidence
 
 
@@ -248,7 +253,7 @@ def group_by_video(list_results):
     """
     result = defaultdict(list)
     for video, subtitles in list_results:
-        result[video] += subtitles
+        result[video] += subtitles or []
     return result
 
 

subliminal/infos.py

@@ -15,4 +15,4 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.6.0'
+__version__ = '0.6.1'

subliminal/language.py

@@ -17,6 +17,10 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .utils import to_unicode
 import re
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 COUNTRIES = [('AF', 'AFG', '004', u'Afghanistan'),
@@ -838,15 +842,26 @@ class Language(object):
         if isinstance(country, Country):
             self.country = country
         elif isinstance(country, basestring):
-            self.country = Country(country, countries)
+            try:
+                self.country = Country(country, countries)
+            except ValueError:
+                logger.warning(u'Country %s could not be identified' % country)
+                if strict:
+                    raise
 
         # Language + Country format
         #TODO: Improve this part
-        for regexp in [r.match(language) for r in self.with_country_regexps]:
-            if regexp:
-                language = regexp.group(1)
-                self.country = Country(regexp.group(2), countries)
-                break
+        if country is None:
+            for regexp in [r.match(language) for r in self.with_country_regexps]:
+                if regexp:
+                    language = regexp.group(1)
+                    try:
+                        self.country = Country(regexp.group(2), countries)
+                    except ValueError:
+                        logger.warning(u'Country %s could not be identified' % country)
+                        if strict:
+                            raise
+                    break
 
         # Try to find the language
         language = to_unicode(language.strip().lower())

subliminal/services/__init__.py

@@ -160,6 +160,7 @@ class ServiceBase(object):
     def download(self, subtitle):
         """Download a subtitle"""
         self.download_file(subtitle.link, subtitle.path)
+        return subtitle
 
     @classmethod
     def check_validity(cls, video, languages):
@@ -188,17 +189,17 @@ class ServiceBase(object):
         :param string filepath: destination path
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
             with open(filepath, 'wb') as f:
                 f.write(r.content)
         except Exception as e:
-            logger.error(u'Download %s failed: %s' % (url, e))
+            logger.error(u'Download failed: %s' % e)
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
-        logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+        logger.debug(u'Download finished')
 
     def download_zip_file(self, url, filepath):
         """Attempt to download a zip file and extract any subtitle file from it, if any.
@@ -208,7 +209,7 @@ class ServiceBase(object):
         :param string filepath: destination path for the subtitle
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             zippath = filepath + '.zip'
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
@@ -218,17 +219,15 @@ class ServiceBase(object):
                 # TODO: could check if maybe we already have a text file and
                 # download it directly
                 raise DownloadFailedError('Downloaded file is not a zip file')
-            zipsub = zipfile.ZipFile(zippath)
-            for subfile in zipsub.namelist():
-                if os.path.splitext(subfile)[1] in EXTENSIONS:
-                    open(filepath, 'w').write(zipsub.open(subfile).read())
-                    break
-            else:
-                logger.debug(u'No subtitles found in zip file')
-                raise DownloadFailedError('No subtitles found in zip file')
+            with zipfile.ZipFile(zippath) as zipsub:
+                for subfile in zipsub.namelist():
+                    if os.path.splitext(subfile)[1] in EXTENSIONS:
+                        with open(filepath, 'w') as f:
+                            f.write(zipsub.open(subfile).read())
+                        break
+                else:
+                    raise DownloadFailedError('No subtitles found in zip file')
             os.remove(zippath)
-            logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
-            return
         except Exception as e:
             logger.error(u'Download %s failed: %s' % (url, e))
             if os.path.exists(zippath):
@@ -236,6 +235,7 @@ class ServiceBase(object):
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
 
 
 class ServiceConfig(object):

subliminal/services/addic7ed.py

@@ -17,12 +17,14 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
 from ..cache import cachedmethod
+from ..exceptions import DownloadFailedError
 from ..language import Language, language_set
 from ..subtitles import get_subtitle_path, ResultSubtitle
 from ..utils import get_keywords
 from ..videos import Episode
 from bs4 import BeautifulSoup
 import logging
+import os
 import re
 
 
@@ -51,7 +53,8 @@ class Addic7ed(ServiceBase):
     #TODO: Complete this
     languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
                               'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
-    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre')}
+    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre'),
+                    'Spanish (Latin America)': Language('spa'), }
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
@@ -144,11 +147,27 @@ class Addic7ed(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/%s' % (self.server_url, suburl['suburl']),
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/%s' % (self.server_url, suburl['suburl']),
                                       keywords=[suburl['release']])
             subtitles.append(subtitle)
         return subtitles
 
+    def download(self, subtitle):
+        logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path))
+        try:
+            r = self.session.get(subtitle.link, headers={'Referer': subtitle.link, 'User-Agent': self.user_agent})
+            soup = BeautifulSoup(r.content, self.required_features)
+            if soup.title is not None and u'Addic7ed.com' in soup.title.text.strip():
+                raise DownloadFailedError('Download limit exceeded')
+            with open(subtitle.path, 'wb') as f:
+                f.write(r.content)
+        except Exception as e:
+            logger.error(u'Download failed: %s' % e)
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
+        return subtitle
+
 
 Service = Addic7ed

subliminal/services/bierdopje.py

@@ -19,7 +19,7 @@ from . import ServiceBase
 from ..cache import cachedmethod
 from ..exceptions import ServiceError
 from ..language import language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
 from ..utils import to_unicode
 from ..videos import Episode
 from bs4 import BeautifulSoup
@@ -87,8 +87,11 @@ class BierDopje(ServiceBase):
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
             for result in soup.results('result'):
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
-                                          release=to_unicode(result.filename.contents[0]))
+                release = to_unicode(result.filename.contents[0])
+                if not release.endswith(tuple(EXTENSIONS)):
+                    release += '.srt'
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result.downloadlink.contents[0],
+                                          release=release)
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/opensubtitles.py

@@ -120,7 +120,7 @@ class OpenSubtitles(ServiceBase):
             language = self.get_language(result['SubLanguageID'])
             path = get_subtitle_path(filepath, language, self.config.multi)
             confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['SubDownloadLink'],
                                       release=to_unicode(result['SubFileName']), confidence=confidence)
             subtitles.append(subtitle)
         return subtitles

subliminal/services/podnapisi.py

@@ -79,7 +79,7 @@ class Podnapisi(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
                                       release=to_unicode(result['release']), confidence=result['weight'])
             subtitles.append(subtitle)
         if not subtitles:

subliminal/services/subswiki.py

@@ -93,7 +93,7 @@ class SubsWiki(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/subtitulos.py

@@ -36,7 +36,7 @@ class Subtitulos(ServiceBase):
     languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
     language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
                     u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
-                    u'English (UK)': Language('eng-GB')}
+                    u'English (UK)': Language('eng-GB'), 'Galego': Language('glg')}
     language_code = 'name'
     videos = [Episode]
     require_video = False
@@ -79,7 +79,8 @@ class Subtitulos(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'],
+                                          keywords=sub_keywords)
                 subtitles.append(subtitle)
         return subtitles
 

subliminal/services/tvsubtitles.py

@@ -44,7 +44,8 @@ class TvSubtitles(ServiceBase):
                               'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
                               'zh', 'pt-br'])
     #TODO: Find more exceptions
-    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr')}
+    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr'),
+                    'cn': Language('chi')}
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
@@ -128,14 +129,14 @@ class TvSubtitles(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/download-%d.html' % (self.server_url, subid['subid']),
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/download-%d.html' % (self.server_url, subid['subid']),
                                       keywords=[subid['rip'], subid['release']])
             subtitles.append(subtitle)
         return subtitles
 
     def download(self, subtitle):
         self.download_zip_file(subtitle.link, subtitle.path)
+        return subtitle
 
 
 Service = TvSubtitles

subliminal/subtitles.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .language import Language
+from .utils import to_unicode
 import os.path
 
 
@@ -34,6 +35,8 @@ class Subtitle(object):
 
     """
     def __init__(self, path, language):
+        if not isinstance(language, Language):
+            raise TypeError('%r is not an instance of Language')
         self.path = path
         self.language = language
 
@@ -44,6 +47,15 @@ class Subtitle(object):
             return os.path.exists(self.path)
         return False
 
+    def __unicode__(self):
+        return to_unicode(self.path)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __repr__(self):
+        return '%s(%s, %s)' % (self.__class__.__name__, self, self.language)
+
 
 class EmbeddedSubtitle(Subtitle):
     """Subtitle embedded in a container
@@ -60,7 +72,7 @@ class EmbeddedSubtitle(Subtitle):
 
     @classmethod
     def from_enzyme(cls, path, subtitle):
-        language = Language(subtitle.language) or None
+        language = Language(subtitle.language, strict=False)
         return cls(path, language, subtitle.trackno)
 
 
@@ -69,15 +81,14 @@ class ExternalSubtitle(Subtitle):
     @classmethod
     def from_path(cls, path):
         """Create an :class:`ExternalSubtitle` from path"""
-        extension = ''
+        extension = None
         for e in EXTENSIONS:
             if path.endswith(e):
                 extension = e
                 break
-        if not extension:
+        if extension is None:
             raise ValueError('Not a supported subtitle extension')
-        language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
-        language = Language(language) or None
+        language = Language(os.path.splitext(path[:len(path) - len(extension)])[1][1:], strict=False)
         return cls(path, language)
 
 
@@ -94,13 +105,13 @@ class ResultSubtitle(ExternalSubtitle):
     :param set keywords: keywords that describe the subtitle
 
     """
-    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
+    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=None):
         super(ResultSubtitle, self).__init__(path, language)
         self.service = service
         self.link = link
         self.release = release
         self.confidence = confidence
-        self.keywords = keywords
+        self.keywords = keywords or set()
 
     @property
     def single(self):
@@ -110,12 +121,12 @@ class ResultSubtitle(ExternalSubtitle):
         :rtype: bool
 
         """
-        extension = os.path.splitext(self.path)[0]
-        language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
-        return Language(language) == Language('und')
+        return self.language == Language('Undetermined')
 
     def __repr__(self):
-        return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
+        if not self.release:
+            return 'ResultSubtitle(%s, %s, %s, %.2f)' % (self.path, self.language, self.service, self.confidence)
+        return 'ResultSubtitle(%s, %s, %s, %.2f, release=%s)' % (self.path, self.language, self.service, self.confidence, self.release.encode('ascii', 'ignore'))
 
 
 def get_subtitle_path(video_path, language, multi):

subliminal/utils.py

@@ -61,4 +61,9 @@ def to_unicode(data):
         raise ValueError('Basestring expected')
     if isinstance(data, unicode):
         return data
+    for encoding in ('utf-8', 'latin-1'):
+        try:
+            return unicode(data, encoding)
+        except UnicodeDecodeError:
+            pass
     return unicode(data, 'utf-8', 'replace')

subliminal/videos.py

@@ -16,6 +16,8 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import subtitles
+from .language import Language
+from .utils import to_unicode
 import enzyme
 import guessit
 import hashlib
@@ -129,7 +131,6 @@ class Video(object):
             logger.debug(u'Failed parsing %s with enzyme' % self.path)
         if isinstance(video_infos, enzyme.core.AVContainer):
             results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
-
         # cannot use glob here because it chokes if there are any square
         # brackets inside the filename, so we have to use basic string
         # startswith/endswith comparisons
@@ -138,17 +139,18 @@ class Video(object):
         for path in existing:
             for ext in subtitles.EXTENSIONS:
                 if path.endswith(ext):
-                    possible_lang = path[len(basename) + 1:-len(ext)]
-                    if possible_lang == '':
-                        results.append(subtitles.ExternalSubtitle(path, None))
-                    else:
-                        lang = guessit.Language(possible_lang)
-                        if lang:
-                            results.append(subtitles.ExternalSubtitle(path, lang))
+                    language = Language(path[len(basename) + 1:-len(ext)], strict=False)
+                    results.append(subtitles.ExternalSubtitle(path, language))
         return results
 
+    def __unicode__(self):
+        return to_unicode(self.path or self.release)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
     def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.release)
+        return '%s(%s)' % (self.__class__.__name__, self)
 
     def __hash__(self):
         return hash(self.path or self.release)
@@ -212,8 +214,6 @@ def scan(entry, max_depth=3, scan_filter=None, depth=0):
     """
     if depth > max_depth and max_depth != 0:  # we do not want to search the whole file system except if max_depth = 0
         return []
-    if depth == 0:
-        entry = os.path.abspath(entry)
     if os.path.isdir(entry):  # a dir? recurse
         logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
         result = []
@@ -273,6 +273,8 @@ def hash_thesubdb(path):
 
     """
     readsize = 64 * 1024
+    if os.path.getsize(path) < readsize:
+        return None
     with open(path, 'rb') as f:
         data = f.read(readsize)
         f.seek(-readsize, os.SEEK_END)

tests/__init__.py

@@ -15,11 +15,11 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from . import test_language, test_services, test_subliminal
+from . import test_language, test_services, test_subliminal, test_videos
 import unittest
 
 
-suite = unittest.TestSuite([test_language.suite(), test_services.suite(), test_subliminal.suite()])
+suite = unittest.TestSuite([test_language.suite(), test_services.suite(), test_subliminal.suite(), test_videos.suite()])
 
 
 if __name__ == '__main__':

tests/test_services.py

@@ -29,7 +29,6 @@ from subliminal.services.subtitulos import Subtitulos
 from subliminal.services.thesubdb import TheSubDB
 from subliminal.services.tvsubtitles import TvSubtitles
 import os
-import sys
 import unittest
 try:
     import cPickle as pickle
@@ -51,7 +50,6 @@ class ServiceTestCase(unittest.TestCase):
         # Setting config to None allows to delete the object, which will in turn save the cache
         self.config = None
 
-    @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows")
     def test_query_series(self):
         with self.service(self.config) as service:
             results = service.query(service, self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
@@ -246,7 +244,7 @@ class OpenSubtitlesTestCase(ServiceTestCase):
         self.fake_file = u'/tmp/fake_file'
         self.episode_path = existing_video
         self.episode_sublanguage = 'en'
-        self.episode_subfilesizes = [33585, 33547, 33563, 33601]
+        self.episode_subfilesizes = [30374, 30358, 33585, 33547, 33563, 33601]
         self.movie = 'Inception'
         self.imdbid = '1375666'
         self.wrong_imdbid = '9999999'
@@ -500,4 +498,4 @@ def suite():
 
 
 if __name__ == '__main__':
-    unittest.TextTestRunner(verbosity=2).run(suite())
+    unittest.TextTestRunner().run(suite())

tests/test_subliminal.py

@@ -20,39 +20,75 @@ from subliminal import Pool, list_subtitles, download_subtitles
 import os
 import time
 import unittest
+import requests
+import tarfile
+import StringIO
 
 
-cache_dir = u'/tmp/sublicache'
-if not os.path.exists(cache_dir):
-    os.mkdir(cache_dir)
-existing_video = u'/something/The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
+test_dir = 'test_subliminal_files'
+cache_dir = 'test_subliminal_cache'
+test_video = 'The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_subliminal_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+    if not os.path.exists(cache_dir):
+        os.mkdir(cache_dir)
 
 
 class ApiTestCase(unittest.TestCase):
     def test_list_subtitles(self):
-        results = list_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+        results = list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) > 0)
 
     def test_download_subtitles(self):
-        results = download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) == 1)
         for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
+            self.assertTrue(video.release == test_video)
             self.assertTrue(len(subtitles) == 1)
             for subtitle in subtitles:
                 self.assertTrue(os.path.exists(subtitle.path))
                 os.remove(subtitle.path)
 
-    def test_download_multi_subtitles(self):
-        results = download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3, multi=True)
+    def test_download_subtitles_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_multi(self):
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir, multi=True)
         self.assertTrue(len(results) == 1)
         for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
+            self.assertTrue(video.release == test_video)
             self.assertTrue(len(subtitles) == 2)
             for subtitle in subtitles:
                 self.assertTrue(os.path.exists(subtitle.path))
                 os.remove(subtitle.path)
 
+    def test_download_subtitles_multi_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_languages(self):
+        results = download_subtitles('Dexter/Season 04/S04E08 - Road Kill - 720p BluRay.mkv', languages=['en'],
+                                     cache_dir=cache_dir, multi=True, force=False, services=['subtitulos', 'tvsubtitles'])
+        self.assertTrue(len(results) == 1)
+        for _, subtitles in results.iteritems():
+            self.assertTrue(len(subtitles) == 1)
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
 
 class AsyncTestCase(unittest.TestCase):
     def test_pool(self):
@@ -71,15 +107,15 @@ class AsyncTestCase(unittest.TestCase):
 
     def test_list_subtitles(self):
         with Pool(4) as p:
-            results = p.list_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+            results = p.list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) > 0)
 
     def test_download_subtitles(self):
         with Pool(4) as p:
-            results = p.download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
+            results = p.download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
         self.assertTrue(len(results) == 1)
         for video, subtitles in results.iteritems():
-            self.assertTrue(video.release == existing_video)
+            self.assertTrue(video.release == test_video)
             self.assertTrue(len(subtitles) == 1)
             for subtitle in subtitles:
                 self.assertTrue(os.path.exists(subtitle.path))

tests/test_videos.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from subliminal.subtitles import EmbeddedSubtitle, ExternalSubtitle
+from subliminal.videos import scan
+from subliminal.language import Language
+import StringIO
+import os
+import requests
+import tarfile
+import unittest
+
+
+test_dir = 'test_videos_files'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_videos_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+
+
+class ScanTestCase(unittest.TestCase):
+    def test_basic(self):
+        results = scan(test_dir)
+        self.assertTrue(len(results) == 1)
+        self.assertTrue(isinstance(results[0], tuple))
+        self.assertTrue(len(results[0]) == 2)
+
+    def test_embedded_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, EmbeddedSubtitle)]
+        self.assertTrue(len(results) == 8)
+        for l in ('fre', 'eng', 'ita', 'spa', 'hun', 'ger', 'jpn', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+    def test_external_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, ExternalSubtitle)]
+        self.assertTrue(len(results) == 3)
+        for l in ('fre', 'eng', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ScanTestCase))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite())