Update documentation

0.7.x

.gitignore

@@ -34,6 +34,9 @@ pip-log.txt
 # Rope
 .ropeproject
 
+# PyCharm
+.idea
+
 # Sphinx
 docs/_build
 

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 
 python:
+  - "2.6"
   - "2.7"
 
 install:

HISTORY.rst

@@ -1,6 +1,33 @@
 Changelog
 =========
 
+0.7.5
+-----
+**release date:** 2015-03-04
+
+* Update requirements
+* Remove BierDopje provider
+* Add pre-guessed video optional argument in scan_video
+* Improve hearing impaired support
+* Fix TVSubtitles and Podnapisi providers
+
+0.7.4
+-----
+**release date:** 2014-01-27
+
+* Fix requirements for guessit and babelfish
+
+
+0.7.3
+-----
+**release date:** 2013-11-22
+
+* Fix windows compatibility
+* Improve subtitle validation
+* Improve embedded subtitle languages detection
+* Improve unittests
+
+
 0.7.2
 -----
 **release date:** 2013-11-10

README.rst

@@ -17,8 +17,8 @@ Subliminal uses multiple providers to give users a vast choice and have a better
 best matching subtitles. Providers are extensible through a dedicated entry point.
 
 * Addic7ed
-* BierDopje
 * OpenSubtitles
+* Podnapisi
 * TheSubDB
 * TvSubtitles
 
@@ -45,7 +45,7 @@ skipping videos that already have subtitles whether they are embedded or not::
     subliminal.cache_region.configure('dogpile.cache.dbm', arguments={'filename': '/path/to/cachefile.dbm'})
 
     # scan for videos in the folder and their subtitles
-    videos = scan_videos(['/path/to/video/folder'], subtitles=True, embedded_subtitles=True) 
+    videos = subliminal.scan_videos(['/path/to/video/folder'], subtitles=True, embedded_subtitles=True, age=timedelta(weeks=1))
 
     # download
     subliminal.download_best_subtitles(videos, {Language('eng'), Language('fra')}, age=timedelta(week=1))

dev-requirements.txt

@@ -2,3 +2,4 @@ sympy>=0.7.3
 sphinx>=1.1.3
 sphinxcontrib-programoutput>=0.8
 Sphinx-PyPI-upload>=0.2.1
+setuptools>=1.4

docs/index.rst

@@ -17,8 +17,8 @@ Subliminal uses multiple providers to give users a vast choice and have a better
 best matching subtitles. Providers are extensible through a dedicated entry point.
 
 * Addic7ed
-* BierDopje
 * OpenSubtitles
+* Podnapisi
 * TheSubDB
 * TvSubtitles
 
@@ -47,7 +47,7 @@ skipping videos that already have subtitles whether they are embedded or not::
     subliminal.cache_region.configure('dogpile.cache.dbm', arguments={'filename': '/path/to/cachefile.dbm'})
 
     # scan for videos in the folder and their subtitles
-    videos = scan_videos(['/path/to/video/folder'], subtitles=True, embedded_subtitles=True) 
+    videos = subliminal.scan_videos(['/path/to/video/folder'], subtitles=True, embedded_subtitles=True, age=timedelta(weeks=1))
 
     # download
     subliminal.download_best_subtitles(videos, {Language('eng'), Language('fra')}, age=timedelta(week=1))

requirements.txt

@@ -1,9 +1,9 @@
 beautifulsoup4>=4.3.2
-guessit>=0.6.2
+guessit>=0.7,<0.10
 requests>=2.0.1
 enzyme>=0.4.0
 html5lib>=0.99
-dogpile.cache>=0.5.1
-babelfish>=0.3.0
+dogpile.cache>=0.5.2
+babelfish>=0.5.0
 charade>=1.0.3
 pysrt>=0.5.0

setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 
 setup(name='subliminal',
-    version='0.7.2',
+    version='0.7.5',
     license='MIT',
     description='Subtitles, faster than your thoughts',
     long_description=open('README.rst').read() + '\n\n' + open('HISTORY.rst').read(),
@@ -19,6 +19,7 @@ setup(name='subliminal',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.3',
@@ -27,14 +28,13 @@ setup(name='subliminal',
     entry_points={
         'console_scripts': ['subliminal = subliminal.cli:subliminal'],
         'subliminal.providers': ['addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
-                                 'bierdopje = subliminal.providers.bierdopje:BierDopjeProvider',
                                  'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
                                  'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
                                  'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
                                  'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'],
-        'babelfish.converters': ['addic7ed = subliminal.converters.addic7ed:Addic7edConverter',
-                                 'podnapisi = subliminal.converters.podnapisi:PodnapisiConverter',
-                                 'tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter']
+        'babelfish.language_converters': ['addic7ed = subliminal.converters.addic7ed:Addic7edConverter',
+                                          'podnapisi = subliminal.converters.podnapisi:PodnapisiConverter',
+                                          'tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter']
     },
     install_requires=open('requirements.txt').readlines(),
     test_suite='subliminal.tests.suite')

subliminal/__init__.py

@@ -1,16 +1,19 @@
 # -*- coding: utf-8 -*-
 __title__ = 'subliminal'
-__version__ = '0.7.2'
+__version__ = '0.7.5'
 __author__ = 'Antoine Bertin'
 __license__ = 'MIT'
 __copyright__ = 'Copyright 2013 Antoine Bertin'
 
 import logging
 from .api import PROVIDERS_ENTRY_POINT, list_subtitles, download_subtitles, download_best_subtitles
-from .cache import region as cache_region
+from .cache import MutexLock, region as cache_region
 from .exceptions import Error, ProviderError, ProviderConfigurationError, ProviderNotAvailable, InvalidSubtitle
 from .subtitle import Subtitle
 from .video import VIDEO_EXTENSIONS, SUBTITLE_EXTENSIONS, Video, Episode, Movie, scan_videos, scan_video
 
+class NullHandler(logging.Handler):
+    def emit(self, record):
+        pass
 
-logging.getLogger(__name__).addHandler(logging.NullHandler())
+logging.getLogger(__name__).addHandler(NullHandler())

subliminal/api.py

@@ -6,9 +6,10 @@ import logging
 import operator
 import babelfish
 import pkg_resources
+from os.path import basename
 from .exceptions import ProviderNotAvailable, InvalidSubtitle
 from .subtitle import get_subtitle_path
-
+from socket import error as socket_error
 
 logger = logging.getLogger(__name__)
 
@@ -67,16 +68,21 @@ def list_subtitles(videos, languages, providers=None, provider_configs=None):
                                 provider_entry_point.name, provider_video, provider_video_languages)
                     try:
                         provider_subtitles = provider.list_subtitles(provider_video, provider_video_languages)
-                    except ProviderNotAvailable:
+                    except ProviderNotAvailable as err:
                         logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
+                        logger.debug('ProviderNotAvailable error: %r', str(err))
                         break
                     except:
                         logger.exception('Unexpected error in provider %r', provider_entry_point.name)
                         continue
-                    logger.info('Found %d subtitles', len(provider_subtitles))
+                    logger.info('Found %d subtitle(s) on %s' % (
+                        len(provider_subtitles),
+                        provider_entry_point.name,
+                    ))
                     subtitles[provider_video].extend(provider_subtitles)
-        except ProviderNotAvailable:
+        except ProviderNotAvailable as err:
             logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
+            logger.debug('ProviderNotAvailable error: %r', str(err))
     return subtitles
 
 
@@ -92,15 +98,19 @@ def download_subtitles(subtitles, provider_configs=None, single=False):
     """
     provider_configs = provider_configs or {}
     discarded_providers = set()
-    providers_by_name = {ep.name: ep.load() for ep in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT)}
+    providers_by_name = dict([(ep.name, ep.load()) for ep in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT)])
+
     initialized_providers = {}
+    downloaded_subtitles = collections.defaultdict(list)
+    fetched_subtitles = set()
     try:
         for video, video_subtitles in subtitles.items():
-            languages = {subtitle.language for subtitle in video_subtitles}
+            languages = set([subtitle.language for subtitle in video_subtitles])
             downloaded_languages = set()
             for subtitle in video_subtitles:
                 # filter
                 if subtitle.language in downloaded_languages:
+                    logger.debug('Skipping subtitle: %r already downloaded', subtitle.language)
                     continue
                 if subtitle.provider_name in discarded_providers:
                     logger.debug('Skipping subtitle from discarded provider %r', subtitle.provider_name)
@@ -113,19 +123,35 @@ def download_subtitles(subtitles, provider_configs=None, single=False):
                     provider = providers_by_name[subtitle.provider_name](**provider_configs.get(subtitle.provider_name, {}))
                     try:
                         provider.initialize()
-                    except ProviderNotAvailable:
+                    except ProviderNotAvailable as err:
                         logger.warning('Provider %r is not available, discarding it', subtitle.provider_name)
+                        logger.debug('ProviderNotAvailable error: %r', str(err))
+                        discarded_providers.add(subtitle.provider_name)
+                        continue
+                    except socket_error as err:
+                        logger.warning('Provider %r is not responding, discarding it', subtitle.provider_name)
+                        logger.debug('Provider socket error: %r', str(err))
+                        discarded_providers.add(subtitle.provider_name)
+                        continue
+                    except:
+                        logger.exception('Unexpected error in provider %r', subtitle.provider_name)
                         discarded_providers.add(subtitle.provider_name)
                         continue
                     initialized_providers[subtitle.provider_name] = provider
 
                 # download subtitles
                 subtitle_path = get_subtitle_path(video.name, None if single else subtitle.language)
+                if basename(subtitle_path) in fetched_subtitles:
+                    logger.debug('Skipping subtitle already retrieved %r', basename(subtitle_path))
+                    continue
+
                 logger.info('Downloading subtitle %r into %r', subtitle, subtitle_path)
                 try:
                     subtitle_text = provider.download_subtitle(subtitle)
-                except ProviderNotAvailable:
+                    downloaded_subtitles[video].append(subtitle)
+                except ProviderNotAvailable as err:
                     logger.warning('Provider %r is not available, discarding it', subtitle.provider_name)
+                    logger.debug('ProviderNotAvailable error: %r', str(err))
                     discarded_providers.add(subtitle.provider_name)
                     continue
                 except InvalidSubtitle:
@@ -136,8 +162,9 @@ def download_subtitles(subtitles, provider_configs=None, single=False):
                     continue
                 with io.open(subtitle_path, 'w', encoding='utf-8') as f:
                     f.write(subtitle_text)
-                downloaded_languages.add(subtitle.language)
-                if single or downloaded_languages == languages:
+                    downloaded_languages.add(subtitle.language)
+                    fetched_subtitles.add(basename(subtitle_path))
+                if single or sorted(downloaded_languages) == sorted(languages):
                     break
     finally:  # terminate providers
         for (provider_name, provider) in initialized_providers.items():
@@ -147,10 +174,11 @@ def download_subtitles(subtitles, provider_configs=None, single=False):
                 logger.warning('Provider %r is not available, unable to terminate', provider_name)
             except:
                 logger.exception('Unexpected error in provider %r', provider_name)
+    return downloaded_subtitles
 
 
 def download_best_subtitles(videos, languages, providers=None, provider_configs=None, single=False, min_score=0,
-                            hearing_impaired=False):
+                            hearing_impaired=False, hi_score_adjust=0):
     """Download the best subtitles for `videos` with the given `languages` using the specified `providers`
 
     :param videos: videos to download subtitles for
@@ -164,11 +192,13 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
     :param bool single: download with .srt extension if `True`, add language identifier otherwise
     :param int min_score: minimum score for subtitles to download
     :param bool hearing_impaired: download hearing impaired subtitles
+    :param int hi_score_adjust: Adjust hearing_impaired_scores if matched.
 
     """
     provider_configs = provider_configs or {}
     discarded_providers = set()
     downloaded_subtitles = collections.defaultdict(list)
+    fetched_subtitles = set()
     # filter videos
     videos = [v for v in videos if v.subtitle_languages & languages < languages
               and (not single or babelfish.Language('und') not in v.subtitle_languages)]
@@ -184,27 +214,37 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
             continue
         Provider = provider_entry_point.load()
         if not Provider.languages & languages - subtitle_languages:
-            logger.info('Skipping provider %r: no language to search for', provider_entry_point.name)
+            logger.debug('Skipping provider %r: no language to search for', provider_entry_point.name)
             continue
         if not [v for v in videos if Provider.check(v)]:
-            logger.info('Skipping provider %r: no video to search for', provider_entry_point.name)
+            logger.debug('Skipping provider %r: video type not hosted here.', provider_entry_point.name)
             continue
         provider = Provider(**provider_configs.get(provider_entry_point.name, {}))
         try:
             provider.initialize()
-        except ProviderNotAvailable:
+        except ProviderNotAvailable as err:
             logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
+            logger.debug('ProviderNotAvailable error: %r', str(err))
+            continue
+        except socket_error as err:
+            logger.warning('Provider %r is not responding, discarding it', provider_entry_point.name)
+            logger.debug('Provider socket error: %r', str(err))
+            continue
+        except:
+            logger.exception('Unexpected error in provider %r', provider_entry_point.name)
             continue
         initialized_providers[provider_entry_point.name] = provider
     try:
         for video in videos:
             # search for subtitles
             subtitles = []
+            downloaded_languages = set()
             for provider_name, provider in initialized_providers.items():
                 if provider.check(video):
                     if provider_name in discarded_providers:
                         logger.debug('Skipping discarded provider %r', provider_name)
                         continue
+
                     provider_video_languages = provider.languages & languages - video.subtitle_languages
                     if not provider_video_languages:
                         logger.debug('Skipping provider %r: no language to search for for video %r', provider_name,
@@ -214,30 +254,38 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                                 provider_name, video, provider_video_languages)
                     try:
                         provider_subtitles = provider.list_subtitles(video, provider_video_languages)
-                    except ProviderNotAvailable:
+                    except ProviderNotAvailable as err:
                         logger.warning('Provider %r is not available, discarding it', provider_name)
+                        logger.debug('ProviderNotAvailable error: %r', str(err))
                         discarded_providers.add(provider_name)
                         continue
                     except:
                         logger.exception('Unexpected error in provider %r', provider_name)
                         continue
-                    logger.info('Found %d subtitles', len(provider_subtitles))
+                    logger.info('Found %d subtitle(s) on %s' % (
+                        len(provider_subtitles),
+                        provider_name,
+                    ))
                     subtitles.extend(provider_subtitles)
 
             # find the best subtitles and download them
-            downloaded_languages = video.subtitle_languages.copy()
-            for subtitle, score in sorted([(s, s.compute_score(video)) for s in subtitles],
-                                          key=operator.itemgetter(1), reverse=True):
+            for subtitle, score in sorted([(s, s.compute_score(video, hi_score_adjust)) \
+                    for s in subtitles], key=operator.itemgetter(1), reverse=True):
+
                 # filter
                 if subtitle.provider_name in discarded_providers:
                     logger.debug('Skipping subtitle from discarded provider %r', subtitle.provider_name)
                     continue
-                if subtitle.hearing_impaired != hearing_impaired:
-                    logger.debug('Skipping subtitle: hearing impaired != %r', hearing_impaired)
-                    continue
+
+                if hearing_impaired is not None:
+                    if subtitle.hearing_impaired != hearing_impaired:
+                        logger.debug('Skipping subtitle: hearing impaired != %r', hearing_impaired)
+                        continue
+
                 if score < min_score:
                     logger.debug('Skipping subtitle: score < %d', min_score)
                     continue
+
                 if subtitle.language in downloaded_languages:
                     logger.debug('Skipping subtitle: %r already downloaded', subtitle.language)
                     continue
@@ -245,12 +293,17 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                 # download
                 provider = initialized_providers[subtitle.provider_name]
                 subtitle_path = get_subtitle_path(video.name, None if single else subtitle.language)
+                if basename(subtitle_path) in fetched_subtitles:
+                    logger.debug('Skipping subtitle already retrieved %r', basename(subtitle_path))
+                    continue
+
                 logger.info('Downloading subtitle %r with score %d into %r', subtitle, score, subtitle_path)
                 try:
                     subtitle_text = provider.download_subtitle(subtitle)
                     downloaded_subtitles[video].append(subtitle)
-                except ProviderNotAvailable:
+                except ProviderNotAvailable as err:
                     logger.warning('Provider %r is not available, discarding it', subtitle.provider_name)
+                    logger.debug('ProviderNotAvailable error: %r', str(err))
                     discarded_providers.add(subtitle.provider_name)
                     continue
                 except InvalidSubtitle:
@@ -261,16 +314,21 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                     continue
                 with io.open(subtitle_path, 'w', encoding='utf-8') as f:
                     f.write(subtitle_text)
-                downloaded_languages.add(subtitle.language)
-                if single or downloaded_languages >= languages:
-                    logger.debug('All languages downloaded')
+                    downloaded_languages.add(subtitle.language)
+                    fetched_subtitles.add(basename(subtitle_path))
+                if single or sorted(downloaded_languages) == sorted(languages):
                     break
+
     finally:  # terminate providers
         for (provider_name, provider) in initialized_providers.items():
             try:
                 provider.terminate()
-            except ProviderNotAvailable:
+            except ProviderNotAvailable as err:
                 logger.warning('Provider %r is not available, unable to terminate', provider_name)
+                logger.debug('ProviderNotAvailable error: %r', str(err))
+            except socket_error as err:
+                logger.warning('Provider %r is not available, unable to terminate', provider_name)
+                logger.debug('Provider socket error: %r', str(err))
             except:
                 logger.exception('Unexpected error in provider %r', provider_name)
     return downloaded_subtitles

subliminal/cache.py

@@ -1,13 +1,16 @@
 # -*- coding: utf-8 -*-
 import inspect
-import dogpile.cache
+from dogpile.cache import make_region  # @UnresolvedImport
+from dogpile.cache.backends.file import AbstractFileLock  # @UnresolvedImport
+from dogpile.cache.compat import string_type  # @UnresolvedImport
+from dogpile.core.readwrite_lock import ReadWriteMutex  # @UnresolvedImport
 
 
 #: Subliminal's cache version
-CACHE_VERSION = 1
+CACHE_VERSION = 2
 
 
-def subliminal_key_generator(namespace, fn, to_str=dogpile.cache.compat.string_type):
+def subliminal_key_generator(namespace, fn, to_str=string_type):
     """Add a :data:`CACHE_VERSION` to dogpile.cache's default function_key_generator"""
     if namespace is None:
         namespace = '%d:%s:%s' % (CACHE_VERSION, fn.__module__, fn.__name__)
@@ -26,5 +29,28 @@ def subliminal_key_generator(namespace, fn, to_str=dogpile.cache.compat.string_t
     return generate_key
 
 
-#: The dogpile.cache region
-region = dogpile.cache.make_region(function_key_generator=subliminal_key_generator)
+class MutexLock(AbstractFileLock):
+    """:class:`MutexLock` is a thread-based rw lock based on :class:`dogpile.core.ReadWriteMutex`"""
+    def __init__(self, filename):
+        self.mutex = ReadWriteMutex()
+
+    def acquire_read_lock(self, wait):
+        ret = self.mutex.acquire_read_lock(wait)
+        return wait or ret
+
+    def acquire_write_lock(self, wait):
+        ret = self.mutex.acquire_write_lock(wait)
+        return wait or ret
+
+    def release_read_lock(self):
+        return self.mutex.release_read_lock()
+
+    def release_write_lock(self):
+        return self.mutex.release_write_lock()
+
+
+#: The dogpile.cache region (long-lived)
+region = make_region(function_key_generator=subliminal_key_generator)
+
+#: The dogpile.cache region for :meth:`~subliminal.providers.Provider.query` (short-lived)
+query_region = make_region(function_key_generator=subliminal_key_generator)

subliminal/cli.py

@@ -9,7 +9,7 @@ import sys
 import babelfish
 import guessit
 import pkg_resources
-from subliminal import (__version__, PROVIDERS_ENTRY_POINT, cache_region, Video, Episode, Movie, scan_videos,
+from subliminal import (__version__, PROVIDERS_ENTRY_POINT, cache_region, MutexLock, Video, Episode, Movie, scan_videos,
     download_best_subtitles)
 try:
     import colorlog
@@ -52,11 +52,6 @@ def subliminal():
     filtering_group.add_argument('-f', '--force', action='store_true',
                                  help='force subtitle download for videos with existing subtitles')
 
-    # addic7ed
-    addic7ed_group = parser.add_argument_group('addic7ed')
-    addic7ed_group.add_argument('--addic7ed-username', metavar='USERNAME', help='username for addic7ed provider')
-    addic7ed_group.add_argument('--addic7ed-password', metavar='PASSWORD', help='password for addic7ed provider')
-
     # output
     output_group = parser.add_argument_group('output')
     output_exclusive_group = output_group.add_mutually_exclusive_group()
@@ -81,7 +76,7 @@ def subliminal():
 
     # parse languages
     try:
-        args.languages = {babelfish.Language.fromietf(l) for l in args.languages}
+        args.languages = set( babelfish.Language.fromietf(l) for l in args.languages )
     except babelfish.Error:
         parser.error('argument -l/--languages: codes are not IETF: %r' % args.languages)
 
@@ -90,7 +85,7 @@ def subliminal():
         match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', args.age)
         if not match:
             parser.error('argument -a/--age: invalid age: %r' % args.age)
-        args.age = datetime.timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
+        args.age = datetime.timedelta(**dict([(k, int(v)) for k, v in match.groupdict(0).items()]))
 
     # parse cache-file
     args.cache_file = os.path.abspath(os.path.expanduser(args.cache_file))
@@ -100,11 +95,6 @@ def subliminal():
 
     # parse provider configs
     provider_configs = {}
-    if (args.addic7ed_username is not None and args.addic7ed_password is None
-        or args.addic7ed_username is None and args.addic7ed_password is not None):
-        parser.error('argument --addic7ed-username/--addic7ed-password: both arguments are required or none')
-    if args.addic7ed_username is not None and args.addic7ed_password is not None:
-        provider_configs['addic7ed'] = {'username': args.addic7ed_username, 'password': args.addic7ed_password}
 
     # parse color
     if args.color and colorlog is None:
@@ -138,15 +128,15 @@ def subliminal():
         logging.getLogger('subliminal.api').setLevel(logging.INFO)
 
     # configure cache
-    cache_region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
-                           arguments={'filename': args.cache_file})
+    cache_region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),  # @UndefinedVariable
+                           arguments={'filename': args.cache_file, 'lock_factory': MutexLock})
 
     # scan videos
     videos = scan_videos([p for p in args.paths if os.path.exists(p)], subtitles=not args.force,
                          embedded_subtitles=not args.force, age=args.age)
 
     # guess videos
-    videos.extend([Video.fromguess(os.path.split(p)[1], guessit.guess_file_info(p, 'autodetect')) for p in args.paths
+    videos.extend([Video.fromguess(os.path.split(p)[1], guessit.guess_file_info(p, info=['filename'])) for p in args.paths
                    if not os.path.exists(p)])
 
     # download best subtitles

subliminal/converters/addic7ed.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-from babelfish.converters.name import NameConverter
+from babelfish import LanguageReverseConverter, language_converters
 
 
-class Addic7edConverter(NameConverter):
+class Addic7edConverter(LanguageReverseConverter):
     def __init__(self):
-        super(Addic7edConverter, self).__init__()
+        self.name_converter = language_converters['name']
         self.from_addic7ed = {'Català': ('cat',), 'Chinese (Simplified)': ('zho',), 'Chinese (Traditional)': ('zho',),
                               'Euskera': ('eus',), 'Galego': ('glg',), 'Greek': ('ell',), 'Malay': ('msa',),
                               'Portuguese (Brazilian)': ('por', 'BR'), 'Serbian (Cyrillic)': ('srp', None, 'Cyrl'),
@@ -14,7 +14,7 @@ class Addic7edConverter(NameConverter):
         self.to_addic7ed = {('cat',): 'Català', ('zho',): 'Chinese (Simplified)', ('eus',): 'Euskera',
                             ('glg',): 'Galego', ('ell',): 'Greek', ('msa',): 'Malay',
                             ('por', 'BR'): 'Portuguese (Brazilian)', ('srp', None, 'Cyrl'): 'Serbian (Cyrillic)'}
-        self.codes |= set(self.from_addic7ed.keys())
+        self.codes = self.name_converter.codes | set(self.from_addic7ed.keys())
 
     def convert(self, alpha3, country=None, script=None):
         if (alpha3, country, script) in self.to_addic7ed:
@@ -23,9 +23,9 @@ class Addic7edConverter(NameConverter):
             return self.to_addic7ed[(alpha3, country)]
         if (alpha3,) in self.to_addic7ed:
             return self.to_addic7ed[(alpha3,)]
-        return super(Addic7edConverter, self).convert(alpha3, country, script)
+        return self.name_converter.convert(alpha3, country, script)
 
     def reverse(self, addic7ed):
         if addic7ed in self.from_addic7ed:
             return self.from_addic7ed[addic7ed]
-        return super(Addic7edConverter, self).reverse(addic7ed)
+        return self.name_converter.reverse(addic7ed)

subliminal/converters/podnapisi.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-from babelfish import ReverseConverter, ConvertError, ReverseError
+from babelfish import LanguageReverseConverter, LanguageConvertError, LanguageReverseError
 
 
-class PodnapisiConverter(ReverseConverter):
+class PodnapisiConverter(LanguageReverseConverter):
     def __init__(self):
-        super(PodnapisiConverter, self).__init__()
         self.from_podnapisi = {2: ('eng',), 28: ('spa',), 26: ('pol',), 36: ('srp',), 1: ('slv',), 38: ('hrv',),
                                9: ('ita',), 8: ('fra',), 48: ('por', 'BR'), 23: ('nld',), 12: ('ara',), 13: ('ron',),
                                33: ('bul',), 32: ('por',), 16: ('ell',), 15: ('hun',), 31: ('fin',), 30: ('tur',),
@@ -15,7 +14,7 @@ class PodnapisiConverter(ReverseConverter):
                                11: ('jpn',), 4: ('kor',), 29: ('sqi',), 6: ('isl',), 19: ('lit',), 46: ('ukr',),
                                44: ('tha',), 53: ('cat',), 56: ('sin',), 21: ('lav',), 40: ('cmn',), 55: ('msa',),
                                42: ('hin',), 50: ('bel',)}
-        self.to_podnapisi = {v: k for k, v in self.from_podnapisi.items()}
+        self.to_podnapisi = dict([(v, k) for k, v in self.from_podnapisi.items()])
         self.codes = set(self.from_podnapisi.keys())
 
     def convert(self, alpha3, country=None, script=None):
@@ -25,9 +24,9 @@ class PodnapisiConverter(ReverseConverter):
             return self.to_podnapisi[(alpha3, country)]
         if (alpha3, country, script) in self.to_podnapisi:
             return self.to_podnapisi[(alpha3, country, script)]
-        raise ConvertError(alpha3, country, script)
+        raise LanguageConvertError(alpha3, country, script)
 
     def reverse(self, podnapisi):
         if podnapisi not in self.from_podnapisi:
-            raise ReverseError(podnapisi)
+            raise LanguageReverseError(podnapisi)
         return self.from_podnapisi[podnapisi]

subliminal/converters/tvsubtitles.py

@@ -1,24 +1,24 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-from babelfish.converters.alpha2 import Alpha2Converter
+from babelfish import LanguageReverseConverter, language_converters
 
 
-class TVsubtitlesConverter(Alpha2Converter):
+class TVsubtitlesConverter(LanguageReverseConverter):
     def __init__(self):
-        super(TVsubtitlesConverter, self).__init__()
+        self.alpha2_converter = language_converters['alpha2']
         self.from_tvsubtitles = {'br': ('por', 'BR'), 'ua': ('ukr',), 'gr': ('ell',), 'cn': ('zho',), 'jp': ('jpn',),
                                  'cz': ('ces',)}
-        self.to_tvsubtitles = {v: k for k, v in self.from_tvsubtitles}
-        self.codes |= set(self.from_tvsubtitles.keys())
+        self.to_tvsubtitles = set([(v, k) for k, v in self.from_tvsubtitles])
+        self.codes = self.alpha2_converter.codes | set(self.from_tvsubtitles.keys())
 
     def convert(self, alpha3, country=None, script=None):
         if (alpha3, country) in self.to_tvsubtitles:
             return self.to_tvsubtitles[(alpha3, country)]
         if (alpha3,) in self.to_tvsubtitles:
             return self.to_tvsubtitles[(alpha3,)]
-        return super(TVsubtitlesConverter, self).convert(alpha3, country, script)
+        return self.alpha2_converter.convert(alpha3, country, script)
 
     def reverse(self, tvsubtitles):
         if tvsubtitles in self.from_tvsubtitles:
             return self.from_tvsubtitles[tvsubtitles]
-        return super(TVsubtitlesConverter, self).reverse(tvsubtitles)
+        return self.alpha2_converter.reverse(tvsubtitles)

subliminal/providers/__init__.py

@@ -2,7 +2,25 @@
 from __future__ import unicode_literals
 import babelfish
 from ..video import Episode, Movie
+from .. import __version__
+from random import randint
 
+# Agent List
+AGENT_LIST = (
+    'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0',
+    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0',
+    'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+    'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
+    'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A',
+    'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
+)
 
 class Provider(object):
     """Base class for providers
@@ -22,6 +40,13 @@ class Provider(object):
     #: Required hash, if any
     required_hash = None
 
+    # Returns a random agent to use from the list above
+    random_user_agent = AGENT_LIST[randint(0, len(AGENT_LIST)-1)]
+
+    # Defines the ideal user agent to use for all providers otherwise
+    primary_user_agent = 'Subliminal/%s' % __version__
+
+
     def __init__(self, **kwargs):
         pass
 

subliminal/providers/addic7ed.py

@@ -6,10 +6,9 @@ import bs4
 import charade
 import requests
 from . import Provider
-from .. import __version__
 from ..cache import region
 from ..exceptions import ProviderConfigurationError, ProviderNotAvailable, InvalidSubtitle
-from ..subtitle import Subtitle, is_valid_subtitle
+from ..subtitle import Subtitle, is_valid_subtitle, sanitize_string
 from ..video import Episode
 
 
@@ -53,49 +52,20 @@ class Addic7edSubtitle(Subtitle):
 
 
 class Addic7edProvider(Provider):
-    languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
+    languages = set([babelfish.Language('por', 'BR')]) | set([babelfish.Language(l)
                  for l in ['ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas',
                            'fin', 'fra', 'glg', 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa',
                            'nld', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha',
-                           'tur', 'ukr', 'vie', 'zho']}
+                           'tur', 'ukr', 'vie', 'zho']])
     video_types = (Episode,)
     server = 'http://www.addic7ed.com'
 
-    def __init__(self, username=None, password=None):
-        if username is not None and password is None or username is None and password is not None:
-            raise ProviderConfigurationError('Username and password must be specified')
-        self.username = username
-        self.password = password
-        self.logged_in = False
-
     def initialize(self):
         self.session = requests.Session()
-        self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
-        # login
-        if self.username is not None and self.password is not None:
-            logger.debug('Logging in')
-            data = {'username': self.username, 'password': self.password, 'Submit': 'Log in'}
-            try:
-                r = self.session.post(self.server + '/dologin.php', data, timeout=10, allow_redirects=False)
-            except requests.Timeout:
-                raise ProviderNotAvailable('Timeout after 10 seconds')
-            if r.status_code == 302:
-                logger.info('Logged in')
-                self.logged_in = True
-            else:
-                logger.error('Failed to login')
-
-    def terminate(self):
-        # logout
-        if self.logged_in:
-            try:
-                r = self.session.get(self.server + '/logout.php', timeout=10)
-                logger.info('Logged out')
-            except requests.Timeout:
-                raise ProviderNotAvailable('Timeout after 10 seconds')
-            if r.status_code != 200:
-                raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
-        self.session.close()
+        self.session.headers = {
+            'User-Agent': self.random_user_agent,
+            'Referer': self.server,
+        }
 
     def get(self, url, params=None):
         """Make a GET request on `url` with the given parameters
@@ -126,7 +96,8 @@ class Addic7edProvider(Provider):
         soup = self.get('/shows.php')
         show_ids = {}
         for html_show in soup.select('td.version > h3 > a[href^="/show/"]'):
-            show_ids[html_show.string.lower()] = int(html_show['href'][6:])
+            show_ids[sanitize_string(html_show.string)] = \
+                    int(html_show['href'][6:])
         return show_ids
 
     @region.cache_on_arguments()
@@ -150,10 +121,11 @@ class Addic7edProvider(Provider):
 
     def query(self, series, season):
         show_ids = self.get_show_ids()
-        if series.lower() in show_ids:
-            show_id = show_ids[series.lower()]
+        sanitized_series = sanitize_string(series)
+        if sanitized_series in show_ids:
+            show_id = show_ids[sanitized_series]
         else:
-            show_id = self.find_show_id(series.lower())
+            show_id = self.find_show_id(sanitized_series)
             if show_id is None:
                 return []
         params = {'show_id': show_id, 'season': season}
@@ -166,6 +138,9 @@ class Addic7edProvider(Provider):
             if cells[5].string != 'Completed':
                 logger.debug('Skipping incomplete subtitle')
                 continue
+            if not cells[3].string:
+                logger.debug('Skipping empty language')
+                continue
             subtitles.append(Addic7edSubtitle(babelfish.Language.fromaddic7ed(cells[3].string), series, season,
                                               int(cells[1].string), cells[2].string, cells[4].string,
                                               bool(cells[6].string), cells[9].a['href'], link))

subliminal/providers/bierdopje.py

@@ -1,135 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-import logging
-import urllib
-import babelfish
-import charade
-import guessit
-import requests
-import xml.etree.ElementTree
-from . import Provider
-from .. import __version__
-from ..cache import region
-from ..exceptions import InvalidSubtitle, ProviderNotAvailable, ProviderError
-from ..subtitle import Subtitle, is_valid_subtitle, compute_guess_matches
-from ..video import Episode
-
-
-logger = logging.getLogger(__name__)
-
-
-class BierDopjeSubtitle(Subtitle):
-    provider_name = 'bierdopje'
-
-    def __init__(self, language, season, episode, tvdb_id, series, filename, download_link):
-        super(BierDopjeSubtitle, self).__init__(language)
-        self.season = season
-        self.episode = episode
-        self.tvdb_id = tvdb_id
-        self.series = series
-        self.filename = filename
-        self.download_link = download_link
-
-    def compute_matches(self, video):
-        matches = set()
-        # tvdb_id
-        if video.tvdb_id and self.tvdb_id == video.tvdb_id:
-            matches.add('tvdb_id')
-        # series
-        if video.series and self.series == video.series:
-            matches.add('series')
-        # season
-        if video.season and self.season == video.season:
-            matches.add('season')
-        # episode
-        if video.episode and self.episode == video.episode:
-            matches.add('episode')
-        matches |= compute_guess_matches(video, guessit.guess_episode_info(self.filename + '.mkv'))
-        return matches
-
-
-class BierDopjeProvider(Provider):
-    languages = {babelfish.Language(l) for l in ['eng', 'nld']}
-    video_types = (Episode,)
-
-    def initialize(self):
-        self.session = requests.Session()
-        self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
-
-    def terminate(self):
-        self.session.close()
-
-    def get(self, url, **params):
-        """Make a GET request on the `url` formatted with `**params`
-
-        :param string url: API part of the URL to reach without the leading slash
-        :param \*\*params: format specs for the `url`
-        :return: the response
-        :rtype: :class:`xml.etree.ElementTree.Element`
-        :raise: :class:`~subliminal.exceptions.ProviderNotAvailable`
-
-        """
-        try:
-            r = self.session.get('http://api.bierdopje.com/A2B638AC5D804C2E/' + url.format(**params), timeout=10)
-        except requests.Timeout:
-            raise ProviderNotAvailable('Timeout after 10 seconds')
-        if r.status_code == 429:
-            raise ProviderNotAvailable('Too Many Requests')
-        elif r.status_code != 200:
-            raise ProviderError('Request failed with status code %d' % r.status_code)
-        return xml.etree.ElementTree.fromstring(r.content)
-
-    @region.cache_on_arguments()
-    def find_show_id(self, series):
-        """Find the show id from series name
-
-        :param string series: series of the episode
-        :return: show id
-        :rtype: int
-
-        """
-        logger.debug('Searching for series %r', series)
-        root = self.get('FindShowByName/{series}', series=urllib.quote(series))
-        if root.find('response/status').text == 'false':
-            logger.info('Series %r not found', series)
-            return None
-        return int(root.find('response/results/result[1]/showid').text)
-
-    def query(self, language, season, episode, tvdb_id=None, series=None):
-        params = {'language': language.alpha2, 'season': season, 'episode': episode}
-        if tvdb_id is not None:
-            params['showid'] = tvdb_id
-            params['istvdbid'] = 'true'
-        elif series is not None:
-            show_id = self.find_show_id(series)
-            if show_id is None:
-                return []
-            params['showid'] = show_id
-            params['istvdbid'] = 'false'
-        else:
-            raise ValueError('Missing parameter tvdb_id or series')
-        logger.debug('Searching subtitles %r', params)
-        root = self.get('GetAllSubsFor/{showid}/{season}/{episode}/{language}/{istvdbid}', **params)
-        if root.find('response/status').text == 'false':
-            logger.debug('No subtitle found')
-            return []
-        logger.debug('Found subtitles %r', root.find('response/results'))
-        return [BierDopjeSubtitle(language, season, episode, tvdb_id, series, result.find('filename').text,
-                                  result.find('downloadlink').text) for result in root.find('response/results')]
-
-    def list_subtitles(self, video, languages):
-        return [s for l in languages for s in self.query(l, video.season, video.episode, video.tvdb_id, video.series)]
-
-    def download_subtitle(self, subtitle):
-        try:
-            r = self.session.get(subtitle.download_link, timeout=10)
-        except requests.Timeout:
-            raise ProviderNotAvailable('Timeout after 10 seconds')
-        if r.status_code == 429:
-            raise ProviderNotAvailable('Too Many Requests')
-        elif r.status_code != 200:
-            raise ProviderError('Request failed with status code %d' % r.status_code)
-        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'], 'replace')
-        if not is_valid_subtitle(subtitle_text):
-            raise InvalidSubtitle
-        return subtitle_text

subliminal/providers/opensubtitles.py

@@ -13,6 +13,7 @@ from . import Provider
 from .. import __version__
 from ..exceptions import ProviderError, ProviderNotAvailable, InvalidSubtitle
 from ..subtitle import Subtitle, is_valid_subtitle, compute_guess_matches
+from ..subtitle import sanitize_string
 from ..video import Episode, Movie
 
 
@@ -23,8 +24,8 @@ class OpenSubtitlesSubtitle(Subtitle):
     provider_name = 'opensubtitles'
     series_re = re.compile('^"(?P<series_name>.*)" (?P<series_title>.*)$')
 
-    def __init__(self, language, hearing_impaired, id, matched_by, movie_kind, hash, movie_name, movie_release_name, movie_year,
-                 movie_imdb_id, series_season, series_episode):
+    def __init__(self, language, hearing_impaired, id, matched_by, movie_kind, hash, movie_name, movie_release_name,  # @ReservedAssignment
+                 movie_year, movie_imdb_id, series_season, series_episode):
         super(OpenSubtitlesSubtitle, self).__init__(language, hearing_impaired)
         self.id = id
         self.matched_by = matched_by
@@ -50,7 +51,9 @@ class OpenSubtitlesSubtitle(Subtitle):
         # episode
         if isinstance(video, Episode) and self.movie_kind == 'episode':
             # series
-            if video.series and self.series_name.lower() == video.series.lower():
+            if video.series and \
+                    sanitize_string(self.series_name) == \
+                    sanitize_string(video.series):
                 matches.add('series')
             # season
             if video.season and self.series_season == video.season:
@@ -77,13 +80,15 @@ class OpenSubtitlesSubtitle(Subtitle):
         if video.imdb_id and self.movie_imdb_id == video.imdb_id:
             matches.add('imdb_id')
         # title
-        if video.title and self.movie_name.lower() == video.title.lower():
+        if video.title and \
+                sanitize_string(self.movie_name) == \
+                sanitize_string(video.title):
             matches.add('title')
         return matches
 
 
 class OpenSubtitlesProvider(Provider):
-    languages = {babelfish.Language.fromopensubtitles(l) for l in babelfish.CONVERTERS['opensubtitles'].codes}
+    languages = set([babelfish.Language.fromopensubtitles(l) for l in babelfish.language_converters['opensubtitles'].codes])
 
     def __init__(self):
         self.server = xmlrpclib.ServerProxy('http://api.opensubtitles.org/xml-rpc')
@@ -106,7 +111,7 @@ class OpenSubtitlesProvider(Provider):
         if response['status'] != '200 OK':
             raise ProviderError('Logout failed with status %r' % response['status'])
 
-    def query(self, languages, hash=None, size=None, imdb_id=None, query=None):
+    def query(self, languages, hash=None, size=None, imdb_id=None, query=None):  # @ReservedAssignment
         searches = []
         if hash and size:
             searches.append({'moviehash': hash, 'moviebytesize': str(size)})

subliminal/providers/podnapisi.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import io
 import logging
 import re
+import contextlib
 import xml.etree.ElementTree
 import zipfile
 import babelfish
@@ -11,25 +12,28 @@ import charade
 import guessit
 import requests
 from . import Provider
-from .. import __version__
 from ..exceptions import InvalidSubtitle, ProviderNotAvailable, ProviderError
 from ..subtitle import Subtitle, is_valid_subtitle, compute_guess_matches
+from ..subtitle import sanitize_string
 from ..video import Episode, Movie
 
 
 logger = logging.getLogger(__name__)
-
+URL_RE = re.compile(
+    '^((http[s]?|ftp):\/)?\/?([^:\/\s]+)(:([^\/]*))?((\/\w+)*\/)' + \
+    '([\w\-\.]+[^#?\s]+)(\?([^#]*))?(#(.*))?$',
+)
 
 class PodnapisiSubtitle(Subtitle):
     provider_name = 'podnapisi'
 
-    def __init__(self, language, id, releases, hearing_impaired, link, series=None, season=None, episode=None,
+    def __init__(self, language, id, releases, hearing_impaired, link, series=None, season=None, episode=None,  # @ReservedAssignment
                  title=None, year=None):
         super(PodnapisiSubtitle, self).__init__(language, hearing_impaired)
         self.id = id
         self.releases = releases
         self.hearing_impaired = hearing_impaired
-        self.link = link
+        self.link = '/ppodnapisi' + link
         self.series = series
         self.season = season
         self.episode = episode
@@ -41,7 +45,9 @@ class PodnapisiSubtitle(Subtitle):
         # episode
         if isinstance(video, Episode):
             # series
-            if video.series and self.series.lower() == video.series.lower():
+            if video.series and \
+                sanitize_string(self.series) == \
+                sanitize_string(video.series):
                 matches.add('series')
             # season
             if video.season and self.season == video.season:
@@ -55,7 +61,9 @@ class PodnapisiSubtitle(Subtitle):
         # movie
         elif isinstance(video, Movie):
             # title
-            if video.title and self.title.lower() == video.title.lower():
+            if video.title and \
+                sanitize_string(self.title) == \
+                sanitize_string(video.title):
                 matches.add('title')
             # year
             if video.year and self.year == video.year:
@@ -67,31 +75,43 @@ class PodnapisiSubtitle(Subtitle):
 
 
 class PodnapisiProvider(Provider):
-    languages = {babelfish.Language.frompodnapisi(l) for l in babelfish.CONVERTERS['podnapisi'].codes}
+    languages = set([babelfish.Language.frompodnapisi(l) for l in babelfish.language_converters['podnapisi'].codes])
     video_types = (Episode, Movie)
     server = 'http://simple.podnapisi.net'
-    link_re = re.compile('^.*(?P<link>/ppodnapisi/download/i/\d+/k/.*$)')
+    pre_link_re = re.compile('^.*(?P<link>/ppodnapisi/predownload/i/\d+/k/.*$)')
+    link_re = re.compile('^.*(?P<link>/[a-zA-Z]{2}/ppodnapisi/download/i/\d+/k/.*$)')
 
     def initialize(self):
         self.session = requests.Session()
-        self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
+        self.session.headers = {'User-Agent': self.primary_user_agent }
 
     def terminate(self):
         self.session.close()
 
-    def get(self, url, params=None, is_xml=True):
+    def get(self, url, params=None, headers=None, is_xml=True):
         """Make a GET request on `url` with the given parameters
 
         :param string url: part of the URL to reach with the leading slash
         :param dict params: params of the request
+        :param dict headers: headers of the request
         :param bool xml: whether the response content is XML or not
         :return: the response
         :rtype: :class:`xml.etree.ElementTree.Element` or :class:`bs4.BeautifulSoup`
         :raise: :class:`~subliminal.exceptions.ProviderNotAvailable`
 
         """
+
+        prefix_url = ''
+        url_result = URL_RE.search(url)
+        if url_result and url_result.group(2) is None:
+            prefix_url = self.server
+
         try:
-            r = self.session.get(self.server + '/ppodnapisi' + url, params=params, timeout=10)
+            r = self.session.get(
+                prefix_url + url, params=params,
+                headers=headers,
+                timeout=10,
+            )
         except requests.Timeout:
             raise ProviderNotAvailable('Timeout after 10 seconds')
         if r.status_code != 200:
@@ -116,20 +136,30 @@ class PodnapisiProvider(Provider):
         logger.debug('Searching episode %r', params)
         subtitles = []
         while True:
-            root = self.get('/search', params)
+            root = self.get('/ppodnapisi/search', params)
             if not int(root.find('pagination/results').text):
                 logger.debug('No subtitle found')
                 break
             if series and season and episode:
-                subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
+                try:
+                    subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
                                                     'h' in (s.find('flags').text or ''), s.find('url').text[38:],
                                                     series=series, season=season, episode=episode)
                                   for s in root.findall('subtitle')])
+                except AttributeError:
+                    # there simply wasn't enough information in the TV Show
+                    # gracefully handle this instead of crashing :)
+                    break
             elif title:
-                subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
+                try:
+                    subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
                                                     'h' in (s.find('flags').text or ''), s.find('url').text[38:],
                                                     title=title, year=year)
                                   for s in root.findall('subtitle')])
+                except AttributeError:
+                    # there simply wasn't enough information in the movie
+                    # gracefully handle this instead of crashing :)
+                    break
             if int(root.find('pagination/current').text) >= int(root.find('pagination/count').text):
                 break
             params['page'] = int(root.find('pagination/current').text) + 1
@@ -144,6 +174,21 @@ class PodnapisiProvider(Provider):
 
     def download_subtitle(self, subtitle):
         soup = self.get(subtitle.link, is_xml=False)
+        pre_link = soup.find('a', href=self.pre_link_re)
+        if not pre_link:
+            raise ProviderError('Cannot find the pre-download link')
+        pre_link = self.server + \
+            self.pre_link_re.match(pre_link['href']).group('link')
+
+        # Continue following the link
+        soup = self.get(
+            pre_link,
+            headers={
+                'Referer': self.server,
+            },
+            is_xml=False,
+        )
+
         link = soup.find('a', href=self.link_re)
         if not link:
             raise ProviderError('Cannot find the download link')
@@ -153,7 +198,7 @@ class PodnapisiProvider(Provider):
             raise ProviderNotAvailable('Timeout after 10 seconds')
         if r.status_code != 200:
             raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
-        with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+        with contextlib.closing(zipfile.ZipFile(io.BytesIO(r.content))) as zf:
             if len(zf.namelist()) > 1:
                 raise ProviderError('More than one file to unzip')
             subtitle_bytes = zf.read(zf.namelist()[0])

subliminal/providers/thesubdb.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 class TheSubDBSubtitle(Subtitle):
     provider_name = 'thesubdb'
 
-    def __init__(self, language, hash):
+    def __init__(self, language, hash):  # @ReservedAssignment
         super(TheSubDBSubtitle, self).__init__(language)
         self.hash = hash
 
@@ -29,7 +29,7 @@ class TheSubDBSubtitle(Subtitle):
 
 
 class TheSubDBProvider(Provider):
-    languages = {babelfish.Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']}
+    languages = set([babelfish.Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']])
     required_hash = 'thesubdb'
 
     def initialize(self):
@@ -55,7 +55,7 @@ class TheSubDBProvider(Provider):
             raise ProviderNotAvailable('Timeout after 10 seconds')
         return r
 
-    def query(self, hash):
+    def query(self, hash):  # @ReservedAssignment
         params = {'action': 'search', 'hash': hash}
         logger.debug('Searching subtitles %r', params)
         r = self.get(params)
@@ -65,7 +65,7 @@ class TheSubDBProvider(Provider):
         elif r.status_code != 200:
             raise ProviderError('Request failed with status code %d' % r.status_code)
         return [TheSubDBSubtitle(language, hash) for language in
-                {babelfish.Language.fromalpha2(l) for l in r.content.split(',')}]
+                set([babelfish.Language.fromalpha2(l) for l in r.content.split(',')])]
 
     def list_subtitles(self, video, languages):
         return [s for s in self.query(video.hashes['thesubdb']) if s.language in languages]

subliminal/providers/tvsubtitles.py

@@ -3,18 +3,19 @@ from __future__ import unicode_literals
 import io
 import logging
 import re
+import contextlib
 import zipfile
 import babelfish
 import bs4
 import charade
 import requests
 from . import Provider
-from .. import __version__
 from ..cache import region
 from ..exceptions import InvalidSubtitle, ProviderNotAvailable, ProviderError
-from ..subtitle import Subtitle, is_valid_subtitle
+from ..subtitle import Subtitle, is_valid_subtitle, sanitize_string
 from ..video import Episode
 
+IGNORE_DATEMATCH=re.compile('^(.*)[ \t0-9-._)(]*$')
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__)
 class TVsubtitlesSubtitle(Subtitle):
     provider_name = 'tvsubtitles'
 
-    def __init__(self, language, series, season, episode, id, rip, release):
+    def __init__(self, language, series, season, episode, id, rip, release):  # @ReservedAssignment
         super(TVsubtitlesSubtitle, self).__init__(language)
         self.series = series
         self.season = season
@@ -56,9 +57,9 @@ class TVsubtitlesSubtitle(Subtitle):
 
 
 class TVsubtitlesProvider(Provider):
-    languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
+    languages = set([babelfish.Language('por', 'BR')]) | set([babelfish.Language(l)
                  for l in ['ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor',
-                           'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho']}
+                           'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho']])
     video_types = (Episode,)
     server = 'http://www.tvsubtitles.net'
     episode_id_re = re.compile('^episode-\d+\.html$')
@@ -67,7 +68,7 @@ class TVsubtitlesProvider(Provider):
 
     def initialize(self):
         self.session = requests.Session()
-        self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
+        self.session.headers = {'User-Agent': self.primary_user_agent }
 
     def terminate(self):
         self.session.close()
@@ -105,15 +106,34 @@ class TVsubtitlesProvider(Provider):
         logger.debug('Searching series %r', data)
         soup = self.request('/search.php', data=data, method='POST')
         links = soup.select('div.left li div a[href^="/tvshow-"]')
+        sanitized_series = IGNORE_DATEMATCH.match(
+            sanitize_string(series).replace('.', ' ').strip(),
+        )
+        if not sanitized_series:
+            sanitized_series = sanitize_string(series)\
+                        .replace('.', ' ').strip()
+        else:
+            sanitized_series = sanitized_series.group(1)
+
         if not links:
             logger.info('Series %r not found', series)
             return None
+
         for link in links:
             match = self.link_re.match(link.string)
             if not match:
                 logger.warning('Could not parse %r', link.string)
                 continue
-            if match.group('series').lower().replace('.', ' ').strip() == series.lower():
+            show = IGNORE_DATEMATCH.match(
+                sanitize_string(match.group('series'))\
+                        .replace('.', ' ').strip(),
+            )
+            if not show:
+                logger.warning('Could not postparse %r', match.group('series'))
+                continue
+            show = show.group(1)
+
+            if show == sanitized_series:
                 return int(link['href'][8:-5])
         return int(links[0]['href'][8:-5])
 
@@ -139,7 +159,7 @@ class TVsubtitlesProvider(Provider):
         return episode_ids
 
     def query(self, series, season, episode):
-        show_id = self.find_show_id(series.lower())
+        show_id = self.find_show_id(series)
         if show_id is None:
             return []
         episode_ids = self.find_episode_ids(show_id, season)
@@ -165,7 +185,7 @@ class TVsubtitlesProvider(Provider):
             raise ProviderNotAvailable('Timeout after 10 seconds')
         if r.status_code != 200:
             raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
-        with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+        with contextlib.closing(zipfile.ZipFile(io.BytesIO(r.content))) as zf:
             if len(zf.namelist()) > 1:
                 raise ProviderError('More than one file to unzip')
             subtitle_bytes = zf.read(zf.namelist()[0])

subliminal/score.py

@@ -6,7 +6,7 @@ from sympy import Eq, symbols, solve
 
 # Symbols
 release_group, resolution, video_codec, audio_codec = symbols('release_group resolution video_codec audio_codec')
-imdb_id, hash, title, series, tvdb_id, season, episode = symbols('imdb_id hash title series tvdb_id season episode')
+imdb_id, hash, title, series, tvdb_id, season, episode = symbols('imdb_id hash title series tvdb_id season episode')  # @ReservedAssignment
 year = symbols('year')
 
 

subliminal/subtitle.py

@@ -4,11 +4,15 @@ import logging
 import os.path
 import babelfish
 import pysrt
+import re
 from .video import Episode, Movie
 
 
 logger = logging.getLogger(__name__)
 
+#:  The following characters are always stripped
+IGNORED_CHARACTERS_RE = re.compile('[!@#$\'"]')
+
 
 class Subtitle(object):
     """Base class for subtitle
@@ -33,7 +37,7 @@ class Subtitle(object):
         """
         raise NotImplementedError
 
-    def compute_score(self, video):
+    def compute_score(self, video, hi_score_adjust=0):
         """Compute the score of the subtitle against the `video`
 
         There are equivalent matches so that a provider can match one element or its equivalent. This is
@@ -47,6 +51,7 @@ class Subtitle(object):
 
         :param video: the video to compute the score against
         :type video: :class:`~subliminal.video.Video`
+        :param hi_score_adjust: adjust hearing impaired matched videos by this value
         :return: score of the subtitle
         :rtype: int
 
@@ -62,20 +67,45 @@ class Subtitle(object):
             # remove equivalences
             if isinstance(video, Episode):
                 if 'imdb_id' in matches:
-                    matches -= {'series', 'tvdb_id', 'season', 'episode', 'title'}
+                    matches -= set(['series', 'tvdb_id', 'season', 'episode', 'title'])
                 if 'tvdb_id' in matches:
-                    matches -= {'series'}
+                    matches -= set(['series',])
                 if 'title' in matches:
-                    matches -= {'season', 'episode'}
+                    matches -= set(['season', 'episode'])
             # add other scores
-            score += sum((video.scores[match] for match in matches))
-        logger.info('Computed score %d with matches %r', score, initial_matches)
+            score += sum([video.scores[match] for match in matches])
+
+            # Adjust scoring if hearing impaired subtitles are detected
+            if self.hearing_impaired and hi_score_adjust != 0:
+                logger.debug('Hearing impaired subtitle score adjusted ' + \
+                             'by %d' % hi_score_adjust)
+                # Priortization (adjust score)
+                score += hi_score_adjust
+
+        logger.debug('Computed score %d with matches %r', score, initial_matches)
         return score
 
     def __repr__(self):
         return '<%s [%s]>' % (self.__class__.__name__, self.language)
 
 
+def sanitize_string(str_in):
+    """
+    Sanitizes a string passed into it by eliminating characters that might
+    otherwise cause issues when attempting to locate a match on websites by
+    striping out any special characters and forcing a consistent string that
+    can be used for caching too.
+
+    :param string str_in: the string to sanitize
+    :return: sanitized string
+    :rtype: string
+    """
+    if not isinstance(str_in, basestring):
+        # handle int, float, etc
+        str_in = str(str_in)
+
+    return IGNORED_CHARACTERS_RE.sub('', str_in).lower().strip()
+
 def get_subtitle_path(video_path, language=None):
     """Create the subtitle path from the given `video_path` and `language`
 
@@ -90,7 +120,7 @@ def get_subtitle_path(video_path, language=None):
     if language is not None:
         try:
             return subtitle_path + '.%s.%s' % (language.alpha2, 'srt')
-        except babelfish.ConvertError:
+        except babelfish.LanguageConvertError:
             return subtitle_path + '.%s.%s' % (language.alpha3, 'srt')
     return subtitle_path + '.srt'
 
@@ -105,8 +135,9 @@ def is_valid_subtitle(subtitle_text):
     try:
         pysrt.from_string(subtitle_text, error_handling=pysrt.ERROR_RAISE)
         return True
-    except pysrt.Error:
-        pass
+    except pysrt.Error as e:
+        if e.args[0] > 80:
+            return True
     except:
         logger.exception('Unexpected error when validating subtitle')
     return False

subliminal/tests/__init__.py

@@ -6,7 +6,7 @@ from subliminal import cache_region
 from . import test_providers, test_subliminal
 
 
-cache_region.configure('dogpile.cache.memory', expiration_time=60 * 30)
+cache_region.configure('dogpile.cache.memory', expiration_time=60 * 30)  # @UndefinedVariable
 suite = TestSuite([test_providers.suite(), test_subliminal.suite()])
 
 

subliminal/tests/test_providers.py

@@ -25,135 +25,66 @@ class Addic7edProviderTestCase(ProviderTestCase):
     def test_find_show_id(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('The Big Bang')
-        self.assertTrue(show_id == 126)
+        self.assertEqual(show_id, 126)
 
     def test_find_show_id_error(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('the big how i met your mother')
-        self.assertTrue(show_id is None)
+        self.assertEqual(show_id, None)
 
     def test_get_show_ids(self):
         with self.Provider() as provider:
             show_ids = provider.get_show_ids()
-        self.assertTrue('the big bang theory' in show_ids and show_ids['the big bang theory'] == 126)
+        self.assertTrue('the big bang theory' in show_ids)
+        self.assertEqual(show_ids['the big bang theory'], 126)
 
     def test_query_episode_0(self):
         video = EPISODES[0]
-        languages = {Language('tur'), Language('rus'), Language('heb'), Language('ita'), Language('fra'),
+        languages = set([Language('rus'), Language('heb'), Language('ita'), Language('fra'),
                      Language('ron'), Language('nld'), Language('eng'), Language('deu'), Language('ell'),
-                     Language('por', 'BR'), Language('bul')}
-        matches = {frozenset(['episode', 'release_group', 'title', 'series', 'resolution', 'season']),
+                     Language('por', 'BR'), Language('bul')])
+        matches = set([frozenset(['episode', 'release_group', 'title', 'series', 'resolution', 'season']),
                    frozenset(['series', 'resolution', 'season']),
                    frozenset(['series', 'episode', 'season', 'title']),
                    frozenset(['series', 'release_group', 'season']),
                    frozenset(['series', 'episode', 'season', 'release_group', 'title']),
-                   frozenset(['series', 'season'])}
+                   frozenset(['series', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(video.series, video.season)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_1(self):
         video = EPISODES[1]
-        languages = {Language('ind'), Language('spa'), Language('hrv'), Language('ita'), Language('fra'),
+        languages = set([Language('ind'), Language('spa'), Language('hrv'), Language('ita'), Language('fra'),
                      Language('cat'), Language('ell'), Language('nld'), Language('eng'), Language('fas'),
                      Language('por'), Language('nor'), Language('deu'), Language('ron'), Language('por', 'BR'),
-                     Language('bul')}
-        matches = {frozenset(['series', 'episode', 'resolution', 'season', 'title']),
+                     Language('bul')])
+        matches = set([frozenset(['series', 'episode', 'resolution', 'season', 'title']),
                    frozenset(['series', 'resolution', 'season']),
                    frozenset(['series', 'episode', 'season', 'title']),
                    frozenset(['series', 'release_group', 'season']),
                    frozenset(['series', 'resolution', 'release_group', 'season']),
                    frozenset(['series', 'episode', 'season', 'release_group', 'title']),
-                   frozenset(['series', 'season'])}
+                   frozenset(['series', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(video.series, video.season)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_list_subtitles(self):
         video = EPISODES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['series', 'episode', 'season', 'release_group', 'title']),
-                   frozenset(['series', 'episode', 'season', 'title'])}
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['series', 'episode', 'season', 'release_group', 'title']),
+                   frozenset(['series', 'episode', 'season', 'title'])])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_download_subtitle(self):
         video = EPISODES[0]
-        languages = {Language('eng'), Language('fra')}
-        with self.Provider() as provider:
-            subtitles = provider.list_subtitles(video, languages)
-            subtitle_text = provider.download_subtitle(subtitles[0])
-        self.assertTrue(is_valid_subtitle(subtitle_text))
-
-
-class BierDopjeProviderTestCase(ProviderTestCase):
-    provider_name = 'bierdopje'
-
-    def test_find_show_id(self):
-        with self.Provider() as provider:
-            show_id = provider.find_show_id('The Big Bang')
-        self.assertTrue(show_id == 9203)
-
-    def test_find_show_id_error(self):
-        with self.Provider() as provider:
-            show_id = provider.find_show_id('the big how i met your mother')
-        self.assertTrue(show_id is None)
-
-    def test_query_episode_0(self):
-        video = EPISODES[0]
-        language = Language('eng')
-        matches = {frozenset(['series', 'video_codec', 'resolution', 'episode', 'season']),
-                   frozenset(['season', 'video_codec', 'episode', 'series']),
-                   frozenset(['episode', 'video_codec', 'season', 'series', 'resolution', 'release_group'])}
-        with self.Provider() as provider:
-            subtitles = provider.query(language, video.season, video.episode, series=video.series)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
-
-    def test_query_episode_1(self):
-        video = EPISODES[1]
-        language = Language('nld')
-        matches = {frozenset(['series', 'video_codec', 'resolution', 'episode', 'season']),
-                   frozenset(['season', 'video_codec', 'episode', 'series']),
-                   frozenset(['series', 'episode', 'season']),
-                   frozenset(['season', 'video_codec', 'episode', 'release_group', 'series']),
-                   frozenset(['episode', 'video_codec', 'season', 'series', 'resolution', 'release_group'])}
-        with self.Provider() as provider:
-            subtitles = provider.query(language, video.season, video.episode, series=video.series)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
-
-    def test_query_episode_0_tvdb_id(self):
-        video = EPISODES[0]
-        language = Language('eng')
-        matches = {frozenset(['video_codec', 'tvdb_id', 'episode', 'season', 'series']),
-                   frozenset(['episode', 'video_codec', 'series', 'season', 'tvdb_id', 'resolution', 'release_group']),
-                   frozenset(['episode', 'series', 'video_codec', 'tvdb_id', 'resolution', 'season'])}
-        with self.Provider() as provider:
-            subtitles = provider.query(language, video.season, video.episode, tvdb_id=video.tvdb_id)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
-
-    def test_list_subtitles(self):
-        video = EPISODES[1]
-        languages = {Language('eng'), Language('nld')}
-        matches = {frozenset(['series', 'video_codec', 'tvdb_id', 'episode', 'season']),
-                   frozenset(['episode', 'video_codec', 'season', 'series', 'tvdb_id', 'resolution', 'release_group']),
-                   frozenset(['season', 'tvdb_id', 'episode', 'series']),
-                   frozenset(['episode', 'video_codec', 'season', 'series', 'tvdb_id', 'resolution']),
-                   frozenset(['episode', 'video_codec', 'season', 'series', 'tvdb_id', 'release_group'])}
-        with self.Provider() as provider:
-            subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
-
-    def test_download_subtitle(self):
-        video = EPISODES[0]
-        languages = {Language('eng'), Language('nld')}
+        languages = set([Language('eng'), Language('fra')])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
             subtitle_text = provider.download_subtitle(subtitles[0])
@@ -165,113 +96,115 @@ class OpenSubtitlesProviderTestCase(ProviderTestCase):
 
     def test_query_movie_0_query(self):
         video = MOVIES[0]
-        languages = {Language('eng')}
-        matches = {frozenset([]), frozenset(['imdb_id', 'resolution', 'title', 'year']),
+        languages = set([Language('eng'), ])
+        matches = set([frozenset([]), frozenset(['imdb_id', 'resolution', 'title', 'year']),
                    frozenset(['imdb_id', 'title', 'year']),
                    frozenset(['imdb_id', 'video_codec', 'title', 'year']),
                    frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year']),
-                   frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group'])}
+                   frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, query=video.title)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_0_query(self):
         video = EPISODES[0]
-        languages = {Language('eng')}
-        matches = {frozenset(['series', 'episode', 'season', 'imdb_id']),
+        languages = set([Language('eng'), ])
+        matches = set([frozenset(['series', 'episode', 'season', 'imdb_id']),
                    frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season']),
-                   frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])}
+                   frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, query=video.name.split(os.sep)[-1])
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_1_query(self):
         video = EPISODES[1]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season']),
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season']),
                    frozenset(['series', 'imdb_id', 'title', 'episode', 'season']),
                    frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season']),
                    frozenset(['episode', 'video_codec', 'series', 'imdb_id', 'resolution', 'season']),
                    frozenset(['series', 'imdb_id', 'resolution', 'episode', 'season']),
-                   frozenset(['series', 'episode', 'season', 'imdb_id'])}
+                   frozenset(['series', 'episode', 'season', 'imdb_id'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, query=video.name.split(os.sep)[-1])
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_movie_0_imdb_id(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['imdb_id', 'video_codec', 'title', 'year']),
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['imdb_id', 'video_codec', 'title', 'year']),
                    frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year']),
                    frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group']),
                    frozenset(['imdb_id', 'title', 'year']),
-                   frozenset(['imdb_id', 'resolution', 'title', 'year'])}
+                   frozenset(['imdb_id', 'resolution', 'title', 'year'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, imdb_id=video.imdb_id)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_0_imdb_id(self):
         video = EPISODES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['series', 'episode', 'season', 'imdb_id']),
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['series', 'episode', 'season', 'imdb_id']),
                    frozenset(['episode', 'release_group', 'video_codec', 'series', 'imdb_id', 'resolution', 'season']),
                    frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season']),
-                   frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])}
+                   frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, imdb_id=video.imdb_id)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_movie_0_hash(self):
         video = MOVIES[0]
-        languages = {Language('eng')}
-        matches = {frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id']),
+        languages = set([Language('eng'), ])
+        matches = set([frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id']),
                    frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
                    frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title']),
+                   frozenset([]),
                    frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title']),
-                   frozenset(['year', 'imdb_id', 'hash', 'title'])}
+                   frozenset(['year', 'imdb_id', 'hash', 'title'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, hash=video.hashes['opensubtitles'], size=video.size)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_0_hash(self):
         video = EPISODES[0]
-        languages = {Language('eng')}
-        matches = {frozenset(['series', 'hash']),
+        languages = set([Language('eng'), ])
+        matches = set([frozenset(['series', 'hash']),
                    frozenset(['episode', 'season', 'series', 'imdb_id', 'video_codec', 'hash']),
                    frozenset(['series', 'episode', 'season', 'hash', 'imdb_id']),
-                   frozenset(['series', 'resolution', 'hash', 'video_codec'])}
+                   frozenset(['series', 'resolution', 'hash', 'video_codec'])])
         with self.Provider() as provider:
             subtitles = provider.query(languages, hash=video.hashes['opensubtitles'], size=video.size)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_list_subtitles(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
                    frozenset(['imdb_id', 'year', 'title']),
                    frozenset(['year', 'video_codec', 'imdb_id', 'resolution', 'title']),
                    frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
                    frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title']),
+                   frozenset([]),
                    frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title']),
                    frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id']),
                    frozenset(['year', 'imdb_id', 'hash', 'title']),
                    frozenset(['video_codec', 'imdb_id', 'year', 'title']),
-                   frozenset(['year', 'imdb_id', 'resolution', 'title'])}
+                   frozenset(['year', 'imdb_id', 'resolution', 'title'])])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_download_subtitle(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
             subtitle_text = provider.download_subtitle(subtitles[0])
@@ -284,42 +217,44 @@ class PodnapisiProviderTestCase(ProviderTestCase):
     def test_query_movie_0(self):
         video = MOVIES[0]
         language = Language('eng')
-        matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+        matches = set([frozenset(['video_codec', 'title', 'resolution', 'year']),
                    frozenset(['title', 'resolution', 'year']),
                    frozenset(['video_codec', 'title', 'year']),
                    frozenset(['title', 'year']),
-                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year'])}
+                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year']),
+                   frozenset(['video_codec', 'title', 'resolution', 'audio_codec', 'year'])])
         with self.Provider() as provider:
             subtitles = provider.query(language, title=video.title, year=video.year)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue([Language, ], [subtitle.language for subtitle in subtitles])
 
     def test_query_episode_0(self):
         video = EPISODES[0]
         language = Language('eng')
-        matches = {frozenset(['episode', 'series', 'season', 'video_codec', 'resolution', 'release_group']),
-                   frozenset(['season', 'video_codec', 'episode', 'resolution', 'series'])}
+        matches = set([frozenset(['episode', 'series', 'season', 'video_codec', 'resolution', 'release_group']),
+                   frozenset(['season', 'video_codec', 'episode', 'resolution', 'series'])])
         with self.Provider() as provider:
             subtitles = provider.query(language, series=video.series, season=video.season, episode=video.episode)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue([Language, ], [subtitle.language for subtitle in subtitles])
 
     def test_list_subtitles(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['video_codec', 'title', 'resolution', 'year']),
                    frozenset(['title', 'resolution', 'year']),
                    frozenset(['video_codec', 'title', 'year']),
                    frozenset(['title', 'year']),
-                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year'])}
+                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year']),
+                   frozenset(['video_codec', 'title', 'resolution', 'audio_codec', 'year'])])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_download_subtitle(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
             subtitle_text = provider.download_subtitle(subtitles[0])
@@ -331,34 +266,34 @@ class TheSubDBProviderTestCase(ProviderTestCase):
 
     def test_query_episode_0(self):
         video = EPISODES[0]
-        languages = {Language('eng'), Language('spa'), Language('por')}
-        matches = {frozenset(['hash'])}
+        languages = set([Language('eng'), Language('spa'), Language('por')])
+        matches = set([frozenset(['hash']), ])
         with self.Provider() as provider:
             subtitles = provider.query(video.hashes['thesubdb'])
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_1(self):
         video = EPISODES[1]
-        languages = {Language('eng'), Language('por')}
-        matches = {frozenset(['hash'])}
+        languages = set([Language('eng'), Language('por')])
+        matches = set([frozenset(['hash']), ])
         with self.Provider() as provider:
             subtitles = provider.query(video.hashes['thesubdb'])
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_list_subtitles(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('por')}
-        matches = {frozenset(['hash'])}
+        languages = set([Language('eng'), Language('por')])
+        matches = set([frozenset(['hash']), ])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_download_subtitle(self):
         video = MOVIES[0]
-        languages = {Language('eng'), Language('por')}
+        languages = (Language('eng'), Language('por'), )
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
             subtitle_text = provider.download_subtitle(subtitles[0])
@@ -371,62 +306,63 @@ class TVsubtitlesProviderTestCase(ProviderTestCase):
     def test_find_show_id(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('The Big Bang')
-        self.assertTrue(show_id == 154)
+        self.assertEqual(show_id, 154)
 
     def test_find_show_id_ambiguous(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('New Girl')
-        self.assertTrue(show_id == 977)
+        self.assertEqual(show_id, 977)
 
     def test_find_show_id_no_dots(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('Marvel\'s Agents of S H I E L D')
-        self.assertTrue(show_id == 1340)
+        self.assertEqual(show_id, 1340)
 
     def test_find_show_id_error(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('the big gaming')
-        self.assertTrue(show_id is None)
+        self.assertEqual(show_id, None)
 
     def test_find_episode_ids(self):
         with self.Provider() as provider:
             episode_ids = provider.find_episode_ids(154, 5)
-        self.assertTrue(set(episode_ids.keys()) == set(range(1, 25)))
+        self.assertEqual(set(episode_ids.keys()), set(range(1, 25)))
 
     def test_query_episode_0(self):
         video = EPISODES[0]
-        languages = {Language('fra'), Language('por'), Language('hun'), Language('ron'), Language('eng')}
-        matches = {frozenset(['series', 'episode', 'season', 'video_codec']),
-                   frozenset(['series', 'episode', 'season'])}
+        languages = set([Language('fra'), Language('por'), Language('hun'), Language('ron'), Language('eng')])
+        matches = set([frozenset(['series', 'episode', 'resolution', 'season']),
+                   frozenset(['series', 'episode', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(video.series, video.season, video.episode)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_query_episode_1(self):
         video = EPISODES[1]
-        languages = {Language('fra'), Language('ell'), Language('ron'), Language('eng'), Language('hun'),
-                     Language('por'), Language('por', 'BR')}
-        matches = {frozenset(['series', 'episode', 'resolution', 'season']),
+        languages = set([Language('fra'), Language('ell'), Language('ron'), Language('eng'), Language('hun'),
+                     Language('por'), Language('por', 'BR')])
+        matches = set([frozenset(['series', 'episode', 'resolution', 'season']),
                    frozenset(['series', 'episode', 'season', 'video_codec']),
-                   frozenset(['series', 'episode', 'season'])}
+                   frozenset(['series', 'episode', 'season'])])
         with self.Provider() as provider:
             subtitles = provider.query(video.series, video.season, video.episode)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_list_subtitles(self):
         video = EPISODES[0]
-        languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['series', 'episode', 'season'])}
+        languages = set([Language('eng'), Language('fra')])
+        matches = set([frozenset(['series', 'episode', 'resolution', 'season']),
+                      frozenset([u'series', u'episode', u'season'])])
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
-        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
-        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+        self.assertTrue(matches - set([frozenset(subtitle.compute_matches(video)) for subtitle in subtitles]) == set([]))
+        self.assertTrue(languages - set([subtitle.language for subtitle in subtitles]) == set([]))
 
     def test_download_subtitle(self):
         video = EPISODES[0]
-        languages = {Language('hun')}
+        languages = (Language('hun'), )
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
             subtitle_text = provider.download_subtitle(subtitles[0])
@@ -436,7 +372,6 @@ class TVsubtitlesProviderTestCase(ProviderTestCase):
 def suite():
     suite = TestSuite()
     suite.addTest(TestLoader().loadTestsFromTestCase(Addic7edProviderTestCase))
-    suite.addTest(TestLoader().loadTestsFromTestCase(BierDopjeProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(OpenSubtitlesProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(PodnapisiProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(TheSubDBProviderTestCase))

subliminal/tests/test_subliminal.py

@@ -21,30 +21,30 @@ class ApiTestCase(TestCase):
 
     def test_list_subtitles_movie_0(self):
         videos = [MOVIES[0]]
-        languages = {Language('eng')}
+        languages = set([ Language('eng'), ])
         subtitles = list_subtitles(videos, languages)
-        self.assertTrue(len(subtitles) == len(videos))
+        self.assertEqual(len(subtitles), len(videos))
         self.assertTrue(len(subtitles[videos[0]]) > 0)
 
     def test_list_subtitles_movie_0_por_br(self):
         videos = [MOVIES[0]]
-        languages = {Language('por', 'BR')}
+        languages = set([Language('por', 'BR'), ])
         subtitles = list_subtitles(videos, languages)
-        self.assertTrue(len(subtitles) == len(videos))
+        self.assertEqual(len(subtitles), len(videos))
         self.assertTrue(len(subtitles[videos[0]]) > 0)
 
     def test_list_subtitles_episodes(self):
         videos = [EPISODES[0], EPISODES[1]]
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = list_subtitles(videos, languages)
-        self.assertTrue(len(subtitles) == len(videos))
+        self.assertEqual(len(subtitles), len(videos))
         self.assertTrue(len(subtitles[videos[0]]) > 0)
 
     def test_download_subtitles(self):
         videos = [EPISODES[0], EPISODES[1]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = list_subtitles(videos, languages)
         download_subtitles(subtitles)
         for video in videos:
@@ -55,7 +55,7 @@ class ApiTestCase(TestCase):
         videos = [EPISODES[0], EPISODES[1]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = list_subtitles(videos, languages)
         download_subtitles(subtitles, single=True)
         for video in videos:
@@ -65,10 +65,11 @@ class ApiTestCase(TestCase):
         videos = [EPISODES[0], EPISODES[1]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = download_best_subtitles(videos, languages)
         for video in videos:
-            self.assertTrue(video in subtitles and len(subtitles[video]) == 2)
+            self.assertTrue(video in subtitles)
+            self.assertTrue(len(subtitles[video]) == 2)
             self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.en.srt'))
             self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.fr.srt'))
 
@@ -76,27 +77,28 @@ class ApiTestCase(TestCase):
         videos = [EPISODES[0], EPISODES[1]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = download_best_subtitles(videos, languages, single=True)
         for video in videos:
-            self.assertTrue(video in subtitles and len(subtitles[video]) == 1)
+            self.assertTrue(video in subtitles)
+            self.assertEqual(len(subtitles[video]), 1)
             self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.srt'))
 
     def test_download_best_subtitles_min_score(self):
         videos = [MOVIES[0]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng'), Language('fra')}
+        languages = set([Language('eng'), Language('fra')])
         subtitles = download_best_subtitles(videos, languages, min_score=1000)
-        self.assertTrue(len(subtitles) == 0)
+        self.assertEqual(len(subtitles), 0)
 
     def test_download_best_subtitles_hearing_impaired(self):
         videos = [MOVIES[0]]
         for video in videos:
             video.name = os.path.join(TEST_DIR, video.name.split(os.sep)[-1])
-        languages = {Language('eng')}
+        languages = set([Language('eng'), ])
         subtitles = download_best_subtitles(videos, languages, hearing_impaired=True)
-        self.assertTrue(subtitles[videos[0]][0].hearing_impaired == True)
+        self.assertTrue(subtitles[videos[0]][0].hearing_impaired)
 
 
 class VideoTestCase(TestCase):
@@ -111,47 +113,47 @@ class VideoTestCase(TestCase):
     def test_scan_video_movie(self):
         video = MOVIES[0]
         scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.name == os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.title.lower() == video.title.lower())
-        self.assertTrue(scanned_video.year == video.year)
-        self.assertTrue(scanned_video.video_codec == video.video_codec)
-        self.assertTrue(scanned_video.resolution == video.resolution)
-        self.assertTrue(scanned_video.release_group == video.release_group)
-        self.assertTrue(scanned_video.subtitle_languages == set())
-        self.assertTrue(scanned_video.hashes == {})
-        self.assertTrue(scanned_video.audio_codec is None)
-        self.assertTrue(scanned_video.imdb_id is None)
-        self.assertTrue(scanned_video.size == 0)
+        self.assertEqual(scanned_video.name, os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertEqual(scanned_video.title.lower(), video.title.lower())
+        self.assertEqual(scanned_video.year, video.year)
+        self.assertEqual(scanned_video.video_codec, video.video_codec)
+        self.assertEqual(scanned_video.resolution, video.resolution)
+        self.assertEqual(scanned_video.release_group, video.release_group)
+        self.assertEqual(scanned_video.subtitle_languages, set())
+        self.assertEqual(scanned_video.hashes, {})
+        self.assertEqual(scanned_video.audio_codec, None)
+        self.assertEqual(scanned_video.imdb_id, None)
+        self.assertEqual(scanned_video.size, 0)
 
     def test_scan_video_episode(self):
         video = EPISODES[0]
         scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.name == os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.series == video.series)
-        self.assertTrue(scanned_video.season == video.season)
-        self.assertTrue(scanned_video.episode == video.episode)
-        self.assertTrue(scanned_video.video_codec == video.video_codec)
-        self.assertTrue(scanned_video.resolution == video.resolution)
-        self.assertTrue(scanned_video.release_group == video.release_group)
-        self.assertTrue(scanned_video.subtitle_languages == set())
-        self.assertTrue(scanned_video.hashes == {})
-        self.assertTrue(scanned_video.title is None)
-        self.assertTrue(scanned_video.tvdb_id is None)
-        self.assertTrue(scanned_video.imdb_id is None)
-        self.assertTrue(scanned_video.audio_codec is None)
-        self.assertTrue(scanned_video.size == 0)
+        self.assertEqual(scanned_video.name, os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertEqual(scanned_video.series, video.series)
+        self.assertEqual(scanned_video.season, video.season)
+        self.assertEqual(scanned_video.episode, video.episode)
+        self.assertEqual(scanned_video.video_codec, video.video_codec)
+        self.assertEqual(scanned_video.resolution, video.resolution)
+        self.assertEqual(scanned_video.release_group, video.release_group)
+        self.assertEqual(scanned_video.subtitle_languages, set())
+        self.assertEqual(scanned_video.hashes, {})
+        self.assertEqual(scanned_video.title, None)
+        self.assertEqual(scanned_video.tvdb_id, None)
+        self.assertEqual(scanned_video.imdb_id, None)
+        self.assertEqual(scanned_video.audio_codec, None)
+        self.assertEqual(scanned_video.size, 0)
 
     def test_scan_video_subtitle_language_und(self):
         video = EPISODES[0]
         open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
         scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.subtitle_languages == {Language('und')})
+        self.assertEqual(scanned_video.subtitle_languages, set([Language('und'), ]))
 
     def test_scan_video_subtitles_language_eng(self):
         video = EPISODES[0]
         open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.en.srt', 'w').close()
         scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.subtitle_languages == {Language('eng')})
+        self.assertEqual(scanned_video.subtitle_languages, set([Language('eng'), ]))
 
     def test_scan_video_subtitles_languages(self):
         video = EPISODES[0]
@@ -159,7 +161,8 @@ class VideoTestCase(TestCase):
         open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.fr.srt', 'w').close()
         open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
         scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
-        self.assertTrue(scanned_video.subtitle_languages == {Language('eng'), Language('fra'), Language('und')})
+        self.assertEqual(scanned_video.subtitle_languages, set([Language('eng'), Language('fra'), Language('und')]))
+
 
 def suite():
     suite = TestSuite()

subliminal/video.py

@@ -71,6 +71,10 @@ class Video(object):
     def __hash__(self):
         return hash(self.name)
 
+    def __eq__(self, other):
+        return self.__class__.__name__ == other.__class__.__name__\
+                and self.name == other.name
+
 
 class Episode(Video):
     """Episode :class:`Video`
@@ -112,6 +116,18 @@ class Episode(Video):
     def __repr__(self):
         return '<%s [%r, %rx%r]>' % (self.__class__.__name__, self.series, self.season, self.episode)
 
+    def __hash__(self):
+        return hash((
+            self.series,
+            self.season,
+            self.episode,
+        ))
+
+    def __eq__(self, other):
+        return self.__class__.__name__ == other.__class__.__name__\
+                and self.series == other.series\
+                and self.season == other.season\
+                and self.episode == other.episode
 
 class Movie(Video):
     """Movie :class:`Video`
@@ -147,6 +163,18 @@ class Movie(Video):
             return '<%s [%r]>' % (self.__class__.__name__, self.title)
         return '<%s [%r, %r]>' % (self.__class__.__name__, self.title, self.year)
 
+    def __hash__(self):
+        if self.year is None:
+            return hash((
+                self.title,
+                self.year,
+            ))
+        return hash(self.title)
+
+    def __eq__(self, other):
+        return self.__class__.__name__ == other.__class__.__name__\
+                and self.title == other.title\
+                and self.year == other.year
 
 def scan_subtitle_languages(path):
     """Search for subtitles with alpha2 extension from a video `path` and return their language
@@ -156,7 +184,7 @@ def scan_subtitle_languages(path):
     :rtype: set
 
     """
-    language_extensions = tuple('.' + c for c in babelfish.CONVERTERS['alpha2'].codes)
+    language_extensions = tuple('.' + c for c in babelfish.language_converters['alpha2'].codes)
     dirpath, filename = os.path.split(path)
     subtitles = set()
     for p in os.listdir(dirpath):
@@ -169,12 +197,14 @@ def scan_subtitle_languages(path):
     return subtitles
 
 
-def scan_video(path, subtitles=True, embedded_subtitles=True):
+def scan_video(path, subtitles=True, embedded_subtitles=True, video=None):
     """Scan a video and its subtitle languages from a video `path`
 
     :param string path: absolute path to the video
     :param bool subtitles: scan for subtitles with the same name
     :param bool embedded_subtitles: scan for embedded subtitles
+    :parm :class:`Video`: optionally specify a video if you've already detected on
+                          by other means.
     :return: the scanned video
     :rtype: :class:`Video`
     :raise: ValueError if cannot guess enough information from the path
@@ -182,7 +212,12 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
     """
     dirpath, filename = os.path.split(path)
     logger.info('Scanning video %r in %r', filename, dirpath)
-    video = Video.fromguess(path, guessit.guess_file_info(path, 'autodetect'))
+    if not video:
+        video = Video.fromguess(
+            path,
+            guessit.guess_file_info(path, info=['filename']),
+        )
+
     video.size = os.path.getsize(path)
     if video.size > 10485760:
         logger.debug('Size is %d', video.size)
@@ -239,10 +274,20 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
                 if embedded_subtitles:
                     embedded_subtitle_languages = set()
                     for st in mkv.subtitle_tracks:
-                        try:
-                            embedded_subtitle_languages.add(babelfish.Language.fromalpha3b(st.language or 'und'))
-                        except babelfish.Error:
-                            logger.error('Embedded subtitle language %r is not a valid language', st.language)
+                        if st.language:
+                            try:
+                                embedded_subtitle_languages.add(babelfish.Language.fromalpha3b(st.language))
+                            except babelfish.Error:
+                                logger.error('Embedded subtitle track language %r is not a valid language', st.language)
+                                embedded_subtitle_languages.add(babelfish.Language('und'))
+                        elif st.name:
+                            try:
+                                embedded_subtitle_languages.add(babelfish.Language.fromname(st.name))
+                            except babelfish.Error:
+                                logger.error('Embedded subtitle track name %r is not a valid language', st.name)
+                                embedded_subtitle_languages.add(babelfish.Language('und'))
+                        else:
+                            embedded_subtitle_languages.add(babelfish.Language('und'))
                     logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages)
                     video.subtitle_languages |= embedded_subtitle_languages
             else:
@@ -282,12 +327,12 @@ def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
         for dirpath, dirnames, filenames in os.walk(path):
             # skip badly encoded directories
             if isinstance(dirpath, bytes):
-                logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', errors='replace'))
+                logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', 'replace'))
                 continue
             # skip badly encoded and hidden sub directories
             for dirname in list(dirnames):
                 if isinstance(dirname, bytes):
-                    logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', errors='replace'),
+                    logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', 'replace'),
                                  dirpath)
                     dirnames.remove(dirname)
                 elif dirname.startswith('.'):
@@ -297,7 +342,7 @@ def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
             for filename in filenames:
                 # skip badly encoded files
                 if isinstance(filename, bytes):
-                    logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', errors='replace'),
+                    logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', 'replace'),
                                  dirpath)
                     continue
                 # filter videos