Release 0.7.2

The alpha2 converter might not yet be loaded at this point.
When subliminal's converters are loaded, they import subliminal
which import the video module.

HISTORY.rst

@@ -1,6 +1,29 @@
 Changelog
 =========
 
+0.7.2
+-----
+**release date:** 2013-11-10
+
+* Fix TVSubtitles for ambiguous series
+* Add a CACHE_VERSION to force cache reloading on version change
+* Set CLI default cache expiration time to 30 days
+* Add podnapisi provider
+* Support script for languages e.g. Latn, Cyrl
+* Improve logging levels
+* Fix subtitle validation in some rare cases
+
+
+0.7.1
+-----
+**release date:** 2013-11-06
+
+* Improve CLI
+* Add login support for Addic7ed
+* Remove lxml dependency
+* Many fixes
+
+
 0.7.0
 -----
 **release date:** 2013-10-29

dev-requirements.txt

@@ -1,3 +1,4 @@
 sympy>=0.7.3
 sphinx>=1.1.3
 sphinxcontrib-programoutput>=0.8
+Sphinx-PyPI-upload>=0.2.1

docs/api/cache.rst

@@ -2,6 +2,10 @@ Cache
 =====
 .. module:: subliminal.cache
 
-.. autodata:: region
+.. autodata:: CACHE_VERSION
+.. autofunction:: subliminal_key_generator
+.. data:: region
 
-Refer to `dogpile.cache's documentation <http://dogpilecache.readthedocs.org>`_ to see how to configure a region
+    The dogpile.cache region
+
+Refer to `dogpile.cache's documentation <http://dogpilecache.readthedocs.org>`_ to see how to configure the region

docs/api/video.rst

@@ -4,7 +4,6 @@ Video
 
 .. autodata:: VIDEO_EXTENSIONS
 .. autodata:: SUBTITLE_EXTENSIONS
-.. autodata:: LANGUAGE_EXTENSIONS
 .. autoclass:: Video
     :members:
 .. autoclass:: Episode

requirements.txt

@@ -1,10 +1,9 @@
 beautifulsoup4>=4.3.2
-guessit>=0.6.1
-requests>=2.0.0
-enzyme>=0.3.1
+guessit>=0.6.2
+requests>=2.0.1
+enzyme>=0.4.0
 html5lib>=0.99
 dogpile.cache>=0.5.1
-babelfish>=0.1.5
-lxml>=3.2.3
+babelfish>=0.3.0
 charade>=1.0.3
 pysrt>=0.5.0

setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 
 setup(name='subliminal',
-    version='0.7.0',
+    version='0.7.2',
     license='MIT',
     description='Subtitles, faster than your thoughts',
     long_description=open('README.rst').read() + '\n\n' + open('HISTORY.rst').read(),
@@ -29,9 +29,11 @@ setup(name='subliminal',
         'subliminal.providers': ['addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
                                  'bierdopje = subliminal.providers.bierdopje:BierDopjeProvider',
                                  'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
+                                 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
                                  'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
                                  'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'],
         'babelfish.converters': ['addic7ed = subliminal.converters.addic7ed:Addic7edConverter',
+                                 'podnapisi = subliminal.converters.podnapisi:PodnapisiConverter',
                                  'tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter']
     },
     install_requires=open('requirements.txt').readlines(),

subliminal/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 __title__ = 'subliminal'
-__version__ = '0.7.0'
+__version__ = '0.7.2'
 __author__ = 'Antoine Bertin'
 __license__ = 'MIT'
 __copyright__ = 'Copyright 2013 Antoine Bertin'

subliminal/api.py

@@ -4,6 +4,7 @@ import collections
 import io
 import logging
 import operator
+import babelfish
 import pkg_resources
 from .exceptions import ProviderNotAvailable, InvalidSubtitle
 from .subtitle import get_subtitle_path
@@ -33,40 +34,49 @@ def list_subtitles(videos, languages, providers=None, provider_configs=None):
     provider_configs = provider_configs or {}
     subtitles = collections.defaultdict(list)
     # filter videos
-    videos = [v for v in videos if v.subtitle_languages != languages]
+    videos = [v for v in videos if v.subtitle_languages & languages < languages]
     if not videos:
         logger.info('No video to download subtitles for with languages %r', languages)
         return subtitles
+    subtitle_languages = set.intersection(*[v.subtitle_languages for v in videos])
     for provider_entry_point in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT):
         # filter and initialize provider
         if providers is not None and provider_entry_point.name not in providers:
             logger.debug('Skipping provider %r: not in the list', provider_entry_point.name)
             continue
         Provider = provider_entry_point.load()
-        provider_languages = Provider.languages & languages
+        provider_languages = Provider.languages & languages - subtitle_languages
         if not provider_languages:
-            logger.debug('Skipping provider %r: no language to search for', provider_entry_point.name)
+            logger.info('Skipping provider %r: no language to search for', provider_entry_point.name)
             continue
         provider_videos = [v for v in videos if Provider.check(v)]
         if not provider_videos:
-            logger.debug('Skipping provider %r: no video to search for', provider_entry_point.name)
+            logger.info('Skipping provider %r: no video to search for', provider_entry_point.name)
             continue
 
         # list subtitles with the provider
-        with Provider(**provider_configs.get(provider_entry_point.name, {})) as provider:
-            for provider_video in provider_videos:
-                logger.info('Listing subtitles with provider %r for video %r with languages %r',
-                            provider_entry_point.name, provider_video, provider_languages)
-                try:
-                    provider_subtitles = provider.list_subtitles(provider_video, provider_languages)
-                except ProviderNotAvailable:
-                    logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
-                    break
-                except:
-                    logger.exception('Unexpected error in provider %r', provider_entry_point.name)
-                    continue
-                logger.info('Found %d subtitles', len(provider_subtitles))
-                subtitles[provider_video].extend(provider_subtitles)
+        try:
+            with Provider(**provider_configs.get(provider_entry_point.name, {})) as provider:
+                for provider_video in provider_videos:
+                    provider_video_languages = provider_languages - provider_video.subtitle_languages
+                    if not provider_video_languages:
+                        logger.debug('Skipping provider %r: no language to search for for video %r',
+                                     provider_entry_point.name, provider_video)
+                        continue
+                    logger.info('Listing subtitles with provider %r for video %r with languages %r',
+                                provider_entry_point.name, provider_video, provider_video_languages)
+                    try:
+                        provider_subtitles = provider.list_subtitles(provider_video, provider_video_languages)
+                    except ProviderNotAvailable:
+                        logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
+                        break
+                    except:
+                        logger.exception('Unexpected error in provider %r', provider_entry_point.name)
+                        continue
+                    logger.info('Found %d subtitles', len(provider_subtitles))
+                    subtitles[provider_video].extend(provider_subtitles)
+        except ProviderNotAvailable:
+            logger.warning('Provider %r is not available, discarding it', provider_entry_point.name)
     return subtitles
 
 
@@ -124,14 +134,19 @@ def download_subtitles(subtitles, provider_configs=None, single=False):
                 except:
                     logger.exception('Unexpected error in provider %r', subtitle.provider_name)
                     continue
-                with io.open(subtitle_path, 'w') as f:
+                with io.open(subtitle_path, 'w', encoding='utf-8') as f:
                     f.write(subtitle_text)
                 downloaded_languages.add(subtitle.language)
                 if single or downloaded_languages == languages:
                     break
     finally:  # terminate providers
-        for provider in initialized_providers.values():
-            provider.terminate()
+        for (provider_name, provider) in initialized_providers.items():
+            try:
+                provider.terminate()
+            except ProviderNotAvailable:
+                logger.warning('Provider %r is not available, unable to terminate', provider_name)
+            except:
+                logger.exception('Unexpected error in provider %r', provider_name)
 
 
 def download_best_subtitles(videos, languages, providers=None, provider_configs=None, single=False, min_score=0,
@@ -155,22 +170,24 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
     discarded_providers = set()
     downloaded_subtitles = collections.defaultdict(list)
     # filter videos
-    videos = [v for v in videos if v.subtitle_languages != languages]
+    videos = [v for v in videos if v.subtitle_languages & languages < languages
+              and (not single or babelfish.Language('und') not in v.subtitle_languages)]
     if not videos:
         logger.info('No video to download subtitles for with languages %r', languages)
         return downloaded_subtitles
     # filter and initialize providers
+    subtitle_languages = set.intersection(*[v.subtitle_languages for v in videos])
     initialized_providers = {}
     for provider_entry_point in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT):
         if providers is not None and provider_entry_point.name not in providers:
             logger.debug('Skipping provider %r: not in the list', provider_entry_point.name)
             continue
         Provider = provider_entry_point.load()
-        if not Provider.languages & languages:
-            logger.debug('Skipping provider %r: no language to search for', provider_entry_point.name)
+        if not Provider.languages & languages - subtitle_languages:
+            logger.info('Skipping provider %r: no language to search for', provider_entry_point.name)
             continue
         if not [v for v in videos if Provider.check(v)]:
-            logger.debug('Skipping provider %r: no video to search for', provider_entry_point.name)
+            logger.info('Skipping provider %r: no video to search for', provider_entry_point.name)
             continue
         provider = Provider(**provider_configs.get(provider_entry_point.name, {}))
         try:
@@ -188,11 +205,15 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                     if provider_name in discarded_providers:
                         logger.debug('Skipping discarded provider %r', provider_name)
                         continue
-                    provider_languages = provider.languages & languages
+                    provider_video_languages = provider.languages & languages - video.subtitle_languages
+                    if not provider_video_languages:
+                        logger.debug('Skipping provider %r: no language to search for for video %r', provider_name,
+                                     video)
+                        continue
                     logger.info('Listing subtitles with provider %r for video %r with languages %r',
-                                provider_name, video, provider_languages)
+                                provider_name, video, provider_video_languages)
                     try:
-                        provider_subtitles = provider.list_subtitles(video, provider_languages)
+                        provider_subtitles = provider.list_subtitles(video, provider_video_languages)
                     except ProviderNotAvailable:
                         logger.warning('Provider %r is not available, discarding it', provider_name)
                         discarded_providers.add(provider_name)
@@ -204,7 +225,7 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                     subtitles.extend(provider_subtitles)
 
             # find the best subtitles and download them
-            downloaded_languages = set()
+            downloaded_languages = video.subtitle_languages.copy()
             for subtitle, score in sorted([(s, s.compute_score(video)) for s in subtitles],
                                           key=operator.itemgetter(1), reverse=True):
                 # filter
@@ -238,13 +259,18 @@ def download_best_subtitles(videos, languages, providers=None, provider_configs=
                 except:
                     logger.exception('Unexpected error in provider %r', subtitle.provider_name)
                     continue
-                with io.open(subtitle_path, 'w') as f:
+                with io.open(subtitle_path, 'w', encoding='utf-8') as f:
                     f.write(subtitle_text)
                 downloaded_languages.add(subtitle.language)
-                if single or downloaded_languages == languages:
+                if single or downloaded_languages >= languages:
                     logger.debug('All languages downloaded')
                     break
     finally:  # terminate providers
-        for provider in initialized_providers.values():
-            provider.terminate()
+        for (provider_name, provider) in initialized_providers.items():
+            try:
+                provider.terminate()
+            except ProviderNotAvailable:
+                logger.warning('Provider %r is not available, unable to terminate', provider_name)
+            except:
+                logger.exception('Unexpected error in provider %r', provider_name)
     return downloaded_subtitles

subliminal/cache.py

@@ -1,6 +1,30 @@
 # -*- coding: utf-8 -*-
+import inspect
 import dogpile.cache
 
 
-#: The subliminal's dogpile.cache region
-region = dogpile.cache.make_region()
+#: Subliminal's cache version
+CACHE_VERSION = 1
+
+
+def subliminal_key_generator(namespace, fn, to_str=dogpile.cache.compat.string_type):
+    """Add a :data:`CACHE_VERSION` to dogpile.cache's default function_key_generator"""
+    if namespace is None:
+        namespace = '%d:%s:%s' % (CACHE_VERSION, fn.__module__, fn.__name__)
+    else:
+        namespace = '%d:%s:%s|%s' % (CACHE_VERSION, fn.__module__, fn.__name__, namespace)
+
+    args = inspect.getargspec(fn)
+    has_self = args[0] and args[0][0] in ('self', 'cls')
+
+    def generate_key(*args, **kw):
+        if kw:
+            raise ValueError('Keyword arguments not supported')
+        if has_self:
+            args = args[1:]
+        return namespace + '|' + ' '.join(map(to_str, args))
+    return generate_key
+
+
+#: The dogpile.cache region
+region = dogpile.cache.make_region(function_key_generator=subliminal_key_generator)

subliminal/cli.py

@@ -11,74 +11,150 @@ import guessit
 import pkg_resources
 from subliminal import (__version__, PROVIDERS_ENTRY_POINT, cache_region, Video, Episode, Movie, scan_videos,
     download_best_subtitles)
+try:
+    import colorlog
+except ImportError:
+    colorlog = None
 
 
 DEFAULT_CACHE_FILE = os.path.join('~', '.config', 'subliminal.cache.dbm')
 
 
-def subliminal_parser():
-    parser = argparse.ArgumentParser(description='Subtitles, faster than your thoughts')
-    parser.add_argument('-l', '--languages', nargs='+', metavar='LANGUAGE', help='wanted languages as alpha2 code (ISO-639-1)')
-    parser.add_argument('-p', '--providers', nargs='+', metavar='PROVIDER', help='providers to use from %s (default: all)' % ', '.join(ep.name for ep in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT)))
-    parser.add_argument('-m', '--min-score', type=int, help='minimum score for subtitles. 0-%d for episodes, 0-%d for movies' % (Episode.scores['hash'], Movie.scores['hash']))
-    parser.add_argument('-s', '--single', action='store_true', help='download without language code in subtitle\'s filename i.e. .srt only')
-    parser.add_argument('-f', '--force', action='store_true', help='overwrite existing subtitles')
-    parser.add_argument('-c', '--cache-file', default=DEFAULT_CACHE_FILE, help='cache file (default: %(default)s)')
-    parser.add_argument('-a', '--age', help='download subtitles for videos newer than AGE e.g. 12h, 1w2d')
-    parser.add_argument('--hearing-impaired', action='store_true', help='download hearing impaired subtitles')
-    group_verbosity = parser.add_mutually_exclusive_group()
-    group_verbosity.add_argument('-q', '--quiet', action='store_true', help='disable output')
-    group_verbosity.add_argument('-v', '--verbose', action='store_true', help='verbose output')
-    parser.add_argument('--version', action='version', version=__version__)
-    parser.add_argument('paths', nargs='+', metavar='PATH', help='path to video file or folder')
-    return parser
-
-
 def subliminal():
-    parser = subliminal_parser()
+    parser = argparse.ArgumentParser(prog='subliminal', description='Subtitles, faster than your thoughts',
+                                     epilog='Suggestions and bug reports are greatly appreciated: '
+                                     'https://github.com/Diaoul/subliminal/issues', add_help=False)
+
+    # required arguments
+    required_arguments_group = parser.add_argument_group('required arguments')
+    required_arguments_group.add_argument('paths', nargs='+', metavar='PATH', help='path to video file or folder')
+    required_arguments_group.add_argument('-l', '--languages', nargs='+', required=True, metavar='LANGUAGE',
+                                          help='wanted languages as IETF codes e.g. fr, pt-BR, sr-Cyrl ')
+
+    # configuration
+    configuration_group = parser.add_argument_group('configuration')
+    configuration_group.add_argument('-s', '--single', action='store_true',
+                                     help='download without language code in subtitle\'s filename i.e. .srt only')
+    configuration_group.add_argument('-c', '--cache-file', default=DEFAULT_CACHE_FILE,
+                                     help='cache file (default: %(default)s)')
+
+    # filtering
+    filtering_group = parser.add_argument_group('filtering')
+    providers = [ep.name for ep in pkg_resources.iter_entry_points(PROVIDERS_ENTRY_POINT)]
+    filtering_group.add_argument('-p', '--providers', nargs='+', metavar='PROVIDER',
+                                 help='providers to use (%s)' % ', '.join(providers))
+    filtering_group.add_argument('-m', '--min-score', type=int,
+                                 help='minimum score for subtitles (0-%d for episodes, 0-%d for movies)'
+                                 % (Episode.scores['hash'], Movie.scores['hash']))
+    filtering_group.add_argument('-a', '--age', help='download subtitles for videos newer than AGE e.g. 12h, 1w2d')
+    filtering_group.add_argument('-h', '--hearing-impaired', action='store_true',
+                                 help='download hearing impaired subtitles')
+    filtering_group.add_argument('-f', '--force', action='store_true',
+                                 help='force subtitle download for videos with existing subtitles')
+
+    # addic7ed
+    addic7ed_group = parser.add_argument_group('addic7ed')
+    addic7ed_group.add_argument('--addic7ed-username', metavar='USERNAME', help='username for addic7ed provider')
+    addic7ed_group.add_argument('--addic7ed-password', metavar='PASSWORD', help='password for addic7ed provider')
+
+    # output
+    output_group = parser.add_argument_group('output')
+    output_exclusive_group = output_group.add_mutually_exclusive_group()
+    output_exclusive_group.add_argument('-q', '--quiet', action='store_true', help='disable output')
+    output_exclusive_group.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+    output_group.add_argument('--color', action='store_true', help='add color to console output (requires colorlog)')
+
+    # troubleshooting
+    troubleshooting_group = parser.add_argument_group('troubleshooting')
+    troubleshooting_group.add_argument('--debug', action='store_true', help='debug output')
+    troubleshooting_group.add_argument('--version', action='version', version=__version__)
+    troubleshooting_group.add_argument('--help', action='help', help='show this help message and exit')
+
+    # parse args
     args = parser.parse_args()
 
     # parse paths
     try:
-        args.paths = [p.decode('utf-8') for p in args.paths]
+        args.paths = [os.path.abspath(os.path.expanduser(p.decode('utf-8'))) for p in args.paths]
     except UnicodeDecodeError:
         parser.error('argument paths: encodings is not utf-8: %r' % args.paths)
 
     # parse languages
     try:
-        args.languages = {babelfish.Language.fromalpha2(l) for l in args.languages}
+        args.languages = {babelfish.Language.fromietf(l) for l in args.languages}
     except babelfish.Error:
-        parser.error('argument -l/--languages: codes are not ISO-639-1: %r' % args.languages)
+        parser.error('argument -l/--languages: codes are not IETF: %r' % args.languages)
 
     # parse age
     if args.age is not None:
         match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', args.age)
         if not match:
             parser.error('argument -a/--age: invalid age: %r' % args.age)
-        args.age = datetime.timedelta(**match.groupdict())
+        args.age = datetime.timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
 
-    # setup verbosity
-    if args.verbose:
-        logging.basicConfig(level=logging.DEBUG)
+    # parse cache-file
+    args.cache_file = os.path.abspath(os.path.expanduser(args.cache_file))
+    if not os.path.exists(os.path.split(args.cache_file)[0]):
+        parser.error('argument -c/--cache-file: directory %r for cache file does not exist'
+                     % os.path.split(args.cache_file)[0])
+
+    # parse provider configs
+    provider_configs = {}
+    if (args.addic7ed_username is not None and args.addic7ed_password is None
+        or args.addic7ed_username is None and args.addic7ed_password is not None):
+        parser.error('argument --addic7ed-username/--addic7ed-password: both arguments are required or none')
+    if args.addic7ed_username is not None and args.addic7ed_password is not None:
+        provider_configs['addic7ed'] = {'username': args.addic7ed_username, 'password': args.addic7ed_password}
+
+    # parse color
+    if args.color and colorlog is None:
+        parser.error('argument --color: colorlog required')
+
+    # setup output
+    if args.debug:
+        handler = logging.StreamHandler()
+        if args.color:
+            handler.setFormatter(colorlog.ColoredFormatter('%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s-%(funcName)s:%(lineno)d%(reset)s] %(message)s',
+                                                           log_colors=dict(colorlog.default_log_colors.items() + [('DEBUG', 'cyan')])))
+        else:
+            handler.setFormatter(logging.Formatter('%(levelname)-8s [%(name)s-%(funcName)s:%(lineno)d] %(message)s'))
+        logging.getLogger().addHandler(handler)
+        logging.getLogger().setLevel(logging.DEBUG)
+    elif args.verbose:
+        handler = logging.StreamHandler()
+        if args.color:
+            handler.setFormatter(colorlog.ColoredFormatter('%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s%(reset)s] %(message)s'))
+        else:
+            handler.setFormatter(logging.Formatter('%(levelname)-8s [%(name)s] %(message)s'))
+        logging.getLogger('subliminal').addHandler(handler)
+        logging.getLogger('subliminal').setLevel(logging.INFO)
     elif not args.quiet:
-        logging.basicConfig(level=logging.WARN)
+        handler = logging.StreamHandler()
+        if args.color:
+            handler.setFormatter(colorlog.ColoredFormatter('[%(log_color)s%(levelname)s%(reset)s] %(message)s'))
+        else:
+            handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+        logging.getLogger('subliminal.api').addHandler(handler)
+        logging.getLogger('subliminal.api').setLevel(logging.INFO)
 
     # configure cache
-    cache_region.configure('dogpile.cache.dbm', arguments={'filename': os.path.expanduser(args.cache_file)})
+    cache_region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
+                           arguments={'filename': args.cache_file})
 
     # scan videos
-    videos = scan_videos([p for p in args.paths if os.path.exists(p)], subtitles=not args.force, age=args.age)
+    videos = scan_videos([p for p in args.paths if os.path.exists(p)], subtitles=not args.force,
+                         embedded_subtitles=not args.force, age=args.age)
 
     # guess videos
     videos.extend([Video.fromguess(os.path.split(p)[1], guessit.guess_file_info(p, 'autodetect')) for p in args.paths
                    if not os.path.exists(p)])
 
     # download best subtitles
-    subtitles = download_best_subtitles(videos, args.languages, providers=args.providers, provider_configs=None,
-                                        single=args.single, min_score=args.min_score,
-                                        hearing_impaired=args.hearing_impaired)
+    subtitles = download_best_subtitles(videos, args.languages, providers=args.providers,
+                                        provider_configs=provider_configs, single=args.single,
+                                        min_score=args.min_score, hearing_impaired=args.hearing_impaired)
 
-    # output result
+    # result output
     if not subtitles:
         if not args.quiet:
             sys.stderr.write('No subtitles downloaded\n')

subliminal/converters/addic7ed.py

@@ -6,22 +6,24 @@ from babelfish.converters.name import NameConverter
 class Addic7edConverter(NameConverter):
     def __init__(self):
         super(Addic7edConverter, self).__init__()
-        self.from_addic7ed = {'Català': ('cat', None), 'Chinese (Simplified)': ('zho', None),
-                              'Chinese (Traditional)': ('zho', None), 'Euskera': ('eus', None),
-                              'Galego': ('glg', None), 'Greek': ('ell', None),
-                              'Malay': ('msa', None), 'Portuguese (Brazilian)': ('por', 'BR'),
-                              'Serbian (Cyrillic)': ('srp', None), 'Serbian (Latin)': ('srp', None),
-                              'Spanish (Latin America)': ('spa', None), 'Spanish (Spain)': ('spa', None)}
-        self.to_addic7ed = {('cat', None): 'Català', ('zho', None): 'Chinese (Simplified)',
-                            ('eus', None): 'Euskera', ('glg', None): 'Galego',
-                            ('ell', None): 'Greek', ('msa', None): 'Malay',
-                            ('por', 'BR'): 'Portuguese (Brazilian)', ('srp', None): 'Serbian (Cyrillic)'}
+        self.from_addic7ed = {'Català': ('cat',), 'Chinese (Simplified)': ('zho',), 'Chinese (Traditional)': ('zho',),
+                              'Euskera': ('eus',), 'Galego': ('glg',), 'Greek': ('ell',), 'Malay': ('msa',),
+                              'Portuguese (Brazilian)': ('por', 'BR'), 'Serbian (Cyrillic)': ('srp', None, 'Cyrl'),
+                              'Serbian (Latin)': ('srp',), 'Spanish (Latin America)': ('spa',),
+                              'Spanish (Spain)': ('spa',)}
+        self.to_addic7ed = {('cat',): 'Català', ('zho',): 'Chinese (Simplified)', ('eus',): 'Euskera',
+                            ('glg',): 'Galego', ('ell',): 'Greek', ('msa',): 'Malay',
+                            ('por', 'BR'): 'Portuguese (Brazilian)', ('srp', None, 'Cyrl'): 'Serbian (Cyrillic)'}
         self.codes |= set(self.from_addic7ed.keys())
 
-    def convert(self, alpha3, country=None):
+    def convert(self, alpha3, country=None, script=None):
+        if (alpha3, country, script) in self.to_addic7ed:
+            return self.to_addic7ed[(alpha3, country, script)]
         if (alpha3, country) in self.to_addic7ed:
             return self.to_addic7ed[(alpha3, country)]
-        return super(Addic7edConverter, self).convert(alpha3, country)
+        if (alpha3,) in self.to_addic7ed:
+            return self.to_addic7ed[(alpha3,)]
+        return super(Addic7edConverter, self).convert(alpha3, country, script)
 
     def reverse(self, addic7ed):
         if addic7ed in self.from_addic7ed:

subliminal/converters/podnapisi.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from babelfish import ReverseConverter, ConvertError, ReverseError
+
+
+class PodnapisiConverter(ReverseConverter):
+    def __init__(self):
+        super(PodnapisiConverter, self).__init__()
+        self.from_podnapisi = {2: ('eng',), 28: ('spa',), 26: ('pol',), 36: ('srp',), 1: ('slv',), 38: ('hrv',),
+                               9: ('ita',), 8: ('fra',), 48: ('por', 'BR'), 23: ('nld',), 12: ('ara',), 13: ('ron',),
+                               33: ('bul',), 32: ('por',), 16: ('ell',), 15: ('hun',), 31: ('fin',), 30: ('tur',),
+                               7: ('ces',), 25: ('swe',), 27: ('rus',), 24: ('dan',), 22: ('heb',), 51: ('vie',),
+                               52: ('fas',), 5: ('deu',), 14: ('spa', 'AR'), 54: ('ind',), 47: ('srp', None, 'Cyrl'),
+                               3: ('nor',), 20: ('est',), 10: ('bos',), 17: ('zho',), 37: ('slk',), 35: ('mkd',),
+                               11: ('jpn',), 4: ('kor',), 29: ('sqi',), 6: ('isl',), 19: ('lit',), 46: ('ukr',),
+                               44: ('tha',), 53: ('cat',), 56: ('sin',), 21: ('lav',), 40: ('cmn',), 55: ('msa',),
+                               42: ('hin',), 50: ('bel',)}
+        self.to_podnapisi = {v: k for k, v in self.from_podnapisi.items()}
+        self.codes = set(self.from_podnapisi.keys())
+
+    def convert(self, alpha3, country=None, script=None):
+        if (alpha3,) in self.to_podnapisi:
+            return self.to_podnapisi[(alpha3,)]
+        if (alpha3, country) in self.to_podnapisi:
+            return self.to_podnapisi[(alpha3, country)]
+        if (alpha3, country, script) in self.to_podnapisi:
+            return self.to_podnapisi[(alpha3, country, script)]
+        raise ConvertError(alpha3, country, script)
+
+    def reverse(self, podnapisi):
+        if podnapisi not in self.from_podnapisi:
+            raise ReverseError(podnapisi)
+        return self.from_podnapisi[podnapisi]

subliminal/converters/tvsubtitles.py

@@ -6,15 +6,17 @@ from babelfish.converters.alpha2 import Alpha2Converter
 class TVsubtitlesConverter(Alpha2Converter):
     def __init__(self):
         super(TVsubtitlesConverter, self).__init__()
-        self.from_tvsubtitles = {'br': ('por', 'BR'), 'ua': ('ukr', None), 'gr': ('ell', None), 'cn': ('zho', None),
-                                 'jp': ('jpn', None), 'cz': ('ces', None)}
+        self.from_tvsubtitles = {'br': ('por', 'BR'), 'ua': ('ukr',), 'gr': ('ell',), 'cn': ('zho',), 'jp': ('jpn',),
+                                 'cz': ('ces',)}
         self.to_tvsubtitles = {v: k for k, v in self.from_tvsubtitles}
         self.codes |= set(self.from_tvsubtitles.keys())
 
-    def convert(self, alpha3, country=None):
+    def convert(self, alpha3, country=None, script=None):
         if (alpha3, country) in self.to_tvsubtitles:
             return self.to_tvsubtitles[(alpha3, country)]
-        return super(TVsubtitlesConverter, self).convert(alpha3, country)
+        if (alpha3,) in self.to_tvsubtitles:
+            return self.to_tvsubtitles[(alpha3,)]
+        return super(TVsubtitlesConverter, self).convert(alpha3, country, script)
 
     def reverse(self, tvsubtitles):
         if tvsubtitles in self.from_tvsubtitles:

subliminal/providers/addic7ed.py

@@ -8,7 +8,7 @@ import requests
 from . import Provider
 from .. import __version__
 from ..cache import region
-from ..exceptions import InvalidSubtitle, ProviderNotAvailable
+from ..exceptions import ProviderConfigurationError, ProviderNotAvailable, InvalidSubtitle
 from ..subtitle import Subtitle, is_valid_subtitle
 from ..video import Episode
 
@@ -58,14 +58,43 @@ class Addic7edProvider(Provider):
                            'fin', 'fra', 'glg', 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa',
                            'nld', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha',
                            'tur', 'ukr', 'vie', 'zho']}
-    videos = (Episode,)
+    video_types = (Episode,)
     server = 'http://www.addic7ed.com'
 
+    def __init__(self, username=None, password=None):
+        if username is not None and password is None or username is None and password is not None:
+            raise ProviderConfigurationError('Username and password must be specified')
+        self.username = username
+        self.password = password
+        self.logged_in = False
+
     def initialize(self):
         self.session = requests.Session()
         self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
+        # login
+        if self.username is not None and self.password is not None:
+            logger.debug('Logging in')
+            data = {'username': self.username, 'password': self.password, 'Submit': 'Log in'}
+            try:
+                r = self.session.post(self.server + '/dologin.php', data, timeout=10, allow_redirects=False)
+            except requests.Timeout:
+                raise ProviderNotAvailable('Timeout after 10 seconds')
+            if r.status_code == 302:
+                logger.info('Logged in')
+                self.logged_in = True
+            else:
+                logger.error('Failed to login')
 
     def terminate(self):
+        # logout
+        if self.logged_in:
+            try:
+                r = self.session.get(self.server + '/logout.php', timeout=10)
+                logger.info('Logged out')
+            except requests.Timeout:
+                raise ProviderNotAvailable('Timeout after 10 seconds')
+            if r.status_code != 200:
+                raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
         self.session.close()
 
     def get(self, url, params=None):
@@ -154,7 +183,9 @@ class Addic7edProvider(Provider):
             raise ProviderNotAvailable('Timeout after 10 seconds')
         if r.status_code != 200:
             raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
-        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'])
+        if r.headers['Content-Type'] == 'text/html':
+            raise ProviderNotAvailable('Download limit exceeded')
+        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'], 'replace')
         if not is_valid_subtitle(subtitle_text):
             raise InvalidSubtitle
         return subtitle_text

subliminal/providers/bierdopje.py

@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import logging
 import urllib
 import babelfish
-import bs4
 import charade
 import guessit
 import requests
+import xml.etree.ElementTree
 from . import Provider
 from .. import __version__
 from ..cache import region
@@ -65,7 +65,7 @@ class BierDopjeProvider(Provider):
         :param string url: API part of the URL to reach without the leading slash
         :param \*\*params: format specs for the `url`
         :return: the response
-        :rtype: :class:`bs4.BeautifulSoup`
+        :rtype: :class:`xml.etree.ElementTree.Element`
         :raise: :class:`~subliminal.exceptions.ProviderNotAvailable`
 
         """
@@ -77,7 +77,7 @@ class BierDopjeProvider(Provider):
             raise ProviderNotAvailable('Too Many Requests')
         elif r.status_code != 200:
             raise ProviderError('Request failed with status code %d' % r.status_code)
-        return bs4.BeautifulSoup(r.content, ['xml'])
+        return xml.etree.ElementTree.fromstring(r.content)
 
     @region.cache_on_arguments()
     def find_show_id(self, series):
@@ -89,11 +89,11 @@ class BierDopjeProvider(Provider):
 
         """
         logger.debug('Searching for series %r', series)
-        soup = self.get('FindShowByName/{series}', series=urllib.quote(series))
-        if soup.status.contents[0] == 'false':
+        root = self.get('FindShowByName/{series}', series=urllib.quote(series))
+        if root.find('response/status').text == 'false':
             logger.info('Series %r not found', series)
             return None
-        return int(soup.showid.contents[0])
+        return int(root.find('response/results/result[1]/showid').text)
 
     def query(self, language, season, episode, tvdb_id=None, series=None):
         params = {'language': language.alpha2, 'season': season, 'episode': episode}
@@ -109,19 +109,16 @@ class BierDopjeProvider(Provider):
         else:
             raise ValueError('Missing parameter tvdb_id or series')
         logger.debug('Searching subtitles %r', params)
-        soup = self.get('GetAllSubsFor/{showid}/{season}/{episode}/{language}/{istvdbid}', **params)
-        if soup.status.contents[0] == 'false':
+        root = self.get('GetAllSubsFor/{showid}/{season}/{episode}/{language}/{istvdbid}', **params)
+        if root.find('response/status').text == 'false':
             logger.debug('No subtitle found')
             return []
-        logger.debug('Found subtitles %r', soup.results('result'))
-        return [BierDopjeSubtitle(language, season, episode, tvdb_id, series, result.filename.contents[0],
-                                  result.downloadlink.contents[0]) for result in soup.results('result')]
+        logger.debug('Found subtitles %r', root.find('response/results'))
+        return [BierDopjeSubtitle(language, season, episode, tvdb_id, series, result.find('filename').text,
+                                  result.find('downloadlink').text) for result in root.find('response/results')]
 
     def list_subtitles(self, video, languages):
-        subtitles = []
-        for language in languages:
-            subtitles.extend(self.query(language, video.season, video.episode, video.tvdb_id, video.series))
-        return subtitles
+        return [s for l in languages for s in self.query(l, video.season, video.episode, video.tvdb_id, video.series)]
 
     def download_subtitle(self, subtitle):
         try:
@@ -132,7 +129,7 @@ class BierDopjeProvider(Provider):
             raise ProviderNotAvailable('Too Many Requests')
         elif r.status_code != 200:
             raise ProviderError('Request failed with status code %d' % r.status_code)
-        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'])
+        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'], 'replace')
         if not is_valid_subtitle(subtitle_text):
             raise InvalidSubtitle
         return subtitle_text

subliminal/providers/opensubtitles.py

@@ -128,7 +128,6 @@ class OpenSubtitlesProvider(Provider):
         if not response['data']:
             logger.debug('No subtitle found')
             return []
-        logger.debug('Found subtitles %r', response['data'])
         return [OpenSubtitlesSubtitle(babelfish.Language.fromopensubtitles(r['SubLanguageID']),
                                       bool(int(r['SubHearingImpaired'])), r['IDSubtitleFile'], r['MatchedBy'],
                                       r['MovieKind'], r['MovieHash'], r['MovieName'], r['MovieReleaseName'],
@@ -141,7 +140,8 @@ class OpenSubtitlesProvider(Provider):
         query = None
         if ('opensubtitles' not in video.hashes or not video.size) and not video.imdb_id:
             query = video.name.split(os.sep)[-1]
-        return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id, query=query)
+        return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id,
+                          query=query)
 
     def download_subtitle(self, subtitle):
         try:
@@ -153,7 +153,7 @@ class OpenSubtitlesProvider(Provider):
         if not response['data']:
             raise ProviderError('Nothing to download')
         subtitle_bytes = zlib.decompress(base64.b64decode(response['data'][0]['data']), 47)
-        subtitle_text = subtitle_bytes.decode(charade.detect(subtitle_bytes)['encoding'])
+        subtitle_text = subtitle_bytes.decode(charade.detect(subtitle_bytes)['encoding'], 'replace')
         if not is_valid_subtitle(subtitle_text):
             raise InvalidSubtitle
         return subtitle_text

subliminal/providers/podnapisi.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import io
+import logging
+import re
+import xml.etree.ElementTree
+import zipfile
+import babelfish
+import bs4
+import charade
+import guessit
+import requests
+from . import Provider
+from .. import __version__
+from ..exceptions import InvalidSubtitle, ProviderNotAvailable, ProviderError
+from ..subtitle import Subtitle, is_valid_subtitle, compute_guess_matches
+from ..video import Episode, Movie
+
+
+logger = logging.getLogger(__name__)
+
+
+class PodnapisiSubtitle(Subtitle):
+    provider_name = 'podnapisi'
+
+    def __init__(self, language, id, releases, hearing_impaired, link, series=None, season=None, episode=None,
+                 title=None, year=None):
+        super(PodnapisiSubtitle, self).__init__(language, hearing_impaired)
+        self.id = id
+        self.releases = releases
+        self.hearing_impaired = hearing_impaired
+        self.link = link
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.title = title
+        self.year = year
+
+    def compute_matches(self, video):
+        matches = set()
+        # episode
+        if isinstance(video, Episode):
+            # series
+            if video.series and self.series.lower() == video.series.lower():
+                matches.add('series')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+            # guess
+            for release in self.releases:
+                matches |= compute_guess_matches(video, guessit.guess_episode_info(release + '.mkv'))
+        # movie
+        elif isinstance(video, Movie):
+            # title
+            if video.title and self.title.lower() == video.title.lower():
+                matches.add('title')
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+            # guess
+            for release in self.releases:
+                matches |= compute_guess_matches(video, guessit.guess_movie_info(release + '.mkv'))
+        return matches
+
+
+class PodnapisiProvider(Provider):
+    languages = {babelfish.Language.frompodnapisi(l) for l in babelfish.CONVERTERS['podnapisi'].codes}
+    video_types = (Episode, Movie)
+    server = 'http://simple.podnapisi.net'
+    link_re = re.compile('^.*(?P<link>/ppodnapisi/download/i/\d+/k/.*$)')
+
+    def initialize(self):
+        self.session = requests.Session()
+        self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__}
+
+    def terminate(self):
+        self.session.close()
+
+    def get(self, url, params=None, is_xml=True):
+        """Make a GET request on `url` with the given parameters
+
+        :param string url: part of the URL to reach with the leading slash
+        :param dict params: params of the request
+        :param bool xml: whether the response content is XML or not
+        :return: the response
+        :rtype: :class:`xml.etree.ElementTree.Element` or :class:`bs4.BeautifulSoup`
+        :raise: :class:`~subliminal.exceptions.ProviderNotAvailable`
+
+        """
+        try:
+            r = self.session.get(self.server + '/ppodnapisi' + url, params=params, timeout=10)
+        except requests.Timeout:
+            raise ProviderNotAvailable('Timeout after 10 seconds')
+        if r.status_code != 200:
+            raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
+        if is_xml:
+            return xml.etree.ElementTree.fromstring(r.content)
+        else:
+            return bs4.BeautifulSoup(r.content, ['permissive'])
+
+    def query(self, language, series=None, season=None, episode=None, title=None, year=None):
+        params = {'sXML': 1, 'sJ': language.podnapisi}
+        if series and season and episode:
+            params['sK'] = series
+            params['sTS'] = season
+            params['sTE'] = episode
+        elif title:
+            params['sK'] = title
+            if year:
+                params['sY'] = year
+        else:
+            raise ValueError('Missing parameters series and season and episode or title')
+        logger.debug('Searching episode %r', params)
+        subtitles = []
+        while True:
+            root = self.get('/search', params)
+            if not int(root.find('pagination/results').text):
+                logger.debug('No subtitle found')
+                break
+            if series and season and episode:
+                subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
+                                                    'h' in (s.find('flags').text or ''), s.find('url').text[38:],
+                                                    series=series, season=season, episode=episode)
+                                  for s in root.findall('subtitle')])
+            elif title:
+                subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text), s.find('release').text.split(),
+                                                    'h' in (s.find('flags').text or ''), s.find('url').text[38:],
+                                                    title=title, year=year)
+                                  for s in root.findall('subtitle')])
+            if int(root.find('pagination/current').text) >= int(root.find('pagination/count').text):
+                break
+            params['page'] = int(root.find('pagination/current').text) + 1
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        if isinstance(video, Episode):
+            return [s for l in languages for s in self.query(l, series=video.series, season=video.season,
+                                                             episode=video.episode)]
+        elif isinstance(video, Movie):
+            return [s for l in languages for s in self.query(l, title=video.title, year=video.year)]
+
+    def download_subtitle(self, subtitle):
+        soup = self.get(subtitle.link, is_xml=False)
+        link = soup.find('a', href=self.link_re)
+        if not link:
+            raise ProviderError('Cannot find the download link')
+        try:
+            r = self.session.get(self.server + self.link_re.match(link['href']).group('link'), timeout=10)
+        except requests.Timeout:
+            raise ProviderNotAvailable('Timeout after 10 seconds')
+        if r.status_code != 200:
+            raise ProviderNotAvailable('Request failed with status code %d' % r.status_code)
+        with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+            if len(zf.namelist()) > 1:
+                raise ProviderError('More than one file to unzip')
+            subtitle_bytes = zf.read(zf.namelist()[0])
+        subtitle_text = subtitle_bytes.decode(charade.detect(subtitle_bytes)['encoding'], 'replace')
+        if not is_valid_subtitle(subtitle_text):
+            raise InvalidSubtitle
+        return subtitle_text

subliminal/providers/thesubdb.py

@@ -8,7 +8,6 @@ from . import Provider
 from .. import __version__
 from ..exceptions import InvalidSubtitle, ProviderNotAvailable, ProviderError
 from ..subtitle import Subtitle, is_valid_subtitle
-from ..video import Episode, Movie
 
 
 logger = logging.getLogger(__name__)
@@ -31,7 +30,6 @@ class TheSubDBSubtitle(Subtitle):
 
 class TheSubDBProvider(Provider):
     languages = {babelfish.Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']}
-    video_types = (Episode, Movie)
     required_hash = 'thesubdb'
 
     def initialize(self):
@@ -77,7 +75,7 @@ class TheSubDBProvider(Provider):
         r = self.get(params)
         if r.status_code != 200:
             raise ProviderError('Request failed with status code %d' % r.status_code)
-        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'])
+        subtitle_text = r.content.decode(charade.detect(r.content)['encoding'], 'replace')
         if not is_valid_subtitle(subtitle_text):
             raise InvalidSubtitle
         return subtitle_text

subliminal/providers/tvsubtitles.py

@@ -59,10 +59,11 @@ class TVsubtitlesProvider(Provider):
     languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
                  for l in ['ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor',
                            'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho']}
-    videos = (Episode,)
+    video_types = (Episode,)
     server = 'http://www.tvsubtitles.net'
-    episode_id_re = re.compile('^episode-(\d+)\.html$')
-    subtitle_re = re.compile('^\/subtitle-(\d+)\.html$')
+    episode_id_re = re.compile('^episode-\d+\.html$')
+    subtitle_re = re.compile('^\/subtitle-\d+\.html$')
+    link_re = re.compile('^(?P<series>.+) \(\d{4}-\d{4}\)$')
 
     def initialize(self):
         self.session = requests.Session()
@@ -77,6 +78,7 @@ class TVsubtitlesProvider(Provider):
         :param string url: part of the URL to reach with the leading slash
         :param dict params: params of the request
         :param dict data: data of the request
+        :param string method: method of the request
         :return: the response
         :rtype: :class:`bs4.BeautifulSoup`
         :raise: :class:`~subliminal.exceptions.ProviderNotAvailable`
@@ -106,6 +108,13 @@ class TVsubtitlesProvider(Provider):
         if not links:
             logger.info('Series %r not found', series)
             return None
+        for link in links:
+            match = self.link_re.match(link.string)
+            if not match:
+                logger.warning('Could not parse %r', link.string)
+                continue
+            if match.group('series').lower().replace('.', ' ').strip() == series.lower():
+                return int(link['href'][8:-5])
         return int(links[0]['href'][8:-5])
 
     @region.cache_on_arguments()
@@ -160,7 +169,7 @@ class TVsubtitlesProvider(Provider):
             if len(zf.namelist()) > 1:
                 raise ProviderError('More than one file to unzip')
             subtitle_bytes = zf.read(zf.namelist()[0])
-        subtitle_text = subtitle_bytes.decode(charade.detect(subtitle_bytes)['encoding'])
+        subtitle_text = subtitle_bytes.decode(charade.detect(subtitle_bytes)['encoding'], 'replace')
         if not is_valid_subtitle(subtitle_text):
             raise InvalidSubtitle
         return subtitle_text

subliminal/subtitle.py

@@ -73,7 +73,7 @@ class Subtitle(object):
         return score
 
     def __repr__(self):
-        return '<%s [%r]>' % (self.__class__.__name__, self.language)
+        return '<%s [%s]>' % (self.__class__.__name__, self.language)
 
 
 def get_subtitle_path(video_path, language=None):
@@ -90,7 +90,7 @@ def get_subtitle_path(video_path, language=None):
     if language is not None:
         try:
             return subtitle_path + '.%s.%s' % (language.alpha2, 'srt')
-        except babelfish.NoConversionError:
+        except babelfish.ConvertError:
             return subtitle_path + '.%s.%s' % (language.alpha3, 'srt')
     return subtitle_path + '.srt'
 
@@ -106,7 +106,10 @@ def is_valid_subtitle(subtitle_text):
         pysrt.from_string(subtitle_text, error_handling=pysrt.ERROR_RAISE)
         return True
     except pysrt.Error:
-        return False
+        pass
+    except:
+        logger.exception('Unexpected error when validating subtitle')
+    return False
 
 
 def compute_guess_matches(video, guess):

subliminal/tests/test_providers.py

@@ -93,6 +93,16 @@ class Addic7edProviderTestCase(ProviderTestCase):
 class BierDopjeProviderTestCase(ProviderTestCase):
     provider_name = 'bierdopje'
 
+    def test_find_show_id(self):
+        with self.Provider() as provider:
+            show_id = provider.find_show_id('The Big Bang')
+        self.assertTrue(show_id == 9203)
+
+    def test_find_show_id_error(self):
+        with self.Provider() as provider:
+            show_id = provider.find_show_id('the big how i met your mother')
+        self.assertTrue(show_id is None)
+
     def test_query_episode_0(self):
         video = EPISODES[0]
         language = Language('eng')
@@ -219,10 +229,11 @@ class OpenSubtitlesProviderTestCase(ProviderTestCase):
     def test_query_movie_0_hash(self):
         video = MOVIES[0]
         languages = {Language('eng')}
-        matches = {frozenset(['imdb_id', 'title', 'hash', 'year']),
-                   frozenset(['imdb_id', 'hash', 'title', 'year', 'video_codec', 'resolution']),
-                   frozenset(['imdb_id', 'video_codec', 'hash', 'title', 'year']),
-                   frozenset(['imdb_id', 'hash', 'title', 'year', 'video_codec', 'resolution', 'release_group'])}
+        matches = {frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id']),
+                   frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
+                   frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title']),
+                   frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title']),
+                   frozenset(['year', 'imdb_id', 'hash', 'title'])}
         with self.Provider() as provider:
             subtitles = provider.query(languages, hash=video.hashes['opensubtitles'], size=video.size)
         self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
@@ -243,15 +254,64 @@ class OpenSubtitlesProviderTestCase(ProviderTestCase):
     def test_list_subtitles(self):
         video = MOVIES[0]
         languages = {Language('eng'), Language('fra')}
-        matches = {frozenset(['imdb_id', 'title', 'hash', 'year']),
-                   frozenset(['imdb_id', 'resolution', 'title', 'year']),
-                   frozenset(['imdb_id', 'title', 'year']),
-                   frozenset(['imdb_id', 'video_codec', 'title', 'year']),
-                   frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year']),
-                   frozenset(['imdb_id', 'hash', 'title', 'year', 'video_codec', 'resolution', 'release_group']),
-                   frozenset(['imdb_id', 'video_codec', 'hash', 'title', 'year']),
-                   frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group']),
-                   frozenset(['imdb_id', 'hash', 'title', 'year', 'video_codec', 'resolution'])}
+        matches = {frozenset(['title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
+                   frozenset(['imdb_id', 'year', 'title']),
+                   frozenset(['year', 'video_codec', 'imdb_id', 'resolution', 'title']),
+                   frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id']),
+                   frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title']),
+                   frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title']),
+                   frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id']),
+                   frozenset(['year', 'imdb_id', 'hash', 'title']),
+                   frozenset(['video_codec', 'imdb_id', 'year', 'title']),
+                   frozenset(['year', 'imdb_id', 'resolution', 'title'])}
+        with self.Provider() as provider:
+            subtitles = provider.list_subtitles(video, languages)
+        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
+        self.assertTrue({subtitle.language for subtitle in subtitles} == languages)
+
+    def test_download_subtitle(self):
+        video = MOVIES[0]
+        languages = {Language('eng'), Language('fra')}
+        with self.Provider() as provider:
+            subtitles = provider.list_subtitles(video, languages)
+            subtitle_text = provider.download_subtitle(subtitles[0])
+        self.assertTrue(is_valid_subtitle(subtitle_text))
+
+
+class PodnapisiProviderTestCase(ProviderTestCase):
+    provider_name = 'podnapisi'
+
+    def test_query_movie_0(self):
+        video = MOVIES[0]
+        language = Language('eng')
+        matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+                   frozenset(['title', 'resolution', 'year']),
+                   frozenset(['video_codec', 'title', 'year']),
+                   frozenset(['title', 'year']),
+                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year'])}
+        with self.Provider() as provider:
+            subtitles = provider.query(language, title=video.title, year=video.year)
+        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
+        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
+
+    def test_query_episode_0(self):
+        video = EPISODES[0]
+        language = Language('eng')
+        matches = {frozenset(['episode', 'series', 'season', 'video_codec', 'resolution', 'release_group']),
+                   frozenset(['season', 'video_codec', 'episode', 'resolution', 'series'])}
+        with self.Provider() as provider:
+            subtitles = provider.query(language, series=video.series, season=video.season, episode=video.episode)
+        self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
+        self.assertTrue({subtitle.language for subtitle in subtitles} == {language})
+
+    def test_list_subtitles(self):
+        video = MOVIES[0]
+        languages = {Language('eng'), Language('fra')}
+        matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+                   frozenset(['title', 'resolution', 'year']),
+                   frozenset(['video_codec', 'title', 'year']),
+                   frozenset(['title', 'year']),
+                   frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year'])}
         with self.Provider() as provider:
             subtitles = provider.list_subtitles(video, languages)
         self.assertTrue({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles} == matches)
@@ -313,6 +373,16 @@ class TVsubtitlesProviderTestCase(ProviderTestCase):
             show_id = provider.find_show_id('The Big Bang')
         self.assertTrue(show_id == 154)
 
+    def test_find_show_id_ambiguous(self):
+        with self.Provider() as provider:
+            show_id = provider.find_show_id('New Girl')
+        self.assertTrue(show_id == 977)
+
+    def test_find_show_id_no_dots(self):
+        with self.Provider() as provider:
+            show_id = provider.find_show_id('Marvel\'s Agents of S H I E L D')
+        self.assertTrue(show_id == 1340)
+
     def test_find_show_id_error(self):
         with self.Provider() as provider:
             show_id = provider.find_show_id('the big gaming')
@@ -368,6 +438,7 @@ def suite():
     suite.addTest(TestLoader().loadTestsFromTestCase(Addic7edProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(BierDopjeProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(OpenSubtitlesProviderTestCase))
+    suite.addTest(TestLoader().loadTestsFromTestCase(PodnapisiProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(TheSubDBProviderTestCase))
     suite.addTest(TestLoader().loadTestsFromTestCase(TVsubtitlesProviderTestCase))
     return suite

subliminal/tests/test_subliminal.py

@@ -5,7 +5,7 @@ import os
 import shutil
 from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
 from babelfish import Language
-from subliminal import list_subtitles, download_subtitles, download_best_subtitles
+from subliminal import list_subtitles, download_subtitles, download_best_subtitles, scan_video
 from subliminal.tests.common import MOVIES, EPISODES
 
 
@@ -99,9 +99,72 @@ class ApiTestCase(TestCase):
         self.assertTrue(subtitles[videos[0]][0].hearing_impaired == True)
 
 
+class VideoTestCase(TestCase):
+    def setUp(self):
+        os.mkdir(TEST_DIR)
+        for video in MOVIES + EPISODES:
+            open(os.path.join(TEST_DIR, os.path.split(video.name)[1]), 'w').close()
+
+    def tearDown(self):
+        shutil.rmtree(TEST_DIR)
+
+    def test_scan_video_movie(self):
+        video = MOVIES[0]
+        scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.name == os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.title.lower() == video.title.lower())
+        self.assertTrue(scanned_video.year == video.year)
+        self.assertTrue(scanned_video.video_codec == video.video_codec)
+        self.assertTrue(scanned_video.resolution == video.resolution)
+        self.assertTrue(scanned_video.release_group == video.release_group)
+        self.assertTrue(scanned_video.subtitle_languages == set())
+        self.assertTrue(scanned_video.hashes == {})
+        self.assertTrue(scanned_video.audio_codec is None)
+        self.assertTrue(scanned_video.imdb_id is None)
+        self.assertTrue(scanned_video.size == 0)
+
+    def test_scan_video_episode(self):
+        video = EPISODES[0]
+        scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.name == os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.series == video.series)
+        self.assertTrue(scanned_video.season == video.season)
+        self.assertTrue(scanned_video.episode == video.episode)
+        self.assertTrue(scanned_video.video_codec == video.video_codec)
+        self.assertTrue(scanned_video.resolution == video.resolution)
+        self.assertTrue(scanned_video.release_group == video.release_group)
+        self.assertTrue(scanned_video.subtitle_languages == set())
+        self.assertTrue(scanned_video.hashes == {})
+        self.assertTrue(scanned_video.title is None)
+        self.assertTrue(scanned_video.tvdb_id is None)
+        self.assertTrue(scanned_video.imdb_id is None)
+        self.assertTrue(scanned_video.audio_codec is None)
+        self.assertTrue(scanned_video.size == 0)
+
+    def test_scan_video_subtitle_language_und(self):
+        video = EPISODES[0]
+        open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
+        scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.subtitle_languages == {Language('und')})
+
+    def test_scan_video_subtitles_language_eng(self):
+        video = EPISODES[0]
+        open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.en.srt', 'w').close()
+        scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.subtitle_languages == {Language('eng')})
+
+    def test_scan_video_subtitles_languages(self):
+        video = EPISODES[0]
+        open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.en.srt', 'w').close()
+        open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.fr.srt', 'w').close()
+        open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
+        scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+        self.assertTrue(scanned_video.subtitle_languages == {Language('eng'), Language('fra'), Language('und')})
+
 def suite():
     suite = TestSuite()
     suite.addTest(TestLoader().loadTestsFromTestCase(ApiTestCase))
+    suite.addTest(TestLoader().loadTestsFromTestCase(VideoTestCase))
     return suite
 
 

subliminal/video.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
+import datetime
 import hashlib
 import logging
 import os
@@ -23,9 +24,6 @@ VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.
 #: Subtitle extensions
 SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl')
 
-#: Language extensions
-LANGUAGE_EXTENSIONS = tuple('.' + c for c in babelfish.CONVERTERS['alpha2'].codes)
-
 
 class Video(object):
     """Base class for videos
@@ -158,10 +156,15 @@ def scan_subtitle_languages(path):
     :rtype: set
 
     """
+    language_extensions = tuple('.' + c for c in babelfish.CONVERTERS['alpha2'].codes)
     dirpath, filename = os.path.split(path)
-    subtitles = {babelfish.Language.fromalpha2(os.path.splitext(p)[0][-2:]) for p in os.listdir(dirpath)
-                 if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0])
-                 and os.path.splitext(p)[0].endswith(LANGUAGE_EXTENSIONS)}
+    subtitles = set()
+    for p in os.listdir(dirpath):
+        if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0]) and p.endswith(SUBTITLE_EXTENSIONS):
+            if os.path.splitext(p)[0].endswith(language_extensions):
+                subtitles.add(babelfish.Language.fromalpha2(os.path.splitext(p)[0][-2:]))
+            else:
+                subtitles.add(babelfish.Language('und'))
     logger.debug('Found subtitles %r', subtitles)
     return subtitles
 
@@ -169,7 +172,7 @@ def scan_subtitle_languages(path):
 def scan_video(path, subtitles=True, embedded_subtitles=True):
     """Scan a video and its subtitle languages from a video `path`
 
-    :param string path: path to the video
+    :param string path: absolute path to the video
     :param bool subtitles: scan for subtitles with the same name
     :param bool embedded_subtitles: scan for embedded subtitles
     :return: the scanned video
@@ -179,63 +182,80 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
     """
     dirpath, filename = os.path.split(path)
     logger.info('Scanning video %r in %r', filename, dirpath)
-    video = Video.fromguess(path, guessit.guess_file_info(filename, 'autodetect'))
-    # mkv container
-    if filename.endswith('.mkv'):
-        with open(path, 'rb') as f:
-            mkv = enzyme.MKV(f)
-        video_track = mkv.video_tracks[0]
-        audio_track = mkv.audio_tracks[0]
-        # resolution
-        if video_track.height in (480, 720, 1080):
-            if video_track.interlaced:
-                video.resolution = '%di' % video_track.height
-                logger.debug('Found resolution %s with enzyme', video.resolution)
-            else:
-                video.resolution = '%dp' % video_track.height
-                logger.debug('Found resolution %s with enzyme', video.resolution)
-        # video codec
-        if video_track.codec_id == 'V_MPEG4/ISO/AVC':
-            video.video_codec = 'h264'
-            logger.debug('Found video_codec %s with enzyme', video.video_codec)
-        elif video_track.codec_id == 'V_MPEG4/ISO/SP':
-            video.video_codec = 'DivX'
-            logger.debug('Found video_codec %s with enzyme', video.video_codec)
-        elif video_track.codec_id == 'V_MPEG4/ISO/ASP':
-            video.video_codec = 'XviD'
-            logger.debug('Found video_codec %s with enzyme', video.video_codec)
-        # audio codec
-        if audio_track.codec_id == 'A_AC3':
-            video.audio_codec = 'AC3'
-            logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
-        elif audio_track.codec_id == 'A_DTS':
-            video.audio_codec = 'DTS'
-            logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
-        elif audio_track.codec_id == 'A_AAC':
-            video.audio_codec = 'AAC'
-            logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
-        # embedded subtitles
-        if embedded_subtitles:
-            embedded_subtitle_languages = {babelfish.Language.fromalpha3b(st.language) for st in
-                                           mkv.subtitle_tracks if st.language != 'und'}
-            if embedded_subtitle_languages:
-                logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages)
-                video.subtitle_languages |= embedded_subtitle_languages
+    video = Video.fromguess(path, guessit.guess_file_info(path, 'autodetect'))
     video.size = os.path.getsize(path)
-    logger.debug('Size is %d', video.size)
-    video.hashes['opensubtitles'] = hash_opensubtitles(path)
-    video.hashes['thesubdb'] = hash_thesubdb(path)
-    logger.debug('Computed hashes %r', video.hashes)
-    # add subtitles
+    if video.size > 10485760:
+        logger.debug('Size is %d', video.size)
+        video.hashes['opensubtitles'] = hash_opensubtitles(path)
+        video.hashes['thesubdb'] = hash_thesubdb(path)
+        logger.debug('Computed hashes %r', video.hashes)
+    else:
+        logger.warning('Size is lower than 10MB: hashes not computed')
     if subtitles:
         video.subtitle_languages |= scan_subtitle_languages(path)
+    # enzyme
+    try:
+        if filename.endswith('.mkv'):
+            with open(path, 'rb') as f:
+                mkv = enzyme.MKV(f)
+            if mkv.video_tracks:
+                video_track = mkv.video_tracks[0]
+                # resolution
+                if video_track.height in (480, 720, 1080):
+                    if video_track.interlaced:
+                        video.resolution = '%di' % video_track.height
+                        logger.debug('Found resolution %s with enzyme', video.resolution)
+                    else:
+                        video.resolution = '%dp' % video_track.height
+                        logger.debug('Found resolution %s with enzyme', video.resolution)
+                # video codec
+                if video_track.codec_id == 'V_MPEG4/ISO/AVC':
+                    video.video_codec = 'h264'
+                    logger.debug('Found video_codec %s with enzyme', video.video_codec)
+                elif video_track.codec_id == 'V_MPEG4/ISO/SP':
+                    video.video_codec = 'DivX'
+                    logger.debug('Found video_codec %s with enzyme', video.video_codec)
+                elif video_track.codec_id == 'V_MPEG4/ISO/ASP':
+                    video.video_codec = 'XviD'
+                    logger.debug('Found video_codec %s with enzyme', video.video_codec)
+            else:
+                logger.warning('MKV has no video track')
+            if mkv.audio_tracks:
+                audio_track = mkv.audio_tracks[0]
+                # audio codec
+                if audio_track.codec_id == 'A_AC3':
+                    video.audio_codec = 'AC3'
+                    logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+                elif audio_track.codec_id == 'A_DTS':
+                    video.audio_codec = 'DTS'
+                    logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+                elif audio_track.codec_id == 'A_AAC':
+                    video.audio_codec = 'AAC'
+                    logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+            else:
+                logger.warning('MKV has no audio track')
+            if mkv.subtitle_tracks:
+                # embedded subtitles
+                if embedded_subtitles:
+                    embedded_subtitle_languages = set()
+                    for st in mkv.subtitle_tracks:
+                        try:
+                            embedded_subtitle_languages.add(babelfish.Language.fromalpha3b(st.language or 'und'))
+                        except babelfish.Error:
+                            logger.error('Embedded subtitle language %r is not a valid language', st.language)
+                    logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages)
+                    video.subtitle_languages |= embedded_subtitle_languages
+            else:
+                logger.debug('MKV has no subtitle track')
+    except enzyme.Error:
+        logger.error('Parsing video metadata with enzyme failed')
     return video
 
 
 def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
     """Scan `paths` for videos and their subtitle languages
 
-    :params paths: paths to scan for videos
+    :params paths: absolute paths to scan for videos
     :type paths: list of string
     :param bool subtitles: scan for subtitles with the same name
     :param bool embedded_subtitles: scan for embedded subtitles
@@ -248,31 +268,57 @@ def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
     videos = []
     # scan files
     for filepath in [p for p in paths if os.path.isfile(p)]:
+        if age and datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath)) > age:
+            logger.info('Skipping video %r: older than %r', filepath, age)
+            continue
         try:
-            videos.append(scan_video(filepath, subtitles))
+            videos.append(scan_video(filepath, subtitles, embedded_subtitles))
         except ValueError as e:
-            logger.info('Skipping video: %s', e)
+            logger.error('Skipping video: %s', e)
             continue
     # scan directories
     for path in [p for p in paths if os.path.isdir(p)]:
         logger.info('Scanning directory %r', path)
-        for dirpath, _, filenames in os.walk(path):
-            # skip badly encoded directories and files
+        for dirpath, dirnames, filenames in os.walk(path):
+            # skip badly encoded directories
             if isinstance(dirpath, bytes):
                 logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', errors='replace'))
                 continue
-            safe_filenames = []
-            for filename in filenames:
-                if isinstance(filename, bytes):
-                    logger.error('Skipping badly encoded filename %r', filename.decode('utf-8', errors='replace'))
-                    continue
-                safe_filenames.append(filename)
+            # skip badly encoded and hidden sub directories
+            for dirname in list(dirnames):
+                if isinstance(dirname, bytes):
+                    logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', errors='replace'),
+                                 dirpath)
+                    dirnames.remove(dirname)
+                elif dirname.startswith('.'):
+                    logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath)
+                    dirnames.remove(dirname)
             # scan for videos
-            for video_filename in [f for f in safe_filenames if f.endswith(VIDEO_EXTENSIONS)]:
+            for filename in filenames:
+                # skip badly encoded files
+                if isinstance(filename, bytes):
+                    logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', errors='replace'),
+                                 dirpath)
+                    continue
+                # filter videos
+                if not filename.endswith(VIDEO_EXTENSIONS):
+                    continue
+                # skip hidden files
+                if filename.startswith('.'):
+                    logger.debug('Skipping hidden filename %r in %r', filename, dirpath)
+                    continue
+                filepath = os.path.join(dirpath, filename)
+                # skip links
+                if os.path.islink(filepath):
+                    logger.debug('Skipping link %r in %r', filename, dirpath)
+                    continue
+                if age and datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath)) > age:
+                    logger.info('Skipping video %r: older than %r', filepath, age)
+                    continue
                 try:
-                    video = scan_video(os.path.join(dirpath, video_filename), subtitles=subtitles)
+                    video = scan_video(filepath, subtitles, embedded_subtitles)
                 except ValueError as e:
-                    logger.info('Skipping video: %s', e)
+                    logger.error('Skipping video: %s', e)
                     continue
                 videos.append(video)
     return videos