Merge branch 'develop'

Matching confidence could not be computed because of the
missing extension

.gitignore

@@ -20,16 +20,24 @@ pip-log.txt
 .coverage
 .tox
 
-#Translations
+# Translations
 *.mo
 
-#Mr Developer
+# Mr Developer
 .mr.developer.cfg
 
-#Pydev
+# Pydev
 .project
 .pydevproject
 .settings
 
-#Sphinx
+# Rope
+.ropeproject
+
+# Sphinx
 docs/_build
+
+# Subliminal unittests
+tests/*.srt
+tests/*_files
+tests/*_cache

.travis.yml

@@ -0,0 +1,9 @@
+language: python
+python:
+  - "2.7"
+install:
+  - pip install -r requirements.txt --use-mirrors
+  - pip install -r optional-requirements.txt --use-mirrors
+script: python setup.py test
+notifications:
+  irc: "irc.freenode.org#subliminal"

NEWS.rst

@@ -1,10 +1,50 @@
 News
 ====
 
+0.6.1
+-----
+**release date:** 2012-06-24
+
+* Fix subtitle release name in BierDopje
+* Fix subtitles being downloaded multiple times
+* Add Chinese support to TvSubtitles
+* Fix encoding issues
+* Fix single download subtitles without the force option
+* Add Spanish (Latin America) exception to Addic7ed
+* Fix group_by_video when a list entry has None as subtitles
+* Add support for Galician language in Subtitulos
+* Add an integrity check after subtitles download for Addic7ed
+* Add error handling for if not strict in Language
+* Fix TheSubDB hash method to return None if the file is too small
+* Fix guessit.Language in Video.scan
+* Fix language detection of subtitles
+
+0.6.0
+-----
+**release date:** 2012-06-16
+**WARNING:** Backward incompatible changes
+
+* Fix --workers option in CLI
+* Use a dedicated module for languages
+* Use beautifulsoup4
+* Improve return types
+* Add scan_filter option
+* Add --age option in CLI
+* Add TvSubtitles service
+* Add Addic7ed service
+
+
+0.5.1
+-----
+**release date:** 2012-03-25
+
+* Improve error handling of enzyme parsing
+
 0.5
 ---
 **release date:** 2012-03-25
 **WARNING:** Backward incompatible changes
+
 * Use more unicode
 * New list_subtitles and download_subtitles methods
 * New Pool object for asynchronous work
@@ -17,12 +57,14 @@ News
 0.4
 ---
 **release date:** 2011-11-11
+
 * Many fixes
 * Better error handling
 
 0.3
 ---
 **release date:** 2011-08-18
+
 * Fix a bug when series is not guessed by guessit
 * Fix dependencies failure when installing package
 * Fix encoding issues with logging
@@ -33,6 +75,7 @@ News
 0.2
 ---
 **release date:** 2011-07-11
+
 * Fix plugin configuration
 * Fix some encoding issues
 * Remove extra logging
@@ -40,4 +83,5 @@ News
 0.1
 ---
 **release date:** not released yet
+
 * Initial release

README.rst

@@ -1,5 +1,8 @@
 Subliminal
 ==========
+
+.. image:: https://secure.travis-ci.org/Diaoul/subliminal.png?branch=develop
+
 Subliminal is a python library to search and download subtitles.
 
 It uses video hashes and the powerful `guessit <http://guessit.readthedocs.org/>`_ library
@@ -11,11 +14,13 @@ Features
 --------
 Multiple subtitles services are available:
 
-* OpenSubtitles
-* TheSubDB
+* Addic7ed
 * BierDopje
+* OpenSubtitles
 * SubsWiki
 * Subtitulos
+* TheSubDB
+* TvSubtitles
 
 You can use main subliminal's functions with a **file path**, a **file name** or a **folder path**.
 
@@ -25,8 +30,8 @@ Download english subtitles::
 
     $ subliminal -l en The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4
     **************************************************
-    Downloaded 1 subtitles
-    (Episode(u'The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'), ResultSubtitle(en, opensubtitles, 0.33, The.Big.Bang.Theory.S05E18.HDTV-LOL.srt))
+    Downloaded 1 subtitle(s) for 1 video(s)
+    The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.srt from opensubtitles
     **************************************************
 
 Module

docs/_templates/sidebar-watch.html

@@ -1,4 +0,0 @@
-<p>
-  <iframe src="http://markdotto.github.com/github-buttons/github-btn.html?user=Diaoul&repo=subliminal&type=watch&count=true&size=large"
-    allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
-</p>

docs/conf.py

@@ -93,12 +93,17 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'flask'
+html_theme = 'diaoul'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {'github_user': 'Diaoul',
+                      'github_repo': 'subliminal',
+                      'github_branch': 'develop',
+                      'fork_me': 1,
+                      'flattr_href': 'http://subliminal.readthedocs.org/',
+                      'flattr_thing_url': 'http://flattr.com/thing/629842/Subliminal'}
 
 # Add any paths that contain custom themes here, relative to this directory.
 html_theme_path = ['_themes']
@@ -134,7 +139,7 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
-    'index':    ['sidebar-intro.html', 'sidebar-watch.html', 'localtoc.html', 'sidebar-links.html', 'searchbox.html'],
+    'index':    ['sidebar-intro.html', 'sidebar-watch.html', 'sidebar-travis-ci.html', 'sidebar-donate.html', 'localtoc.html', 'sidebar-links.html', 'searchbox.html'],
     '**':       ['localtoc.html', 'relations.html', 'sourcelink.html']
     }
 
@@ -245,3 +250,6 @@ texinfo_documents = [
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
+
+# -- Options for autodoc -------------------------------------------------------
+autodoc_member_order = 'bysource'

docs/dev.rst

@@ -8,7 +8,7 @@ combining different websites but there is no guarantee of a perfect match.
 Even if OpenSubtitles has a gigantic subtitles database, you may not be able to
 find a subtitle on it but you will find it elsewhere, say BierDopje. Sometimes,
 it just takes some time before it shows up on a website even if already available
-on another, but you don't wanna wait to watch the latest Big Bang Theory, right?
+on another, but you do not want to wait to watch the latest Big Bang Theory, right?
 
 Given this, to be reliable, subliminal has to use different :mod:`~subliminal.services`
 and use a unified method to gather them all. The :class:`~subliminal.services.ServiceBase`
@@ -19,11 +19,22 @@ class will achieve this.
 
 Languages
 ---------
-To be able to support many languages, subliminal has a :mod:`~subliminal.languages`
-module that contains utility functions and ISO languages code (639-1 and 639-2)
+To be able to support many languages, subliminal makes heavy use of ISO-3166 and ISO-639
+in a dedicated module.
 
-.. automodule:: subliminal.languages
+.. automodule:: subliminal.language
     :members:
+    
+    .. data:: subliminal.language.COUNTRIES
+
+        ISO-3166-1 countries list from `Debian package iso-codes 3.36-1 <http://packages.debian.org/fr/sid/iso-codes>`_.
+        Each item of this list is a tuple like ``(‘alpha2’, ‘alpha3’, ‘numeric’, ‘name’)``
+    
+    .. data:: subliminal.language.LANGUAGES
+
+        ISO-639-2 languages list from `the official source <http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt>`_.
+        Each item of this list is a tuple like ``(‘alpha3’, ‘terminologic’, ‘alpha2’, ‘name’, ‘french_name’)``
+    
 
 Tasks
 -----

docs/index.rst

@@ -18,11 +18,13 @@ Features
 --------
 Multiple subtitles services are available:
 
-* OpenSubtitles
-* TheSubDB
+* Addic7ed
 * BierDopje
+* OpenSubtitles
 * SubsWiki
 * Subtitulos
+* TheSubDB
+* TvSubtitles
 
 You can use main subliminal's functions with a **file path**, a **file name** or a **folder path**.
 
@@ -32,8 +34,8 @@ Download english subtitles::
 
     $ subliminal -l en The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4
     **************************************************
-    Downloaded 1 subtitles
-    (Episode(u'The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'), ResultSubtitle(en, opensubtitles, 0.33, The.Big.Bang.Theory.S05E18.HDTV-LOL.srt))
+    Downloaded 1 subtitle(s) for 1 video(s)
+    The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.srt from opensubtitles
     **************************************************
 
 Module

docs/user.rst

@@ -1,5 +1,4 @@
-There are 4 different ways of using subliminal and each one
-is described in a dedicated section below.
+There are 4 different ways of using subliminal and each one is described in a dedicated section below.
 
 First, here are some basics
 
@@ -14,32 +13,38 @@ Current available services are available in the :data:`subliminal.SERVICES` vari
 
 Languages
 ^^^^^^^^^
-Subliminal supports multiple languages that are represented with their extended ISO 639-1 code.
-The current extensions to the ISO 639-1 are:
-
-* *po* for Brazilian Portuguese
+Subliminal supports multiple languages representations based on `ISO-639 <http://en.wikipedia.org/wiki/ISO_639>`_
+and `ISO-3166 <http://en.wikipedia.org/wiki/ISO_3166>`_. Any single ISO-639 string or combination of ISO-639 and
+ISO-3166 is acceptable. For example, you can use ``pt-br`` for Portuguese (Brazil) or ``en`` for English.
 
 Paths
 ^^^^^
 All paths parameters in subliminal most commont functions can be either *a file path*,
 *a file name* or a *folder path*
 
-* File path (existing): hashes of the file will be computed and used during the search for services that supports
-  this functionnality.
-* File name (or non-existing file path): the guessit python library will be used to guess informations and a text-based search will
-  be done with services.
-* Folder path (containing video files): the given folder will be searched for video files using their :data:`~subliminal.videos.MIMETYPES`
-  and/or :data:`~subliminal.videos.EXTENSIONS`. The default maximum depth to scan is 3
+* File path (existing): hashes of the file will be computed and used during the search for services that
+  supports this functionnality.
+* File name (or non-existing file path): the guessit python library will be used to guess informations
+  and a text-based search will be done with services.
+* Folder path (containing video files): the given folder will be searched for video files using their
+  :data:`~subliminal.videos.MIMETYPES` and/or :data:`~subliminal.videos.EXTENSIONS`.
+  The default maximum depth to scan is 3
 
 CLI
 ---
-Subliminal is shipped with a basic Command Line Interface that allows you to
-download subtitles for one or more videos in a multi-threaded way.
+Subliminal is shipped with a Command Line Interface that allows you to
+download subtitles for one or more videos in a multithreaded way.
 
+.. note::
+
+    The cache directory defaults to *~/.config/subliminal*. Even on Windows
+
+Usage
+^^^^^
 You can have the documentation of the CLI using ``subliminal --help``::
 
-    usage: subliminal [-h] [-l LG] [-s NAME] [-m] [-f] [-w N] [-c] [-q | -v]
-                      [--cache-dir DIR | --no-cache-dir] [--version]
+    usage: subliminal [-h] [-l LG] [-s NAME] [-m] [-f] [-w N] [-a AGE] [-c]
+                      [-q | -v] [--cache-dir DIR | --no-cache-dir] [--version]
                       PATH [PATH ...]
 
     Subtitles, faster than your thoughts
@@ -55,6 +60,8 @@ You can have the documentation of the CLI using ``subliminal --help``::
       -m, --multi           download multiple subtitle languages
       -f, --force           replace existing subtitle file
       -w N, --workers N     use N threads (default: 4)
+      -a AGE, --age AGE     scan only for files newer or older (prefix with +)
+                            than AGE (e.g. 12h, 1w2d, +3d6h)
       -c, --compatibility   try not to use unicode (use this if you have encoding
                             errors)
       -q, --quiet           disable output
@@ -64,9 +71,13 @@ You can have the documentation of the CLI using ``subliminal --help``::
                             work)
       --version             show program's version number and exit
 
-.. note::
+Cron job
+^^^^^^^^
+This CLI is well suited for automatic subtitles downloads. For example, to download english and french
+subtitles for videos newer than one week under /path/to/videos/ each day at 1:00AM with a single worker,
+you can use the following crontab line::
 
-    The cache directory defaults to *~/.config/subliminal*. Even on Windows
+    0 1 * * * user /path/to/subliminal -m -l en -l fr -w 1 -a 1w -q /path/to/videos/
 
 Simple module use
 -----------------
@@ -105,9 +116,3 @@ that takes care of that for you::
 
     >>> with subliminal.Pool(4) as p:
     ...     p.list_subtitles('The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4', ['en'])
-
-
-* from the command line
-* basic functions :func:`~subliminal.list_subtitles` and :func:`~subliminal.download_subtitles`
-* multi-threaded :class:`~subliminal.async.Pool` which implements the abovementioned functions
-* using your own algorithm that produces and gather results of elementary :class:`~subliminal.tasks.Task`

optional-requirements.txt

@@ -0,0 +1 @@
+lxml

requirements.txt

@@ -1,4 +1,5 @@
-BeautifulSoup>=3.2.0
-guessit>=0.2
+beautifulsoup4>=4.0
+guessit>=0.4.1
 requests
 enzyme>=0.1
+html5lib

scripts/subliminal

@@ -17,9 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 import argparse
-import subliminal
+import datetime
 import logging
 import os
+import re
+import subliminal
 import sys
 
 
@@ -29,7 +31,8 @@ def main():
     parser.add_argument('-s', '--service', action='append', dest='services', help='service to use', metavar='NAME')
     parser.add_argument('-m', '--multi', action='store_true', help='download multiple subtitle languages')
     parser.add_argument('-f', '--force', action='store_true', help='replace existing subtitle file')
-    parser.add_argument('-w', '--workers', action='store', help='use N threads (default: %(default)s)', metavar='N', default=4)
+    parser.add_argument('-w', '--workers', action='store', help='use N threads (default: %(default)s)', metavar='N', type=int, default=4)
+    parser.add_argument('-a', '--age', action='store', help='scan only for files newer or older (prefix with +) than AGE (e.g. 12h, 1w2d, +3d6h)', metavar='AGE', default=None)
     parser.add_argument('-c', '--compatibility', action='store_true', help='try not to use unicode (use this if you have encoding errors)')
     group_verbosity = parser.add_mutually_exclusive_group()
     group_verbosity.add_argument('-q', '--quiet', action='store_true', help='disable output')
@@ -51,6 +54,23 @@ def main():
     if not os.path.exists(args.cache_dir):
         os.mkdir(args.cache_dir)
 
+    # Create filter function
+    scan_filter = None
+    if args.age:
+        regex = re.compile(r'^(?P<sign>\+?)((?P<weeks>\d+?)w)?((?P<days>\d+?)d)?((?P<hours>\d+?)h)?')
+        parts = regex.match(args.age)
+        if not parts:
+            raise ValueError('Incorrect age format')
+        time_params = {}
+        parts = parts.groupdict()
+        for name, param in parts.iteritems():
+            if param and name != 'sign':
+                time_params[name] = int(param)
+        if parts['sign'] == '+':
+            scan_filter = lambda x: datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(x)) < datetime.timedelta(**time_params)
+        else:
+            scan_filter = lambda x: datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(x)) > datetime.timedelta(**time_params)
+
     # Compatibility mode
     if args.compatibility:
         paths = args.paths
@@ -59,19 +79,19 @@ def main():
 
     # Download subtitles
     with subliminal.Pool(args.workers) as p:
-        subtitles = p.download_subtitles(paths, languages=args.languages,
-                             services=args.services, cache_dir=args.cache_dir,
-                             force=args.force, multi=args.multi)
+        results = p.download_subtitles(paths, languages=args.languages, services=args.services, cache_dir=args.cache_dir,
+                                       force=args.force, multi=args.multi, scan_filter=scan_filter)
 
-    if not subtitles:
+    if not results:
         if not args.quiet:
-            sys.stderr.write('No subtitles found\n')
+            sys.stderr.write('No subtitles downloaded\n')
         exit(1)
     if not args.quiet:
         print '*' * 50
-        print 'Downloaded %d subtitles' % len(subtitles)
-        for subtitle in subtitles:
-            print subtitle
+        print 'Downloaded %d subtitle(s) for %d video(s)' % (sum([len(s) for s in results.itervalues()]), len(results))
+        for _, subtitles in results.iteritems():
+            for subtitle in subtitles:
+                print '%s from %s' % (subtitle.path, subtitle.service)
         print '*' * 50
 
 

setup.py

@@ -17,12 +17,17 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 import os.path
+import sys
 from setuptools import setup, find_packages
 
 
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
+required = ['beautifulsoup4 >= 4.0', 'guessit >= 0.4.1', 'requests', 'enzyme >= 0.1', 'html5lib']
+if sys.hexversion < 0x20700f0:
+    required.append('argparse >= 1.1')
+
 execfile(os.path.join(os.path.dirname(__file__), 'subliminal', 'infos.py'))
 setup(name='subliminal',
     version=__version__,
@@ -43,4 +48,6 @@ setup(name='subliminal',
     url='https://github.com/Diaoul/subliminal',
     packages=find_packages(),
     scripts=['scripts/subliminal'],
-    install_requires=['BeautifulSoup >= 3.2.0', 'guessit >= 0.2', 'requests', 'enzyme >= 0.1'])
+    test_suite='tests.suite',
+    install_requires=required,
+    extras_require={'full': ['lxml']})

subliminal/api.py

@@ -18,7 +18,7 @@
 from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
     MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
     group_by_video, key_subtitles)
-from .languages import list_languages
+from .language import language_set, language_list, LANGUAGES
 import logging
 
 
@@ -26,30 +26,32 @@ __all__ = ['list_subtitles', 'download_subtitles']
 logger = logging.getLogger(__name__)
 
 
-def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
     """List subtitles in given paths according to the criteria
 
     :param paths: path(s) to video file or folder
     :type paths: string or list
-    :param list languages: languages to search for, in preferred order
+    :param languages: languages to search for, in preferred order
+    :type languages: list of :class:`~subliminal.language.Language` or string
     :param list services: services to use for the search, in preferred order
     :param bool force: force searching for subtitles even if some are detected
     :param bool multi: search multiple languages for the same video
     :param string cache_dir: path to the cache directory to use
     :param int max_depth: maximum depth for scanning entries
+    :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
     :return: found subtitles
     :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
 
     """
     services = services or SERVICES
-    languages = set(languages or list_languages(1))
+    languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
     if isinstance(paths, basestring):
         paths = [paths]
     if any([not isinstance(p, unicode) for p in paths]):
         logger.warning(u'Not all entries are unicode')
     results = []
     service_instances = {}
-    tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+    tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
     for task in tasks:
         try:
             result = consume_task(task, service_instances)
@@ -61,40 +63,47 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
     return group_by_video(results)
 
 
-def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, order=None):
+def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
     """Download subtitles in given paths according to the criteria
 
     :param paths: path(s) to video file or folder
     :type paths: string or list
-    :param list languages: languages to search for, in preferred order
+    :param languages: languages to search for, in preferred order
+    :type languages: list of :class:`~subliminal.language.Language` or string
     :param list services: services to use for the search, in preferred order
     :param bool force: force searching for subtitles even if some are detected
     :param bool multi: search multiple languages for the same video
     :param string cache_dir: path to the cache directory to use
     :param int max_depth: maximum depth for scanning entries
+    :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
     :param order: preferred order for subtitles sorting
     :type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
-    :return: found subtitles
-    :rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`])
+    :return: downloaded subtitles
+    :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
+
+    .. note::
+
+        If you use ``multi=True``, :data:`~subliminal.core.LANGUAGE_INDEX` has to be the first item of the ``order`` list
+        or you might get unexpected results.
 
     """
     services = services or SERVICES
-    languages = languages or list_languages(1)
+    languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
     if isinstance(paths, basestring):
         paths = [paths]
     order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
-    subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+    subtitles_by_video = list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
     for video, subtitles in subtitles_by_video.iteritems():
         subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
     results = []
     service_instances = {}
-    tasks = create_download_tasks(subtitles_by_video, multi)
+    tasks = create_download_tasks(subtitles_by_video, languages, multi)
     for task in tasks:
         try:
             result = consume_task(task, service_instances)
-            results.append(result)
+            results.append((task.video, result))
         except:
             logger.error(u'Error consuming task %r' % task, exc_info=True)
     for service_instance in service_instances.itervalues():
         service_instance.terminate()
-    return results
+    return group_by_video(results)

subliminal/async.py

@@ -18,13 +18,14 @@
 from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
     SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
     create_download_tasks, group_by_video, key_subtitles)
-from .languages import list_languages
+from .language import language_list, language_set, LANGUAGES
 from .tasks import StopTask
 import Queue
 import logging
 import threading
 
 
+__all__ = ['Worker', 'Pool']
 logger = logging.getLogger(__name__)
 
 
@@ -108,34 +109,34 @@ class Pool(object):
                 break
         return results
 
-    def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+    def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
         """See :meth:`subliminal.list_subtitles`"""
         services = services or SERVICES
-        languages = set(languages or list_languages(1))
+        languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
         if isinstance(paths, basestring):
             paths = [paths]
         if any([not isinstance(p, unicode) for p in paths]):
             logger.warning(u'Not all entries are unicode')
-        tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+        tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
         for task in tasks:
             self.tasks.put(task)
         self.join()
         results = self.collect()
         return group_by_video(results)
 
-    def download_subtitles(self, paths, languages=None, services=None, cache_dir=None, max_depth=3, force=True, multi=False, order=None):
+    def download_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
         """See :meth:`subliminal.download_subtitles`"""
         services = services or SERVICES
-        languages = languages or list_languages(1)
+        languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
         if isinstance(paths, basestring):
             paths = [paths]
         order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
-        subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+        subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
         for video, subtitles in subtitles_by_video.iteritems():
             subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
-        tasks = create_download_tasks(subtitles_by_video, multi)
+        tasks = create_download_tasks(subtitles_by_video, languages, multi)
         for task in tasks:
             self.tasks.put(task)
         self.join()
         results = self.collect()
-        return results
+        return group_by_video(results)

subliminal/cache.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Nicolas Wack <wackou@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from collections import defaultdict
+from functools import wraps
+import logging
+import os.path
+import threading
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+
+__all__ = ['Cache', 'cachedmethod']
+logger = logging.getLogger(__name__)
+
+
+class Cache(object):
+    """A Cache object contains cached values for methods. It can have
+    separate internal caches, one for each service
+
+    """
+    def __init__(self, cache_dir):
+        self.cache_dir = cache_dir
+        self.cache = defaultdict(dict)
+        self.lock = threading.RLock()
+
+    def __del__(self):
+        for service_name in self.cache:
+            self.save(service_name)
+
+    def cache_location(self, service_name):
+        return os.path.join(self.cache_dir, 'subliminal_%s.cache' % service_name)
+
+    def load(self, service_name):
+        with self.lock:
+            if service_name in self.cache:
+                # already loaded
+                return
+
+            self.cache[service_name] = defaultdict(dict)
+            filename = self.cache_location(service_name)
+            logger.debug(u'Cache: loading cache from %s' % filename)
+            try:
+                self.cache[service_name] = pickle.load(open(filename, 'rb'))
+            except IOError:
+                logger.info('Cache: Cache file "%s" doesn\'t exist, creating it' % filename)
+            except EOFError:
+                logger.error('Cache: cache file "%s" is corrupted... Removing it.' % filename)
+                os.remove(filename)
+
+    def save(self, service_name):
+        filename = self.cache_location(service_name)
+        logger.debug(u'Cache: saving cache to %s' % filename)
+        with self.lock:
+            pickle.dump(self.cache[service_name], open(filename, 'wb'))
+
+    def clear(self, service_name):
+        try:
+            os.remove(self.cache_location(service_name))
+        except OSError:
+            pass
+        self.cache[service_name] = defaultdict(dict)
+
+    def cached_func_key(self, func, cls=None):
+        try:
+            cls = func.im_class
+        except:
+            pass
+        return ('%s.%s' % (cls.__module__, cls.__name__), func.__name__)
+
+    def function_cache(self, service_name, func):
+        func_key = self.cached_func_key(func)
+        return self.cache[service_name][func_key]
+
+    def cache_for(self, service_name, func, args, result):
+        # no need to lock here, dict ops are atomic
+        self.function_cache(service_name, func)[args] = result
+
+    def cached_value(self, service_name, func, args):
+        """Raises KeyError if not found"""
+        # no need to lock here, dict ops are atomic
+        return self.function_cache(service_name, func)[args]
+
+
+def cachedmethod(function):
+    """Decorator to make a method use the cache.
+
+    .. note::
+
+        This can NOT be used with static functions, it has to be used on
+        methods of some class
+
+    """
+    @wraps(function)
+    def cached(*args):
+        c = args[0].config.cache
+        service_name = args[0].__class__.__name__
+        func_key = c.cached_func_key(function, cls=args[0].__class__)
+        func_cache = c.cache[service_name][func_key]
+
+        # we need to remove the first element of args for the key, as it is the
+        # instance pointer and we don't want the cache to know which instance
+        # called it, it is shared among all instances of the same class
+        key = args[1:]
+
+        if key in func_cache:
+            result = func_cache[key]
+            logger.debug(u'Using cached value for %s(%s), returns: %s' % (func_key, key, result))
+            return result
+
+        result = function(*args)
+
+        # note: another thread could have already cached a value in the
+        # meantime, but that's ok as we prefer to keep the latest value in
+        # the cache
+        func_cache[key] = result
+        return result
+    return cached

subliminal/core.py

@@ -20,8 +20,10 @@ from .services import ServiceConfig
 from .tasks import DownloadTask, ListTask
 from .utils import get_keywords
 from .videos import Episode, Movie, scan
+from .language import Language
 from collections import defaultdict
 from itertools import groupby
+import bs4
 import guessit
 import logging
 
@@ -30,11 +32,11 @@ __all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
            'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
            'key_subtitles', 'group_by_video']
 logger = logging.getLogger(__name__)
-SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb']
+SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles']
 LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
 
 
-def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth):
+def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter):
     """Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria
 
     :param paths: path(s) to video file or folder
@@ -45,51 +47,48 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
     :param bool multi: search multiple languages for the same video
     :param string cache_dir: path to the cache directory to use
     :param int max_depth: maximum depth for scanning entries
+    :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
     :return: the created tasks
     :rtype: list of :class:`~subliminal.tasks.ListTask`
 
     """
     scan_result = []
     for p in paths:
-        scan_result.extend(scan(p, max_depth))
+        scan_result.extend(scan(p, max_depth, scan_filter))
     logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth))
     tasks = []
     config = ServiceConfig(multi, cache_dir)
+    services = filter_services(services)
     for video, detected_subtitles in scan_result:
-        detected_languages = set([s.language for s in detected_subtitles])
+        detected_languages = set(s.language for s in detected_subtitles)
         wanted_languages = languages.copy()
         if not force and multi:
             wanted_languages -= detected_languages
             if not wanted_languages:
                 logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
                 continue
-        if not force and not multi and None in detected_languages:
+        if not force and not multi and Language('Undetermined') in detected_languages:
             logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
             continue
         logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
         for service_name in services:
             mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
             service = mod.Service
-            service_languages = wanted_languages & service.available_languages()
-            if not service_languages:
-                logger.debug(u'Skipping %r: none of wanted languages %r available for service %s' % (video, wanted_languages, service_name))
+            if not service.check_validity(video, wanted_languages):
                 continue
-            if not service.is_valid_video(video):
-                logger.debug(u'Skipping %r: not part of supported videos %r for service %s' % (video, service.videos, service_name))
-                continue
-            task = ListTask(video, service_languages, service_name, config)
+            task = ListTask(video, wanted_languages & service.languages, service_name, config)
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
     return tasks
 
 
-def create_download_tasks(subtitles_by_video, multi):
+def create_download_tasks(subtitles_by_video, languages, multi):
     """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
 
-    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
+    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results with ordered subtitles
     :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
-    :param order: preferred order for subtitles sorting
-    :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+    :param languages: languages in preferred order
+    :type languages: :class:`~subliminal.language.language_list`
     :param bool multi: download multiple languages for the same video
     :return: the created tasks
     :rtype: list of :class:`~subliminal.tasks.DownloadTask`
@@ -104,7 +103,7 @@ def create_download_tasks(subtitles_by_video, multi):
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
             continue
-        for _, by_language in groupby(subtitles, lambda s: s.language):
+        for _, by_language in groupby(subtitles, lambda s: languages.index(s.language)):
             task = DownloadTask(video, list(by_language))
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
@@ -120,7 +119,7 @@ def consume_task(task, services=None):
     :type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
     :param dict services: mapping between the service name and an instance of this service
     :return: the result of the task
-    :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle`
+    :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle`
 
     """
     if services is None:
@@ -128,21 +127,14 @@ def consume_task(task, services=None):
     logger.info(u'Consuming %r' % task)
     result = None
     if isinstance(task, ListTask):
-        if task.service not in services:
-            mod = __import__('services.' + task.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
-            services[task.service] = mod.Service(task.config)
-            services[task.service].init()
-        subtitles = services[task.service].list(task.video, task.languages)
-        result = subtitles
+        service = get_service(services, task.service, config=task.config)
+        result = service.list(task.video, task.languages)
     elif isinstance(task, DownloadTask):
         for subtitle in task.subtitles:
-            if subtitle.service not in services:
-                mod = __import__('services.' + subtitle.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
-                services[subtitle.service] = mod.Service()
-                services[subtitle.service].init()
+            service = get_service(services, subtitle.service)
             try:
-                services[subtitle.service].download(subtitle)
-                result = subtitle
+                service.download(subtitle)
+                result = [subtitle]
                 break
             except DownloadFailedError:
                 logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
@@ -166,6 +158,7 @@ def matching_confidence(video, subtitle):
     guess = guessit.guess_file_info(subtitle.release, 'autodetect')
     video_keywords = get_keywords(video.guess)
     subtitle_keywords = get_keywords(guess) | subtitle.keywords
+    logger.debug(u'Video keywords %r - Subtitle keywords %r' % (video_keywords, subtitle_keywords))
     replacement = {'keywords': len(video_keywords & subtitle_keywords)}
     if isinstance(video, Episode):
         replacement.update({'series': 0, 'season': 0, 'episode': 0})
@@ -188,11 +181,34 @@ def matching_confidence(video, subtitle):
             if 'year' in guess and guess['year'] == video.year:
                 replacement['year'] = 1
     else:
-        return 0
+        logger.debug(u'Not able to compute confidence for %r' % video)
+        return 0.0
+    logger.debug(u'Found %r' % replacement)
     confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
+    logger.info(u'Computed confidence %.4f for %r and %r' % (confidence, video, subtitle))
     return confidence
 
 
+def get_service(services, service_name, config=None):
+    """Get a service from its name in the service dict with the specified config.
+    If the service does not exist in the service dict, it is created and added to the dict.
+
+    :param dict services: dict where to get existing services or put created ones
+    :param string service_name: name of the service to get
+    :param config: config to use for the service
+    :type config: :class:`~subliminal.services.ServiceConfig` or None
+    :return: the corresponding service
+    :rtype: :class:`~subliminal.services.ServiceBase`
+
+    """
+    if service_name not in services:
+        mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+        services[service_name] = mod.Service()
+        services[service_name].init()
+    services[service_name].config = config
+    return services[service_name]
+
+
 def key_subtitles(subtitle, video, languages, services, order):
     """Create a key to sort subtitle using the given order
 
@@ -212,6 +228,7 @@ def key_subtitles(subtitle, video, languages, services, order):
     for sort_item in order:
         if sort_item == LANGUAGE_INDEX:
             key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
+            key += '{0:01d}'.format(subtitle.language == languages[languages.index(subtitle.language)])
         elif sort_item == SERVICE_INDEX:
             key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
         elif sort_item == SERVICE_CONFIDENCE:
@@ -236,5 +253,23 @@ def group_by_video(list_results):
     """
     result = defaultdict(list)
     for video, subtitles in list_results:
-        result[video] += subtitles
+        result[video] += subtitles or []
     return result
+
+
+def filter_services(services):
+    """Filter out services that are not available because of a missing feature
+
+    :param list services: service names to filter
+    :return: a copy of the initial list of service names without unavailable ones
+    :rtype: list
+
+    """
+    filtered_services = services[:]
+    for service_name in services:
+        mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+        service = mod.Service
+        if service.required_features is not None and bs4.builder_registry.lookup(*service.required_features) is None:
+            logger.warning(u'Service %s not available: none of available features could be used. One of %r required' % (service_name, service.required_features))
+            filtered_services.remove(service_name)
+    return filtered_services

subliminal/exceptions.py

@@ -22,60 +22,11 @@ class Error(Exception):
     pass
 
 
-class InvalidLanguageError(Error):
-    """Exception raised when invalid language is submitted
-
-    Attributes:
-        language -- language that cause the error
-    """
-    def __init__(self, language):
-        self.language = language
-
-    def __str__(self):
-        return self.language
-
-
-class MissingLanguageError(Error):
-    """Exception raised when a missing language is found
-
-    Attributes:
-        language -- the missing language
-    """
-    def __init__(self, language):
-        self.language = language
-
-    def __str__(self):
-        return self.language
-
-
-class InvalidServiceError(Error):
-    """Exception raised when invalid service is submitted
-
-    :param string service: service that causes the error
-
-    """
-    def __init__(self, service):
-        self.service = service
-
-    def __str__(self):
-        return self.service
-
-
 class ServiceError(Error):
     """"Exception raised by services"""
     pass
 
 
-class WrongTaskError(Error):
-    """"Exception raised when invalid task is submitted"""
-    pass
-
-
 class DownloadFailedError(Error):
     """"Exception raised when a download task has failed in service"""
     pass
-
-
-class UnknownVideoError(Error):
-    """"Exception raised when a video could not be identified"""
-    pass

subliminal/infos.py

@@ -15,4 +15,4 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.5'
+__version__ = '0.6.1'

subliminal/languages.py

@@ -1,547 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['convert_language', 'list_languages', 'LANGUAGES']
-
-
-def convert_language(language, to_iso, from_iso=None):
-    """Convert a language into another format
-
-    :param string language: language
-    :param int to_iso: convert language to ISO-639-x
-    :param int from_iso: convert language from ISO-639-x
-    :return: converted language
-    :rtype: string
-
-    """
-    if from_iso == None:  # if no from_iso is given, try to guess it
-        if language.startswith(language[:1].upper()):
-            from_iso = 0
-        elif len(language) == 2:
-            from_iso = 1
-        elif len(language) == 3:
-            from_iso = 2
-        else:
-            raise ValueError('Invalid input language format')
-    if isinstance(language, unicode):
-        language = language.encode('utf-8')
-    converted_language = None
-    for language_tuple in LANGUAGES:
-        if language_tuple[from_iso] == language and language_tuple[to_iso]:
-            converted_language = language_tuple[to_iso]
-            break
-    return converted_language
-
-
-def list_languages(iso):
-    """List languages in the given ISO-639-x format
-
-    :param int iso: ISO-639-x format to list
-    :return: languages in the requested format
-    :rtype: list
-
-    """
-    return [l[iso] for l in LANGUAGES if l[iso]]
-
-#: ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-#: + ('Brazilian', 'po', 'pob')
-LANGUAGES = [('Afar', 'aa', 'aar'),
-             ('Abkhazian', 'ab', 'abk'),
-             ('Achinese', '', 'ace'),
-             ('Acoli', '', 'ach'),
-             ('Adangme', '', 'ada'),
-             ('Adyghe; Adygei', '', 'ady'),
-             ('Afro-Asiatic languages', '', 'afa'),
-             ('Afrihili', '', 'afh'),
-             ('Afrikaans', 'af', 'afr'),
-             ('Ainu', '', 'ain'),
-             ('Akan', 'ak', 'aka'),
-             ('Akkadian', '', 'akk'),
-             ('Albanian', 'sq', 'alb'),
-             ('Aleut', '', 'ale'),
-             ('Algonquian languages', '', 'alg'),
-             ('Southern Altai', '', 'alt'),
-             ('Amharic', 'am', 'amh'),
-             ('English, Old (ca.450-1100)', '', 'ang'),
-             ('Angika', '', 'anp'),
-             ('Apache languages', '', 'apa'),
-             ('Arabic', 'ar', 'ara'),
-             ('Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)', '', 'arc'),
-             ('Aragonese', 'an', 'arg'),
-             ('Armenian', 'hy', 'arm'),
-             ('Mapudungun; Mapuche', '', 'arn'),
-             ('Arapaho', '', 'arp'),
-             ('Artificial languages', '', 'art'),
-             ('Arawak', '', 'arw'),
-             ('Assamese', 'as', 'asm'),
-             ('Asturian; Bable; Leonese; Asturleonese', '', 'ast'),
-             ('Athapascan languages', '', 'ath'),
-             ('Australian languages', '', 'aus'),
-             ('Avaric', 'av', 'ava'),
-             ('Avestan', 'ae', 'ave'),
-             ('Awadhi', '', 'awa'),
-             ('Aymara', 'ay', 'aym'),
-             ('Azerbaijani', 'az', 'aze'),
-             ('Banda languages', '', 'bad'),
-             ('Bamileke languages', '', 'bai'),
-             ('Bashkir', 'ba', 'bak'),
-             ('Baluchi', '', 'bal'),
-             ('Bambara', 'bm', 'bam'),
-             ('Balinese', '', 'ban'),
-             ('Basque', 'eu', 'baq'),
-             ('Basa', '', 'bas'),
-             ('Baltic languages', '', 'bat'),
-             ('Beja; Bedawiyet', '', 'bej'),
-             ('Belarusian', 'be', 'bel'),
-             ('Bemba', '', 'bem'),
-             ('Bengali', 'bn', 'ben'),
-             ('Berber languages', '', 'ber'),
-             ('Bhojpuri', '', 'bho'),
-             ('Bihari languages', 'bh', 'bih'),
-             ('Bikol', '', 'bik'),
-             ('Bini; Edo', '', 'bin'),
-             ('Bislama', 'bi', 'bis'),
-             ('Siksika', '', 'bla'),
-             ('Bantu (Other)', '', 'bnt'),
-             ('Bosnian', 'bs', 'bos'),
-             ('Braj', '', 'bra'),
-             ('Breton', 'br', 'bre'),
-             ('Batak languages', '', 'btk'),
-             ('Buriat', '', 'bua'),
-             ('Buginese', '', 'bug'),
-             ('Bulgarian', 'bg', 'bul'),
-             ('Burmese', 'my', 'bur'),
-             ('Blin; Bilin', '', 'byn'),
-             ('Caddo', '', 'cad'),
-             ('Central American Indian languages', '', 'cai'),
-             ('Galibi Carib', '', 'car'),
-             ('Catalan; Valencian', 'ca', 'cat'),
-             ('Caucasian languages', '', 'cau'),
-             ('Cebuano', '', 'ceb'),
-             ('Celtic languages', '', 'cel'),
-             ('Chamorro', 'ch', 'cha'),
-             ('Chibcha', '', 'chb'),
-             ('Chechen', 'ce', 'che'),
-             ('Chagatai', '', 'chg'),
-             ('Chinese', 'zh', 'chi'),
-             ('Chuukese', '', 'chk'),
-             ('Mari', '', 'chm'),
-             ('Chinook jargon', '', 'chn'),
-             ('Choctaw', '', 'cho'),
-             ('Chipewyan; Dene Suline', '', 'chp'),
-             ('Cherokee', '', 'chr'),
-             ('Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic', 'cu', 'chu'),
-             ('Chuvash', 'cv', 'chv'),
-             ('Cheyenne', '', 'chy'),
-             ('Chamic languages', '', 'cmc'),
-             ('Coptic', '', 'cop'),
-             ('Cornish', 'kw', 'cor'),
-             ('Corsican', 'co', 'cos'),
-             ('Creoles and pidgins, English based', '', 'cpe'),
-             ('Creoles and pidgins, French-based ', '', 'cpf'),
-             ('Creoles and pidgins, Portuguese-based ', '', 'cpp'),
-             ('Cree', 'cr', 'cre'),
-             ('Crimean Tatar; Crimean Turkish', '', 'crh'),
-             ('Creoles and pidgins ', '', 'crp'),
-             ('Kashubian', '', 'csb'),
-             ('Cushitic languages', '', 'cus'),
-             ('Czech', 'cs', 'cze'),
-             ('Dakota', '', 'dak'),
-             ('Danish', 'da', 'dan'),
-             ('Dargwa', '', 'dar'),
-             ('Land Dayak languages', '', 'day'),
-             ('Delaware', '', 'del'),
-             ('Slave (Athapascan)', '', 'den'),
-             ('Dogrib', '', 'dgr'),
-             ('Dinka', '', 'din'),
-             ('Divehi; Dhivehi; Maldivian', 'dv', 'div'),
-             ('Dogri', '', 'doi'),
-             ('Dravidian languages', '', 'dra'),
-             ('Lower Sorbian', '', 'dsb'),
-             ('Duala', '', 'dua'),
-             ('Dutch, Middle (ca.1050-1350)', '', 'dum'),
-             ('Dutch; Flemish', 'nl', 'dut'),
-             ('Dyula', '', 'dyu'),
-             ('Dzongkha', 'dz', 'dzo'),
-             ('Efik', '', 'efi'),
-             ('Egyptian (Ancient)', '', 'egy'),
-             ('Ekajuk', '', 'eka'),
-             ('Elamite', '', 'elx'),
-             ('English', 'en', 'eng'),
-             ('English, Middle (1100-1500)', '', 'enm'),
-             ('Esperanto', 'eo', 'epo'),
-             ('Estonian', 'et', 'est'),
-             ('Ewe', 'ee', 'ewe'),
-             ('Ewondo', '', 'ewo'),
-             ('Fang', '', 'fan'),
-             ('Faroese', 'fo', 'fao'),
-             ('Fanti', '', 'fat'),
-             ('Fijian', 'fj', 'fij'),
-             ('Filipino; Pilipino', '', 'fil'),
-             ('Finnish', 'fi', 'fin'),
-             ('Finno-Ugrian languages', '', 'fiu'),
-             ('Fon', '', 'fon'),
-             ('French', 'fr', 'fre'),
-             ('French, Middle (ca.1400-1600)', '', 'frm'),
-             ('French, Old (842-ca.1400)', '', 'fro'),
-             ('Northern Frisian', '', 'frr'),
-             ('Eastern Frisian', '', 'frs'),
-             ('Western Frisian', 'fy', 'fry'),
-             ('Fulah', 'ff', 'ful'),
-             ('Friulian', '', 'fur'),
-             ('Ga', '', 'gaa'),
-             ('Gayo', '', 'gay'),
-             ('Gbaya', '', 'gba'),
-             ('Germanic languages', '', 'gem'),
-             ('Georgian', 'ka', 'geo'),
-             ('German', 'de', 'ger'),
-             ('Geez', '', 'gez'),
-             ('Gilbertese', '', 'gil'),
-             ('Gaelic; Scottish Gaelic', 'gd', 'gla'),
-             ('Irish', 'ga', 'gle'),
-             ('Galician', 'gl', 'glg'),
-             ('Manx', 'gv', 'glv'),
-             ('German, Middle High (ca.1050-1500)', '', 'gmh'),
-             ('German, Old High (ca.750-1050)', '', 'goh'),
-             ('Gondi', '', 'gon'),
-             ('Gorontalo', '', 'gor'),
-             ('Gothic', '', 'got'),
-             ('Grebo', '', 'grb'),
-             ('Greek, Ancient (to 1453)', '', 'grc'),
-             ('Greek, Modern (1453-)', 'el', 'gre'),
-             ('Guarani', 'gn', 'grn'),
-             ('Swiss German; Alemannic; Alsatian', '', 'gsw'),
-             ('Gujarati', 'gu', 'guj'),
-             ('Gwich\'in', '', 'gwi'),
-             ('Haida', '', 'hai'),
-             ('Haitian; Haitian Creole', 'ht', 'hat'),
-             ('Hausa', 'ha', 'hau'),
-             ('Hawaiian', '', 'haw'),
-             ('Hebrew', 'he', 'heb'),
-             ('Herero', 'hz', 'her'),
-             ('Hiligaynon', '', 'hil'),
-             ('Himachali languages; Western Pahari languages', '', 'him'),
-             ('Hindi', 'hi', 'hin'),
-             ('Hittite', '', 'hit'),
-             ('Hmong; Mong', '', 'hmn'),
-             ('Hiri Motu', 'ho', 'hmo'),
-             ('Croatian', 'hr', 'hrv'),
-             ('Upper Sorbian', '', 'hsb'),
-             ('Hungarian', 'hu', 'hun'),
-             ('Hupa', '', 'hup'),
-             ('Iban', '', 'iba'),
-             ('Igbo', 'ig', 'ibo'),
-             ('Icelandic', 'is', 'ice'),
-             ('Ido', 'io', 'ido'),
-             ('Sichuan Yi; Nuosu', 'ii', 'iii'),
-             ('Ijo languages', '', 'ijo'),
-             ('Inuktitut', 'iu', 'iku'),
-             ('Interlingue; Occidental', 'ie', 'ile'),
-             ('Iloko', '', 'ilo'),
-             ('Interlingua (International Auxiliary Language Association)', 'ia', 'ina'),
-             ('Indic languages', '', 'inc'),
-             ('Indonesian', 'id', 'ind'),
-             ('Indo-European languages', '', 'ine'),
-             ('Ingush', '', 'inh'),
-             ('Inupiaq', 'ik', 'ipk'),
-             ('Iranian languages', '', 'ira'),
-             ('Iroquoian languages', '', 'iro'),
-             ('Italian', 'it', 'ita'),
-             ('Javanese', 'jv', 'jav'),
-             ('Lojban', '', 'jbo'),
-             ('Japanese', 'ja', 'jpn'),
-             ('Judeo-Persian', '', 'jpr'),
-             ('Judeo-Arabic', '', 'jrb'),
-             ('Kara-Kalpak', '', 'kaa'),
-             ('Kabyle', '', 'kab'),
-             ('Kachin; Jingpho', '', 'kac'),
-             ('Kalaallisut; Greenlandic', 'kl', 'kal'),
-             ('Kamba', '', 'kam'),
-             ('Kannada', 'kn', 'kan'),
-             ('Karen languages', '', 'kar'),
-             ('Kashmiri', 'ks', 'kas'),
-             ('Kanuri', 'kr', 'kau'),
-             ('Kawi', '', 'kaw'),
-             ('Kazakh', 'kk', 'kaz'),
-             ('Kabardian', '', 'kbd'),
-             ('Khasi', '', 'kha'),
-             ('Khoisan languages', '', 'khi'),
-             ('Central Khmer', 'km', 'khm'),
-             ('Khotanese; Sakan', '', 'kho'),
-             ('Kikuyu; Gikuyu', 'ki', 'kik'),
-             ('Kinyarwanda', 'rw', 'kin'),
-             ('Kirghiz; Kyrgyz', 'ky', 'kir'),
-             ('Kimbundu', '', 'kmb'),
-             ('Konkani', '', 'kok'),
-             ('Komi', 'kv', 'kom'),
-             ('Kongo', 'kg', 'kon'),
-             ('Korean', 'ko', 'kor'),
-             ('Kosraean', '', 'kos'),
-             ('Kpelle', '', 'kpe'),
-             ('Karachay-Balkar', '', 'krc'),
-             ('Karelian', '', 'krl'),
-             ('Kru languages', '', 'kro'),
-             ('Kurukh', '', 'kru'),
-             ('Kuanyama; Kwanyama', 'kj', 'kua'),
-             ('Kumyk', '', 'kum'),
-             ('Kurdish', 'ku', 'kur'),
-             ('Kutenai', '', 'kut'),
-             ('Ladino', '', 'lad'),
-             ('Lahnda', '', 'lah'),
-             ('Lamba', '', 'lam'),
-             ('Lao', 'lo', 'lao'),
-             ('Latin', 'la', 'lat'),
-             ('Latvian', 'lv', 'lav'),
-             ('Lezghian', '', 'lez'),
-             ('Limburgan; Limburger; Limburgish', 'li', 'lim'),
-             ('Lingala', 'ln', 'lin'),
-             ('Lithuanian', 'lt', 'lit'),
-             ('Mongo', '', 'lol'),
-             ('Lozi', '', 'loz'),
-             ('Luxembourgish; Letzeburgesch', 'lb', 'ltz'),
-             ('Luba-Lulua', '', 'lua'),
-             ('Luba-Katanga', 'lu', 'lub'),
-             ('Ganda', 'lg', 'lug'),
-             ('Luiseno', '', 'lui'),
-             ('Lunda', '', 'lun'),
-             ('Luo (Kenya and Tanzania)', '', 'luo'),
-             ('Lushai', '', 'lus'),
-             ('Macedonian', 'mk', 'mac'),
-             ('Madurese', '', 'mad'),
-             ('Magahi', '', 'mag'),
-             ('Marshallese', 'mh', 'mah'),
-             ('Maithili', '', 'mai'),
-             ('Makasar', '', 'mak'),
-             ('Malayalam', 'ml', 'mal'),
-             ('Mandingo', '', 'man'),
-             ('Maori', 'mi', 'mao'),
-             ('Austronesian languages', '', 'map'),
-             ('Marathi', 'mr', 'mar'),
-             ('Masai', '', 'mas'),
-             ('Malay', 'ms', 'may'),
-             ('Moksha', '', 'mdf'),
-             ('Mandar', '', 'mdr'),
-             ('Mende', '', 'men'),
-             ('Irish, Middle (900-1200)', '', 'mga'),
-             ('Mi\'kmaq; Micmac', '', 'mic'),
-             ('Minangkabau', '', 'min'),
-             ('Uncoded languages', '', 'mis'),
-             ('Mon-Khmer languages', '', 'mkh'),
-             ('Malagasy', 'mg', 'mlg'),
-             ('Maltese', 'mt', 'mlt'),
-             ('Manchu', '', 'mnc'),
-             ('Manipuri', '', 'mni'),
-             ('Manobo languages', '', 'mno'),
-             ('Mohawk', '', 'moh'),
-             ('Mongolian', 'mn', 'mon'),
-             ('Mossi', '', 'mos'),
-             ('Multiple languages', '', 'mul'),
-             ('Munda languages', '', 'mun'),
-             ('Creek', '', 'mus'),
-             ('Mirandese', '', 'mwl'),
-             ('Marwari', '', 'mwr'),
-             ('Mayan languages', '', 'myn'),
-             ('Erzya', '', 'myv'),
-             ('Nahuatl languages', '', 'nah'),
-             ('North American Indian languages', '', 'nai'),
-             ('Neapolitan', '', 'nap'),
-             ('Nauru', 'na', 'nau'),
-             ('Navajo; Navaho', 'nv', 'nav'),
-             ('Ndebele, South; South Ndebele', 'nr', 'nbl'),
-             ('Ndebele, North; North Ndebele', 'nd', 'nde'),
-             ('Ndonga', 'ng', 'ndo'),
-             ('Low German; Low Saxon; German, Low; Saxon, Low', '', 'nds'),
-             ('Nepali', 'ne', 'nep'),
-             ('Nepal Bhasa; Newari', '', 'new'),
-             ('Nias', '', 'nia'),
-             ('Niger-Kordofanian languages', '', 'nic'),
-             ('Niuean', '', 'niu'),
-             ('Norwegian Nynorsk; Nynorsk, Norwegian', 'nn', 'nno'),
-             ('Bokmål, Norwegian; Norwegian Bokmål', 'nb', 'nob'),
-             ('Nogai', '', 'nog'),
-             ('Norse, Old', '', 'non'),
-             ('Norwegian', 'no', 'nor'),
-             ('N\'Ko', '', 'nqo'),
-             ('Pedi; Sepedi; Northern Sotho', '', 'nso'),
-             ('Nubian languages', '', 'nub'),
-             ('Classical Newari; Old Newari; Classical Nepal Bhasa', '', 'nwc'),
-             ('Chichewa; Chewa; Nyanja', 'ny', 'nya'),
-             ('Nyamwezi', '', 'nym'),
-             ('Nyankole', '', 'nyn'),
-             ('Nyoro', '', 'nyo'),
-             ('Nzima', '', 'nzi'),
-             ('Occitan (post 1500); Provençal', 'oc', 'oci'),
-             ('Ojibwa', 'oj', 'oji'),
-             ('Oriya', 'or', 'ori'),
-             ('Oromo', 'om', 'orm'),
-             ('Osage', '', 'osa'),
-             ('Ossetian; Ossetic', 'os', 'oss'),
-             ('Turkish, Ottoman (1500-1928)', '', 'ota'),
-             ('Otomian languages', '', 'oto'),
-             ('Papuan languages', '', 'paa'),
-             ('Pangasinan', '', 'pag'),
-             ('Pahlavi', '', 'pal'),
-             ('Pampanga; Kapampangan', '', 'pam'),
-             ('Panjabi; Punjabi', 'pa', 'pan'),
-             ('Papiamento', '', 'pap'),
-             ('Palauan', '', 'pau'),
-             ('Persian, Old (ca.600-400 B.C.)', '', 'peo'),
-             ('Persian', 'fa', 'per'),
-             ('Philippine languages', '', 'phi'),
-             ('Phoenician', '', 'phn'),
-             ('Pali', 'pi', 'pli'),
-             ('Polish', 'pl', 'pol'),
-             ('Pohnpeian', '', 'pon'),
-             ('Portuguese', 'pt', 'por'),
-             ('Prakrit languages', '', 'pra'),
-             ('Provençal, Old (to 1500)', '', 'pro'),
-             ('Pushto; Pashto', 'ps', 'pus'),
-             ('Reserved for local use', '', 'qaa-qtz'),
-             ('Quechua', 'qu', 'que'),
-             ('Rajasthani', '', 'raj'),
-             ('Rapanui', '', 'rap'),
-             ('Rarotongan; Cook Islands Maori', '', 'rar'),
-             ('Romance languages', '', 'roa'),
-             ('Romansh', 'rm', 'roh'),
-             ('Romany', '', 'rom'),
-             ('Romanian; Moldavian; Moldovan', 'ro', 'rum'),
-             ('Rundi', 'rn', 'run'),
-             ('Aromanian; Arumanian; Macedo-Romanian', '', 'rup'),
-             ('Russian', 'ru', 'rus'),
-             ('Sandawe', '', 'sad'),
-             ('Sango', 'sg', 'sag'),
-             ('Yakut', '', 'sah'),
-             ('South American Indian (Other)', '', 'sai'),
-             ('Salishan languages', '', 'sal'),
-             ('Samaritan Aramaic', '', 'sam'),
-             ('Sanskrit', 'sa', 'san'),
-             ('Sasak', '', 'sas'),
-             ('Santali', '', 'sat'),
-             ('Sicilian', '', 'scn'),
-             ('Scots', '', 'sco'),
-             ('Selkup', '', 'sel'),
-             ('Semitic languages', '', 'sem'),
-             ('Irish, Old (to 900)', '', 'sga'),
-             ('Sign Languages', '', 'sgn'),
-             ('Shan', '', 'shn'),
-             ('Sidamo', '', 'sid'),
-             ('Sinhala; Sinhalese', 'si', 'sin'),
-             ('Siouan languages', '', 'sio'),
-             ('Sino-Tibetan languages', '', 'sit'),
-             ('Slavic languages', '', 'sla'),
-             ('Slovak', 'sk', 'slo'),
-             ('Slovenian', 'sl', 'slv'),
-             ('Southern Sami', '', 'sma'),
-             ('Northern Sami', 'se', 'sme'),
-             ('Sami languages', '', 'smi'),
-             ('Lule Sami', '', 'smj'),
-             ('Inari Sami', '', 'smn'),
-             ('Samoan', 'sm', 'smo'),
-             ('Skolt Sami', '', 'sms'),
-             ('Shona', 'sn', 'sna'),
-             ('Sindhi', 'sd', 'snd'),
-             ('Soninke', '', 'snk'),
-             ('Sogdian', '', 'sog'),
-             ('Somali', 'so', 'som'),
-             ('Songhai languages', '', 'son'),
-             ('Sotho, Southern', 'st', 'sot'),
-             ('Spanish; Castilian', 'es', 'spa'),
-             ('Sardinian', 'sc', 'srd'),
-             ('Sranan Tongo', '', 'srn'),
-             ('Serbian', 'sr', 'srp'),
-             ('Serer', '', 'srr'),
-             ('Nilo-Saharan languages', '', 'ssa'),
-             ('Swati', 'ss', 'ssw'),
-             ('Sukuma', '', 'suk'),
-             ('Sundanese', 'su', 'sun'),
-             ('Susu', '', 'sus'),
-             ('Sumerian', '', 'sux'),
-             ('Swahili', 'sw', 'swa'),
-             ('Swedish', 'sv', 'swe'),
-             ('Classical Syriac', '', 'syc'),
-             ('Syriac', '', 'syr'),
-             ('Tahitian', 'ty', 'tah'),
-             ('Tai languages', '', 'tai'),
-             ('Tamil', 'ta', 'tam'),
-             ('Tatar', 'tt', 'tat'),
-             ('Telugu', 'te', 'tel'),
-             ('Timne', '', 'tem'),
-             ('Tereno', '', 'ter'),
-             ('Tetum', '', 'tet'),
-             ('Tajik', 'tg', 'tgk'),
-             ('Tagalog', 'tl', 'tgl'),
-             ('Thai', 'th', 'tha'),
-             ('Tibetan', 'bo', 'tib'),
-             ('Tigre', '', 'tig'),
-             ('Tigrinya', 'ti', 'tir'),
-             ('Tiv', '', 'tiv'),
-             ('Tokelau', '', 'tkl'),
-             ('Klingon; tlhIngan-Hol', '', 'tlh'),
-             ('Tlingit', '', 'tli'),
-             ('Tamashek', '', 'tmh'),
-             ('Tonga (Nyasa)', '', 'tog'),
-             ('Tonga (Tonga Islands)', 'to', 'ton'),
-             ('Tok Pisin', '', 'tpi'),
-             ('Tsimshian', '', 'tsi'),
-             ('Tswana', 'tn', 'tsn'),
-             ('Tsonga', 'ts', 'tso'),
-             ('Turkmen', 'tk', 'tuk'),
-             ('Tumbuka', '', 'tum'),
-             ('Tupi languages', '', 'tup'),
-             ('Turkish', 'tr', 'tur'),
-             ('Altaic languages', '', 'tut'),
-             ('Tuvalu', '', 'tvl'),
-             ('Twi', 'tw', 'twi'),
-             ('Tuvinian', '', 'tyv'),
-             ('Udmurt', '', 'udm'),
-             ('Ugaritic', '', 'uga'),
-             ('Uighur; Uyghur', 'ug', 'uig'),
-             ('Ukrainian', 'uk', 'ukr'),
-             ('Umbundu', '', 'umb'),
-             ('Undetermined', '', 'und'),
-             ('Urdu', 'ur', 'urd'),
-             ('Uzbek', 'uz', 'uzb'),
-             ('Vai', '', 'vai'),
-             ('Venda', 've', 'ven'),
-             ('Vietnamese', 'vi', 'vie'),
-             ('Volapük', 'vo', 'vol'),
-             ('Votic', '', 'vot'),
-             ('Wakashan languages', '', 'wak'),
-             ('Walamo', '', 'wal'),
-             ('Waray', '', 'war'),
-             ('Washo', '', 'was'),
-             ('Welsh', 'cy', 'wel'),
-             ('Sorbian languages', '', 'wen'),
-             ('Walloon', 'wa', 'wln'),
-             ('Wolof', 'wo', 'wol'),
-             ('Kalmyk; Oirat', '', 'xal'),
-             ('Xhosa', 'xh', 'xho'),
-             ('Yao', '', 'yao'),
-             ('Yapese', '', 'yap'),
-             ('Yiddish', 'yi', 'yid'),
-             ('Yoruba', 'yo', 'yor'),
-             ('Yupik languages', '', 'ypk'),
-             ('Zapotec', '', 'zap'),
-             ('Blissymbols; Blissymbolics; Bliss', '', 'zbl'),
-             ('Zenaga', '', 'zen'),
-             ('Zhuang; Chuang', 'za', 'zha'),
-             ('Zande languages', '', 'znd'),
-             ('Zulu', 'zu', 'zul'),
-             ('Zuni', '', 'zun'),
-             ('No linguistic content; Not applicable', '', 'zxx'),
-             ('Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki', '', 'zza'),
-             ('Brazilian', 'po', 'pob')]

subliminal/services/__init__.py

@@ -15,11 +15,15 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from ..exceptions import MissingLanguageError, DownloadFailedError
+from ..cache import Cache
+from ..exceptions import DownloadFailedError, ServiceError
+from ..language import language_set, Language
+from ..subtitles import EXTENSIONS
 import logging
 import os
 import requests
 import threading
+import zipfile
 
 
 __all__ = ['ServiceBase', 'ServiceConfig']
@@ -37,7 +41,7 @@ class ServiceBase(object):
     server_url = ''
 
     #: User Agent for any HTTP-based requests
-    user_agent = 'subliminal v0.5'
+    user_agent = 'subliminal v0.6'
 
     #: Whether based on an API or not
     api_based = False
@@ -45,14 +49,14 @@ class ServiceBase(object):
     #: Timeout for web requests
     timeout = 5
 
-    #: Lock for cache interactions
-    lock = threading.Lock()
+    #: :class:`~subliminal.language.language_set` of available languages
+    languages = language_set()
 
-    #: Mapping to Service's language codes and subliminal's
-    languages = {}
+    #: Map between language objects and language codes used in the service
+    language_map = {}
 
-    #: Whether the mapping is reverted or not
-    reverted_languages = False
+    #: Default attribute of a :class:`~subliminal.language.Language` to get with :meth:`get_code`
+    language_code = 'alpha2'
 
     #: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
     videos = []
@@ -60,8 +64,12 @@ class ServiceBase(object):
     #: Whether the video has to exist or not
     require_video = False
 
+    #: List of required features for BeautifulSoup
+    required_features = None
+
     def __init__(self, config=None):
         self.config = config or ServiceConfig()
+        self.session = None
 
     def __enter__(self):
         self.init()
@@ -75,33 +83,84 @@ class ServiceBase(object):
         logger.debug(u'Initializing %s' % self.__class__.__name__)
         self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
 
+    def init_cache(self):
+        """Initialize cache, make sure it is loaded from disk"""
+        if not self.config or not self.config.cache:
+            raise ServiceError('Cache directory is required')
+        self.config.cache.load(self.__class__.__name__)
+
+    def save_cache(self):
+        self.config.cache.save(self.__class__.__name__)
+
+    def clear_cache(self):
+        self.config.cache.clear(self.__class__.__name__)
+
+    def cache_for(self, func, args, result):
+        return self.config.cache.cache_for(self.__class__.__name__, func, args, result)
+
+    def cached_value(self, func, args):
+        return self.config.cache.cached_value(self.__class__.__name__, func, args)
+
     def terminate(self):
         """Terminate connection"""
         logger.debug(u'Terminating %s' % self.__class__.__name__)
 
+    def get_code(self, language):
+        """Get the service code for a :class:`~subliminal.language.Language`
+
+        It uses the :data:`language_map` and if there's no match, falls back
+        on the :data:`language_code` attribute of the given :class:`~subliminal.language.Language`
+
+        """
+        if language in self.language_map:
+            return self.language_map[language]
+        if self.language_code is None:
+            raise ValueError('%r has no matching code' % language)
+        return getattr(language, self.language_code)
+
+    def get_language(self, code):
+        """Get a :class:`~subliminal.language.Language` from a service code
+
+        It uses the :data:`language_map` and if there's no match, uses the
+        given code as ``language`` parameter for the :class:`~subliminal.language.Language`
+        constructor
+
+        .. note::
+
+            A warning is emitted if the generated :class:`~subliminal.language.Language`
+            is "Undetermined"
+
+        """
+        if code in self.language_map:
+            return self.language_map[code]
+        language = Language(code, strict=False)
+        if language == Language('Undetermined'):
+            logger.warning(u'Code %s could not be identified as a language for %s' % (code, self.__class__.__name__))
+        return language
+
     def query(self, *args):
         """Make the actual query"""
-        pass
+        raise NotImplementedError()
 
     def list(self, video, languages):
-        """List subtitles"""
-        pass
+        """List subtitles
+
+        As a service writer, you can either override this method or implement
+        :meth:`list_checked` instead to have the languages pre-filtered for you
+
+        """
+        if not self.check_validity(video, languages):
+            return []
+        return self.list_checked(video, languages)
+
+    def list_checked(self, video, languages):
+        """List subtitles without having to check parameters for validity"""
+        raise NotImplementedError()
 
     def download(self, subtitle):
         """Download a subtitle"""
         self.download_file(subtitle.link, subtitle.path)
-
-    @classmethod
-    def available_languages(cls):
-        """Available languages in the Service
-
-        :return: available languages
-        :rtype: set
-
-        """
-        if not cls.reverted_languages:
-            return set(cls.languages.keys())
-        return set(cls.languages.values())
+        return subtitle
 
     @classmethod
     def check_validity(cls, video, languages):
@@ -109,76 +168,20 @@ class ServiceBase(object):
 
         :param video: the video to check
         :type video: :class:`~subliminal.videos.video`
-        :param set languages: languages to check
+        :param languages: languages to check
+        :type languages: :class:`~subliminal.language.Language`
         :rtype: bool
 
         """
-        languages &= cls.available_languages()
+        languages = (languages & cls.languages) - language_set(['Undetermined'])
         if not languages:
-            logger.debug(u'No language available for service %s' % cls.__class__.__name__.lower())
+            logger.debug(u'No language available for service %s' % cls.__name__.lower())
             return False
-        if not cls.is_valid_video(video):
-            logger.debug(u'%r is not valid for service %s' % (video, cls.__class__.__name__.lower()))
+        if cls.require_video and not video.exists or not isinstance(video, tuple(cls.videos)):
+            logger.debug(u'%r is not valid for service %s' % (video, cls.__name__.lower()))
             return False
         return True
 
-    @classmethod
-    def is_valid_video(cls, video):
-        """Check if video is valid in the Service
-
-        :param video: the video to check
-        :type video: :class:`~subliminal.videos.Video`
-        :rtype: bool
-
-        """
-        if cls.require_video and not video.exists:
-            return False
-        if not isinstance(video, tuple(cls.videos)):
-            return False
-        return True
-
-    @classmethod
-    def is_valid_language(cls, language):
-        """Check if language is valid in the Service
-
-        :param string language: the language to check
-        :rtype: bool
-
-        """
-        if language in cls.available_languages():
-            return True
-        return False
-
-    @classmethod
-    def get_revert_language(cls, language):
-        """ISO-639-1 language code from service language code
-
-        :param string language: service language code
-        :return: ISO-639-1 language code
-        :rtype: string
-
-        """
-        if not cls.reverted_languages and language in cls.languages.values():
-            return [k for k, v in cls.languages.iteritems() if v == language][0]
-        if cls.reverted_languages and language in cls.languages.keys():
-            return cls.languages[language]
-        raise MissingLanguageError(language)
-
-    @classmethod
-    def get_language(cls, language):
-        """Service language code from ISO-639-1 language code
-
-        :param string language: ISO-639-1 language code
-        :return: service language code
-        :rtype: string
-
-        """
-        if not cls.reverted_languages and language in cls.languages.keys():
-            return cls.languages[language]
-        if cls.reverted_languages and language in cls.languages.values():
-            return [k for k, v in cls.languages.iteritems() if v == language][0]
-        raise MissingLanguageError(language)
-
     def download_file(self, url, filepath):
         """Attempt to download a file and remove it in case of failure
 
@@ -186,17 +189,53 @@ class ServiceBase(object):
         :param string filepath: destination path
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
             with open(filepath, 'wb') as f:
                 f.write(r.content)
         except Exception as e:
-            logger.error(u'Download %s failed: %s' % (url, e))
+            logger.error(u'Download failed: %s' % e)
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
-        logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+        logger.debug(u'Download finished')
+
+    def download_zip_file(self, url, filepath):
+        """Attempt to download a zip file and extract any subtitle file from it, if any.
+        This cleans up after itself if anything fails.
+
+        :param string url: URL of the zip file to download
+        :param string filepath: destination path for the subtitle
+
+        """
+        logger.info(u'Downloading %s in %s' % (url, filepath))
+        try:
+            zippath = filepath + '.zip'
+            r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
+            with open(zippath, 'wb') as f:
+                f.write(r.content)
+            if not zipfile.is_zipfile(zippath):
+                # TODO: could check if maybe we already have a text file and
+                # download it directly
+                raise DownloadFailedError('Downloaded file is not a zip file')
+            with zipfile.ZipFile(zippath) as zipsub:
+                for subfile in zipsub.namelist():
+                    if os.path.splitext(subfile)[1] in EXTENSIONS:
+                        with open(filepath, 'w') as f:
+                            f.write(zipsub.open(subfile).read())
+                        break
+                else:
+                    raise DownloadFailedError('No subtitles found in zip file')
+            os.remove(zippath)
+        except Exception as e:
+            logger.error(u'Download %s failed: %s' % (url, e))
+            if os.path.exists(zippath):
+                os.remove(zippath)
+            if os.path.exists(filepath):
+                os.remove(filepath)
+            raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
 
 
 class ServiceConfig(object):
@@ -209,6 +248,9 @@ class ServiceConfig(object):
     def __init__(self, multi=False, cache_dir=None):
         self.multi = multi
         self.cache_dir = cache_dir
+        self.cache = None
+        if cache_dir is not None:
+            self.cache = Cache(cache_dir)
 
     def __repr__(self):
-        return 'ServiceConfig(%r, %s)' % (self.multi, self.cache_dir)
+        return 'ServiceConfig(%r, %s)' % (self.multi, self.cache.cache_dir)

subliminal/services/addic7ed.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Olivier Leveau <olifozzy@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from . import ServiceBase
+from ..cache import cachedmethod
+from ..exceptions import DownloadFailedError
+from ..language import Language, language_set
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords
+from ..videos import Episode
+from bs4 import BeautifulSoup
+import logging
+import os
+import re
+
+
+logger = logging.getLogger(__name__)
+
+
+def match(pattern, string):
+    try:
+        return re.search(pattern, string).group(1)
+    except AttributeError:
+        logger.debug(u'Could not match %r on %r' % (pattern, string))
+        return None
+
+
+def matches(pattern, string):
+    try:
+        return re.search(pattern, string).group(1, 2)
+    except AttributeError:
+        logger.debug(u'Could not match %r on %r' % (pattern, string))
+        return None
+
+
+class Addic7ed(ServiceBase):
+    server_url = 'http://www.addic7ed.com'
+    api_based = False
+    #TODO: Complete this
+    languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
+                              'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
+    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre'),
+                    'Spanish (Latin America)': Language('spa'), }
+    videos = [Episode]
+    require_video = False
+    required_features = ['permissive']
+
+    @cachedmethod
+    def get_likely_series_id(self, name):
+        r = self.session.get('%s/shows.php' % self.server_url)
+        soup = BeautifulSoup(r.content, self.required_features)
+        for elem in soup.find_all('h3'):
+            show_name = elem.a.text.lower()
+            show_id = int(match('show/([0-9]+)', elem.a['href']))
+            # we could just return the id of the queried show, but as we
+            # already downloaded the whole page we might as well fill in the
+            # information for all the shows
+            self.cache_for(self.get_likely_series_id, args=(show_name,), result=show_id)
+        return self.cached_value(self.get_likely_series_id, args=(name,))
+
+    @cachedmethod
+    def get_episode_url(self, series_id, season, number):
+        """Get the Addic7ed id for the given episode. Raises KeyError if none
+        could be found
+
+        """
+        # download the page of the show, contains ids for all episodes all seasons
+        r = self.session.get('%s/show/%d' % (self.server_url, series_id))
+        soup = BeautifulSoup(r.content, self.required_features)
+        form = soup.find('form', attrs={'name': 'multidl'})
+        for table in form.find_all('table'):
+            for row in table.find_all('tr'):
+                cell = row.find('td', 'MultiDldS')
+                if not cell:
+                    continue
+                m = matches('/serie/.+/([0-9]+)/([0-9]+)/', cell.a['href'])
+                if not m:
+                    continue
+                episode_url = cell.a['href']
+                season_number = int(m[0])
+                episode_number = int(m[1])
+                # we could just return the url of the queried episode, but as we
+                # already downloaded the whole page we might as well fill in the
+                # information for all the episodes of the show
+                self.cache_for(self.get_episode_url, args=(series_id, season_number, episode_number), result=episode_url)
+        # raises KeyError if not found
+        return self.cached_value(self.get_episode_url, args=(series_id, season, number))
+
+    # Do not cache this method in order to always check for the most recent
+    # subtitles
+    def get_sub_urls(self, episode_url):
+        suburls = []
+        r = self.session.get('%s/%s' % (self.server_url, episode_url))
+        epsoup = BeautifulSoup(r.content, self.required_features)
+        for releaseTable in epsoup.find_all('table', 'tabel95'):
+            releaseRow = releaseTable.find('td', 'NewsTitle')
+            if not releaseRow:
+                continue
+            release = releaseRow.text.strip()
+            for row in releaseTable.find_all('tr'):
+                link = row.find('a', 'buttonDownload')
+                if not link:
+                    continue
+                if 'href' not in link.attrs or not (link['href'].startswith('/original') or link['href'].startswith('/updated')):
+                    continue
+                suburl = link['href']
+                lang = self.get_language(row.find('td', 'language').text.strip())
+                result = {'suburl': suburl, 'language': lang, 'release': release}
+                suburls.append(result)
+        return suburls
+
+    def list_checked(self, video, languages):
+        return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
+
+    def query(self, filepath, languages, keywords, series, season, episode):
+        logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+        self.init_cache()
+        try:
+            sid = self.get_likely_series_id(series.lower())
+        except KeyError:
+            logger.debug(u'Could not find series id for %s' % series)
+            return []
+        try:
+            ep_url = self.get_episode_url(sid, season, episode)
+        except KeyError:
+            logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
+            return []
+        suburls = self.get_sub_urls(ep_url)
+        # filter the subtitles with our queried languages
+        subtitles = []
+        for suburl in suburls:
+            language = suburl['language']
+            if language not in languages:
+                continue
+            path = get_subtitle_path(filepath, language, self.config.multi)
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/%s' % (self.server_url, suburl['suburl']),
+                                      keywords=[suburl['release']])
+            subtitles.append(subtitle)
+        return subtitles
+
+    def download(self, subtitle):
+        logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path))
+        try:
+            r = self.session.get(subtitle.link, headers={'Referer': subtitle.link, 'User-Agent': self.user_agent})
+            soup = BeautifulSoup(r.content, self.required_features)
+            if soup.title is not None and u'Addic7ed.com' in soup.title.text.strip():
+                raise DownloadFailedError('Download limit exceeded')
+            with open(subtitle.path, 'wb') as f:
+                f.write(r.content)
+        except Exception as e:
+            logger.error(u'Download failed: %s' % e)
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
+        return subtitle
+
+
+Service = Addic7ed

subliminal/services/bierdopje.py

@@ -16,13 +16,14 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
+from ..cache import cachedmethod
 from ..exceptions import ServiceError
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode
+from ..language import language_set
+from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
 from ..utils import to_unicode
-import BeautifulSoup
+from ..videos import Episode
+from bs4 import BeautifulSoup
 import logging
-import os.path
 import urllib
 try:
     import cPickle as pickle
@@ -36,30 +37,22 @@ logger = logging.getLogger(__name__)
 class BierDopje(ServiceBase):
     server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
     api_based = True
-    languages = {'en': 'en', 'nl': 'nl'}
-    reverted_languages = False
+    languages = language_set(['eng', 'dut'])
     videos = [Episode]
     require_video = False
+    required_features = ['xml']
 
-    def __init__(self, config=None):
-        super(BierDopje, self).__init__(config)
-        self.showids = {}
-        if self.config and self.config.cache_dir:
-            self.init_cache()
-
-    def init_cache(self):
-        logger.debug(u'Initializing cache...')
-        if not self.config or not self.config.cache_dir:
-            raise ServiceError('Cache directory is required')
-        self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
-        if not os.path.exists(self.showids_cache):
-            self.save_cache()
-
-    def save_cache(self):
-        logger.debug(u'Saving showids to cache...')
-        with self.lock:
-            with open(self.showids_cache, 'w') as f:
-                pickle.dump(self.showids, f)
+    @cachedmethod
+    def get_show_id(self, series):
+        r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
+        if r.status_code != 200:
+            logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+            return None
+        soup = BeautifulSoup(r.content, self.required_features)
+        if soup.status.contents[0] == 'false':
+            logger.debug(u'Could not find show %s' % series)
+            return None
+        return int(soup.showid.contents[0])
 
     def load_cache(self):
         logger.debug(u'Loading showids from cache...')
@@ -67,25 +60,12 @@ class BierDopje(ServiceBase):
             with open(self.showids_cache, 'r') as f:
                 self.showids = pickle.load(f)
 
-    def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
-        self.load_cache()
+    def query(self, filepath, season, episode, languages, tvdbid=None, series=None):
+        self.init_cache()
         if series:
-            if series.lower() in self.showids:  # from cache
-                request_id = self.showids[series.lower()]
-                logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
-            else:  # query to get showid
-                logger.debug(u'Getting showid from show name %s...' % series)
-                r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
-                if r.status_code != 200:
-                    logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
-                    return []
-                soup = BeautifulSoup.BeautifulStoneSoup(r.content)
-                if soup.status.contents[0] == 'false':
-                    logger.debug(u'Could not find show %s' % series)
-                    return []
-                request_id = int(soup.showid.contents[0])
-                self.showids[series.lower()] = request_id
-                self.save_cache()
+            request_id = self.get_show_id(series.lower())
+            if request_id is None:
+                return []
             request_source = 'showid'
             request_is_tvdbid = 'false'
         elif tvdbid:
@@ -96,27 +76,27 @@ class BierDopje(ServiceBase):
             raise ServiceError('One or more parameter missing')
         subtitles = []
         for language in languages:
-            logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
-            r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
+            logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2))
+            r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language.alpha2, request_is_tvdbid))
             if r.status_code != 200:
                 logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
                 return []
-            soup = BeautifulSoup.BeautifulStoneSoup(r.content)
+            soup = BeautifulSoup(r.content, self.required_features)
             if soup.status.contents[0] == 'false':
-                logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
+                logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2))
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
             for result in soup.results('result'):
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
-                                          release=to_unicode(result.filename.contents[0]))
+                release = to_unicode(result.filename.contents[0])
+                if not release.endswith(tuple(EXTENSIONS)):
+                    release += '.srt'
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result.downloadlink.contents[0],
+                                          release=release)
                 subtitles.append(subtitle)
         return subtitles
 
-    def list(self, video, languages):
-        if not self.check_validity(video, languages):
-            return []
-        results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
-        return results
+    def list_checked(self, video, languages):
+        return self.query(video.path or video.release, video.season, video.episode, languages, video.tvdbid, video.series)
 
 
 Service = BierDopje

subliminal/services/opensubtitles.py

@@ -17,9 +17,10 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
 from ..exceptions import ServiceError, DownloadFailedError
+from ..language import Language, language_set
 from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode, Movie
 from ..utils import to_unicode
+from ..videos import Episode, Movie
 import gzip
 import logging
 import os.path
@@ -32,34 +33,50 @@ logger = logging.getLogger(__name__)
 class OpenSubtitles(ServiceBase):
     server_url = 'http://api.opensubtitles.org/xml-rpc'
     api_based = True
-    languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
-                 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
-                 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
-                 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
-                 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
-                 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
-                 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
-                 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
-                 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
-                 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
-                 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
-                 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
-                 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
-                 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
-                 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
-                 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
-                 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
-                 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
-                 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
-                 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
-                 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
-                 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
-                 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
-                 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
-                 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
-                 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
-                 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
-    reverted_languages = False
+    # Source: http://www.opensubtitles.org/addons/export_languages.php
+    languages = language_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh', 'afr', 'ain', 'aka', 'akk',
+                              'alb', 'ale', 'alg', 'alt', 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
+                              'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava', 'ave', 'awa', 'aym', 'aze',
+                              'bad', 'bai', 'bak', 'bal', 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
+                              'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla', 'bnt', 'bos', 'bra', 'bre',
+                              'btk', 'bua', 'bug', 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau', 'ceb',
+                              'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk', 'chm', 'chn', 'cho', 'chp', 'chr',
+                              'chu', 'chv', 'chy', 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre', 'crh',
+                              'crp', 'csb', 'cus', 'cze', 'dak', 'dan', 'dar', 'day', 'del', 'den', 'dgr', 'din',
+                              'div', 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi', 'egy', 'eka', 'ell',
+                              'elx', 'eng', 'enm', 'epo', 'est', 'ewe', 'ewo', 'fan', 'fao', 'fat', 'fij', 'fil',
+                              'fin', 'fiu', 'fon', 'fre', 'frm', 'fro', 'fry', 'ful', 'fur', 'gaa', 'gay', 'gba',
+                              'gem', 'geo', 'ger', 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh', 'gon',
+                              'gor', 'got', 'grb', 'grc', 'grn', 'guj', 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb',
+                              'her', 'hil', 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup', 'iba', 'ibo',
+                              'ice', 'ido', 'iii', 'ijo', 'iku', 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh',
+                              'ipk', 'ira', 'iro', 'ita', 'jav', 'jpn', 'jpr', 'jrb', 'kaa', 'kab', 'kac', 'kal',
+                              'kam', 'kan', 'kar', 'kas', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm', 'kho',
+                              'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon', 'kor', 'kos', 'kpe', 'krc', 'kro',
+                              'kru', 'kua', 'kum', 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav', 'lez',
+                              'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua', 'lub', 'lug', 'lui', 'lun', 'luo',
+                              'lus', 'mac', 'mad', 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map', 'mar',
+                              'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic', 'min', 'mkh', 'mlg', 'mlt', 'mnc',
+                              'mni', 'mno', 'moh', 'mon', 'mos', 'mun', 'mus', 'mwl', 'mwr', 'myn', 'myv', 'nah',
+                              'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds', 'nep', 'new', 'nia', 'nic',
+                              'niu', 'nno', 'nob', 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym', 'nyn',
+                              'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa', 'oss', 'ota', 'oto', 'paa', 'pag',
+                              'pal', 'pam', 'pan', 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol', 'pon',
+                              'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap', 'rar', 'roa', 'roh', 'rom', 'rum',
+                              'run', 'rup', 'rus', 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas', 'sat',
+                              'scn', 'sco', 'sel', 'sem', 'sga', 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla',
+                              'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo', 'sms', 'sna', 'snd', 'snk',
+                              'sog', 'som', 'son', 'sot', 'spa', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk', 'sun',
+                              'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai', 'tam', 'tat', 'tel', 'tem', 'ter',
+                              'tet', 'tgk', 'tgl', 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli', 'tmh',
+                              'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk', 'tum', 'tup', 'tur', 'tut', 'tvl',
+                              'twi', 'tyv', 'udm', 'uga', 'uig', 'ukr', 'umb', 'urd', 'uzb', 'vai', 'ven', 'vie',
+                              'vol', 'vot', 'wak', 'wal', 'war', 'was', 'wel', 'wen', 'wln', 'wol', 'xal', 'xho',
+                              'yao', 'yap', 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'znd', 'zul', 'zun',
+                              'por-BR', 'rum-MD'])
+    language_map = {'mol': Language('rum-MD'), 'scc': Language('srp'), 'pob': Language('por-BR'),
+                    Language('rum-MD'): 'mol', Language('srp'): 'scc', Language('por-BR'): 'pob'}
+    language_code = 'alpha3'
     videos = [Episode, Movie]
     require_video = False
     confidence_order = ['moviehash', 'imdbid', 'fulltext']
@@ -92,7 +109,7 @@ class OpenSubtitles(ServiceBase):
         if not searches:
             raise ServiceError('One or more parameter missing')
         for search in searches:
-            search['sublanguageid'] = ','.join([self.get_language(l) for l in languages])
+            search['sublanguageid'] = ','.join(self.get_code(l) for l in languages)
         logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
         results = self.server.SearchSubtitles(self.token, searches)
         if not results['data']:
@@ -100,17 +117,15 @@ class OpenSubtitles(ServiceBase):
             return []
         subtitles = []
         for result in results['data']:
-            language = self.get_revert_language(result['SubLanguageID'])
+            language = self.get_language(result['SubLanguageID'])
             path = get_subtitle_path(filepath, language, self.config.multi)
             confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['SubDownloadLink'],
                                       release=to_unicode(result['SubFileName']), confidence=confidence)
             subtitles.append(subtitle)
         return subtitles
 
-    def list(self, video, languages):
-        if not self.check_validity(video, languages):
-            return []
+    def list_checked(self, video, languages):
         results = []
         if video.exists:
             results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))

subliminal/services/podnapisi.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from . import ServiceBase
+from ..exceptions import ServiceError, DownloadFailedError
+from ..language import language_set, Language
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import to_unicode
+from ..videos import Episode, Movie
+from hashlib import md5, sha256
+import logging
+import xmlrpclib
+
+
+logger = logging.getLogger(__name__)
+
+
+class Podnapisi(ServiceBase):
+    server_url = 'http://ssp.podnapisi.net:8000'
+    api_based = True
+    languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en',
+                              'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id',
+                              'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl',
+                              'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk',
+                              'vi', 'zh', 'es-ar', 'pt-br'])
+    language_map = {'jp': Language('jpn'), Language('jpn'): 'jp',
+                    'gr': Language('gre'), Language('gre'): 'gr',
+                    'pb': Language('por-BR'), Language('por-BR'): 'pb',
+                    'ag': Language('spa-AR'), Language('spa-AR'): 'ag',
+                    'cyr': Language('srp')}
+    videos = [Episode, Movie]
+    require_video = True
+
+    def __init__(self, config=None):
+        super(Podnapisi, self).__init__(config)
+        self.server = xmlrpclib.ServerProxy(self.server_url)
+        self.token = None
+
+    def init(self):
+        super(Podnapisi, self).init()
+        result = self.server.initiate(self.user_agent)
+        if result['status'] != 200:
+            raise ServiceError('Initiate failed')
+        username = 'python_subliminal'
+        password = sha256(md5('XWFXQ6gE5Oe12rv4qxXX').hexdigest() + result['nonce']).hexdigest()
+        self.token = result['session']
+        result = self.server.authenticate(self.token, username, password)
+        if result['status'] != 200:
+            raise ServiceError('Authenticate failed')
+
+    def terminate(self):
+        super(Podnapisi, self).terminate()
+
+    def query(self, filepath, languages, moviehash):
+        results = self.server.search(self.token, [moviehash])
+        if results['status'] != 200:
+            logger.error('Search failed with error code %d' % results['status'])
+            return []
+        if not results['results'] or not results['results'][moviehash]['subtitles']:
+            logger.debug(u'Could not find subtitles for %r with token %s' % (moviehash, self.token))
+            return []
+        subtitles = []
+        for result in results['results'][moviehash]['subtitles']:
+            language = self.get_language(result['lang'])
+            if language not in languages:
+                continue
+            path = get_subtitle_path(filepath, language, self.config.multi)
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
+                                      release=to_unicode(result['release']), confidence=result['weight'])
+            subtitles.append(subtitle)
+        if not subtitles:
+            return []
+        # Convert weight to confidence
+        max_weight = float(max([s.confidence for s in subtitles]))
+        min_weight = float(min([s.confidence for s in subtitles]))
+        for subtitle in subtitles:
+            if max_weight == 0 and min_weight == 0:
+                subtitle.confidence = 1.0
+            else:
+                subtitle.confidence = (subtitle.confidence - min_weight) / (max_weight - min_weight)
+        return subtitles
+
+    def list_checked(self, video, languages):
+        results = self.query(video.path, languages, video.hashes['OpenSubtitles'])
+        return results
+
+    def download(self, subtitle):
+        results = self.server.download(self.token, [subtitle.link])
+        if results['status'] != 200:
+            raise DownloadFailedError()
+        subtitle.link = 'http://www.podnapisi.net/static/podnapisi/' + results['names'][0]['filename']
+        self.download_file(subtitle.link, subtitle.path)
+        return subtitle
+
+
+Service = Podnapisi

subliminal/services/subswiki.py

@@ -17,10 +17,11 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
 from ..exceptions import ServiceError
+from ..language import language_set, Language
 from ..subtitles import get_subtitle_path, ResultSubtitle
 from ..videos import Episode, Movie
+from bs4 import BeautifulSoup
 from subliminal.utils import get_keywords, split_keyword
-import BeautifulSoup
 import logging
 import re
 import urllib
@@ -32,17 +33,17 @@ logger = logging.getLogger(__name__)
 class SubsWiki(ServiceBase):
     server_url = 'http://www.subswiki.com'
     api_based = False
-    languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
-                 u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
-                 u'Italian': 'it', u'Català': 'ca'}
-    reverted_languages = True
+    languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
+    language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
+                    u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
+                    u'English (UK)': Language('eng-GB')}
+    language_code = 'name'
     videos = [Episode, Movie]
     require_video = False
     release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
+    required_features = ['permissive']
 
-    def list(self, video, languages):
-        if not self.check_validity(video, languages):
-            return []
+    def list_checked(self, video, languages):
         results = []
         if isinstance(video, Episode):
             results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
@@ -74,7 +75,7 @@ class SubsWiki(ServiceBase):
         if r.status_code != 200:
             logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
             return []
-        soup = BeautifulSoup.BeautifulSoup(r.content)
+        soup = BeautifulSoup(r.content, self.required_features)
         subtitles = []
         for sub in soup('td', {'class': 'NewsTitle'}):
             sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
@@ -82,8 +83,8 @@ class SubsWiki(ServiceBase):
                 logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
                 continue
             for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
-                language = self.get_revert_language(html_language.string.strip())
-                if not language in languages:
+                language = self.get_language(html_language.string.strip())
+                if language not in languages:
                     logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
                     continue
                 html_status = html_language.findNextSibling('td')
@@ -92,8 +93,9 @@ class SubsWiki(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
                 subtitles.append(subtitle)
         return subtitles
 
+
 Service = SubsWiki

subliminal/services/subtitulos.py

@@ -16,10 +16,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
+from ..language import language_set, Language
 from ..subtitles import get_subtitle_path, ResultSubtitle
 from ..videos import Episode
+from bs4 import BeautifulSoup
 from subliminal.utils import get_keywords, split_keyword
-import BeautifulSoup
 import logging
 import re
 import unicodedata
@@ -32,19 +33,21 @@ logger = logging.getLogger(__name__)
 class Subtitulos(ServiceBase):
     server_url = 'http://www.subtitulos.es'
     api_based = False
-    languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
-                 u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
-                 u'Italian': 'it', u'Català': 'ca'}
-    reverted_languages = True
+    languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
+    language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
+                    u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
+                    u'English (UK)': Language('eng-GB'), 'Galego': Language('glg')}
+    language_code = 'name'
     videos = [Episode]
     require_video = False
-    release_pattern = re.compile('Versi&oacute;n (.+) ([0-9]+).([0-9])+ megabytes')
+    required_features = ['permissive']
+    # the '.+' in the pattern for Version allows us to match both '&oacute;'
+    # and the 'ó' char directly. This is because now BS4 converts the html
+    # code chars into their equivalent unicode char
+    release_pattern = re.compile('Versi.+n (.+) ([0-9]+).([0-9])+ megabytes')
 
-    def list(self, video, languages):
-        if not self.check_validity(video, languages):
-            return []
-        results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
-        return results
+    def list_checked(self, video, languages):
+        return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
 
     def query(self, filepath, languages, keywords, series, season, episode):
         request_series = series.lower().replace(' ', '_')
@@ -58,7 +61,7 @@ class Subtitulos(ServiceBase):
         if r.status_code != 200:
             logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
             return []
-        soup = BeautifulSoup.BeautifulSoup(r.content)
+        soup = BeautifulSoup(r.content, self.required_features)
         subtitles = []
         for sub in soup('div', {'id': 'version'}):
             sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
@@ -66,8 +69,8 @@ class Subtitulos(ServiceBase):
                 logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
                 continue
             for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
-                language = self.get_revert_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
-                if not language in languages:
+                language = self.get_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
+                if language not in languages:
                     logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
                     continue
                 html_status = html_language.findNext('li', {'class': 'li-estado green'})
@@ -76,8 +79,10 @@ class Subtitulos(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'],
+                                          keywords=sub_keywords)
                 subtitles.append(subtitle)
         return subtitles
 
+
 Service = Subtitulos

subliminal/services/thesubdb.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
+from ..language import language_set
 from ..subtitles import get_subtitle_path, ResultSubtitle
 from ..videos import Episode, Movie, UnknownVideo
 import logging
@@ -25,22 +26,18 @@ logger = logging.getLogger(__name__)
 
 
 class TheSubDB(ServiceBase):
-    server_url = 'http://api.thesubdb.com/'  # for testing purpose, use http://sandbox.thesubdb.com/ instead
-    user_agent = 'SubDB/1.0 (subliminal/0.5; https://github.com/Diaoul/subliminal)'  # defined by the API
+    server_url = 'http://api.thesubdb.com'
+    user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)'
     api_based = True
-    languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
-                 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
-                 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
-                 'sv': 'sv', 'tr': 'tr'}  # list available with the API at http://sandbox.thesubdb.com/?action=languages
-    reverted_languages = False
+    # Source: http://api.thesubdb.com/?action=languages
+    languages = language_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'id', 'it',
+                              'la', 'nl', 'no', 'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr', 'sv',
+                              'tr'])
     videos = [Movie, Episode, UnknownVideo]
     require_video = True
 
-    def list(self, video, languages):
-        if not self.check_validity(video, languages):
-            return []
-        results = self.query(video.path, video.hashes['TheSubDB'], languages)
-        return results
+    def list_checked(self, video, languages):
+        return self.query(video.path, video.hashes['TheSubDB'], languages)
 
     def query(self, filepath, moviehash, languages):
         r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
@@ -50,7 +47,7 @@ class TheSubDB(ServiceBase):
         if r.status_code != 200:
             logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
             return []
-        available_languages = set([self.get_revert_language(l) for l in r.content.split(',')])
+        available_languages = language_set(r.content.split(','))
         languages &= available_languages
         if not languages:
             logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
@@ -58,8 +55,9 @@ class TheSubDB(ServiceBase):
         subtitles = []
         for language in languages:
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.get_language(language)))
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, language.alpha2))
             subtitles.append(subtitle)
         return subtitles
 
+
 Service = TheSubDB

subliminal/services/tvsubtitles.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Nicolas Wack <wackou@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from . import ServiceBase
+from ..cache import cachedmethod
+from ..language import language_set, Language
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords
+from ..videos import Episode
+from bs4 import BeautifulSoup
+import logging
+import re
+
+
+logger = logging.getLogger(__name__)
+
+
+def match(pattern, string):
+    try:
+        return re.search(pattern, string).group(1)
+    except AttributeError:
+        logger.debug(u'Could not match %r on %r' % (pattern, string))
+        return None
+
+
+class TvSubtitles(ServiceBase):
+    server_url = 'http://www.tvsubtitles.net'
+    api_based = False
+    languages = language_set(['ar', 'bg', 'cs', 'da', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hu',
+                              'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
+                              'zh', 'pt-br'])
+    #TODO: Find more exceptions
+    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr'),
+                    'cn': Language('chi')}
+    videos = [Episode]
+    require_video = False
+    required_features = ['permissive']
+
+    @cachedmethod
+    def get_likely_series_id(self, name):
+        r = self.session.post('%s/search.php' % self.server_url, data={'q': name})
+        soup = BeautifulSoup(r.content, self.required_features)
+        maindiv = soup.find('div', 'left')
+        results = []
+        for elem in maindiv.find_all('li'):
+            sid = int(match('tvshow-([0-9]+)\.html', elem.a['href']))
+            show_name = match('(.*) \(', elem.a.text)
+            results.append((show_name, sid))
+        #TODO: pick up the best one in a smart way
+        result = results[0]
+        return result[1]
+
+    @cachedmethod
+    def get_episode_id(self, series_id, season, number):
+        """Get the TvSubtitles id for the given episode. Raises KeyError if none
+        could be found."""
+        # download the page of the season, contains ids for all episodes
+        episode_id = None
+        r = self.session.get('%s/tvshow-%d-%d.html' % (self.server_url, series_id, season))
+        soup = BeautifulSoup(r.content, self.required_features)
+        table = soup.find('table', id='table5')
+        for row in table.find_all('tr'):
+            cells = row.find_all('td')
+            if not cells:
+                continue
+            episode_number = match('x([0-9]+)', cells[0].text)
+            if not episode_number:
+                continue
+            episode_number = int(episode_number)
+            episode_id = int(match('episode-([0-9]+)', cells[1].a['href']))
+            # we could just return the id of the queried episode, but as we
+            # already downloaded the whole page we might as well fill in the
+            # information for all the episodes of the season
+            self.cache_for(self.get_episode_id, args=(series_id, season, episode_number), result=episode_id)
+        # raises KeyError if not found
+        return self.cached_value(self.get_episode_id, args=(series_id, season, number))
+
+    # Do not cache this method in order to always check for the most recent
+    # subtitles
+    def get_sub_ids(self, episode_id):
+        subids = []
+        r = self.session.get('%s/episode-%d.html' % (self.server_url, episode_id))
+        epsoup = BeautifulSoup(r.content, self.required_features)
+        for subdiv in epsoup.find_all('a'):
+            if 'href' not in subdiv.attrs or not subdiv['href'].startswith('/subtitle'):
+                continue
+            subid = int(match('([0-9]+)', subdiv['href']))
+            lang = self.get_language(match('flags/(.*).gif', subdiv.img['src']))
+            result = {'subid': subid, 'language': lang}
+            for p in subdiv.find_all('p'):
+                if 'alt' in p.attrs and p['alt'] == 'rip':
+                    result['rip'] = p.text.strip()
+                if 'alt' in p.attrs and p['alt'] == 'release':
+                    result['release'] = p.text.strip()
+            subids.append(result)
+        return subids
+
+    def list_checked(self, video, languages):
+        return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
+
+    def query(self, filepath, languages, keywords, series, season, episode):
+        logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+        self.init_cache()
+        sid = self.get_likely_series_id(series.lower())
+        try:
+            ep_id = self.get_episode_id(sid, season, episode)
+        except KeyError:
+            logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
+            return []
+        subids = self.get_sub_ids(ep_id)
+        # filter the subtitles with our queried languages
+        subtitles = []
+        for subid in subids:
+            language = subid['language']
+            if language not in languages:
+                continue
+            path = get_subtitle_path(filepath, language, self.config.multi)
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/download-%d.html' % (self.server_url, subid['subid']),
+                                      keywords=[subid['rip'], subid['release']])
+            subtitles.append(subtitle)
+        return subtitles
+
+    def download(self, subtitle):
+        self.download_zip_file(subtitle.link, subtitle.path)
+        return subtitle
+
+
+Service = TvSubtitles

subliminal/subtitles.py

@@ -15,13 +15,13 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from .languages import list_languages, convert_language
+from .language import Language
+from .utils import to_unicode
 import os.path
 
 
 __all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
 
-
 #: Subtitles extensions
 EXTENSIONS = ['.srt', '.sub', '.txt']
 
@@ -30,10 +30,13 @@ class Subtitle(object):
     """Base class for subtitles
 
     :param string path: path to the subtitle
-    :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+    :param language: language of the subtitle
+    :type language: :class:`~subliminal.language.Language`
 
     """
     def __init__(self, path, language):
+        if not isinstance(language, Language):
+            raise TypeError('%r is not an instance of Language')
         self.path = path
         self.language = language
 
@@ -44,12 +47,22 @@ class Subtitle(object):
             return os.path.exists(self.path)
         return False
 
+    def __unicode__(self):
+        return to_unicode(self.path)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __repr__(self):
+        return '%s(%s, %s)' % (self.__class__.__name__, self, self.language)
+
 
 class EmbeddedSubtitle(Subtitle):
     """Subtitle embedded in a container
 
     :param string path: path to the subtitle
-    :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+    :param language: language of the subtitle
+    :type language: :class:`~subliminal.language.Language`
     :param int track_id: id of the subtitle track in the container
 
     """
@@ -59,7 +72,7 @@ class EmbeddedSubtitle(Subtitle):
 
     @classmethod
     def from_enzyme(cls, path, subtitle):
-        language = convert_language(subtitle.language, 1, 2)
+        language = Language(subtitle.language, strict=False)
         return cls(path, language, subtitle.trackno)
 
 
@@ -68,16 +81,14 @@ class ExternalSubtitle(Subtitle):
     @classmethod
     def from_path(cls, path):
         """Create an :class:`ExternalSubtitle` from path"""
-        extension = ''
+        extension = None
         for e in EXTENSIONS:
             if path.endswith(e):
                 extension = e
                 break
-        if not extension:
+        if extension is None:
             raise ValueError('Not a supported subtitle extension')
-        language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
-        if not language in list_languages(1):
-            language = None
+        language = Language(os.path.splitext(path[:len(path) - len(extension)])[1][1:], strict=False)
         return cls(path, language)
 
 
@@ -85,7 +96,8 @@ class ResultSubtitle(ExternalSubtitle):
     """Subtitle found using :mod:`~subliminal.services`
 
     :param string path: path to the subtitle
-    :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+    :param language: language of the subtitle
+    :type language: :class:`~subliminal.language.Language`
     :param string service: name of the service
     :param string link: download link for the subtitle
     :param string release: release name of the video
@@ -93,13 +105,13 @@ class ResultSubtitle(ExternalSubtitle):
     :param set keywords: keywords that describe the subtitle
 
     """
-    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
+    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=None):
         super(ResultSubtitle, self).__init__(path, language)
         self.service = service
         self.link = link
         self.release = release
         self.confidence = confidence
-        self.keywords = keywords
+        self.keywords = keywords or set()
 
     @property
     def single(self):
@@ -109,22 +121,29 @@ class ResultSubtitle(ExternalSubtitle):
         :rtype: bool
 
         """
-        extension = os.path.splitext(self.path)[0]
-        language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
-        if not language in list_languages(1):
-            return True
-        return False
+        return self.language == Language('Undetermined')
 
     def __repr__(self):
-        return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
+        if not self.release:
+            return 'ResultSubtitle(%s, %s, %s, %.2f)' % (self.path, self.language, self.service, self.confidence)
+        return 'ResultSubtitle(%s, %s, %s, %.2f, release=%s)' % (self.path, self.language, self.service, self.confidence, self.release.encode('ascii', 'ignore'))
 
 
 def get_subtitle_path(video_path, language, multi):
-    """Create the subtitle path from the given video path using language if multi"""
+    """Create the subtitle path from the given video path using language if multi
+
+    :param string video_path: path to the video
+    :param language: language of the subtitle
+    :type language: :class:`~subliminal.language.Language`
+    :param bool multi: whether to use multi language naming or not
+    :return: path of the subtitle
+    :rtype: string
+
+    """
     if not os.path.exists(video_path):
         path = os.path.splitext(os.path.basename(video_path))[0]
     else:
         path = os.path.splitext(video_path)[0]
     if multi and language:
-        return path + '.%s%s' % (language, EXTENSIONS[0])
+        return path + '.%s%s' % (language.alpha2, EXTENSIONS[0])
     return path + '%s' % EXTENSIONS[0]

subliminal/tasks.py

@@ -35,6 +35,7 @@ class ListTask(Task):
 
     """
     def __init__(self, video, languages, service, config):
+        super(ListTask, self).__init__()
         self.video = video
         self.service = service
         self.languages = languages
@@ -54,6 +55,7 @@ class DownloadTask(Task):
 
     """
     def __init__(self, video, subtitles):
+        super(DownloadTask, self).__init__()
         self.video = video
         self.subtitles = subtitles
 

subliminal/utils.py

@@ -61,4 +61,9 @@ def to_unicode(data):
         raise ValueError('Basestring expected')
     if isinstance(data, unicode):
         return data
+    for encoding in ('utf-8', 'latin-1'):
+        try:
+            return unicode(data, encoding)
+        except UnicodeDecodeError:
+            pass
     return unicode(data, 'utf-8', 'replace')

subliminal/videos.py

@@ -16,7 +16,8 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import subtitles
-from .languages import list_languages
+from .language import Language
+from .utils import to_unicode
 import enzyme
 import guessit
 import hashlib
@@ -126,22 +127,33 @@ class Video(object):
         try:
             video_infos = enzyme.parse(self.path)
             logger.debug(u'Succeeded parsing %s with enzyme: %r' % (self.path, video_infos))
-        except enzyme.ParseError:
+        except:
             logger.debug(u'Failed parsing %s with enzyme' % self.path)
         if isinstance(video_infos, enzyme.core.AVContainer):
             results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
-        for l in list_languages(1):
-            for e in subtitles.EXTENSIONS:
-                single_path = basepath + '%s' % e
-                if os.path.exists(single_path):
-                    results.append(subtitles.ExternalSubtitle(single_path, None))
-                multi_path = basepath + '.%s%s' % (l, e)
-                if os.path.exists(multi_path):
-                    results.append(subtitles.ExternalSubtitle(multi_path, l))
+        # cannot use glob here because it chokes if there are any square
+        # brackets inside the filename, so we have to use basic string
+        # startswith/endswith comparisons
+        folder, basename = os.path.split(basepath)
+        existing = [f for f in os.listdir(folder) if f.startswith(basename)]
+        for path in existing:
+            for ext in subtitles.EXTENSIONS:
+                if path.endswith(ext):
+                    language = Language(path[len(basename) + 1:-len(ext)], strict=False)
+                    results.append(subtitles.ExternalSubtitle(path, language))
         return results
 
+    def __unicode__(self):
+        return to_unicode(self.path or self.release)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
     def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.release)
+        return '%s(%s)' % (self.__class__.__name__, self)
+
+    def __hash__(self):
+        return hash(self.path or self.release)
 
 
 class Episode(Video):
@@ -189,11 +201,12 @@ class UnknownVideo(Video):
     pass
 
 
-def scan(entry, max_depth=3, depth=0):
+def scan(entry, max_depth=3, scan_filter=None, depth=0):
     """Scan a path for videos and subtitles
 
     :param string entry: path
     :param int max_depth: maximum folder depth
+    :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
     :param int depth: starting depth
     :return: found videos and subtitles
     :rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`])
@@ -201,19 +214,19 @@ def scan(entry, max_depth=3, depth=0):
     """
     if depth > max_depth and max_depth != 0:  # we do not want to search the whole file system except if max_depth = 0
         return []
-    if depth == 0:
-        entry = os.path.abspath(entry)
     if os.path.isdir(entry):  # a dir? recurse
         logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
         result = []
         for e in os.listdir(entry):
-            result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
+            result.extend(scan(os.path.join(entry, e), max_depth, scan_filter, depth + 1))
         return result
     if os.path.isfile(entry) or depth == 0:
         logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth))
         if depth != 0:  # trust the user: only check for valid format if recursing
             if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
                 return []
+            if scan_filter is not None and scan_filter(entry):
+                return []
         video = Video.from_path(entry)
         return [(video, video.scan())]
     logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth))
@@ -260,6 +273,8 @@ def hash_thesubdb(path):
 
     """
     readsize = 64 * 1024
+    if os.path.getsize(path) < readsize:
+        return None
     with open(path, 'rb') as f:
         data = f.read(readsize)
         f.seek(-readsize, os.SEEK_END)

tests/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from . import test_language, test_services, test_subliminal, test_videos
+import unittest
+
+
+suite = unittest.TestSuite([test_language.suite(), test_services.suite(), test_subliminal.suite(), test_videos.suite()])
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite)

tests/test_async.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from subliminal import Pool
-import os
-import time
-import unittest
-
-
-cache_dir = u'/tmp/sublicache'
-if not os.path.exists(cache_dir):
-    os.mkdir(cache_dir)
-existing_video = u'/something/The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
-
-
-class AsyncTestCase(unittest.TestCase):
-    def test_pool(self):
-        p = Pool(4)
-        self.assertTrue(len(p.workers) == 4)
-        for w in p.workers:
-            self.assertTrue(w.isAlive() == False)
-        p.start()
-        for w in p.workers:
-            self.assertTrue(w.isAlive() == True)
-        p.stop()
-        p.join()
-        time.sleep(0.2)  # so terminate is finished on Worker and proper Thread methods finished
-        for w in p.workers:
-            self.assertTrue(w.isAlive() == False)
-
-    def test_list_subtitles(self):
-        with Pool(4) as p:
-            results = p.list_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
-        self.assertTrue(len(results) > 0)
-
-    def test_download_subtitles(self):
-        with Pool(4) as p:
-            results = p.download_subtitles(existing_video, languages=['en', 'fr'], cache_dir=cache_dir, max_depth=3)
-        self.assertTrue(len(results) > 0)
-
-
-if __name__ == '__main__':
-    unittest.main()

tests/test_language.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from subliminal.language import Language, Country, language_set, language_list
+import unittest
+
+
+class LanguageListTestCase(unittest.TestCase):
+    def test_list_contains(self):
+        languages = list([Language('fr'), Language('en-US'), Language('en-GB')])
+        self.assertTrue(Language('fr') in languages)
+        self.assertTrue(Language('en-US') in languages)
+        self.assertTrue(Language('en') not in languages)
+        self.assertTrue(Language('fr-BE') not in languages)
+
+    def test_language_list_contains(self):
+        languages = language_list(['fr', 'en-US', 'en-GB'])
+        self.assertTrue(Language('fr') in languages)
+        self.assertTrue(Language('en-US') in languages)
+        self.assertTrue(Language('en') not in languages)
+        self.assertTrue(Language('fr-BE') in languages)
+
+    def test_list_index(self):
+        languages = [Language('fr'), Language('en-US'), Language('en-GB')]
+        self.assertTrue(languages.index(Language('fr')) == 0)
+        self.assertTrue(languages.index(Language('en-US')) == 1)
+        self.assertTrue(languages.index(Language('en-GB')) == 2)
+        with self.assertRaises(ValueError):
+            languages.index(Language('fr-BE'))
+
+    def test_language_list_index(self):
+        languages = language_list(['fr', 'en-US', 'en-GB'])
+        self.assertTrue(languages.index(Language('fr')) == 0)
+        self.assertTrue(languages.index(Language('en-US')) == 1)
+        self.assertTrue(languages.index(Language('en-GB')) == 2)
+        self.assertTrue(languages.index(Language('fr-BE')) == 0)
+
+
+class LanguageSetTestCase(unittest.TestCase):
+    def test_set_contains(self):
+        languages = set([Language('fr'), Language('en-US'), Language('en-GB')])
+        self.assertTrue(Language('fr') in languages)
+        self.assertTrue(Language('en-US') in languages)
+        self.assertTrue(Language('en') not in languages)
+        self.assertTrue(Language('fr-BE') not in languages)
+
+    def test_language_set_contains(self):
+        languages = language_set(['fr', 'en-US', 'en-GB'])
+        self.assertTrue(Language('fr') in languages)
+        self.assertTrue(Language('en-US') in languages)
+        self.assertTrue(Language('en') not in languages)
+        self.assertTrue(Language('fr-BE') in languages)
+
+    def test_language_set_intersect(self):
+        languages = language_set(['fr', 'en-US', 'en-GB'])
+        self.assertTrue(len(languages & language_set([Language('en')])) == 2)
+        self.assertTrue(len(language_set([Language('en')]) & languages) == 2)
+        self.assertTrue(len(languages & language_set([Language('fr')])) == 1)
+
+    def test_language_set_substract(self):
+        languages = language_set(['fr', 'en-US', 'en-GB'])
+        self.assertTrue(len(languages - language_set(['en'])) == 1)
+        self.assertTrue(len(languages - language_set(['en-US'])) == 2)
+        self.assertTrue(len(languages - language_set(['en-US', 'fr'])) == 1)
+
+
+class LanguageTestCase(unittest.TestCase):
+    def test_attrs(self):
+        language = Language('French')
+        self.assertTrue(language.alpha2 == 'fr')
+        self.assertTrue(language.alpha3 == 'fre')
+        self.assertTrue(language.terminologic == 'fra')
+        self.assertTrue(language.name == 'French')
+        self.assertTrue(language.french_name == u'français')
+
+    def test_eq(self):
+        language = Language('French')
+        self.assertTrue(language == Language('fr'))
+        self.assertTrue(language == Language('fre'))
+        self.assertTrue(language == Language('fra'))
+        self.assertTrue(language == Language('Français'))
+
+    def test_ne(self):
+        self.assertTrue(Language('French') != Language('en'))
+
+    def test_in(self):
+        self.assertTrue(Language('Portuguese (BR)') in Language('Portuguese - Brazil'))
+        self.assertTrue(Language('Portuguese (BR)') in Language('Portuguese'))
+        self.assertTrue(Language('Portuguese') not in Language('Portuguese (BR)'))
+
+    def test_with_country(self):
+        self.assertTrue(Language('Portuguese (BR)').country == Country('Brazil'))
+        self.assertTrue(Language('pt_BR').country == Country('Brazil'))
+        self.assertTrue(Language('fr - France').country == Country('France'))
+        self.assertTrue(Language('fra', country='FR').country == Country('France'))
+        self.assertTrue(Language('fra', country=Country('FRA')).country == Country('France'))
+
+    def test_eq_with_country(self):
+        self.assertTrue(Language('Portuguese (BR)') == Language('Portuguese - Brazil'))
+        self.assertTrue(Language('English') == Language('en'))
+
+    def test_ne_with_country(self):
+        self.assertTrue(Language('Portuguese') != Language('Portuguese (BR)'))
+        self.assertTrue(Language('English (US)') != Language('English (GB)'))
+
+    def test_hash(self):
+        self.assertTrue(hash(Language('French')) == hash('fre'))
+
+    def test_missing(self):
+        with self.assertRaises(ValueError):
+            Language('zzz')
+
+
+class CountryTestCase(unittest.TestCase):
+    def test_attrs(self):
+        country = Country('France')
+        self.assertTrue(country.alpha2 == 'FR')
+        self.assertTrue(country.alpha3 == 'FRA')
+        self.assertTrue(country.name == 'France')
+
+    def test_eq(self):
+        country = Country('France')
+        self.assertTrue(country == Country('FR'))
+        self.assertTrue(country == Country('FRA'))
+        self.assertTrue(country == Country('250'))
+
+    def test_ne(self):
+        self.assertTrue(Country('France') != Country('GB'))
+
+    def test_hash(self):
+        self.assertTrue(hash(Country('France')) == hash('FRA'))
+
+    def test_missing(self):
+        with self.assertRaises(ValueError):
+            Country('ZZ')
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(CountryTestCase))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(LanguageTestCase))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(LanguageSetTestCase))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(LanguageListTestCase))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite())

tests/test_services.py

@@ -17,16 +17,23 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from subliminal import videos
-from subliminal.exceptions import MissingLanguageError, ServiceError
+from subliminal.exceptions import ServiceError
+from subliminal.language import language_set, LANGUAGES
 from subliminal.services import ServiceConfig
+from subliminal.services.addic7ed import Addic7ed
 from subliminal.services.bierdopje import BierDopje
 from subliminal.services.opensubtitles import OpenSubtitles
+from subliminal.services.podnapisi import Podnapisi
 from subliminal.services.subswiki import SubsWiki
 from subliminal.services.subtitulos import Subtitulos
 from subliminal.services.thesubdb import TheSubDB
-from subliminal.subtitles import Subtitle
+from subliminal.services.tvsubtitles import TvSubtitles
 import os
 import unittest
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
 
 
 cache_dir = u'/tmp/sublicache'
@@ -35,20 +42,144 @@ if not os.path.exists(cache_dir):
 existing_video = u'/something/The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
 
 
-class BierDopjeTestCase(unittest.TestCase):
-    query_tests = ['test_query_series', 'test_query_wrong_series', 'test_query_wrong_languages',
-                   'test_query_tvdbid', 'test_query_wrong_tvdbid', 'test_query_series_and_tvdbid']
-    list_tests = ['test_list_episode', 'test_list_movie', 'test_list_wrong_languages']
-    download_tests = ['test_download']
+class ServiceTestCase(unittest.TestCase):
+    def setUp(self):
+        self.wrong_languages = language_set(list(language_set(LANGUAGES) - self.service.languages)[:2])
+
+    def tearDown(self):
+        # Setting config to None allows to delete the object, which will in turn save the cache
+        self.config = None
+
+    def test_query_series(self):
+        with self.service(self.config) as service:
+            results = service.query(service, self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) > 0)
+
+    def test_query_wrong_series(self):
+        with self.service(self.config) as service:
+            results = service.query(service, self.fake_file, self.languages, self.episode_keywords, self.wrong_series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+    def test_query_wrong_languages(self):
+        with self.service(self.config) as service:
+            results = service.query(service, self.fake_file, self.wrong_languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+    def test_list_episode(self):
+        video = videos.Video.from_path(self.episode_path)
+        with self.service(self.config) as service:
+            results = service.list(video, self.languages)
+        self.assertTrue(len(results) > 0)
+
+    def test_list_movie(self):
+        video = videos.Video.from_path(self.movie_path)
+        with self.service(self.config) as service:
+            results = service.list(video, self.languages)
+        self.assertTrue(len(results) > 0)
+
+    def test_list_wrong_languages(self):
+        video = videos.Video.from_path(self.episode_path)
+        with self.service(self.config) as service:
+            results = service.list(video, self.wrong_languages)
+        self.assertTrue(len(results) == 0)
+
+    def test_download_episode(self):
+        video = videos.Video.from_path(self.episode_path)
+        with self.service(self.config) as service:
+            subtitle = service.list(video, language_set([self.episode_sublanguage]))[0]
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            service.download(subtitle)
+        self.assertTrue(os.path.exists(subtitle.path))
+        self.assertTrue(os.path.getsize(subtitle.path) in self.episode_subfilesizes, msg='Size %d not in %r' % (os.path.getsize(subtitle.path), self.episode_subfilesizes))
+        os.remove(subtitle.path)
+
+    def test_download_movie(self):
+        video = videos.Video.from_path(self.movie_path)
+        with self.service(self.config) as service:
+            subtitle = service.list(video, language_set([self.movie_sublanguage]))[0]
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            service.download(subtitle)
+        self.assertTrue(os.path.exists(subtitle.path))
+        self.assertTrue(os.path.getsize(subtitle.path) in self.movie_subfilesizes, msg='Size %d not in %r' % (os.path.getsize(subtitle.path), self.movie_subfilesizes))
+        os.remove(subtitle.path)
+
+    def test_cached_series(self):
+        with self.service(self.config) as service:
+            service.clear_cache()
+            service.query(self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
+            service.save_cache()
+        c = pickle.load(open(os.path.join(cache_dir, 'subliminal_%s.cache' % self.service.__name__)))
+        found = False
+        for _, cached_values in c.items():
+            for args, __ in cached_values.items():
+                if args == (self.series.lower(),):
+                    found = True
+        self.assertTrue(found)
+
+
+class Addic7edTestCase(ServiceTestCase):
+    query_tests = ['test_query_series', 'test_query_wrong_series', 'test_query_wrong_languages']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = ['test_cached_series']
+    service = Addic7ed
 
     def setUp(self):
+        super(Addic7edTestCase, self).setUp()
+        self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
+        self.fake_file = u'/tmp/fake_file'
+        self.languages = language_set(['en', 'fr'])
+        self.episode_path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
+        self.episode_sublanguage = 'en'
+        # FIXME: this is the size of the first subtitle that appears on the page
+        # which is the original one, not the most updated one. We should make
+        # sure the Addic7ed service picks up the most recent one instead
+        self.episode_subfilesizes = [33538, 33643]
+        self.episode_keywords = set(['asap', 'hdtv'])
+        self.series = 'The Big Bang Theory'
+        self.wrong_series = 'No Existent Show Name'
+        self.season = 5
+        self.episode = 6
+
+    def test_query_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) > 0)
+
+    def test_query_wrong_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.wrong_series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+    def test_query_wrong_languages(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+
+class BierDopjeTestCase(ServiceTestCase):
+    query_tests = ['test_query_series', 'test_query_wrong_series', 'test_query_wrong_languages',
+                   'test_query_tvdbid', 'test_query_wrong_tvdbid', 'test_query_series_and_tvdbid']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = ['test_cached_series']
+    service = BierDopje
+
+    def setUp(self):
+        super(BierDopjeTestCase, self).setUp()
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
         self.episode_path = u'The Big Bang Theory/Season 05/S05E06 - The Rhinitis Revelation - HD TV.mkv'
+        self.episode_sublanguage = 'en'
+        self.episode_subfilesizes = [33469]
         self.movie_path = u'Inception (2010)/Inception - 1080p.mkv'
-        self.languages = set(['en', 'nl'])
-        self.wrong_languages = set(['zz', 'es'])
+        self.movie_sublanguage = 'en'
+        self.movie_subfilesizes = []
+        self.languages = language_set(['en', 'nl'])
         self.fake_file = u'/tmp/fake_file'
         self.series = 'The Big Bang Theory'
+        self.episode_keywords = set()
         self.wrong_series = 'No Existent Show Name'
         self.season = 5
         self.episode = 6
@@ -56,202 +187,159 @@ class BierDopjeTestCase(unittest.TestCase):
         self.wrong_tvdbid = 9999999999
 
     def test_query_series(self):
-        with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.languages, self.fake_file, series=self.series)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.season, self.episode, self.languages, series=self.series)
         self.assertTrue(len(results) > 0)
 
     def test_query_wrong_series(self):
-        with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.languages, self.fake_file, series=self.wrong_series)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.season, self.episode, self.languages, series=self.wrong_series)
         self.assertTrue(len(results) == 0)
 
     def test_query_wrong_languages(self):
-        with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.wrong_languages, self.fake_file, series=self.series)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.season, self.episode, self.wrong_languages, series=self.series)
         self.assertTrue(len(results) == 0)
 
     def test_query_tvdbid(self):
         with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.languages, self.fake_file, tvdbid=self.tvdbid)
+            results = service.query(self.fake_file, self.season, self.episode, self.languages, tvdbid=self.tvdbid)
         self.assertTrue(len(results) > 0)
 
     def test_query_series_and_tvdbid(self):
         with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.languages, self.fake_file, series=self.series, tvdbid=self.tvdbid)
+            results = service.query(self.fake_file, self.season, self.episode, self.languages, series=self.series, tvdbid=self.tvdbid)
         self.assertTrue(len(results) > 0)
 
     def test_query_wrong_tvdbid(self):
         with BierDopje(self.config) as service:
-            results = service.query(self.season, self.episode, self.languages, self.fake_file, tvdbid=self.wrong_tvdbid)
+            results = service.query(self.fake_file, self.season, self.episode, self.languages, tvdbid=self.wrong_tvdbid)
         self.assertTrue(len(results) == 0)
 
-    def test_list_episode(self):
-        episode = videos.Video.from_path(self.episode_path)
-        with BierDopje(self.config) as service:
-            results = service.list(episode, self.languages)
-        self.assertTrue(len(results) > 0)
-
-    def test_list_movie(self):
-        movie = videos.Video.from_path(self.movie_path)
-        with BierDopje(self.config) as service:
-            results = service.list(movie, self.languages)
-        self.assertTrue(len(results) == 0)
-
-    def test_list_wrong_languages(self):
-        episode = videos.Video.from_path(self.episode_path)
-        with BierDopje(self.config) as service:
-            results = service.list(episode, self.wrong_languages)
-        self.assertTrue(len(results) == 0)
-
-    def test_download(self):
-        episode = videos.Video.from_path(self.episode_path)
-        with BierDopje(self.config) as service:
-            subtitle = service.list(episode, self.languages)[0]
-            if os.path.exists(subtitle.path):
-                os.remove(subtitle.path)
-            service.download(subtitle)
-        self.assertTrue(os.path.exists(subtitle.path))
+    def test_cached_series(self):
+        with self.service(self.config) as service:
+            service.clear_cache()
+            service.query(self.fake_file, self.season, self.episode, self.languages, series=self.series)
+            service.save_cache()
+        c = pickle.load(open(os.path.join(cache_dir, 'subliminal_%s.cache' % self.service.__name__)))
+        found = False
+        for _, cached_values in c.items():
+            for args, __ in cached_values.items():
+                if args == (self.series.lower(),):
+                    found = True
+        self.assertTrue(found)
 
 
-class OpenSubtitlesTestCase(unittest.TestCase):
+class OpenSubtitlesTestCase(ServiceTestCase):
     query_tests = ['test_query_query', 'test_query_imdbid', 'test_query_hash', 'test_query_wrong_languages']
-    list_tests = ['test_list', 'test_list_wrong_languages']
-    download_tests = ['test_download']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = []
+    service = OpenSubtitles
 
     def setUp(self):
+        super(OpenSubtitlesTestCase, self).setUp()
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
-        self.languages = set(['en', 'fr'])
-        self.wrong_languages = set(['zz', 'yy'])
+        self.languages = language_set(['en', 'fr'])
         self.fake_file = u'/tmp/fake_file'
-        self.path = existing_video
-        self.series = 'The Big Bang Theory'
+        self.episode_path = existing_video
+        self.episode_sublanguage = 'en'
+        self.episode_subfilesizes = [30374, 30358, 33585, 33547, 33563, 33601]
         self.movie = 'Inception'
-        self.wrong_series = 'No Existent Show Name'
-        self.imdbid = 'tt1375666'
-        self.wrong_imdbid = 'tt9999999'
+        self.imdbid = '1375666'
+        self.wrong_imdbid = '9999999'
         self.hash = '51e57c4e8fd77990'
         self.size = 882571264L
 
     def test_query_query(self):
-        with OpenSubtitles(self.config) as service:
+        with self.service(self.config) as service:
             results = service.query(self.fake_file, self.languages, query=self.movie)
         self.assertTrue(len(results) > 0)
 
     def test_query_imdbid(self):
-        with OpenSubtitles(self.config) as service:
+        with self.service(self.config) as service:
             results = service.query(self.fake_file, self.languages, imdbid=self.imdbid)
         self.assertTrue(len(results) > 0)
 
     def test_query_hash(self):
-        with OpenSubtitles(self.config) as service:
+        with self.service(self.config) as service:
             results = service.query(self.fake_file, self.languages, moviehash=self.hash, size=self.size)
         self.assertTrue(len(results) > 0)
 
     def test_query_wrong_languages(self):
-        with OpenSubtitles(self.config) as service:
-            with self.assertRaises(MissingLanguageError):
-                service.query(self.fake_file, self.wrong_languages, moviehash=self.hash, size=self.size)
-
-    def test_list(self):
-        video = videos.Video.from_path(self.path)
-        with OpenSubtitles(self.config) as service:
-            results = service.list(video, self.languages)
-        self.assertTrue(len(results) > 0)
-
-    def test_list_wrong_languages(self):
-        video = videos.Video.from_path(self.path)
-        with OpenSubtitles(self.config) as service:
-            results = service.list(video, self.wrong_languages)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, moviehash=self.hash, size=self.size)
         self.assertTrue(len(results) == 0)
 
-    def test_download(self):
-        video = videos.Video.from_path(self.path)
-        with OpenSubtitles(self.config) as service:
-            subtitle = service.list(video, self.languages)[0]
-            if os.path.exists(subtitle.path):
-                os.remove(subtitle.path)
-            result = service.download(subtitle)
-        self.assertTrue(isinstance(result, Subtitle))
-        self.assertTrue(os.path.exists(subtitle.path))
 
-
-class TheSubDBTestCase(unittest.TestCase):
-    query_tests = ['test_query', 'test_query_wrong_hash', 'test_query_wrong_languages']
-    list_tests = ['test_list', 'test_list_wrong_languages']
-    download_tests = ['test_download']
+class PodnapisiTestCase(ServiceTestCase):
+    query_tests = ['test_query', 'test_query_wrong_languages']
+    list_tests = [] #'test_list', 'test_list_wrong_languages'
+    download_tests = [] #'test_download'
+    cache_tests = []
+    service = Podnapisi
 
     def setUp(self):
+        super(PodnapisiTestCase, self).setUp()
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
-        self.path = existing_video
-        self.hash = u'edc1981d6459c6111fe36205b4aff6c2'
-        self.wrong_hash = u'ffffffffffffffffffffffffffffffff'
-        self.languages = set(['en', 'nl'])
-        self.wrong_languages = set(['zz', 'cs'])
+        self.languages = language_set(['en', 'fr'])
         self.fake_file = u'/tmp/fake_file'
+        self.path = existing_video
+        self.hash = 'e1b45885346cfa0b'
 
     def test_query(self):
-        with TheSubDB(self.config) as service:
-            results = service.query(self.fake_file, self.hash, self.languages)
+        with Podnapisi(self.config) as service:
+            results = service.query(self.fake_file, self.languages, moviehash=self.hash)
         self.assertTrue(len(results) > 0)
 
-    def test_query_wrong_hash(self):
-        with TheSubDB(self.config) as service:
-            results = service.query(self.fake_file, self.wrong_hash, self.languages)
-        self.assertTrue(len(results) == 0)
-
     def test_query_wrong_languages(self):
-        with TheSubDB(self.config) as service:
-            results = service.query(self.fake_file, self.hash, self.wrong_languages)
+        with Podnapisi(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, moviehash=self.hash)
         self.assertTrue(len(results) == 0)
 
-    def test_list(self):
-        video = videos.Video.from_path(self.path)
-        with TheSubDB(self.config) as service:
-            results = service.list(video, self.languages)
-        self.assertTrue(len(results) > 0)
 
-    def test_list_wrong_languages(self):
-        video = videos.Video.from_path(self.path)
-        with TheSubDB(self.config) as service:
-            results = service.list(video, self.wrong_languages)
-        self.assertTrue(len(results) == 0)
-
-    def test_download(self):
-        video = videos.Video.from_path(self.path)
-        with TheSubDB(self.config) as service:
-            subtitle = service.list(video, self.languages)[0]
-            if os.path.exists(subtitle.path):
-                os.remove(subtitle.path)
-            service.download(subtitle)
-        self.assertTrue(os.path.exists(subtitle.path))
-
-
-class SubsWikiTestCase(unittest.TestCase):
+class SubsWikiTestCase(ServiceTestCase):
     query_tests = ['test_query_series', 'test_query_movie', 'test_query_wrong_parameters', 'test_query_wrong_series', 'test_query_wrong_languages']
-    list_tests = ['test_list_series', 'test_list_movie', 'test_list_series_wrong_languages']
-    download_tests = ['test_download']
+    list_tests = ['test_list_episode', 'test_list_movie', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode', 'test_download_movie']
+    cache_tests = []
+    service = SubsWiki
 
     def setUp(self):
+        super(SubsWikiTestCase, self).setUp()
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
         self.fake_file = u'/tmp/fake_file'
-        self.languages = set(['en', 'es'])
-        self.wrong_languages = set(['zz', 'ay'])
+        self.languages = language_set(['en', 'es'])
         self.movie_path = u'Soul Surfer (2011)/Soul.Surfer.(2011).DVDRip.XviD-TWiZTED.mkv'
+        self.movie_sublanguage = 'es'
+        self.movie_subfilesizes = [87528]
         self.movie_keywords = set(['twizted'])
         self.movie = u'Soul Surfer'
         self.movie_year = 2011
-        self.series_path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
-        self.series_keywords = set(['asap', 'hdtv'])
+        self.episode_path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
+        self.episode_sublanguage = 'es'
+        self.episode_subfilesizes = [34040]
+        self.episode_keywords = set(['asap', 'hdtv'])
         self.series = 'The Big Bang Theory'
         self.wrong_series = 'No Existent Show Name'
-        self.series_season = 5
-        self.series_episode = 6
+        self.season = 5
+        self.episode = 6
 
     def test_query_series(self):
-        with SubsWiki(self.config) as service:
-            results = service.query(self.fake_file, self.languages, keywords=self.series_keywords, series=self.series, season=self.series_season, episode=self.series_episode)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
         self.assertTrue(len(results) > 0)
 
+    def test_query_wrong_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.wrong_series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+    def test_query_wrong_languages(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
     def test_query_movie(self):
         with SubsWiki(self.config) as service:
             results = service.query(self.fake_file, self.languages, keywords=self.movie_keywords, movie=self.movie, year=self.movie_year)
@@ -259,134 +347,155 @@ class SubsWikiTestCase(unittest.TestCase):
 
     def test_query_wrong_parameters(self):
         with SubsWiki(self.config) as service:
-            with self.assertRaises(ServiceError):
-                service.query(self.fake_file, self.languages, keywords=self.movie_keywords, movie=self.movie, series=self.series)
-
-    def test_query_wrong_series(self):
-        with SubsWiki(self.config) as service:
-            results = service.query(self.fake_file, self.languages, keywords=self.series_keywords, series=self.wrong_series, season=self.series_season, episode=self.series_episode)
-        self.assertTrue(len(results) == 0)
-
-    def test_query_wrong_languages(self):
-        with SubsWiki(self.config) as service:
-            results = service.query(self.fake_file, self.wrong_languages, keywords=self.series_keywords, series=self.series, season=self.series_season, episode=self.series_episode)
-        self.assertTrue(len(results) == 0)
-
-    def test_list_series(self):
-        video = videos.Video.from_path(self.series_path)
-        with SubsWiki(self.config) as service:
-            results = service.list(video, self.languages)
-        self.assertTrue(len(results) > 0)
-
-    def test_list_movie(self):
-        video = videos.Video.from_path(self.movie_path)
-        with SubsWiki(self.config) as service:
-            results = service.list(video, self.languages)
-        self.assertTrue(len(results) > 0)
-
-    def test_list_series_wrong_languages(self):
-        video = videos.Video.from_path(self.series_path)
-        with SubsWiki(self.config) as service:
-            results = service.list(video, self.wrong_languages)
-        self.assertTrue(len(results) == 0)
-
-    def test_download(self):
-        video = videos.Video.from_path(self.series_path)
-        with SubsWiki(self.config) as service:
-            subtitle = service.list(video, self.languages)[0]
-            if os.path.exists(subtitle.path):
-                os.remove(subtitle.path)
-            service.download(subtitle)
-        self.assertTrue(os.path.exists(subtitle.path))
+            self.assertRaises(ServiceError, service.query,
+                              self.fake_file, self.languages, keywords=self.movie_keywords, movie=self.movie, series=self.series)
 
 
-class SubtitulosTestCase(unittest.TestCase):
-    query_tests = ['test_query', 'test_query_wrong_series', 'test_query_wrong_languages']
-    list_tests = ['test_list', 'test_list_wrong_languages']
-    download_tests = ['test_download']
+class SubtitulosTestCase(ServiceTestCase):
+    query_tests = ['test_query_series', 'test_query_wrong_series', 'test_query_wrong_languages']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = []
+    service = Subtitulos
 
     def setUp(self):
+        super(SubtitulosTestCase, self).setUp()
         self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
         self.fake_file = u'/tmp/fake_file'
-        self.languages = set(['en', 'es'])
-        self.wrong_languages = set(['zz', 'ay'])
-        self.path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
-        self.keywords = set(['asap', 'hdtv'])
+        self.languages = language_set(['en', 'es'])
+        self.episode_path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
+        self.episode_sublanguage = 'en'
+        self.episode_subfilesizes = [32986]
+        self.episode_keywords = set(['asap', 'hdtv'])
         self.series = 'The Big Bang Theory'
         self.wrong_series = 'No Existent Show Name'
         self.season = 5
         self.episode = 6
 
-    def test_query(self):
-        with Subtitulos(self.config) as service:
-            results = service.query(self.fake_file, self.languages, self.keywords, self.series, self.season, self.episode)
+    def test_query_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
         self.assertTrue(len(results) > 0)
 
     def test_query_wrong_series(self):
-        with Subtitulos(self.config) as service:
-            results = service.query(self.fake_file, self.languages, self.keywords, self.wrong_series, self.season, self.episode)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.wrong_series, self.season, self.episode)
         self.assertTrue(len(results) == 0)
 
     def test_query_wrong_languages(self):
-        with Subtitulos(self.config) as service:
-            results = service.query(self.fake_file, self.wrong_languages, self.keywords, self.series, self.season, self.episode)
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, self.episode_keywords, self.series, self.season, self.episode)
         self.assertTrue(len(results) == 0)
 
-    def test_list(self):
-        video = videos.Video.from_path(self.path)
-        with Subtitulos(self.config) as service:
-            results = service.list(video, self.languages)
+
+class TheSubDBTestCase(ServiceTestCase):
+    query_tests = ['test_query', 'test_query_wrong_languages']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = []
+    service = TheSubDB
+
+    def setUp(self):
+        super(TheSubDBTestCase, self).setUp()
+        self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
+        self.episode_path = existing_video
+        self.episode_sublanguage = 'en'
+        self.episode_subfilesizes = [33536]
+        self.hash = u'edc1981d6459c6111fe36205b4aff6c2'
+        self.languages = language_set(['en', 'nl'])
+        self.fake_file = u'/tmp/fake_file'
+
+    def test_query(self):
+        with TheSubDB(self.config) as service:
+            results = service.query(self.fake_file, self.hash, self.languages)
         self.assertTrue(len(results) > 0)
 
-    def test_list_wrong_languages(self):
-        video = videos.Video.from_path(self.path)
-        with Subtitulos(self.config) as service:
-            results = service.list(video, self.wrong_languages)
+    def test_query_wrong_languages(self):
+        with TheSubDB(self.config) as service:
+            results = service.query(self.fake_file, self.hash, self.wrong_languages)
         self.assertTrue(len(results) == 0)
 
-    def test_download(self):
-        video = videos.Video.from_path(self.path)
-        with Subtitulos(self.config) as service:
-            subtitle = service.list(video, self.languages)[0]
-            if os.path.exists(subtitle.path):
-                os.remove(subtitle.path)
-            result = service.download(subtitle)
-        self.assertTrue(os.path.exists(subtitle.path))
+    @unittest.skipUnless(os.path.exists(existing_video), 'No existing video')
+    def test_list_episode(self):
+        super(TheSubDBTestCase, self).test_list_episode()
+
+    @unittest.skipUnless(os.path.exists(existing_video), 'No existing video')
+    def test_download_episode(self):
+        super(TheSubDBTestCase, self).test_download_episode()
+
+
+class TvSubtitlesTestCase(ServiceTestCase):
+    query_tests = ['test_query_series', 'test_query_wrong_series', 'test_query_wrong_languages']
+    list_tests = ['test_list_episode', 'test_list_wrong_languages']
+    download_tests = ['test_download_episode']
+    cache_tests = ['test_cached_series']
+    service = TvSubtitles
+
+    def setUp(self):
+        super(TvSubtitlesTestCase, self).setUp()
+        self.config = ServiceConfig(multi=True, cache_dir=cache_dir)
+        self.fake_file = u'/tmp/fake_file'
+        self.languages = language_set(['en', 'es'])
+        self.episode_path = u'The Big Bang Theory/Season 05/The.Big.Bang.Theory.S05E06.HDTV.XviD-ASAP.mkv'
+        self.episode_sublanguage = 'en'
+        self.episode_subfilesizes = [33078]
+        self.episode_keywords = set(['asap', 'hdtv'])
+        self.series = 'The Big Bang Theory'
+        self.wrong_series = 'No Existent Show Name'
+        self.season = 5
+        self.episode = 6
+
+    def test_query_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) > 0)
+
+    def test_query_wrong_series(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.languages, self.episode_keywords, self.wrong_series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+    def test_query_wrong_languages(self):
+        with self.service(self.config) as service:
+            results = service.query(self.fake_file, self.wrong_languages, self.episode_keywords, self.series, self.season, self.episode)
+        self.assertTrue(len(results) == 0)
+
+
+TESTCASES = [Addic7edTestCase, BierDopjeTestCase, OpenSubtitlesTestCase, PodnapisiTestCase, SubsWikiTestCase,
+             SubtitulosTestCase, TheSubDBTestCase, TvSubtitlesTestCase]
 
 
 def query_suite():
     suite = unittest.TestSuite()
-    suite.addTests(map(BierDopjeTestCase, BierDopjeTestCase.query_tests))
-    suite.addTests(map(OpenSubtitlesTestCase, OpenSubtitlesTestCase.query_tests))
-    suite.addTests(map(TheSubDBTestCase, TheSubDBTestCase.query_tests))
-    suite.addTests(map(SubsWikiTestCase, SubsWikiTestCase.query_tests))
-    suite.addTests(map(SubtitulosTestCase, SubtitulosTestCase.query_tests))
+    for testcase in TESTCASES:
+        suite.addTests(map(testcase, testcase.query_tests))
     return suite
 
 
 def list_suite():
     suite = unittest.TestSuite()
-    suite.addTests(map(BierDopjeTestCase, BierDopjeTestCase.list_tests))
-    suite.addTests(map(OpenSubtitlesTestCase, OpenSubtitlesTestCase.list_tests))
-    suite.addTests(map(TheSubDBTestCase, TheSubDBTestCase.list_tests))
-    suite.addTests(map(SubsWikiTestCase, SubsWikiTestCase.list_tests))
-    suite.addTests(map(SubtitulosTestCase, SubtitulosTestCase.list_tests))
+    for testcase in TESTCASES:
+        suite.addTests(map(testcase, testcase.list_tests))
     return suite
 
 
 def download_suite():
     suite = unittest.TestSuite()
-    suite.addTests(map(BierDopjeTestCase, BierDopjeTestCase.download_tests))
-    suite.addTests(map(OpenSubtitlesTestCase, OpenSubtitlesTestCase.download_tests))
-    suite.addTests(map(TheSubDBTestCase, TheSubDBTestCase.download_tests))
-    suite.addTests(map(SubsWikiTestCase, SubsWikiTestCase.download_tests))
-    suite.addTests(map(SubtitulosTestCase, SubtitulosTestCase.download_tests))
+    for testcase in TESTCASES:
+        suite.addTests(map(testcase, testcase.download_tests))
     return suite
 
 
+def cache_suite():
+    suite = unittest.TestSuite()
+    for testcase in TESTCASES:
+        suite.addTests(map(testcase, testcase.cache_tests))
+    return suite
+
+
+def suite():
+    return unittest.TestSuite([query_suite(), list_suite(), download_suite(), cache_suite()])
+
+
 if __name__ == '__main__':
-    suites = []
-    suites.append(query_suite())
-    suites.append(list_suite())
-    suites.append(download_suite())
-    unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite(suites))
+    unittest.TextTestRunner().run(suite())

tests/test_subliminal.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from subliminal import Pool, list_subtitles, download_subtitles
+import os
+import time
+import unittest
+import requests
+import tarfile
+import StringIO
+
+
+test_dir = 'test_subliminal_files'
+cache_dir = 'test_subliminal_cache'
+test_video = 'The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_subliminal_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+    if not os.path.exists(cache_dir):
+        os.mkdir(cache_dir)
+
+
+class ApiTestCase(unittest.TestCase):
+    def test_list_subtitles(self):
+        results = list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
+        self.assertTrue(len(results) > 0)
+
+    def test_download_subtitles(self):
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
+        self.assertTrue(len(results) == 1)
+        for video, subtitles in results.iteritems():
+            self.assertTrue(video.release == test_video)
+            self.assertTrue(len(subtitles) == 1)
+            for subtitle in subtitles:
+                self.assertTrue(os.path.exists(subtitle.path))
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_multi(self):
+        results = download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir, multi=True)
+        self.assertTrue(len(results) == 1)
+        for video, subtitles in results.iteritems():
+            self.assertTrue(video.release == test_video)
+            self.assertTrue(len(subtitles) == 2)
+            for subtitle in subtitles:
+                self.assertTrue(os.path.exists(subtitle.path))
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_multi_noforce(self):
+        results_first = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        results = download_subtitles(test_dir, languages=['en', 'fr'], cache_dir=cache_dir, multi=True, force=False, services=['thesubdb'])
+        self.assertTrue(len(results) == 0)
+        for _, subtitles in results_first.iteritems():
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+    def test_download_subtitles_languages(self):
+        results = download_subtitles('Dexter/Season 04/S04E08 - Road Kill - 720p BluRay.mkv', languages=['en'],
+                                     cache_dir=cache_dir, multi=True, force=False, services=['subtitulos', 'tvsubtitles'])
+        self.assertTrue(len(results) == 1)
+        for _, subtitles in results.iteritems():
+            self.assertTrue(len(subtitles) == 1)
+            for subtitle in subtitles:
+                os.remove(subtitle.path)
+
+
+class AsyncTestCase(unittest.TestCase):
+    def test_pool(self):
+        p = Pool(4)
+        self.assertTrue(len(p.workers) == 4)
+        for w in p.workers:
+            self.assertTrue(w.isAlive() == False)
+        p.start()
+        for w in p.workers:
+            self.assertTrue(w.isAlive() == True)
+        p.stop()
+        p.join()
+        time.sleep(0.2)  # so terminate is finished on Worker and proper Thread methods finished
+        for w in p.workers:
+            self.assertTrue(w.isAlive() == False)
+
+    def test_list_subtitles(self):
+        with Pool(4) as p:
+            results = p.list_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
+        self.assertTrue(len(results) > 0)
+
+    def test_download_subtitles(self):
+        with Pool(4) as p:
+            results = p.download_subtitles(test_video, languages=['en', 'fr'], cache_dir=cache_dir)
+        self.assertTrue(len(results) == 1)
+        for video, subtitles in results.iteritems():
+            self.assertTrue(video.release == test_video)
+            self.assertTrue(len(subtitles) == 1)
+            for subtitle in subtitles:
+                self.assertTrue(os.path.exists(subtitle.path))
+                os.remove(subtitle.path)
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ApiTestCase))
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(AsyncTestCase))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite())

tests/test_videos.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+from subliminal.subtitles import EmbeddedSubtitle, ExternalSubtitle
+from subliminal.videos import scan
+from subliminal.language import Language
+import StringIO
+import os
+import requests
+import tarfile
+import unittest
+
+
+test_dir = 'test_videos_files'
+
+
+def setUpModule():
+    if not os.path.exists(test_dir):
+        r = requests.get('https://github.com/downloads/Diaoul/subliminal/test_videos_files.tar.gz')
+        with tarfile.open(fileobj=StringIO.StringIO(r.content), mode='r:gz') as f:
+            f.extractall(test_dir)
+
+
+class ScanTestCase(unittest.TestCase):
+    def test_basic(self):
+        results = scan(test_dir)
+        self.assertTrue(len(results) == 1)
+        self.assertTrue(isinstance(results[0], tuple))
+        self.assertTrue(len(results[0]) == 2)
+
+    def test_embedded_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, EmbeddedSubtitle)]
+        self.assertTrue(len(results) == 8)
+        for l in ('fre', 'eng', 'ita', 'spa', 'hun', 'ger', 'jpn', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+    def test_external_subtitles(self):
+        results = [s for s in scan(test_dir)[0][1] if isinstance(s, ExternalSubtitle)]
+        self.assertTrue(len(results) == 3)
+        for l in ('fre', 'eng', 'und'):
+            self.assertTrue(any([s.language == Language(l) for s in results]))
+
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ScanTestCase))
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner().run(suite())