release 2.0.24.1565

# Conflicts:
#	Contents/Info.plist
#	Contents/Libraries/Shared/subliminal_patch/patch_provider_pool.py
#	README.md

.gitattributes

@@ -0,0 +1,3 @@
+.gitattributes export-ignore
+/Wiki export-ignore
+.gitignore export-ignore

CHANGELOG.md

@@ -1,3 +1,141 @@
+2.0.23.1464 RC10.1
+ - core: huge bugfix; please check `Library/Application Support/Plex Media\ Server/Plug-in Support/Data/com.plexapp.agents.subzero/DataItems`
+         for any `subs_XXXXX.json.gz` file bigger than 500kb and delete them
+
+
+2.0.23.1456 RC10
+- core: findBetterSubtitles: increase series cutoff by 2 (resolution match)
+- core: add VTT format
+- core: fix crashes regarding DBM/cache management
+- core: update rarfile.py
+- core: add missing encodings
+- core: full support for Serbian subtitles (Cyrillic and Latin)
+- podnapisi: fix pt-BR, srp-cyrl and srp-latn
+- core: implement own provider registry and ditch the subliminal one
+- core: use ftfy library to fix re-encoding errors inside subtitles introduced by the subtitle author
+- core: always store and save subtitles normalized to UTF-8
+- core: replace spaced dashes in movie/series names before re-refining with plex metadata info
+- submod: remove_HI: handle multiline brackets correctly
+
+
+2.0.20.1364 RC9
+- core: performance improvements
+- core: if info couldn't be guessed from the filename, fill missing info from PMS #270
+- submod: OCR: add more to the eng dictionary
+- submod: HI: fixed some issues with font style tags
+- core: don't ignore subtitles from providers that don't have hearing impaired info, when hearing impaired mode is set to "force non-HI"
+- legendastv/menu: fix manual subtitle selection issues in menu
+- core: improve specials matching on OpenSubtitles
+- core: update guessit
+
+
+2.0.19.1337 RC8
+- napiprojekt: fixed: couldn't convert microdvd to SRT in certain occasions
+- core: when normalize to UTF-8 is enabled, also store the subtitle in UTF-8 encoding in the internal storage
+- core: add more encodings for western/eastern/northern europe
+- submod: OCR: update dictionaries from SubtitleEdit
+- submod: common: be smarter about uppercase i's in words that should have lowercase L's
+- submod: fix unopened/unclosed font style tags after modification
+- core: re-enable OMDB support
+- core: update guessit for better matching
+- core: fix SearchAllRecentlyMissing (was broken since RC3)
+
+
+2.0.19.1299 RC7
+- submod: offset mods now get merged internally when applied multiple times (to avoid errors and increase performance)
+- submod: improve performance
+- submod: core mods (OCR, common, remove_HI) now are always applied in a fixed order internally, regardless of the order they were added in
+- submod: CM_spaces_in_numbers: don't break up ellipses (30... 29... 28...)
+- submod: CM_spaces_in_numbers: don't fix countdown numbers (30, 29, 28)
+- submod: remove_HI: make bracket removal more aggressive
+- submod: remove_HI: be less aggressive when removing text-before-colon
+- submod: remove_HI: remove all-uppercase-before-sentence (THIS IS ALL UPPERCASE And here starts a sentence -> And here starts a sentence)
+- submod: fix all character ranges to include non-ASCII characters
+- add new README for 2.0
+
+
+2.0.19.1267 RC6
+- core: add new SZ subtitle storage format
+  - smaller data files and less cumbersome
+  - it will auto migrate when old data is accessed - to speed this up, use "Trigger subtitle storage migration (expensive)" in advanced menu)
+- core: performance optimizations
+- addic7ed: when release group matches, assume the format matches, too (leftover change from RC5)
+- submod: fix patterns for beginlines/endlines
+- submod: add our own dictionaries to OCR fixes (english)
+- submod: hearing impaired: also remove full-caps with punctuation inside
+- submod: correctly handle partiallines
+- submod: in numbers with spaces (incorrect), also allow for some punctuation (,.:')
+
+
+2.0.18.1245 RC5
+- core: add more debug info
+- core: fix subtitle modifications (was broken in RC4, created non-usable subtitles)
+- submod: add ANSI colors
+- menu/submod: add color mod menu
+- submod: exclusive mods now are mutually exclusive and get cleaned on duplicate
+- menu/core: naming
+
+For everyone who runs RC4: your subtitles are broken. Go to the advanced menu and trigger `Re-Apply mods of all stored subtitles` to fix them.
+
+
+2.0.17.1234 RC4
+- core: backport provider-download-retry implementation
+- core: implement custom user agent (for OpenSubtitles)
+- core/menu: correct handling of media with multiple files
+- core: fix SearchAllRecentlyMissing; also wait 5 seconds between searches
+- core: SearchAllRecentlyMissing: honor physical ignores
+- submod: pattern fixes
+- submod: better unicode handling
+- submod: add color mod (only automatic by now)
+
+
+2.0.15.1216 RC3
+- core: fixes
+- scheduler: revert some of the aggressive changes in RC2
+- submod: be smarter about WholeLine matches
+
+
+2.0.15.1209 RC2
+- core: fixes
+- core: submod-common: fix multiple dots at start of line
+- core/menu: add subtitle modification debug setting
+- core/menu: when manually listing available subtitles in menu, display those with wrong FPS also (opensubtitles), because you can fix them later
+- core/menu: advanced-menu: add apply-all-default-mods menu item; add re-apply all mods menu item
+- core: always look for currently (not-) existing subtitles when called; hopefully fixes #276
+- scheduler/menu: be faster; also launch scheduled tasks in threads, not just manually launched ones
+- core: don't delete subtitles with .custom or .embedded in their filenames when running auto cleanup, if the correct media file exists
+- menu: add back-to-previous menu items
+
+
+2.0.12.1180 RC1
+- core: update subliminal to version 2
+- core: update all dependencies
+- core: add new providers: legendastv (pt-BR), napiprojekt (pl), shooter (cn), subscenter (heb)
+- core: rewritten all subliminal patches for version 2
+- menu: add icons for menu items; update main channel icon
+- core: use SSL again for opensubtitles
+- core: improved matching due to subliminal 2 (and SZ custom) tvdb/omdb refiners
+- menu: add "Get my logs" function to the advanced menu, which zips up all necessary logs suitable for posting in the forums
+- core: on non-windows systems, utilize a file-based cache database for provider media lists and subliminal refiner results
+- core: add manual and automatic subtitle modification framework (fix common OCR issues, remove hearing impaired etc.)
+- menu: add subtitle modifications (subtitle content fixes, offset-based shifting, framerate conversion)
+- menu: add recently played menu
+- improve almost everything Sub-Zero did in 1.4 :)
+
+
+1.4.27.973
+- core: ignore "obfuscated" and "scrambled" tags in filenames when searching for subtitles
+- core: exotic embedded subtitles are now also considered when searching (and when the option is enabled); fixes #264
+
+
+1.4.27.967
+- core: remember the last 10 played items; only consider on_playback for "playing" state within the first 60 seconds of an item
+
+
+1.4.27.965
+- core: on_playback activity bugfixes
+
+
 1.4.27.957
 - core: correctly fall back to the next best subtitle if the current one couldn't be downloaded; hopefully fixes #231
 - core: add "Scan: which external subtitles should be picked up?"-setting

Contents/Code/__init__.py

@@ -3,7 +3,6 @@ import sys
 import datetime
 import os
 
-from subliminal_patch import compute_score
 from subzero.sandbox import restore_builtins
 
 module = sys.modules['__main__']
@@ -18,18 +17,15 @@ import logger
 
 sys.modules["logger"] = logger
 
-import subliminal
 import support
 
 import interface
 sys.modules["interface"] = interface
 
-from subliminal.cli import MutexLock
 from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
 from interface.menu import *
 from support.plex_media import media_to_videos, get_media_item_ids, scan_videos
-from support.subtitlehelpers import get_subtitles_from_metadata
-from support.storage import whack_missing_parts, save_subtitles, get_subtitle_storage
+from support.storage import save_subtitles, store_subtitle_info
 from support.items import is_ignored
 from support.config import config
 from support.lib import get_intent
@@ -37,19 +33,14 @@ from support.helpers import track_usage, get_title_for_video_metadata, get_ident
 from support.history import get_history
 from support.data import dispatch_migrate
 from support.activities import activity
+from support.download import download_best_subtitles
 
 
 def Start():
     HTTP.CacheTime = 0
     HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
 
-    try:
-        subliminal.region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
-                                    arguments={'filename': os.path.join(config.data_items_path, 'subzero.dbm'),
-                                               'lock_factory': MutexLock})
-    except:
-        Log.Warn("Not using file based cache!")
-        subliminal.region.configure('dogpile.cache.memory')
+    config.init_cache()
 
     # clear expired intents
     intent = get_intent()
@@ -58,9 +49,12 @@ def Start():
     # clear expired menu history items
     now = datetime.datetime.now()
     if "menu_history" in Dict:
-        for key, timeout in Dict["menu_history"].items():
+        for key, timeout in Dict["menu_history"].copy().items():
             if now > timeout:
-                del Dict["menu_history"][key]
+                try:
+                    del Dict["menu_history"][key]
+                except:
+                    pass
 
     # run migrations
     if "subs" in Dict or "history" in Dict:
@@ -96,45 +90,6 @@ def Start():
         track_usage("General", "plugin", "start", config.version)
 
 
-def download_best_subtitles(video_part_map, min_score=0):
-    hearing_impaired = Prefs['subtitles.search.hearingImpaired']
-    languages = config.lang_list
-    if not languages:
-        return
-
-    missing_languages = False
-    for video, part in video_part_map.iteritems():
-        if not Prefs['subtitles.save.filesystem']:
-            # scan for existing metadata subtitles
-            meta_subs = get_subtitles_from_metadata(part)
-            for language, subList in meta_subs.iteritems():
-                if subList:
-                    video.subtitle_languages.add(language)
-                    Log.Debug("Found metadata subtitle %s for %s", language, video)
-
-        missing_subs = (languages - video.subtitle_languages)
-
-        # all languages are found if we either really have subs for all languages or we only want to have exactly one language
-        # and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
-        found_one_which_is_enough = len(video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
-        if not missing_subs or found_one_which_is_enough:
-            if found_one_which_is_enough:
-                Log.Debug('Only one language was requested, and we\'ve got a subtitle for %s', video)
-            else:
-                Log.Debug('All languages %r exist for %s', languages, video)
-            continue
-        missing_languages = True
-        break
-
-    if missing_languages:
-        Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" % (min_score, hearing_impaired))
-
-        return subliminal.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
-                                                  provider_configs=config.provider_settings, pool_class=config.provider_pool,
-                                                  compute_score=compute_score)
-    Log.Debug("All languages for all requested videos exist. Doing nothing.")
-
-
 def update_local_media(metadata, media, media_type="movies"):
     # Look for subtitles
     if media_type == "movies":
@@ -175,10 +130,6 @@ class SubZeroAgent(object):
         results.Append(MetadataSearchResult(id='null', score=100))
 
     def update(self, metadata, media, lang):
-        if not config.enable_agent:
-            Log.Debug("Skipping Sub-Zero agent(s)")
-            return
-
         Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))
         intent = get_intent()
 
@@ -191,6 +142,9 @@ class SubZeroAgent(object):
             config.init_subliminal_patches()
             videos = media_to_videos(media, kind=self.agent_type)
 
+            # find local media
+            update_local_media(metadata, media, media_type=self.agent_type)
+
             # media ignored?
             use_any_parts = False
             for video in videos:
@@ -211,20 +165,24 @@ class SubZeroAgent(object):
 
             set_refresh_menu_state(media, media_type=self.agent_type)
 
-            # find local media
-            update_local_media(metadata, media, media_type=self.agent_type)
-
             # scanned_video_part_map = {subliminal.Video: plex_part, ...}
             scanned_video_part_map = scan_videos(videos, kind=self.agent_type)
 
-            # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
-            downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score)
-            item_ids = get_media_item_ids(media, kind=self.agent_type)
+            downloaded_subtitles = None
+            if not config.enable_agent:
+                Log.Debug("Skipping Sub-Zero agent(s)")
 
-            whack_missing_parts(scanned_video_part_map)
+            else:
+                # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
+                downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score)
+                item_ids = get_media_item_ids(media, kind=self.agent_type)
 
+            downloaded_any = False
             if downloaded_subtitles:
-                save_subtitles(scanned_video_part_map, downloaded_subtitles)
+                downloaded_any = any(downloaded_subtitles.values())
+
+            if downloaded_any:
+                save_subtitles(scanned_video_part_map, downloaded_subtitles, mods=config.default_mods)
                 track_usage("Subtitle", "refreshed", "download", 1)
 
                 for video, video_subtitles in downloaded_subtitles.items():
@@ -234,6 +192,10 @@ class SubZeroAgent(object):
                         history = get_history()
                         history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
                                     subtitle=subtitle)
+            else:
+                # store subtitle info even if we've downloaded none
+                store_subtitle_info(scanned_video_part_map, dict((k, []) for k in scanned_video_part_map.keys()),
+                                    None, mode="a")
 
             update_local_media(metadata, media, media_type=self.agent_type)
 
@@ -243,7 +205,7 @@ class SubZeroAgent(object):
 
             # notify any running tasks about our finished update
             for item_id in item_ids:
-                scheduler.signal("updated_metadata", item_id)
+                #scheduler.signal("updated_metadata", item_id)
 
                 # resolve existing intent for that id
                 intent.resolve("force", item_id)

Contents/Code/interface/__init__.py

@@ -18,3 +18,6 @@ sys.modules["interface.refresh_item"] = refresh_item
 
 import item_details
 sys.modules["interface.item_details"] = item_details
+
+import sub_mod
+sys.modules["interface.modification"] = sub_mod

Contents/Code/interface/advanced.py

@@ -3,19 +3,23 @@ import datetime
 import StringIO
 import glob
 import os
+import traceback
 import urlparse
 
 from zipfile import ZipFile, ZIP_DEFLATED
 
+from babelfish import Language
+
 from subzero.lib.io import FileIO
 from subzero.constants import PREFIX, PLUGIN_IDENTIFIER
-from menu_helpers import SubFolderObjectContainer, debounce, set_refresh_menu_state, ZipObject
+from menu_helpers import SubFolderObjectContainer, debounce, set_refresh_menu_state, ZipObject, ObjectContainer
 from main import fatality
 from support.helpers import timestamp, pad_title
 from support.config import config
 from support.lib import Plex
-from support.storage import reset_storage, log_storage
-from support.background import scheduler
+from support.storage import reset_storage, log_storage, get_subtitle_storage
+from support.scheduler import scheduler
+from support.items import set_mods_for_part, get_item_kind_from_rating_key
 
 
 @route(PREFIX + '/advanced')
@@ -49,6 +53,18 @@ def AdvancedMenu(randomize=None, header=None, message=None):
         key=Callback(TriggerStorageMaintenance, randomize=timestamp()),
         title=pad_title("Trigger subtitle storage maintenance"),
     ))
+    oc.add(DirectoryObject(
+        key=Callback(TriggerStorageMigration, randomize=timestamp()),
+        title=pad_title("Trigger subtitle storage migration (expensive)"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(ApplyDefaultMods, randomize=timestamp()),
+        title=pad_title("Apply configured default subtitle mods to all (active) stored subtitles"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(ReApplyMods, randomize=timestamp()),
+        title=pad_title("Re-Apply mods of all stored subtitles"),
+    ))
     oc.add(DirectoryObject(
         key=Callback(LogStorage, key="tasks", randomize=timestamp()),
         title=pad_title("Log the plugin's scheduled tasks state storage"),
@@ -57,6 +73,10 @@ def AdvancedMenu(randomize=None, header=None, message=None):
         key=Callback(LogStorage, key="ignore", randomize=timestamp()),
         title=pad_title("Log the plugin's internal ignorelist storage"),
     ))
+    oc.add(DirectoryObject(
+        key=Callback(LogStorage, key=None, randomize=timestamp()),
+        title=pad_title("Log the plugin's complete state storage"),
+    ))
     oc.add(DirectoryObject(
         key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
         title=pad_title("Reset the plugin's scheduled tasks state storage"),
@@ -92,6 +112,7 @@ def Restart():
 
 
 @route(PREFIX + '/storage/reset', sure=bool)
+@debounce
 def ResetStorage(key, randomize=None, sure=False):
     if not sure:
         oc = SubFolderObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
@@ -127,6 +148,7 @@ def LogStorage(key, randomize=None):
 
 
 @route(PREFIX + '/triggerbetter')
+@debounce
 def TriggerBetterSubtitles(randomize=None):
     scheduler.dispatch_task("FindBetterSubtitles")
     return AdvancedMenu(
@@ -137,6 +159,7 @@ def TriggerBetterSubtitles(randomize=None):
 
 
 @route(PREFIX + '/triggermaintenance')
+@debounce
 def TriggerStorageMaintenance(randomize=None):
     scheduler.dispatch_task("SubtitleStorageMaintenance")
     return AdvancedMenu(
@@ -146,27 +169,111 @@ def TriggerStorageMaintenance(randomize=None):
     )
 
 
+@route(PREFIX + '/triggerstoragemigration')
+@debounce
+def TriggerStorageMigration(randomize=None):
+    scheduler.dispatch_task("MigrateSubtitleStorage")
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='MigrateSubtitleStorage triggered'
+    )
+
+
+def apply_default_mods(reapply_current=False):
+    storage = get_subtitle_storage()
+    subs_applied = 0
+    for fn in storage.get_all_files():
+        data = storage.load(None, filename=fn)
+        if data:
+            video_id = data.video_id
+            item_type = get_item_kind_from_rating_key(video_id)
+            if not item_type:
+                continue
+
+            for part_id, part in data.parts.iteritems():
+                for lang, subs in part.iteritems():
+                    current_sub = subs.get("current")
+                    if not current_sub:
+                        continue
+                    sub = subs[current_sub]
+
+                    if not sub.content:
+                        continue
+
+                    current_mods = sub.mods or []
+                    if not reapply_current:
+                        add_mods = list(set(config.default_mods).difference(set(current_mods)))
+                        if not add_mods:
+                            continue
+                    else:
+                        if not current_mods:
+                            continue
+                        add_mods = []
+
+                    try:
+                        set_mods_for_part(video_id, part_id, Language.fromietf(lang), item_type, add_mods, mode="add")
+                    except:
+                        Log.Error("Couldn't set mods for %s:%s: %s", video_id, part_id, traceback.format_exc())
+                        continue
+
+                    subs_applied += 1
+    Log.Debug("Applied mods to %i items" % subs_applied)
+
+
+@route(PREFIX + '/applydefaultmods')
+@debounce
+def ApplyDefaultMods(randomize=None):
+    Thread.CreateTimer(1.0, apply_default_mods)
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='This may take some time ...'
+    )
+
+
+@route(PREFIX + '/reapplyallmods')
+@debounce
+def ReApplyMods(randomize=None):
+    Thread.CreateTimer(1.0, apply_default_mods, reapply_current=True)
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='This may take some time ...'
+    )
+
+
 @route(PREFIX + '/get_logs_link')
 def GetLogsLink():
+    if not config.plex_token:
+        oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
+                             header="Sorry, feature unavailable",
+                             message="Universal Plex token not available")
+        return oc
+
     # try getting the link base via the request in context, first, otherwise use the public ip
     req_headers = Core.sandbox.context.request.headers
+    get_external_ip = True
+    link_base = ""
 
     if "Origin" in req_headers:
         link_base = req_headers["Origin"]
         Log.Debug("Using origin-based link_base")
+        get_external_ip = False
 
     elif "Referer" in req_headers:
         parsed = urlparse.urlparse(req_headers["Referer"])
         link_base = "%s://%s:%s" % (parsed.scheme, parsed.hostname, parsed.port)
         Log.Debug("Using referer-based link_base")
+        get_external_ip = False
 
-    else:
+    if get_external_ip or "plex.tv" in link_base:
         ip = Core.networking.http_request("http://www.plexapp.com/ip.php", cacheTime=7200).content.strip()
         link_base = "https://%s:32400" % ip
         Log.Debug("Using ip-based fallback link_base")
 
-    logs_link = "%s%s?X-Plex-Token=%s" % (link_base, PREFIX + '/logs', config.universal_plex_token)
-    oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
+    logs_link = "%s%s?X-Plex-Token=%s" % (link_base, PREFIX + '/logs', config.plex_token)
+    oc = ObjectContainer(title2=logs_link, no_cache=True, no_history=True,
                          header="Copy this link and open this in your browser, please",
                          message=logs_link)
     return oc
@@ -189,6 +296,7 @@ def DownloadLogs():
 
 
 @route(PREFIX + '/invalidatecache')
+@debounce
 def InvalidateCache(randomize=None):
     from subliminal.cache import region
     region.invalidate()

Contents/Code/interface/item_details.py

@@ -1,18 +1,22 @@
 # coding=utf-8
 import os
 
-from subzero.constants import PREFIX
+from sub_mod import SubtitleModificationsMenu
 from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_ignore_options, get_item_task_data, \
     set_refresh_menu_state
-from refresh_item import RefreshItem
-from support.helpers import timestamp, cast_bool, df, get_language
-from support.items import get_item_kind_from_rating_key, get_item
-from support.plex_media import get_plex_metadata, scan_videos
-from support.lib import Plex
-from support.storage import get_subtitle_storage
-from support.config import config
-from support.background import scheduler
 
+from refresh_item import RefreshItem
+from subzero.constants import PREFIX
+from support.config import config
+from support.helpers import timestamp, cast_bool, df, get_language
+from support.items import get_item_kind_from_rating_key, get_item, get_current_sub
+from support.lib import Plex
+from support.plex_media import get_plex_metadata, scan_videos, PMSMediaProxy
+from support.scheduler import scheduler
+from support.storage import get_subtitle_storage
+
+
+# fixme: needs kwargs cleanup
 
 @route(PREFIX + '/item/{rating_key}/actions')
 @debounce
@@ -35,6 +39,22 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
     timeout = 30
 
     oc = SubFolderObjectContainer(title2=title, replace_parent=True)
+
+    # add back to season for episode
+    if current_kind == "episode":
+        from interface.menu import MetadataMenu
+        show = get_item(item.show.rating_key)
+        season = get_item(item.season.rating_key)
+
+        oc.add(DirectoryObject(
+            key=Callback(MetadataMenu, rating_key=season.rating_key, title=season.title, base_title=show.title,
+                         previous_item_type="show", previous_rating_key=show.rating_key,
+                         display_items=True, randomize=timestamp()),
+            title=u"< Back to %s" % season.title,
+            summary="Back to %s > %s" % (show.title, season.title),
+            thumb=season.thumb or default_thumb
+        ))
+
     oc.add(DirectoryObject(
         key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp(),
                      timeout=timeout * 1000),
@@ -45,7 +65,7 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
     oc.add(DirectoryObject(
         key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(),
                      timeout=timeout * 1000),
-        title=u"Auto-search: %s" % item_title,
+        title=u"Force-find subtitles: %s" % item_title,
         summary="Issues a forced refresh, ignoring known subtitles and searching for new ones",
         thumb=item.thumb or default_thumb
     ))
@@ -55,58 +75,108 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
     stored_subs = subtitle_storage.load_or_new(item)
 
     # get the plex item
-    plex_item = list(Plex["library"].metadata(rating_key))[0]
-
-    # get current media info for that item
-    media = plex_item.media
+    plex_item = get_item(rating_key)
 
     # look for subtitles for all available media parts and all of their languages
-    for part in media.parts:
-        filename = os.path.basename(part.file)
-        part_id = str(part.id)
+    has_multiple_parts = len(plex_item.media) > 1
+    part_index = 0
+    for media in plex_item.media:
+        for part in media.parts:
+            filename = os.path.basename(part.file)
+            if not os.path.exists(part.file):
+                continue
 
-        # iterate through all configured languages
-        for lang in config.lang_list:
-            lang_a2 = lang.alpha2
-            # ietf lang?
-            if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
-                lang_a2 = lang_a2.split("-")[0]
+            part_id = str(part.id)
+            part_index += 1
 
-            # get corresponding stored subtitle data for that media part (physical media item), for language
-            current_sub = stored_subs.get_any(part_id, lang_a2)
-            current_sub_id = None
-            current_sub_provider_name = None
+            # iterate through all configured languages
+            for lang in config.lang_list:
+                # get corresponding stored subtitle data for that media part (physical media item), for language
+                current_sub = stored_subs.get_any(part_id, lang)
+                current_sub_id = None
+                current_sub_provider_name = None
 
-            summary = u"No current subtitle in storage"
-            current_score = None
-            if current_sub:
-                current_sub_id = current_sub.id
-                current_sub_provider_name = current_sub.provider_name
-                current_score = current_sub.score
+                part_index_addon = ""
+                part_summary_addon = ""
+                if has_multiple_parts:
+                    part_index_addon = u"File %s: " % part_index
+                    part_summary_addon = "%s " % filename
 
-                summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
-                          (current_sub.provider_name, df(current_sub.date_added), current_sub.mode_verbose, lang,
-                           current_sub.score, current_sub.storage_type)
+                summary = u"%sNo current subtitle in storage" % part_summary_addon
+                current_score = None
+                if current_sub:
+                    current_sub_id = current_sub.id
+                    current_sub_provider_name = current_sub.provider_name
+                    current_score = current_sub.score
 
-            oc.add(DirectoryObject(
-                key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title,
-                             item_title=item_title, language=lang, current_id=current_sub_id,
-                             item_type=plex_item.type, filename=filename, current_data=summary,
-                             randomize=timestamp(), current_provider=current_sub_provider_name,
-                             current_score=current_score),
-                title=u"List %s subtitles" % lang.name,
-                summary=summary
-            ))
+                    summary = u"%sCurrent subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
+                              (part_summary_addon, current_sub.provider_name, df(current_sub.date_added),
+                               current_sub.mode_verbose, lang, current_sub.score, current_sub.storage_type)
+
+                    oc.add(DirectoryObject(
+                        key=Callback(SubtitleOptionsMenu, rating_key=rating_key, part_id=part_id, title=title,
+                                     item_title=item_title, language=lang, language_name=lang.name,
+                                     current_id=current_sub_id,
+                                     item_type=plex_item.type, filename=filename, current_data=summary,
+                                     randomize=timestamp(), current_provider=current_sub_provider_name,
+                                     current_score=current_score),
+                        title=u"%sActions for %s subtitle" % (part_index_addon, lang.name),
+                        summary=summary
+                    ))
+                else:
+                    oc.add(DirectoryObject(
+                        key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title,
+                                     item_title=item_title, language=lang, language_name=lang.name,
+                                     current_id=current_sub_id,
+                                     item_type=plex_item.type, filename=filename, current_data=summary,
+                                     randomize=timestamp(), current_provider=current_sub_provider_name,
+                                     current_score=current_score),
+                        title=u"%sList %s subtitles" % (part_index_addon, lang.name),
+                        summary=summary
+                    ))
 
     add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
     return oc
 
 
+@route(PREFIX + '/item/current_sub/{rating_key}/{part_id}', force=bool)
+@debounce
+def SubtitleOptionsMenu(**kwargs):
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs["part_id"]
+    language = kwargs["language"]
+
+    current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
+    kwargs.pop("randomize")
+
+    oc.add(DirectoryObject(
+        key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"],
+                     title=kwargs["title"], randomize=timestamp()),
+        title=u"< Back to %s" % kwargs["title"],
+        summary=kwargs["current_data"],
+        thumb=default_thumb
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(ListAvailableSubsForItemMenu, randomize=timestamp(), **kwargs),
+        title=u"List %s subtitles" % kwargs["language_name"],
+        summary=kwargs["current_data"]
+    ))
+    if current_sub:
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
+            title=u"Modify %s subtitle" % kwargs["language_name"],
+            summary=u"Currently applied mods: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
+        ))
+    return oc
+
+
 @route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool)
 @debounce
 def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None,
-                                 item_type="episode", language=None, force=False, current_id=None, current_data=None,
+                                 item_type="episode", language=None, language_name=None, force=False, current_id=None,
+                                 current_data=None,
                                  current_provider=None, current_score=None, randomize=None):
     assert rating_key, part_id
 
@@ -121,7 +191,7 @@ def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item
     oc = SubFolderObjectContainer(title2=unicode(title), replace_parent=True)
     oc.add(DirectoryObject(
         key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()),
-        title=u"Back to: %s" % title,
+        title=u"< Back to %s" % title,
         summary=current_data,
         thumb=default_thumb
     ))
@@ -151,6 +221,18 @@ def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item
             summary=u"%sFilename: %s" % (current_display, filename),
             thumb=default_thumb
         ))
+
+        if search_results == "found_none":
+            oc.add(DirectoryObject(
+                key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
+                             language=language, filename=filename, current_data=current_data, force=True,
+                             part_id=part_id, title=title, current_id=current_id, item_type=item_type,
+                             current_provider=current_provider, current_score=current_score,
+                             randomize=timestamp()),
+                title=u"No subtitles found",
+                summary=u"%sFilename: %s" % (current_display, filename),
+                thumb=default_thumb
+            ))
     else:
         oc.add(DirectoryObject(
             key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
@@ -163,19 +245,29 @@ def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item
             thumb=default_thumb
         ))
 
-    if not search_results:
+    if not search_results or search_results == "found_none":
         return oc
 
+    seen = []
     for subtitle in search_results:
+        if subtitle.id in seen:
+            continue
+
+        wrong_fps_addon = ""
+        if subtitle.wrong_fps:
+            wrong_fps_addon = " (wrong FPS, sub: %s, media: %s)" % (subtitle.fps, plex_part.fps)
+
         oc.add(DirectoryObject(
             key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title,
                          subtitle_id=str(subtitle.id), language=language),
-            title=u"%s: %s, score: %s" % ("Available" if current_id != subtitle.id else "Current",
-                                          subtitle.provider_name, subtitle.score),
+            title=u"%s: %s, score: %s%s" % ("Available" if current_id != subtitle.id else "Current",
+                                            subtitle.provider_name, subtitle.score, wrong_fps_addon),
             summary=u"Release: %s, Matches: %s" % (subtitle.release_info, ", ".join(subtitle.matches)),
             thumb=default_thumb
         ))
 
+        seen.append(current_id)
+
     return oc
 
 

Contents/Code/interface/main.py

@@ -2,11 +2,13 @@
 
 from subzero.constants import PREFIX, TITLE, ART
 from support.config import config
-from support.helpers import pad_title, timestamp, df
-from support.background import scheduler
+from support.helpers import pad_title, timestamp, df, get_plex_item_display_title
+from support.scheduler import scheduler
 from support.ignore import ignore_list
-from support.items import get_item_thumb, get_on_deck_items, get_all_items, get_items_info
-from menu_helpers import main_icon, debounce, SubFolderObjectContainer, default_thumb, dig_tree, add_ignore_options
+from support.items import get_item_thumb, get_on_deck_items, get_all_items, get_items_info, get_item, \
+    get_item_kind_from_item
+from menu_helpers import main_icon, debounce, SubFolderObjectContainer, default_thumb, dig_tree, add_ignore_options,\
+    ObjectContainer
 from item_details import ItemDetailsMenu
 
 
@@ -69,16 +71,24 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
 
         oc.add(DirectoryObject(
             key=Callback(OnDeckMenu),
-            title="On Deck items",
+            title="On-deck items",
             summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/"
                     "subtitles.",
             thumb=R("icon-ondeck.jpg")
         ))
+        if "last_played_items" in Dict and Dict["last_played_items"]:
+            oc.add(DirectoryObject(
+                key=Callback(RecentlyPlayedMenu),
+                title=pad_title("Recently played items"),
+                summary="Shows the %i recently played items and allows you to individually (force-) refresh their "
+                        "metadata/subtitles." % config.store_recently_played_amount,
+                thumb=R("icon-played.jpg")
+            ))
         oc.add(DirectoryObject(
             key=Callback(RecentlyAddedMenu),
-            title="Recently Added items",
+            title="Recently-added items",
             summary="Shows the recently added items per section.",
-            thumb=R("icon-recent.jpg")
+            thumb=R("icon-added.jpg")
         ))
         oc.add(DirectoryObject(
             key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()),
@@ -100,7 +110,7 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
         task = scheduler.task(task_name)
 
         if task.ready_for_display:
-            task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
+            task_state = "Running: %s/%s (%s%%)" % (task.items_done, task.items_searching, task.percentage)
         else:
             task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (
                 df(scheduler.last_run(task_name)) or "never",
@@ -168,6 +178,34 @@ def OnDeckMenu(message=None):
     return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=get_on_deck_items)
 
 
+@route(PREFIX + '/recently_played')
+def RecentlyPlayedMenu():
+    base_title = "Recently Played"
+    oc = SubFolderObjectContainer(title2=base_title, replace_parent=True)
+
+    for item in [get_item(rating_key) for rating_key in Dict["last_played_items"]]:
+        if not item:
+            continue
+
+        kind = get_item_kind_from_item(item)
+        if kind not in ("episode", "movie"):
+            continue
+
+        if kind == "episode":
+            item_title = get_plex_item_display_title(item, "show", parent=item.season, section_title=None,
+                                                     parent_title=item.show.title)
+        else:
+            item_title = get_plex_item_display_title(item, kind, section_title=None)
+
+        oc.add(DirectoryObject(
+            title=item_title,
+            key=Callback(ItemDetailsMenu, title=base_title + " > " + item.title, item_title=item.title,
+                         rating_key=item.rating_key)
+        ))
+
+    return oc
+
+
 @route(PREFIX + '/recently_added')
 def RecentlyAddedMenu(message=None):
     """
@@ -215,8 +253,6 @@ def RecentMissingSubtitlesMenu(force=False, randomize=None):
                 thumb=get_item_thumb(item) or default_thumb
             ))
 
-        scheduler.clear_task_data("MissingSubtitles")
-
     return oc
 
 

Contents/Code/interface/menu.py

@@ -1,5 +1,8 @@
 # coding=utf-8
+import locale
 import logging
+import os
+
 import logger
 
 from item_details import ItemDetailsMenu
@@ -9,12 +12,12 @@ from menu_helpers import add_ignore_options, dig_tree, set_refresh_menu_state, \
 from main import fatality, IgnoreMenu
 from advanced import DispatchRestart
 from subzero.constants import ART, PREFIX, DEPENDENCY_MODULE_NAMES
-from support.background import scheduler
+from support.scheduler import scheduler
 from support.config import config
-from support.helpers import timestamp,  df
+from support.helpers import timestamp, df
 from support.ignore import ignore_list
 from support.items import get_all_items, get_items_info, \
-    get_item_kind_from_rating_key
+    get_item_kind_from_rating_key, get_item
 
 # init GUI
 ObjectContainer.art = R(ART)
@@ -53,7 +56,7 @@ def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, displa
 
 @route(PREFIX + '/section/contents', display_items=bool)
 def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None,
-                 previous_rating_key=None):
+                 previous_rating_key=None, randomize=None):
     """
     displays the contents of a section based on whether it has a deeper tree or not (movies->movie (item) list; series->series list)
     :param rating_key:
@@ -72,6 +75,22 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
     current_kind = get_item_kind_from_rating_key(rating_key)
 
     if display_items:
+        timeout = 30
+
+        # add back to series for season
+        if current_kind == "season":
+            timeout = 360
+
+            show = get_item(previous_rating_key)
+            oc.add(DirectoryObject(
+                key=Callback(MetadataMenu, rating_key=show.rating_key, title=show.title, base_title=show.section.title,
+                             previous_item_type="section", display_items=True, randomize=timestamp()),
+                title=u"< Back to %s" % show.title,
+                thumb=show.thumb or default_thumb
+            ))
+        elif current_kind == "series":
+            timeout = 1800
+
         items = get_all_items(key="children", value=rating_key, base="library/metadata")
         kind, deeper = get_items_info(items)
         dig_tree(oc, items, MetadataMenu,
@@ -81,12 +100,6 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
         if should_display_ignore(items, previous=previous_item_type):
             add_ignore_options(oc, "series", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
-        timeout = 30
-        if current_kind == "season":
-            timeout = 360
-        elif current_kind == "series":
-            timeout = 1800
-
         # add refresh
         oc.add(DirectoryObject(
             key=Callback(RefreshItem, rating_key=rating_key, item_title=title, refresh_kind=current_kind,
@@ -147,7 +160,6 @@ def RefreshMissing(randomize=None):
 @route(PREFIX + '/ValidatePrefs', enforce_route=True)
 def ValidatePrefs():
     Core.log.setLevel(logging.DEBUG)
-    Log.Debug("Validate Prefs called.")
 
     # cache the channel state
     update_dict = False
@@ -182,9 +194,51 @@ def ValidatePrefs():
         Core.log.removeHandler(logger.console_handler)
         Log.Debug("Stop logging to console")
 
+    Log.Debug("Validate Prefs called.")
+
+    # SZ config debug
+    Log.Debug("--- SZ Config-Debug ---")
+    for attr in [
+            "app_support_path", "data_path", "data_items_path", "enable_agent",
+            "enable_channel", "permissions_ok", "missing_permissions", "fs_encoding",
+            "subtitle_destination_folder", "dbm_supported", "lang_list"]:
+        Log.Debug("config.%s: %s", attr, getattr(config, attr))
+
+    for attr in ["plugin_log_path", "server_log_path"]:
+        value = getattr(config, attr)
+        access = os.access(value, os.R_OK)
+        if Core.runtime.os == "Windows":
+            try:
+                f = open(value, "r")
+                f.read(1)
+                f.close()
+            except:
+                access = False
+
+        Log.Debug("config.%s: %s (accessible: %s)", attr, value, access)
+
+    for attr in [
+            "subtitles.save.filesystem", ]:
+        Log.Debug("Pref.%s: %s", attr, Prefs[attr])
+
+    # fixme: check existance of and os access of logs
+    Log.Debug("Platform: %s", Core.runtime.platform)
+    Log.Debug("OS: %s", Core.runtime.os)
+    Log.Debug("----- Environment -----")
+    for key, value in os.environ.iteritems():
+        if key.startswith("PLEX") or key.startswith("SZ_"):
+            if "TOKEN" in key:
+                outval = "xxxxxxxxxxxxxxxxxxx"
+
+            else:
+                outval = value
+            Log.Debug("%s: %s", key, outval)
+    Log.Debug("Locale: %s", locale.getdefaultlocale())
+    Log.Debug("-----------------------")
+
     Log.Debug("Setting log-level to %s", Prefs["log_level"])
     logger.register_logging_handler(DEPENDENCY_MODULE_NAMES, level=Prefs["log_level"])
     Core.log.setLevel(logging.getLevelName(Prefs["log_level"]))
+    os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'
 
     return
-

Contents/Code/interface/menu_helpers.py

@@ -8,7 +8,7 @@ from support.ignore import ignore_list
 from support.lib import get_intent
 from support.config import config
 from subzero.constants import ICON_SUB, ICON
-from support.background import scheduler
+from support.scheduler import scheduler
 
 default_thumb = R(ICON_SUB)
 main_icon = ICON if not config.is_development else "icon-dev.jpg"
@@ -43,8 +43,8 @@ def add_ignore_options(oc, kind, callback_menu=None, title=None, rating_key=None
 
     oc.add(DirectoryObject(
         key=Callback(callback_menu, kind=use_kind, rating_key=rating_key, title=title),
-        title=u"%s %s \"%s\" %s the ignore list" % (
-            "Remove" if in_list else "Add", ignore_list.verbose(kind) if add_kind else "", unicode(title), "from" if in_list else "to")
+        title=u"%s %s \"%s\"" % (
+            "Un-Ignore" if in_list else "Ignore", ignore_list.verbose(kind) if add_kind else "", unicode(title))
     )
     )
 
@@ -148,7 +148,7 @@ def debounce(func):
 
     def wrap(*args, **kwargs):
         if "randomize" in kwargs:
-            if not "menu_history" in Dict:
+            if "menu_history" not in Dict:
                 Dict["menu_history"] = {}
 
             key = get_lookup_key([func] + list(args), kwargs)
@@ -156,8 +156,13 @@ def debounce(func):
                 Log.Debug("not triggering %s twice with %s, %s" % (func, args, kwargs))
                 return ObjectContainer()
             else:
-                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(days=1)
-                Dict.Save()
+                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(hours=6)
+                try:
+                    Dict.Save()
+                except TypeError:
+                    Log.Error("Can't save menu history for: %r", key)
+                    del Dict["menu_history"][key]
+
         return func(*args, **kwargs)
 
     return wrap

Contents/Code/interface/sub_mod.py

@@ -0,0 +1,251 @@
+# coding=utf-8
+
+import traceback
+import types
+
+from babelfish import Language
+
+from menu_helpers import debounce, SubFolderObjectContainer, default_thumb
+from subzero.modification import registry as mod_registry, SubtitleModifications
+from subzero.constants import PREFIX
+from support.plex_media import get_plex_metadata, scan_videos
+from support.helpers import timestamp, pad_title
+from support.items import get_current_sub, set_mods_for_part
+
+
+@route(PREFIX + '/item/sub_mods/{rating_key}/{part_id}', force=bool)
+@debounce
+def SubtitleModificationsMenu(**kwargs):
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs["part_id"]
+    language = kwargs["language"]
+    current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
+    kwargs.pop("randomize")
+
+    current_mods = current_sub.mods or []
+
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    from interface.item_details import SubtitleOptionsMenu
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleOptionsMenu, randomize=timestamp(), **kwargs),
+        title=u"< Back to subtitle options for: %s" % kwargs["title"],
+        summary=kwargs["current_data"],
+        thumb=default_thumb
+    ))
+
+    for identifier, mod in mod_registry.mods.iteritems():
+        if mod.advanced:
+            continue
+
+        if mod.exclusive and identifier in current_mods:
+            continue
+
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=identifier, mode="add", randomize=timestamp(), **kwargs),
+            title=pad_title(mod.description), summary=mod.long_description or ""
+        ))
+
+    fps_mod = SubtitleModifications.get_mod_class("change_FPS")
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleFPSModMenu, randomize=timestamp(), **kwargs),
+        title=pad_title(fps_mod.description), summary=fps_mod.long_description or ""
+    ))
+
+    shift_mod = SubtitleModifications.get_mod_class("shift_offset")
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs),
+        title=pad_title(shift_mod.description), summary=shift_mod.long_description or ""
+    ))
+
+    color_mod = SubtitleModifications.get_mod_class("color")
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleColorModMenu, randomize=timestamp(), **kwargs),
+        title=pad_title(color_mod.description), summary=color_mod.long_description or ""
+    ))
+
+    if current_mods:
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=None, mode="remove_last", randomize=timestamp(), **kwargs),
+            title=pad_title("Remove last applied mod (%s)" % current_mods[-1]),
+            summary=u"Currently applied mods: %s" % (", ".join(current_mods) if current_mods else "none")
+        ))
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleListMods, randomize=timestamp(), **kwargs),
+            title=pad_title("Manage applied mods"),
+            summary=u"Currently applied mods: %s" % (", ".join(current_mods))
+        ))
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleSetMods, mods=None, mode="clear", randomize=timestamp(), **kwargs),
+        title=pad_title("Restore original version"),
+        summary=u"Currently applied mods: %s" % (", ".join(current_mods) if current_mods else "none")
+    ))
+
+    return oc
+
+
+@route(PREFIX + '/item/sub_mod_fps/{rating_key}/{part_id}', force=bool)
+def SubtitleFPSModMenu(**kwargs):
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs["part_id"]
+    item_type = kwargs["item_type"]
+
+    kwargs.pop("randomize")
+
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
+        title="< Back to subtitle modification menu"
+    ))
+
+    metadata = get_plex_metadata(rating_key, part_id, item_type)
+    scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+    video, plex_part = scanned_parts.items()[0]
+
+    target_fps = plex_part.fps
+
+    for fps in ["23.976", "24.000", "25.000", "29.970", "30.000", "50.000", "59.940", "60.000"]:
+        if float(fps) == float(target_fps):
+            continue
+
+        if float(fps) > float(target_fps):
+            indicator = "subs constantly getting faster"
+        else:
+            indicator = "subs constantly getting slower"
+
+        mod_ident = SubtitleModifications.get_mod_signature("change_FPS", **{"from": fps, "to": target_fps})
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs),
+            title="%s fps -> %s fps (%s)" % (fps, target_fps, indicator)
+        ))
+
+    return oc
+
+
+POSSIBLE_UNITS = (("ms", "milliseconds"), ("s", "seconds"), ("m", "minutes"), ("h", "hours"))
+POSSIBLE_UNITS_D = dict(POSSIBLE_UNITS)
+
+
+@route(PREFIX + '/item/sub_mod_shift_unit/{rating_key}/{part_id}', force=bool)
+def SubtitleShiftModUnitMenu(**kwargs):
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    kwargs.pop("randomize")
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
+        title="< Back to subtitle modifications"
+    ))
+
+    for unit, title in POSSIBLE_UNITS:
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleShiftModMenu, unit=unit, randomize=timestamp(), **kwargs),
+            title="Adjust by %s" % title
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/item/sub_mod_shift/{rating_key}/{part_id}/{unit}', force=bool)
+def SubtitleShiftModMenu(unit=None, **kwargs):
+    if unit not in POSSIBLE_UNITS_D:
+        raise NotImplementedError
+
+    kwargs.pop("randomize")
+
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs),
+        title="< Back to unit selection"
+    ))
+
+    rng = []
+    if unit == "h":
+        rng = range(-10, 11)
+    elif unit in ("m", "s"):
+        rng = range(-15, 15)
+    elif unit == "ms":
+        rng = range(-900, 1000, 100)
+
+    for i in rng:
+        if i == 0:
+            continue
+
+        mod_ident = SubtitleModifications.get_mod_signature("shift_offset", **{unit: i})
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs),
+            title="%s %s" % (("%s" if i < 0 else "+%s") % i, unit)
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/item/sub_mod_colors/{rating_key}/{part_id}', force=bool)
+def SubtitleColorModMenu(**kwargs):
+    kwargs.pop("randomize")
+
+    color_mod = SubtitleModifications.get_mod_class("color")
+
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
+        title="< Back to subtitle modification menu"
+    ))
+
+    for color, code in color_mod.colors.iteritems():
+        mod_ident = SubtitleModifications.get_mod_signature("color", **{"name": color})
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs),
+            title="%s (%s)" % (color, code)
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/item/sub_set_mods/{rating_key}/{part_id}/{mods}/{mode}', force=bool)
+@debounce
+def SubtitleSetMods(mods=None, mode=None, **kwargs):
+    if not isinstance(mods, types.ListType) and mods:
+        mods = [mods]
+
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs["part_id"]
+    lang_a2 = kwargs["language"]
+    item_type = kwargs["item_type"]
+
+    language = Language.fromietf(lang_a2)
+
+    set_mods_for_part(rating_key, part_id, language, item_type, mods, mode=mode)
+
+    kwargs.pop("randomize")
+    return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
+
+
+@route(PREFIX + '/item/sub_list_mods/{rating_key}/{part_id}', force=bool)
+@debounce
+def SubtitleListMods(**kwargs):
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs["part_id"]
+    language = kwargs["language"]
+    current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
+
+    kwargs.pop("randomize")
+
+    oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
+
+    oc.add(DirectoryObject(
+        key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
+        title="< Back to subtitle modifications"
+    ))
+
+    for identifier in current_sub.mods:
+        oc.add(DirectoryObject(
+            key=Callback(SubtitleSetMods, mods=identifier, mode="remove", randomize=timestamp(), **kwargs),
+            title="Remove: %s" % identifier
+        ))
+
+    return oc

Contents/Code/support/__init__.py

@@ -18,7 +18,7 @@ sys.modules["support.plex_media"] = plex_media
 
 import localmedia
 
-sys.modules["subzero.localmedia"] = localmedia
+sys.modules["support.localmedia"] = localmedia
 
 import subtitlehelpers
 
@@ -32,9 +32,9 @@ import missing_subtitles
 
 sys.modules["support.missing_subtitles"] = missing_subtitles
 
-import background
+import scheduler
 
-sys.modules["support.background"] = background
+sys.modules["support.scheduler"] = scheduler
 
 import tasks
 
@@ -58,3 +58,6 @@ sys.modules["support.data"] = data
 
 import activities
 sys.modules["support.activities"] = activities
+
+import download
+sys.modules["support.download"] = download

Contents/Code/support/activities.py

@@ -11,9 +11,9 @@ class PlexActivityManager(object):
     def start(self):
         activity_sources_enabled = None
 
-        if config.universal_plex_token:
+        if config.plex_token:
             from plex import Plex
-            Plex.configuration.defaults.authentication(config.universal_plex_token)
+            Plex.configuration.defaults.authentication(config.plex_token)
             activity_sources_enabled = ["websocket"]
             Activity.on('websocket.playing', self.on_playing)
 
@@ -27,9 +27,6 @@ class PlexActivityManager(object):
 
     @throttle(5, instance_method=True)
     def on_playing(self, info):
-        if not config.use_activities:
-            return
-
         # ignore non-playing states and anything too far in
         if info["state"] != "playing" or info["viewOffset"] > 60000:
             return
@@ -41,19 +38,37 @@ class PlexActivityManager(object):
             return
 
         rating_key = info["ratingKey"]
-        if rating_key not in Dict["last_played_items"]:
-            # new playing; store last 10 recently played items
+
+        # only use integer based rating keys
+        try:
+            int(rating_key)
+        except ValueError:
+            return
+
+        if rating_key in Dict["last_played_items"] and rating_key != Dict["last_played_items"][0]:
+            # shift last played
+            Dict["last_played_items"].insert(0,
+                                             Dict["last_played_items"].pop(Dict["last_played_items"].index(rating_key)))
+            Dict.Save()
+
+        elif rating_key not in Dict["last_played_items"]:
+            # new playing; store last X recently played items
             Dict["last_played_items"].insert(0, rating_key)
-            Dict["last_played_items"] = Dict["last_played_items"][:10]
+            Dict["last_played_items"] = Dict["last_played_items"][:config.store_recently_played_amount]
 
             Dict.Save()
 
+            if not config.react_to_activities:
+                return
+
             debug_msg = "Started playing %s. Refreshing it." % rating_key
 
-            key_to_refresh = None
-            if config.activity_mode in ["refresh", "next_episode", "hybrid"]:
+            # todo: cleanup debug messages for hybrid-plus
+
+            keys_to_refresh = []
+            if config.activity_mode in ["refresh", "next_episode", "hybrid", "hybrid-plus"]:
                 # next episode or next episode and current movie
-                if config.activity_mode in ["next_episode", "hybrid"]:
+                if config.activity_mode in ["next_episode", "hybrid", "hybrid-plus"]:
                     plex_item = get_item(rating_key)
                     if not plex_item:
                         Log.Warn("Can't determine media type of %s, skipping" % rating_key)
@@ -61,20 +76,24 @@ class PlexActivityManager(object):
 
                     if get_item_kind_from_item(plex_item) == "episode":
                         next_ep = self.get_next_episode(rating_key)
+                        if config.activity_mode == "hybrid-plus":
+                            keys_to_refresh.append(rating_key)
                         if next_ep:
-                            key_to_refresh = next_ep.rating_key
+                            keys_to_refresh.append(next_ep.rating_key)
                             debug_msg = "Started playing %s. Refreshing next episode (%s, S%02iE%02i)." % \
                                         (rating_key, next_ep.rating_key, int(next_ep.season.index), int(next_ep.index))
 
                     else:
                         if config.activity_mode == "hybrid":
-                            key_to_refresh = rating_key
+                            keys_to_refresh.append(rating_key)
                 elif config.activity_mode == "refresh":
-                    key_to_refresh = rating_key
+                    keys_to_refresh.append(rating_key)
 
-                if key_to_refresh:
+                if keys_to_refresh:
                     Log.Debug(debug_msg)
-                    refresh_item(key_to_refresh)
+                    Log.Debug("Refreshing %s", keys_to_refresh)
+                    for key in keys_to_refresh:
+                        refresh_item(key)
 
     def get_next_episode(self, rating_key):
         plex_item = get_item(rating_key)
@@ -108,4 +127,5 @@ class PlexActivityManager(object):
                                 if ep.index == 1:
                                     return ep
 
+
 activity = PlexActivityManager()

Contents/Code/support/config.py

@@ -3,18 +3,23 @@
 import os
 import re
 import inspect
+import sys
 
 import datetime
 
 import subliminal
 import subliminal_patch
+
+from whichdb import whichdb
 from babelfish import Language
+from subliminal.cli import MutexLock
 from subzero.lib.io import FileIO, get_viable_encoding
-from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW
+from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW, MEDIA_TYPE_TO_STRING
 from lib import Plex
 from helpers import check_write_permissions, cast_bool
 
-SUBTITLE_EXTS = ['utf', 'utf8', 'utf-8', 'srt', 'smi', 'rt', 'ssa', 'aqt', 'jss', 'ass', 'idx', 'sub', 'txt', 'psb']
+SUBTITLE_EXTS = ['utf', 'utf8', 'utf-8', 'srt', 'smi', 'rt', 'ssa', 'aqt', 'jss', 'ass', 'idx', 'sub', 'txt', 'psb',
+                 'vtt']
 VIDEO_EXTS = ['3g2', '3gp', 'asf', 'asx', 'avc', 'avi', 'avs', 'bivx', 'bup', 'divx', 'dv', 'dvr-ms', 'evo', 'fli',
               'flv',
               'm2t', 'm2ts', 'm2v', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'mts', 'nsv', 'nuv', 'ogm', 'ogv', 'tp',
@@ -27,6 +32,8 @@ IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")
 VERSION_RE = re.compile(ur'CFBundleVersion.+?<string>([0-9\.]+)</string>', re.DOTALL)
 DEV_RE = re.compile(ur'PlexPluginDevMode.+?<string>([01]+)</string>', re.DOTALL)
 
+impawrt = getattr(sys.modules['__main__'], "__builtins__").get("__import__")
+
 
 def int_or_default(s, default):
     try:
@@ -45,7 +52,9 @@ class Config(object):
     data_path = None
     data_items_path = None
     universal_plex_token = None
+    plex_token = None
     is_development = False
+    dbm_supported = False
 
     enable_channel = True
     enable_agent = True
@@ -56,6 +65,7 @@ class Config(object):
     pin_valid_minutes = 10
     lang_list = None
     subtitle_destination_folder = None
+    subtitle_formats = None
     providers = None
     provider_settings = None
     max_recent_items_per_library = 200
@@ -67,14 +77,23 @@ class Config(object):
     notify_executable = None
     sections = None
     enabled_sections = None
-    enforce_encoding = False
+    remove_hi = False
+    fix_ocr = False
+    fix_common = False
+    colors = ""
     chmod = None
     forced_only = False
     exotic_ext = False
     treat_und_as_first = False
     ext_match_strictness = False
-    use_activities = False
+    default_mods = None
+    debug_mods = False
+    react_to_activities = False
     activity_mode = None
+    subtitles_save_to = None
+    no_refresh = False
+
+    store_recently_played_amount = 20
 
     initialized = False
 
@@ -89,6 +108,9 @@ class Config(object):
         self.data_path = getattr(Data, "_core").storage.data_path
         self.data_items_path = os.path.join(self.data_path, "DataItems")
         self.universal_plex_token = self.get_universal_plex_token()
+        self.plex_token = os.environ.get("PLEXTOKEN", self.universal_plex_token)
+
+        os.environ["SZ_USER_AGENT"] = self.get_user_agent()
 
         self.set_plugin_mode()
         self.set_plugin_lock()
@@ -96,6 +118,8 @@ class Config(object):
 
         self.lang_list = self.get_lang_list()
         self.subtitle_destination_folder = self.get_subtitle_destination_folder()
+        self.subtitle_formats = self.get_subtitle_formats()
+        self.forced_only = cast_bool(Prefs["subtitles.only_foreign"])
         self.providers = self.get_providers()
         self.provider_settings = self.get_provider_settings()
         self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 2000)
@@ -106,14 +130,59 @@ class Config(object):
         self.enabled_sections = self.check_enabled_sections()
         self.permissions_ok = self.check_permissions()
         self.notify_executable = self.check_notify_executable()
-        self.enforce_encoding = cast_bool(Prefs['subtitles.enforce_encoding'])
+        self.remove_hi = cast_bool(Prefs['subtitles.remove_hi'])
+        self.fix_ocr = cast_bool(Prefs['subtitles.fix_ocr'])
+        self.fix_common = cast_bool(Prefs['subtitles.fix_common'])
+        self.colors = Prefs['subtitles.colors'] if Prefs['subtitles.colors'] != "don't change" else None
         self.chmod = self.check_chmod()
-        self.forced_only = cast_bool(Prefs["subtitles.only_foreign"])
         self.exotic_ext = cast_bool(Prefs["subtitles.scan.exotic_ext"])
         self.treat_und_as_first = cast_bool(Prefs["subtitles.language.treat_und_as_first"])
         self.ext_match_strictness = self.determine_ext_sub_strictness()
+        self.default_mods = self.get_default_mods()
+        self.debug_mods = cast_bool(Prefs['log_debug_mods'])
+        self.subtitles_save_to = Prefs['subtitles.save.filesystem']
+        self.no_refresh = os.environ.get("SZ_NO_REFRESH", False)
         self.initialized = True
 
+    def init_cache(self):
+        names = ['dbhash', 'gdbm', 'dbm']
+        dbfn = None
+        self.dbm_supported = False
+
+        # try importing dbm modules
+        if impawrt:
+            for name in names:
+                try:
+                    impawrt(name)
+                except:
+                    continue
+                if not self.dbm_supported:
+                    self.dbm_supported = name
+                    break
+
+            if self.dbm_supported:
+                # anydbm checks; try guessing the format and importing the correct module
+                dbfn = os.path.join(config.data_items_path, 'subzero.dbm')
+                db_which = whichdb(dbfn)
+                if db_which is not None and db_which != "":
+                    try:
+                        impawrt(db_which)
+                    except ImportError:
+                        self.dbm_supported = False
+
+        if Core.runtime.os != "Windows" and self.dbm_supported:
+            try:
+                subliminal.region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
+                                            arguments={'filename': dbfn,
+                                                       'lock_factory': MutexLock})
+                Log.Info("Using file based cache!")
+                return
+            except:
+                self.dbm_supported = False
+
+        Log.Warn("Not using file based cache!")
+        subliminal.region.configure('dogpile.cache.memory')
+
     def set_log_paths(self):
         # find log handler
         for handler in Core.log.handlers:
@@ -138,7 +207,9 @@ class Config(object):
             except:
                 Log.Warn("Couldn't determine Plex Token")
         else:
-            Log("Did NOT find Preferences file - please check logfile and hierarchy. Aborting!")
+            Log("Did NOT find Preferences file - most likely Windows OS. Otherwise please check logfile and hierarchy.")
+
+        # fixme: windows
 
     def set_plugin_mode(self):
         if Prefs["plugin_mode"] == "only agent":
@@ -213,11 +284,17 @@ class Config(object):
         return all_permissions_ok
 
     def get_version(self):
+        return self.get_bare_version() + ("" if not self.is_development else " DEV")
+
+    def get_bare_version(self):
         result = VERSION_RE.search(self.plugin_info)
-        add = "" if not self.is_development else " DEV"
 
         if result:
-            return result.group(1) + add
+            return result.group(1)
+        return "2.x.x.x"
+
+    def get_user_agent(self):
+        return "Sub-Zero/%s" % (self.get_bare_version() + ("" if not self.is_development else "-dev"))
 
     def get_dev_mode(self):
         dev = DEV_RE.search(self.plugin_info)
@@ -270,7 +347,7 @@ class Config(object):
         self.enabled_sections = self.check_enabled_sections()
 
     def check_enabled_sections(self):
-        enabled_for_primary_agents = []
+        enabled_for_primary_agents = {"movie": [], "show": []}
         enabled_sections = {}
 
         # find which agents we're enabled for
@@ -283,11 +360,11 @@ class Config(object):
                     related_agents = Plex.primary_agent(agent.identifier, t.media_type)
                     for a in related_agents:
                         if a.identifier == PLUGIN_IDENTIFIER and a.enabled:
-                            enabled_for_primary_agents.append(agent.identifier)
+                            enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[t.media_type]].append(agent.identifier)
 
         # find the libraries that use them
         for library in self.sections:
-            if library.agent in enabled_for_primary_agents:
+            if library.agent in enabled_for_primary_agents.get(library.type, []):
                 enabled_sections[library.key] = library
 
         Log.Debug(u"I'm enabled for: %s" % [lib.title for key, lib in enabled_sections.iteritems()])
@@ -330,6 +407,15 @@ class Config(object):
         return fld_custom or (
             Prefs["subtitles.save.subFolder"] if Prefs["subtitles.save.subFolder"] != "current folder" else None)
 
+    def get_subtitle_formats(self):
+        formats = Prefs["subtitles.save.formats"]
+        out = []
+        if "SRT" in formats:
+            out.append("srt")
+        if "VTT" in formats:
+            out.append("vtt")
+        return out
+
     def get_providers(self):
         providers = {'opensubtitles': cast_bool(Prefs['provider.opensubtitles.enabled']),
                      # 'thesubdb': Prefs['provider.thesubdb.enabled'],
@@ -343,10 +429,13 @@ class Config(object):
                      }
 
         # ditch non-forced-subtitles-reporting providers
-        if cast_bool(Prefs['subtitles.only_foreign']):
+        if self.forced_only:
             providers["addic7ed"] = False
             providers["tvsubtitles"] = False
             providers["legendastv"] = False
+            providers["napiprojekt"] = False
+            providers["shooter"] = False
+            providers["subscenter"] = False
 
         return filter(lambda prov: providers[prov], providers)
 
@@ -404,17 +493,32 @@ class Config(object):
             return "loose"
         return "strict"
 
+    def get_default_mods(self):
+        mods = []
+        if self.remove_hi:
+            mods.append("remove_HI")
+        if self.fix_ocr:
+            mods.append("OCR_fixes")
+        if self.fix_common:
+            mods.append("common")
+        if self.colors:
+            mods.append("color(name=%s)" % self.colors)
+
+        return mods
+
     def set_activity_modes(self):
         val = Prefs["activity.on_playback"]
         if val == "never":
-            self.use_activities = False
+            self.react_to_activities = False
             return
 
-        self.use_activities = True
+        self.react_to_activities = True
         if val == "current media item":
             self.activity_mode = "refresh"
         elif val == "hybrid: current item or next episode":
             self.activity_mode = "hybrid"
+        elif val == "hybrid-plus: current item and next episode":
+            self.activity_mode = "hybrid-plus"
         else:
             self.activity_mode = "next_episode"
 

Contents/Code/support/download.py

@@ -0,0 +1,46 @@
+# coding=utf-8
+
+import subliminal_patch as subliminal
+
+from support.config import config
+from subtitlehelpers import get_subtitles_from_metadata
+from subliminal_patch import compute_score
+
+
+def download_best_subtitles(video_part_map, min_score=0):
+    hearing_impaired = Prefs['subtitles.search.hearingImpaired']
+    languages = config.lang_list
+    if not languages:
+        return
+
+    missing_languages = False
+    for video, part in video_part_map.iteritems():
+        if not Prefs['subtitles.save.filesystem']:
+            # scan for existing metadata subtitles
+            meta_subs = get_subtitles_from_metadata(part)
+            for language, subList in meta_subs.iteritems():
+                if subList:
+                    video.subtitle_languages.add(language)
+                    Log.Debug("Found metadata subtitle %s for %s", language, video)
+
+        missing_subs = (languages - video.subtitle_languages)
+
+        # all languages are found if we either really have subs for all languages or we only want to have exactly one language
+        # and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
+        found_one_which_is_enough = len(video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
+        if not missing_subs or found_one_which_is_enough:
+            if found_one_which_is_enough:
+                Log.Debug('Only one language was requested, and we\'ve got a subtitle for %s', video)
+            else:
+                Log.Debug('All languages %r exist for %s', languages, video)
+            continue
+        missing_languages = True
+        break
+
+    if missing_languages:
+        Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" % (min_score, hearing_impaired))
+
+        return subliminal.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
+                                                  provider_configs=config.provider_settings, pool_class=config.provider_pool,
+                                                  compute_score=compute_score)
+    Log.Debug("All languages for all requested videos exist. Doing nothing.")

Contents/Code/support/helpers.py

@@ -9,15 +9,26 @@ import time
 import re
 import platform
 import subprocess
-
-from bs4 import UnicodeDammit
+import sys
+from collections import OrderedDict
 
 import chardet
 
+from bs4 import UnicodeDammit
 from babelfish import Language
-
 from subzero.analytics import track_event
 
+mswindows = (sys.platform == "win32")
+if mswindows:
+    from subprocess import list2cmdline
+    quote_args = list2cmdline
+else:
+    # POSIX
+    from pipes import quote
+
+    def quote_args(seq):
+        return ' '.join(quote(arg) for arg in seq)
+
 # Unicode control characters can appear in ID3v2 tags but are not legal in XML.
 RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
                      u'|' + \
@@ -30,7 +41,7 @@ RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])'
 
 
 def cast_bool(value):
-    return str(value) in ("true", "True")
+    return str(value).strip() in ("true", "True")
 
 
 # A platform independent way to split paths which might come in with different separators.
@@ -110,9 +121,9 @@ def str_pad(s, length, align='left', pad_char=' ', trim=False):
         raise ValueError("Unknown align type, expected either 'left' or 'right'")
 
 
-def pad_title(value):
+def pad_title(value, width=49):
     """Pad a title to 30 characters to force the 'details' view."""
-    return str_pad(value, 49, pad_char=' ')
+    return str_pad(value, width, pad_char=' ')
 
 
 def get_plex_item_display_title(item, kind, parent=None, parent_title=None, section_title=None,
@@ -236,13 +247,13 @@ def get_item_hints(data):
     :param data: video item dict of media_to_videos 
     :return: 
     """
-    hints = {"title": data["title"], "type": "movie"}
+    hints = {"title": data["original_title"] or data["title"], "type": "movie"}
     if data["type"] == "episode":
         hints.update(
             {
                 "type": "episode",
                 "episode_title": data["title"],
-                "title": data["series"],
+                "title": data["original_title"] or data["series"],
             }
         )
     return hints
@@ -256,7 +267,7 @@ def notify_executable(exe_info, videos, subtitles, storage):
     exe, arguments = exe_info
     for video, video_subtitles in subtitles.items():
         for subtitle in video_subtitles:
-            lang = Locale.Language.Match(subtitle.language.alpha2)
+            lang = str(subtitle.language)
             data = video.plexapi_metadata.copy()
             data.update({
                 "subtitle_language": lang,
@@ -273,9 +284,21 @@ def notify_executable(exe_info, videos, subtitles, storage):
             prepared_arguments = [arg % prepared_data for arg in arguments]
 
             Log.Debug(u"Calling %s with arguments: %s" % (exe, prepared_arguments))
+            env = os.environ
+            if not mswindows:
+                env_path = {"PATH": os.pathsep.join(
+                                        [
+                                            "/usr/local/bin",
+                                            "/usr/bin",
+                                            os.environ.get("PATH", "")
+                                        ]
+                                    )
+                            }
+                env = dict(os.environ, **env_path)
+
             try:
-                output = subprocess.check_output(subprocess.list2cmdline([exe] + prepared_arguments),
-                                                 stderr=subprocess.STDOUT, shell=True)
+                output = subprocess.check_output(quote_args([exe] + prepared_arguments),
+                                                 stderr=subprocess.STDOUT, shell=True, env=env)
             except subprocess.CalledProcessError:
                 Log.Error(u"Calling %s failed: %s" % (exe, traceback.format_exc()))
             else:
@@ -286,6 +309,26 @@ def track_usage(category=None, action=None, label=None, value=None):
     if not cast_bool(Prefs["track_usage"]):
         return
 
+    if "last_tracked" not in Dict:
+        Dict["last_tracked"] = OrderedDict()
+        Dict.Save()
+
+    event_key = (category, action, label, value)
+    now = datetime.datetime.now()
+    if event_key in Dict["last_tracked"] and (Dict["last_tracked"][event_key] + datetime.timedelta(minutes=30)) < now:
+        return
+
+    Dict["last_tracked"][event_key] = now
+
+    # maintenance
+    for key, value in Dict["last_tracked"].copy().iteritems():
+        # kill day old values
+        if value < now - datetime.timedelta(days=1):
+            try:
+                del Dict["last_tracked"][key]
+            except:
+                pass
+
     Thread.Create(dispatch_track_usage, category, action, label, value,
                   identifier=Dict["anon_id"], first_use=Dict["first_use"],
                   add=Network.PublicAddress)
@@ -303,3 +346,7 @@ def dispatch_track_usage(*args, **kwargs):
 
 def get_language(lang_short):
     return Language.fromietf(lang_short)
+
+
+class PartUnknownException(Exception):
+    pass

Contents/Code/support/items.py

@@ -2,12 +2,15 @@
 
 import logging
 import re
+import traceback
 import types
 import os
 from ignore import ignore_list
-from helpers import is_recent, get_plex_item_display_title, query_plex
+from helpers import is_recent, get_plex_item_display_title, query_plex, PartUnknownException
 from lib import Plex, get_intent
 from config import config, IGNORE_FN
+from subliminal_patch.subtitle import ModifiedSubtitle
+from subzero.modification import registry as mod_registry, SubtitleModifications
 
 logger = logging.getLogger(__name__)
 
@@ -17,12 +20,16 @@ container_size_re = re.compile(ur'totalSize="(\d+)"')
 
 
 def get_item(key):
-    item_id = int(key)
+    try:
+        item_id = int(key)
+    except ValueError:
+        return
+
     item_container = Plex["library"].metadata(item_id)
 
     try:
         return list(item_container)[0]
-    except IndexError:
+    except:
         pass
 
 
@@ -40,11 +47,11 @@ PLEX_API_TYPE_MAP = {
 
 def get_item_kind_from_rating_key(key):
     item = get_item(key)
-    return PLEX_API_TYPE_MAP[get_item_kind(item)]
+    return PLEX_API_TYPE_MAP.get(get_item_kind(item))
 
 
 def get_item_kind_from_item(item):
-    return PLEX_API_TYPE_MAP[get_item_kind(item)]
+    return PLEX_API_TYPE_MAP.get(get_item_kind(item))
 
 
 def get_item_thumb(item):
@@ -164,14 +171,17 @@ def get_recent_items():
         "X-Plex-Container-Size": "%s" % config.max_recent_items_per_library
     }
 
-    episode_re = re.compile(ur'ratingKey="(?P<key>\d+)"'
+    episode_re = re.compile(ur'(?su)ratingKey="(?P<key>\d+)"'
                             ur'.+?grandparentRatingKey="(?P<parent_key>\d+)"'
                             ur'.+?title="(?P<title>.*?)"'
                             ur'.+?grandparentTitle="(?P<parent_title>.*?)"'
                             ur'.+?index="(?P<episode>\d+?)"'
-                            ur'.+?parentIndex="(?P<season>\d+?)".+?addedAt="(?P<added>\d+)"')
-    movie_re = re.compile(ur'ratingKey="(?P<key>\d+)".+?title="(?P<title>.*?)".+?addedAt="(?P<added>\d+)"')
-    available_keys = ("key", "title", "parent_key", "parent_title", "season", "episode", "added")
+                            ur'.+?parentIndex="(?P<season>\d+?)".+?addedAt="(?P<added>\d+)"'
+                            ur'.+?<Part.+? file="(?P<filename>[^"]+?)"')
+    movie_re = re.compile(ur'(?su)ratingKey="(?P<key>\d+)".+?title="(?P<title>.*?)'
+                          ur'".+?addedAt="(?P<added>\d+)"'
+                          ur'.+?<Part.+? file="(?P<filename>[^"]+?)"')
+    available_keys = ("key", "title", "parent_key", "parent_title", "season", "episode", "added", "filename")
     recent = []
 
     for section in Plex["library"].sections():
@@ -182,8 +192,10 @@ def get_recent_items():
             continue
 
         use_args = args.copy()
+        plex_item_type = "Movie"
         if section.type == "show":
             use_args["type"] = "4"
+            plex_item_type = "Episode"
 
         url = "http://127.0.0.1:32400/library/sections/%s/all" % int(section.key)
         response = query_plex(url, use_args)
@@ -198,6 +210,10 @@ def get_recent_items():
             if data["key"] in ignore_list.videos:
                 Log.Debug(u"Skipping item: %s" % data["title"])
                 continue
+            if is_physically_ignored(data["filename"], plex_item_type):
+                Log.Debug(u"Skipping item: %s" % data["title"])
+                continue
+
             if is_recent(int(data["added"])):
                 recent.append((int(data["added"]), section.type, section.title, data["key"]))
 
@@ -242,6 +258,16 @@ def is_ignored(rating_key, item=None):
         return True
 
     # physical/path ignore
+    if config.ignore_sz_files or config.ignore_paths:
+        for media in item.media:
+            for part in media.parts:
+                if is_physically_ignored(part.file, kind):
+                    return True
+
+    return False
+
+
+def is_physically_ignored(fn, kind):
     if config.ignore_sz_files or config.ignore_paths:
         # normally check current item folder and the library
         check_ignore_paths = [".", "../"]
@@ -249,18 +275,15 @@ def is_ignored(rating_key, item=None):
             # series/episode, we've got a season folder here, also
             check_ignore_paths.append("../../")
 
-        for part in item.media.parts:
-            if config.ignore_paths and config.is_path_ignored(part.file):
-                Log.Debug("Item %s's path is manually ignored" % rating_key)
-                return True
+        if config.ignore_paths and config.is_path_ignored(fn):
+            Log.Debug("Item %s's path is manually ignored" % fn)
+            return True
 
-            if config.ignore_sz_files:
-                for sub_path in check_ignore_paths:
-                    if config.is_physically_ignored(os.path.abspath(os.path.join(os.path.dirname(part.file), sub_path))):
-                        Log.Debug("An ignore file exists in either the items or its parent folders")
-                        return True
-
-    return False
+        if config.ignore_sz_files:
+            for sub_path in check_ignore_paths:
+                if config.is_physically_ignored(os.path.normpath(os.path.join(os.path.dirname(fn), sub_path))):
+                    Log.Debug("An ignore file exists in either the items or its parent folders")
+                    return True
 
 
 def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, parent_rating_key=None):
@@ -283,3 +306,81 @@ def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, paren
     for key in refresh:
         Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", key)
         Plex["library/metadata"].refresh(key)
+
+
+def get_current_sub(rating_key, part_id, language):
+    from support.storage import get_subtitle_storage
+
+    item = get_item(rating_key)
+    subtitle_storage = get_subtitle_storage()
+    stored_subs = subtitle_storage.load_or_new(item)
+    current_sub = stored_subs.get_any(part_id, language)
+    return current_sub, stored_subs, subtitle_storage
+
+
+def set_mods_for_part(rating_key, part_id, language, item_type, mods, mode="add"):
+    from support.plex_media import get_plex_metadata, scan_videos
+    from support.storage import save_subtitles
+
+    current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
+    if mode == "add":
+        for mod in mods:
+            identifier, args = SubtitleModifications.parse_identifier(mod)
+            mod_class = SubtitleModifications.get_mod_class(identifier)
+
+            if identifier not in mod_registry.mods_available:
+                raise NotImplementedError("Mod unknown or not registered")
+
+            # clean exclusive mods
+            if mod_class.exclusive and current_sub.mods:
+                for current_mod in current_sub.mods[:]:
+                    if current_mod.startswith(identifier):
+                        current_sub.mods.remove(current_mod)
+                        Log.Info("Removing superseded mod %s" % current_mod)
+
+            current_sub.add_mod(mod)
+    elif mode == "clear":
+        current_sub.add_mod(None)
+    elif mode == "remove":
+        for mod in mods:
+            current_sub.mods.remove(mod)
+
+    elif mode == "remove_last":
+        if current_sub.mods:
+            current_sub.mods.pop()
+    else:
+        raise NotImplementedError("Wrong mode given")
+    storage.save(stored_subs)
+
+    try:
+        metadata = get_plex_metadata(rating_key, part_id, item_type)
+    except PartUnknownException:
+        return
+
+    scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True,
+                                no_refining=True)
+    video, plex_part = scanned_parts.items()[0]
+
+    subtitle = ModifiedSubtitle(language, mods=current_sub.mods)
+    subtitle.content = current_sub.content
+    if current_sub.encoding:
+        # thanks plex
+        setattr(subtitle, "_guessed_encoding", current_sub.encoding)
+
+        if current_sub.encoding != "utf-8":
+            subtitle.set_encoding("utf-8")
+            current_sub.content = subtitle.content
+            current_sub.encoding = "utf-8"
+            storage.save(stored_subs)
+
+    subtitle.plex_media_fps = plex_part.fps
+    subtitle.page_link = "modify subtitles with: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
+    subtitle.language = language
+    subtitle.id = current_sub.id
+
+    try:
+        save_subtitles(scanned_parts, {video: [subtitle]}, mode="m", bare_save=True)
+        Log.Debug("Modified %s subtitle for: %s:%s with: %s", language.name, rating_key, part_id,
+                  ", ".join(current_sub.mods) if current_sub.mods else "none")
+    except:
+        Log.Error("Something went wrong when modifying subtitle: %s", traceback.format_exc())

Contents/Code/support/localmedia.py

@@ -108,7 +108,8 @@ def find_subtitles(part):
                     if ext.lower()[1:] in config.SUBTITLE_EXTS:
                         # get fn without forced/default/normal tag
                         split_tag = root.rsplit(".", 1)
-                        if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default']:
+                        if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default', 'embedded',
+                                                                           'custom']:
                             root = split_tag[0]
 
                         # get associated media file name without language
@@ -160,9 +161,8 @@ def find_subtitles(part):
         # determine whether to pick up the subtitle based on our match strictness
         elif not filename_matches_part:
             if sz_config.ext_match_strictness == "strict" or (
-                    sz_config.ext_match_strictness == "loose" and not filename_contains_part):
-
-                #Log.Debug("%s doesn't match %s, skipping" % (helpers.unicodize(local_filename),
+                            sz_config.ext_match_strictness == "loose" and not filename_contains_part):
+                # Log.Debug("%s doesn't match %s, skipping" % (helpers.unicodize(local_filename),
                 #                                             helpers.unicodize(part_basename)))
                 continue
 

Contents/Code/support/missing_subtitles.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 import traceback
+import time
 
 from support.config import config
 from support.helpers import get_plex_item_display_title, cast_bool
@@ -8,8 +9,6 @@ from support.lib import Plex
 
 
 def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
-    existing_subs = {"internal": [], "external": [], "count": 0}
-
     item_id = int(rating_key)
     item = get_item(rating_key)
 
@@ -18,36 +17,41 @@ def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_t
     else:
         item_title = get_plex_item_display_title(item, kind, section_title=section_title)
 
-    video = item.media
+    missing = set()
+    languages_set = set(languages)
+    for media in item.media:
+        existing_subs = {"internal": [], "external": [], "count": 0}
+        for part in media.parts:
+            for stream in part.streams:
+                if stream.stream_type == 3:
+                    if stream.index:
+                        key = "internal"
+                    else:
+                        key = "external"
 
-    for part in video.parts:
-        for stream in part.streams:
-            if stream.stream_type == 3:
-                if stream.index:
-                    key = "internal"
-                else:
-                    key = "external"
+                    existing_subs[key].append(Locale.Language.Match(stream.language_code or ""))
+                    existing_subs["count"] = existing_subs["count"] + 1
 
-                existing_subs[key].append(Locale.Language.Match(stream.language_code or ""))
-                existing_subs["count"] = existing_subs["count"] + 1
+        missing_from_part = set(languages_set)
+        if existing_subs["count"]:
+            existing_flat = set((existing_subs["internal"] if internal else []) + (existing_subs["external"] if external else []))
+            if languages_set.issubset(existing_flat) or (len(existing_flat) >= 1 and Prefs['subtitles.only_one']):
+                # all subs found
+                #Log.Info(u"All subtitles exist for '%s'", item_title)
+                continue
 
-    missing = languages
-    if existing_subs["count"]:
-        existing_flat = (existing_subs["internal"] if internal else []) + (existing_subs["external"] if external else [])
-        languages_set = set(languages)
-        if languages_set.issubset(existing_flat) or (len(existing_flat) >= 1 and Prefs['subtitles.only_one']):
-            # all subs found
-            Log.Info(u"All subtitles exist for '%s'", item_title)
-            return
+            missing_from_part = languages_set - existing_flat
 
-        missing = languages_set - set(existing_flat)
-        Log.Info(u"Subs still missing for '%s': %s", item_title, missing)
+        if missing_from_part:
+            Log.Info(u"Subs still missing for '%s' (%s: %s): %s", item_title, rating_key, media.id,
+                     missing_from_part)
+            missing.update(missing_from_part)
 
     if missing:
         return added_at, item_id, item_title, item, missing
 
 
-def items_get_all_missing_subs(items):
+def items_get_all_missing_subs(items, sleep_after_request=False):
     missing = []
     for added_at, kind, section_title, key in items:
         try:
@@ -65,13 +69,13 @@ def items_get_all_missing_subs(items):
                 missing.append(state)
         except:
             Log.Error("Something went wrong when getting the state of item %s: %s", key, traceback.format_exc())
+        if sleep_after_request:
+            time.sleep(sleep_after_request)
     return missing
 
 
 def refresh_item(item):
-    Plex["library/metadata"].refresh(item)
+    if not config.no_refresh:
+        Plex["library/metadata"].refresh(item)
 
 
-def refresh_items(items):
-    for item, title in items:
-        refresh_item(item)

Contents/Code/support/plex_media.py

@@ -1,15 +1,14 @@
 # coding=utf-8
 
 import os
+from urllib2 import URLError
 
 import helpers
-
+from config import config
 from items import get_item
 from lib import get_intent, Plex
-from config import config
 from subzero.video import parse_video
 
-
 def get_metadata_dict(item, part, add):
     data = {
         "item": item,
@@ -22,6 +21,54 @@ def get_metadata_dict(item, part, add):
     return data
 
 
+imdb_guid_identifier = "com.plexapp.agents.imdb://"
+tvdb_guid_identifier = "com.plexapp.agents.thetvdb://"
+
+
+def get_plexapi_stream_info(plex_item, part_id=None):
+    d = {"stream": {}}
+    data = d["stream"]
+
+    # find current part
+    current_part = None
+    current_media = None
+    for media in plex_item.media:
+        for part in media.parts:
+            if not part_id or str(part.id) == part_id:
+                current_part = part
+                current_media = media
+                break
+        if current_part:
+            break
+
+    if not current_part:
+        return d
+
+    data["video_codec"] = current_media.video_codec
+    data["audio_codec"] = current_media.audio_codec.upper()
+
+    if data["audio_codec"] == "DCA":
+        data["audio_codec"] = "DTS"
+
+    if current_media.audio_channels == 8:
+        data["audio_channels"] = "7.1"
+
+    elif current_media.audio_channels == 6:
+        data["audio_channels"] = "5.1"
+    else:
+        data["audio_channels"] = "%s.0" % str(current_media.audio_channels)
+
+    # iter streams
+    for stream in current_part.streams:
+        if stream.stream_type == 1:
+            # video stream
+            data["resolution"] = "%s%s" % (current_media.video_resolution,
+                                           "i" if stream.scan_type != "progressive" else "p")
+            break
+
+    return d
+
+
 def media_to_videos(media, kind="series"):
     """
     iterates through media and returns the associated parts (videos)
@@ -31,36 +78,61 @@ def media_to_videos(media, kind="series"):
     """
     videos = []
 
+    # this is a Show or a Movie object
+    plex_item = get_item(media.id)
+    year = plex_item.year
+    original_title = plex_item.title_original
+
     if kind == "series":
         for season in media.seasons:
             season_object = media.seasons[season]
             for episode in media.seasons[season].episodes:
                 ep = media.seasons[season].episodes[episode]
 
+                tvdb_id = None
+                series_tvdb_id = None
+                if tvdb_guid_identifier in ep.guid:
+                    tvdb_id = ep.guid[len(tvdb_guid_identifier):].split("?")[0]
+                    series_tvdb_id = tvdb_id.split("/")[0]
+
                 # get plex item via API for additional metadata
                 plex_episode = get_item(ep.id)
+                stream_info = get_plexapi_stream_info(plex_episode)
 
                 for item in media.seasons[season].episodes[episode].items:
                     for part in item.parts:
                         videos.append(
                             get_metadata_dict(plex_episode, part,
-                                              {"plex_part": part, "type": "episode", "title": ep.title,
-                                               "series": media.title, "id": ep.id,
-                                               "series_id": media.id, "season_id": season_object.id,
-                                               "episode": plex_episode.index, "season": plex_episode.season.index,
-                                               "section": plex_episode.section.title
-                                               })
+                                              dict(stream_info, **{"plex_part": part, "type": "episode",
+                                                                    "title": ep.title,
+                                                                    "series": media.title, "id": ep.id, "year": year,
+                                                                    "series_id": media.id,
+                                                                    "season_id": season_object.id,
+                                                                    "imdb_id": None, "series_tvdb_id": series_tvdb_id,
+                                                                    "tvdb_id": tvdb_id,
+                                                                    "original_title": original_title,
+                                                                    "episode": plex_episode.index,
+                                                                    "season": plex_episode.season.index,
+                                                                    "section": plex_episode.section.title
+                                                                    })
+                                              )
                         )
     else:
-        plex_item = get_item(media.id)
+        stream_info = get_plexapi_stream_info(plex_item)
+        imdb_id = None
+        if imdb_guid_identifier in media.guid:
+            imdb_id = media.guid[len(imdb_guid_identifier):].split("?")[0]
         for item in media.items:
             for part in item.parts:
                 videos.append(
-                    get_metadata_dict(plex_item, part, {"plex_part": part, "type": "movie",
-                                                        "title": media.title, "id": media.id,
-                                                        "series_id": None,
-                                                        "season_id": None,
-                                                        "section": plex_item.section.title})
+                    get_metadata_dict(plex_item, part, dict(stream_info, **{"plex_part": part, "type": "movie",
+                                                                             "title": media.title, "id": media.id,
+                                                                             "series_id": None, "year": year,
+                                                                             "season_id": None, "imdb_id": imdb_id,
+                                                                             "original_title": original_title,
+                                                                             "series_tvdb_id": None, "tvdb_id": None,
+                                                                             "section": plex_item.section.title})
+                                      )
                 )
     return videos
 
@@ -92,10 +164,10 @@ def get_media_item_ids(media, kind="series"):
     return ids
 
 
-def scan_video(plex_part, ignore_all=False, hints=None, rating_key=None):
+def scan_video(pms_video_info, ignore_all=False, hints=None, rating_key=None, no_refining=False):
     """
     returnes a subliminal/guessit-refined parsed video
-    :param plex_part: 
+    :param pms_video_info: 
     :param ignore_all: 
     :param hints: 
     :param rating_key: 
@@ -104,14 +176,19 @@ def scan_video(plex_part, ignore_all=False, hints=None, rating_key=None):
     embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
     external_subtitles = not ignore_all and Prefs['subtitles.scan.external']
 
+    plex_part = pms_video_info["plex_part"]
+
     if ignore_all:
         Log.Debug("Force refresh intended.")
 
-    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (
+    Log.Debug("Scanning video: %s, external_subtitles=%s, embedded_subtitles=%s" % (
         plex_part.file, external_subtitles, embedded_subtitles))
 
     known_embedded = []
-    parts = list(Plex["library"].metadata(rating_key))[0].media.parts
+    parts = []
+    for media in list(Plex["library"].metadata(rating_key))[0].media:
+        parts += media.parts
+
     plexpy_part = None
     for part in parts:
         if int(part.id) == int(plex_part.id):
@@ -139,17 +216,19 @@ def scan_video(plex_part, ignore_all=False, hints=None, rating_key=None):
 
     try:
         # get basic video info scan (filename)
-        video = parse_video(plex_part.file, hints, external_subtitles=external_subtitles,
+        video = parse_video(plex_part.file, pms_video_info, hints, external_subtitles=external_subtitles,
                             embedded_subtitles=embedded_subtitles, known_embedded=known_embedded,
-                            forced_only=config.forced_only, video_fps=plex_part.fps)
+                            forced_only=config.forced_only, no_refining=no_refining)
 
+        # add video fps info
+        video.fps = plex_part.fps
         return video
 
     except ValueError:
         Log.Warn("File could not be guessed by subliminal: %s" % plex_part.file)
 
 
-def scan_videos(videos, kind="series", ignore_all=False):
+def scan_videos(videos, kind="series", ignore_all=False, no_refining=False):
     """
     receives a list of videos containing dictionaries returned by media_to_videos
     :param videos:
@@ -165,8 +244,8 @@ def scan_videos(videos, kind="series", ignore_all=False):
 
         hints = helpers.get_item_hints(video)
         video["plex_part"].fps = get_stream_fps(video["plex_part"].streams)
-        scanned_video = scan_video(video["plex_part"], ignore_all=force_refresh or ignore_all, hints=hints,
-                                   rating_key=video["id"])
+        scanned_video = scan_video(video, ignore_all=force_refresh or ignore_all, hints=hints,
+                                   rating_key=video["id"], no_refining=no_refining)
 
         if not scanned_video:
             continue
@@ -179,49 +258,79 @@ def scan_videos(videos, kind="series", ignore_all=False):
     return ret
 
 
-class PartUnknownException(Exception):
-    pass
-
-
-def get_plex_metadata(rating_key, part_id, item_type):
+def get_plex_metadata(rating_key, part_id, item_type, plex_item=None):
     """
     uses the Plex 3rd party API accessor to get metadata information
 
-    :param rating_key:
+    :param rating_key: movie or episode
     :param part_id:
     :param item_type:
     :return:
     """
 
-    plex_item = list(Plex["library"].metadata(rating_key))[0]
+    if not plex_item:
+        plex_item = get_item(rating_key)
+
+    if not plex_item:
+        return
 
     # find current part
     current_part = None
-    for part in plex_item.media.parts:
-        if str(part.id) == part_id:
-            current_part = part
+    for media in plex_item.media:
+        for part in media.parts:
+            if str(part.id) == str(part_id):
+                current_part = part
 
     if not current_part:
-        raise PartUnknownException("Part unknown")
+        raise helpers.PartUnknownException("Part unknown")
+
+    stream_info = get_plexapi_stream_info(plex_item, part_id)
 
     # get normalized metadata
+    # fixme: duplicated logic of media_to_videos
     if item_type == "episode":
+        show = list(Plex["library"].metadata(plex_item.show.rating_key))[0]
+        year = show.year
+        tvdb_id = None
+        series_tvdb_id = None
+        original_title = show.title_original
+        if tvdb_guid_identifier in plex_item.guid:
+            tvdb_id = plex_item.guid[len(tvdb_guid_identifier):].split("?")[0]
+            series_tvdb_id = tvdb_id.split("/")[0]
         metadata = get_metadata_dict(plex_item, current_part,
-                                     {"plex_part": current_part, "type": "episode", "title": plex_item.title,
-                                      "series": plex_item.show.title, "id": plex_item.rating_key,
-                                      "series_id": plex_item.show.rating_key,
-                                      "season_id": plex_item.season.rating_key,
-                                      "season": plex_item.season.index,
-                                      "episode": plex_item.index
-                                      })
+                                     dict(stream_info,
+                                          **{"plex_part": current_part, "type": "episode", "title": plex_item.title,
+                                             "series": plex_item.show.title, "id": plex_item.rating_key,
+                                             "series_id": plex_item.show.rating_key,
+                                             "season_id": plex_item.season.rating_key,
+                                             "imdb_id": None,
+                                             "year": year,
+                                             "tvdb_id": tvdb_id,
+                                             "series_tvdb_id": series_tvdb_id,
+                                             "original_title": original_title,
+                                             "season": plex_item.season.index,
+                                             "episode": plex_item.index
+                                             })
+                                     )
     else:
-        metadata = get_metadata_dict(plex_item, current_part, {"plex_part": current_part, "type": "movie",
-                                                               "title": plex_item.title, "id": plex_item.rating_key,
-                                                               "series_id": None,
-                                                               "season_id": None,
-                                                               "season": None,
-                                                               "episode": None,
-                                                               "section": plex_item.section.title})
+        imdb_id = None
+        original_title = plex_item.title_original
+        if imdb_guid_identifier in plex_item.guid:
+            imdb_id = plex_item.guid[len(imdb_guid_identifier):].split("?")[0]
+        metadata = get_metadata_dict(plex_item, current_part,
+                                     dict(stream_info, **{"plex_part": current_part, "type": "movie",
+                                                           "title": plex_item.title, "id": plex_item.rating_key,
+                                                           "series_id": None,
+                                                           "season_id": None,
+                                                           "imdb_id": imdb_id,
+                                                           "year": plex_item.year,
+                                                           "tvdb_id": None,
+                                                           "series_tvdb_id": None,
+                                                           "original_title": original_title,
+                                                           "season": None,
+                                                           "episode": None,
+                                                           "section": plex_item.section.title})
+                                     )
     return metadata
 
 
@@ -257,3 +366,24 @@ class PMSMediaProxy(object):
                 break
 
             m = m.children[0]
+
+    def get_all_parts(self):
+        """
+        walk the mediatree until the given part was found; if no part was given, return the first one
+        :param part_id:
+        :return:
+        """
+        m = self.mediatree
+        parts = []
+        while 1:
+            if m.items:
+                media_item = m.items[0]
+                for part in media_item.parts:
+                    parts.append(part)
+                break
+
+            if not m.children:
+                break
+
+            m = m.children[0]
+        return parts

Contents/Code/support/scheduler.py

@@ -168,6 +168,7 @@ class DefaultScheduler(object):
                 for args, kwargs in queue:
                     Log.Debug("Dispatching single task: %s, %s", args, kwargs)
                     Thread.Create(self.run_task, True, *args, **kwargs)
+                    Thread.Sleep(5.0)
 
             # scheduled tasks
             for name, info in self.tasks.iteritems():
@@ -185,9 +186,13 @@ class DefaultScheduler(object):
                     continue
 
                 if not task.last_run or (task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now):
-                    self.run_task(name)
+                    # fixme: scheduled tasks run synchronously. is this the best idea?
+                    Thread.Create(self.run_task, True, name)
+                    #Thread.Sleep(5.0)
+                    #self.run_task(name)
+                    Thread.Sleep(5.0)
 
-            Thread.Sleep(5.0)
+            Thread.Sleep(1)
 
 
 scheduler = DefaultScheduler()

Contents/Code/support/storage.py

@@ -5,61 +5,24 @@ import os
 import pprint
 import copy
 
-import subliminal
-from items import get_item
+from subliminal_patch.core import save_subtitles as subliminal_save_subtitles
 from subzero.subtitle_storage import StoredSubtitlesManager
 
 from subtitlehelpers import force_utf8
 from config import config
 from helpers import notify_executable, get_title_for_video_metadata, cast_bool, force_unicode
 from plex_media import PMSMediaProxy
+from support.items import get_item
 
 
-get_subtitle_storage = lambda: StoredSubtitlesManager(Data, get_item)
-
-
-def whack_missing_parts(scanned_video_part_map, existing_parts=None):
-    """
-    cleans out our internal storage's video parts (parts may get updated/deleted/whatever)
-    :param existing_parts: optional list of part ids known
-    :param scanned_video_part_map: videos to check for
-    :return:
-    """
-    # shortcut
-
-    if "subs" not in Dict:
-        return
-
-    if not existing_parts:
-        existing_parts = []
-        for part in scanned_video_part_map.viewvalues():
-            existing_parts.append(str(part.id))
-
-    whacked_parts = False
-    for video in scanned_video_part_map.keys():
-        video_id = str(video.id)
-        if video_id not in Dict["subs"]:
-            continue
-
-        parts = Dict["subs"][video_id].keys()
-
-        for part_id in parts:
-            part_id = str(part_id)
-            if part_id not in existing_parts:
-                Log.Info("Whacking part %s in internal storage of video %s (%s, %s)", part_id, video_id,
-                         repr(existing_parts), repr(parts))
-                del Dict["subs"][video_id][part_id]
-                whacked_parts = True
-
-    if whacked_parts:
-        Dict.Save()
+def get_subtitle_storage():
+    return StoredSubtitlesManager(Data, get_item)
 
 
 def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_type, mode="a"):
     """
     stores information about downloaded subtitles in plex's Dict()
     """
-    existing_parts = []
     for video, video_subtitles in downloaded_subtitles.items():
         part = scanned_video_part_map[video]
         part_id = str(part.id)
@@ -71,27 +34,21 @@ def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_ty
         subtitle_storage = get_subtitle_storage()
         stored_subs = subtitle_storage.load_or_new(plex_item)
 
-        existing_parts.append(part_id)
-
-        stored_any = False
         for subtitle in video_subtitles:
-            lang = Locale.Language.Match(subtitle.language.alpha2)
-            Log.Debug(u"Adding subtitle to storage: %s, %s, %s" % (video_id, part_id, title))
+            lang = str(subtitle.language)
+            subtitle.set_encoding("utf-8")
+            Log.Debug(u"Adding subtitle to storage: %s, %s, %s, %s" % (video_id, part_id, title,
+                                                                       subtitle.guess_encoding()))
             ret_val = stored_subs.add(part_id, lang, subtitle, storage_type, mode=mode)
 
             if ret_val:
                 Log.Debug("Subtitle stored")
-                stored_any = True
 
             else:
                 Log.Debug("Subtitle already existing in storage")
 
-        if stored_any:
-            Log.Debug("Saving subtitle storage for %s" % video_id)
-            subtitle_storage.save(stored_subs)
-
-    #if existing_parts:
-    #    whack_missing_parts(scanned_video_part_map, existing_parts=existing_parts)
+        Log.Debug("Saving subtitle storage for %s" % video_id)
+        subtitle_storage.save(stored_subs)
 
 
 def reset_storage(key):
@@ -107,6 +64,8 @@ def reset_storage(key):
 
 
 def log_storage(key):
+    if not key:
+        Log.Debug(pprint.pformat(getattr(Dict, "_dict")))
     if key in Dict:
         Log.Debug(pprint.pformat(Dict[key]))
 
@@ -134,9 +93,9 @@ def save_subtitles_to_file(subtitles):
             fld = force_unicode(fld)
             if not os.path.exists(fld):
                 os.makedirs(fld)
-        subliminal.save_subtitles(video, video_subtitles, directory=fld, single=cast_bool(Prefs['subtitles.only_one']),
-                                      encode_with=force_utf8 if config.enforce_encoding else None,
-                                      chmod=config.chmod, forced_tag=config.forced_only, path_decoder=force_unicode)
+        subliminal_save_subtitles(video, video_subtitles, directory=fld, single=cast_bool(Prefs['subtitles.only_one']),
+                                  chmod=config.chmod, forced_tag=config.forced_only, path_decoder=force_unicode,
+                                  debug_mods=config.debug_mods, formats=config.subtitle_formats)
     return True
 
 
@@ -144,7 +103,7 @@ def save_subtitles_to_metadata(videos, subtitles):
     for video, video_subtitles in subtitles.items():
         mediaPart = videos[video]
         for subtitle in video_subtitles:
-            content = force_utf8(subtitle.text) if config.enforce_encoding else subtitle.content
+            content = subtitle.get_modified_content(debug=config.debug_mods)
 
             if not isinstance(mediaPart, Framework.api.agentkit.MediaPart):
                 # we're being handed a Plex.py model instance here, not an internal PMS MediaPart object.
@@ -156,9 +115,29 @@ def save_subtitles_to_metadata(videos, subtitles):
     return True
 
 
-def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
+def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_save=False, mods=None):
+    """
+     
+    :param scanned_video_part_map: 
+    :param downloaded_subtitles: 
+    :param mode: 
+    :param bare_save: don't trigger anything; don't store information
+    :param mods: enabled mods
+    :return: 
+    """
     meta_fallback = False
     save_successful = False
+
+    if mods:
+        for video, video_subtitles in downloaded_subtitles.items():
+            if not video_subtitles:
+                continue
+
+            for subtitle in video_subtitles:
+                Log.Info("Applying mods: %s to %s", mods, subtitle)
+                subtitle.mods = mods
+                subtitle.plex_media_fps = video.fps
+
     storage = "metadata"
     if Prefs['subtitles.save.filesystem']:
         storage = "filesystem"
@@ -180,8 +159,11 @@ def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
             Log.Debug("Using metadata as subtitle storage")
         save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles)
 
-    if save_successful and config.notify_executable:
+    if not bare_save and save_successful and config.notify_executable:
         notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage)
 
-    store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)
+    if not bare_save and save_successful:
+        store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)
+
+    return save_successful
 

Contents/Code/support/subtitlehelpers.py

@@ -86,7 +86,7 @@ class VobSubSubtitleHelper(SubtitleHelper):
 
 
 IETF_MATCH = ".+\.([^-.]+)(?:-[A-Za-z]+)?$"
-ENDSWITH_LANGUAGECODE_RE = re.compile("\.([^-.]{2,3})(?:-[A-Za-z]{2})?$")
+ENDSWITH_LANGUAGECODE_RE = re.compile("\.([^-.]{2,3})(?:-[A-Za-z]{2,})?$")
 
 
 def match_ietf_language(s):
@@ -129,13 +129,12 @@ class DefaultSubtitleHelper(SubtitleHelper):
                 default = '1'
 
         # Attempt to extract the language from the filename (e.g. Avatar (2009).eng)
-        language = ""
-
-        # IETF support thanks to https://github.com/hpsbranco/LocalMedia.bundle/commit/4fad9aefedece78a1fa96401304351347f644369
+        # IETF support thanks to
+        # https://github.com/hpsbranco/LocalMedia.bundle/commit/4fad9aefedece78a1fa96401304351347f644369
         language = Locale.Language.Match(match_ietf_language(file))
 
         # skip non-SRT if wanted
-        if not helpers.cast_bool(Prefs["subtitles.scan.exotic_ext"]) and ext not in ["srt", "ass", "ssa"]:
+        if not helpers.cast_bool(Prefs["subtitles.scan.exotic_ext"]) and ext not in ["srt", "ass", "ssa", "vtt"]:
             return lang_sub_map
 
         codec = None
@@ -158,7 +157,7 @@ class DefaultSubtitleHelper(SubtitleHelper):
                 Log("An error occurred while attempting to parse the subtitle file, skipping... : " + self.filename)
                 return lang_sub_map
 
-        if codec is None and ext in ['ass', 'ssa', 'smi', 'srt', 'psb']:
+        if codec is None and ext in ['ass', 'ssa', 'smi', 'srt', 'psb', 'vtt']:
             codec = ext.replace('ass', 'ssa')
 
         if format is None:
@@ -194,7 +193,10 @@ def get_subtitles_from_metadata(part):
 def force_utf8(content):
     a = UnicodeDammit(content)
 
-    Log.Debug("detected encoding: %s (None: most likely already successfully decoded)" % a.original_encoding)
+    if a.original_encoding:
+        Log.Debug("detected encoding: %s (None: most likely already successfully decoded)" % a.original_encoding)
+    else:
+        Log.Debug("detected encoding: unicode (already decoded)")
 
     # easy way out - already utf-8
     if a.original_encoding and a.original_encoding == "utf-8":

Contents/Code/support/tasks.py

@@ -11,13 +11,13 @@ from subliminal import list_subtitles as list_all_subtitles
 from babelfish import Language
 
 from missing_subtitles import items_get_all_missing_subs, refresh_item
-from background import scheduler
-from storage import save_subtitles, whack_missing_parts, get_subtitle_storage
+from scheduler import scheduler
+from storage import save_subtitles, get_subtitle_storage
 from support.config import config
-from support.items import get_recent_items, is_ignored, get_item
-from support.lib import Plex
-from support.helpers import track_usage, get_title_for_video_metadata, cast_bool
-from support.plex_media import scan_videos, get_plex_metadata, PartUnknownException
+from support.items import get_recent_items, get_item, is_ignored
+from support.helpers import track_usage, get_title_for_video_metadata, cast_bool, PartUnknownException
+from support.plex_media import scan_videos, get_plex_metadata
+from download import download_best_subtitles
 
 
 class Task(object):
@@ -80,124 +80,52 @@ class Task(object):
         return
 
     def run(self):
+        Log.Info(u"Task: running: %s", self.name)
         self.time_start = datetime.datetime.now()
 
     def post_run(self, data_holder):
         self.running = False
         self.last_run = datetime.datetime.now()
-        if self.time_start:
+        if self.time_start and self.last_run:
             self.last_run_time = self.last_run - self.time_start
         self.time_start = None
-
-
-class SearchAllRecentlyAddedMissing(Task):
-    periodic = True
-    items_done = None
-    items_searching = None
-    items_searching_ids = None
-    items_failed = None
-    percentage = 0
-
-    stall_time = 30
-
-    def __init__(self, scheduler):
-        super(SearchAllRecentlyAddedMissing, self).__init__(scheduler)
-        self.items_done = None
-        self.items_searching = None
-        self.items_searching_ids = None
-        self.items_failed = None
-        self.percentage = 0
-
-    def signal(self, signal_name, *args, **kwargs):
-        handler = getattr(self, "signal_%s" % signal_name)
-        return handler(*args, **kwargs) if handler else None
-
-    def signal_updated_metadata(self, *args, **kwargs):
-        item_id = int(args[0])
-
-        if self.items_searching_ids is not None and item_id in self.items_searching_ids:
-            self.items_done.append(item_id)
-            return True
-
-    def prepare(self, *args, **kwargs):
-        self.items_done = []
-        recent_items = get_recent_items()
-        missing = items_get_all_missing_subs(recent_items)
-        ids = set([id for added_at, id, title, item, missing_languages in missing if not is_ignored(id, item=item)])
-        self.items_searching = missing
-        self.items_searching_ids = ids
-        self.items_failed = []
-        self.percentage = 0
-        self.ready_for_display = True
-
-    def run(self):
-        super(SearchAllRecentlyAddedMissing, self).run()
-        self.running = True
-        missing_count = len(self.items_searching)
-        items_done_count = 0
-
-        for added_at, item_id, title, item, missing_languages in self.items_searching:
-            Log.Debug(u"Task: %s, triggering refresh for %s (%s)", self.name, title, item_id)
-            refresh_item(item_id)
-            search_started = datetime.datetime.now()
-            tries = 1
-            while 1:
-                if item_id in self.items_done:
-                    items_done_count += 1
-                    Log.Debug(u"Task: %s, item %s done", self.name, item_id)
-                    self.percentage = int(items_done_count * 100 / missing_count)
-                    break
-
-                # item considered stalled after self.stall_time seconds passed after last refresh
-                if (datetime.datetime.now() - search_started).total_seconds() > self.stall_time:
-                    if tries > 3:
-                        self.items_failed.append(item_id)
-                        Log.Debug(u"Task: %s, item stalled for %s times: %s, skipping", self.name, tries, item_id)
-                        break
-
-                    Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time,
-                              item_id)
-                    tries += 1
-                    refresh_item(item_id)
-                    search_started = datetime.datetime.now()
-                    time.sleep(1)
-                time.sleep(0.1)
-            # we can't hammer the PMS, otherwise requests will be stalled
-            time.sleep(1)
-
-        Log.Debug("Task: %s, done. Failed items: %s", self.name, self.items_failed)
-        self.running = False
-
-    def post_run(self, task_data):
-        super(SearchAllRecentlyAddedMissing, self).post_run(task_data)
-        self.ready_for_display = False
-        self.percentage = 0
-        self.items_done = None
-        self.items_failed = None
-        self.items_searching = None
-        self.items_searching_ids = None
+        Log.Info(u"Task: ran: %s", self.name)
 
 
 class SubtitleListingMixin(object):
-    def list_subtitles(self, rating_key, item_type, part_id, language):
-        metadata = get_plex_metadata(rating_key, part_id, item_type)
+    def list_subtitles(self, rating_key, item_type, part_id, language, skip_wrong_fps=True, metadata=None,
+                       scanned_parts=None):
 
-        if item_type == "episode":
-            min_score = 240
-        else:
-            min_score = 60
+        if not metadata:
+            metadata = get_plex_metadata(rating_key, part_id, item_type)
 
-        scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
-        if not scanned_parts:
-            Log.Error("Couldn't list available subtitles for %s", rating_key)
+        if not metadata:
             return
 
+        if not scanned_parts:
+            scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+            if not scanned_parts:
+                Log.Error("Couldn't list available subtitles for %s", rating_key)
+                return
+
         video, plex_part = scanned_parts.items()[0]
         config.init_subliminal_patches()
 
+        provider_settings = config.provider_settings.copy()
+        if not skip_wrong_fps:
+            provider_settings = config.provider_settings.copy()
+            provider_settings["opensubtitles"]["skip_wrong_fps"] = False
+
+        if item_type == "episode":
+            min_score = 240
+            if video.is_special:
+                min_score = 180
+        else:
+            min_score = 60
+
         available_subs = list_all_subtitles(scanned_parts, {Language.fromietf(language)},
                                             providers=config.providers,
-                                            provider_configs=config.provider_settings,
+                                            provider_configs=provider_settings,
                                             pool_class=config.provider_pool)
 
         use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in ("prefer", "force HI")
@@ -247,8 +175,7 @@ class DownloadSubtitleMixin(object):
 
         if subtitle.content:
             try:
-                whack_missing_parts(scanned_parts)
-                save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode)
+                save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode, mods=config.default_mods)
                 Log.Debug("Manually downloaded subtitle for: %s", rating_key)
                 download_successful = True
                 refresh_item(rating_key)
@@ -266,6 +193,8 @@ class DownloadSubtitleMixin(object):
                     history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
                                 subtitle=subtitle,
                                 mode=mode)
+        else:
+            set_refresh_menu_state("Subtitle download failed (%s)" % rating_key)
         return download_successful
 
 
@@ -291,7 +220,13 @@ class AvailableSubsForItem(SubtitleListingMixin, Task):
         super(AvailableSubsForItem, self).run()
         self.running = True
         track_usage("Subtitle", "manual", "list", 1)
-        self.data = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language)
+        subs = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language, skip_wrong_fps=False)
+        if not subs:
+            self.data = "found_none"
+            return
+
+        # we can't have nasty unpicklable stuff like ZipFile, BytesIO etc in self.data
+        self.data = [s.make_picklable() for s in subs]
 
     def post_run(self, task_data):
         super(AvailableSubsForItem, self).post_run(task_data)
@@ -335,11 +270,178 @@ class MissingSubtitles(Task):
         task_data["missing_subtitles"] = self.data
 
 
+class SearchAllRecentlyAddedMissing(Task):
+    periodic = True
+
+    items_done = None
+    items_searching = None
+    percentage = 0
+
+    def __init__(self, scheduler):
+        super(SearchAllRecentlyAddedMissing, self).__init__(scheduler)
+        self.items_done = None
+        self.items_searching = None
+        self.percentage = 0
+
+    def signal_updated_metadata(self, *args, **kwargs):
+        return True
+
+    def prepare(self):
+        self.items_done = 0
+        self.items_searching = 0
+        self.percentage = 0
+        self.ready_for_display = True
+
+    def run(self):
+        super(SearchAllRecentlyAddedMissing, self).run()
+
+        self.running = True
+        self.prepare()
+
+        from support.history import get_history
+        history = get_history()
+
+        now = datetime.datetime.now()
+        min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
+        min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
+
+        is_recent_str = Prefs["scheduler.item_is_recent_age"]
+        num, ident = is_recent_str.split()
+
+        max_search_days = 0
+        if ident == "days":
+            max_search_days = int(num)
+        elif ident == "weeks":
+            max_search_days = int(num) * 7
+
+        subtitle_storage = get_subtitle_storage()
+        recent_sub_fns = subtitle_storage.get_recent_files(age_days=max_search_days)
+        viable_items = {}
+
+        # determine viable items
+        for fn in recent_sub_fns:
+            # added_date <= max_search_days?
+            stored_subs = subtitle_storage.load(filename=fn)
+            if not stored_subs:
+                continue
+
+            if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
+                continue
+
+            viable_items[fn] = stored_subs
+
+        self.items_searching = len(viable_items)
+
+        download_count = 0
+        videos_with_downloads = 0
+
+        config.init_subliminal_patches()
+
+        Log.Info("%s: Searching for subtitles for %s items", self.name, self.items_searching)
+
+        # search for subtitles in viable items
+        for fn, stored_subs in viable_items.iteritems():
+            video_id = stored_subs.video_id
+
+            if stored_subs.item_type == "episode":
+                min_score = min_score_series
+            else:
+                min_score = min_score_movies
+
+            parts = []
+            plex_item = get_item(video_id)
+
+            if not plex_item:
+                Log.Info("%s: Item %s unknown", self.name, video_id)
+                continue
+
+            if is_ignored(video_id, item=plex_item):
+                continue
+
+            for media in plex_item.media:
+                parts += media.parts
+
+            downloads_per_video = 0
+            for part in parts:
+                part_id = part.id
+
+                try:
+                    metadata = get_plex_metadata(video_id, part_id, stored_subs.item_type)
+                except PartUnknownException:
+                    Log.Info("%s: Part %s:%s unknown", self.name, video_id, part_id)
+                    continue
+
+                if not metadata:
+                    Log.Info("%s: Part %s:%s unknown", self.name, video_id, part_id)
+                    continue
+
+                Log.Debug("%s: Looking for missing subtitles: %s:%s", self.name, video_id, part_id)
+                scanned_parts = scan_videos([metadata], kind="series"
+                                            if stored_subs.item_type == "episode" else "movie")
+
+                downloaded_subtitles = download_best_subtitles(scanned_parts, min_score=min_score)
+                download_successful = False
+
+                if downloaded_subtitles:
+                    downloaded_any = any(downloaded_subtitles.values())
+                    if not downloaded_any:
+                        continue
+
+                    try:
+                        save_subtitles(scanned_parts, downloaded_subtitles, mode="a", mods=config.default_mods)
+                        Log.Debug("%s: Downloaded subtitle for item with missing subs: %s", self.name, video_id)
+                        download_successful = True
+                        refresh_item(video_id)
+                        track_usage("Subtitle", "manual", "download", 1)
+                    except:
+                        Log.Error("%s: Something went wrong when downloading specific subtitle: %s", self.name,
+                                  traceback.format_exc())
+                    finally:
+                        item_title = get_title_for_video_metadata(metadata, add_section_title=False)
+                        if download_successful:
+                            # store item in history
+                            for video, video_subtitles in downloaded_subtitles.items():
+                                if not video_subtitles:
+                                    continue
+
+                                for subtitle in video_subtitles:
+                                    downloads_per_video += 1
+                                    history.add(item_title, video.id, section_title=metadata["section"],
+                                                subtitle=subtitle,
+                                                mode="a")
+
+            download_count += downloads_per_video
+
+            if downloads_per_video:
+                videos_with_downloads += 1
+
+            self.items_done = self.items_done + 1
+            self.percentage = int(self.items_done * 100 / self.items_searching)
+
+            if downloads_per_video:
+                time.sleep(5)
+            else:
+                time.sleep(1)
+
+        if download_count:
+            Log.Debug("Task: %s, done. Missing subtitles found for %s/%s items (%s subs downloaded)", self.name,
+                      videos_with_downloads, self.items_searching, download_count)
+        else:
+            Log.Debug("Task: %s, done. No subtitles found for %s items", self.name, self.items_searching)
+
+    def post_run(self, task_data):
+        super(SearchAllRecentlyAddedMissing, self).post_run(task_data)
+        self.ready_for_display = False
+        self.percentage = 0
+        self.items_done = None
+        self.items_searching = None
+
+
 class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
     periodic = True
 
-    # TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired
-    series_cutoff = 355
+    # TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired, resolution
+    series_cutoff = 357
 
     # movies: format, title, release_group, year, video_codec, resolution, hearing_impaired
     movies_cutoff = 117
@@ -362,13 +464,26 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
                 return
 
         now = datetime.datetime.now()
+        min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
+        min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
+        overwrite_manually_modified = cast_bool(
+            Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
+        overwrite_manually_selected = cast_bool(
+            Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"])
 
         subtitle_storage = get_subtitle_storage()
         recent_subs = subtitle_storage.load_recent_files(age_days=max_search_days)
+        viable_item_count = 0
 
         for fn, stored_subs in recent_subs.iteritems():
             video_id = stored_subs.video_id
-            cutoff = self.series_cutoff if stored_subs.item_type == "episode" else self.movies_cutoff
+
+            if stored_subs.item_type == "episode":
+                cutoff = self.series_cutoff
+                min_score = min_score_series
+            else:
+                cutoff = self.movies_cutoff
+                min_score = min_score_movies
 
             # don't search for better subtitles until at least 30 minutes have passed
             if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
@@ -379,6 +494,7 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
             if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
                 continue
 
+            viable_item_count += 1
             ditch_parts = []
 
             # look through all stored subtitle data
@@ -398,14 +514,20 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
 
                     # late cutoff met? skip
                     if current_score >= cutoff:
-                        Log.Debug(u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s",
-                                  current_score, cutoff, stored_subs.title)
+                        Log.Debug(u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s (%s)",
+                                  current_score, cutoff, stored_subs.title, video_id)
                         continue
 
                     # got manual subtitle but don't want to touch those?
-                    if current_mode == "m" and \
-                            not cast_bool(Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]):
-                        Log.Debug(u"Skipping finding better subs, had manual: %s", stored_subs.title)
+                    if current_mode == "m" and not overwrite_manually_selected:
+                        Log.Debug(u"Skipping finding better subs, had manual: %s (%s)", stored_subs.title, video_id)
+                        continue
+
+                    # subtitle modifications different from default
+                    if not overwrite_manually_modified and current.mods \
+                            and set(current.mods).difference(set(config.default_mods)):
+                        Log.Debug(u"Skipping finding better subs, it has manual modifications: %s (%s)",
+                                  stored_subs.title, video_id)
                         continue
 
                     try:
@@ -420,7 +542,7 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
                         better_downloaded = False
                         better_tried_download = 0
                         for sub in subs:
-                            if sub.score > current_score:
+                            if sub.score > current_score and sub.score > min_score:
                                 Log.Debug("Better subtitle found for %s, downloading", video_id)
                                 better_tried_download += 1
                                 ret = self.download_subtitle(sub, video_id, mode="b")
@@ -444,8 +566,13 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
                         pass
                 subtitle_storage.save(stored_subs)
 
+            time.sleep(1)
+
         if better_found:
-            Log.Debug("Task: %s, done. Better subtitles found for %s items", self.name, better_found)
+            Log.Debug("Task: %s, done. Better subtitles found for %s/%s items", self.name, better_found,
+                      viable_item_count)
+        else:
+            Log.Debug("Task: %s, done. No better subtitles found for %s items", self.name, viable_item_count)
 
 
 class SubtitleStorageMaintenance(Task):
@@ -457,7 +584,7 @@ class SubtitleStorageMaintenance(Task):
         self.running = True
         Log.Info("Running subtitle storage maintenance")
         storage = get_subtitle_storage()
-        deleted_items = storage.delete_missing_files()
+        deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list))
         if deleted_items:
             Log.Info("Subtitle information for %d non-existant videos have been cleaned up" % len(deleted_items))
             Log.Debug("Videos: %s" % deleted_items)
@@ -465,9 +592,45 @@ class SubtitleStorageMaintenance(Task):
             Log.Info("Nothing to do")
 
 
+class MenuHistoryMaintenance(Task):
+    periodic = True
+    frequency = "every 7 days"
+
+    def run(self):
+        super(MenuHistoryMaintenance, self).run()
+        self.running = True
+        Log.Info("Running menu history maintenance")
+        now = datetime.datetime.now()
+        if "menu_history" in Dict:
+            for key, timeout in Dict["menu_history"].copy().items():
+                if now > timeout:
+                    try:
+                        del Dict["menu_history"][key]
+                    except:
+                        pass
+
+
+class MigrateSubtitleStorage(Task):
+    periodic = False
+    frequency = None
+
+    def run(self):
+        super(MigrateSubtitleStorage, self).run()
+        self.running = True
+        Log.Info("Running subtitle storage migration")
+        storage = get_subtitle_storage()
+        for fn in storage.get_all_files():
+            if fn.endswith(".json.gz"):
+                continue
+            Log.Debug("Migrating %s", fn)
+            storage.load(None, fn)
+
+
 scheduler.register(SearchAllRecentlyAddedMissing)
 scheduler.register(AvailableSubsForItem)
 scheduler.register(DownloadSubtitleForItem)
 scheduler.register(MissingSubtitles)
 scheduler.register(FindBetterSubtitles)
 scheduler.register(SubtitleStorageMaintenance)
+scheduler.register(MigrateSubtitleStorage)
+scheduler.register(MenuHistoryMaintenance)

Contents/DefaultPrefs.json

@@ -40,6 +40,8 @@
       "ro",
       "ru",
       "sr",
+      "sr-cyrl",
+      "sr-latn",
       "sk",
       "sl",
       "es",
@@ -94,6 +96,8 @@
       "ro",
       "ru",
       "sr",
+      "sr-cyrl",
+      "sr-latn",
       "sk",
       "sl",
       "es",
@@ -148,6 +152,8 @@
       "ro",
       "ru",
       "sr",
+      "sr-cyrl",
+      "sr-latn",
       "sk",
       "sl",
       "es",
@@ -258,13 +264,14 @@
       "35",
       "30",
       "25",
+      "21",
       "20",
       "15",
       "10",
       "5",
       "0"
     ],
-    "default": "25"
+    "default": "21"
   },
   {
     "id": "provider.addic7ed.use_random_agents",
@@ -332,7 +339,7 @@
   },
   {
     "id": "providers.multithreading",
-    "label": "Search enabled providers simuntaneously (multithreading)",
+    "label": "Search enabled providers simultaneously (multithreading)",
     "type": "bool",
     "default": "true"
   },
@@ -356,7 +363,7 @@
   },
   {
     "id": "subtitles.scan.exotic_ext",
-    "label": "Scan: include \"exotic\" external subtitle formats (anything else than .srt/.ssa/.ass)",
+    "label": "Scan: include \"exotic\" subtitle formats (anything else than .srt/.ssa/.ass/.vtt; embedded or external)",
     "type": "bool",
     "default": "false"
   },
@@ -381,7 +388,7 @@
     "id": "subtitles.search.minimumMovieScore2",
     "label": "Minimum score for movies (min: 60, def/sane: 69, min-ideal: 82; see http://v.ht/szscores)",
     "type": "text",
-    "default": "69"
+    "default": "60"
   },
   {
     "id": "subtitles.search.hearingImpaired",
@@ -396,17 +403,65 @@
     "default": "don't prefer"
   },
   {
-    "id": "subtitles.enforce_encoding",
-    "label": "Normalize subtitle encoding to UTF-8",
+    "id": "subtitles.remove_hi",
+    "label": "Remove Hearing Impaired tags from downloaded subtitles",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "subtitles.fix_common",
+    "label": "Fix common whitespace/punctuation issues in subtitles",
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "subtitles.fix_ocr",
+    "label": "Fix common OCR errors in downloaded subtitles",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "subtitles.colors",
+    "label": "Change colors of subtitles to",
+    "type": "enum",
+    "values": [
+      "don't change",
+      "white",
+      "light-grey",
+      "red",
+      "green",
+      "yellow",
+      "blue",
+      "magenta",
+      "cyan",
+      "black",
+      "dark-red",
+      "dark-green",
+      "dark-yellow",
+      "dark-blue",
+      "dark-magenta",
+      "dark-cyan",
+      "dark-grey"
+    ],
+    "default": "don't change"
+  },
   {
     "id": "subtitles.save.filesystem",
     "label": "Store subtitles next to media files (instead of metadata)",
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "subtitles.save.formats",
+    "label": "Subtitle formats to save (non-SRT only works if the previous option is enabled)",
+    "type": "enum",
+    "values": [
+      "SRT",
+      "VTT",
+      "SRT+VTT"
+    ],
+    "default": "SRT"
+  },
   {
     "id": "subtitles.save.subFolder",
     "label": "Subtitle Folder (\"current folder\" is the folder the current media file lives in)",
@@ -452,7 +507,8 @@
       "never",
       "current media item",
       "next episode (series)",
-      "hybrid: current item or next episode"
+      "hybrid: current item or next episode",
+      "hybrid-plus: current item and next episode"
     ],
     "default": "never"
   },
@@ -492,7 +548,7 @@
     "id": "scheduler.max_recent_items_per_library",
     "label": "Scheduler: Recent items to consider per library",
     "type": "text",
-    "default": "500"
+    "default": "1000"
   },
   {
     "id": "scheduler.tasks.FindBetterSubtitles.frequency",
@@ -516,7 +572,13 @@
     "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected",
     "label": "Scheduler: Overwrite manually selected subtitles when better found",
     "type": "bool",
-    "default": "true"
+    "default": "false"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified",
+    "label": "Scheduler: Overwrite subtitles with non-default subtitle modifications when better found",
+    "type": "bool",
+    "default": "false"
   },
   {
     "id": "history_size",
@@ -593,7 +655,7 @@
   },
   {
     "id": "notify_executable",
-    "label": "Call this executable upon successful subtitle download",
+    "label": "Call this executable upon successful subtitle download (see Wiki for details)",
     "type": "text",
     "default": ""
   },
@@ -616,6 +678,12 @@
     ],
     "default": "WARNING"
   },
+  {
+    "id": "log_debug_mods",
+    "label": "Log subtitle modification (debug)",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "log_console",
     "label": "Log to console (for development/debugging)",

Contents/Info.plist

@@ -9,11 +9,11 @@
         <key>CFBundleInfoDictionaryVersion</key>
         <string>6.0</string>
         <key>CFBundleShortVersionString</key>
-        <string>2.0.0</string>
+        <string>2.0.24</string>
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.0.0.9</string>
+        <string>2.0.24.1565</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -23,7 +23,7 @@
         <key>PlexPluginConsoleLogging</key>
         <string>0</string>
         <key>PlexPluginDevMode</key>
-        <string>1</string>
+        <string>0</string>
          <key>PlexPluginCodePolicy</key>
             <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API -->
             <string>Elevated</string>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.0.0.9 DEV
+Version 2.0.24.1565
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 

Contents/Libraries/Shared/anydbm.py

@@ -0,0 +1,85 @@
+"""Generic interface to all dbm clones.
+
+Instead of
+
+        import dbm
+        d = dbm.open(file, 'w', 0666)
+
+use
+
+        import anydbm
+        d = anydbm.open(file, 'w')
+
+The returned object is a dbhash, gdbm, dbm or dumbdbm object,
+dependent on the type of database being opened (determined by whichdb
+module) in the case of an existing dbm. If the dbm does not exist and
+the create or new flag ('c' or 'n') was specified, the dbm type will
+be determined by the availability of the modules (tested in the above
+order).
+
+It has the following interface (key and data are strings):
+
+        d[key] = data   # store data at key (may override data at
+                        # existing key)
+        data = d[key]   # retrieve data at key (raise KeyError if no
+                        # such key)
+        del d[key]      # delete data stored at key (raises KeyError
+                        # if no such key)
+        flag = key in d   # true if the key exists
+        list = d.keys() # return a list of all existing keys (slow!)
+
+Future versions may change the order in which implementations are
+tested for existence, and add interfaces to other dbm-like
+implementations.
+"""
+
+class error(Exception):
+    pass
+
+_names = ['dbhash', 'gdbm', 'dbm', 'dumbdbm']
+_errors = [error]
+_defaultmod = None
+
+for _name in _names:
+    try:
+        _mod = __import__(_name)
+    except ImportError:
+        continue
+    if not _defaultmod:
+        _defaultmod = _mod
+    _errors.append(_mod.error)
+
+if not _defaultmod:
+    raise ImportError, "no dbm clone found; tried %s" % _names
+
+error = tuple(_errors)
+
+def open(file, flag='r', mode=0666):
+    """Open or create database at path given by *file*.
+
+    Optional argument *flag* can be 'r' (default) for read-only access, 'w'
+    for read-write access of an existing database, 'c' for read-write access
+    to a new or existing database, and 'n' for read-write access to a new
+    database.
+
+    Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it
+    only if it doesn't exist; and 'n' always creates a new database.
+    """
+
+    # guess the type of an existing database
+    from whichdb import whichdb
+    result=whichdb(file)
+    if result is None:
+        # db doesn't exist
+        if 'c' in flag or 'n' in flag:
+            # file doesn't exist and the new
+            # flag was used so use default type
+            mod = _defaultmod
+        else:
+            raise error, "need 'c' or 'n' flag to open new db"
+    elif result == "":
+        # db type cannot be determined
+        raise error, "db type could not be determined"
+    else:
+        mod = __import__(result)
+    return mod.open(file, flag, mode)

Contents/Libraries/Shared/dbhash.py

@@ -0,0 +1,18 @@
+"""Provide a (g)dbm-compatible interface to bsddb.hashopen."""
+
+import sys
+import warnings
+warnings.warnpy3k("in 3.x, the dbhash module has been removed", stacklevel=2)
+try:
+    import bsddb
+except ImportError:
+    # prevent a second import of this module from spuriously succeeding
+    del sys.modules[__name__]
+    raise
+
+__all__ = ["error","open"]
+
+error = bsddb.error                     # Exported for anydbm
+
+def open(file, flag = 'r', mode=0666):
+    return bsddb.hashopen(file, flag, mode)

Contents/Libraries/Shared/dumbdbm.py

@@ -0,0 +1,249 @@
+"""A dumb and slow but simple dbm clone.
+
+For database spam, spam.dir contains the index (a text file),
+spam.bak *may* contain a backup of the index (also a text file),
+while spam.dat contains the data (a binary file).
+
+XXX TO DO:
+
+- seems to contain a bug when updating...
+
+- reclaim free space (currently, space once occupied by deleted or expanded
+items is never reused)
+
+- support concurrent access (currently, if two processes take turns making
+updates, they can mess up the index)
+
+- support efficient access to large databases (currently, the whole index
+is read when the database is opened, and some updates rewrite the whole index)
+
+- support opening for read-only (flag = 'm')
+
+"""
+
+import ast as _ast
+import os as _os
+import __builtin__
+import UserDict
+
+_open = __builtin__.open
+
+_BLOCKSIZE = 512
+
+error = IOError                         # For anydbm
+
+class _Database(UserDict.DictMixin):
+
+    # The on-disk directory and data files can remain in mutually
+    # inconsistent states for an arbitrarily long time (see comments
+    # at the end of __setitem__).  This is only repaired when _commit()
+    # gets called.  One place _commit() gets called is from __del__(),
+    # and if that occurs at program shutdown time, module globals may
+    # already have gotten rebound to None.  Since it's crucial that
+    # _commit() finish successfully, we can't ignore shutdown races
+    # here, and _commit() must not reference any globals.
+    _os = _os       # for _commit()
+    _open = _open   # for _commit()
+
+    def __init__(self, filebasename, mode):
+        self._mode = mode
+
+        # The directory file is a text file.  Each line looks like
+        #    "%r, (%d, %d)\n" % (key, pos, siz)
+        # where key is the string key, pos is the offset into the dat
+        # file of the associated value's first byte, and siz is the number
+        # of bytes in the associated value.
+        self._dirfile = filebasename + _os.extsep + 'dir'
+
+        # The data file is a binary file pointed into by the directory
+        # file, and holds the values associated with keys.  Each value
+        # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
+        # binary 8-bit string value.
+        self._datfile = filebasename + _os.extsep + 'dat'
+        self._bakfile = filebasename + _os.extsep + 'bak'
+
+        # The index is an in-memory dict, mirroring the directory file.
+        self._index = None  # maps keys to (pos, siz) pairs
+
+        # Mod by Jack: create data file if needed
+        try:
+            f = _open(self._datfile, 'r')
+        except IOError:
+            with _open(self._datfile, 'w') as f:
+                self._chmod(self._datfile)
+        else:
+            f.close()
+        self._update()
+
+    # Read directory file into the in-memory index dict.
+    def _update(self):
+        self._index = {}
+        try:
+            f = _open(self._dirfile)
+        except IOError:
+            pass
+        else:
+            with f:
+                for line in f:
+                    line = line.rstrip()
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
+                    self._index[key] = pos_and_siz_pair
+
+    # Write the index dict to the directory file.  The original directory
+    # file (if any) is renamed with a .bak extension first.  If a .bak
+    # file currently exists, it's deleted.
+    def _commit(self):
+        # CAUTION:  It's vital that _commit() succeed, and _commit() can
+        # be called from __del__().  Therefore we must never reference a
+        # global in this routine.
+        if self._index is None:
+            return  # nothing to do
+
+        try:
+            self._os.unlink(self._bakfile)
+        except self._os.error:
+            pass
+
+        try:
+            self._os.rename(self._dirfile, self._bakfile)
+        except self._os.error:
+            pass
+
+        with self._open(self._dirfile, 'w') as f:
+            self._chmod(self._dirfile)
+            for key, pos_and_siz_pair in self._index.iteritems():
+                f.write("%r, %r\n" % (key, pos_and_siz_pair))
+
+    sync = _commit
+
+    def __getitem__(self, key):
+        pos, siz = self._index[key]     # may raise KeyError
+        with _open(self._datfile, 'rb') as f:
+            f.seek(pos)
+            dat = f.read(siz)
+        return dat
+
+    # Append val to the data file, starting at a _BLOCKSIZE-aligned
+    # offset.  The data file is first padded with NUL bytes (if needed)
+    # to get to an aligned offset.  Return pair
+    #     (starting offset of val, len(val))
+    def _addval(self, val):
+        with _open(self._datfile, 'rb+') as f:
+            f.seek(0, 2)
+            pos = int(f.tell())
+            npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
+            f.write('\0'*(npos-pos))
+            pos = npos
+            f.write(val)
+        return (pos, len(val))
+
+    # Write val to the data file, starting at offset pos.  The caller
+    # is responsible for ensuring that there's enough room starting at
+    # pos to hold val, without overwriting some other value.  Return
+    # pair (pos, len(val)).
+    def _setval(self, pos, val):
+        with _open(self._datfile, 'rb+') as f:
+            f.seek(pos)
+            f.write(val)
+        return (pos, len(val))
+
+    # key is a new key whose associated value starts in the data file
+    # at offset pos and with length siz.  Add an index record to
+    # the in-memory index dict, and append one to the directory file.
+    def _addkey(self, key, pos_and_siz_pair):
+        self._index[key] = pos_and_siz_pair
+        with _open(self._dirfile, 'a') as f:
+            self._chmod(self._dirfile)
+            f.write("%r, %r\n" % (key, pos_and_siz_pair))
+
+    def __setitem__(self, key, val):
+        if not type(key) == type('') == type(val):
+            raise TypeError, "keys and values must be strings"
+        if key not in self._index:
+            self._addkey(key, self._addval(val))
+        else:
+            # See whether the new value is small enough to fit in the
+            # (padded) space currently occupied by the old value.
+            pos, siz = self._index[key]
+            oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
+            newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
+            if newblocks <= oldblocks:
+                self._index[key] = self._setval(pos, val)
+            else:
+                # The new value doesn't fit in the (padded) space used
+                # by the old value.  The blocks used by the old value are
+                # forever lost.
+                self._index[key] = self._addval(val)
+
+            # Note that _index may be out of synch with the directory
+            # file now:  _setval() and _addval() don't update the directory
+            # file.  This also means that the on-disk directory and data
+            # files are in a mutually inconsistent state, and they'll
+            # remain that way until _commit() is called.  Note that this
+            # is a disaster (for the database) if the program crashes
+            # (so that _commit() never gets called).
+
+    def __delitem__(self, key):
+        # The blocks used by the associated value are lost.
+        del self._index[key]
+        # XXX It's unclear why we do a _commit() here (the code always
+        # XXX has, so I'm not changing it).  _setitem__ doesn't try to
+        # XXX keep the directory file in synch.  Why should we?  Or
+        # XXX why shouldn't __setitem__?
+        self._commit()
+
+    def keys(self):
+        return self._index.keys()
+
+    def has_key(self, key):
+        return key in self._index
+
+    def __contains__(self, key):
+        return key in self._index
+
+    def iterkeys(self):
+        return self._index.iterkeys()
+    __iter__ = iterkeys
+
+    def __len__(self):
+        return len(self._index)
+
+    def close(self):
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
+
+    __del__ = close
+
+    def _chmod (self, file):
+        if hasattr(self._os, 'chmod'):
+            self._os.chmod(file, self._mode)
+
+
+def open(file, flag=None, mode=0666):
+    """Open the database file, filename, and return corresponding object.
+
+    The flag argument, used to control how the database is opened in the
+    other DBM implementations, is ignored in the dumbdbm module; the
+    database is always opened for update, and will be created if it does
+    not exist.
+
+    The optional mode argument is the UNIX mode of the file, used only when
+    the database has to be created.  It defaults to octal code 0666 (and
+    will be modified by the prevailing umask).
+
+    """
+    # flag argument is currently ignored
+
+    # Modify mode depending on the umask
+    try:
+        um = _os.umask(0)
+        _os.umask(um)
+    except AttributeError:
+        pass
+    else:
+        # Turn off any bits that are set in the umask
+        mode = mode & (~um)
+
+    return _Database(file, mode)

Contents/Libraries/Shared/enzyme/mkv.py

@@ -369,7 +369,8 @@ class Chapter(object):
         if chapterdisplays:
             string = chapterdisplays[0].get('ChapString')
             language = chapterdisplays[0].get('ChapLanguage')
-        return cls(start, hidden, enabled, end, string, language)
+            return cls(start, hidden, enabled, end, string, language)
+        return cls(start, hidden, enabled, end)
 
     def __repr__(self):
         return '<%s [%s, enabled=%s]>' % (self.__class__.__name__, self.start, self.enabled)

Contents/Libraries/Shared/enzyme/parsers/ebml/core.py

@@ -168,9 +168,13 @@ def parse(stream, specs, size=None, ignore_element_types=None, ignore_element_na
     while size is None or stream.tell() - start < size:
         try:
             element = parse_element(stream, specs)
+            if not element or not hasattr(element, "type"):
+                stream.seek(element.size, 1)
+                continue
+
             if element.type is None:
-                logger.error('Element with id 0x%x is not in the specs' % element_id)
-                stream.seek(element_size, 1)
+                logger.error('Element with id 0x%x is not in the specs' % element.id)
+                stream.seek(element.size, 1)
                 continue
             elif element.type in ignore_element_types or element.name in ignore_element_names:
                 logger.info('%s %s %s ignored', element.__class__.__name__, element.name, element.type)

Contents/Libraries/Shared/ftfy/__init__.py

@@ -0,0 +1,411 @@
+# -*- coding: utf-8 -*-
+"""
+ftfy: fixes text for you
+
+This is a module for making text less broken. See the `fix_text` function
+for more information.
+"""
+
+from __future__ import unicode_literals
+import unicodedata
+import ftfy.bad_codecs
+from ftfy import fixes
+from ftfy.formatting import display_ljust
+from ftfy.compatibility import is_printable
+
+__version__ = '4.4.3'
+
+
+# See the docstring for ftfy.bad_codecs to see what we're doing here.
+ftfy.bad_codecs.ok()
+
+
+def fix_text(text,
+             fix_entities='auto',
+             remove_terminal_escapes=True,
+             fix_encoding=True,
+             fix_latin_ligatures=True,
+             fix_character_width=True,
+             uncurl_quotes=True,
+             fix_line_breaks=True,
+             fix_surrogates=True,
+             remove_control_chars=True,
+             remove_bom=True,
+             normalization='NFC',
+             max_decode_length=10**6):
+    r"""
+    Given Unicode text as input, fix inconsistencies and glitches in it,
+    such as mojibake.
+
+    Let's start with some examples:
+
+        >>> print(fix_text('ünicode'))
+        ünicode
+
+        >>> print(fix_text('Broken text… it’s flubberific!',
+        ...                normalization='NFKC'))
+        Broken text... it's flubberific!
+
+        >>> print(fix_text('HTML entities <3'))
+        HTML entities <3
+
+        >>> print(fix_text('<em>HTML entities &lt;3</em>'))
+        <em>HTML entities &lt;3</em>
+
+        >>> print(fix_text("&macr;\\_(ã\x83\x84)_/&macr;"))
+        ¯\_(ツ)_/¯
+
+        >>> # This example string starts with a byte-order mark, even if
+        >>> # you can't see it on the Web.
+        >>> print(fix_text('\ufeffParty like\nit&rsquo;s 1999!'))
+        Party like
+        it's 1999!
+
+        >>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ'))
+        LOUD NOISES
+
+        >>> len(fix_text('fi' * 100000))
+        200000
+
+        >>> len(fix_text(''))
+        0
+
+    Based on the options you provide, ftfy applies these steps in order:
+
+    - If `remove_terminal_escapes` is True, remove sequences of bytes that are
+      instructions for Unix terminals, such as the codes that make text appear
+      in different colors.
+
+    - If `fix_encoding` is True, look for common mistakes that come from
+      encoding or decoding Unicode text incorrectly, and fix them if they are
+      reasonably fixable. See `fixes.fix_encoding` for details.
+
+    - If `fix_entities` is True, replace HTML entities with their equivalent
+      characters. If it's "auto" (the default), then consider replacing HTML
+      entities, but don't do so in text where you have seen a pair of actual
+      angle brackets (that's probably actually HTML and you shouldn't mess
+      with the entities).
+
+    - If `uncurl_quotes` is True, replace various curly quotation marks with
+      plain-ASCII straight quotes.
+
+    - If `fix_latin_ligatures` is True, then ligatures made of Latin letters,
+      such as `fi`, will be separated into individual letters. These ligatures
+      are usually not meaningful outside of font rendering, and often represent
+      copy-and-paste errors.
+
+    - If `fix_character_width` is True, half-width and full-width characters
+      will be replaced by their standard-width form.
+
+    - If `fix_line_breaks` is true, convert all line breaks to Unix style
+      (CRLF and CR line breaks become LF line breaks).
+
+    - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates
+      in the resulting string, by converting them to the correct characters
+      when they're appropriately paired, or replacing them with \ufffd
+      otherwise.
+
+    - If `remove_control_chars` is true, remove control characters that
+      are not suitable for use in text. This includes most of the ASCII control
+      characters, plus some Unicode controls such as the byte order mark
+      (U+FEFF). Useful control characters, such as Tab, Line Feed, and
+      bidirectional marks, are left as they are.
+
+    - If `remove_bom` is True, remove the Byte-Order Mark at the start of the
+      string if it exists. (This is largely redundant, because it's a special
+      case of `remove_control_characters`. This option will become deprecated
+      in a later version.)
+
+    - If `normalization` is not None, apply the specified form of Unicode
+      normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'.
+
+      - The default normalization, NFC, combines characters and diacritics that
+        are written using separate code points, such as converting "e" plus an
+        acute accent modifier into "é", or converting "ka" (か) plus a dakuten
+        into the single character "ga" (が). Unicode can be converted to NFC
+        form without any change in its meaning.
+
+      - If you ask for NFKC normalization, it will apply additional
+        normalizations that can change the meanings of characters. For example,
+        ellipsis characters will be replaced with three periods, all ligatures
+        will be replaced with the individual characters that make them up,
+        and characters that differ in font style will be converted to the same
+        character.
+
+    - If anything was changed, repeat all the steps, so that the function is
+      idempotent. "&amp;amp;" will become "&", for example, not "&amp;".
+
+    `fix_text` will work one line at a time, with the possibility that some
+    lines are in different encodings, allowing it to fix text that has been
+    concatenated together from different sources.
+
+    When it encounters lines longer than `max_decode_length` (1 million
+    codepoints by default), it will not run the `fix_encoding` step, to avoid
+    unbounded slowdowns.
+
+    If you're certain that any decoding errors in the text would have affected
+    the entire text in the same way, and you don't mind operations that scale
+    with the length of the text, you can use `fix_text_segment` directly to
+    fix the whole string in one batch.
+    """
+    if isinstance(text, bytes):
+        raise UnicodeError(fixes.BYTES_ERROR_TEXT)
+
+    out = []
+    pos = 0
+    while pos < len(text):
+        textbreak = text.find('\n', pos) + 1
+        fix_encoding_this_time = fix_encoding
+        if textbreak == 0:
+            textbreak = len(text)
+        if (textbreak - pos) > max_decode_length:
+            fix_encoding_this_time = False
+
+        substring = text[pos:textbreak]
+
+        if fix_entities == 'auto' and '<' in substring and '>' in substring:
+            # we see angle brackets together; this could be HTML
+            fix_entities = False
+
+        out.append(
+            fix_text_segment(
+                substring,
+                fix_entities=fix_entities,
+                remove_terminal_escapes=remove_terminal_escapes,
+                fix_encoding=fix_encoding_this_time,
+                uncurl_quotes=uncurl_quotes,
+                fix_latin_ligatures=fix_latin_ligatures,
+                fix_character_width=fix_character_width,
+                fix_line_breaks=fix_line_breaks,
+                fix_surrogates=fix_surrogates,
+                remove_control_chars=remove_control_chars,
+                remove_bom=remove_bom,
+                normalization=normalization
+            )
+        )
+        pos = textbreak
+
+    return ''.join(out)
+
+# Some alternate names for the main functions
+ftfy = fix_text
+fix_encoding = fixes.fix_encoding
+fix_text_encoding = fixes.fix_text_encoding  # deprecated
+
+
+def fix_file(input_file,
+             encoding=None,
+             fix_entities='auto',
+             remove_terminal_escapes=True,
+             fix_encoding=True,
+             fix_latin_ligatures=True,
+             fix_character_width=True,
+             uncurl_quotes=True,
+             fix_line_breaks=True,
+             fix_surrogates=True,
+             remove_control_chars=True,
+             remove_bom=True,
+             normalization='NFC'):
+    """
+    Fix text that is found in a file.
+
+    If the file is being read as Unicode text, use that. If it's being read as
+    bytes, then we hope an encoding was supplied. If not, unfortunately, we
+    have to guess what encoding it is. We'll try a few common encodings, but we
+    make no promises. See the `guess_bytes` function for how this is done.
+
+    The output is a stream of fixed lines of text.
+    """
+    entities = fix_entities
+    for line in input_file:
+        if isinstance(line, bytes):
+            if encoding is None:
+                line, encoding = guess_bytes(line)
+            else:
+                line = line.decode(encoding)
+        if fix_entities == 'auto' and '<' in line and '>' in line:
+            entities = False
+        yield fix_text_segment(
+            line,
+            fix_entities=entities,
+            remove_terminal_escapes=remove_terminal_escapes,
+            fix_encoding=fix_encoding,
+            fix_latin_ligatures=fix_latin_ligatures,
+            fix_character_width=fix_character_width,
+            uncurl_quotes=uncurl_quotes,
+            fix_line_breaks=fix_line_breaks,
+            fix_surrogates=fix_surrogates,
+            remove_control_chars=remove_control_chars,
+            remove_bom=remove_bom,
+            normalization=normalization
+        )
+
+
+def fix_text_segment(text,
+                     fix_entities='auto',
+                     remove_terminal_escapes=True,
+                     fix_encoding=True,
+                     fix_latin_ligatures=True,
+                     fix_character_width=True,
+                     uncurl_quotes=True,
+                     fix_line_breaks=True,
+                     fix_surrogates=True,
+                     remove_control_chars=True,
+                     remove_bom=True,
+                     normalization='NFC'):
+    """
+    Apply fixes to text in a single chunk. This could be a line of text
+    within a larger run of `fix_text`, or it could be a larger amount
+    of text that you are certain is in a consistent encoding.
+
+    See `fix_text` for a description of the parameters.
+    """
+    if isinstance(text, bytes):
+        raise UnicodeError(fixes.BYTES_ERROR_TEXT)
+
+    if fix_entities == 'auto' and '<' in text and '>' in text:
+        fix_entities = False
+    while True:
+        origtext = text
+        if remove_terminal_escapes:
+            text = fixes.remove_terminal_escapes(text)
+        if fix_encoding:
+            text = fixes.fix_encoding(text)
+        if fix_entities:
+            text = fixes.unescape_html(text)
+        if fix_latin_ligatures:
+            text = fixes.fix_latin_ligatures(text)
+        if fix_character_width:
+            text = fixes.fix_character_width(text)
+        if uncurl_quotes:
+            text = fixes.uncurl_quotes(text)
+        if fix_line_breaks:
+            text = fixes.fix_line_breaks(text)
+        if fix_surrogates:
+            text = fixes.fix_surrogates(text)
+        if remove_control_chars:
+            text = fixes.remove_control_chars(text)
+        if remove_bom and not remove_control_chars:
+            # Skip this step if we've already done `remove_control_chars`,
+            # because it would be redundant.
+            text = fixes.remove_bom(text)
+        if normalization is not None:
+            text = unicodedata.normalize(normalization, text)
+        if text == origtext:
+            return text
+
+
+def guess_bytes(bstring):
+    """
+    NOTE: Using `guess_bytes` is not the recommended way of using ftfy. ftfy
+    is not designed to be an encoding detector.
+
+    In the unfortunate situation that you have some bytes in an unknown
+    encoding, ftfy can guess a reasonable strategy for decoding them, by trying
+    a few common encodings that can be distinguished from each other.
+
+    Unlike the rest of ftfy, this may not be accurate, and it may *create*
+    Unicode problems instead of solving them!
+
+    It doesn't try East Asian encodings at all, and if you have East Asian text
+    that you don't know how to decode, you are somewhat out of luck.  East
+    Asian encodings require some serious statistics to distinguish from each
+    other, so we can't support them without decreasing the accuracy of ftfy.
+
+    If you don't know which encoding you have at all, I recommend
+    trying the 'chardet' module, and being appropriately skeptical about its
+    results.
+
+    The encodings we try here are:
+
+    - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks
+      like nothing else
+    - UTF-8, because it's the global standard, which has been used by a
+      majority of the Web since 2008
+    - "utf-8-variants", because it's what people actually implement when they
+      think they're doing UTF-8
+    - MacRoman, because Microsoft Office thinks it's still a thing, and it
+      can be distinguished by its line breaks. (If there are no line breaks in
+      the string, though, you're out of luck.)
+    - "sloppy-windows-1252", the Latin-1-like encoding that is the most common
+      single-byte encoding
+    """
+    if type(bstring) == type(''):
+        raise UnicodeError(
+            "This string was already decoded as Unicode. You should pass "
+            "bytes to guess_bytes, not Unicode."
+        )
+
+    if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'):
+        return bstring.decode('utf-16'), 'utf-16'
+
+    byteset = set(bytes(bstring))
+    byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n'
+
+    try:
+        if byte_ed in byteset or byte_c0 in byteset:
+            # Byte 0xed can be used to encode a range of codepoints that
+            # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates,
+            # so when we see 0xed, it's very likely we're being asked to
+            # decode CESU-8, the variant that encodes UTF-16 surrogates
+            # instead of the original characters themselves.
+            #
+            # This will occasionally trigger on standard UTF-8, as there
+            # are some Korean characters that also use byte 0xed, but that's
+            # not harmful.
+            #
+            # Byte 0xc0 is impossible because, numerically, it would only
+            # encode characters lower than U+0040. Those already have
+            # single-byte representations, and UTF-8 requires using the
+            # shortest possible representation. However, Java hides the null
+            # codepoint, U+0000, in a non-standard longer representation -- it
+            # encodes it as 0xc0 0x80 instead of 0x00, guaranteeing that 0x00
+            # will never appear in the encoded bytes.
+            #
+            # The 'utf-8-variants' decoder can handle both of these cases, as
+            # well as standard UTF-8, at the cost of a bit of speed.
+            return bstring.decode('utf-8-variants'), 'utf-8-variants'
+        else:
+            return bstring.decode('utf-8'), 'utf-8'
+    except UnicodeDecodeError:
+        pass
+
+    if byte_CR in bstring and byte_LF not in bstring:
+        return bstring.decode('macroman'), 'macroman'
+    else:
+        return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252'
+
+
+def explain_unicode(text):
+    """
+    A utility method that's useful for debugging mysterious Unicode.
+
+    It breaks down a string, showing you for each codepoint its number in
+    hexadecimal, its glyph, its category in the Unicode standard, and its name
+    in the Unicode standard.
+
+        >>> explain_unicode('(╯°□°)╯︵ ┻━┻')
+        U+0028  (       [Ps] LEFT PARENTHESIS
+        U+256F  ╯       [So] BOX DRAWINGS LIGHT ARC UP AND LEFT
+        U+00B0  °       [So] DEGREE SIGN
+        U+25A1  □       [So] WHITE SQUARE
+        U+00B0  °       [So] DEGREE SIGN
+        U+0029  )       [Pe] RIGHT PARENTHESIS
+        U+256F  ╯       [So] BOX DRAWINGS LIGHT ARC UP AND LEFT
+        U+FE35  ︵      [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS
+        U+0020          [Zs] SPACE
+        U+253B  ┻       [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL
+        U+2501  ━       [So] BOX DRAWINGS HEAVY HORIZONTAL
+        U+253B  ┻       [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL
+    """
+    for char in text:
+        if is_printable(char):
+            display = char
+        else:
+            display = char.encode('unicode-escape').decode('ascii')
+        print('U+{code:04X}  {display} [{category}] {name}'.format(
+            display=display_ljust(display, 7),
+            code=ord(char),
+            category=unicodedata.category(char),
+            name=unicodedata.name(char, '<unknown>')
+        ))

Contents/Libraries/Shared/ftfy/bad_codecs/__init__.py

@@ -0,0 +1,94 @@
+# coding: utf-8
+r"""
+Give Python the ability to decode some common, flawed encodings.
+
+Python does not want you to be sloppy with your text. Its encoders and decoders
+("codecs") follow the relevant standards whenever possible, which means that
+when you get text that *doesn't* follow those standards, you'll probably fail
+to decode it. Or you might succeed at decoding it for implementation-specific
+reasons, which is perhaps worse.
+
+There are some encodings out there that Python wishes didn't exist, which are
+widely used outside of Python:
+
+- "utf-8-variants", a family of not-quite-UTF-8 encodings, including the
+  ever-popular CESU-8 and "Java modified UTF-8".
+- "Sloppy" versions of character map encodings, where bytes that don't map to
+  anything will instead map to the Unicode character with the same number.
+
+Simply importing this module, or in fact any part of the `ftfy` package, will
+make these new "bad codecs" available to Python through the standard Codecs
+API. You never have to actually call any functions inside `ftfy.bad_codecs`.
+
+However, if you want to call something because your code checker insists on it,
+you can call ``ftfy.bad_codecs.ok()``.
+
+A quick example of decoding text that's encoded in CESU-8:
+
+    >>> import ftfy.bad_codecs
+    >>> print(b'\xed\xa0\xbd\xed\xb8\x8d'.decode('utf-8-variants'))
+    😍
+"""
+from __future__ import unicode_literals
+from encodings import normalize_encoding
+import codecs
+
+_CACHE = {}
+
+# Define some aliases for 'utf-8-variants'. All hyphens get turned into
+# underscores, because of `normalize_encoding`.
+UTF8_VAR_NAMES = (
+    'utf_8_variants', 'utf8_variants',
+    'utf_8_variant', 'utf8_variant',
+    'utf_8_var', 'utf8_var',
+    'cesu_8', 'cesu8',
+    'java_utf_8', 'java_utf8'
+)
+
+
+def search_function(encoding):
+    """
+    Register our "bad codecs" with Python's codecs API. This involves adding
+    a search function that takes in an encoding name, and returns a codec
+    for that encoding if it knows one, or None if it doesn't.
+
+    The encodings this will match are:
+
+    - Encodings of the form 'sloppy-windows-NNNN' or 'sloppy-iso-8859-N',
+      where the non-sloppy version is an encoding that leaves some bytes
+      unmapped to characters.
+    - The 'utf-8-variants' encoding, which has the several aliases seen
+      above.
+    """
+    if encoding in _CACHE:
+        return _CACHE[encoding]
+
+    norm_encoding = normalize_encoding(encoding)
+    codec = None
+    if norm_encoding in UTF8_VAR_NAMES:
+        from ftfy.bad_codecs.utf8_variants import CODEC_INFO
+        codec = CODEC_INFO
+    elif norm_encoding.startswith('sloppy_'):
+        from ftfy.bad_codecs.sloppy import CODECS
+        codec = CODECS.get(norm_encoding)
+
+    if codec is not None:
+        _CACHE[encoding] = codec
+
+    return codec
+
+
+def ok():
+    """
+    A feel-good function that gives you something to call after importing
+    this package.
+
+    Why is this here? Pyflakes. Pyflakes gets upset when you import a module
+    and appear not to use it. It doesn't know that you're using it when
+    you use the ``unicode.encode`` and ``bytes.decode`` methods with certain
+    encodings.
+    """
+    pass
+
+
+codecs.register(search_function)

Contents/Libraries/Shared/ftfy/bad_codecs/sloppy.py

@@ -0,0 +1,164 @@
+# coding: utf-8
+r"""
+Decodes single-byte encodings, filling their "holes" in the same messy way that
+everyone else does.
+
+A single-byte encoding maps each byte to a Unicode character, except that some
+bytes are left unmapped. In the commonly-used Windows-1252 encoding, for
+example, bytes 0x81 and 0x8D, among others, have no meaning.
+
+Python, wanting to preserve some sense of decorum, will handle these bytes
+as errors. But Windows knows that 0x81 and 0x8D are possible bytes and they're
+different from each other. It just hasn't defined what they are in terms of
+Unicode.
+
+Software that has to interoperate with Windows-1252 and Unicode -- such as all
+the common Web browsers -- will pick some Unicode characters for them to map
+to, and the characters they pick are the Unicode characters with the same
+numbers: U+0081 and U+008D. This is the same as what Latin-1 does, and the
+resulting characters tend to fall into a range of Unicode that's set aside for
+obselete Latin-1 control characters anyway.
+
+These sloppy codecs let Python do the same thing, thus interoperating with
+other software that works this way. It defines a sloppy version of many
+single-byte encodings with holes. (There is no need for a sloppy version of
+an encoding without holes: for example, there is no such thing as
+sloppy-iso-8859-2 or sloppy-macroman.)
+
+The following encodings will become defined:
+
+- sloppy-windows-1250 (Central European, sort of based on ISO-8859-2)
+- sloppy-windows-1251 (Cyrillic)
+- sloppy-windows-1252 (Western European, based on Latin-1)
+- sloppy-windows-1253 (Greek, sort of based on ISO-8859-7)
+- sloppy-windows-1254 (Turkish, based on ISO-8859-9)
+- sloppy-windows-1255 (Hebrew, based on ISO-8859-8)
+- sloppy-windows-1256 (Arabic)
+- sloppy-windows-1257 (Baltic, based on ISO-8859-13)
+- sloppy-windows-1258 (Vietnamese)
+- sloppy-cp874 (Thai, based on ISO-8859-11)
+- sloppy-iso-8859-3 (Maltese and Esperanto, I guess)
+- sloppy-iso-8859-6 (different Arabic)
+- sloppy-iso-8859-7 (Greek)
+- sloppy-iso-8859-8 (Hebrew)
+- sloppy-iso-8859-11 (Thai)
+
+Aliases such as "sloppy-cp1252" for "sloppy-windows-1252" will also be
+defined.
+
+Only sloppy-windows-1251 and sloppy-windows-1252 are used by the rest of ftfy;
+the rest are rather uncommon.
+
+Here are some examples, using `ftfy.explain_unicode` to illustrate how
+sloppy-windows-1252 merges Windows-1252 with Latin-1:
+
+    >>> from ftfy import explain_unicode
+    >>> some_bytes = b'\x80\x81\x82'
+    >>> explain_unicode(some_bytes.decode('latin-1'))
+    U+0080  \x80    [Cc] <unknown>
+    U+0081  \x81    [Cc] <unknown>
+    U+0082  \x82    [Cc] <unknown>
+
+    >>> explain_unicode(some_bytes.decode('windows-1252', 'replace'))
+    U+20AC  €       [Sc] EURO SIGN
+    U+FFFD  �       [So] REPLACEMENT CHARACTER
+    U+201A  ‚       [Ps] SINGLE LOW-9 QUOTATION MARK
+
+    >>> explain_unicode(some_bytes.decode('sloppy-windows-1252'))
+    U+20AC  €       [Sc] EURO SIGN
+    U+0081  \x81    [Cc] <unknown>
+    U+201A  ‚       [Ps] SINGLE LOW-9 QUOTATION MARK
+"""
+from __future__ import unicode_literals
+import codecs
+from encodings import normalize_encoding
+import sys
+
+REPLACEMENT_CHAR = '\ufffd'
+PY26 = sys.version_info[:2] == (2, 6)
+
+def make_sloppy_codec(encoding):
+    """
+    Take a codec name, and return a 'sloppy' version of that codec that can
+    encode and decode the unassigned bytes in that encoding.
+
+    Single-byte encodings in the standard library are defined using some
+    boilerplate classes surrounding the functions that do the actual work,
+    `codecs.charmap_decode` and `charmap_encode`. This function, given an
+    encoding name, *defines* those boilerplate classes.
+    """
+    # Make an array of all 256 possible bytes.
+    all_bytes = bytearray(range(256))
+
+    # Get a list of what they would decode to in Latin-1.
+    sloppy_chars = list(all_bytes.decode('latin-1'))
+
+    # Get a list of what they decode to in the given encoding. Use the
+    # replacement character for unassigned bytes.
+    if PY26:
+        decoded_chars = all_bytes.decode(encoding, 'replace')
+    else:
+        decoded_chars = all_bytes.decode(encoding, errors='replace')
+
+    # Update the sloppy_chars list. Each byte that was successfully decoded
+    # gets its decoded value in the list. The unassigned bytes are left as
+    # they are, which gives their decoding in Latin-1.
+    for i, char in enumerate(decoded_chars):
+        if char != REPLACEMENT_CHAR:
+            sloppy_chars[i] = char
+
+    # For ftfy's own purposes, we're going to allow byte 1A, the "Substitute"
+    # control code, to encode the Unicode replacement character U+FFFD.
+    sloppy_chars[0x1a] = REPLACEMENT_CHAR
+
+    # Create the data structures that tell the charmap methods how to encode
+    # and decode in this sloppy encoding.
+    decoding_table = ''.join(sloppy_chars)
+    encoding_table = codecs.charmap_build(decoding_table)
+
+    # Now produce all the class boilerplate. Look at the Python source for
+    # `encodings.cp1252` for comparison; this is almost exactly the same,
+    # except I made it follow pep8.
+    class Codec(codecs.Codec):
+        def encode(self, input, errors='strict'):
+            return codecs.charmap_encode(input, errors, encoding_table)
+
+        def decode(self, input, errors='strict'):
+            return codecs.charmap_decode(input, errors, decoding_table)
+
+    class IncrementalEncoder(codecs.IncrementalEncoder):
+        def encode(self, input, final=False):
+            return codecs.charmap_encode(input, self.errors, encoding_table)[0]
+
+    class IncrementalDecoder(codecs.IncrementalDecoder):
+        def decode(self, input, final=False):
+            return codecs.charmap_decode(input, self.errors, decoding_table)[0]
+
+    class StreamWriter(Codec, codecs.StreamWriter):
+        pass
+
+    class StreamReader(Codec, codecs.StreamReader):
+        pass
+
+    return codecs.CodecInfo(
+        name='sloppy-' + encoding,
+        encode=Codec().encode,
+        decode=Codec().decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamreader=StreamReader,
+        streamwriter=StreamWriter,
+    )
+
+# Define a codec for each incomplete encoding. The resulting CODECS dictionary
+# can be used by the main module of ftfy.bad_codecs.
+CODECS = {}
+INCOMPLETE_ENCODINGS = (
+    ['windows-%s' % num for num in range(1250, 1259)] +
+    ['iso-8859-%s' % num for num in (3, 6, 7, 8, 11)] +
+    ['cp%s' % num for num in range(1250, 1259)] + ['cp874']
+)
+
+for _encoding in INCOMPLETE_ENCODINGS:
+    _new_name = normalize_encoding('sloppy-' + _encoding)
+    CODECS[_new_name] = make_sloppy_codec(_encoding)

Contents/Libraries/Shared/ftfy/bad_codecs/utf8_variants.py

@@ -0,0 +1,282 @@
+r"""
+This file defines a codec called "utf-8-variants" (or "utf-8-var"), which can
+decode text that's been encoded with a popular non-standard version of UTF-8.
+This includes CESU-8, the accidental encoding made by layering UTF-8 on top of
+UTF-16, as well as Java's twist on CESU-8 that contains a two-byte encoding for
+codepoint 0.
+
+This is particularly relevant in Python 3, which provides no other way of
+decoding CESU-8 [1]_.
+
+The easiest way to use the codec is to simply import `ftfy.bad_codecs`:
+
+    >>> import ftfy.bad_codecs
+    >>> result = b'here comes a null! \xc0\x80'.decode('utf-8-var')
+    >>> print(repr(result).lstrip('u'))
+    'here comes a null! \x00'
+
+The codec does not at all enforce "correct" CESU-8. For example, the Unicode
+Consortium's not-quite-standard describing CESU-8 requires that there is only
+one possible encoding of any character, so it does not allow mixing of valid
+UTF-8 and CESU-8. This codec *does* allow that, just like Python 2's UTF-8
+decoder does.
+
+Characters in the Basic Multilingual Plane still have only one encoding. This
+codec still enforces the rule, within the BMP, that characters must appear in
+their shortest form. There is one exception: the sequence of bytes `0xc0 0x80`,
+instead of just `0x00`, may be used to encode the null character `U+0000`, like
+in Java.
+
+If you encode with this codec, you get legitimate UTF-8. Decoding with this
+codec and then re-encoding is not idempotent, although encoding and then
+decoding is. So this module won't produce CESU-8 for you. Look for that
+functionality in the sister module, "Breaks Text For You", coming approximately
+never.
+
+.. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec:
+   first decode the bytes (incorrectly), then encode them, then decode them
+   again, using UTF-8 as the codec every time.
+"""
+
+from __future__ import unicode_literals
+import re
+import codecs
+from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder,
+                             IncrementalEncoder as UTF8IncrementalEncoder)
+from ftfy.compatibility import bytes_to_ints, unichr, PYTHON2
+
+NAME = 'utf-8-variants'
+
+# This regular expression matches all possible six-byte CESU-8 sequences,
+# plus truncations of them at the end of the string. (If any of the
+# subgroups matches $, then all the subgroups after it also have to match $,
+# as there are no more characters to match.)
+CESU8_EXPR = (
+    b'('
+    b'\xed'
+    b'([\xa0-\xaf]|$)'
+    b'([\x80-\xbf]|$)'
+    b'(\xed|$)'
+    b'([\xb0-\xbf]|$)'
+    b'([\x80-\xbf]|$)'
+    b')'
+)
+
+CESU8_RE = re.compile(CESU8_EXPR)
+
+# This expression matches isolated surrogate characters that aren't
+# CESU-8, which have to be handled carefully on Python 2.
+SURROGATE_EXPR = (b'(\xed([\xa0-\xbf]|$)([\x80-\xbf]|$))')
+
+# This expression matches the Java encoding of U+0, including if it's
+# truncated and we need more bytes.
+NULL_EXPR = b'(\xc0(\x80|$))'
+
+# This regex matches cases that we need to decode differently from
+# standard UTF-8.
+SPECIAL_BYTES_RE = re.compile(b'|'.join([NULL_EXPR, CESU8_EXPR, SURROGATE_EXPR]))
+
+
+class IncrementalDecoder(UTF8IncrementalDecoder):
+    """
+    An incremental decoder that extends Python's built-in UTF-8 decoder.
+
+    This encoder needs to take in bytes, possibly arriving in a stream, and
+    output the correctly decoded text. The general strategy for doing this
+    is to fall back on the real UTF-8 decoder whenever possible, because
+    the real UTF-8 decoder is way optimized, but to call specialized methods
+    we define here for the cases the real encoder isn't expecting.
+    """
+    def _buffer_decode(self, input, errors, final):
+        """
+        Decode bytes that may be arriving in a stream, following the Codecs
+        API.
+
+        `input` is the incoming sequence of bytes. `errors` tells us how to
+        handle errors, though we delegate all error-handling cases to the real
+        UTF-8 decoder to ensure correct behavior. `final` indicates whether
+        this is the end of the sequence, in which case we should raise an
+        error given incomplete input.
+
+        Returns as much decoded text as possible, and the number of bytes
+        consumed.
+        """
+        # decoded_segments are the pieces of text we have decoded so far,
+        # and position is our current position in the byte string. (Bytes
+        # before this position have been consumed, and bytes after it have
+        # yet to be decoded.)
+        decoded_segments = []
+        position = 0
+        while True:
+            # Use _buffer_decode_step to decode a segment of text.
+            decoded, consumed = self._buffer_decode_step(
+                input[position:],
+                errors,
+                final
+            )
+            if consumed == 0:
+                # Either there's nothing left to decode, or we need to wait
+                # for more input. Either way, we're done for now.
+                break
+
+            # Append the decoded text to the list, and update our position.
+            decoded_segments.append(decoded)
+            position += consumed
+
+        if final:
+            # _buffer_decode_step must consume all the bytes when `final` is
+            # true.
+            assert position == len(input)
+
+        return ''.join(decoded_segments), position
+
+    def _buffer_decode_step(self, input, errors, final):
+        """
+        There are three possibilities for each decoding step:
+
+        - Decode as much real UTF-8 as possible.
+        - Decode a six-byte CESU-8 sequence at the current position.
+        - Decode a Java-style null at the current position.
+
+        This method figures out which step is appropriate, and does it.
+        """
+        # Get a reference to the superclass method that we'll be using for
+        # most of the real work.
+        sup = UTF8IncrementalDecoder._buffer_decode
+
+        # Find the next byte position that indicates a variant of UTF-8.
+        match = SPECIAL_BYTES_RE.search(input)
+        if match is None:
+            return sup(input, errors, final)
+
+        cutoff = match.start()
+        if cutoff > 0:
+            return sup(input[:cutoff], errors, True)
+
+        # Some byte sequence that we intend to handle specially matches
+        # at the beginning of the input.
+        if input.startswith(b'\xc0'):
+            if len(input) > 1:
+                # Decode the two-byte sequence 0xc0 0x80.
+                return '\u0000', 2
+            else:
+                if final:
+                    # We hit the end of the stream. Let the superclass method
+                    # handle it.
+                    return sup(input, errors, True)
+                else:
+                    # Wait to see another byte.
+                    return '', 0
+        else:
+            # Decode a possible six-byte sequence starting with 0xed.
+            return self._buffer_decode_surrogates(sup, input, errors, final)
+
+    @staticmethod
+    def _buffer_decode_surrogates(sup, input, errors, final):
+        """
+        When we have improperly encoded surrogates, we can still see the
+        bits that they were meant to represent.
+
+        The surrogates were meant to encode a 20-bit number, to which we
+        add 0x10000 to get a codepoint. That 20-bit number now appears in
+        this form:
+
+          11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst
+
+        The CESU8_RE above matches byte sequences of this form. Then we need
+        to extract the bits and assemble a codepoint number from them.
+        """
+        if len(input) < 6:
+            if final:
+                # We found 0xed near the end of the stream, and there aren't
+                # six bytes to decode. Delegate to the superclass method to
+                # handle it as an error.
+                if PYTHON2 and len(input) >= 3:
+                    # We can't trust Python 2 to raise an error when it's
+                    # asked to decode a surrogate, so let's force the issue.
+                    input = mangle_surrogates(input)
+                return sup(input, errors, final)
+            else:
+                # We found a surrogate, the stream isn't over yet, and we don't
+                # know enough of the following bytes to decode anything, so
+                # consume zero bytes and wait.
+                return '', 0
+        else:
+            if CESU8_RE.match(input):
+                # Given this is a CESU-8 sequence, do some math to pull out
+                # the intended 20-bit value, and consume six bytes.
+                bytenums = bytes_to_ints(input[:6])
+                codepoint = (
+                    ((bytenums[1] & 0x0f) << 16) +
+                    ((bytenums[2] & 0x3f) << 10) +
+                    ((bytenums[4] & 0x0f) << 6) +
+                    (bytenums[5] & 0x3f) +
+                    0x10000
+                )
+                return unichr(codepoint), 6
+            else:
+                # This looked like a CESU-8 sequence, but it wasn't one.
+                # 0xed indicates the start of a three-byte sequence, so give
+                # three bytes to the superclass to decode as usual -- except
+                # for working around the Python 2 discrepancy as before.
+                if PYTHON2:
+                    input = mangle_surrogates(input)
+                return sup(input[:3], errors, False)
+
+
+def mangle_surrogates(bytestring):
+    """
+    When Python 3 sees the UTF-8 encoding of a surrogate codepoint, it treats
+    it as an error (which it is). In 'replace' mode, it will decode as three
+    replacement characters. But Python 2 will just output the surrogate
+    codepoint.
+
+    To ensure consistency between Python 2 and Python 3, and protect downstream
+    applications from malformed strings, we turn surrogate sequences at the
+    start of the string into the bytes `ff ff ff`, which we're *sure* won't
+    decode, and which turn into three replacement characters in 'replace' mode.
+
+    This function does nothing in Python 3, and it will be deprecated in ftfy
+    5.0.
+    """
+    if PYTHON2:
+        if bytestring.startswith(b'\xed') and len(bytestring) >= 3:
+            decoded = bytestring[:3].decode('utf-8', 'replace')
+            if '\ud800' <= decoded <= '\udfff':
+                return b'\xff\xff\xff' + mangle_surrogates(bytestring[3:])
+        return bytestring
+    else:
+        # On Python 3, nothing needs to be done.
+        return bytestring
+
+# The encoder is identical to UTF-8.
+IncrementalEncoder = UTF8IncrementalEncoder
+
+
+# Everything below here is boilerplate that matches the modules in the
+# built-in `encodings` package.
+def encode(input, errors='strict'):
+    return IncrementalEncoder(errors).encode(input, final=True), len(input)
+
+
+def decode(input, errors='strict'):
+    return IncrementalDecoder(errors).decode(input, final=True), len(input)
+
+
+class StreamWriter(codecs.StreamWriter):
+    encode = encode
+
+
+class StreamReader(codecs.StreamReader):
+    decode = decode
+
+
+CODEC_INFO = codecs.CodecInfo(
+    name=NAME,
+    encode=encode,
+    decode=decode,
+    incrementalencoder=IncrementalEncoder,
+    incrementaldecoder=IncrementalDecoder,
+    streamreader=StreamReader,
+    streamwriter=StreamWriter,
+)

Contents/Libraries/Shared/ftfy/badness.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+"""
+Heuristics to determine whether re-encoding text is actually making it
+more reasonable.
+"""
+
+from __future__ import unicode_literals
+import re
+import unicodedata
+from ftfy.chardata import chars_to_classes
+
+# The following regex uses the mapping of character classes to ASCII
+# characters defined in chardata.py and build_data.py:
+#
+# L = Latin capital letter
+# l = Latin lowercase letter
+# A = Non-latin capital or title-case letter
+# a = Non-latin lowercase letter
+# C = Non-cased letter (Lo)
+# X = Control character (Cc)
+# m = Letter modifier (Lm)
+# M = Mark (Mc, Me, Mn)
+# N = Miscellaneous numbers (No)
+# 1 = Math symbol (Sm) or currency symbol (Sc)
+# 2 = Symbol modifier (Sk)
+# 3 = Other symbol (So)
+# S = UTF-16 surrogate
+# _ = Unassigned character
+#   = Whitespace
+# o = Other
+
+
+def _make_weirdness_regex():
+    """
+    Creates a list of regexes that match 'weird' character sequences.
+    The more matches there are, the weirder the text is.
+    """
+    groups = []
+
+    # Match lowercase letters that are followed by non-ASCII uppercase letters
+    groups.append('lA')
+
+    # Match diacritical marks, except when they modify a non-cased letter or
+    # another mark.
+    #
+    # You wouldn't put a diacritical mark on a digit or a space, for example.
+    # You might put it on a Latin letter, but in that case there will almost
+    # always be a pre-composed version, and we normalize to pre-composed
+    # versions first. The cases that can't be pre-composed tend to be in
+    # large scripts without case, which are in class C.
+    groups.append('[^CM]M')
+
+    # Match non-Latin characters adjacent to Latin characters.
+    #
+    # This is a simplification from ftfy version 2, which compared all
+    # adjacent scripts. However, the ambiguities we need to resolve come from
+    # encodings designed to represent Latin characters.
+    groups.append('[Ll][AaC]')
+    groups.append('[AaC][Ll]')
+
+    # Match IPA letters next to capital letters.
+    #
+    # IPA uses lowercase letters only. Some accented capital letters next to
+    # punctuation can accidentally decode as IPA letters, and an IPA letter
+    # appearing next to a capital letter is a good sign that this happened.
+    groups.append('[LA]i')
+    groups.append('i[LA]')
+
+    # Match non-combining diacritics. We've already set aside the common ones
+    # like ^ (the CIRCUMFLEX ACCENT, repurposed as a caret, exponent sign,
+    # or happy eye) and assigned them to category 'o'. The remaining ones,
+    # like the diaeresis (¨), are pretty weird to see on their own instead
+    # of combined with a letter.
+    groups.append('2')
+
+    # Match C1 control characters, which are almost always the result of
+    # decoding Latin-1 that was meant to be Windows-1252.
+    groups.append('X')
+
+    # Match private use and unassigned characters.
+    groups.append('P')
+    groups.append('_')
+
+    # Match adjacent characters from any different pair of these categories:
+    # - Modifier marks (M)
+    # - Letter modifiers (m)
+    # - Miscellaneous numbers (N)
+    # - Symbols (1 or 3, because 2 is already weird on its own)
+
+    exclusive_categories = 'MmN13'
+    for cat1 in exclusive_categories:
+        others_range = ''.join(c for c in exclusive_categories if c != cat1)
+        groups.append('{cat1}[{others_range}]'.format(
+            cat1=cat1, others_range=others_range
+        ))
+    regex = '|'.join('({0})'.format(group) for group in groups)
+    return re.compile(regex)
+
+WEIRDNESS_RE = _make_weirdness_regex()
+
+# These characters appear in mojibake but also appear commonly on their own.
+# We have a slight preference to leave them alone.
+COMMON_SYMBOL_RE = re.compile(
+    '['
+    '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}'
+    '\N{LEFT SINGLE QUOTATION MARK}\N{LEFT DOUBLE QUOTATION MARK}'
+    '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}'
+    '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}\N{DEGREE SIGN}'
+    '\N{TRADE MARK SIGN}'
+    '\N{REGISTERED SIGN}'
+    '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}'
+    '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}'
+    '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}'
+    '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}'
+    '\N{NO-BREAK SPACE}'
+    '\N{ACUTE ACCENT}\N{MULTIPLICATION SIGN}\N{LATIN SMALL LETTER SHARP S}'
+    '\ufeff'  # The byte-order mark, whose encoding '' looks common
+    ']'
+)
+
+def sequence_weirdness(text):
+    """
+    Determine how often a text has unexpected characters or sequences of
+    characters. This metric is used to disambiguate when text should be
+    re-decoded or left as is.
+
+    We start by normalizing text in NFC form, so that penalties for
+    diacritical marks don't apply to characters that know what to do with
+    them.
+
+    The following things are deemed weird:
+
+    - Lowercase letters followed by non-ASCII uppercase letters
+    - Non-Latin characters next to Latin characters
+    - Un-combined diacritical marks, unless they're stacking on non-alphabetic
+      characters (in languages that do that kind of thing a lot) or other
+      marks
+    - C1 control characters
+    - Adjacent symbols from any different pair of these categories:
+
+        - Modifier marks
+        - Letter modifiers
+        - Non-digit numbers
+        - Symbols (including math and currency)
+
+    The return value is the number of instances of weirdness.
+    """
+    text2 = unicodedata.normalize('NFC', text)
+    weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2)))
+    punct_discount = len(COMMON_SYMBOL_RE.findall(text2))
+    return weirdness * 2 - punct_discount
+
+
+def text_cost(text):
+    """
+    An overall cost function for text. Weirder is worse, but all else being
+    equal, shorter strings are better.
+
+    The overall cost is measured as the "weirdness" (see
+    :func:`sequence_weirdness`) plus the length.
+    """
+    return sequence_weirdness(text) + len(text)

Contents/Libraries/Shared/ftfy/build_data.py

@@ -0,0 +1,132 @@
+"""
+A script to make the char_classes.dat file.
+
+This never needs to run in normal usage. It needs to be run if the character
+classes we care about change, or if a new version of Python supports a new
+Unicode standard and we want it to affect our string decoding.
+
+The file that we generate is based on Unicode 9.0, as supported by Python 3.6.
+You can certainly use it in earlier versions. This simply makes sure that we
+get consistent results from running ftfy on different versions of Python.
+
+The file will be written to the current directory.
+"""
+from __future__ import unicode_literals
+import unicodedata
+import sys
+import zlib
+if sys.hexversion >= 0x03000000:
+    unichr = chr
+
+# L = Latin capital letter
+# l = Latin lowercase letter
+# A = Non-latin capital or title-case letter
+# a = Non-latin lowercase letter
+# C = Non-cased letter (Lo)
+# X = Control character (Cc)
+# m = Letter modifier (Lm)
+# M = Mark (Mc, Me, Mn)
+# N = Miscellaneous numbers (No)
+# P = Private use (Co)
+# 1 = Math symbol (Sm) or currency symbol (Sc)
+# 2 = Symbol modifier (Sk)
+# 3 = Other symbol (So)
+# S = UTF-16 surrogate
+# _ = Unassigned character
+#   = Whitespace
+# o = Other
+
+
+def make_char_data_file(do_it_anyway=False):
+    """
+    Build the compressed data file 'char_classes.dat' and write it to the
+    current directory.
+
+    If you run this, run it in Python 3.6 or later. It will run in earlier
+    versions, but you won't get the Unicode 9 standard, leading to inconsistent
+    behavior.
+
+    To protect against this, running this in the wrong version of Python will
+    raise an error unless you pass `do_it_anyway=True`.
+    """
+    if sys.hexversion < 0x03060000 and not do_it_anyway:
+        raise RuntimeError(
+            "This function should be run in Python 3.6 or later."
+        )
+
+    cclasses = [None] * 0x110000
+    for codepoint in range(0x0, 0x110000):
+        char = unichr(codepoint)
+        category = unicodedata.category(char)
+
+        if (0x250 <= codepoint < 0x300) and char != 'ə':
+            # IPA symbols and modifiers.
+            #
+            # This category excludes the schwa (ə), which is used as a normal
+            # Latin letter in some languages.
+            cclasses[codepoint] = 'i'
+        elif category.startswith('L'):  # letters
+            if unicodedata.name(char, '').startswith('LATIN'):
+                if category == 'Lu':
+                    cclasses[codepoint] = 'L'
+                else:
+                    cclasses[codepoint] = 'l'
+            else:
+                if category == 'Lu' or category == 'Lt':
+                    cclasses[codepoint] = 'A'
+                elif category == 'Ll':
+                    cclasses[codepoint] = 'a'
+                elif category == 'Lo':
+                    cclasses[codepoint] = 'C'
+                elif category == 'Lm':
+                    cclasses[codepoint] = 'm'
+                else:
+                    raise ValueError('got some weird kind of letter')
+        elif 0xfe00 <= codepoint <= 0xfe0f or 0x1f3fb <= codepoint <= 0x1f3ff:
+            # Variation selectors and skin-tone modifiers have the category
+            # of non-spacing marks, but they act like symbols
+            cclasses[codepoint] = '3'
+        elif category.startswith('M'):  # marks
+            cclasses[codepoint] = 'M'
+        elif category == 'No':
+            cclasses[codepoint] = 'N'
+        elif category == 'Sm' or category == 'Sc':
+            cclasses[codepoint] = '1'
+        elif category == 'Sk':
+            cclasses[codepoint] = '2'
+        elif category == 'So':
+            cclasses[codepoint] = '3'
+        elif category == 'Cc':
+            cclasses[codepoint] = 'X'
+        elif category == 'Cs':
+            cclasses[codepoint] = 'S'
+        elif category == 'Co':
+            cclasses[codepoint] = 'P'
+        elif category.startswith('Z'):
+            cclasses[codepoint] = ' '
+        elif 0x1f000 <= codepoint <= 0x1ffff:
+            # This range is rapidly having emoji added to it. Assume that
+            # an unassigned codepoint in this range is just a symbol we
+            # don't know yet.
+            cclasses[codepoint] = '3'
+        elif category == 'Cn':
+            cclasses[codepoint] = '_'
+        else:
+            cclasses[codepoint] = 'o'
+
+    # Mark whitespace control characters as whitespace
+    cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' '
+
+    # Some other exceptions for characters that are more commonly used as
+    # punctuation or decoration than for their ostensible purpose.
+    # For example, tilde is not usually a "math symbol", and the accents
+    # `´ are much more like quotation marks than modifiers.
+    for char in "^~`´˝＾｀":
+        cclasses[ord(char)] = 'o'
+
+    out = open('char_classes.dat', 'wb')
+    out.write(zlib.compress(''.join(cclasses).encode('ascii')))
+    out.close()
+
+if __name__ == '__main__':
+    make_char_data_file()

Contents/Libraries/Shared/ftfy/chardata.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+"""
+This gives other modules access to the gritty details about characters and the
+encodings that use them.
+"""
+
+from __future__ import unicode_literals
+import re
+import zlib
+import unicodedata
+import itertools
+from pkg_resources import resource_string
+from ftfy.compatibility import unichr
+
+# These are the encodings we will try to fix in ftfy, in the
+# order that they should be tried.
+CHARMAP_ENCODINGS = [
+    'latin-1',
+    'sloppy-windows-1252',
+    'sloppy-windows-1250',
+    'iso-8859-2',
+    'sloppy-windows-1251',
+    'macroman',
+    'cp437',
+]
+
+
+def _build_regexes():
+    """
+    ENCODING_REGEXES contain reasonably fast ways to detect if we
+    could represent a given string in a given encoding. The simplest one is
+    the 'ascii' detector, which of course just determines if all characters
+    are between U+0000 and U+007F.
+    """
+    # Define a regex that matches ASCII text.
+    encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')}
+
+    for encoding in CHARMAP_ENCODINGS:
+        # Make a sequence of characters that bytes \x80 to \xFF decode to
+        # in each encoding, as well as byte \x1A, which is used to represent
+        # the replacement character � in the sloppy-* encodings.
+        latin1table = ''.join(unichr(i) for i in range(128, 256)) + '\x1a'
+        charlist = latin1table.encode('latin-1').decode(encoding)
+
+        # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B
+        # to \x7F -- will decode as those ASCII characters in any encoding we
+        # support, so we can just include them as ranges. This also lets us
+        # not worry about escaping regex special characters, because all of
+        # them are in the \x1B to \x7F range.
+        regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist)
+        encoding_regexes[encoding] = re.compile(regex)
+    return encoding_regexes
+ENCODING_REGEXES = _build_regexes()
+
+
+def _build_utf8_punct_regex():
+    """
+    Recognize UTF-8 mojibake that's so blatant that we can fix it even when the
+    rest of the string doesn't decode as UTF-8 -- namely, UTF-8 sequences for
+    the 'General Punctuation' characters U+2000 to U+2040, re-encoded in
+    Windows-1252.
+
+    These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they
+    all begin with when decoded as Windows-1252.
+    """
+    # We're making a regex that has all the literal bytes from 0x80 to 0xbf in
+    # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask.
+    # However, when we decode the regex as Windows-1252, the resulting
+    # characters won't even be remotely contiguous.
+    #
+    # Unrelatedly, the expression that generates these bytes will be so much
+    # prettier when we deprecate Python 2.
+    continuation_char_list = ''.join(
+        unichr(i) for i in range(0x80, 0xc0)
+    ).encode('latin-1')
+    obvious_utf8 = ('â€['
+                    + continuation_char_list.decode('sloppy-windows-1252')
+                    + ']')
+    return re.compile(obvious_utf8)
+PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex()
+
+
+# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0'
+# that some Windows-1252 program converted to a plain space.
+#
+# The smaller values are included on a case-by-case basis, because we don't want
+# to decode likely input sequences to unlikely characters. These are the ones
+# that *do* form likely characters before 0xa0:
+#
+#   0xc2 -> U+A0 NO-BREAK SPACE
+#   0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE
+#   0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON
+#   0xce -> U+3A0 GREEK CAPITAL LETTER PI
+#   0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER
+#
+# These still need to come with a cost, so that they only get converted when
+# there's evidence that it fixes other things. Any of these could represent
+# characters that legitimately appear surrounded by spaces, particularly U+C5
+# (Å), which is a word in multiple languages!
+#
+# We should consider checking for b'\x85' being converted to ... in the future.
+# I've seen it once, but the text still wasn't recoverable.
+
+ALTERED_UTF8_RE = re.compile(b'[\xc2\xc3\xc5\xce\xd0][ ]'
+                             b'|[\xe0-\xef][ ][\x80-\xbf]'
+                             b'|[\xe0-\xef][\x80-\xbf][ ]'
+                             b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]'
+                             b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]'
+                             b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]')
+
+# This expression matches UTF-8 and CESU-8 sequences where some of the
+# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is
+# used within ftfy to represent a byte that produced the replacement character
+# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8
+# sequence as \ufffd instead of failing to re-decode it at all.
+LOSSY_UTF8_RE = re.compile(
+    b'[\xc2-\xdf][\x1a]'
+    b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]'
+    b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]'
+    b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]'
+    b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a]'
+    b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]'
+    b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]'
+    b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]'
+    b'|\x1a'
+)
+
+# These regexes match various Unicode variations on single and double quotes.
+SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]')
+DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]')
+
+
+def possible_encoding(text, encoding):
+    """
+    Given text and a single-byte encoding, check whether that text could have
+    been decoded from that single-byte encoding.
+
+    In other words, check whether it can be encoded in that encoding, possibly
+    sloppily.
+    """
+    return bool(ENCODING_REGEXES[encoding].match(text))
+
+
+CHAR_CLASS_STRING = zlib.decompress(
+    resource_string(__name__, 'char_classes.dat')
+).decode('ascii')
+
+def chars_to_classes(string):
+    """
+    Convert each Unicode character to a letter indicating which of many
+    classes it's in.
+
+    See build_data.py for where this data comes from and what it means.
+    """
+    return string.translate(CHAR_CLASS_STRING)
+
+
+def _build_control_char_mapping():
+    """
+    Build a translate mapping that strips likely-unintended control characters.
+    See :func:`ftfy.fixes.remove_control_chars` for a description of these
+    codepoint ranges and why they should be removed.
+    """
+    control_chars = {}
+
+    for i in itertools.chain(
+        range(0x00, 0x09), [0x0b],
+        range(0x0e, 0x20), [0x7f],
+        range(0x206a, 0x2070),
+        [0xfeff],
+        range(0xfff9, 0xfffd),
+        range(0x1d173, 0x1d17b),
+        range(0xe0000, 0xe0080)
+    ):
+        control_chars[i] = None
+
+    return control_chars
+CONTROL_CHARS = _build_control_char_mapping()
+
+
+# A translate mapping that breaks ligatures made of Latin letters. While
+# ligatures may be important to the representation of other languages, in
+# Latin letters they tend to represent a copy/paste error.
+#
+# Ligatures may also be separated by NFKC normalization, but that is sometimes
+# more normalization than you want.
+LIGATURES = {
+    ord('IJ'): 'IJ',
+    ord('ij'): 'ij',
+    ord('ff'): 'ff',
+    ord('fi'): 'fi',
+    ord('fl'): 'fl',
+    ord('ffi'): 'ffi',
+    ord('ffl'): 'ffl',
+    ord('ſt'): 'ſt',
+    ord('st'): 'st'
+}
+
+
+def _build_width_map():
+    """
+    Build a translate mapping that replaces halfwidth and fullwidth forms
+    with their standard-width forms.
+    """
+    # Though it's not listed as a fullwidth character, we'll want to convert
+    # U+3000 IDEOGRAPHIC SPACE to U+20 SPACE on the same principle, so start
+    # with that in the dictionary.
+    width_map = {0x3000: ' '}
+    for i in range(0xff01, 0xfff0):
+        char = unichr(i)
+        alternate = unicodedata.normalize('NFKC', char)
+        if alternate != char:
+            width_map[i] = alternate
+    return width_map
+WIDTH_MAP = _build_width_map()

Contents/Libraries/Shared/ftfy/cli.py

@@ -0,0 +1,122 @@
+"""
+A command-line utility for fixing text found in a file.
+"""
+
+import sys
+import io
+import codecs
+from ftfy import fix_file, __version__
+from ftfy.compatibility import PYTHON2
+
+
+ENCODE_ERROR_TEXT_UNIX = """ftfy error:
+Unfortunately, this output stream does not support Unicode.
+
+Your system locale may be very old or misconfigured. You should use a locale
+that supports UTF-8. One way to do this is to `export LANG=C.UTF-8`.
+"""
+
+ENCODE_ERROR_TEXT_WINDOWS = """ftfy error:
+Unfortunately, this output stream does not support Unicode.
+
+You might be trying to output to the Windows Command Prompt (cmd.exe), which
+does not fully support Unicode for historical reasons. In general, we recommend
+finding a way to run Python without using cmd.exe.
+
+You can work around this problem by using the '-o filename' option in ftfy to
+output to a file instead.
+"""
+
+DECODE_ERROR_TEXT = """ftfy error:
+This input couldn't be decoded as %r. We got the following error:
+
+    %s
+
+ftfy works best when its input is in a known encoding. You can use `ftfy -g`
+to guess, if you're desperate. Otherwise, give the encoding name with the
+`-e` option, such as `ftfy -e latin-1`.
+"""
+
+
+def main():
+    """
+    Run ftfy as a command-line utility.
+    """
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="ftfy (fixes text for you), version %s" % __version__
+    )
+    parser.add_argument('filename', default='-', nargs='?',
+                        help='The file whose Unicode is to be fixed. Defaults '
+                             'to -, meaning standard input.')
+    parser.add_argument('-o', '--output', type=str, default='-',
+                        help='The file to output to. Defaults to -, meaning '
+                             'standard output.')
+    parser.add_argument('-g', '--guess', action='store_true',
+                        help="Ask ftfy to guess the encoding of your input. "
+                             "This is risky. Overrides -e.")
+    parser.add_argument('-e', '--encoding', type=str, default='utf-8',
+                        help='The encoding of the input. Defaults to UTF-8.')
+    parser.add_argument('-n', '--normalization', type=str, default='NFC',
+                        help='The normalization of Unicode to apply. '
+                             'Defaults to NFC. Can be "none".')
+    parser.add_argument('--preserve-entities', action='store_true',
+                        help="Leave HTML entities as they are. The default "
+                             "is to decode them, as long as no HTML tags "
+                             "have appeared in the file.")
+
+    args = parser.parse_args()
+
+    encoding = args.encoding
+    if args.guess:
+        encoding = None
+
+    if args.filename == '-':
+        # Get a standard input stream made of bytes, so we can decode it as
+        # whatever encoding is necessary.
+        if PYTHON2:
+            file = sys.stdin
+        else:
+            file = sys.stdin.buffer
+    else:
+        file = open(args.filename, 'rb')
+
+    if args.output == '-':
+        encode_output = PYTHON2
+        outfile = sys.stdout
+    else:
+        encode_output = False
+        outfile = io.open(args.output, 'w', encoding='utf-8')
+
+    normalization = args.normalization
+    if normalization.lower() == 'none':
+        normalization = None
+
+    if args.preserve_entities:
+        fix_entities = False
+    else:
+        fix_entities = 'auto'
+
+    try:
+        for line in fix_file(file, encoding=encoding,
+                             fix_entities=fix_entities,
+                             normalization=normalization):
+            if encode_output:
+                outfile.write(line.encode('utf-8'))
+            else:
+                try:
+                    outfile.write(line)
+                except UnicodeEncodeError:
+                    if sys.platform == 'win32':
+                        sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS)
+                    else:
+                        sys.stderr.write(ENCODE_ERROR_TEXT_UNIX)
+                    sys.exit(1)
+    except UnicodeDecodeError as err:
+        sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

Contents/Libraries/Shared/ftfy/compatibility.py

@@ -0,0 +1,55 @@
+"""
+Makes some function names and behavior consistent between Python 2 and
+Python 3, and also between narrow and wide builds.
+"""
+from __future__ import unicode_literals
+import sys
+import unicodedata
+
+if sys.hexversion >= 0x03000000:
+    unichr = chr
+    xrange = range
+    PYTHON2 = False
+else:
+    unichr = unichr
+    xrange = xrange
+    PYTHON2 = True
+
+PYTHON34_OR_LATER = (sys.hexversion >= 0x03040000)
+
+
+def _narrow_unichr_workaround(codepoint):
+    """
+    A replacement for unichr() on narrow builds of Python. This will get
+    us the narrow representation of an astral character, which will be
+    a string of length two, containing two UTF-16 surrogates.
+    """
+    escaped = b'\\U%08x' % codepoint
+    return escaped.decode('unicode-escape')
+
+
+if sys.maxunicode < 0x10000:
+    unichr = _narrow_unichr_workaround
+
+
+def bytes_to_ints(bytestring):
+    """
+    No matter what version of Python this is, make a sequence of integers from
+    a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a
+    sequence of integers.
+    """
+    if PYTHON2:
+        return [ord(b) for b in bytestring]
+    else:
+        return bytestring
+
+
+def is_printable(char):
+    """
+    str.isprintable() is new in Python 3. It's useful in `explain_unicode`, so
+    let's make a crude approximation in Python 2.
+    """
+    if PYTHON2:
+        return not unicodedata.category(char).startswith('C')
+    else:
+        return char.isprintable()

Contents/Libraries/Shared/ftfy/fixes.py

@@ -0,0 +1,664 @@
+# -*- coding: utf-8 -*-
+"""
+This module contains the individual fixes that the main fix_text function
+can perform.
+"""
+
+from __future__ import unicode_literals
+import re
+import sys
+import codecs
+import warnings
+from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS,
+                           CONTROL_CHARS, LIGATURES, WIDTH_MAP,
+                           PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE,
+                           LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE)
+from ftfy.badness import text_cost
+from ftfy.compatibility import unichr
+from html5lib.constants import entities
+
+
+BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode.
+
+ftfy is designed to fix problems that were introduced by handling Unicode
+incorrectly. It might be able to fix the bytes you just handed it, but the
+fact that you just gave a pile of bytes to a function that fixes text means
+that your code is *also* handling Unicode incorrectly.
+
+ftfy takes Unicode text as input. You should take these bytes and decode
+them from the encoding you think they are in. If you're not sure what encoding
+they're in:
+
+- First, try to find out. 'utf-8' is a good assumption.
+- If the encoding is simply unknowable, try running your bytes through
+  ftfy.guess_bytes. As the name implies, this may not always be accurate.
+
+If you're confused by this, please read the Python Unicode HOWTO:
+
+    http://docs.python.org/%d/howto/unicode.html
+""" % sys.version_info[0]
+
+
+def fix_encoding(text):
+    r"""
+    Fix text with incorrectly-decoded garbage ("mojibake") whenever possible.
+
+    This function looks for the evidence of mojibake, formulates a plan to fix
+    it, and applies the plan.  It determines whether it should replace nonsense
+    sequences of single-byte characters that were really meant to be UTF-8
+    characters, and if so, turns them into the correctly-encoded Unicode
+    character that they were meant to represent.
+
+    The input to the function must be Unicode. If you don't have Unicode text,
+    you're not using the right tool to solve your problem.
+
+    `fix_encoding` decodes text that looks like it was decoded incorrectly. It
+    leaves alone text that doesn't.
+
+        >>> print(fix_encoding('único'))
+        único
+
+        >>> print(fix_encoding('This text is fine already :þ'))
+        This text is fine already :þ
+
+    Because these characters often come from Microsoft products, we allow
+    for the possibility that we get not just Unicode characters 128-255, but
+    also Windows's conflicting idea of what characters 128-160 are.
+
+        >>> print(fix_encoding('This — should be an em dash'))
+        This — should be an em dash
+
+    We might have to deal with both Windows characters and raw control
+    characters at the same time, especially when dealing with characters like
+    0x81 that have no mapping in Windows. This is a string that Python's
+    standard `.encode` and `.decode` methods cannot correct.
+
+        >>> print(fix_encoding('This text is sad .â\x81”.'))
+        This text is sad .⁔.
+
+    However, it has safeguards against fixing sequences of letters and
+    punctuation that can occur in valid text. In the following example,
+    the last three characters are not replaced with a Korean character,
+    even though they could be.
+
+        >>> print(fix_encoding('not such a fan of Charlotte Brontë…”'))
+        not such a fan of Charlotte Brontë…”
+
+    This function can now recover some complex manglings of text, such as when
+    UTF-8 mojibake has been normalized in a way that replaces U+A0 with a
+    space:
+
+        >>> print(fix_encoding('The more you know 🌠'))
+        The more you know 🌠
+
+    Cases of genuine ambiguity can sometimes be addressed by finding other
+    characters that are not double-encoded, and expecting the encoding to
+    be consistent:
+
+        >>> print(fix_encoding('AHÅ™, the new sofa from IKEA®'))
+        AHÅ™, the new sofa from IKEA®
+
+    Finally, we handle the case where the text is in a single-byte encoding
+    that was intended as Windows-1252 all along but read as Latin-1:
+
+        >>> print(fix_encoding('This text was never UTF-8 at all\x85'))
+        This text was never UTF-8 at all…
+
+    The best version of the text is found using
+    :func:`ftfy.badness.text_cost`.
+    """
+    text, _ = fix_encoding_and_explain(text)
+    return text
+
+
+def fix_text_encoding(text):
+    """
+    A deprecated name for :func:`ftfy.fixes.fix_encoding`.
+    """
+    warnings.warn('fix_text_encoding is now known as fix_encoding',
+                  DeprecationWarning)
+    return fix_encoding(text)
+
+
+# When we support discovering mojibake in more encodings, we run the risk
+# of more false positives. We can mitigate false positives by assigning an
+# additional cost to using encodings that are rarer than Windows-1252, so
+# that these encodings will only be used if they fix multiple problems.
+ENCODING_COSTS = {
+    'macroman': 2,
+    'iso-8859-2': 2,
+    'sloppy-windows-1250': 2,
+    'sloppy-windows-1251': 3,
+    'cp437': 3,
+}
+
+
+def fix_encoding_and_explain(text):
+    """
+    Re-decodes text that has been decoded incorrectly, and also return a
+    "plan" indicating all the steps required to fix it.
+
+    The resulting plan could be used with :func:`ftfy.fixes.apply_plan`
+    to fix additional strings that are broken in the same way.
+    """
+    best_version = text
+    best_cost = text_cost(text)
+    best_plan = []
+    plan_so_far = []
+    while True:
+        prevtext = text
+        text, plan = fix_one_step_and_explain(text)
+        plan_so_far.extend(plan)
+        cost = text_cost(text)
+        for _, _, step_cost in plan_so_far:
+            cost += step_cost
+
+        if cost < best_cost:
+            best_cost = cost
+            best_version = text
+            best_plan = list(plan_so_far)
+        if text == prevtext:
+            return best_version, best_plan
+
+
+def fix_one_step_and_explain(text):
+    """
+    Performs a single step of re-decoding text that's been decoded incorrectly.
+
+    Returns the decoded text, plus a "plan" for how to reproduce what it did.
+    """
+    if isinstance(text, bytes):
+        raise UnicodeError(BYTES_ERROR_TEXT)
+    if len(text) == 0:
+        return text, []
+
+    # The first plan is to return ASCII text unchanged.
+    if possible_encoding(text, 'ascii'):
+        return text, []
+
+    # As we go through the next step, remember the possible encodings
+    # that we encounter but don't successfully fix yet. We may need them
+    # later.
+    possible_1byte_encodings = []
+
+    # Suppose the text was supposed to be UTF-8, but it was decoded using
+    # a single-byte encoding instead. When these cases can be fixed, they
+    # are usually the correct thing to do, so try them next.
+    for encoding in CHARMAP_ENCODINGS:
+        if possible_encoding(text, encoding):
+            encoded_bytes = text.encode(encoding)
+            encode_step = ('encode', encoding, ENCODING_COSTS.get(encoding, 0))
+            transcode_steps = []
+
+            # Now, find out if it's UTF-8 (or close enough). Otherwise,
+            # remember the encoding for later.
+            try:
+                decoding = 'utf-8'
+                # Check encoded_bytes for sequences that would be UTF-8,
+                # except they have b' ' where b'\xa0' would belong.
+                if ALTERED_UTF8_RE.search(encoded_bytes):
+                    encoded_bytes = restore_byte_a0(encoded_bytes)
+                    cost = encoded_bytes.count(b'\xa0') * 2
+                    transcode_steps.append(('transcode', 'restore_byte_a0', cost))
+
+                # Check for the byte 0x1a, which indicates where one of our
+                # 'sloppy' codecs found a replacement character.
+                if encoding.startswith('sloppy') and b'\x1a' in encoded_bytes:
+                    encoded_bytes = replace_lossy_sequences(encoded_bytes)
+                    transcode_steps.append(('transcode', 'replace_lossy_sequences', 0))
+
+                if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes:
+                    decoding = 'utf-8-variants'
+
+                decode_step = ('decode', decoding, 0)
+                steps = [encode_step] + transcode_steps + [decode_step]
+                fixed = encoded_bytes.decode(decoding)
+                return fixed, steps
+
+            except UnicodeDecodeError:
+                possible_1byte_encodings.append(encoding)
+
+    # Look for a-hat-euro sequences that remain, and fix them in isolation.
+    if PARTIAL_UTF8_PUNCT_RE.search(text):
+        steps = [('transcode', 'fix_partial_utf8_punct_in_1252', 1)]
+        fixed = fix_partial_utf8_punct_in_1252(text)
+        return fixed, steps
+
+    # The next most likely case is that this is Latin-1 that was intended to
+    # be read as Windows-1252, because those two encodings in particular are
+    # easily confused.
+    if 'latin-1' in possible_1byte_encodings:
+        if 'windows-1252' in possible_1byte_encodings:
+            # This text is in the intersection of Latin-1 and
+            # Windows-1252, so it's probably legit.
+            return text, []
+        else:
+            # Otherwise, it means we have characters that are in Latin-1 but
+            # not in Windows-1252. Those are C1 control characters. Nobody
+            # wants those. Assume they were meant to be Windows-1252. Don't
+            # use the sloppy codec, because bad Windows-1252 characters are
+            # a bad sign.
+            encoded = text.encode('latin-1')
+            try:
+                fixed = encoded.decode('windows-1252')
+                steps = []
+                if fixed != text:
+                    steps = [('encode', 'latin-1', 0),
+                             ('decode', 'windows-1252', 1)]
+                return fixed, steps
+            except UnicodeDecodeError:
+                # This text contained characters that don't even make sense
+                # if you assume they were supposed to be Windows-1252. In
+                # that case, let's not assume anything.
+                pass
+
+    # The cases that remain are mixups between two different single-byte
+    # encodings, and not the common case of Latin-1 vs. Windows-1252.
+    #
+    # These cases may be unsolvable without adding false positives, though
+    # I have vague ideas about how to optionally address them in the future.
+
+    # Return the text unchanged; the plan is empty.
+    return text, []
+
+
+def apply_plan(text, plan):
+    """
+    Apply a plan for fixing the encoding of text.
+
+    The plan is a list of tuples of the form (operation, encoding, cost):
+
+    - `operation` is 'encode' if it turns a string into bytes, 'decode' if it
+      turns bytes into a string, and 'transcode' if it keeps the type the same.
+    - `encoding` is the name of the encoding to use, such as 'utf-8' or
+      'latin-1', or the function name in the case of 'transcode'.
+    - The `cost` does not affect how the plan itself works. It's used by other
+      users of plans, namely `fix_encoding_and_explain`, which has to decide
+      *which* plan to use.
+    """
+    obj = text
+    for operation, encoding, _ in plan:
+        if operation == 'encode':
+            obj = obj.encode(encoding)
+        elif operation == 'decode':
+            obj = obj.decode(encoding)
+        elif operation == 'transcode':
+            if encoding in TRANSCODERS:
+                obj = TRANSCODERS[encoding](obj)
+            else:
+                raise ValueError("Unknown transcode operation: %s" % encoding)
+        else:
+            raise ValueError("Unknown plan step: %s" % operation)
+
+    return obj
+
+
+HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};")
+
+
+def unescape_html(text):
+    """
+    Decode all three types of HTML entities/character references.
+
+    Code by Fredrik Lundh of effbot.org. Rob Speer made a slight change
+    to it for efficiency: it won't match entities longer than 8 characters,
+    because there are no valid entities like that.
+
+        >>> print(unescape_html('&lt;tag&gt;'))
+        <tag>
+    """
+    def fixup(match):
+        """
+        Replace one matched HTML entity with the character it represents,
+        if possible.
+        """
+        text = match.group(0)
+        if text[:2] == "&#":
+            # character reference
+            try:
+                if text[:3] == "&#x":
+                    codept = int(text[3:-1], 16)
+                else:
+                    codept = int(text[2:-1])
+                if 0x80 <= codept < 0xa0:
+                    # Decode this range of characters as Windows-1252, as Web
+                    # browsers do in practice.
+                    return unichr(codept).encode('latin-1').decode('sloppy-windows-1252')
+                else:
+                    return unichr(codept)
+            except ValueError:
+                pass
+        else:
+            # named entity
+            try:
+                text = entities[text[1:]]
+            except KeyError:
+                pass
+        return text  # leave as is
+    return HTML_ENTITY_RE.sub(fixup, text)
+
+
+ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])')
+
+def remove_terminal_escapes(text):
+    r"""
+    Strip out "ANSI" terminal escape sequences, such as those that produce
+    colored text on Unix.
+
+        >>> print(remove_terminal_escapes(
+        ...     "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m"
+        ... ))
+        I'm blue, da ba dee da ba doo...
+    """
+    return ANSI_RE.sub('', text)
+
+
+def uncurl_quotes(text):
+    r"""
+    Replace curly quotation marks with straight equivalents.
+
+        >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d'))
+        "here's a test"
+    """
+    return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text))
+
+
+def fix_latin_ligatures(text):
+    """
+    Replace single-character ligatures of Latin letters, such as 'fi', with the
+    characters that they contain, as in 'fi'. Latin ligatures are usually not
+    intended in text strings (though they're lovely in *rendered* text).  If
+    you have such a ligature in your string, it is probably a result of a
+    copy-and-paste glitch.
+
+    We leave ligatures in other scripts alone to be safe. They may be intended,
+    and removing them may lose information. If you want to take apart nearly
+    all ligatures, use NFKC normalization.
+
+        >>> print(fix_latin_ligatures("fluffiest"))
+        fluffiest
+    """
+    return text.translate(LIGATURES)
+
+
+def fix_character_width(text):
+    """
+    The ASCII characters, katakana, and Hangul characters have alternate
+    "halfwidth" or "fullwidth" forms that help text line up in a grid.
+
+    If you don't need these width properties, you probably want to replace
+    these characters with their standard form, which is what this function
+    does.
+
+    Note that this replaces the ideographic space, U+3000, with the ASCII
+    space, U+20.
+
+        >>> print(fix_character_width("ＬＯＵＤ　ＮＯＩＳＥＳ"))
+        LOUD NOISES
+        >>> print(fix_character_width("Ｕﾀｰﾝ"))   # this means "U-turn"
+        Uターン
+    """
+    return text.translate(WIDTH_MAP)
+
+
+def fix_line_breaks(text):
+    r"""
+    Convert all line breaks to Unix style.
+
+    This will convert the following sequences into the standard \\n
+    line break:
+
+        - CRLF (\\r\\n), used on Windows and in some communication
+          protocols
+        - CR (\\r), once used on Mac OS Classic, and now kept alive
+          by misguided software such as Microsoft Office for Mac
+        - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029),
+          defined by Unicode and used to sow confusion and discord
+        - NEXT LINE (\\x85), a C1 control character that is certainly
+          not what you meant
+
+    The NEXT LINE character is a bit of an odd case, because it
+    usually won't show up if `fix_encoding` is also being run.
+    \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS.
+
+        >>> print(fix_line_breaks(
+        ...     "This string is made of two things:\u2029"
+        ...     "1. Unicode\u2028"
+        ...     "2. Spite"
+        ... ))
+        This string is made of two things:
+        1. Unicode
+        2. Spite
+
+    For further testing and examples, let's define a function to make sure
+    we can see the control characters in their escaped form:
+
+        >>> def eprint(text):
+        ...     print(text.encode('unicode-escape').decode('ascii'))
+
+        >>> eprint(fix_line_breaks("Content-type: text/plain\r\n\r\nHi."))
+        Content-type: text/plain\n\nHi.
+
+        >>> eprint(fix_line_breaks("This is how Microsoft \r trolls Mac users"))
+        This is how Microsoft \n trolls Mac users
+
+        >>> eprint(fix_line_breaks("What is this \x85 I don't even"))
+        What is this \n I don't even
+    """
+    return text.replace('\r\n', '\n').replace('\r', '\n')\
+               .replace('\u2028', '\n').replace('\u2029', '\n')\
+               .replace('\u0085', '\n')
+
+
+SURROGATE_RE = re.compile('[\ud800-\udfff]')
+SURROGATE_PAIR_RE = re.compile('[\ud800-\udbff][\udc00-\udfff]')
+
+
+def convert_surrogate_pair(match):
+    """
+    Convert a surrogate pair to the single codepoint it represents.
+
+    This implements the formula described at:
+    http://en.wikipedia.org/wiki/Universal_Character_Set_characters#Surrogates
+    """
+    pair = match.group(0)
+    codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00)
+    return unichr(codept)
+
+
+def fix_surrogates(text):
+    """
+    Replace 16-bit surrogate codepoints with the characters they represent
+    (when properly paired), or with \ufffd otherwise.
+
+        >>> high_surrogate = unichr(0xd83d)
+        >>> low_surrogate = unichr(0xdca9)
+        >>> print(fix_surrogates(high_surrogate + low_surrogate))
+        💩
+        >>> print(fix_surrogates(low_surrogate + high_surrogate))
+        ��
+
+    The above doctest had to be very carefully written, because even putting
+    the Unicode escapes of the surrogates in the docstring was causing
+    various tools to fail, which I think just goes to show why this fixer is
+    necessary.
+    """
+    if SURROGATE_RE.search(text):
+        text = SURROGATE_PAIR_RE.sub(convert_surrogate_pair, text)
+        text = SURROGATE_RE.sub('\ufffd', text)
+    return text
+
+
+def remove_control_chars(text):
+    """
+    Remove various control characters that you probably didn't intend to be in
+    your text. Many of these characters appear in the table of "Characters not
+    suitable for use with markup" at
+    http://www.unicode.org/reports/tr20/tr20-9.html.
+
+    This includes:
+
+    - ASCII control characters, except for the important whitespace characters
+      (U+00 to U+08, U+0B, U+0E to U+1F, U+7F)
+    - Deprecated Arabic control characters (U+206A to U+206F)
+    - Interlinear annotation characters (U+FFF9 to U+FFFB)
+    - The Object Replacement Character (U+FFFC)
+    - The byte order mark (U+FEFF)
+    - Musical notation control characters (U+1D173 to U+1D17A)
+    - Tag characters (U+E0000 to U+E007F)
+
+    However, these similar characters are left alone:
+
+    - Control characters that produce whitespace (U+09, U+0A, U+0C, U+0D,
+      U+2028, and U+2029)
+    - C1 control characters (U+80 to U+9F) -- even though they are basically
+      never used intentionally, they are important clues about what mojibake
+      has happened
+    - Control characters that affect glyph rendering, such as joiners and
+      right-to-left marks (U+200C to U+200F, U+202A to U+202E)
+    """
+    return text.translate(CONTROL_CHARS)
+
+
+def remove_bom(text):
+    r"""
+    Remove a byte-order mark that was accidentally decoded as if it were part
+    of the text.
+
+    >>> print(remove_bom("\ufeffWhere do you want to go today?"))
+    Where do you want to go today?
+    """
+    return text.lstrip(unichr(0xfeff))
+
+
+# Define a regex to match valid escape sequences in Python string literals.
+ESCAPE_SEQUENCE_RE = re.compile(r'''
+    ( \\U........      # 8-digit hex escapes
+    | \\u....          # 4-digit hex escapes
+    | \\x..            # 2-digit hex escapes
+    | \\[0-7]{1,3}     # Octal escapes
+    | \\N\{[^}]+\}     # Unicode characters by name
+    | \\[\\'"abfnrtv]  # Single-character escapes
+    )''', re.UNICODE | re.VERBOSE)
+
+
+def decode_escapes(text):
+    r"""
+    Decode backslashed escape sequences, including \\x, \\u, and \\U character
+    references, even in the presence of other Unicode.
+
+    This is what Python's "string-escape" and "unicode-escape" codecs were
+    meant to do, but in contrast, this actually works. It will decode the
+    string exactly the same way that the Python interpreter decodes its string
+    literals.
+
+        >>> factoid = '\\u20a1 is the currency symbol for the colón.'
+        >>> print(factoid[1:])
+        u20a1 is the currency symbol for the colón.
+        >>> print(decode_escapes(factoid))
+        ₡ is the currency symbol for the colón.
+
+    Even though Python itself can read string literals with a combination of
+    escapes and literal Unicode -- you're looking at one right now -- the
+    "unicode-escape" codec doesn't work on literal Unicode. (See
+    http://stackoverflow.com/a/24519338/773754 for more details.)
+
+    Instead, this function searches for just the parts of a string that
+    represent escape sequences, and decodes them, leaving the rest alone. All
+    valid escape sequences are made of ASCII characters, and this allows
+    "unicode-escape" to work correctly.
+
+    This fix cannot be automatically applied by the `ftfy.fix_text` function,
+    because escaped text is not necessarily a mistake, and there is no way
+    to distinguish text that's supposed to be escaped from text that isn't.
+    """
+    def decode_match(match):
+        "Given a regex match, decode the escape sequence it contains."
+        return codecs.decode(match.group(0), 'unicode-escape')
+
+    return ESCAPE_SEQUENCE_RE.sub(decode_match, text)
+
+
+def restore_byte_a0(byts):
+    """
+    Some mojibake has been additionally altered by a process that said "hmm,
+    byte A0, that's basically a space!" and replaced it with an ASCII space.
+    When the A0 is part of a sequence that we intend to decode as UTF-8,
+    changing byte A0 to 20 would make it fail to decode.
+
+    This process finds sequences that would convincingly decode as UTF-8 if
+    byte 20 were changed to A0, and puts back the A0. For the purpose of
+    deciding whether this is a good idea, this step gets a cost of twice
+    the number of bytes that are changed.
+
+    This is used as a step within `fix_encoding`.
+    """
+    def replacement(match):
+        "The function to apply when this regex matches."
+        return match.group(0).replace(b'\x20', b'\xa0')
+
+    return ALTERED_UTF8_RE.sub(replacement, byts)
+
+
+def replace_lossy_sequences(byts):
+    """
+    This function identifies sequences where information has been lost in
+    a "sloppy" codec, indicated by byte 1A, and if they would otherwise look
+    like a UTF-8 sequence, it replaces them with the UTF-8 sequence for U+FFFD.
+
+    A further explanation:
+
+    ftfy can now fix text in a few cases that it would previously fix
+    incompletely, because of the fact that it can't successfully apply the fix
+    to the entire string. A very common case of this is when characters have
+    been erroneously decoded as windows-1252, but instead of the "sloppy"
+    windows-1252 that passes through unassigned bytes, the unassigned bytes get
+    turned into U+FFFD (�), so we can't tell what they were.
+
+    This most commonly happens with curly quotation marks that appear
+    ``“ like this �``.
+
+    We can do better by building on ftfy's "sloppy codecs" to let them handle
+    less-sloppy but more-lossy text. When they encounter the character ``�``,
+    instead of refusing to encode it, they encode it as byte 1A -- an
+    ASCII control code called SUBSTITUTE that once was meant for about the same
+    purpose. We can then apply a fixer that looks for UTF-8 sequences where
+    some continuation bytes have been replaced by byte 1A, and decode the whole
+    sequence as �; if that doesn't work, it'll just turn the byte back into �
+    itself.
+
+    As a result, the above text ``“ like this �`` will decode as
+    ``“ like this �``.
+
+    If U+1A was actually in the original string, then the sloppy codecs will
+    not be used, and this function will not be run, so your weird control
+    character will be left alone but wacky fixes like this won't be possible.
+
+    This is used as a step within `fix_encoding`.
+    """
+    return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts)
+
+
+def fix_partial_utf8_punct_in_1252(text):
+    """
+    Fix particular characters that seem to be found in the wild encoded in
+    UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be
+    consistently applied.
+
+    For this function, we assume the text has been decoded in Windows-1252.
+    If it was decoded in Latin-1, we'll call this right after it goes through
+    the Latin-1-to-Windows-1252 fixer.
+
+    This is used as a step within `fix_encoding`.
+    """
+    def replacement(match):
+        "The function to apply when this regex matches."
+        return match.group(0).encode('sloppy-windows-1252').decode('utf-8')
+    return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text)
+
+
+TRANSCODERS = {
+    'restore_byte_a0': restore_byte_a0,
+    'replace_lossy_sequences': replace_lossy_sequences,
+    'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252
+}

Contents/Libraries/Shared/ftfy/formatting.py

@@ -0,0 +1,154 @@
+# coding: utf-8
+"""
+This module provides functions for justifying Unicode text in a monospaced
+display such as a terminal.
+
+We used to have our own implementation here, but now we mostly rely on
+the 'wcwidth' library.
+"""
+from __future__ import unicode_literals, division
+from unicodedata import normalize
+from wcwidth import wcwidth, wcswidth
+
+
+def character_width(char):
+    r"""
+    Determine the width that a character is likely to be displayed as in
+    a monospaced terminal. The width for a printable character will
+    always be 0, 1, or 2.
+
+    Nonprintable or control characters will return -1, a convention that comes
+    from wcwidth.
+
+    >>> character_width('車')
+    2
+    >>> character_width('A')
+    1
+    >>> character_width('\N{ZERO WIDTH JOINER}')
+    0
+    >>> character_width('\n')
+    -1
+    """
+    return wcwidth(char)
+
+
+def monospaced_width(text):
+    """
+    Return the number of character cells that this string is likely to occupy
+    when displayed in a monospaced, modern, Unicode-aware terminal emulator.
+    We refer to this as the "display width" of the string.
+
+    This can be useful for formatting text that may contain non-spacing
+    characters, or CJK characters that take up two character cells.
+
+    Returns -1 if the string contains a non-printable or control character.
+
+    >>> monospaced_width('ちゃぶ台返し')
+    12
+    >>> len('ちゃぶ台返し')
+    6
+    >>> monospaced_width('owl\N{SOFT HYPHEN}flavored')
+    12
+    >>> monospaced_width('example\x80')
+    -1
+
+    # The Korean word 'ibnida' can be written with 3 characters or 7 jamo.
+    # Either way, it *looks* the same and takes up 6 character cells.
+    >>> monospaced_width('입니다')
+    6
+    >>> monospaced_width('\u110b\u1175\u11b8\u1102\u1175\u1103\u1161')
+    6
+    """
+    # NFC-normalize the text first, so that we don't need special cases for
+    # Hangul jamo.
+    return wcswidth(normalize('NFC', text))
+
+
+def display_ljust(text, width, fillchar=' '):
+    """
+    Return `text` left-justified in a Unicode string whose display width,
+    in a monospaced terminal, should be at least `width` character cells.
+    The rest of the string will be padded with `fillchar`, which must be
+    a width-1 character.
+
+    "Left" here means toward the beginning of the string, which may actually
+    appear on the right in an RTL context. This is similar to the use of the
+    word "left" in "left parenthesis".
+
+    >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し']
+    >>> for line in lines:
+    ...     print(display_ljust(line, 20, '▒'))
+    Table flip▒▒▒▒▒▒▒▒▒▒
+    (╯°□°)╯︵ ┻━┻▒▒▒▒▒▒▒
+    ちゃぶ台返し▒▒▒▒▒▒▒▒
+
+    This example, and the similar ones that follow, should come out justified
+    correctly when viewed in a monospaced terminal. It will probably not look
+    correct if you're viewing this code or documentation in a Web browser.
+    """
+    if character_width(fillchar) != 1:
+        raise ValueError("The padding character must have display width 1")
+
+    text_width = monospaced_width(text)
+    if text_width == -1:
+        # There's a control character here, so just don't add padding
+        return text
+
+    padding = max(0, width - text_width)
+    return text + fillchar * padding
+
+
+def display_rjust(text, width, fillchar=' '):
+    """
+    Return `text` right-justified in a Unicode string whose display width,
+    in a monospaced terminal, should be at least `width` character cells.
+    The rest of the string will be padded with `fillchar`, which must be
+    a width-1 character.
+
+    "Right" here means toward the end of the string, which may actually be on
+    the left in an RTL context. This is similar to the use of the word "right"
+    in "right parenthesis".
+
+    >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し']
+    >>> for line in lines:
+    ...     print(display_rjust(line, 20, '▒'))
+    ▒▒▒▒▒▒▒▒▒▒Table flip
+    ▒▒▒▒▒▒▒(╯°□°)╯︵ ┻━┻
+    ▒▒▒▒▒▒▒▒ちゃぶ台返し
+    """
+    if character_width(fillchar) != 1:
+        raise ValueError("The padding character must have display width 1")
+
+    text_width = monospaced_width(text)
+    if text_width == -1:
+        return text
+
+    padding = max(0, width - text_width)
+    return fillchar * padding + text
+
+
+def display_center(text, width, fillchar=' '):
+    """
+    Return `text` centered in a Unicode string whose display width, in a
+    monospaced terminal, should be at least `width` character cells. The rest
+    of the string will be padded with `fillchar`, which must be a width-1
+    character.
+
+    >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し']
+    >>> for line in lines:
+    ...     print(display_center(line, 20, '▒'))
+    ▒▒▒▒▒Table flip▒▒▒▒▒
+    ▒▒▒(╯°□°)╯︵ ┻━┻▒▒▒▒
+    ▒▒▒▒ちゃぶ台返し▒▒▒▒
+    """
+    if character_width(fillchar) != 1:
+        raise ValueError("The padding character must have display width 1")
+
+    text_width = monospaced_width(text)
+    if text_width == -1:
+        return text
+
+    padding = max(0, width - text_width)
+    left_padding = padding // 2
+    right_padding = padding - left_padding
+    return fillchar * left_padding + text + fillchar * right_padding

Contents/Libraries/Shared/ftfy/streamtester/__init__.py

@@ -0,0 +1,47 @@
+# coding: utf-8
+"""
+This file defines a general method for evaluating ftfy using data that arrives
+in a stream. A concrete implementation of it is found in `twitter_tester.py`.
+"""
+from __future__ import print_function, unicode_literals
+from ftfy import fix_text
+from ftfy.fixes import fix_encoding, unescape_html
+from ftfy.chardata import possible_encoding
+
+
+class StreamTester:
+    """
+    Take in a sequence of texts, and show the ones that will be changed by
+    ftfy. This will also periodically show updates, such as the proportion of
+    texts that changed.
+    """
+    def __init__(self):
+        self.num_fixed = 0
+        self.count = 0
+
+    def check_ftfy(self, text, encoding_only=True):
+        """
+        Given a single text input, check whether `ftfy.fix_text_encoding`
+        would change it. If so, display the change.
+        """
+        self.count += 1
+        text = unescape_html(text)
+        if not possible_encoding(text, 'ascii'):
+            if encoding_only:
+                fixed = fix_encoding(text)
+            else:
+                fixed = fix_text(text, uncurl_quotes=False, fix_character_width=False)
+            if text != fixed:
+                # possibly filter common bots before printing
+                print('\nText:\t{text!r}\nFixed:\t{fixed!r}\n'.format(
+                    text=text, fixed=fixed
+                ))
+                self.num_fixed += 1
+            elif 'â€' in text or '\x80' in text:
+                print('\nNot fixed:\t{text!r}'.format(text=text))
+
+        # Print status updates once in a while
+        if self.count % 100 == 0:
+            print('.', end='', flush=True)
+        if self.count % 10000 == 0:
+            print('\n%d/%d fixed' % (self.num_fixed, self.count))

Contents/Libraries/Shared/ftfy/streamtester/oauth.py

@@ -0,0 +1,72 @@
+# coding: utf-8
+"""
+Do what is necessary to authenticate this tester as a Twitter "app", using
+somebody's Twitter account.
+"""
+from __future__ import unicode_literals
+import os
+
+
+AUTH_TOKEN_PATH = os.path.expanduser('~/.cache/oauth/twitter_ftfy.auth')
+
+def get_auth():
+    """
+    Twitter has some bizarre requirements about how to authorize an "app" to
+    use its API.
+
+    The user of the app has to log in to get a secret token. That's fine. But
+    the app itself has its own "consumer secret" token. The app has to know it,
+    and the user of the app has to not know it.
+
+    This is, of course, impossible. It's equivalent to DRM. Your computer can't
+    *really* make use of secret information while hiding the same information
+    from you.
+
+    The threat appears to be that, if you have this super-sekrit token, you can
+    impersonate the app while doing something different. Well, of course you
+    can do that, because you *have the source code* and you can change it to do
+    what you want. You still have to log in as a particular user who has a
+    token that's actually secret, you know.
+
+    Even developers of closed-source applications that use the Twitter API are
+    unsure what to do, for good reason. These "secrets" are not secret in any
+    cryptographic sense. A bit of Googling shows that the secret tokens for
+    every popular Twitter app are already posted on the Web.
+
+    Twitter wants us to pretend this string can be kept secret, and hide this
+    secret behind a fig leaf like everybody else does. So that's what we've
+    done.
+    """
+
+    from twitter.oauth import OAuth
+    from twitter import oauth_dance, read_token_file
+
+    def unhide(secret):
+        """
+        Do something mysterious and exactly as secure as every other Twitter
+        app.
+        """
+        return ''.join([chr(ord(c) - 0x2800) for c in secret])
+
+    fig_leaf = '⠴⡹⠹⡩⠶⠴⡶⡅⡂⡩⡅⠳⡏⡉⡈⠰⠰⡹⡥⡶⡈⡐⡍⡂⡫⡍⡗⡬⡒⡧⡶⡣⡰⡄⡧⡸⡑⡣⠵⡓⠶⠴⡁'
+    consumer_key = 'OFhyNd2Zt4Ba6gJGJXfbsw'
+
+    if os.path.exists(AUTH_TOKEN_PATH):
+        token, token_secret = read_token_file(AUTH_TOKEN_PATH)
+    else:
+        authdir = os.path.dirname(AUTH_TOKEN_PATH)
+        if not os.path.exists(authdir):
+            os.makedirs(authdir)
+        token, token_secret = oauth_dance(
+            app_name='ftfy-tester',
+            consumer_key=consumer_key,
+            consumer_secret=unhide(fig_leaf),
+            token_filename=AUTH_TOKEN_PATH
+        )
+
+    return OAuth(
+        token=token,
+        token_secret=token_secret,
+        consumer_key=consumer_key,
+        consumer_secret=unhide(fig_leaf)
+    )

Contents/Libraries/Shared/ftfy/streamtester/twitter_tester.py

@@ -0,0 +1,88 @@
+"""
+Implements a StreamTester that runs over Twitter data. See the class
+docstring.
+
+This module is written for Python 3 only. The __future__ imports you see here
+are just to let Python 2 scan the file without crashing with a SyntaxError.
+"""
+from __future__ import print_function, unicode_literals
+import os
+from collections import defaultdict
+from ftfy.streamtester import StreamTester
+
+
+class TwitterTester(StreamTester):
+    """
+    This class uses the StreamTester code (defined in `__init__.py`) to
+    evaluate ftfy's real-world performance, by feeding it live data from
+    Twitter.
+
+    This is a semi-manual evaluation. It requires a human to look at the
+    results and determine if they are good. The three possible cases we
+    can see here are:
+
+        - Success: the process takes in mojibake and outputs correct text.
+        - False positive: the process takes in correct text, and outputs
+          mojibake. Every false positive should be considered a bug, and
+          reported on GitHub if it isn't already.
+        - Confusion: the process takes in mojibake and outputs different
+          mojibake. Not a great outcome, but not as dire as a false
+          positive.
+
+    This tester cannot reveal false negatives. So far, that can only be
+    done by the unit tests.
+    """
+    OUTPUT_DIR = './twitterlogs'
+
+    def __init__(self):
+        self.lines_by_lang = defaultdict(list)
+        super().__init__()
+
+    def save_files(self):
+        """
+        When processing data from live Twitter, save it to log files so that
+        it can be replayed later.
+        """
+        if not os.path.exists(self.OUTPUT_DIR):
+            os.makedirs(self.OUTPUT_DIR)
+        for lang, lines in self.lines_by_lang.items():
+            filename = 'tweets.{}.txt'.format(lang)
+            fullname = os.path.join(self.OUTPUT_DIR, filename)
+            langfile = open(fullname, 'a', encoding='utf-8')
+            for line in lines:
+                print(line.replace('\n', ' '), file=langfile)
+            langfile.close()
+        self.lines_by_lang = defaultdict(list)
+
+    def run_sample(self):
+        """
+        Listen to live data from Twitter, and pass on the fully-formed tweets
+        to `check_ftfy`. This requires the `twitter` Python package as a
+        dependency.
+        """
+        from twitter import TwitterStream
+        from ftfy.streamtester.oauth import get_auth
+        twitter_stream = TwitterStream(auth=get_auth())
+        iterator = twitter_stream.statuses.sample()
+        for tweet in iterator:
+            if 'text' in tweet:
+                self.check_ftfy(tweet['text'])
+                if 'user' in tweet:
+                    lang = tweet['user'].get('lang', 'NONE')
+                    self.lines_by_lang[lang].append(tweet['text'])
+                if self.count % 10000 == 100:
+                    self.save_files()
+
+
+def main():
+    """
+    When run from the command line, this script connects to the Twitter stream
+    and runs the TwitterTester on it forever. Or at least until the stream
+    drops.
+    """
+    tester = TwitterTester()
+    tester.run_sample()
+
+
+if __name__ == '__main__':
+    main()

Contents/Libraries/Shared/guessit/__main__.py

@@ -33,8 +33,6 @@ def guess_filename(filename, options):
     if not options.get('yaml') and not options.get('json') and not options.get('show_property'):
         print('For:', filename)
 
-    options['implicit'] = True  # Force implicit option in CLI
-
     guess = api.guessit(filename, options)
 
     if options.get('show_property'):

Contents/Libraries/Shared/guessit/__version__.py

@@ -4,4 +4,4 @@
 Version module
 """
 # pragma: no cover
-__version__ = '2.1.3.dev0'
+__version__ = '3.0.0.dev0'

Contents/Libraries/Shared/guessit/api.py

@@ -126,7 +126,8 @@ class GuessItApi(object):
                 for match in matches:
                     if isinstance(match.value, six.text_type):
                         match.value = match.value.encode("ascii")
-            return matches.to_dict(options.get('advanced', False), options.get('implicit', False))
+            return matches.to_dict(options.get('advanced', False), options.get('single_value', False),
+                                   options.get('enforce_list', False))
         except:
             raise GuessitException(string, options)
 

Contents/Libraries/Shared/guessit/options.py

@@ -54,6 +54,10 @@ def build_argument_parser():
                              help='Display the value of a single property (title, series, video_codec, year, ...)')
     output_opts.add_argument('-a', '--advanced', dest='advanced', action='store_true', default=None,
                              help='Display advanced information for filename guesses, as json output')
+    output_opts.add_argument('-s', '--single-value', dest='single_value', action='store_true', default=None,
+                             help='Keep only first value found for each property')
+    output_opts.add_argument('-l', '--enforce-list', dest='enforce_list', action='store_true', default=None,
+                             help='Wrap each found value in a list even when property has a single value')
     output_opts.add_argument('-j', '--json', dest='json', action='store_true', default=None,
                              help='Display information for filename guesses as json output')
     output_opts.add_argument('-y', '--yaml', dest='yaml', action='store_true', default=None,

Contents/Libraries/Shared/guessit/rules/properties/audio_codec.py

@@ -39,12 +39,12 @@ def audio_codec():
     rebulk.defaults(name="audio_codec", conflict_solver=audio_codec_priority)
 
     rebulk.regex("MP3", "LAME", r"LAME(?:\d)+-?(?:\d)+", value="MP3")
-    rebulk.regex("Dolby", "DolbyDigital", "Dolby-Digital", "DDP?", value="DolbyDigital")
+    rebulk.regex('Dolby', 'DolbyDigital', 'Dolby-Digital', 'DD', 'AC3D?', value='AC3')
     rebulk.regex("DolbyAtmos", "Dolby-Atmos", "Atmos", value="DolbyAtmos")
-    rebulk.regex("AAC", value="AAC")
-    rebulk.regex("AC3D?", value="AC3")
-    rebulk.regex("Flac", value="FLAC")
-    rebulk.regex("DTS", value="DTS")
+    rebulk.string("AAC", value="AAC")
+    rebulk.string('EAC3', 'DDP', 'DD+', value="EAC3")
+    rebulk.string("Flac", value="FLAC")
+    rebulk.string("DTS", value="DTS")
     rebulk.regex("True-?HD", value="TrueHD")
 
     rebulk.defaults(name="audio_profile")

Contents/Libraries/Shared/guessit/rules/properties/container.py

@@ -34,15 +34,17 @@ def container():
               'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv',
               'iso', 'vob']
     torrent = ['torrent']
+    nzb = ['nzb']
 
     rebulk.regex(r'\.'+build_or_pattern(subtitles)+'$', exts=subtitles, tags=['extension', 'subtitle'])
     rebulk.regex(r'\.'+build_or_pattern(info)+'$', exts=info, tags=['extension', 'info'])
     rebulk.regex(r'\.'+build_or_pattern(videos)+'$', exts=videos, tags=['extension', 'video'])
     rebulk.regex(r'\.'+build_or_pattern(torrent)+'$', exts=torrent, tags=['extension', 'torrent'])
+    rebulk.regex(r'\.'+build_or_pattern(nzb)+'$', exts=nzb, tags=['extension', 'nzb'])
 
     rebulk.defaults(name='container',
                     validator=seps_surround,
-                    formatter=lambda s: s.upper(),
+                    formatter=lambda s: s.lower(),
                     conflict_solver=lambda match, other: match
                     if other.name in ['format',
                                       'video_codec'] or other.name == 'container' and 'extension' in other.tags
@@ -51,5 +53,6 @@ def container():
     rebulk.string(*[sub for sub in subtitles if sub not in ['sub']], tags=['subtitle'])
     rebulk.string(*videos, tags=['video'])
     rebulk.string(*torrent, tags=['torrent'])
+    rebulk.string(*nzb, tags=['nzb'])
 
     return rebulk

Contents/Libraries/Shared/guessit/rules/properties/edition.py

@@ -24,12 +24,18 @@ def edition():
                  conflict_solver=lambda match, other: other
                  if other.name == 'episode_details' and other.value == 'Special'
                  else '__default__')
-    rebulk.string('SE', value='Special Edition', tags='has-neighbor')
+    rebulk.string('se', value='Special Edition', tags='has-neighbor')
     rebulk.regex('criterion-edition', 'edition-criterion', value='Criterion Edition')
     rebulk.regex('deluxe', 'deluxe-edition', 'edition-deluxe', value='Deluxe Edition')
-    rebulk.regex('limited', 'limited-edition', value='Limited Edition')
+    rebulk.regex('limited', 'limited-edition', value='Limited Edition', tags=['has-neighbor', 'release-group-prefix'])
     rebulk.regex(r'theatrical-cut', r'theatrical-edition', r'theatrical', value='Theatrical Edition')
     rebulk.regex(r"director'?s?-cut", r"director'?s?-cut-edition", r"edition-director'?s?-cut", 'DC',
-                 value="Director's cut")
+                 value="Director's Cut")
+    rebulk.regex('extended', 'extended-?cut', 'extended-?version',
+                 value='Extended', tags=['has-neighbor', 'release-group-prefix'])
+    rebulk.regex('alternat(e|ive)(?:-?Cut)?', value='Alternative Cut', tags=['has-neighbor', 'release-group-prefix'])
+    for value in ('Remastered', 'Uncensored', 'Uncut', 'Unrated'):
+        rebulk.string(value, value=value, tags=['has-neighbor', 'release-group-prefix'])
+    rebulk.string('Festival', value='Festival', tags=['has-neighbor-before', 'has-neighbor-after'])
 
     return rebulk

Contents/Libraries/Shared/guessit/rules/properties/episode_title.py

@@ -5,7 +5,7 @@ Episode title
 """
 from collections import defaultdict
 
-from rebulk import Rebulk, Rule, AppendMatch, RenameMatch, POST_PROCESS
+from rebulk import Rebulk, Rule, AppendMatch, RemoveMatch, RenameMatch, POST_PROCESS
 
 from ..common import seps, title_seps
 from ..common.formatters import cleanup
@@ -19,8 +19,12 @@ def episode_title():
     :return: Created Rebulk object
     :rtype: Rebulk
     """
-    rebulk = Rebulk().rules(EpisodeTitleFromPosition,
-                            AlternativeTitleReplace,
+    previous_names = ('episode', 'episode_details', 'episode_count',
+                      'season', 'season_count', 'date', 'title', 'year')
+
+    rebulk = Rebulk().rules(RemoveConflictsWithEpisodeTitle(previous_names),
+                            EpisodeTitleFromPosition(previous_names),
+                            AlternativeTitleReplace(previous_names),
                             TitleToEpisodeTitle,
                             Filepart3EpisodeTitle,
                             Filepart2EpisodeTitle,
@@ -28,6 +32,62 @@ def episode_title():
     return rebulk
 
 
+class RemoveConflictsWithEpisodeTitle(Rule):
+    """
+    Remove conflicting matches that might lead to wrong episode_title parsing.
+    """
+
+    priority = 64
+    consequence = RemoveMatch
+
+    def __init__(self, previous_names):
+        super(RemoveConflictsWithEpisodeTitle, self).__init__()
+        self.previous_names = previous_names
+        self.next_names = ('streaming_service', 'screen_size', 'format',
+                           'video_codec', 'audio_codec', 'other', 'container')
+        self.affected_if_holes_after = ('part', )
+        self.affected_names = ('part', 'year')
+
+    def when(self, matches, context):
+        to_remove = []
+        for filepart in matches.markers.named('path'):
+            for match in matches.range(filepart.start, filepart.end,
+                                       predicate=lambda m: m.name in self.affected_names):
+                before = matches.previous(match, index=0,
+                                          predicate=lambda m, fp=filepart: not m.private and m.start >= fp.start)
+                if not before or before.name not in self.previous_names:
+                    continue
+
+                after = matches.next(match, index=0,
+                                     predicate=lambda m, fp=filepart: not m.private and m.end <= fp.end)
+                if not after or after.name not in self.next_names:
+                    continue
+
+                group = matches.markers.at_match(match, predicate=lambda m: m.name == 'group', index=0)
+
+                def has_value_in_same_group(current_match, current_group=group):
+                    """Return true if current match has value and belongs to the current group."""
+                    return current_match.value.strip(seps) and (
+                        current_group == matches.markers.at_match(current_match,
+                                                                  predicate=lambda mm: mm.name == 'group', index=0)
+                    )
+
+                holes_before = matches.holes(before.end, match.start, predicate=has_value_in_same_group)
+                holes_after = matches.holes(match.end, after.start, predicate=has_value_in_same_group)
+
+                if not holes_before and not holes_after:
+                    continue
+
+                if match.name in self.affected_if_holes_after and not holes_after:
+                    continue
+
+                to_remove.append(match)
+                if match.parent:
+                    to_remove.append(match.parent)
+
+        return to_remove
+
+
 class TitleToEpisodeTitle(Rule):
     """
     If multiple different title are found, convert the one following episode number to episode_title.
@@ -65,12 +125,14 @@ class EpisodeTitleFromPosition(TitleBaseRule):
     """
     dependency = TitleToEpisodeTitle
 
+    def __init__(self, previous_names):
+        super(EpisodeTitleFromPosition, self).__init__('episode_title', ['title'])
+        self.previous_names = previous_names
+
     def hole_filter(self, hole, matches):
         episode = matches.previous(hole,
                                    lambda previous: any(name in previous.names
-                                                        for name in ['episode', 'episode_details',
-                                                                     'episode_count', 'season', 'season_count',
-                                                                     'date', 'title', 'year']),
+                                                        for name in self.previous_names),
                                    0)
 
         crc32 = matches.named('crc32')
@@ -88,9 +150,6 @@ class EpisodeTitleFromPosition(TitleBaseRule):
             return False
         return super(EpisodeTitleFromPosition, self).should_remove(match, matches, filepart, hole, context)
 
-    def __init__(self):
-        super(EpisodeTitleFromPosition, self).__init__('episode_title', ['title'])
-
     def when(self, matches, context):
         if matches.named('episode_title'):
             return
@@ -104,6 +163,10 @@ class AlternativeTitleReplace(Rule):
     dependency = EpisodeTitleFromPosition
     consequence = RenameMatch
 
+    def __init__(self, previous_names):
+        super(AlternativeTitleReplace, self).__init__()
+        self.previous_names = previous_names
+
     def when(self, matches, context):
         if matches.named('episode_title'):
             return
@@ -115,10 +178,7 @@ class AlternativeTitleReplace(Rule):
             if main_title:
                 episode = matches.previous(main_title,
                                            lambda previous: any(name in previous.names
-                                                                for name in ['episode', 'episode_details',
-                                                                             'episode_count', 'season',
-                                                                             'season_count',
-                                                                             'date', 'title', 'year']),
+                                                                for name in self.previous_names),
                                            0)
 
                 crc32 = matches.named('crc32')

Contents/Libraries/Shared/guessit/rules/properties/episodes.py

@@ -98,7 +98,7 @@ def episodes():
         episode/season separated by a weak discrete separator should be consecutive, unless a strong discrete separator
         or a range separator is present in the chain (1.3&5 is valid, but 1.3-5 is not valid and 1.3.5 is not valid)
         """
-        values = match.children.to_dict(implicit=True)
+        values = match.children.to_dict()
         if 'season' in values and is_iterable(values['season']):
             # Season numbers must be in natural order to be validated.
             if not list(sorted(values['season'])) == values['season']:
@@ -231,14 +231,16 @@ def episodes():
                  formatter={'season': int, 'other': lambda match: 'Complete'})
 
     # 12, 13
-    rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode'], formatter={'episode': int, 'version': int}) \
+    rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode'], formatter={'episode': int, 'version': int},
+                 disabled=lambda context: context.get('type') == 'movie') \
         .defaults(validator=None) \
         .regex(r'(?P<episode>\d{2})') \
         .regex(r'v(?P<version>\d+)').repeater('?') \
         .regex(r'(?P<episodeSeparator>[x-])(?P<episode>\d{2})').repeater('*')
 
     # 012, 013
-    rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode'], formatter={'episode': int, 'version': int}) \
+    rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode'], formatter={'episode': int, 'version': int},
+                 disabled=lambda context: context.get('type') == 'movie') \
         .defaults(validator=None) \
         .regex(r'0(?P<episode>\d{1,2})') \
         .regex(r'v(?P<version>\d+)').repeater('?') \
@@ -246,7 +248,8 @@ def episodes():
 
     # 112, 113
     rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode'], formatter={'episode': int, 'version': int},
-                 disabled=lambda context: not context.get('episode_prefer_number', False)) \
+                 disabled=lambda context: (not context.get('episode_prefer_number', False) or
+                                           context.get('type') == 'movie')) \
         .defaults(validator=None) \
         .regex(r'(?P<episode>\d{3,4})') \
         .regex(r'v(?P<version>\d+)').repeater('?') \
@@ -287,7 +290,8 @@ def episodes():
     rebulk.chain(tags=['bonus-conflict', 'weak-movie', 'weak-episode', 'weak-duplicate'],
                  formatter={'season': int, 'episode': int, 'version': int},
                  conflict_solver=lambda match, other: match if other.name == 'year' else '__default__',
-                 disabled=lambda context: context.get('episode_prefer_number', False)) \
+                 disabled=lambda context: (context.get('episode_prefer_number', False) or
+                                           context.get('type') == 'movie')) \
         .defaults(validator=None) \
         .regex(r'(?P<season>\d{1,2})(?P<episode>\d{2})') \
         .regex(r'v(?P<version>\d+)').repeater('?') \
@@ -460,8 +464,21 @@ class RemoveWeakIfMovie(Rule):
         return context.get('type') != 'episode'
 
     def when(self, matches, context):
-        if matches.named('year'):
-            return matches.tagged('weak-movie')
+        to_remove = []
+        to_ignore = set()
+        remove = False
+        for filepart in matches.markers.named('path'):
+            year = matches.range(filepart.start, filepart.end, predicate=lambda m: m.name == 'year', index=0)
+            if year:
+                remove = True
+                next_match = matches.next(year, predicate=lambda m, fp=filepart: m.private and m.end <= fp.end, index=0)
+                if next_match and not matches.at_match(next_match, predicate=lambda m: m.name == 'year'):
+                    to_ignore.add(next_match.initiator)
+
+        if remove:
+            to_remove.extend(matches.tagged('weak-movie', predicate=lambda m: m.initiator not in to_ignore))
+
+        return to_remove
 
 
 class RemoveWeakIfSxxExx(Rule):

Contents/Libraries/Shared/guessit/rules/properties/language.py

@@ -39,8 +39,7 @@ COMMON_WORDS_STRICT = frozenset(['brazil'])
 
 UNDETERMINED = babelfish.Language('und')
 
-SYN = {('und', None): ['unknown', 'inconnu', 'unk'],
-       ('ell', None): ['gr', 'greek'],
+SYN = {('ell', None): ['gr', 'greek'],
        ('spa', None): ['esp', 'español', 'espanol'],
        ('fra', None): ['français', 'vf', 'vff', 'vfi', 'vfq'],
        ('swe', None): ['se'],

Contents/Libraries/Shared/guessit/rules/properties/other.py

@@ -66,28 +66,27 @@ def other():
     rebulk.regex('(?:PS-?)?Vita', value='PS Vita')
 
     for value in (
-            'Screener', 'Remux', 'Remastered', '3D', 'mHD', 'HDLight', 'HQ', 'DDC', 'HR', 'PAL', 'SECAM', 'NTSC',
+            'Screener', 'Remux', '3D', 'mHD', 'HDLight', 'HQ', 'DDC', 'HR', 'PAL', 'SECAM', 'NTSC',
             'CC', 'LD', 'MD', 'XXX'):
         rebulk.string(value, value=value)
 
     rebulk.string('LDTV', value='LD')
     rebulk.string('HD', value='HD', validator=None,
                   tags=['streaming_service.prefix', 'streaming_service.suffix'])
+    rebulk.regex('Full-?HD', 'FHD', value='FullHD', validator=None,
+                 tags=['streaming_service.prefix', 'streaming_service.suffix'])
     rebulk.regex('Ultra-?(?:HD)?', 'UHD', value='UltraHD', validator=None,
                  tags=['streaming_service.prefix', 'streaming_service.suffix'])
 
-    for value in ('Limited', 'Complete', 'Classic', 'Unrated', 'LiNE', 'Bonus', 'Trailer', 'FINAL', 'Retail', 'Uncut',
-                  'Extended', 'Extended Cut', 'Colorized', 'Internal', 'Uncensored'):
+    for value in ('Complete', 'Classic', 'LiNE', 'Bonus', 'Trailer', 'FINAL', 'Retail',
+                  'Colorized', 'Internal'):
         rebulk.string(value, value=value, tags=['has-neighbor', 'release-group-prefix'])
-    rebulk.regex('Extended-?version', value='Extended', tags=['has-neighbor', 'release-group-prefix'])
-    rebulk.regex('Alternat(e|ive)(?:-?Cut)?', value='Alternative Cut', tags=['has-neighbor', 'release-group-prefix'])
     rebulk.regex('Read-?NFO', value='Read NFO')
     rebulk.string('CONVERT', value='Converted', tags='has-neighbor')
     rebulk.string('DOCU', value='Documentary', tags='has-neighbor')
     rebulk.string('OM', value='Open Matte', tags='has-neighbor')
     rebulk.string('STV', value='Straight to Video', tags='has-neighbor')
     rebulk.string('OAR', value='Original Aspect Ratio', tags='has-neighbor')
-    rebulk.string('Festival', value='Festival', tags=['has-neighbor-before', 'has-neighbor-after'])
     rebulk.string('Complet', value='Complete', tags=['has-neighbor', 'release-group-prefix'])
 
     for coast in ('East', 'West'):
@@ -134,6 +133,7 @@ class ValidateHasNeighbor(Rule):
     Validate tag has-neighbor
     """
     consequence = RemoveMatch
+    priority = 64
 
     def when(self, matches, context):
         ret = []
@@ -159,6 +159,7 @@ class ValidateHasNeighborBefore(Rule):
     Validate tag has-neighbor-before that previous match exists.
     """
     consequence = RemoveMatch
+    priority = 64
 
     def when(self, matches, context):
         ret = []
@@ -178,6 +179,7 @@ class ValidateHasNeighborAfter(Rule):
     Validate tag has-neighbor-after that next match exists.
     """
     consequence = RemoveMatch
+    priority = 64
 
     def when(self, matches, context):
         ret = []

Contents/Libraries/Shared/guessit/rules/properties/website.py

@@ -85,6 +85,7 @@ class ValidateWebsitePrefix(Rule):
     """
     Validate website prefixes
     """
+    priority = 64
     consequence = RemoveMatch
 
     def when(self, matches, context):

Contents/Libraries/Shared/guessit/test/episodes.yml

@@ -330,7 +330,7 @@
   screen_size: 720p
   format: WEB-DL
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   video_codec: h264
   release_group: CtrlHD
 
@@ -356,7 +356,7 @@
   screen_size: 720p
   format: WEB-DL
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   video_codec: h264
   release_group: CtrlHD
 
@@ -388,7 +388,7 @@
   screen_size: 720p
   format: WEB-DL
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   video_codec: h264
 
 ? Game of Thrones S03E06 1080i HDTV DD5.1 MPEG2-TrollHD.ts
@@ -398,7 +398,7 @@
   screen_size: 1080i
   format: HDTV
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   video_codec: Mpeg2
   release_group: TrollHD
 
@@ -548,7 +548,7 @@
   screen_size: 720p
   season: 1
   video_profile: BP
-  audio_codec: DolbyDigital
+  audio_codec: AC3
 
 ? Sleepy.Hollow.S01E09.720p.WEB-DL.DD5.1.H.264-BS.mkv
 : episode: 9
@@ -559,7 +559,7 @@
   screen_size: 720p
   season: 1
   release_group: BS
-  audio_codec: DolbyDigital
+  audio_codec: AC3
 
 ? Battlestar.Galactica.S00.Pilot.FRENCH.DVDRip.XviD-NOTAG.avi
 : title: Battlestar Galactica
@@ -621,7 +621,7 @@
   streaming_service: Netflix
   format: WEBRip
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   video_codec: h264
   release_group: NTb
 
@@ -1130,7 +1130,7 @@
   episode: 21
   episode_title: Al Sah-Him
   screen_size: 1080p
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: "5.1"
   video_codec: h264
   release_group: BS
@@ -1167,7 +1167,7 @@
   audio_codec: AAC
   date: 2015-07-01
   format: WEBRip
-  other: Extended
+  edition: Extended
   release_group: BTW
   screen_size: 720p
   streaming_service: Comedy Central
@@ -1653,7 +1653,7 @@
 
 ? The.Good.Wife.S06E01.E10.720p.WEB-DL.DD5.1.H.264-CtrlHD/The.Good.Wife.S06E09.Trust.Issues.720p.WEB-DL.DD5.1.H.264-CtrlHD.mkv
 : audio_channels: '5.1'
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   episode: 9
   format: WEB-DL
   release_group: CtrlHD
@@ -1814,7 +1814,7 @@
   format: HDTV
   video_codec: h264
   audio_codec: AAC
-  container: MP4
+  container: mp4
   release_group: k3n
   type: episode
 
@@ -1853,7 +1853,7 @@
 
 ? Game.of.Thrones.S6.Ep5.X265.Dolby.2.0.KTM3.mp4
 : audio_channels: '2.0'
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   container: mp4
   episode: 5
   release_group: KTM3
@@ -1885,7 +1885,7 @@
 
 ? Breaking.Bad.S01E01.2008.BluRay.VC1.1080P.5.1.WMV-NOVO
 : audio_channels: '5.1'
-  container: WMV
+  container: wmv
   episode: 1
   format: BluRay
   release_group: NOVO
@@ -1922,9 +1922,7 @@
 
 ? Fear.The.Walking.Dead.S02E01.HDTV.x264.AAC.MP4-k3n.mp4
 : audio_codec: AAC
-  container:
-  - MP4
-  - mp4
+  container: mp4
   episode: 1
   format: HDTV
   mimetype: video/mp4
@@ -2063,7 +2061,7 @@
 
 ? The.Walking.Dead.S06E01.FRENCH.1080p.WEB-DL.DD5.1.HEVC.x265-GOLF68
 : audio_channels: '5.1'
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   episode: 1
   format: WEB-DL
   language: fr
@@ -2202,7 +2200,7 @@
   season: 1
   screen_size: 720p
   format: HDTV
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: Group
@@ -2242,7 +2240,7 @@
   screen_size: 1080p
   streaming_service: Amazon Prime
   format: WEBRip
-  audio_codec: DolbyDigital
+  audio_codec: EAC3
   audio_channels: '5.1'
   video_codec: h264
   type: episode
@@ -2252,7 +2250,7 @@
 : title: Show Name
   date: 2016-09-28
   episode_title: Nice Title
-  other: Extended
+  edition: Extended
   screen_size: 1080p
   streaming_service: Comedy Central
   format: WEBRip
@@ -2403,7 +2401,7 @@
   screen_size: 1080p
   streaming_service: Netflix
   format: WEBRip
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: NTb
@@ -2692,7 +2690,7 @@
   screen_size: 4K
   streaming_service: Amazon Prime
   format: WEBRip
-  audio_codec: DolbyDigital
+  audio_codec: EAC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: Group
@@ -2909,7 +2907,7 @@
   episode: 10
   screen_size: 1080p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: KINGS
@@ -2935,7 +2933,7 @@
   screen_size: 1080p
   streaming_service: Netflix
   format: WEBRip
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: ViSUM
@@ -2956,7 +2954,7 @@
   episode: 5
   screen_size: 1080p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: HKD
@@ -2998,7 +2996,7 @@
   episode_title: The Brain In The Bot
   screen_size: 1080p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: R2D2
@@ -3012,7 +3010,7 @@
   episode: 7
   screen_size: 1080p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   subtitle_language: nl
@@ -3040,7 +3038,7 @@
   episode: 12
   screen_size: 1080p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   release_group: Het.Robot.Team.OYM
   type: episode
@@ -3311,7 +3309,7 @@
   screen_size: 720p
   format: WEBRip
   video_codec: h264
-  container: MKV
+  container: mkv
   audio_codec: AC3
   audio_channels: '5.1'
   release_group: Ehhhh
@@ -3501,7 +3499,7 @@
   screen_size: 1080p
   streaming_service: Amazon Prime
   format: WEBRip
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: GROUP
@@ -3592,7 +3590,7 @@
   episode: 13
   other: FINAL
   language: mul
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   screen_size: 4K
   streaming_service: Netflix
@@ -3664,7 +3662,7 @@
   season: 1
   episode: 1
   episode_title: Spark of Rebellion
-  other: Alternative Cut
+  edition: Alternative Cut
   format: HDTV
   video_codec: h264
   release_group: W4F
@@ -3760,7 +3758,7 @@
 ? Rick and Morty Season 1 [UNCENSORED] [BDRip] [1080p] [HEVC]
 : title: Rick and Morty
   season: 1
-  other: Uncensored
+  edition: Uncensored
   format: BluRay
   screen_size: 1080p
   video_codec: h265
@@ -3830,7 +3828,7 @@
   other: East Coast Feed
   screen_size: 720p
   format: WEB-DL
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   video_codec: h264
   release_group: NTb
@@ -3846,3 +3844,133 @@
   release_group: 0SEC [GloDLS]
   container: mkv
   type: episode
+
+? Anthony.Bourdain.Parts.Unknown.S09E01.Los.Angeles.720p.HDTV.x264-MiNDTHEGAP
+: title: Anthony Bourdain Parts Unknown
+  season: 9
+  episode: 1
+  episode_title: Los Angeles
+  screen_size: 720p
+  format: HDTV
+  video_codec: h264
+  release_group: MiNDTHEGAP
+  type: episode
+
+? -feud.s01e05.and.the.winner.is.(the.oscars.of.1963).720p.amzn.webrip.dd5.1.x264-casstudio.mkv
+: year: 1963
+
+? feud.s01e05.and.the.winner.is.(the.oscars.of.1963).720p.amzn.webrip.dd5.1.x264-casstudio.mkv
+: title: feud
+  season: 1
+  episode: 5
+  episode_title: and the winner is
+  screen_size: 720p
+  streaming_service: Amazon Prime
+  format: WEBRip
+  audio_codec: AC3
+  audio_channels: '5.1'
+  video_codec: h264
+  release_group: casstudio
+  container: mkv
+  type: episode
+
+? Adventure.Time.S08E16.Elements.Part.1.Skyhooks.720p.WEB-DL.AAC2.0.H.264-RTN.mkv
+: title: Adventure Time
+  season: 8
+  episode: 16
+  season: 8
+  episode: 16
+  episode_title: Elements Part 1 Skyhooks
+  screen_size: 720p
+  format: WEB-DL
+  audio_codec: AAC
+  audio_channels: '2.0'
+  video_codec: h264
+  release_group: RTN
+  container: mkv
+  type: episode
+
+? D:\TV\SITCOMS (CLASSIC)\That '70s Show\Season 07\That '70s Show - S07E22 - 2000 Light Years from Home.mkv
+: title: That '70s Show
+  season: 7
+  episode: 22
+  episode_title: 2000 Light Years from Home
+  container: mkv
+  mimetype: video/x-matroska
+  type: episode
+
+? Show.Name.S02E01.Super.Title.720p.WEB-DL.DD5.1.H.264-ABC.nzb
+: title: Show Name
+  season: 2
+  episode: 1
+  episode_title: Super Title
+  screen_size: 720p
+  format: WEB-DL
+  audio_codec: AC3
+  audio_channels: '5.1'
+  video_codec: h264
+  release_group: ABC
+  container: nzb
+  type: episode
+
+? "[SGKK] Bleach 312v1 [720p/mkv]-Group.mkv"
+: title: Bleach
+  season: 3
+  episode: 12
+  version: 1
+  screen_size: 720p
+  release_group: Group
+  container: mkv
+  type: episode
+
+? The.Expanse.S02E08.720p.WEBRip.x264.EAC3-KiNGS.mkv
+: title: The Expanse
+  season: 2
+  episode: 8
+  screen_size: 720p
+  format: WEBRip
+  video_codec: h264
+  audio_codec: EAC3
+  release_group: KiNGS
+  container: mkv
+  type: episode
+
+? Series_name.2005.211.episode.title.avi
+: title: Series name
+  year: 2005
+  season: 2
+  episode: 11
+  episode_title: episode title
+  container: avi
+  type: episode
+
+? the.flash.2014.208.hdtv-lol[ettv].mkv
+: title: the flash
+  year: 2014
+  season: 2
+  episode: 8
+  format: HDTV
+  release_group: lol[ettv]
+  container: mkv
+  type: episode
+
+? "[Despair-Paradise].Kono.Subarashii.Sekai.ni.Shukufuku.wo!.2.-..09.vostfr.FHD"
+: options: -E -t episode
+  release_group: Despair-Paradise
+  title: Kono Subarashii Sekai ni Shukufuku wo! 2
+  episode: 9
+  subtitle_language: fr
+  other: FullHD
+  type: episode
+
+? Whose Line is it anyway/Season 01/Whose.Line.is.it.Anyway.US.S13E01.720p.WEB.x264-TBS.mkv
+: title: Whose Line is it Anyway
+  season: 13
+  episode: 1
+  country: US
+  screen_size: 720p
+  format: WEB-DL
+  video_codec: h264
+  release_group: TBS
+  container: mkv
+  type: episode

Contents/Libraries/Shared/guessit/test/movies.yml

@@ -111,7 +111,7 @@
 ? Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director's.Cut).CD1.DVDRip.XviD.AC3-WAF.avi
 : title: Blade Runner
   year: 1982
-  edition: Director's cut
+  edition: Director's Cut
   cd: 1
   format: DVD
   video_codec: XviD
@@ -147,7 +147,8 @@
   format: DVD
   video_codec: XviD
   release_group: ARROW
-  other: ['Proper', 'Limited']
+  other: Proper
+  edition: Limited Edition
   proper_count: 1
 
 ? Movies/Fr - Paris 2054, Renaissance (2005) - De Christian Volckman - (Film Divx Science Fiction Fantastique Thriller Policier N&B).avi
@@ -363,7 +364,7 @@
   video_codec: h264
   release_group: AN0NYM0US[bb]
   format: BluRay
-  other: Limited
+  edition: Limited Edition
 
 ? movies/La Science des Rêves (2006)/La.Science.Des.Reves.FRENCH.DVDRip.XviD-MP-AceBot.avi
 : title: La Science des Rêves
@@ -439,7 +440,7 @@
   format: BluRay
   video_codec: XviD
   audio_channels: "5.1"
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   uuid: XD607ebb-BRc59935-5155473f-1c5f49
 
 ? Pacific.Rim.3D.2013.COMPLETE.BLURAY-PCH.avi
@@ -644,7 +645,7 @@
     - Timsit
     - Lindon
   screen_size: 1080p
-  container: MKV
+  container: mkv
   format: HDTV
 
 ? some.movie.720p.bluray.x264-mind
@@ -688,20 +689,18 @@
 
 ? h265 - HEVC Riddick Unrated Director Cut French 1080p DTS.mkv
 : audio_codec: DTS
-  edition: Director's cut
+  edition: [Unrated, Director's Cut]
   language: fr
   screen_size: 1080p
   title: Riddick
-  other: Unrated
   video_codec: h265
 
 ? "[h265 - HEVC] Riddick Unrated Director Cut French [1080p DTS].mkv"
 : audio_codec: DTS
-  edition: Director's cut
+  edition: [Unrated, Director's Cut]
   language: fr
   screen_size: 1080p
   title: Riddick
-  other: Unrated
   video_codec: h265
 
 ? Barbecue-2014-French-mHD-1080p
@@ -892,7 +891,8 @@
 
 ? Suicide Squad EXTENDED (2016) 2160p 4K UltraHD Blu-Ray x265 (HEVC 10bit BT709) Dolby Atmos 7.1 -DDR
 : title: Suicide Squad
-  other: [Extended, UltraHD]
+  edition: Extended
+  other: UltraHD
   year: 2016
   screen_size: 4K
   format: BluRay
@@ -906,7 +906,7 @@
 ? Queen - A Kind of Magic (Alternative Extended Version) 2CD 2014
 : title: Queen
   alternative_title: A Kind of Magic
-  other: [Alternative Cut, Extended]
+  edition: [Alternative Cut, Extended]
   cd_count: 2
   year: 2014
   type: movie
@@ -914,7 +914,7 @@
 ? Jour.de.Fete.1949.ALTERNATiVE.CUT.1080p.BluRay.x264-SADPANDA[rarbg]
 : title: Jour de Fete
   year: 1949
-  other: Alternative Cut
+  edition: Alternative Cut
   screen_size: 1080p
   format: BluRay
   video_codec: h264
@@ -941,7 +941,7 @@
 
 ? Alien DC (1979) [1080p]
 : title: Alien
-  edition: Director's cut
+  edition: Director's Cut
   year: 1979
   screen_size: 1080p
   type: movie
@@ -949,7 +949,7 @@
 ? Requiem.For.A.Dream.2000.DC.1080p.BluRay.x264.anoXmous
 : title: Requiem For A Dream
   year: 2000
-  edition: Director's cut
+  edition: Director's Cut
   screen_size: 1080p
   format: BluRay
   video_codec: h264
@@ -963,7 +963,7 @@
   screen_size: 1080p
   format: WEBRip
   video_codec: h264
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   audio_channels: '5.1'
   release_group: FGT
   type: movie
@@ -980,7 +980,7 @@
 ? Suntan.2016.FESTiVAL.DVDRip.x264-IcHoR
 : title: Suntan
   year: 2016
-  other: Festival
+  edition: Festival
   format: DVD
   video_codec: h264
   release_group: IcHoR
@@ -1058,7 +1058,7 @@
 ? The Heartbreak Kid (1993) UNCUT 720p WEBRip x264
 : title: The Heartbreak Kid
   year: 1993
-  other: Uncut
+  edition: Uncut
   screen_size: 720p
   format: WEBRip
   video_codec: h264
@@ -1082,3 +1082,27 @@
   format: BluRay
   screen_size: 1080p
   type: movie
+
+? 10 Cloverfield Lane.[Blu-Ray 1080p].[MULTI]
+: options: --type movie
+  title: 10 Cloverfield Lane
+  format: BluRay
+  screen_size: 1080p
+  language: Multiple languages
+  type: movie
+
+? 007.Spectre.[HDTC.MD].[TRUEFRENCH]
+: options: --type movie
+  title: 007 Spectre
+  format: HDTC
+  language: French
+  type: movie
+
+? We.Are.X.2016.LIMITED.BDRip.x264-BiPOLAR
+: title: We Are X
+  year: 2016
+  edition: Limited Edition
+  format: BluRay
+  video_codec: h264
+  release_group: BiPOLAR
+  type: movie

Contents/Libraries/Shared/guessit/test/rules/audio_codec.yml

@@ -10,9 +10,14 @@
 
 ? +DolbyDigital
 ? +DD
-? +DDP
 ? +Dolby Digital
-: audio_codec: DolbyDigital
+? +AC3
+: audio_codec: AC3
+
+? +DDP
+? +DD+
+? +EAC3
+: audio_codec: EAC3
 
 ? +DolbyAtmos
 ? +Dolby Atmos
@@ -23,9 +28,6 @@
 ? +AAC
 : audio_codec: AAC
 
-? +AC3
-: audio_codec: AC3
-
 ? +Flac
 : audio_codec: FLAC
 
@@ -88,7 +90,7 @@
 
 ? DD5.1
 ? DD51
-: audio_codec: DolbyDigital
+: audio_codec: AC3
   audio_channels: '5.1'
 
 ? -51

Contents/Libraries/Shared/guessit/test/rules/edition.yml

@@ -2,7 +2,7 @@
 # Use - marker to check inputs that should not match results.
 ? Director's cut
 ? Edition Director's cut
-: edition: Director's cut
+: edition: Director's Cut
 
 ? Collector
 ? Collector Edition
@@ -23,3 +23,9 @@
 ? Deluxe Edition
 ? Edition Deluxe
 : edition: Deluxe Edition
+
+? Super Movie Alternate XViD
+? Super Movie Alternative XViD
+? Super Movie Alternate Cut XViD
+? Super Movie Alternative Cut XViD
+: edition: Alternative Cut

Contents/Libraries/Shared/guessit/test/rules/episodes.yml

@@ -146,7 +146,7 @@
 ? Show.Name.-.Season.1.to.3.-.Mp4.1080p
 ? Show.Name.-.Season.1~3.-.Mp4.1080p
 ? Show.Name.-.Saison.1.a.3.-.Mp4.1080p
-: container: MP4
+: container: mp4
   screen_size: 1080p
   season:
   - 1

Contents/Libraries/Shared/guessit/test/rules/other.yml

@@ -87,6 +87,11 @@
 ? HD
 : other: HD
 
+? FHD
+? FullHD
+? Full HD
+: other: FullHD
+
 ? UHD
 ? Ultra
 ? UltraHD
@@ -145,11 +150,5 @@
 ? reencoded
 : other: ReEncoded
 
-? Super Movie Alternate XViD
-? Super Movie Alternative XViD
-? Super Movie Alternate Cut XViD
-? Super Movie Alternative Cut XViD
-: other: Alternative Cut
-
 ? CONVERT XViD
 : other: Converted

Contents/Libraries/Shared/guessit/test/test_yml.py

@@ -207,8 +207,6 @@ class TestYml(object):
             options = {}
         if not isinstance(options, dict):
             options = parse_options(options)
-        if 'implicit' not in options:
-            options['implicit'] = True
         options['config'] = False
         options = load_config(options)
         try:

Contents/Libraries/Shared/guessit/test/various.yml

@@ -191,7 +191,7 @@
   format: WEB-DL
   audio_channels: "5.1"
   video_codec: h264
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   release_group: NTb
 
 ? Despicable.Me.2.2013.1080p.BluRay.x264-VeDeTT.nfo
@@ -546,7 +546,7 @@
 
 ? TEST.S01E02.2160p.NF.WEBRip.x264.DD5.1-ABC
 : audio_channels: '5.1'
-  audio_codec: DolbyDigital
+  audio_codec: AC3
   episode: 2
   format: WEBRip
   release_group: ABC
@@ -744,7 +744,7 @@
 
 ? Gangs of New York 2002 REMASTERED 1080p BluRay x264-AVCHD
 : format: BluRay
-  other: Remastered
+  edition: Remastered
   screen_size: 1080p
   title: Gangs of New York
   type: movie
@@ -761,14 +761,15 @@
   type: episode
   video_codec: h264
 
+# Episode title is indeed 'October 8, 2014'
+# https://thetvdb.com/?tab=episode&seriesid=82483&seasonid=569935&id=4997362&lid=7
 ? The Soup - 11x41 - October 8, 2014.mp4
 : container: mp4
   episode: 41
-  episode_title: October 8
+  episode_title: October 8, 2014
   season: 11
   title: The Soup
   type: episode
-  year: 2014
 
 ? Red.Rock.S02E59.WEB-DLx264-JIVE
 : episode: 59

Contents/Libraries/Shared/json_tricks/__init__.py

@@ -0,0 +1,24 @@
+
+from .utils import hashodict, NoNumpyException, NoPandasException, get_scalar_repr, encode_scalars_inplace
+from .comment import strip_comment_line_with_symbol, strip_comments
+from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, json_complex_encode, \
+	numeric_types_encode, ClassInstanceEncoder, json_set_encode, pandas_encode, nopandas_encode, \
+	numpy_encode, NumpyEncoder, nonumpy_encode, NoNumpyEncoder
+from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, json_complex_hook, \
+	numeric_types_hook, ClassInstanceHook, json_set_hook, pandas_hook, nopandas_hook, json_numpy_obj_hook, \
+	json_nonumpy_obj_hook
+from .nonp import dumps, dump, loads, load
+
+
+try:
+	# find_module takes just as long as importing, so no optimization possible
+	import numpy
+except ImportError:
+	NUMPY_MODE = False
+	# from .nonp import dumps, dump, loads, load, nonumpy_encode as numpy_encode, json_nonumpy_obj_hook as json_numpy_obj_hook
+else:
+	NUMPY_MODE = True
+	# from .np import dumps, dump, loads, load, numpy_encode, NumpyEncoder, json_numpy_obj_hook
+	# from .np_utils import encode_scalars_inplace
+
+

Contents/Libraries/Shared/json_tricks/comment.py

@@ -0,0 +1,29 @@
+
+from re import findall
+
+
+def strip_comment_line_with_symbol(line, start):
+	parts = line.split(start)
+	counts = [len(findall(r'(?:^|[^"\\]|(?:\\\\|\\")+)(")', part)) for part in parts]
+	total = 0
+	for nr, count in enumerate(counts):
+		total += count
+		if total % 2 == 0:
+			return start.join(parts[:nr+1]).rstrip()
+	else:
+		return line.rstrip()
+
+
+def strip_comments(string, comment_symbols=frozenset(('#', '//'))):
+	"""
+	:param string: A string containing json with comments started by comment_symbols.
+	:param comment_symbols: Iterable of symbols that start a line comment (default # or //).
+	:return: The string with the comments removed.
+	"""
+	lines = string.splitlines()
+	for k in range(len(lines)):
+		for symbol in comment_symbols:
+			lines[k] = strip_comment_line_with_symbol(lines[k], start=symbol)
+	return '\n'.join(lines)
+
+

Contents/Libraries/Shared/json_tricks/decoders.py

@@ -0,0 +1,248 @@
+
+from datetime import datetime, date, time, timedelta
+from fractions import Fraction
+from importlib import import_module
+from collections import OrderedDict
+from decimal import Decimal
+from logging import warning
+from json_tricks import NoPandasException, NoNumpyException
+
+
+class DuplicateJsonKeyException(Exception):
+	""" Trying to load a json map which contains duplicate keys, but allow_duplicates is False """
+
+
+class TricksPairHook(object):
+	"""
+	Hook that converts json maps to the appropriate python type (dict or OrderedDict)
+	and then runs any number of hooks on the individual maps.
+	"""
+	def __init__(self, ordered=True, obj_pairs_hooks=None, allow_duplicates=True):
+		"""
+		:param ordered: True if maps should retain their ordering.
+		:param obj_pairs_hooks: An iterable of hooks to apply to elements.
+		"""
+		self.map_type = OrderedDict
+		if not ordered:
+			self.map_type = dict
+		self.obj_pairs_hooks = []
+		if obj_pairs_hooks:
+			self.obj_pairs_hooks = list(obj_pairs_hooks)
+		self.allow_duplicates = allow_duplicates
+
+	def __call__(self, pairs):
+		if not self.allow_duplicates:
+			known = set()
+			for key, value in pairs:
+				if key in known:
+					raise DuplicateJsonKeyException(('Trying to load a json map which contains a' +
+						' duplicate key "{0:}" (but allow_duplicates is False)').format(key))
+				known.add(key)
+		map = self.map_type(pairs)
+		for hook in self.obj_pairs_hooks:
+			map = hook(map)
+		return map
+
+
+def json_date_time_hook(dct):
+	"""
+	Return an encoded date, time, datetime or timedelta to it's python representation, including optional timezone.
+
+	:param dct: (dict) json encoded date, time, datetime or timedelta
+	:return: (date/time/datetime/timedelta obj) python representation of the above
+	"""
+	def get_tz(dct):
+		if not 'tzinfo' in dct:
+			return None
+		try:
+			import pytz
+		except ImportError as err:
+			raise ImportError(('Tried to load a json object which has a timezone-aware (date)time. '
+				'However, `pytz` could not be imported, so the object could not be loaded. '
+				'Error: {0:}').format(str(err)))
+		return pytz.timezone(dct['tzinfo'])
+
+	if isinstance(dct, dict):
+		if '__date__' in dct:
+			return date(year=dct.get('year', 0), month=dct.get('month', 0), day=dct.get('day', 0))
+		elif '__time__' in dct:
+			tzinfo = get_tz(dct)
+			return time(hour=dct.get('hour', 0), minute=dct.get('minute', 0), second=dct.get('second', 0),
+				microsecond=dct.get('microsecond', 0), tzinfo=tzinfo)
+		elif '__datetime__' in dct:
+			tzinfo = get_tz(dct)
+			return datetime(year=dct.get('year', 0), month=dct.get('month', 0), day=dct.get('day', 0),
+				hour=dct.get('hour', 0), minute=dct.get('minute', 0), second=dct.get('second', 0),
+				microsecond=dct.get('microsecond', 0), tzinfo=tzinfo)
+		elif '__timedelta__' in dct:
+			return timedelta(days=dct.get('days', 0), seconds=dct.get('seconds', 0),
+				microseconds=dct.get('microseconds', 0))
+	return dct
+
+
+def json_complex_hook(dct):
+	"""
+	Return an encoded complex number to it's python representation.
+
+	:param dct: (dict) json encoded complex number (__complex__)
+	:return: python complex number
+	"""
+	if isinstance(dct, dict):
+		if '__complex__' in dct:
+			parts = dct['__complex__']
+			assert len(parts) == 2
+			return parts[0] + parts[1] * 1j
+	return dct
+
+
+def numeric_types_hook(dct):
+	if isinstance(dct, dict):
+		if '__decimal__' in dct:
+			return Decimal(dct['__decimal__'])
+		if '__fraction__' in dct:
+			return Fraction(numerator=dct['numerator'], denominator=dct['denominator'])
+	return dct
+
+
+class ClassInstanceHook(object):
+	"""
+	This hook tries to convert json encoded by class_instance_encoder back to it's original instance.
+	It only works if the environment is the same, e.g. the class is similarly importable and hasn't changed.
+	"""
+	def __init__(self, cls_lookup_map=None):
+		self.cls_lookup_map = cls_lookup_map or {}
+
+	def __call__(self, dct):
+		if isinstance(dct, dict) and '__instance_type__' in dct:
+			mod, name = dct['__instance_type__']
+			attrs = dct['attributes']
+			if mod is None:
+				try:
+					Cls = getattr((__import__('__main__')), name)
+				except (ImportError, AttributeError) as err:
+					if not name in self.cls_lookup_map:
+						raise ImportError(('class {0:s} seems to have been exported from the main file, which means '
+							'it has no module/import path set; you need to provide cls_lookup_map which maps names '
+							'to classes').format(name))
+					Cls = self.cls_lookup_map[name]
+			else:
+				imp_err = None
+				try:
+					module = import_module('{0:}'.format(mod, name))
+				except ImportError as err:
+					imp_err = ('encountered import error "{0:}" while importing "{1:}" to decode a json file; perhaps '
+						'it was encoded in a different environment where {1:}.{2:} was available').format(err, mod, name)
+				else:
+					if not hasattr(module, name):
+						imp_err = 'imported "{0:}" but could find "{1:}" inside while decoding a json file (found {2:}'.format(
+							module, name, ', '.join(attr for attr in dir(module) if not attr.startswith('_')))
+					Cls = getattr(module, name)
+				if imp_err:
+					if 'name' in self.cls_lookup_map:
+						Cls = self.cls_lookup_map[name]
+					else:
+						raise ImportError(imp_err)
+			try:
+				obj = Cls.__new__(Cls)
+			except TypeError:
+				raise TypeError(('problem while decoding instance of "{0:s}"; this instance has a special '
+					'__new__ method and can\'t be restored').format(name))
+			if hasattr(obj, '__json_decode__'):
+				obj.__json_decode__(**attrs)
+			else:
+				obj.__dict__ = dict(attrs)
+			return  obj
+		return dct
+
+
+def json_set_hook(dct):
+	"""
+	Return an encoded set to it's python representation.
+	"""
+	if isinstance(dct, dict):
+		if '__set__' in dct:
+			return set((tuple(item) if isinstance(item, list) else item) for item in dct['__set__'])
+	return dct
+
+
+def pandas_hook(dct):
+	if '__pandas_dataframe__' in dct or '__pandas_series__' in dct:
+		# todo: this is experimental
+		if not getattr(pandas_hook, '_warned', False):
+			pandas_hook._warned = True
+			warning('Pandas loading support in json-tricks is experimental and may change in future versions.')
+	if '__pandas_dataframe__' in dct:
+		try:
+			from pandas import DataFrame
+		except ImportError:
+			raise NoPandasException('Trying to decode a map which appears to represent a pandas data structure, but pandas appears not to be installed.')
+		from numpy import dtype, array
+		meta = dct.pop('__pandas_dataframe__')
+		indx = dct.pop('index') if 'index' in dct else None
+		dtypes = dict((colname, dtype(tp)) for colname, tp in zip(meta['column_order'], meta['types']))
+		data = OrderedDict()
+		for name, col in dct.items():
+			data[name] = array(col, dtype=dtypes[name])
+		return DataFrame(
+			data=data,
+			index=indx,
+			columns=meta['column_order'],
+			# mixed `dtypes` argument not supported, so use duct of numpy arrays
+		)
+	elif '__pandas_series__' in dct:
+		from pandas import Series
+		from numpy import dtype, array
+		meta = dct.pop('__pandas_series__')
+		indx = dct.pop('index') if 'index' in dct else None
+		return Series(
+			data=dct['data'],
+			index=indx,
+			name=meta['name'],
+			dtype=dtype(meta['type']),
+		)
+	return dct
+
+
+def nopandas_hook(dct):
+	if isinstance(dct, dict) and ('__pandas_dataframe__' in dct or '__pandas_series__' in dct):
+		raise NoPandasException(('Trying to decode a map which appears to represent a pandas '
+			'data structure, but pandas support is not enabled, perhaps it is not installed.'))
+	return dct
+
+
+def json_numpy_obj_hook(dct):
+	"""
+	Replace any numpy arrays previously encoded by NumpyEncoder to their proper
+	shape, data type and data.
+
+	:param dct: (dict) json encoded ndarray
+	:return: (ndarray) if input was an encoded ndarray
+	"""
+	if isinstance(dct, dict) and '__ndarray__' in dct:
+		try:
+			from numpy import asarray
+			import numpy as nptypes
+		except ImportError:
+			raise NoNumpyException('Trying to decode a map which appears to represent a numpy '
+				'array, but numpy appears not to be installed.')
+		order = 'A'
+		if 'Corder' in dct:
+			order = 'C' if dct['Corder'] else 'F'
+		if dct['shape']:
+			return asarray(dct['__ndarray__'], dtype=dct['dtype'], order=order)
+		else:
+			dtype = getattr(nptypes, dct['dtype'])
+			return dtype(dct['__ndarray__'])
+	return dct
+
+
+def json_nonumpy_obj_hook(dct):
+	"""
+	This hook has no effect except to check if you're trying to decode numpy arrays without support, and give you a useful message.
+	"""
+	if isinstance(dct, dict) and '__ndarray__' in dct:
+		raise NoNumpyException(('Trying to decode a map which appears to represent a numpy array, '
+			'but numpy support is not enabled, perhaps it is not installed.'))
+	return dct
+
+

Contents/Libraries/Shared/json_tricks/encoders.py

@@ -0,0 +1,311 @@
+
+from datetime import datetime, date, time, timedelta
+from fractions import Fraction
+from logging import warning
+from json import JSONEncoder
+from sys import version
+from decimal import Decimal
+from .utils import hashodict, call_with_optional_kwargs, NoPandasException, NoNumpyException
+
+
+class TricksEncoder(JSONEncoder):
+	"""
+	Encoder that runs any number of encoder functions or instances on
+	the objects that are being encoded.
+
+	Each encoder should make any appropriate changes and return an object,
+	changed or not. This will be passes to the other encoders.
+	"""
+	def __init__(self, obj_encoders=None, silence_typeerror=False, primitives=False, **json_kwargs):
+		"""
+		:param obj_encoders: An iterable of functions or encoder instances to try.
+		:param silence_typeerror: If set to True, ignore the TypeErrors that Encoder instances throw (default False).
+		"""
+		self.obj_encoders = []
+		if obj_encoders:
+			self.obj_encoders = list(obj_encoders)
+		self.silence_typeerror = silence_typeerror
+		self.primitives = primitives
+		super(TricksEncoder, self).__init__(**json_kwargs)
+
+	def default(self, obj, *args, **kwargs):
+		"""
+		This is the method of JSONEncoders that is called for each object; it calls
+		all the encoders with the previous one's output used as input.
+
+		It works for Encoder instances, but they are expected not to throw
+		`TypeError` for unrecognized types (the super method does that by default).
+
+		It never calls the `super` method so if there are non-primitive types
+		left at the end, you'll get an encoding error.
+		"""
+		prev_id = id(obj)
+		for encoder in self.obj_encoders:
+			if hasattr(encoder, 'default'):
+				#todo: write test for this scenario (maybe ClassInstanceEncoder?)
+				try:
+					obj = call_with_optional_kwargs(encoder.default, obj, primitives=self.primitives)
+				except TypeError as err:
+					if not self.silence_typeerror:
+						raise
+			elif hasattr(encoder, '__call__'):
+				obj = call_with_optional_kwargs(encoder, obj, primitives=self.primitives)
+			else:
+				raise TypeError('`obj_encoder` {0:} does not have `default` method and is not callable'.format(encoder))
+		if id(obj) == prev_id:
+			#todo: test
+			raise TypeError('Object of type {0:} could not be encoded by {1:} using encoders [{2:s}]'.format(
+				type(obj), self.__class__.__name__, ', '.join(str(encoder) for encoder in self.obj_encoders)))
+		return obj
+
+
+def json_date_time_encode(obj, primitives=False):
+	"""
+	Encode a date, time, datetime or timedelta to a string of a json dictionary, including optional timezone.
+
+	:param obj: date/time/datetime/timedelta obj
+	:return: (dict) json primitives representation of date, time, datetime or timedelta
+	"""
+	if primitives and isinstance(obj, (date, time, datetime)):
+		return obj.isoformat()
+	if isinstance(obj, datetime):
+		dct = hashodict([('__datetime__', None), ('year', obj.year), ('month', obj.month),
+			('day', obj.day), ('hour', obj.hour), ('minute', obj.minute),
+			('second', obj.second), ('microsecond', obj.microsecond)])
+		if obj.tzinfo:
+			dct['tzinfo'] = obj.tzinfo.zone
+	elif isinstance(obj, date):
+		dct = hashodict([('__date__', None), ('year', obj.year), ('month', obj.month), ('day', obj.day)])
+	elif isinstance(obj, time):
+		dct = hashodict([('__time__', None), ('hour', obj.hour), ('minute', obj.minute),
+			('second', obj.second), ('microsecond', obj.microsecond)])
+		if obj.tzinfo:
+			dct['tzinfo'] = obj.tzinfo.zone
+	elif isinstance(obj, timedelta):
+		if primitives:
+			return obj.total_seconds()
+		else:
+			dct = hashodict([('__timedelta__', None), ('days', obj.days), ('seconds', obj.seconds),
+				('microseconds', obj.microseconds)])
+	else:
+		return obj
+	for key, val in tuple(dct.items()):
+		if not key.startswith('__') and not val:
+			del dct[key]
+	return dct
+
+
+def class_instance_encode(obj, primitives=False):
+	"""
+	Encodes a class instance to json. Note that it can only be recovered if the environment allows the class to be
+	imported in the same way.
+	"""
+	if isinstance(obj, list) or isinstance(obj, dict):
+		return obj
+	if hasattr(obj, '__class__') and hasattr(obj, '__dict__'):
+		if not hasattr(obj, '__new__'):
+			raise TypeError('class "{0:s}" does not have a __new__ method; '.format(obj.__class__) +
+				('perhaps it is an old-style class not derived from `object`; add `object` as a base class to encode it.'
+					if (version[:2] == '2.') else 'this should not happen in Python3'))
+		try:
+			obj.__new__(obj.__class__)
+		except TypeError:
+			raise TypeError(('instance "{0:}" of class "{1:}" cannot be encoded because it\'s __new__ method '
+				'cannot be called, perhaps it requires extra parameters').format(obj, obj.__class__))
+		mod = obj.__class__.__module__
+		if mod == '__main__':
+			mod = None
+			warning(('class {0:} seems to have been defined in the main file; unfortunately this means'
+				' that it\'s module/import path is unknown, so you might have to provide cls_lookup_map when '
+				'decoding').format(obj.__class__))
+		name = obj.__class__.__name__
+		if hasattr(obj, '__json_encode__'):
+			attrs = obj.__json_encode__()
+		else:
+			attrs = hashodict(obj.__dict__.items())
+		if primitives:
+			return attrs
+		else:
+			return hashodict((('__instance_type__', (mod, name)), ('attributes', attrs)))
+	return obj
+
+
+def json_complex_encode(obj, primitives=False):
+	"""
+	Encode a complex number as a json dictionary of it's real and imaginary part.
+
+	:param obj: complex number, e.g. `2+1j`
+	:return: (dict) json primitives representation of `obj`
+	"""
+	if isinstance(obj, complex):
+		if primitives:
+			return [obj.real, obj.imag]
+		else:
+			return hashodict(__complex__=[obj.real, obj.imag])
+	return obj
+
+
+def numeric_types_encode(obj, primitives=False):
+	"""
+	Encode Decimal and Fraction.
+	
+	:param primitives: Encode decimals and fractions as standard floats. You may lose precision. If you do this, you may need to enable `allow_nan` (decimals always allow NaNs but floats do not).
+	"""
+	if isinstance(obj, Decimal):
+		if primitives:
+			return float(obj)
+		else:
+			return {
+				'__decimal__': str(obj.canonical()),
+			}
+	if isinstance(obj, Fraction):
+		if primitives:
+			return float(obj)
+		else:
+			return hashodict((
+				('__fraction__', True),
+				('numerator', obj.numerator),
+				('denominator', obj.denominator),
+			))
+	return obj
+
+
+class ClassInstanceEncoder(JSONEncoder):
+	"""
+	See `class_instance_encoder`.
+	"""
+	# Not covered in tests since `class_instance_encode` is recommended way.
+	def __init__(self, obj, encode_cls_instances=True, **kwargs):
+		self.encode_cls_instances = encode_cls_instances
+		super(ClassInstanceEncoder, self).__init__(obj, **kwargs)
+
+	def default(self, obj, *args, **kwargs):
+		if self.encode_cls_instances:
+			obj = class_instance_encode(obj)
+		return super(ClassInstanceEncoder, self).default(obj, *args, **kwargs)
+
+
+def json_set_encode(obj, primitives=False):
+	"""
+	Encode python sets as dictionary with key __set__ and a list of the values.
+
+	Try to sort the set to get a consistent json representation, use arbitrary order if the data is not ordinal.
+	"""
+	if isinstance(obj, set):
+		try:
+			repr = sorted(obj)
+		except Exception:
+			repr = list(obj)
+		if primitives:
+			return repr
+		else:
+			return hashodict(__set__=repr)
+	return obj
+
+
+def pandas_encode(obj, primitives=False):
+	from pandas import DataFrame, Series
+	if isinstance(obj, (DataFrame, Series)):
+		#todo: this is experimental
+		if not getattr(pandas_encode, '_warned', False):
+			pandas_encode._warned = True
+			warning('Pandas dumping support in json-tricks is experimental and may change in future versions.')
+	if isinstance(obj, DataFrame):
+		repr = hashodict()
+		if not primitives:
+			repr['__pandas_dataframe__'] = hashodict((
+				('column_order', tuple(obj.columns.values)),
+				('types', tuple(str(dt) for dt in obj.dtypes)),
+			))
+		repr['index'] = tuple(obj.index.values)
+		for k, name in enumerate(obj.columns.values):
+			repr[name] = tuple(obj.ix[:, k].values)
+		return repr
+	if isinstance(obj, Series):
+		repr = hashodict()
+		if not primitives:
+			repr['__pandas_series__'] = hashodict((
+				('name', str(obj.name)),
+				('type', str(obj.dtype)),
+			))
+		repr['index'] = tuple(obj.index.values)
+		repr['data'] = tuple(obj.values)
+		return repr
+	return obj
+
+
+def nopandas_encode(obj):
+	if ('DataFrame' in getattr(obj.__class__, '__name__', '') or 'Series' in getattr(obj.__class__, '__name__', '')) \
+			and 'pandas.' in getattr(obj.__class__, '__module__', ''):
+		raise NoPandasException(('Trying to encode an object of type {0:} which appears to be '
+			'a numpy array, but numpy support is not enabled, perhaps it is not installed.').format(type(obj)))
+	return obj
+
+
+def numpy_encode(obj, primitives=False):
+	"""
+	Encodes numpy `ndarray`s as lists with meta data.
+	
+	Encodes numpy scalar types as Python equivalents. Special encoding is not possible,
+	because int64 (in py2) and float64 (in py2 and py3) are subclasses of primitives,
+	which never reach the encoder.
+	
+	:param primitives: If True, arrays are serialized as (nested) lists without meta info.
+	"""
+	from numpy import ndarray, generic
+	if isinstance(obj, ndarray):
+		if primitives:
+			return obj.tolist()
+		else:
+			dct = hashodict((
+				('__ndarray__', obj.tolist()),
+				('dtype', str(obj.dtype)),
+				('shape', obj.shape),
+			))
+			if len(obj.shape) > 1:
+				dct['Corder'] = obj.flags['C_CONTIGUOUS']
+			return dct
+	elif isinstance(obj, generic):
+		if NumpyEncoder.SHOW_SCALAR_WARNING:
+			NumpyEncoder.SHOW_SCALAR_WARNING = False
+			warning('json-tricks: numpy scalar serialization is experimental and may work differently in future versions')
+		return obj.item()
+	return obj
+
+
+class NumpyEncoder(ClassInstanceEncoder):
+	"""
+	JSON encoder for numpy arrays.
+	"""
+	SHOW_SCALAR_WARNING = True  # show a warning that numpy scalar serialization is experimental
+	
+	def default(self, obj, *args, **kwargs):
+		"""
+		If input object is a ndarray it will be converted into a dict holding
+		data type, shape and the data. The object can be restored using json_numpy_obj_hook.
+		"""
+		warning('`NumpyEncoder` is deprecated, use `numpy_encode`')  #todo
+		obj = numpy_encode(obj)
+		return super(NumpyEncoder, self).default(obj, *args, **kwargs)
+
+
+def nonumpy_encode(obj):
+	"""
+	Raises an error for numpy arrays.
+	"""
+	if 'ndarray' in getattr(obj.__class__, '__name__', '') and 'numpy.' in getattr(obj.__class__, '__module__', ''):
+		raise NoNumpyException(('Trying to encode an object of type {0:} which appears to be '
+			'a pandas data stucture, but pandas support is not enabled, perhaps it is not installed.').format(type(obj)))
+	return obj
+
+
+class NoNumpyEncoder(JSONEncoder):
+	"""
+	See `nonumpy_encode`.
+	"""
+	def default(self, obj, *args, **kwargs):
+		warning('`NoNumpyEncoder` is deprecated, use `nonumpy_encode`')  #todo
+		obj = nonumpy_encode(obj)
+		return super(NoNumpyEncoder, self).default(obj, *args, **kwargs)
+
+

Contents/Libraries/Shared/json_tricks/nonp.py

@@ -0,0 +1,207 @@
+
+from gzip import GzipFile
+from io import BytesIO
+from json import loads as json_loads
+from os import fsync
+from sys import exc_info, version
+from .utils import NoNumpyException  # keep 'unused' imports
+from .comment import strip_comment_line_with_symbol, strip_comments  # keep 'unused' imports
+from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, ClassInstanceEncoder, \
+	json_complex_encode, json_set_encode, numeric_types_encode, numpy_encode, nonumpy_encode, NoNumpyEncoder, \
+	nopandas_encode, pandas_encode  # keep 'unused' imports
+from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, ClassInstanceHook, \
+	json_complex_hook, json_set_hook, numeric_types_hook, json_numpy_obj_hook, json_nonumpy_obj_hook, \
+	nopandas_hook, pandas_hook  # keep 'unused' imports
+from json import JSONEncoder
+
+
+is_py3 = (version[:2] == '3.')
+str_type = str if is_py3 else (basestring, unicode,)
+ENCODING = 'UTF-8'
+
+
+_cih_instance = ClassInstanceHook()
+DEFAULT_ENCODERS = [json_date_time_encode, class_instance_encode, json_complex_encode, json_set_encode, numeric_types_encode,]
+DEFAULT_HOOKS = [json_date_time_hook, _cih_instance, json_complex_hook, json_set_hook, numeric_types_hook,]
+
+try:
+	import numpy
+except ImportError:
+	DEFAULT_ENCODERS = [nonumpy_encode,] + DEFAULT_ENCODERS
+	DEFAULT_HOOKS = [json_nonumpy_obj_hook,] + DEFAULT_HOOKS
+else:
+	# numpy encode needs to be before complex
+	DEFAULT_ENCODERS = [numpy_encode,] + DEFAULT_ENCODERS
+	DEFAULT_HOOKS = [json_numpy_obj_hook,] + DEFAULT_HOOKS
+
+try:
+	import pandas
+except ImportError:
+	DEFAULT_ENCODERS = [nopandas_encode,] + DEFAULT_ENCODERS
+	DEFAULT_HOOKS = [nopandas_hook,] + DEFAULT_HOOKS
+else:
+	DEFAULT_ENCODERS = [pandas_encode,] + DEFAULT_ENCODERS
+	DEFAULT_HOOKS = [pandas_hook,] + DEFAULT_HOOKS
+
+
+DEFAULT_NONP_ENCODERS = [nonumpy_encode,] + DEFAULT_ENCODERS    # DEPRECATED
+DEFAULT_NONP_HOOKS = [json_nonumpy_obj_hook,] + DEFAULT_HOOKS   # DEPRECATED
+
+
+def dumps(obj, sort_keys=None, cls=TricksEncoder, obj_encoders=DEFAULT_ENCODERS, extra_obj_encoders=(),
+		primitives=False, compression=None, allow_nan=False, conv_str_byte=False, **jsonkwargs):
+	"""
+	Convert a nested data structure to a json string.
+
+	:param obj: The Python object to convert.
+	:param sort_keys: Keep this False if you want order to be preserved.
+	:param cls: The json encoder class to use, defaults to NoNumpyEncoder which gives a warning for numpy arrays.
+	:param obj_encoders: Iterable of encoders to use to convert arbitrary objects into json-able promitives.
+	:param extra_obj_encoders: Like `obj_encoders` but on top of them: use this to add encoders without replacing defaults. Since v3.5 these happen before default encoders.
+	:param allow_nan: Allow NaN and Infinity values, which is a (useful) violation of the JSON standard (default False).
+	:param conv_str_byte: Try to automatically convert between strings and bytes (assuming utf-8) (default False).
+	:return: The string containing the json-encoded version of obj.
+
+	Other arguments are passed on to `cls`. Note that `sort_keys` should be false if you want to preserve order.
+	"""
+	if not hasattr(extra_obj_encoders, '__iter__'):
+		raise TypeError('`extra_obj_encoders` should be a tuple in `json_tricks.dump(s)`')
+	encoders = tuple(extra_obj_encoders) + tuple(obj_encoders)
+	txt = cls(sort_keys=sort_keys, obj_encoders=encoders, allow_nan=allow_nan,
+		primitives=primitives, **jsonkwargs).encode(obj)
+	if not is_py3 and isinstance(txt, str):
+		txt = unicode(txt, ENCODING)
+	if not compression:
+		return txt
+	if compression is True:
+		compression = 5
+	txt = txt.encode(ENCODING)
+	sh = BytesIO()
+	with GzipFile(mode='wb', fileobj=sh, compresslevel=compression) as zh:
+		zh.write(txt)
+	gzstring = sh.getvalue()
+	return gzstring
+
+
+def dump(obj, fp, sort_keys=None, cls=TricksEncoder, obj_encoders=DEFAULT_ENCODERS, extra_obj_encoders=(),
+		 primitives=False, compression=None, force_flush=False, allow_nan=False, conv_str_byte=False, **jsonkwargs):
+	"""
+	Convert a nested data structure to a json string.
+
+	:param fp: File handle or path to write to.
+	:param compression: The gzip compression level, or None for no compression.
+	:param force_flush: If True, flush the file handle used, when possibly also in the operating system (default False).
+	
+	The other arguments are identical to `dumps`.
+	"""
+	txt = dumps(obj, sort_keys=sort_keys, cls=cls, obj_encoders=obj_encoders, extra_obj_encoders=extra_obj_encoders,
+		primitives=primitives, compression=compression, allow_nan=allow_nan, conv_str_byte=conv_str_byte, **jsonkwargs)
+	if isinstance(fp, str_type):
+		fh = open(fp, 'wb+')
+	else:
+		fh = fp
+		if conv_str_byte:
+			try:
+				fh.write(b'')
+			except TypeError:
+				pass
+				# if not isinstance(txt, str_type):
+				# 	# Cannot write bytes, so must be in text mode, but we didn't get a text
+				# 	if not compression:
+				# 		txt = txt.decode(ENCODING)
+			else:
+				try:
+					fh.write(u'')
+				except TypeError:
+					if isinstance(txt, str_type):
+						txt = txt.encode(ENCODING)
+	try:
+		if 'b' not in getattr(fh, 'mode', 'b?') and not isinstance(txt, str_type) and compression:
+			raise IOError('If compression is enabled, the file must be opened in binary mode.')
+		try:
+			fh.write(txt)
+		except TypeError as err:
+			err.args = (err.args[0] + '. A possible reason is that the file is not opened in binary mode; '
+				'be sure to set file mode to something like "wb".',)
+			raise
+	finally:
+		if force_flush:
+			fh.flush()
+			try:
+				if fh.fileno() is not None:
+					fsync(fh.fileno())
+			except (ValueError,):
+				pass
+		if isinstance(fp, str_type):
+			fh.close()
+	return txt
+
+
+def loads(string, preserve_order=True, ignore_comments=True, decompression=None, obj_pairs_hooks=DEFAULT_HOOKS,
+		extra_obj_pairs_hooks=(), cls_lookup_map=None, allow_duplicates=True, conv_str_byte=False, **jsonkwargs):
+	"""
+	Convert a nested data structure to a json string.
+
+	:param string: The string containing a json encoded data structure.
+	:param decode_cls_instances: True to attempt to decode class instances (requires the environment to be similar the the encoding one).
+	:param preserve_order: Whether to preserve order by using OrderedDicts or not.
+	:param ignore_comments: Remove comments (starting with # or //).
+	:param decompression: True to use gzip decompression, False to use raw data, None to automatically determine (default). Assumes utf-8 encoding!
+	:param obj_pairs_hooks: A list of dictionary hooks to apply.
+	:param extra_obj_pairs_hooks: Like `obj_pairs_hooks` but on top of them: use this to add hooks without replacing defaults. Since v3.5 these happen before default hooks.
+	:param cls_lookup_map: If set to a dict, for example ``globals()``, then classes encoded from __main__ are looked up this dict.
+	:param allow_duplicates: If set to False, an error will be raised when loading a json-map that contains duplicate keys.
+	:param parse_float: A function to parse strings to integers (e.g. Decimal). There is also `parse_int`.
+	:param conv_str_byte: Try to automatically convert between strings and bytes (assuming utf-8) (default False).
+	:return: The string containing the json-encoded version of obj.
+
+	Other arguments are passed on to json_func.
+	"""
+	if not hasattr(extra_obj_pairs_hooks, '__iter__'):
+		raise TypeError('`extra_obj_pairs_hooks` should be a tuple in `json_tricks.load(s)`')
+	if decompression is None:
+		decompression = string[:2] == b'\x1f\x8b'
+	if decompression:
+		with GzipFile(fileobj=BytesIO(string), mode='rb') as zh:
+			string = zh.read()
+			string = string.decode(ENCODING)
+	if not isinstance(string, str_type):
+		if conv_str_byte:
+			string = string.decode(ENCODING)
+		else:
+			raise TypeError(('Cannot automatically encode object of type "{0:}" in `json_tricks.load(s)` since '
+				'the encoding is not known. You should instead encode the bytes to a string and pass that '
+				'string to `load(s)`, for example bytevar.encode("utf-8") if utf-8 is the encoding.').format(type(string)))
+	if ignore_comments:
+		string = strip_comments(string)
+	obj_pairs_hooks = tuple(obj_pairs_hooks)
+	_cih_instance.cls_lookup_map = cls_lookup_map or {}
+	hooks = tuple(extra_obj_pairs_hooks) + obj_pairs_hooks
+	hook = TricksPairHook(ordered=preserve_order, obj_pairs_hooks=hooks, allow_duplicates=allow_duplicates)
+	return json_loads(string, object_pairs_hook=hook, **jsonkwargs)
+
+
+def load(fp, preserve_order=True, ignore_comments=True, decompression=None, obj_pairs_hooks=DEFAULT_HOOKS,
+		extra_obj_pairs_hooks=(), cls_lookup_map=None, allow_duplicates=True, conv_str_byte=False, **jsonkwargs):
+	"""
+	Convert a nested data structure to a json string.
+
+	:param fp: File handle or path to load from.
+
+	The other arguments are identical to loads.
+	"""
+	try:
+		if isinstance(fp, str_type):
+			with open(fp, 'rb') as fh:
+				string = fh.read()
+		else:
+			string = fp.read()
+	except UnicodeDecodeError as err:
+		# todo: not covered in tests, is it relevant?
+		raise Exception('There was a problem decoding the file content. A possible reason is that the file is not ' +
+			'opened  in binary mode; be sure to set file mode to something like "rb".').with_traceback(exc_info()[2])
+	return loads(string, preserve_order=preserve_order, ignore_comments=ignore_comments, decompression=decompression,
+		obj_pairs_hooks=obj_pairs_hooks, extra_obj_pairs_hooks=extra_obj_pairs_hooks, cls_lookup_map=cls_lookup_map,
+		allow_duplicates=allow_duplicates, conv_str_byte=conv_str_byte, **jsonkwargs)
+
+

Contents/Libraries/Shared/json_tricks/np.py

@@ -0,0 +1,28 @@
+
+"""
+This file exists for backward compatibility reasons.
+"""
+
+from logging import warning
+from .nonp import NoNumpyException, DEFAULT_ENCODERS, DEFAULT_HOOKS, dumps, dump, loads, load  # keep 'unused' imports
+from .utils import hashodict, NoPandasException
+from .comment import strip_comment_line_with_symbol, strip_comments  # keep 'unused' imports
+from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, ClassInstanceEncoder, \
+	numpy_encode, NumpyEncoder # keep 'unused' imports
+from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, ClassInstanceHook, \
+	json_complex_hook, json_set_hook, json_numpy_obj_hook  # keep 'unused' imports
+
+try:
+	import numpy
+except ImportError:
+	raise NoNumpyException('Could not load numpy, maybe it is not installed? If you do not want to use numpy encoding '
+		'or decoding, you can import the functions from json_tricks.nonp instead, which do not need numpy.')
+
+
+# todo: warning('`json_tricks.np` is deprecated, you can import directly from `json_tricks`')
+
+
+DEFAULT_NP_ENCODERS = [numpy_encode,] + DEFAULT_ENCODERS    # DEPRECATED
+DEFAULT_NP_HOOKS = [json_numpy_obj_hook,] + DEFAULT_HOOKS   # DEPRECATED
+
+

Contents/Libraries/Shared/json_tricks/np_utils.py

@@ -0,0 +1,15 @@
+
+"""
+This file exists for backward compatibility reasons.
+"""
+
+from .utils import hashodict, get_scalar_repr, encode_scalars_inplace
+from .nonp import NoNumpyException
+from . import np
+
+# try:
+# 	from numpy import generic, complex64, complex128
+# except ImportError:
+# 	raise NoNumpyException('Could not load numpy, maybe it is not installed?')
+
+

Contents/Libraries/Shared/json_tricks/utils.py

@@ -0,0 +1,81 @@
+
+from collections import OrderedDict
+
+
+class hashodict(OrderedDict):
+	"""
+	This dictionary is hashable. It should NOT be mutated, or all kinds of weird
+	bugs may appear. This is not enforced though, it's only used for encoding.
+	"""
+	def __hash__(self):
+		return hash(frozenset(self.items()))
+
+
+try:
+	from inspect import signature
+except ImportError:
+	try:
+		from inspect import getfullargspec
+	except ImportError:
+		from inspect import getargspec
+		def get_arg_names(callable):
+			argspec = getargspec(callable)
+			return set(argspec.args)
+	else:
+		#todo: this is not covered in test case (py 3+ uses `signature`, py2 `getfullargspec`); consider removing it
+		def get_arg_names(callable):
+			argspec = getfullargspec(callable)
+			return set(argspec.args) | set(argspec.kwonlyargs)
+else:
+	def get_arg_names(callable):
+		sig = signature(callable)
+		return set(sig.parameters.keys())
+
+
+def call_with_optional_kwargs(callable, *args, **optional_kwargs):
+	accepted_kwargs = get_arg_names(callable)
+	use_kwargs = {}
+	for key, val in optional_kwargs.items():
+		if key in accepted_kwargs:
+			use_kwargs[key] = val
+	return callable(*args, **use_kwargs)
+
+
+class NoNumpyException(Exception):
+	""" Trying to use numpy features, but numpy cannot be found. """
+
+
+class NoPandasException(Exception):
+	""" Trying to use pandas features, but pandas cannot be found. """
+
+
+def get_scalar_repr(npscalar):
+	return hashodict((
+		('__ndarray__', npscalar.item()),
+		('dtype', str(npscalar.dtype)),
+		('shape', ()),
+	))
+
+
+def encode_scalars_inplace(obj):
+	"""
+	Searches a data structure of lists, tuples and dicts for numpy scalars
+	and replaces them by their dictionary representation, which can be loaded
+	by json-tricks. This happens in-place (the object is changed, use a copy).
+	"""
+	from numpy import generic, complex64, complex128
+	if isinstance(obj, (generic, complex64, complex128)):
+		return get_scalar_repr(obj)
+	if isinstance(obj, dict):
+		for key, val in tuple(obj.items()):
+			obj[key] = encode_scalars_inplace(val)
+		return obj
+	if isinstance(obj, list):
+		for k, val in enumerate(obj):
+			obj[k] = encode_scalars_inplace(val)
+		return obj
+	if isinstance(obj, (tuple, set)):
+		return type(obj)(encode_scalars_inplace(val) for val in obj)
+	return obj
+
+

Contents/Libraries/Shared/plex/objects/library/media.py

@@ -23,6 +23,17 @@ class Media(Descriptor):
     bitrate = Property(type=int)
     duration = Property(type=int)
 
+    #@classmethod
+    #def from_node(cls, client, node):
+    #    return cls.construct(client, cls.helpers.find(node, 'Media'), child=True)
+
     @classmethod
     def from_node(cls, client, node):
-        return cls.construct(client, cls.helpers.find(node, 'Media'), child=True)
+        items = []
+
+        for genre in cls.helpers.findall(node, 'Media'):
+            _, obj = Media.construct(client, genre, child=True)
+
+            items.append(obj)
+
+        return [], items

Contents/Libraries/Shared/plex/objects/library/section.py

@@ -14,6 +14,7 @@ class Section(Directory):
     language = Property
 
     composite = Property
+    type = Property
 
     created_at = Property('createdAt', int)
 

Contents/Libraries/Shared/provider_test.sh

@@ -1,28 +1,28 @@
 # addic7ed
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; ProviderPool(providers=['addic7ed'], provider_configs={'addic7ed': {'use_random_agents': True}})['addic7ed'].query('Game of Thrones', 2)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; ProviderPool(providers=['addic7ed'], provider_configs={'addic7ed': {'use_random_agents': True}})['addic7ed'].query('Game of Thrones', 2)"
 
 # opensubtitles
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; ProviderPool(providers=['opensubtitles'], )['opensubtitles'].query([Language('eng')], query='Game of Thrones', season=2, episode=1, tag='Game.of.Thrones.S06E01.The.Red.Woman.720p.WEB-DL.DD5.1.H.264-NTB.mkv', use_tag_search=True)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subzero.video import parse_video; SZProviderPool(providers=['opensubtitles'], )['opensubtitles'].list_subtitles(parse_video('FULL_PATH', {}, {'type': 'episode'}), languages=[Language('eng')])"
 
 # podnapisi
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; ProviderPool(providers=['podnapisi'], )['podnapisi'].query([Language('eng')], 'Game of Thrones', season=2, episode=1)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; SZProviderPool(providers=['podnapisi'], )['podnapisi'].query([Language('eng')], 'Game of Thrones', season=2, episode=1)"
 
 # tvsubtitles
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; ProviderPool(providers=['tvsubtitles'], )['tvsubtitles'].query('Game of Thrones', 2, 1)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; SZProviderPool(providers=['tvsubtitles'], )['tvsubtitles'].query('Game of Thrones', 2, 1)"
 
 # napiprojekt:list
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; from subliminal.core import scan_video; print ProviderPool(providers=['napiprojekt'], )['napiprojekt'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('pol')])"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subliminal.core import scan_video; print SZProviderPool(providers=['napiprojekt'], )['napiprojekt'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('pol')])"
 
 # napiprojekt:download
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import PatchedProviderPool; from subliminal import download_best_subtitles; from babelfish import Language; from subliminal.core import scan_video; subs = download_best_subtitles([scan_video('FULL_PATH')], languages={Language('eng')}, providers=['napiprojekt'], ); print subs.values()[0][0].is_valid()"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from subliminal_patch.score import compute_score; from subliminal import download_best_subtitles; from babelfish import Language; from subliminal.core import scan_video; subs = download_best_subtitles([scan_video('FULL_PATH')], languages={Language('eng')}, providers=['napiprojekt'], pool_class=SZProviderPool, compute_score=compute_score); print subs.values()[0][0].is_valid()"
 
 
 # shooter:list
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; from subliminal.core import scan_video; print ProviderPool(providers=['shooter'], )['shooter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('zho')])"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subliminal.core import scan_video; print SZProviderPool(providers=['shooter'], )['shooter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('zho')])"
 
 # subscenter:list
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal import ProviderPool; from babelfish import Language; from subliminal.core import scan_video; print ProviderPool(providers=['subscenter'], )['subscenter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('heb')])"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subliminal.core import scan_video; print SZProviderPool(providers=['subscenter'], )['subscenter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('heb')])"
 
 
 # refining
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; print parse_video('FILE_NAME', hints={'type': 'episode'}, dry_run=True)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; print parse_video('FILE_NAME', {}, hints={'type': 'episode'}, dry_run=True)"

Contents/Libraries/Shared/rarfile.py

@@ -2672,7 +2672,7 @@ def _parse_xtime(flag, data, pos, basetime=None):
 def is_filelike(obj):
     """Filename or file object?
     """
-    if isinstance(obj, str) or isinstance(obj, unicode):
+    if isinstance(obj, (bytes, unicode)):
         return False
     res = True
     for a in ('read', 'tell', 'seek'):

Contents/Libraries/Shared/rebulk/__version__.py

@@ -4,4 +4,4 @@
 Version module
 """
 # pragma: no cover
-__version__ = '0.8.2'
+__version__ = '0.9.0'

Contents/Libraries/Shared/rebulk/chain.py

@@ -300,14 +300,14 @@ class Chain(Pattern):
 
         if not is_chain_start:
             separator = chain_input_string[0:chain_part_matches[0].initiator.raw_start]
-            if len(separator) > 0:
+            if separator:
                 return []
 
         j = 1
         for i in range(0, len(chain_part_matches) - 1):
             separator = chain_input_string[chain_part_matches[i].initiator.raw_end:
                                            chain_part_matches[i + 1].initiator.raw_start]
-            if len(separator) > 0:
+            if separator:
                 break
             j += 1
         truncated = chain_part_matches[:j]

Contents/Libraries/Shared/rebulk/match.py

@@ -38,7 +38,7 @@ class _BaseMatches(MutableSequence):
     _base_remove = _base.remove
     _base_extend = _base.extend
 
-    def __init__(self, matches=None, input_string=None):
+    def __init__(self, matches=None, input_string=None):  # pylint: disable=super-init-not-called
         self.input_string = input_string
         self._max_end = 0
         self._delegate = []
@@ -493,14 +493,17 @@ class _BaseMatches(MutableSequence):
         """
         return self._tag_dict.keys()
 
-    def to_dict(self, details=False, implicit=False):
+    def to_dict(self, details=False, first_value=False, enforce_list=False):
         """
         Converts matches to a dict object.
         :param details if True, values will be complete Match object, else it will be only string Match.value property
         :type details: bool
-        :param implicit if True, multiple values will be set as a list in the dict. Else, only the first value
-        will be kept.
-        :type implicit: bool
+        :param first_value if True, only the first value will be kept. Else, multiple values will be set as a list in
+        the dict.
+        :type first_value: bool
+        :param enforce_list: if True, value is wrapped in a list even when a single value is found. Else, list values
+        are available under `values_list` property of the returned dict object.
+        :type enforce_list: bool
         :return:
         :rtype: dict
         """
@@ -508,10 +511,10 @@ class _BaseMatches(MutableSequence):
         for match in sorted(self):
             value = match if details else match.value
             ret.matches[match.name].append(match)
-            if value not in ret.values_list[match.name]:
+            if not enforce_list and value not in ret.values_list[match.name]:
                 ret.values_list[match.name].append(value)
             if match.name in ret.keys():
-                if implicit:
+                if not first_value:
                     if not isinstance(ret[match.name], list):
                         if ret[match.name] == value:
                             continue
@@ -521,7 +524,10 @@ class _BaseMatches(MutableSequence):
                             continue
                     ret[match.name].append(value)
             else:
-                ret[match.name] = value
+                if enforce_list and not isinstance(value, list):
+                    ret[match.name] = [value]
+                else:
+                    ret[match.name] = value
         return ret
 
     if six.PY2:  # pragma: no cover
@@ -561,9 +567,9 @@ class _BaseMatches(MutableSequence):
     def __repr__(self):
         return self._delegate.__repr__()
 
-    def insert(self, index, match):
-        self._delegate.insert(index, match)
-        self._add_match(match)
+    def insert(self, index, value):
+        self._delegate.insert(index, value)
+        self._add_match(value)
 
 
 class Matches(_BaseMatches):
@@ -671,12 +677,11 @@ class Match(object):
         """
         if not self.children:
             return set([self.name])
-        else:
-            ret = set()
-            for child in self.children:
-                for name in child.names:
-                    ret.add(name)
-            return ret
+        ret = set()
+        for child in self.children:
+            for name in child.names:
+                ret.add(name)
+        return ret
 
     @property
     def raw_start(self):
@@ -768,10 +773,10 @@ class Match(object):
                     # crop is included in self, split current ...
                     right = copy.deepcopy(current)
                     current.end = start
-                    if len(current) <= 0:
+                    if not current:
                         ret.remove(current)
                     right.start = end
-                    if len(right) > 0:
+                    if right:
                         ret.append(right)
                 elif end <= current.end and end > current.start:
                     current.start = end

Contents/Libraries/Shared/rebulk/pattern.py

@@ -136,7 +136,7 @@ class Pattern(object):
         :return:
         :rtype:
         """
-        if len(match) == 0 or match.value == "":
+        if not match or match.value == "":
             return False
 
         pattern_value = get_first_defined(self.values, [match.name, '__parent__', None],
@@ -164,7 +164,7 @@ class Pattern(object):
         :return:
         :rtype:
         """
-        if len(child) == 0 or child.value == "":
+        if not child or child.value == "":
             return False
 
         pattern_value = get_first_defined(self.values, [child.name, '__children__', None],

Contents/Libraries/Shared/rebulk/processors.py

@@ -51,6 +51,7 @@ class ConflictSolver(Rule):
         return _default_conflict_solver
 
     def when(self, matches, context):
+        # pylint:disable=too-many-nested-blocks
         to_remove_matches = IdentitySet()
 
         public_matches = [match for match in matches if not match.private]

Contents/Libraries/Shared/rebulk/test/default_rules_module.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, len-as-condition
 from ..match import Match
 from ..rules import Rule, RemoveMatch, AppendMatch, RenameMatch, AppendTags, RemoveTags
 

Contents/Libraries/Shared/rebulk/test/rebulk_rules_module.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, len-as-condition
 from rebulk.rules import Rule, RemoveMatch, CustomRule
 
 

Contents/Libraries/Shared/rebulk/test/rules_module.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, len-as-condition
 from ..match import Match
 from ..rules import Rule
 

Contents/Libraries/Shared/rebulk/test/test_chain.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member, len-as-condition
 import re
 
 from functools import partial

Contents/Libraries/Shared/rebulk/test/test_debug.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, protected-access, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, protected-access, invalid-name, len-as-condition
 
 from ..pattern import StringPattern
 from ..rebulk import Rebulk

Contents/Libraries/Shared/rebulk/test/test_introspector.py

@@ -3,7 +3,7 @@
 """
 Introspector tests
 """
-# pylint: disable=no-self-use,pointless-statement,missing-docstring,protected-access,invalid-name
+# pylint: disable=no-self-use,pointless-statement,missing-docstring,protected-access,invalid-name,len-as-condition
 from ..rebulk import Rebulk
 from .. import introspector
 from .default_rules_module import RuleAppend2, RuleAppend3

Contents/Libraries/Shared/rebulk/test/test_loose.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, len-as-condition
 
 from ..loose import call
 

Contents/Libraries/Shared/rebulk/test/test_match.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, unneeded-not
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, unneeded-not, len-as-condition
 
 import pytest
 import six
@@ -419,7 +419,7 @@ class TestMaches(object):
         matches.extend(RePattern("Three", name="3bis", tags=["Three", "re"]).matches(input_string))
         matches.extend(RePattern(r"(\w+)", name="words").matches(input_string))
 
-        kvalues = matches.to_dict()
+        kvalues = matches.to_dict(first_value=True)
         assert kvalues == {"1": "One",
                            "2": "Two",
                            "3": "Three",
@@ -427,7 +427,10 @@ class TestMaches(object):
                            "words": "One"}
         assert kvalues.values_list["words"] == ["One", "Two", "Three"]
 
-        kvalues = matches.to_dict(details=True, implicit=True)
+        kvalues = matches.to_dict(enforce_list=True)
+        assert kvalues["words"] == ["One", "Two", "Three"]
+
+        kvalues = matches.to_dict(details=True)
         assert kvalues["1"].value == "One"
 
         assert len(kvalues["2"]) == 2

Contents/Libraries/Shared/rebulk/test/test_pattern.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, unbalanced-tuple-unpacking
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, unbalanced-tuple-unpacking, len-as-condition
 
 import re
 import pytest
@@ -396,10 +396,10 @@ class TestRePattern(object):
 
         children = matches[0].children
         assert len(children) == 2
-        assert children[0].name is "test"
+        assert children[0].name == "test"
         assert children[0].value == "HE"
 
-        assert children[1].name is "test"
+        assert children[1].name == "test"
         assert children[1].value == "HE"
 
         pattern = RePattern("H(?P<first>e.)(?P<second>rew)", name="test", value="HE")
@@ -807,8 +807,7 @@ class TestValidator(object):
         def invalid_func(match):
             if match.name == 'intParam':
                 return True
-            else:
-                return match.value.startswith('abc')
+            return match.value.startswith('abc')
 
         pattern = RePattern(r"contains (?P<intParam>\d+)", formatter=int, validator=invalid_func, validate_all=True,
                             children=True)
@@ -819,8 +818,7 @@ class TestValidator(object):
         def func(match):
             if match.name == 'intParam':
                 return True
-            else:
-                return match.value.startswith('contains')
+            return match.value.startswith('contains')
 
         pattern = RePattern(r"contains (?P<intParam>\d+)", formatter=int, validator=func, validate_all=True,
                             children=True)

Contents/Libraries/Shared/rebulk/test/test_processors.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member, len-as-condition
 
 from ..pattern import StringPattern, RePattern
 from ..processors import ConflictSolver

Contents/Libraries/Shared/rebulk/test/test_rebulk.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, no-member, len-as-condition
 
 from ..rebulk import Rebulk
 from ..rules import Rule

Contents/Libraries/Shared/rebulk/test/test_rules.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, no-member
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name, no-member, len-as-condition
 import pytest
 from rebulk.test.default_rules_module import RuleRemove0, RuleAppend0, RuleRename0, RuleAppend1, RuleRemove1, \
     RuleRename1, RuleAppend2, RuleRename2, RuleAppend3, RuleRename3, RuleAppendTags0, RuleRemoveTags0, \

Contents/Libraries/Shared/rebulk/test/test_validators.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name
+# pylint: disable=no-self-use, pointless-statement, missing-docstring, invalid-name,len-as-condition
 
 from functools import partial
 

Contents/Libraries/Shared/rebulk/utils.py

@@ -83,6 +83,7 @@ def is_iterable(obj):
     We don't need to check for the Python 2 `unicode` type, because it doesn't
     have an `__iter__` attribute anyway.
     """
+    # pylint: disable=consider-using-ternary
     return hasattr(obj, '__iter__') and not isinstance(obj, str) or isinstance(obj, GeneratorType)
 
 
@@ -117,7 +118,7 @@ class IdentitySet(MutableSet):  # pragma: no cover
     """
     Set based on identity
     """
-    def __init__(self, items=None):
+    def __init__(self, items=None):  # pylint: disable=super-init-not-called
         if items is None:
             items = []
         self.refs = set(map(_Ref, items))
@@ -131,11 +132,11 @@ class IdentitySet(MutableSet):  # pragma: no cover
     def __len__(self):
         return len(self.refs)
 
-    def add(self, elem):
-        self.refs.add(_Ref(elem))
+    def add(self, value):
+        self.refs.add(_Ref(value))
 
-    def discard(self, elem):
-        self.refs.discard(_Ref(elem))
+    def discard(self, value):
+        self.refs.discard(_Ref(value))
 
     def update(self, iterable):
         """

Contents/Libraries/Shared/subliminal/providers/legendastv.py

@@ -48,7 +48,13 @@ timestamp_re = re.compile(r'(?P<day>\d+)/(?P<month>\d+)/(?P<year>\d+) - (?P<hour
 title_re = re.compile(r'^(?P<series>.*?)(?: \((?:(?P<year>\d{4})|(?P<country>[A-Z]{2}))\))?$')
 
 #: Cache key for releases
-releases_key = __name__ + ':releases|{archive_id}'
+releases_key = 'ltv:releases|{archive_id}|{archive_name}'
+
+
+#: Check if the value should actually be cached in dogpile or not
+def should_cache_titles(titles):
+    """Return False if title is an empty dict to avoid caching it."""
+    return titles != {}
 
 
 class LegendasTVArchive(object):
@@ -200,17 +206,24 @@ class LegendasTVProvider(Provider):
 
         self.session.close()
 
-    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
-    def search_titles(self, title):
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME, should_cache_fn=should_cache_titles)
+    def search_titles(self, title, season):
         """Search for titles matching the `title`.
 
+        For episodes, each season has it own title, so we must cache with season number
+
         :param str title: the title to search for.
+        :param int season: season of the title (used only for dogpile cache)
         :return: found titles.
         :rtype: dict
 
         """
         # make the query
-        logger.info('Searching title %r', title)
+        if season:
+            logger.info('Searching episode title %r for season %r', title, season)
+        else:
+            logger.info('Searching movie title %r', title)
+
         r = self.session.get(self.server_url + 'legenda/sugestao/{}'.format(title), timeout=10)
         r.raise_for_status()
         results = json.loads(r.text)
@@ -274,21 +287,23 @@ class LegendasTVProvider(Provider):
         """
         logger.info('Getting archives for title %d and language %d', title_id, language_code)
         archives = []
-        page = 1
+        page = 0
         while True:
             # get the archive page
-            url = self.server_url + 'util/carrega_legendas_busca_filme/{title}/{language}/-/{page}'.format(
-                title=title_id, language=language_code, page=page)
+            url = self.server_url + 'legenda/busca/-/{language}/-/{page}/{title}'.format(
+                language=language_code, page=page, title=title_id)
             r = self.session.get(url)
             r.raise_for_status()
 
             # parse the results
             soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
-            for archive_soup in soup.select('div.list_element > article > div'):
+            for archive_soup in soup.select('div.list_element > article > div > div.f_left'):
                 # create archive
-                archive = LegendasTVArchive(archive_soup.p.a['href'].split('/')[2], archive_soup.p.a.text,
-                                            'pack' in archive_soup['class'], 'destaque' in archive_soup['class'],
-                                            self.server_url + archive_soup.p.a['href'][1:])
+                archive = LegendasTVArchive(archive_soup.a['href'].split('/')[2],
+                                            archive_soup.a.text,
+                                            'pack' in archive_soup.parent['class'],
+                                            'destaque' in archive_soup.parent['class'],
+                                            self.server_url + archive_soup.a['href'][1:])
 
                 # extract text containing downloads, rating and timestamp
                 data_text = archive_soup.find('p', class_='data').text
@@ -308,6 +323,8 @@ class LegendasTVProvider(Provider):
                     raise ProviderError('Archive timestamp is in the future')
 
                 # add archive
+                logger.info('Found archive for title %d and language %d at page %s: %s',
+                            title_id, language_code, page, archive)
                 archives.append(archive)
 
             # stop on last page
@@ -345,12 +362,23 @@ class LegendasTVProvider(Provider):
 
     def query(self, language, title, season=None, episode=None, year=None):
         # search for titles
-        titles = self.search_titles(sanitize(title))
+        titles = self.search_titles(sanitize(title), season)
+        titles_cache_key = 'ltv:search_titles|{title} {season}'.format(title=title.lower(), season=season)
+        if region.get(titles_cache_key) != NO_VALUE:
+            logger.debug('Found %d cached titles for %s', len(titles), title)
 
         # search for titles with the quote or dot character
         ignore_characters = {'\'', '.'}
         if any(c in title for c in ignore_characters):
-            titles.update(self.search_titles(sanitize(title, ignore_characters=ignore_characters)))
+            sanitized_title = sanitize(title, ignore_characters=ignore_characters)
+            additional_titles = self.search_titles(sanitized_title, season)
+            titles.update(additional_titles)
+
+            titles_cache_key = 'ltv:search_titles|{title} {season}'.format(title=sanitized_title.lower(),
+                                                                                   season=season)
+
+            if region.get(titles_cache_key) != NO_VALUE:
+                logger.debug('Found %d cached titles for %s', len(additional_titles), sanitized_title)
 
         subtitles = []
         # iterate over titles
@@ -398,7 +426,8 @@ class LegendasTVProvider(Provider):
                 expiration_time = (datetime.utcnow().replace(tzinfo=pytz.utc) - a.timestamp).total_seconds()
 
                 # attempt to get the releases from the cache
-                releases = region.get(releases_key.format(archive_id=a.id), expiration_time=expiration_time)
+                cache_item = releases_key.format(archive_id=a.id, archive_name=a.name)
+                releases = region.get(cache_item, expiration_time=expiration_time)
 
                 # the releases are not in cache or cache is expired
                 if releases == NO_VALUE:
@@ -425,7 +454,7 @@ class LegendasTVProvider(Provider):
                         releases.append(name)
 
                     # cache the releases
-                    region.set(releases_key.format(archive_id=a.id), releases)
+                    region.set(cache_item, releases)
 
                 # iterate over releases
                 for r in releases:

Contents/Libraries/Shared/subliminal_patch/__init__.py

@@ -1,40 +1,26 @@
 # coding=utf-8
 
-import importlib
 import subliminal
 
 # patch subliminal's subtitle and provider base
 from .subtitle import PatchedSubtitle
 from .providers import Provider
-from .http import RetryingSession
 subliminal.subtitle.Subtitle = PatchedSubtitle
 
-try:
-    subliminal.provider_manager.register('napiprojekt = subliminal.providers.napiprojekt:NapiProjektProvider',)
-except ValueError:
-    # already registered
-    pass
-
-# add our patched base classes
-for name in ("Addic7ed", "Podnapisi", "TVsubtitles", "OpenSubtitles", "LegendasTV", "NapiProjekt", "Shooter",
-             "SubsCenter"):
-    mod = importlib.import_module("subliminal.providers.%s" % name.lower())
-    setattr(getattr(mod, "%sSubtitle" % name), "__bases__", (PatchedSubtitle,))
-    setattr(getattr(mod, "%sProvider" % name), "__bases__", (Provider,))
-
-    # inject our requests.Session wrapper for automatic retry
-    setattr(mod, "Session", RetryingSession)
-
-from .core import scan_video, search_external_subtitles, list_all_subtitles, save_subtitles, refine
+from .core import scan_video, search_external_subtitles, list_all_subtitles, save_subtitles, refine, \
+    download_best_subtitles
 from .score import compute_score
-from .extensions import provider_manager
+from .video import Video
+import extensions
 
 # patch subliminal's core functions
 subliminal.scan_video = subliminal.core.scan_video = scan_video
 subliminal.core.search_external_subtitles = search_external_subtitles
 subliminal.save_subtitles = subliminal.core.save_subtitles = save_subtitles
 subliminal.refine = subliminal.core.refine = refine
+subliminal.video.Video = subliminal.Video = Video
+subliminal.video.Episode.__bases__ = (Video,)
+subliminal.video.Movie.__bases__ = (Video,)
 
 # add our own list_all_subtitles
 subliminal.list_all_subtitles = subliminal.core.list_all_subtitles = list_all_subtitles
-subliminal.provider_manager = subliminal.core.provider_manager = provider_manager

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+import codecs
 import json
 import re
 import os
@@ -10,6 +11,7 @@ import operator
 
 import itertools
 
+import rarfile
 from concurrent.futures import ThreadPoolExecutor
 
 import requests
@@ -20,11 +22,11 @@ from babelfish import LanguageReverseError
 from guessit.jsonutils import GuessitEncoder
 from subliminal import ProviderError, refiner_manager
 
+from extensions import provider_registry
 from subliminal.score import compute_score as default_compute_score
-from subliminal.subtitle import SUBTITLE_EXTENSIONS
 from subliminal.utils import hash_napiprojekt, hash_opensubtitles, hash_shooter, hash_thesubdb
 from subliminal.video import VIDEO_EXTENSIONS, Video, Episode, Movie
-from subliminal.core import guessit, Language, ProviderPool, io
+from subliminal.core import guessit, Language, ProviderPool, io, download_best_subtitles
 
 logger = logging.getLogger(__name__)
 
@@ -37,8 +39,94 @@ DOWNLOAD_RETRY_SLEEP = 2
 
 REMOVE_CRAP_FROM_FILENAME = re.compile(r"(?i)[\s_-]+(obfuscated|scrambled)(\.\w+)$")
 
+SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '.vtt')
+
 
 class SZProviderPool(ProviderPool):
+    def __init__(self, providers=None, provider_configs=None):
+        #: Name of providers to use
+        self.providers = providers or provider_registry.names()
+
+        #: Provider configuration
+        self.provider_configs = provider_configs or {}
+
+        #: Initialized providers
+        self.initialized_providers = {}
+
+        #: Discarded providers
+        self.discarded_providers = set()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.terminate()
+
+    def __getitem__(self, name):
+        if name not in self.providers:
+            raise KeyError
+        if name not in self.initialized_providers:
+            logger.info('Initializing provider %s', name)
+            provider = provider_registry[name](**self.provider_configs.get(name, {}))
+            provider.initialize()
+            self.initialized_providers[name] = provider
+
+        return self.initialized_providers[name]
+
+    def __delitem__(self, name):
+        if name not in self.initialized_providers:
+            raise KeyError(name)
+
+        try:
+            logger.info('Terminating provider %s', name)
+            self.initialized_providers[name].terminate()
+        except (requests.Timeout, socket.timeout):
+            logger.error('Provider %r timed out, improperly terminated', name)
+        except:
+            logger.exception('Provider %r terminated unexpectedly', name)
+
+        del self.initialized_providers[name]
+
+    def list_subtitles_provider(self, provider, video, languages):
+        """List subtitles with a single provider.
+
+        The video and languages are checked against the provider.
+        
+        patch: add traceback info
+
+        :param str provider: name of the provider.
+        :param video: video to list subtitles for.
+        :type video: :class:`~subliminal.video.Video`
+        :param languages: languages to search for.
+        :type languages: set of :class:`~babelfish.language.Language`
+        :return: found subtitles.
+        :rtype: list of :class:`~subliminal.subtitle.Subtitle` or None
+
+        """
+        # check video validity
+        if not provider_registry[provider].check(video):
+            logger.info('Skipping provider %r: not a valid video', provider)
+            return []
+
+        # check supported languages
+        provider_languages = provider_registry[provider].languages & languages
+        if not provider_languages:
+            logger.info('Skipping provider %r: no language to search for', provider)
+            return []
+
+        # list subtitles
+        logger.info('Listing subtitles with provider %r and languages %r', provider, provider_languages)
+        try:
+            ret = self[provider].list_subtitles(video, provider_languages)
+            for s in ret:
+                s.plex_media_fps = float(video.fps)
+            return ret
+
+        except (requests.Timeout, socket.timeout):
+            logger.error('Provider %r timed out', provider)
+        except:
+            logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
+
     def list_subtitles(self, video, languages):
         """List subtitles.
         
@@ -102,14 +190,22 @@ class SZProviderPool(ProviderPool):
             try:
                 self[subtitle.provider_name].download_subtitle(subtitle)
                 break
-            except (requests.Timeout, socket.timeout):
-                logger.error('Provider %r timed out', subtitle.provider_name)
-            except ProviderError:
-                logger.error('Unexpected error in provider %r, Traceback: %s', subtitle.provider_name,
-                             traceback.format_exc())
+            except (requests.ConnectionError,
+                    requests.exceptions.ProxyError,
+                    requests.exceptions.SSLError,
+                    requests.Timeout,
+                    socket.timeout):
+                logger.error('Provider %r connection error', subtitle.provider_name)
+
+            except rarfile.BadRarFile:
+                logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)
+                return False
+
             except:
                 logger.exception('Unexpected error in provider %r, Traceback: %s', subtitle.provider_name,
                                  traceback.format_exc())
+                self.discarded_providers.add(subtitle.provider_name)
+                return False
 
             if tries == DOWNLOAD_TRIES:
                 self.discarded_providers.add(subtitle.provider_name)
@@ -126,6 +222,8 @@ class SZProviderPool(ProviderPool):
             logger.error('Invalid subtitle')
             return False
 
+        subtitle.set_encoding("utf-8")
+
         return True
 
     def download_best_subtitles(self, subtitles, video, languages, min_score=0, hearing_impaired=False, only_one=False,
@@ -192,7 +290,8 @@ class SZProviderPool(ProviderPool):
                 continue
 
             # bail out if hearing_impaired was wrong
-            if "hearing_impaired" not in matches and hearing_impaired in ("force HI", "force non-HI"):
+            if subtitle.hearing_impaired_verifiable and "hearing_impaired" not in matches and \
+                            hearing_impaired in ("force HI", "force non-HI"):
                 logger.debug('%r: Skipping subtitle with score %d because hearing-impaired set to %s', subtitle,
                              score, hearing_impaired)
                 continue
@@ -290,7 +389,8 @@ def scan_video(path, dont_use_actual_file=False, hints=None):
     guess_from = REMOVE_CRAP_FROM_FILENAME.sub(r"\2", guess_from)
 
     # guess
-    guessed_result = guessit(guess_from, options=hints or {})
+    hints["single_value"] = True
+    guessed_result = guessit(guess_from, options=hints)
     logger.debug('GuessIt found: %s', json.dumps(guessed_result, cls=GuessitEncoder, indent=4, ensure_ascii=False))
     video = Video.fromguess(path, guessed_result)
 
@@ -327,7 +427,7 @@ def _search_external_subtitles(path, forced_tag=False):
             continue
 
         p_root, p_ext = os.path.splitext(p)
-        if not INCLUDE_EXOTIC_SUBS and p_ext not in (".srt", ".ass", ".ssa"):
+        if not INCLUDE_EXOTIC_SUBS and p_ext not in (".srt", ".ass", ".ssa", ".vtt"):
             continue
 
         # extract potential forced/normal/default tag
@@ -459,8 +559,8 @@ def get_subtitle_path(video_path, language=None, extension='.srt', forced_tag=Fa
     return subtitle_root + extension
 
 
-def save_subtitles(video, subtitles, single=False, directory=None, encoding=None, encode_with=None, chmod=None,
-                   forced_tag=False, path_decoder=None):
+def save_subtitles(video, subtitles, single=False, directory=None, chmod=None, formats=("srt",), forced_tag=False,
+                   path_decoder=None, debug_mods=False):
     """Save subtitles on filesystem.
 
     Subtitles are saved in the order of the list. If a subtitle with a language has already been saved, other subtitles
@@ -469,18 +569,21 @@ def save_subtitles(video, subtitles, single=False, directory=None, encoding=None
     The extension used is `.lang.srt` by default or `.srt` is `single` is `True`, with `lang` being the IETF code for
     the :attr:`~subliminal.subtitle.Subtitle.language` of the subtitle.
 
+    :param formats: list of "srt" and "vtt"
     :param video: video of the subtitles.
     :type video: :class:`~subliminal.video.Video`
     :param subtitles: subtitles to save.
     :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
     :param bool single: save a single subtitle, default is to save one subtitle per language.
     :param str directory: path to directory where to save the subtitles, default is next to the video.
-    :param str encoding: encoding in which to save the subtitles, default is to keep original encoding.
     :return: the saved subtitles
     :rtype: list of :class:`~subliminal.subtitle.Subtitle`
 
     patch: unicode path problems
     """
+
+    logger.debug("Subtitle formats requested: %r", formats)
+
     saved_subtitles = []
     for subtitle in subtitles:
         # check content
@@ -506,31 +609,13 @@ def save_subtitles(video, subtitles, single=False, directory=None, encoding=None
 
         subtitle.storage_path = subtitle_path
 
-        # save content as is or in the specified encoding
-        logger.info('Saving %r to %r', subtitle, subtitle_path)
-        has_encoder = callable(encode_with)
+        for format in formats:
+            if format != "srt":
+                subtitle_path = os.path.splitext(subtitle_path)[0] + (u".%s" % format)
 
-        if has_encoder:
-            logger.info('Using encoder %s' % encode_with.__name__)
-
-        # save normalized subtitle if encoder or no encoding is given
-        if has_encoder or encoding is None:
-            content = encode_with(subtitle.text) if has_encoder else subtitle.content
-            with io.open(subtitle_path, 'wb') as f:
-                f.write(content)
-
-            # change chmod if requested
-            if chmod:
-                os.chmod(subtitle_path, chmod)
-
-            if single:
-                break
-            continue
-
-        # save subtitle if encoding given
-        if encoding is not None:
-            with io.open(subtitle_path, 'w', encoding=encoding) as f:
-                f.write(subtitle.text)
+            logger.debug(u"Saving %r to %r", subtitle, subtitle_path)
+            with open(subtitle_path, 'w') as f:
+                f.write(subtitle.get_modified_content(format=format))
 
         # change chmod if requested
         if chmod:

Contents/Libraries/Shared/subliminal_patch/extensions.py

@@ -1,22 +1,57 @@
 # coding=utf-8
+from collections import OrderedDict
+
 import subliminal
 import babelfish
-from subliminal.extensions import RegistrableExtensionManager
 
-provider_manager = RegistrableExtensionManager('subliminal.providers', [
-    'addic7ed = subliminal_patch.providers.addic7ed:Addic7edProvider',
-    'legendastv = subliminal_patch.providers.legendastv:LegendasTVProvider',
-    'opensubtitles = subliminal_patch.providers.opensubtitles:OpenSubtitlesProvider',
-    'podnapisi = subliminal_patch.providers.podnapisi:PodnapisiProvider',
-    'shooter = subliminal_patch.providers.shooter:ShooterProvider',
-    'napiprojekt = subliminal_patch.providers.napiprojekt:NapiProjektProvider',
-    'subscenter = subliminal_patch.providers.subscenter:SubsCenterProvider',
-    'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
-    'tvsubtitles = subliminal_patch.providers.tvsubtitles:TVsubtitlesProvider'
-])
+
+class ProviderRegistry(object):
+    providers = None
+
+    def __init__(self):
+        self.providers = OrderedDict()
+
+    def __cmp__(self, d):
+        return cmp(self.providers, d)
+
+    def __contains__(self, item):
+        return item in self.providers
+
+    def __setitem__(self, key, item):
+        self.providers[key] = item
+
+    def __iter__(self):
+        return iter(self.providers)
+
+    def __getitem__(self, key):
+        if key in self.providers:
+            return self.providers[key]
+
+    def __repr__(self):
+        return repr(self.providers)
+
+    def __str__(self):
+        return str(self.providers)
+
+    def __len__(self):
+        return len(self.providers)
+
+    def __delitem__(self, key):
+        del self.providers[key]
+
+    def register(self, name, cls):
+        self.providers[name] = cls
+
+    def names(self):
+        return self.providers.keys()
+
+
+provider_registry = ProviderRegistry()
 
 # add language converters
 babelfish.language_converters.unregister('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
 babelfish.language_converters.register('addic7ed = subliminal_patch.language:PatchedAddic7edConverter')
+babelfish.language_converters.register('szopensubtitles = subliminal_patch.language:PatchedOpenSubtitlesConverter')
 subliminal.refiner_manager.register('sz_metadata = subliminal_patch.refiners.metadata:refine')
+subliminal.refiner_manager.register('sz_omdb = subliminal_patch.refiners.omdb:refine')