release 1.4.11.781

# Conflicts:
#	Contents/Code/__init__.py
#	Contents/Code/interface/menu_helpers.py
#	Contents/Code/support/items.py
#	Contents/Code/support/plex_media.py
#	Contents/Info.plist
#	Contents/Libraries/Shared/subzero/intent.py
#	Contents/Libraries/Shared/subzero/lib/dict.py

CHANGELOG.md

@@ -1,3 +1,66 @@
+1.4.5.742
+- core: fix force-refresh in certain situations
+- menu: add history
+- menu: add manual subtitle selection
+- menu: run Items with missing subtitles in separate thread for big libraries
+- settings: add history list size option (default: 100)
+- settings: add new default scores (TV: 110); use input instead of dropdown
+- settings: increase default missing subtitles amount per library to 2000
+- core: generic rewrites and optimizations
+- core: better hash verification
+- core: add anonymous usage data (opt-out in settings)
+- core: fix pt-BR display (IETF) again
+- wiki: update (thanks @dane22!) - quick URL: http://v.ht/szwiki
+- wiki: add score explanation - quick URL: http://v.ht/szscores
+- core: add persian/farsi encoding support
+
+
+1.3.49.636
+- core/menu: fix force refreshing (again)
+- core/menu: fix redundant route calls
+
+
+1.3.49.630 (backported some changes of the develop-1.4 branch to 1.3)
+- core/menu: make addic7ed boost configurable; lower the default boost value massively (to 10)
+- core: fix force refreshing (hopefully)
+- core: add (thai) tis-620 subtitle encoding support
+- menu: lower letter based menu browsing from 200 to 80 items
+- core: support greek encodings (windows-1253, cp1253, cp737, iso8859_7, cp875, cp869, iso2022_jp_2, mac_greek); hopefully fixes badly saved greek subs
+- menu: add generic back-to-home button to the top of every container view
+- menu: warn the user when SZ isn't enabled for any sections/libraries
+- menu: always re-check permissions status and enabled sections when opening the main menu; no server restart necessary anymore
+
+
+1.3.46.606
+- core: hotfix for new users (who've never downloaded a subtitle with SZ before); fixes #169
+
+
+1.3.46.605
+
+- add wiki (thanks @ukdtom / @dane22)
+- core: remove necessity of Plex credentials; fixes #148
+- core: fix non-SRT subtitle support; fixes #138
+- core: generic source overhaul in preparation for release 1.4
+- core: better filesystem encoding detection; may fix #159
+- core: add encoding handling for windows-1250 and windows-1251 encoding (eastern europe); fixes #162
+- core: overhaul ignore handling; fixes #164
+- core: implement ignore by path setting; fixes #134
+- core: add setting for optional fallback to metadata storage, if filesystem storage failed; fixes #100
+- core: add setting for notifying an executable after a subtitle has been downloaded (see Wiki); fixes #65
+- core: only handle sections for which Sub-Zero is enabled (in PMS agent settings); fixes #167
+- menu: add series/season force-refresh
+- menu: show item thumbnail/art where applicable
+- menu: mitigate PlexWeb behaviour of calling our handlers twice; fixes #168
+
+
+1.3.33.522
+
+- core: fix library permission detection on windows; fixes #151
+- core: "Restrict to one language" now behaves like it should (one found subtitle of any language is treated as sufficient); fixes #149
+- core: add support for other subtitle formats such as ssa/ass/microdvd, convert to srt; fixes #138
+- core: hopefully more consistent force-refresh handling (intent); fixes #118
+
+
 1.3.31.513
 
 - core: add option to only download one language again (and skip the addition of .lang to the subtitle filename) (default: off); fixes #126 

Contents/Code/__init__.py

@@ -1,8 +1,7 @@
 # coding=utf-8
-import os
+import datetime
 import sys
 
-# just some slight modifications to support sum and iter again
 from subzero.sandbox import restore_builtins
 
 module = sys.modules['__main__']
@@ -14,30 +13,25 @@ for key, value in getattr(module, "__builtins__").iteritems():
         globals()[key] = value
 
 import logger
-import logging
-
-# temporarily add the console handler and set it to DEBUG to catch errors upon imports
-Core.log.addHandler(logger.console_handler)
-Core.log.setLevel(logging.DEBUG)
 
 sys.modules["logger"] = logger
 
 import subliminal
-import subliminal_patch
 import support
 
 import interface
 sys.modules["interface"] = interface
 
 from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
-from subzero import intent
 from interface.menu import *
-from support.plex_media import convert_media_to_parts, get_media_item_ids, scan_parts
-from support.subtitlehelpers import get_subtitles_from_metadata, force_utf8
-from support.helpers import notify_executable
-from support.storage import store_subtitle_info, whack_missing_parts
+from support.plex_media import media_to_videos, get_media_item_ids, scan_videos
+from support.subtitlehelpers import get_subtitles_from_metadata
+from support.storage import whack_missing_parts, save_subtitles
 from support.items import is_ignored
 from support.config import config
+from support.lib import get_intent
+from support.helpers import track_usage, get_title_for_video_metadata, get_identifier, cast_bool
+from support.history import get_history
 
 
 def Start():
@@ -47,6 +41,17 @@ def Start():
     # configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
     subliminal.region.configure('dogpile.cache.memory')
 
+    # clear expired intents
+    intent = get_intent()
+    intent.cleanup()
+
+    # clear expired menu history items
+    now = datetime.datetime.now()
+    if "menu_history" in Dict:
+        for key, timeout in Dict["menu_history"].items():
+            if now > timeout:
+                del Dict["menu_history"][key]
+
     # init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
     ValidatePrefs()
     Log.Debug(config.full_version)
@@ -57,15 +62,20 @@ def Start():
             Log.Error("Insufficient permissions on library %s, folder: %s" % (title, path))
         return
 
+    # run task scheduler
     scheduler.run()
 
+    if "anon_id" not in Dict:
+        Dict["anon_id"] = get_identifier()
 
-def init_subliminal_patches():
-    # configure custom subtitle destination folders for scanning pre-existing subs
-    dest_folder = config.subtitle_destination_folder
-    subliminal_patch.patch_video.CUSTOM_PATHS = [dest_folder] if dest_folder else []
-    subliminal_patch.patch_provider_pool.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
-    subliminal_patch.patch_providers.addic7ed.USE_BOOST = bool(Prefs['provider.addic7ed.boost'])
+    # track usage
+    if cast_bool(Prefs["track_usage"]):
+        if "first_use" not in Dict:
+            Dict["first_use"] = datetime.datetime.utcnow()
+            Dict.Save()
+            track_usage("General", "plugin", "first_start", 1)
+        else:
+            track_usage("General", "plugin", "start", 1)
 
 
 def download_best_subtitles(video_part_map, min_score=0):
@@ -106,71 +116,6 @@ def download_best_subtitles(video_part_map, min_score=0):
     Log.Debug("All languages for all requested videos exist. Doing nothing.")
 
 
-def save_subtitles(videos, subtitles):
-    meta_fallback = False
-    save_successful = False
-    storage = "metadata"
-    if Prefs['subtitles.save.filesystem']:
-        storage = "filesystem"
-        try:
-            Log.Debug("Using filesystem as subtitle storage")
-            save_subtitles_to_file(subtitles)
-        except OSError:
-            if Prefs["subtitles.save.metadata_fallback"]:
-                meta_fallback = True
-            else:
-                raise
-        else:
-            save_successful = True
-
-    if not Prefs['subtitles.save.filesystem'] or meta_fallback:
-        if meta_fallback:
-            Log.Debug("Using metadata as subtitle storage, because filesystem storage failed")
-        else:
-            Log.Debug("Using metadata as subtitle storage")
-        save_successful = save_subtitles_to_metadata(videos, subtitles)
-
-    if save_successful and config.notify_executable:
-        notify_executable(config.notify_executable, videos, subtitles, storage)
-
-    store_subtitle_info(videos, subtitles, storage)
-
-
-def save_subtitles_to_file(subtitles):
-    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
-
-    for video, video_subtitles in subtitles.items():
-        if not video_subtitles:
-            continue
-
-        fld = None
-        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
-            # specific subFolder requested, create it if it doesn't exist
-            fld_base = os.path.split(video.name)[0]
-            if fld_custom:
-                if fld_custom.startswith("/"):
-                    # absolute folder
-                    fld = fld_custom
-                else:
-                    fld = os.path.join(fld_base, fld_custom)
-            else:
-                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
-            if not os.path.exists(fld):
-                os.makedirs(fld)
-        subliminal.api.save_subtitles(video, video_subtitles, directory=fld, single=Prefs['subtitles.only_one'],
-                                      encode_with=force_utf8 if Prefs['subtitles.enforce_encoding'] else None)
-    return True
-
-
-def save_subtitles_to_metadata(videos, subtitles):
-    for video, video_subtitles in subtitles.items():
-        mediaPart = videos[video]
-        for subtitle in video_subtitles:
-            content = force_utf8(subtitle.text) if Prefs['subtitles.enforce_encoding'] else subtitle.content
-            mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.page_link] = Proxy.Media(content, ext="srt")
-    return True
-
-
 def update_local_media(metadata, media, media_type="movies"):
     # Look for subtitles
     if media_type == "movies":
@@ -212,23 +157,22 @@ class SubZeroAgent(object):
 
     def update(self, metadata, media, lang):
         Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))
+        intent = get_intent()
 
         if not media:
             Log.Error("Called with empty media, something is really wrong with your setup!")
             return
 
-        set_refresh_menu_state(media, media_type=self.agent_type)
-
         item_ids = []
         try:
-            init_subliminal_patches()
-            parts = convert_media_to_parts(media, kind=self.agent_type)
+            config.init_subliminal_patches()
+            videos = media_to_videos(media, kind=self.agent_type)
 
             # media ignored?
             use_any_parts = False
-            for part in parts:
-                if is_ignored(part["id"]):
-                    Log.Debug(u"Ignoring %s" % part)
+            for video in videos:
+                if is_ignored(video["id"]):
+                    Log.Debug(u"Ignoring %s" % video)
                     continue
                 use_any_parts = True
 
@@ -236,15 +180,34 @@ class SubZeroAgent(object):
                 Log.Debug(u"Nothing to do.")
                 return
 
-            use_score = Prefs[self.score_prefs_key]
-            scanned_parts = scan_parts(parts, kind=self.agent_type)
-            subtitles = download_best_subtitles(scanned_parts, min_score=int(use_score))
+            try:
+                use_score = int(Prefs[self.score_prefs_key].strip())
+            except ValueError:
+                Log.Error("Please only put numbers into the scores setting. Exiting")
+                return
+
+            set_refresh_menu_state(media, media_type=self.agent_type)
+
+            # scanned_video_part_map = {subliminal.Video: plex_part, ...}
+            scanned_video_part_map = scan_videos(videos, kind=self.agent_type)
+
+            # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
+            downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score)
             item_ids = get_media_item_ids(media, kind=self.agent_type)
 
-            whack_missing_parts(scanned_parts)
+            whack_missing_parts(scanned_video_part_map)
 
-            if subtitles:
-                save_subtitles(scanned_parts, subtitles)
+            if downloaded_subtitles:
+                save_subtitles(scanned_video_part_map, downloaded_subtitles)
+                track_usage("Subtitle", "refreshed", "download", 1)
+
+                for video, video_subtitles in downloaded_subtitles.items():
+                    # store item(s) in history
+                    for subtitle in video_subtitles:
+                        item_title = get_title_for_video_metadata(video.plexapi_metadata, add_section_title=False)
+                        history = get_history()
+                        history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
+                                    subtitle=subtitle)
 
             update_local_media(metadata, media, media_type=self.agent_type)
 
@@ -258,17 +221,18 @@ class SubZeroAgent(object):
 
                 # resolve existing intent for that id
                 intent.resolve("force", item_id)
+
             Dict.Save()
 
 
 class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies):
     contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', 'com.plexapp.agents.hama']
-    score_prefs_key = "subtitles.search.minimumMovieScore"
+    score_prefs_key = "subtitles.search.minimumMovieScore1"
     agent_type_verbose = "Movies"
 
 
 class SubZeroSubtitlesAgentTvShows(SubZeroAgent, Agent.TV_Shows):
     contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.themoviedb',
                       'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv', 'com.plexapp.agents.hama']
-    score_prefs_key = "subtitles.search.minimumTVScore"
+    score_prefs_key = "subtitles.search.minimumTVScore1"
     agent_type_verbose = "TV"

Contents/Code/interface/menu.py

@@ -1,19 +1,22 @@
 # coding=utf-8
 import logging
 import logger
+import os
 
 from menu_helpers import add_ignore_options, dig_tree, set_refresh_menu_state, \
-    should_display_ignore, enable_channel_wrapper, default_thumb, debounce
+    should_display_ignore, enable_channel_wrapper, default_thumb, debounce, SZObjectContainer
 from subzero.constants import TITLE, ART, ICON, PREFIX, PLUGIN_IDENTIFIER, DEPENDENCY_MODULE_NAMES
+from subzero.history_storage import mode_map
 from support.background import scheduler
 from support.config import config
-from support.helpers import pad_title, timestamp
+from support.helpers import pad_title, timestamp, get_language, df, cast_bool
 from support.ignore import ignore_list
-from support.items import get_item, get_on_deck_items, refresh_item, get_all_items, get_recent_items, get_items_info, get_item_thumb
+from support.items import get_item, get_on_deck_items, refresh_item, get_all_items, get_recent_items, get_items_info, \
+    get_item_thumb, get_item_kind_from_rating_key
 from support.lib import Plex
 from support.missing_subtitles import items_get_all_missing_subs
+from support.plex_media import get_plex_metadata, scan_videos
 from support.storage import reset_storage, log_storage, get_subtitle_info
-from support.plex_media import scan_parts
 
 # init GUI
 ObjectContainer.art = R(ART)
@@ -35,10 +38,16 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
     """
     subzero main menu
     """
-    title = force_title if force_title is not None else config.full_version
-    oc = ObjectContainer(title1=title, title2=None, header=unicode(header) if header else header, message=message, no_history=no_history,
+    title = config.full_version#force_title if force_title is not None else config.full_version
+    oc = ObjectContainer(title1=title, title2=title, header=unicode(header) if header else title, message=message, no_history=no_history,
                          replace_parent=replace_parent, no_cache=True)
 
+    # always re-check permissions
+    config.refresh_permissions_status()
+
+    # always re-check enabled sections
+    config.refresh_enabled_sections()
+
     if not config.permissions_ok and config.missing_permissions:
         for title, path in config.missing_permissions:
             oc.add(DirectoryObject(
@@ -48,6 +57,14 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
             ))
         return oc
 
+    if not config.enabled_sections:
+        oc.add(DirectoryObject(
+            key=Callback(fatality, randomize=timestamp()),
+            title=pad_title("I'm not enabled!"),
+            summary="Please enable me for some of your libraries in your server settings; currently I do nothing",
+        ))
+        return oc
+
     if not only_refresh:
         if Dict["current_refresh_state"]:
             oc.add(DirectoryObject(
@@ -65,7 +82,7 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
             summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/subtitles."
         ))
         oc.add(DirectoryObject(
-            key=Callback(RecentlyAddedMenu),
+            key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()),
             title="Items with missing subtitles",
             summary="Shows the items honoring the configured 'Item age to be considered recent'-setting (%s)"
                     " and allowing you to individually (force-) refresh their metadata/subtitles. " % Prefs["scheduler.item_is_recent_age"]
@@ -77,14 +94,14 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
                     "(force-) refresh the metadata/subtitles of individual items."
         ))
 
-        task_name = "searchAllRecentlyAddedMissing"
+        task_name = "SearchAllRecentlyAddedMissing"
         task = scheduler.task(task_name)
 
         if task.ready_for_display:
             task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
         else:
-            task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (scheduler.last_run(task_name) or "never",
-                                                                                               scheduler.next_run(task_name) or "never",
+            task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (df(scheduler.last_run(task_name)) or "never",
+                                                                                               df(scheduler.next_run(task_name)) or "never",
                                                                                                str(task.last_run_time).split(".")[0])
 
         oc.add(DirectoryObject(
@@ -99,6 +116,12 @@ def fatality(randomize=None, force_title=None, header=None, message=None, only_r
             summary="Show the current ignore list (mainly used for the automatic tasks)"
         ))
 
+        oc.add(DirectoryObject(
+            key=Callback(HistoryMenu),
+            title="History",
+            summary="Show the last %i downloaded subtitles" % int(Prefs["history_size"])
+        ))
+
     oc.add(DirectoryObject(
         key=Callback(fatality, force_title=" ", randomize=timestamp()),
         title=pad_title("Refresh"),
@@ -128,28 +151,43 @@ def OnDeckMenu(message=None):
     return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=get_on_deck_items)
 
 
-@route(PREFIX + '/recent')
-def RecentlyAddedMenu(message=None):
-    """
-    displays the recently added items with missing subtitles
-    :param message:
-    :return:
-    """
-    return recentItemsMenu(title="Missing Subtitles", base_title="Missing Subtitles")
+@route(PREFIX + '/recent', force=bool)
+@debounce
+def RecentMissingSubtitlesMenu(force=False, randomize=None):
+    title="Items with missing subtitles"
+    oc = SZObjectContainer(title2=title, no_cache=True, no_history=True)
 
+    running = scheduler.is_task_running("MissingSubtitles")
+    task_data = scheduler.get_task_data("MissingSubtitles")
+    missing_items = task_data["missing_subtitles"] if task_data else None
 
-def recentItemsMenu(title, base_title=None):
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
-    recent_items = get_recent_items()
-    if recent_items:
-        missing_items = items_get_all_missing_subs(recent_items)
-        if missing_items:
-            for added_at, item_id, title, item in missing_items:
-                oc.add(DirectoryObject(
-                    key=Callback(ItemDetailsMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id),
-                    title=title,
-                    thumb=get_item_thumb(item) or default_thumb
-                ))
+    if ((missing_items is None) or force) and not running:
+        scheduler.dispatch_task("MissingSubtitles")
+        running = True
+
+    if not running:
+        oc.add(DirectoryObject(
+            key=Callback(RecentMissingSubtitlesMenu, force=True, randomize=timestamp()),
+            title=u"Get items with missing subtitles",
+            thumb=default_thumb
+        ))
+    else:
+        oc.add(DirectoryObject(
+            key=Callback(RecentMissingSubtitlesMenu, force=False, randomize=timestamp()),
+            title=u"Updating, refresh here ...",
+            thumb=default_thumb
+        ))
+
+    if missing_items is not None:
+        for added_at, item_id, item_title, item, missing_languages in missing_items:
+            oc.add(DirectoryObject(
+                key=Callback(ItemDetailsMenu, title=title + " > " + item_title, item_title=item_title, rating_key=item_id),
+                title=item_title,
+                summary="Missing: %s" % ", ".join(l.name for l in missing_languages),
+                thumb=get_item_thumb(item) or default_thumb
+            ))
+
+        scheduler.clear_task_data("MissingSubtitles")
 
     return oc
 
@@ -165,7 +203,7 @@ def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *
     :param kwargs:
     :return:
     """
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SZObjectContainer(title2=title, no_cache=True, no_history=True)
     items = itemGetter(*args, **kwargs)
 
     for kind, title, item_id, deeper, item in items:
@@ -185,7 +223,7 @@ def determine_section_display(kind, item):
     :param item:
     :return:
     """
-    if item.size > 200:
+    if item.size > 80:
         return SectionFirstLetterMenu
     return SectionMenu
 
@@ -203,7 +241,7 @@ def IgnoreMenu(kind, rating_key, title=None, sure=False, todo="not_set"):
     """
     is_ignored = rating_key in ignore_list[kind]
     if not sure:
-        oc = ObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
+        oc = SZObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
             "Add" if not is_ignored else "Remove", ignore_list.verbose(kind), title, "to" if not is_ignored else "from"), title2="Are you sure?")
         oc.add(DirectoryObject(
             key=Callback(IgnoreMenu, kind=kind, rating_key=rating_key, title=title, sure=True, todo="add" if not is_ignored else "remove"),
@@ -248,7 +286,7 @@ def SectionsMenu():
     """
     items = get_all_items("sections")
 
-    return dig_tree(ObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
+    return dig_tree(SZObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
                     menu_determination_callback=determine_section_display, pass_kwargs={"base_title": "Sections"},
                     fill_args={"title": "section_title"})
 
@@ -271,7 +309,7 @@ def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ign
 
     section_title = title
     title = base_title + " > " + title
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SZObjectContainer(title2=title, no_cache=True, no_history=True)
     if ignore_options:
         add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
@@ -295,7 +333,7 @@ def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_titl
     kind, deeper = get_items_info(items)
 
     title = unicode(title)
-    oc = ObjectContainer(title2=section_title, no_cache=True, no_history=True)
+    oc = SZObjectContainer(title2=section_title, no_cache=True, no_history=True)
     title = base_title + " > " + title
     add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
@@ -320,7 +358,7 @@ def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, displa
     :return:
     """
     title = base_title + " > " + unicode(title)
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SZObjectContainer(title2=title, no_cache=True, no_history=True)
 
     items = get_all_items(key="first_character", value=[rating_key, key], base="library/sections", flat=False)
     kind, deeper = get_items_info(items)
@@ -330,7 +368,8 @@ def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, displa
 
 
 @route(PREFIX + '/section/contents', display_items=bool)
-def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None, previous_rating_key=None):
+def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None,
+                 previous_rating_key=None):
     """
     displays the contents of a section based on whether it has a deeper tree or not (movies->movie (item) list; series->series list)
     :param rating_key:
@@ -344,7 +383,9 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
     title = unicode(title)
     item_title = title
     title = base_title + " > " + title
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SZObjectContainer(title2=title, no_cache=True, no_history=True)
+
+    current_kind = get_item_kind_from_rating_key(rating_key)
 
     if display_items:
         items = get_all_items(key="children", value=rating_key, base="library/metadata")
@@ -355,17 +396,23 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
         if should_display_ignore(items, previous=previous_item_type):
             add_ignore_options(oc, "series", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
+        timeout = 30
+        if current_kind == "season":
+            timeout = 90
+        elif current_kind == "series":
+            timeout = 360
+
         # add refresh
         oc.add(DirectoryObject(
-            key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, refresh_kind=kind, previous_rating_key=previous_rating_key,
-                         timeout=16000, randomize=timestamp()),
+            key=Callback(RefreshItem, rating_key=rating_key, item_title=title, refresh_kind=current_kind,
+                         previous_rating_key=previous_rating_key, timeout=timeout*1000, randomize=timestamp()),
             title=u"Refresh: %s" % item_title,
             summary="Refreshes the item, possibly picking up new subtitles on disk"
         ))
         oc.add(DirectoryObject(
-            key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, refresh_kind=kind,
-                         previous_rating_key=previous_rating_key, timeout=16000),
-            title=u"Force-Refresh: %s" % item_title,
+            key=Callback(RefreshItem, rating_key=rating_key, item_title=title, force=True,
+                         refresh_kind=current_kind, previous_rating_key=previous_rating_key, timeout=timeout*1000),
+            title=u"Auto-Find subtitles: %s" % item_title,
             summary="Issues a forced refresh, ignoring known subtitles and searching for new ones"
         ))
     else:
@@ -376,7 +423,7 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
 
 @route(PREFIX + '/ignore_list')
 def IgnoreListMenu():
-    oc = ObjectContainer(title2="Ignore list", replace_parent=True)
+    oc = SZObjectContainer(title2="Ignore list", replace_parent=True)
     for key in ignore_list.key_order:
         values = ignore_list[key]
         for value in values:
@@ -384,7 +431,26 @@ def IgnoreListMenu():
     return oc
 
 
+@route(PREFIX + '/history')
+def HistoryMenu():
+    from support.history import get_history
+    history = get_history()
+    oc = SZObjectContainer(title2="History", replace_parent=True)
+
+    for item in history.history_items:
+        oc.add(DirectoryObject(
+            key=Callback(ItemDetailsMenu, title=item.title, item_title=item.item_title,
+                         rating_key=item.rating_key),
+            title=u"%s (%s)" % (item.item_title, item.mode_verbose),
+            summary=u"%s in %s (%s, score: %s), %s" % (item.lang_name, item.section_title,
+                                                       item.provider_name, item.score, df(item.time))
+        ))
+
+    return oc
+
+
 @route(PREFIX + '/item/{rating_key}/actions')
+@debounce
 def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None):
     """
     displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
@@ -398,56 +464,218 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
     title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title)
     item = get_item(rating_key)
 
-    oc = ObjectContainer(title2=title, replace_parent=True)
+    timeout = 30
+
+    oc = SZObjectContainer(title2=title, replace_parent=True)
     oc.add(DirectoryObject(
-        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp()),
+        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp(),
+                     timeout=timeout*1000),
         title=u"Refresh: %s" % item_title,
         summary="Refreshes the item, possibly picking up new subtitles on disk",
         thumb=item.thumb or default_thumb
     ))
     oc.add(DirectoryObject(
-        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp()),
-        title=u"Force-Refresh: %s" % item_title,
+        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(),
+                     timeout=timeout*1000),
+        title=u"Auto-search: %s" % item_title,
         summary="Issues a forced refresh, ignoring known subtitles and searching for new ones",
         thumb=item.thumb or default_thumb
     ))
+
+    # get stored subtitle info for item id
+    current_subtitle_info = get_subtitle_info(rating_key)
+
+    # get the plex item
+    plex_item = list(Plex["library"].metadata(rating_key))[0]
+
+    # get current media info for that item
+    media = plex_item.media
+
+    # look for subtitles for all available media parts and all of their languages
+    for part in media.parts:
+        filename = os.path.basename(part.file)
+        part_id = str(part.id)
+
+        # get corresponding stored subtitle data for that media part (physical media item)
+        sub_part_data = current_subtitle_info.get(part_id, {}) if current_subtitle_info else {}
+
+        # iterate through all configured languages
+        for lang in config.lang_list:
+            lang_a2 = lang.alpha2
+            # ietf lang?
+            if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
+                lang_a2 = lang_a2.split("-")[0]
+
+            sub_data_for_lang = sub_part_data.get(lang_a2, {})
+
+            # try getting current subtitle information for that language
+            current_subtitle_key = sub_data_for_lang.get("current", (None, None))
+            current_sub_provider_name, current_sub_id = current_subtitle_key
+
+            summary = u"No current subtitle in storage"
+            current_score = None
+            if current_sub_provider_name:
+                current_subtitle = sub_part_data[lang_a2][current_subtitle_key]
+                current_score = current_subtitle["score"]
+
+                summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
+                          (current_sub_provider_name,
+                           df(current_subtitle["date_added"]), mode_map.get(current_subtitle.get("mode", "a")), lang,
+                           current_subtitle["score"], current_subtitle["storage"])
+
+            oc.add(DirectoryObject(
+                key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title,
+                             item_title=item_title, language=lang, current_id=current_sub_id,
+                             item_type=plex_item.type, filename=filename, current_data=summary,
+                             randomize=timestamp(), current_provider=current_sub_provider_name,
+                             current_score=current_score),
+                title=u"List %s subtitles" % lang.name,
+                summary=summary
+            ))
+
     add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
 
     return oc
 
 
+def get_item_task_data(task_name, rating_key, language):
+    task_data = scheduler.get_task_data(task_name)
+    search_results = task_data.get(rating_key, {}) if task_data else {}
+    return search_results.get(language)
+
+
+@route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool)
+@debounce
+def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None,
+                                 item_type="episode", language=None, force=False, current_id=None, current_data=None,
+                                 current_provider=None, current_score=None, randomize=None):
+    assert rating_key, part_id
+
+    running = scheduler.is_task_running("AvailableSubsForItem")
+    search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
+
+    if (search_results is None or force) and not running:
+        scheduler.dispatch_task("AvailableSubsForItem", rating_key=rating_key, item_type=item_type, part_id=part_id,
+                                language=language)
+        running = True
+
+    oc = SZObjectContainer(title2=unicode(title), replace_parent=True)
+    oc.add(DirectoryObject(
+        key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()),
+        title=u"Back to: %s" % title,
+        summary=current_data,
+        thumb=default_thumb
+    ))
+
+    metadata = get_plex_metadata(rating_key, part_id, item_type)
+    scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+
+    if not scanned_parts:
+        Log.Error("Couldn't list available subtitles for %s", rating_key)
+        return oc
+
+    video, plex_part = scanned_parts.items()[0]
+
+    video_display_data = [video.format] if video.format else []
+    if video.release_group:
+        video_display_data.append(u"by %s" % video.release_group)
+    video_display_data = " ".join(video_display_data)
+
+    current_display = (u"Current: %s (%s) " % (current_provider, current_score) if current_provider else "")
+    if not running:
+        oc.add(DirectoryObject(
+            key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, language=language,
+                         filename=filename, part_id=part_id, title=title, current_id=current_id, force=True,
+                         current_provider=current_provider, current_score=current_score,
+                         current_data=current_data, item_type=item_type, randomize=timestamp()),
+            title=u"Search for %s subs (%s)" % (get_language(language).name, video_display_data),
+            summary=u"%sFilename: %s" % (current_display, filename),
+            thumb=default_thumb
+        ))
+    else:
+        oc.add(DirectoryObject(
+            key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
+                         language=language, filename=filename, current_data=current_data,
+                         part_id=part_id, title=title, current_id=current_id, item_type=item_type,
+                         current_provider=current_provider, current_score=current_score,
+                         randomize=timestamp()),
+            title=u"Searching for %s subs (%s), refresh here ..." % (get_language(language).name, video_display_data),
+            summary=u"%sFilename: %s" % (current_display, filename),
+            thumb=default_thumb
+        ))
+
+    if not search_results:
+        return oc
+
+    for subtitle in search_results:
+        oc.add(DirectoryObject(
+            key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title,
+                         subtitle_id=str(subtitle.id), language=language),
+            title=u"%s: %s, score: %s" % ("Available" if current_id != subtitle.id else "Current",
+                                    subtitle.provider_name, subtitle.score),
+            summary=u"Release: %s, Matches: %s" % (subtitle.release_info, ", ".join(subtitle.matches)),
+            thumb=default_thumb
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/download_subtitle/{rating_key}')
+@debounce
+def TriggerDownloadSubtitle(rating_key=None, subtitle_id=None, item_title=None, language=None, randomize=None):
+    set_refresh_menu_state("Downloading subtitle for %s" % item_title or rating_key)
+    search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
+
+    download_subtitle = None
+    for subtitle in search_results:
+        if str(subtitle.id) == subtitle_id:
+            download_subtitle = subtitle
+            break
+    if not download_subtitle:
+        Log.Error(u"Something went horribly wrong")
+
+    else:
+        scheduler.dispatch_task("DownloadSubtitleForItem", rating_key=rating_key, subtitle=download_subtitle)
+
+    return fatality(randomize=timestamp(), header=" ", replace_parent=True)
+
+
 @route(PREFIX + '/item/{rating_key}')
 @debounce
-def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None, previous_rating_key=None, timeout=8000, randomize=None, trigger=True):
+def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None,
+                previous_rating_key=None, timeout=8000, randomize=None, trigger=True):
     assert rating_key
     header = " "
     if trigger:
         set_refresh_menu_state(u"Triggering %sRefresh for %s" % ("Force-" if force else "", item_title))
-        Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind, parent_rating_key=previous_rating_key,
-                      timeout=int(timeout))
+        Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind,
+                      parent_rating_key=previous_rating_key, timeout=int(timeout))
+
         header = u"%s of item %s triggered" % ("Refresh" if not force else "Forced-refresh", rating_key)
     return fatality(randomize=timestamp(), header=header, replace_parent=True)
 
 
 @route(PREFIX + '/missing/refresh')
 @debounce
-def RefreshMissing(randomize=None, trigger=True):
-    header = " "
-    if trigger:
-        Thread.CreateTimer(1.0, lambda: scheduler.run_task("searchAllRecentlyAddedMissing"))
-        header = "Refresh of recently added items with missing subtitles triggered"
+def RefreshMissing(randomize=None):
+    Thread.CreateTimer(1.0, lambda: scheduler.run_task("SearchAllRecentlyAddedMissing"))
+    header = "Refresh of recently added items with missing subtitles triggered"
     return fatality(header=header, replace_parent=True)
 
 
 @route(PREFIX + '/advanced')
 def AdvancedMenu(randomize=None, header=None, message=None):
-    oc = ObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True, no_history=True,
-                         replace_parent=True, title2="Advanced")
+    oc = SZObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True, no_history=True,
+                         replace_parent=False, title2="Advanced")
 
     oc.add(DirectoryObject(
         key=Callback(TriggerRestart, randomize=timestamp()),
         title=pad_title("Restart the plugin"),
     ))
+    oc.add(DirectoryObject(
+        key=Callback(TriggerBetterSubtitles, randomize=timestamp()),
+        title=pad_title("Trigger find better subtitles"),
+    ))
     oc.add(DirectoryObject(
         key=Callback(LogStorage, key="tasks", randomize=timestamp()),
         title=pad_title("Log the plugin's scheduled tasks state storage"),
@@ -460,6 +688,10 @@ def AdvancedMenu(randomize=None, header=None, message=None):
         key=Callback(LogStorage, key="ignore", randomize=timestamp()),
         title=pad_title("Log the plugin's internal ignorelist storage"),
     ))
+    oc.add(DirectoryObject(
+        key=Callback(LogStorage, key="history", randomize=timestamp()),
+        title=pad_title("Log the plugin's internal history storage"),
+    ))
     oc.add(DirectoryObject(
         key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
         title=pad_title("Reset the plugin's scheduled tasks state storage"),
@@ -472,6 +704,10 @@ def AdvancedMenu(randomize=None, header=None, message=None):
         key=Callback(ResetStorage, key="ignore", randomize=timestamp()),
         title=pad_title("Reset the plugin's internal ignorelist storage"),
     ))
+    oc.add(DirectoryObject(
+        key=Callback(ResetStorage, key="history", randomize=timestamp()),
+        title=pad_title("Reset the plugin's internal history storage"),
+    ))
     return oc
 
 
@@ -522,10 +758,9 @@ def DispatchRestart():
 
 @route(PREFIX + '/advanced/restart/trigger')
 @debounce
-def TriggerRestart(randomize=None, trigger=True):
-    if trigger:
-        set_refresh_menu_state("Restarting the plugin")
-        DispatchRestart()
+def TriggerRestart(randomize=None):
+    set_refresh_menu_state("Restarting the plugin")
+    DispatchRestart()
     return fatality(header="Restart triggered, please wait about 5 seconds", force_title=" ", only_refresh=True, replace_parent=True,
                     no_history=True, randomize=timestamp())
 
@@ -538,7 +773,7 @@ def Restart():
 @route(PREFIX + '/storage/reset', sure=bool)
 def ResetStorage(key, randomize=None, sure=False):
     if not sure:
-        oc = ObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
+        oc = SZObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
         oc.add(DirectoryObject(
             key=Callback(ResetStorage, key=key, sure=True, randomize=timestamp()),
             title=pad_title("Are you really sure?"),
@@ -568,3 +803,13 @@ def LogStorage(key, randomize=None):
         header='Success',
         message='Information Storage (%s) logged' % key
     )
+
+
+@route(PREFIX + '/triggerbetter')
+def TriggerBetterSubtitles(randomize=None):
+    scheduler.dispatch_task("FindBetterSubtitles")
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='FindBetterSubtitles triggered'
+    )

Contents/Code/interface/menu_helpers.py

@@ -1,12 +1,12 @@
 # coding=utf-8
 import types
+import datetime
 
 from support.items import get_kind, get_item_thumb
-from subzero import intent
-from support.helpers import format_video
+from support.helpers import get_video_display_title
 from support.ignore import ignore_list
+from support.lib import get_intent
 from subzero.constants import ICON
-from subzero.func import debouncer
 
 default_thumb = R(ICON)
 
@@ -58,8 +58,8 @@ def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_r
             add_kwargs.update(pass_kwargs)
 
         oc.add(DirectoryObject(
-            key=Callback(menu_callback or menu_determination_callback(kind, item), title=title, rating_key=force_rating_key or key,
-                         **add_kwargs),
+            key=Callback(menu_callback or menu_determination_callback(kind, item), title=title,
+                         rating_key=force_rating_key or key, **add_kwargs),
             title=title, thumb=thumb
         ))
     return oc
@@ -90,9 +90,11 @@ def set_refresh_menu_state(state_or_media, media_type="movies"):
             for episode in media.seasons[season].episodes:
                 ep = media.seasons[season].episodes[episode]
                 media_id = ep.id
-                title = format_video("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
+                title = get_video_display_title("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
     else:
-        title = format_video("movie", media.title)
+        title = get_video_display_title("movie", media.title)
+
+    intent = get_intent()
     force_refresh = intent.get("force", media_id)
 
     Dict["current_refresh_state"] = u"%sRefreshing %s" % ("Force-" if force_refresh else "", unicode(title))
@@ -128,13 +130,37 @@ def debounce(func):
     :param func:
     :return:
     """
+    def get_lookup_key(args, kwargs):
+        func_name = list(args).pop(0).__name__
+        return tuple([func_name] + [(key, value) for key, value in kwargs.iteritems()])
+
     def wrap(*args, **kwargs):
         if "randomize" in kwargs:
-            if ([func] + list(args), kwargs) in debouncer:
-                kwargs["trigger"] = False
+            if not "menu_history" in Dict:
+                Dict["menu_history"] = {}
+
+            key = get_lookup_key([func] + list(args), kwargs)
+            if key in Dict["menu_history"]:
                 Log.Debug("not triggering %s twice with %s, %s" % (func, args, kwargs))
+                return ObjectContainer()
             else:
-                debouncer.add([func] + list(args), kwargs)
+                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(days=1)
+                Dict.Save()
         return func(*args, **kwargs)
 
     return wrap
+
+
+class SZObjectContainer(ObjectContainer):
+    def __init__(self, *args, **kwargs):
+        super(SZObjectContainer, self).__init__(*args, **kwargs)
+        from interface.menu import fatality
+        from support.helpers import pad_title, timestamp
+        self.add(DirectoryObject(
+            key=Callback(fatality, force_title=" ", randomize=timestamp()),
+            title=pad_title("<< Back to home"),
+            summary="Current state: %s; Last state: %s" % (
+                (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
+                (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
+            )
+        ))

Contents/Code/support/__init__.py

@@ -47,3 +47,7 @@ sys.modules["support.storage"] = storage
 import ignore
 
 sys.modules["support.ignore"] = ignore
+
+import history
+
+sys.modules["support.history"] = history

Contents/Code/support/background.py

@@ -6,7 +6,7 @@ import traceback
 
 
 def parse_frequency(s):
-    if s == "never":
+    if s == "never" or s == None:
         return None, None
     kind, num, unit = s.split()
     return int(num), unit
@@ -30,6 +30,21 @@ class DefaultScheduler(object):
             Dict["tasks"] = {}
             Dict.Save()
 
+    def get_task_data(self, name):
+        if name not in Dict["tasks"]:
+            raise NotImplementedError("Task missing! %s" % name)
+
+        if "data" in Dict["tasks"][name]:
+            return Dict["tasks"][name]["data"]
+
+    def clear_task_data(self, name):
+        if name not in Dict["tasks"]:
+            raise NotImplementedError("Task missing! %s" % name)
+
+        Dict["tasks"][name]["data"] = {}
+        Dict.Save()
+        Log.Debug("Task data cleared: %s", name)
+
     def register(self, task):
         self.registry.append(task)
 
@@ -38,7 +53,12 @@ class DefaultScheduler(object):
         self.tasks = {}
         for cls in self.registry:
             task = cls(self)
-            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task.name])}
+            try:
+                task_frequency = Prefs["scheduler.tasks.%s.frequency" % task.name]
+            except KeyError:
+                task_frequency = None
+
+            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)}
 
     def run(self):
         self.running = True
@@ -52,13 +72,18 @@ class DefaultScheduler(object):
             return None
         return self.tasks[name]["task"]
 
+    def is_task_running(self, name):
+        task = self.task(name)
+        if task:
+            return task.running
+
     def last_run(self, task):
         if task not in self.tasks:
             return None
         return self.tasks[task]["task"].last_run
 
     def next_run(self, task):
-        if task not in self.tasks:
+        if task not in self.tasks or not self.tasks[task]["task"].periodic:
             return None
         frequency_num, frequency_key = self.tasks[task]["frequency"]
         if not frequency_num:
@@ -70,24 +95,34 @@ class DefaultScheduler(object):
             use_date = now
         return max(use_date + datetime.timedelta(**{frequency_key: frequency_num}), now)
 
-    def run_task(self, name):
+    def run_task(self, name, *args, **kwargs):
         task = self.tasks[name]["task"]
         if task.running:
             Log.Debug("Scheduler: Not running %s, as it's currently running.", name)
-            return
+            return False
 
         Log.Debug("Scheduler: Running task %s", name)
         try:
-            task.prepare()
+            task.prepare(*args, **kwargs)
+            task.time_start = datetime.datetime.now()
             task.run()
         except Exception, e:
             Log.Error("Scheduler: Something went wrong when running %s: %s", name, traceback.format_exc())
         finally:
-            task.post_run()
+            task.last_run = datetime.datetime.now()
+            task.time_start = None
+            task.post_run(Dict["tasks"][name]["data"])
+
+    def dispatch_task(self, *args, **kwargs):
+        Thread.Create(self.run_task, True, *args, **kwargs)
+        Log.Debug("Dispatching single task: %s, %s", args, kwargs)
 
     def signal(self, name, *args, **kwargs):
         for task_name, info in self.tasks.iteritems():
             task = info["task"]
+            if not task.periodic:
+                continue
+
             if task.running:
                 Log.Debug("Scheduler: Sending signal %s to task %s (%s, %s)", name, task_name, args, kwargs)
                 status = task.signal(name, *args, **kwargs)
@@ -108,7 +143,7 @@ class DefaultScheduler(object):
                 now = datetime.datetime.now()
                 task = info["task"]
 
-                if name not in Dict["tasks"]:
+                if name not in Dict["tasks"] or not task.periodic:
                     continue
 
                 if task.running:

Contents/Code/support/config.py

@@ -3,11 +3,14 @@
 import os
 import re
 import inspect
+
+import subliminal
+import subliminal_patch
 from babelfish import Language
 from subzero.lib.io import FileIO, get_viable_encoding
 from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW
 from lib import Plex
-from helpers import check_write_permissions
+from helpers import check_write_permissions, cast_bool
 
 SUBTITLE_EXTS = ['utf', 'utf8', 'utf-8', 'srt', 'smi', 'rt', 'ssa', 'aqt', 'jss', 'ass', 'idx', 'sub', 'txt', 'psb']
 VIDEO_EXTS = ['3g2', '3gp', 'asf', 'asx', 'avc', 'avi', 'avs', 'bivx', 'bup', 'divx', 'dv', 'dvr-ms', 'evo', 'fli', 'flv',
@@ -53,22 +56,30 @@ class Config(object):
         self.subtitle_destination_folder = self.get_subtitle_destination_folder()
         self.providers = self.get_providers()
         self.provider_settings = self.get_provider_settings()
-        self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 200)
+        self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 2000)
         self.sections = list(Plex["library"].sections())
         self.missing_permissions = []
         self.ignore_paths = self.parse_ignore_paths()
+        self.enabled_sections = self.check_enabled_sections()
         self.permissions_ok = self.check_permissions()
         self.notify_executable = self.check_notify_executable()
-        self.enabled_sections = self.check_enabled_sections()
+        self.chmod = self.check_chmod()
         self.initialized = True
 
+    def refresh_permissions_status(self):
+        self.permissions_ok = self.check_permissions()
+
     def check_permissions(self):
         if not Prefs["subtitles.save.filesystem"] or not Prefs["check_permissions"]:
             return True
 
+        self.missing_permissions = []
         use_ignore_fs = Prefs["subtitles.ignore_fs"]
         all_permissions_ok = True
         for section in self.sections:
+            if section.key not in self.enabled_sections:
+                continue
+
             title = section.title
             for location in section:
                 path_str = location.path
@@ -137,6 +148,9 @@ class Config(object):
             return exe_fn, arguments
         Log.Error("Notify executable not existing or not executable: %s" % exe_fn)
 
+    def refresh_enabled_sections(self):
+        self.enabled_sections = self.check_enabled_sections()
+
     def check_enabled_sections(self):
         enabled_for_primary_agents = []
         enabled_sections = {}
@@ -193,7 +207,7 @@ class Config(object):
         if not Prefs["subtitles.save.filesystem"]:
             return
 
-        fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
+        fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if cast_bool(Prefs["subtitles.save.subFolder.Custom"]) else None
         return fld_custom or (Prefs["subtitles.save.subFolder"] if Prefs["subtitles.save.subFolder"] != "current folder" else None)
 
     def get_providers(self):
@@ -218,5 +232,30 @@ class Config(object):
 
         return provider_settings
 
+    def check_chmod(self):
+        val = Prefs["subtitles.save.chmod"]
+        if not val or not len(val):
+            return
+
+        wrong_chmod = False
+        if len(val) != 4:
+            wrong_chmod = True
+
+        try:
+            return int(val, 8)
+        except ValueError:
+            wrong_chmod = True
+
+        if wrong_chmod:
+            Log.Warning("Chmod setting ignored, please use only 4-digit integers with leading 0 (e.g.: 775)")
+
+    def init_subliminal_patches(self):
+        # configure custom subtitle destination folders for scanning pre-existing subs
+        Log.Debug("Patching subliminal ...")
+        dest_folder = self.subtitle_destination_folder
+        subliminal_patch.patch_video.CUSTOM_PATHS = [dest_folder] if dest_folder else []
+        subliminal_patch.patch_provider_pool.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
+        subliminal.video.Episode.scores["addic7ed_boost"] = int(Prefs['provider.addic7ed.boost_by'])
+
 
 config = Config()

Contents/Code/support/helpers.py

@@ -9,6 +9,10 @@ import re
 import platform
 import subprocess
 
+from babelfish import Language
+
+from subzero.analytics import track_event
+
 # Unicode control characters can appear in ID3v2 tags but are not legal in XML.
 RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
                      u'|' + \
@@ -20,6 +24,10 @@ RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])'
                      )
 
 
+def cast_bool(value):
+    return str(value) in ("true", "True")
+
+
 # A platform independent way to split paths which might come in with different separators.
 def split_path(str):
     if str.find('\\') != -1:
@@ -89,7 +97,8 @@ def pad_title(value):
     return str_pad(value, 30, pad_char=' ')
 
 
-def format_item(item, kind, parent=None, parent_title=None, section_title=None, add_section_title=False):
+def get_plex_item_display_title(item, kind, parent=None, parent_title=None, section_title=None,
+                                add_section_title=False):
     """
     :param item: plex item
     :param kind: show or movie
@@ -97,28 +106,64 @@ def format_item(item, kind, parent=None, parent_title=None, section_title=None,
     :param parent_title: parentTitle or None
     :return:
     """
-    return format_video(kind, item.title,
-                        section_title=(
-                            section_title or (parent.section.title if parent and getattr(parent, "section") else None)),
-                        parent_title=(parent_title or (parent.show.title if parent else None)),
-                        season=parent.index if parent else None,
-                        episode=item.index if kind == "show" else None,
-                        add_section_title=add_section_title)
+    return get_video_display_title(kind, item.title,
+                                   section_title=(
+                                       section_title or (parent.section.title if parent and getattr(parent, "section")
+                                                         else None)),
+                                   parent_title=(parent_title or (parent.show.title if parent else None)),
+                                   season=parent.index if parent else None,
+                                   episode=item.index if kind == "show" else None,
+                                   add_section_title=add_section_title)
 
 
-def format_video(kind, title, section_title=None, parent_title=None, season=None, episode=None,
-                 add_section_title=False):
+def get_video_display_title(kind, title, section_title=None, parent_title=None, season=None, episode=None,
+                            add_section_title=False):
     section_add = ""
     if add_section_title:
         section_add = ("%s: " % section_title) if section_title else ""
 
     if kind == "show" and parent_title:
         if season and episode:
-            return '%s%s S%02dE%02d, %s' % (section_add, parent_title, season or 0, episode or 0, title)
-        return '%s%s, %s' % (section_add, parent_title, title)
+            return '%s%s S%02dE%02d%s' % (section_add, parent_title, season or 0, episode or 0,
+                                          (", %s" % title if title else ""))
+        return '%s%s%s' % (section_add, parent_title, (", %s" % title if title else ""))
     return "%s%s" % (section_add, title)
 
 
+def get_title_for_video_metadata(metadata, add_section_title=True, add_episode_title=False):
+    """
+
+    :param metadata:
+    :param add_section_title:
+    :param add_episode_title: add the episode's title if its an episode else always add title
+    :return:
+    """
+    # compute item title
+    add_title = (add_episode_title and metadata["series_id"]) or not metadata["series_id"]
+    return get_video_display_title(
+        "show" if metadata["series_id"] else "movie",
+        metadata["title"] if add_title else "",
+        parent_title=metadata.get("series", None),
+        season=metadata.get("season", None),
+        episode=metadata.get("episode", None),
+        section_title=metadata.get("section", None),
+        add_section_title=add_section_title
+    )
+
+
+def get_identifier():
+    identifier = None
+    try:
+        identifier = Platform.MachineIdentifier
+    except:
+        pass
+
+    if not identifier:
+        identifier = String.UUID()
+
+    return Hash.SHA1(identifier + "SUBZEROOOOOOOOOO")
+
+
 def encode_message(base, s):
     return "%s?message=%s" % (base, urllib.quote_plus(s))
 
@@ -131,6 +176,10 @@ def timestamp():
     return int(time.time())
 
 
+def df(d):
+    return d.strftime("%Y-%m-%d %H:%M:%S") if d else "legacy data"
+
+
 def query_plex(url, args):
     """
     simple http query to the plex API without parsing anything too complicated
@@ -202,3 +251,25 @@ def notify_executable(exe_info, videos, subtitles, storage):
             else:
                 Log.Debug(u"Process output: %s" % output)
 
+
+def track_usage(category=None, action=None, label=None, value=None):
+    if not cast_bool(Prefs["track_usage"]):
+        return
+
+    Thread.Create(dispatch_track_usage, category, action, label, value,
+                  identifier=Dict["anon_id"], first_use=Dict["first_use"],
+                  add=Network.PublicAddress)
+
+
+def dispatch_track_usage(*args, **kwargs):
+    identifier = kwargs.pop("identifier")
+    first_use = kwargs.pop("first_use")
+    add = kwargs.pop("add")
+    try:
+        track_event(identifier=identifier, first_use=first_use, add=add, *[str(a) for a in args])
+    except:
+        Log.Debug("Something went wrong when reporting anonymous user statistics: %s", traceback.format_exc())
+
+
+def get_language(lang_short):
+    return Language.fromietf(lang_short)

Contents/Code/support/history.py

@@ -0,0 +1,4 @@
+# coding=utf-8
+from subzero.history_storage import SubtitleHistory
+
+get_history = lambda: SubtitleHistory(Dict, int(Prefs["history_size"]))

Contents/Code/support/items.py

@@ -5,9 +5,8 @@ import re
 import types
 import os
 from ignore import ignore_list
-from helpers import is_recent, format_item, query_plex
-from subzero import intent
-from lib import Plex
+from helpers import is_recent, get_plex_item_display_title, query_plex
+from lib import Plex, get_intent
 from config import config, IGNORE_FN
 
 logger = logging.getLogger(__name__)
@@ -29,6 +28,19 @@ def get_item_kind(item):
     return type(item).__name__
 
 
+PLEX_API_TYPE_MAP = {
+    "Show": "series",
+    "Season": "season",
+    "Episode": "episode",
+    "Movie": "movie",
+}
+
+
+def get_item_kind_from_rating_key(key):
+    item = get_item(key)
+    return PLEX_API_TYPE_MAP[get_item_kind(item)]
+
+
 def get_item_thumb(item):
     kind = get_item_kind(item)
     if kind == "Episode":
@@ -104,7 +116,7 @@ def get_items(key="recently_added", base="library", value=None, flat=False, add_
             if flat:
                 # return episodes
                 for child in item.children():
-                    items.append(("episode", format_item(child, "show", parent=item, add_section_title=add_section_title), int(item.rating_key),
+                    items.append(("episode", get_plex_item_display_title(child, "show", parent=item, add_section_title=add_section_title), int(item.rating_key),
                                   False, child))
             else:
                 # return seasons
@@ -120,16 +132,16 @@ def get_items(key="recently_added", base="library", value=None, flat=False, add_
 
         elif kind == "episode":
             items.append(
-                (kind, format_item(item, "show", parent=item.season, parent_title=item.show.title, section_title=item.section.title,
-                                   add_section_title=add_section_title), int(item.rating_key), False, item))
+                (kind, get_plex_item_display_title(item, "show", parent=item.season, parent_title=item.show.title, section_title=item.section.title,
+                                                   add_section_title=add_section_title), int(item.rating_key), False, item))
 
         elif kind in ("movie", "artist", "photo"):
-            items.append((kind, format_item(item, kind, section_title=item.section.title, add_section_title=add_section_title),
+            items.append((kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title),
                           int(item.rating_key), False, item))
 
         elif kind == "show":
             items.append((
-                kind, format_item(item, kind, section_title=item.section.title, add_section_title=add_section_title), int(item.rating_key), True,
+                kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title), int(item.rating_key), True,
                 item))
 
     return items
@@ -247,13 +259,22 @@ def is_ignored(rating_key, item=None):
 
 
 def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, parent_rating_key=None):
+    intent = get_intent()
+
     # timeout actually is the time for which the intent will be valid
     if force:
+        Log.Debug("Setting intent for force-refresh of %s to timeout: %s", rating_key, timeout)
         intent.set("force", rating_key, timeout=timeout)
 
-    if refresh_kind == "episode":
-        # season refresh
-        rating_key = parent_rating_key
+        # force Dict.Save()
+        intent.store.save()
 
-    Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", rating_key)
-    Plex["library/metadata"].refresh(rating_key)
+    refresh = [rating_key]
+
+    if refresh_kind == "season":
+        # season refresh, needs explicit per-episode refresh
+        refresh = [item.rating_key for item in list(Plex["library/metadata"].children(int(rating_key)))]
+
+    for key in refresh:
+        Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", key)
+        Plex["library/metadata"].refresh(key)

Contents/Code/support/lib.py

@@ -1,6 +1,8 @@
 # coding=utf-8
 
 import plex
+from subzero.intent import TempIntent
+from subzero.lib.dict import DictProxy
 from subzero.lib.httpfake import PlexPyNativeResponseProxy
 
 
@@ -35,3 +37,18 @@ class PlexPyNativeRequestProxy(object):
 plex.request.Request = PlexPyNativeRequestProxy
 
 Plex = plex.Plex
+
+
+class IntentDictStorage(DictProxy):
+    store = "intent"
+
+    def setup_defaults(self):
+        return {"force": {}}
+
+
+def get_intent():
+    """
+    use this to get an intent from inside a separate thread
+    :return:
+    """
+    return TempIntent(store=IntentDictStorage(Dict))

Contents/Code/support/localmedia.py

@@ -12,7 +12,7 @@ def find_subtitles(part):
     lang_sub_map = {}
     part_filename = helpers.unicodize(part.file)
     part_basename = os.path.splitext(os.path.basename(part_filename))[0]
-    use_filesystem = bool(Prefs["subtitles.save.filesystem"])
+    use_filesystem = helpers.cast_bool(Prefs["subtitles.save.filesystem"])
     paths = [os.path.dirname(part_filename)] if use_filesystem else []
 
     global_subtitle_folder = None
@@ -27,7 +27,7 @@ def find_subtitles(part):
             # got selected subfolder
             sub_dir_list.append(os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"]))
 
-        sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
+        sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if helpers.cast_bool(Prefs["subtitles.save.subFolder.Custom"]) else None
         if sub_dir_custom:
             # got custom subfolder
             if sub_dir_custom.startswith("/"):

Contents/Code/support/missing_subtitles.py

@@ -2,7 +2,7 @@
 import traceback
 
 from support.config import config
-from support.helpers import format_item
+from support.helpers import get_plex_item_display_title, cast_bool
 from support.items import get_item
 from support.lib import Plex
 
@@ -14,9 +14,9 @@ def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_t
     item = get_item(rating_key)
 
     if kind == "show":
-        item_title = format_item(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
+        item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
     else:
-        item_title = format_item(item, kind, section_title=section_title)
+        item_title = get_plex_item_display_title(item, kind, section_title=section_title)
 
     video = item.media
 
@@ -44,7 +44,7 @@ def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_t
         Log.Info(u"Subs still missing for '%s': %s", item_title, missing)
 
     if missing:
-        return added_at, item_id, item_title, item
+        return added_at, item_id, item_title, item, missing
 
 
 def items_get_all_missing_subs(items):
@@ -57,21 +57,21 @@ def items_get_all_missing_subs(items):
                 added_at=added_at,
                 section_title=section_title,
                 languages=config.lang_list,
-                internal=bool(Prefs["subtitles.scan.embedded"]),
-                external=bool(Prefs["subtitles.scan.external"])
+                internal=cast_bool(Prefs["subtitles.scan.embedded"]),
+                external=cast_bool(Prefs["subtitles.scan.external"])
             )
             if state:
-                # (added_at, item_id, title)
+                # (added_at, item_id, title, item, missing_languages)
                 missing.append(state)
         except:
             Log.Error("Something went wrong when getting the state of item %s: %s", key, traceback.format_exc())
     return missing
 
 
-def refresh_item(item, title):
+def refresh_item(item):
     Plex["library/metadata"].refresh(item)
 
 
 def refresh_items(items):
     for item, title in items:
-        refresh_item(item, title)
+        refresh_item(item)

Contents/Code/support/plex_media.py

@@ -5,27 +5,29 @@ import subliminal
 import helpers
 
 from items import get_item
-from subzero import intent
+from lib import get_intent, Plex
 
 
-def flatten_media(media, kind="series"):
+def get_metadata_dict(item, part, add):
+    data = {
+        "item": item,
+        "section": item.section.title,
+        "path": part.file,
+        "folder": os.path.dirname(part.file),
+        "filename": os.path.basename(part.file)
+    }
+    data.update(add)
+    return data
+
+
+def media_to_videos(media, kind="series"):
     """
     iterates through media and returns the associated parts (videos)
     :param media:
     :param kind:
     :return:
     """
-    parts = []
-
-    def get_metadata_dict(item, part, add):
-        data = {
-            "section": item.section.title,
-            "path": part.file,
-            "folder": os.path.dirname(part.file),
-            "filename": os.path.basename(part.file)
-        }
-        data.update(add)
-        return data
+    videos = []
 
     if kind == "series":
         for season in media.seasons:
@@ -38,41 +40,32 @@ def flatten_media(media, kind="series"):
 
                 for item in media.seasons[season].episodes[episode].items:
                     for part in item.parts:
-                        parts.append(
+                        videos.append(
                             get_metadata_dict(plex_episode, part,
-                                              {"video": part, "type": "episode", "title": ep.title,
+                                              {"plex_part": part, "type": "episode", "title": ep.title,
                                                "series": media.title, "id": ep.id,
                                                "series_id": media.id, "season_id": season_object.id,
-                                               "season": plex_episode.season.index,
+                                               "episode": plex_episode.index, "season": plex_episode.season.index,
+                                               "section": plex_episode.section.title
                                                })
                         )
     else:
         plex_item = get_item(media.id)
         for item in media.items:
             for part in item.parts:
-                parts.append(
-                    get_metadata_dict(plex_item, part, {"video": part, "type": "movie",
+                videos.append(
+                    get_metadata_dict(plex_item, part, {"plex_part": part, "type": "movie",
                                                         "title": media.title, "id": media.id,
                                                         "series_id": None,
                                                         "season_id": None,
                                                         "section": plex_item.section.title})
                 )
-    return parts
+    return videos
 
 
 IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")
 
 
-def convert_media_to_parts(media, kind="series"):
-    """
-    returns a list of parts to be used later on; ignores folders with an existing "subzero.ignore" file
-    :param media:
-    :param kind:
-    :return:
-    """
-    return flatten_media(media, kind=kind)
-
-
 def get_stream_fps(streams):
     """
     accepts a list of plex streams or a list of the plex api streams
@@ -97,43 +90,84 @@ def get_media_item_ids(media, kind="series"):
     return ids
 
 
-def scan_video(plex_video, ignore_all=False, hints=None):
+def scan_video(plex_part, ignore_all=False, hints=None):
     embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
     external_subtitles = not ignore_all and Prefs['subtitles.scan.external']
 
     if ignore_all:
         Log.Debug("Force refresh intended.")
 
-    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (plex_video.file, external_subtitles, embedded_subtitles))
+    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (plex_part.file, external_subtitles, embedded_subtitles))
 
     try:
-        return subliminal.video.scan_video(plex_video.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles,
-                                           hints=hints or {}, video_fps=plex_video.fps)
+        return subliminal.video.scan_video(plex_part.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles,
+                                           hints=hints or {}, video_fps=plex_part.fps)
 
     except ValueError:
         Log.Warn("File could not be guessed by subliminal")
 
 
-def scan_parts(parts, kind="series"):
+def scan_videos(videos, kind="series", ignore_all=False):
     """
-    receives a list of parts containing dictionaries returned by flattenToParts
-    :param parts:
+    receives a list of videos containing dictionaries returned by media_to_videos
+    :param videos:
     :param kind: series or movies
     :return: dictionary of subliminal.video.scan_video, key=subliminal scanned video, value=plex file part
     """
     ret = {}
-    for part in parts:
-        force_refresh = intent.get("force", part["id"], part["series_id"], part["season_id"])
+    for video in videos:
+        intent = get_intent()
+        force_refresh = intent.get("force", video["id"], video["series_id"], video["season_id"])
+        Log.Debug("Determining force-refresh (video: %s, series: %s, season: %s), result: %s"
+                  % (video["id"], video["series_id"], video["season_id"], force_refresh))
+
+        hints = helpers.get_item_hints(video["title"], kind, series=video["series"] if kind == "series" else None)
+        video["plex_part"].fps = get_stream_fps(video["plex_part"].streams)
+        scanned_video = scan_video(video["plex_part"], ignore_all=force_refresh or ignore_all, hints=hints)
 
-        hints = helpers.get_item_hints(part["title"], kind, series=part["series"] if kind == "series" else None)
-        part["video"].fps = get_stream_fps(part["video"].streams)
-        scanned_video = scan_video(part["video"], ignore_all=force_refresh, hints=hints)
         if not scanned_video:
             continue
 
-        scanned_video.id = part["id"]
-        part_metadata = part.copy()
-        del part_metadata["video"]
+        scanned_video.id = video["id"]
+        part_metadata = video.copy()
+        del part_metadata["plex_part"]
         scanned_video.plexapi_metadata = part_metadata
-        ret[scanned_video] = part["video"]
-    return ret
+        ret[scanned_video] = video["plex_part"]
+    return ret
+
+
+class PartUnknownException(Exception):
+    pass
+
+
+def get_plex_metadata(rating_key, part_id, item_type):
+    plex_item = list(Plex["library"].metadata(rating_key))[0]
+
+    # find current part
+    current_part = None
+    for part in plex_item.media.parts:
+        if str(part.id) == part_id:
+            current_part = part
+
+    if not current_part:
+        raise PartUnknownException("Part unknown")
+
+    # get normalized metadata
+    if item_type == "episode":
+        metadata = get_metadata_dict(plex_item, current_part,
+                                     {"plex_part": current_part, "type": "episode", "title": plex_item.title,
+                                      "series": plex_item.show.title, "id": plex_item.rating_key,
+                                      "series_id": plex_item.show.rating_key,
+                                      "season_id": plex_item.season.rating_key,
+                                      "season": plex_item.season.index,
+                                      "episode": plex_item.index
+                                      })
+    else:
+        metadata = get_metadata_dict(plex_item, current_part, {"plex_part": current_part, "type": "movie",
+                                                               "title": plex_item.title, "id": plex_item.rating_key,
+                                                               "series_id": None,
+                                                               "season_id": None,
+                                                               "season": None,
+                                                               "episode": None,
+                                                               "section": plex_item.section.title})
+    return metadata

Contents/Code/support/storage.py

@@ -1,77 +1,105 @@
 # coding=utf-8
 
 import datetime
+import os
 import pprint
+import copy
+
+import subliminal
+
+from subtitlehelpers import force_utf8
+from config import config
+from helpers import notify_executable, get_title_for_video_metadata, cast_bool
 
 
 def get_subtitle_info(rating_key):
+    if "subs" not in Dict:
+        Dict["subs"] = {}
+
     return Dict["subs"].get(rating_key)
 
 
-def whack_missing_parts(videos, existing_parts=None):
+def whack_missing_parts(scanned_video_part_map, existing_parts=None):
     """
     cleans out our internal storage's video parts (parts may get updated/deleted/whatever)
     :param existing_parts: optional list of part ids known
-    :param videos: videos to check for
+    :param scanned_video_part_map: videos to check for
     :return:
     """
     # shortcut
 
+    if "subs" not in Dict:
+        return
+
     if not existing_parts:
         existing_parts = []
-        for part in videos.viewvalues():
-            existing_parts.append(part.id)
+        for part in scanned_video_part_map.viewvalues():
+            existing_parts.append(str(part.id))
 
     whacked_parts = False
-    for video in videos.keys():
-        if video.id not in Dict["subs"]:
+    for video in scanned_video_part_map.keys():
+        video_id = str(video.id)
+        if video_id not in Dict["subs"]:
             continue
 
-        for part_id in Dict["subs"][video.id].keys():
+        parts = Dict["subs"][video_id].keys()
+
+        for part_id in parts:
+            part_id = str(part_id)
             if part_id not in existing_parts:
-                del Dict["subs"][video.id][part_id]
-                Log.Info("Whacking part %s in internal storage of video %s", part_id, video.id)
+                Log.Info("Whacking part %s in internal storage of video %s (%s, %s)", part_id, video_id,
+                         repr(existing_parts), repr(parts))
+                del Dict["subs"][video_id][part_id]
                 whacked_parts = True
 
     if whacked_parts:
         Dict.Save()
 
 
-def store_subtitle_info(videos, subtitles, storage_type):
+def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_type, mode="a"):
     """
     stores information about downloaded subtitles in plex's Dict()
     """
     if "subs" not in Dict:
         Dict["subs"] = {}
 
-    storage = Dict["subs"]
-
     existing_parts = []
-    for video, video_subtitles in subtitles.items():
-        part = videos[video]
+    for video, video_subtitles in downloaded_subtitles.items():
+        part = scanned_video_part_map[video]
+        part_id = str(part.id)
+        video_id = str(video.id)
 
-        if video.id not in storage:
-            storage[video.id] = {}
+        if video_id not in Dict["subs"]:
+            Dict["subs"][video_id] = {}
 
-        video_dict = storage[video.id]
-        if part.id not in video_dict:
-            video_dict[part.id] = {}
+        video_dict = copy.deepcopy(Dict["subs"][video_id])
 
-        existing_parts.append(part.id)
+        if part_id not in video_dict:
+            video_dict[part_id] = {}
 
-        part_dict = video_dict[part.id]
+        existing_parts.append(part_id)
+
+        part_dict = video_dict[part_id]
         for subtitle in video_subtitles:
             lang = Locale.Language.Match(subtitle.language.alpha2)
-            if lang not in part_dict:
-                part_dict[lang] = {}
+            # always overwrite the old subtitle
+            part_dict[lang] = {}
+
             lang_dict = part_dict[lang]
-            sub_key = (subtitle.provider_name, subtitle.id)
-            lang_dict[sub_key] = dict(score=subtitle.score, link=subtitle.page_link, storage=storage_type, hash=Hash.MD5(subtitle.content),
-                                      date_added=datetime.datetime.now())
+            sub_key = subtitle.provider_name, str(subtitle.id)
+            metadata = video.plexapi_metadata
+
+            # compute title
+            title = get_title_for_video_metadata(metadata)
+            lang_dict[sub_key] = dict(score=subtitle.score, storage=storage_type, hash=Hash.MD5(subtitle.content),
+                                      date_added=datetime.datetime.now(), title=title, mode=mode)
             lang_dict["current"] = sub_key
 
+        Dict["subs"][video_id] = video_dict
+
     if existing_parts:
-        whack_missing_parts(videos, existing_parts=existing_parts)
+        whack_missing_parts(scanned_video_part_map, existing_parts=existing_parts)
+
     Dict.Save()
 
 
@@ -90,3 +118,70 @@ def reset_storage(key):
 def log_storage(key):
     if key in Dict:
         Log.Debug(pprint.pformat(Dict[key]))
+
+
+def save_subtitles_to_file(subtitles):
+    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \
+        if cast_bool(Prefs["subtitles.save.subFolder.Custom"]) else None
+
+    for video, video_subtitles in subtitles.items():
+        if not video_subtitles:
+            continue
+
+        fld = None
+        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
+            # specific subFolder requested, create it if it doesn't exist
+            fld_base = os.path.split(video.name)[0]
+            if fld_custom:
+                if fld_custom.startswith("/"):
+                    # absolute folder
+                    fld = fld_custom
+                else:
+                    fld = os.path.join(fld_base, fld_custom)
+            else:
+                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
+            if not os.path.exists(fld):
+                os.makedirs(fld)
+        subliminal.api.save_subtitles(video, video_subtitles, directory=fld, single=Prefs['subtitles.only_one'],
+                                      encode_with=force_utf8 if Prefs['subtitles.enforce_encoding'] else None,
+                                      chmod=config.chmod)
+    return True
+
+
+def save_subtitles_to_metadata(videos, subtitles):
+    for video, video_subtitles in subtitles.items():
+        mediaPart = videos[video]
+        for subtitle in video_subtitles:
+            content = force_utf8(subtitle.text) if Prefs['subtitles.enforce_encoding'] else subtitle.content
+            mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.id] = Proxy.Media(content, ext="srt")
+    return True
+
+
+def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
+    meta_fallback = False
+    save_successful = False
+    storage = "metadata"
+    if Prefs['subtitles.save.filesystem']:
+        storage = "filesystem"
+        try:
+            Log.Debug("Using filesystem as subtitle storage")
+            save_subtitles_to_file(downloaded_subtitles)
+        except OSError:
+            if Prefs["subtitles.save.metadata_fallback"]:
+                meta_fallback = True
+            else:
+                raise
+        else:
+            save_successful = True
+
+    if not Prefs['subtitles.save.filesystem'] or meta_fallback:
+        if meta_fallback:
+            Log.Debug("Using metadata as subtitle storage, because filesystem storage failed")
+        else:
+            Log.Debug("Using metadata as subtitle storage")
+        save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles)
+
+    if save_successful and config.notify_executable:
+        notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage)
+
+    store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)

Contents/Code/support/subtitlehelpers.py

@@ -79,6 +79,16 @@ class VobSubSubtitleHelper(SubtitleHelper):
 
 #####################################################################################################################
 
+
+def match_ietf_language(s):
+    language_match = re.match(".+\.([^\.]+)$" if not helpers.cast_bool(Prefs["subtitles.language.ietf"])
+                              else ".+\.([^-.]+)(?:-[A-Za-z]+)?$", s)
+    if language_match and len(language_match.groups()) == 1:
+        language = language_match.groups()[0]
+        return language
+    return s
+
+
 class DefaultSubtitleHelper(SubtitleHelper):
     @classmethod
     def is_helper_for(cls, filename):
@@ -99,10 +109,7 @@ class DefaultSubtitleHelper(SubtitleHelper):
         language = ""
 
         # IETF support thanks to https://github.com/hpsbranco/LocalMedia.bundle/commit/4fad9aefedece78a1fa96401304351347f644369
-        language_match = re.match(".+\.([^\.]+)$" if not Prefs["subtitles.language.ietf"] else ".+\.([^-.]+)(?:-[A-Za-z]+)?$", file)
-        if language_match and len(language_match.groups()) == 1:
-            language = language_match.groups()[0]
-        language = Locale.Language.Match(language)
+        language = Locale.Language.Match(match_ietf_language(file))
 
         codec = None
         format = None

Contents/Code/support/tasks.py

@@ -3,29 +3,49 @@
 import datetime
 import time
 
+import operator
+import traceback
+
+import subliminal
+import subliminal_patch
+
+from subliminal_patch.patch_api import list_all_subtitles, download_subtitles
+from babelfish import Language
+from subliminal_patch.patch_subtitle import compute_score
 from missing_subtitles import items_get_all_missing_subs, refresh_item
 from background import scheduler
-from support.items import get_recent_items, is_ignored
+from storage import save_subtitles, whack_missing_parts
+from support.config import config
+from support.items import get_recent_items, is_ignored, get_item
+from support.lib import Plex
+from support.helpers import track_usage, get_title_for_video_metadata, cast_bool
+from support.plex_media import scan_videos, get_plex_metadata, PartUnknownException
 
 
 class Task(object):
     name = None
     scheduler = None
+    periodic = False
     running = False
     time_start = None
+    data = None
 
     stored_attributes = ("last_run", "last_run_time")
+    default_data = {"last_run": None, "last_run_time": None, "data": {}}
 
     # task ready for being status-displayed?
     ready_for_display = False
 
     def __init__(self, scheduler):
+        self.name = self.get_class_name()
         self.ready_for_display = False
         self.running = False
         self.time_start = None
         self.scheduler = scheduler
-        if self.name not in Dict["tasks"]:
-            Dict["tasks"][self.name] = {"last_run": None, "last_run_time": None}
+        self.setup_defaults()
+
+    def get_class_name(self):
+        return getattr(getattr(self, "__class__"), "__name__")
 
     def __getattribute__(self, name):
         if name in object.__getattribute__(self, "stored_attributes"):
@@ -41,18 +61,36 @@ class Task(object):
 
         object.__setattr__(self, name, value)
 
+    def setup_defaults(self):
+        if self.name not in Dict["tasks"]:
+            Dict["tasks"][self.name] = self.default_data.copy()
+            return
+
+        sd = Dict["tasks"][self.name]
+
+        # forward-migration
+        for key, def_value in self.default_data.iteritems():
+            hasval = key in sd
+            if not hasval:
+                sd[key] = def_value
+
     def signal(self, *args, **kwargs):
         raise NotImplementedError
 
-    def prepare(self):
-        raise NotImplementedError
+    def prepare(self, *args, **kwargs):
+        return
 
     def run(self):
         raise NotImplementedError
 
+    def post_run(self, data_holder):
+        self.running = False
+        if self.time_start:
+            self.last_run_time = self.last_run - self.time_start
+
 
 class SearchAllRecentlyAddedMissing(Task):
-    name = "searchAllRecentlyAddedMissing"
+    periodic = True
     items_done = None
     items_searching = None
     items_searching_ids = None
@@ -80,16 +118,15 @@ class SearchAllRecentlyAddedMissing(Task):
             self.items_done.append(item_id)
             return True
 
-    def prepare(self):
+    def prepare(self, *args, **kwargs):
         self.items_done = []
         recent_items = get_recent_items()
         missing = items_get_all_missing_subs(recent_items)
-        ids = set([id for added_at, id, title, item in missing if not is_ignored(id, item=item)])
+        ids = set([id for added_at, id, title, item, missing_languages in missing if not is_ignored(id, item=item)])
         self.items_searching = missing
         self.items_searching_ids = ids
         self.items_failed = []
         self.percentage = 0
-        self.time_start = datetime.datetime.now()
         self.ready_for_display = True
 
     def run(self):
@@ -97,9 +134,9 @@ class SearchAllRecentlyAddedMissing(Task):
         missing_count = len(self.items_searching)
         items_done_count = 0
 
-        for added_at, item_id, title, item in self.items_searching:
+        for added_at, item_id, title, item, missing_languages in self.items_searching:
             Log.Debug(u"Task: %s, triggering refresh for %s (%s)", self.name, title, item_id)
-            refresh_item(item_id, title)
+            refresh_item(item_id)
             search_started = datetime.datetime.now()
             tries = 1
             while 1:
@@ -116,9 +153,10 @@ class SearchAllRecentlyAddedMissing(Task):
                         Log.Debug(u"Task: %s, item stalled for %s times: %s, skipping", self.name, tries, item_id)
                         break
 
-                    Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time, item_id)
+                    Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time,
+                              item_id)
                     tries += 1
-                    refresh_item(item_id, title)
+                    refresh_item(item_id)
                     search_started = datetime.datetime.now()
                     time.sleep(1)
                 time.sleep(0.1)
@@ -128,12 +166,9 @@ class SearchAllRecentlyAddedMissing(Task):
         Log.Debug("Task: %s, done. Failed items: %s", self.name, self.items_failed)
         self.running = False
 
-    def post_run(self):
+    def post_run(self, task_data):
+        super(SearchAllRecentlyAddedMissing, self).post_run(task_data)
         self.ready_for_display = False
-        self.last_run = datetime.datetime.now()
-        if self.time_start:
-            self.last_run_time = self.last_run - self.time_start
-        self.time_start = None
         self.percentage = 0
         self.items_done = None
         self.items_failed = None
@@ -141,4 +176,253 @@ class SearchAllRecentlyAddedMissing(Task):
         self.items_searching_ids = None
 
 
+class SubtitleListingMixin(object):
+    def list_subtitles(self, rating_key, item_type, part_id, language):
+        metadata = get_plex_metadata(rating_key, part_id, item_type)
+
+        if item_type == "episode":
+            min_score = 66
+        else:
+            min_score = 23
+
+        scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+        if not scanned_parts:
+            Log.Error("Couldn't list available subtitles for %s", rating_key)
+            return
+
+        video, plex_part = scanned_parts.items()[0]
+        config.init_subliminal_patches()
+
+        available_subs = list_all_subtitles(scanned_parts, {Language.fromietf(language)},
+                                            providers=config.providers,
+                                            provider_configs=config.provider_settings)
+
+        use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in ("prefer", "force HI")
+
+        # sort subtitles by score
+        unsorted_subtitles = []
+        for s in available_subs[video]:
+            Log.Debug("Starting score computation for %s", s)
+            try:
+                matches = s.get_matches(video, hearing_impaired=use_hearing_impaired)
+            except AttributeError:
+                Log.Error("Match computation failed for %s: %s", s, traceback.format_exc())
+                continue
+
+            unsorted_subtitles.append((s, compute_score(matches, video), matches))
+        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1), reverse=True)
+
+        subtitles = []
+        for subtitle, score, matches in scored_subtitles:
+            # check score
+            if score < min_score:
+                Log.Info('Score %d is below min_score (%d)', score, min_score)
+                continue
+            subtitle.score = score
+            subtitle.matches = matches
+            subtitle.part_id = part_id
+            subtitle.item_type = item_type
+            subtitles.append(subtitle)
+        return subtitles
+
+
+class DownloadSubtitleMixin(object):
+    def download_subtitle(self, subtitle, rating_key, mode="m"):
+        from interface.menu_helpers import set_refresh_menu_state
+
+        item_type = subtitle.item_type
+        part_id = subtitle.part_id
+        metadata = get_plex_metadata(rating_key, part_id, item_type)
+        scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+        video, plex_part = scanned_parts.items()[0]
+
+        # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
+        download_subtitles([subtitle], providers=config.providers, provider_configs=config.provider_settings)
+
+        if subtitle.content:
+            try:
+                whack_missing_parts(scanned_parts)
+                save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode)
+                Log.Debug("Manually downloaded subtitle for: %s", rating_key)
+                refresh_item(rating_key)
+                track_usage("Subtitle", "manual", "download", 1)
+            except:
+                Log.Error("Something went wrong when downloading specific subtitle: %s", traceback.format_exc())
+            finally:
+                set_refresh_menu_state(None)
+
+                # store item in history
+                from support.history import get_history
+                item_title = get_title_for_video_metadata(metadata, add_section_title=False)
+                history = get_history()
+                history.add(item_title, video.id, section_title=video.plexapi_metadata["section"], subtitle=subtitle,
+                            mode=mode)
+
+
+class AvailableSubsForItem(SubtitleListingMixin, Task):
+    item_type = None
+    part_id = None
+    language = None
+    rating_key = None
+
+    def prepare(self, *args, **kwargs):
+        self.item_type = kwargs.get("item_type")
+        self.part_id = kwargs.get("part_id")
+        self.language = kwargs.get("language")
+        self.rating_key = kwargs.get("rating_key")
+
+    def setup_defaults(self):
+        super(AvailableSubsForItem, self).setup_defaults()
+
+        # reset any previous data
+        Dict["tasks"][self.name]["data"] = {}
+
+    def run(self):
+        self.running = True
+        track_usage("Subtitle", "manual", "list", 1)
+        self.data = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language)
+
+    def post_run(self, task_data):
+        super(AvailableSubsForItem, self).post_run(task_data)
+        if self.rating_key not in task_data:
+            task_data[self.rating_key] = {}
+
+        task_data[self.rating_key][self.language] = self.data
+
+
+class DownloadSubtitleForItem(DownloadSubtitleMixin, Task):
+    subtitle = None
+    rating_key = None
+
+    def prepare(self, *args, **kwargs):
+        self.subtitle = kwargs["subtitle"]
+        self.rating_key = kwargs["rating_key"]
+
+    def run(self):
+        self.running = True
+        self.download_subtitle(self.subtitle, self.rating_key)
+        self.running = False
+
+
+class MissingSubtitles(Task):
+    rating_key = None
+    item_type = None
+    part_id = None
+    language = None
+
+    def run(self):
+        self.running = True
+        self.data = []
+        recent_items = get_recent_items()
+        if recent_items:
+            self.data = items_get_all_missing_subs(recent_items)
+
+    def post_run(self, task_data):
+        super(MissingSubtitles, self).post_run(task_data)
+        task_data["missing_subtitles"] = self.data
+
+
+class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
+    periodic = True
+
+    # TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired
+    series_cutoff = 132
+
+    # movies: format, title, release_group, year, video_codec, resolution, hearing_impaired
+    movies_cutoff = 61
+
+    def run(self):
+        self.running = True
+        better_found = 0
+        try:
+            max_search_days = int(Prefs["scheduler.tasks.FindBetterSubtitles.max_days_after_added"].strip())
+        except ValueError:
+            Log.Error("Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting")
+            return
+        else:
+            if max_search_days > 30:
+                Log.Error("FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.")
+                return
+
+        now = datetime.datetime.now()
+
+        for video_id, parts in Dict["subs"].iteritems():
+            video_id = str(video_id)
+            try:
+                plex_item = get_item(video_id)
+            except:
+                Log.Error("Couldn't get item info for %s", video_id)
+                continue
+            cutoff = self.series_cutoff if plex_item.type == "episode" else self.movies_cutoff
+            added_date = datetime.datetime.fromtimestamp(plex_item.added_at)
+
+            # don't search for better subtitles until at least 30 minutes have passed
+            if added_date + datetime.timedelta(minutes=30) > now:
+                Log.Debug("Item %s too new, skipping", video_id)
+                continue
+
+            # added_date <= max_search_days?
+            if added_date + datetime.timedelta(days=max_search_days) <= now:
+                continue
+
+            ditch_parts = []
+
+            # look through all stored subtitle data
+            for part_id, languages in parts.iteritems():
+                part_id = str(part_id)
+
+                # all languages
+                for language, current_subs in languages.iteritems():
+                    current_key = current_subs.get("current")
+                    current = current_subs.get(current_key)
+
+                    # currently got subtitle?
+                    if not current:
+                        continue
+                    current_score = int(current["score"])
+                    current_mode = current.get("mode", "a")
+
+                    # late cutoff met? skip
+                    if current_score >= cutoff:
+                        Log.Debug(u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s",
+                                  current_score, cutoff, current["title"])
+                        continue
+
+                    # got manual subtitle but don't want to touch those?
+                    if current_mode == "m" and \
+                            not cast_bool(Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]):
+                        Log.Debug(u"Skipping finding better subs, had manual: %s", current["title"])
+                        continue
+
+                    try:
+                        subs = self.list_subtitles(video_id, plex_item.type, part_id, language)
+                    except PartUnknownException:
+                        Log.Info("Part %s unknown/gone; ditching subtitle info", part_id)
+                        ditch_parts.append(part_id)
+                        continue
+
+                    if subs:
+                        # subs are already sorted by score
+                        sub = subs[0]
+                        if sub.score > current_score:
+                            Log.Debug("Better subtitle found for %s, downloading", video_id)
+                            self.download_subtitle(sub, video_id, mode="b")
+                            better_found += 1
+
+            if ditch_parts:
+                for part_id in ditch_parts:
+                    try:
+                        del parts[part_id]
+                    except KeyError:
+                        pass
+
+        if better_found:
+            Log.Debug("Task: %s, done. Better subtitles found for %s items", self.name, better_found)
+        self.running = False
+
+
 scheduler.register(SearchAllRecentlyAddedMissing)
+scheduler.register(AvailableSubsForItem)
+scheduler.register(DownloadSubtitleForItem)
+scheduler.register(MissingSubtitles)
+scheduler.register(FindBetterSubtitles)

Contents/DefaultPrefs.json

@@ -236,12 +236,6 @@
     "type": "bool",
     "default": "true"
   },
-  {
-    "id": "provider.thesubdb.enabled",
-    "label": "Provider: Enable TheSubDB",
-    "type": "bool",
-    "default": "true"
-  },
   {
     "id": "provider.podnapisi.enabled",
     "label": "Provider: Enable Podnapisi.NET",
@@ -255,10 +249,34 @@
     "default": "true"
   },
   {
-    "id": "provider.addic7ed.boost",
-    "label": "Addic7ed: prefer over other providers (if requirements met)",
-    "type": "bool",
-    "default": "false"
+    "id": "provider.addic7ed.boost_by",
+    "label": "Addic7ed: boost score (if requirements met)",
+    "type": "enum",
+    "values": [
+      "100",
+      "95",
+      "90",
+      "85",
+      "80",
+      "75",
+      "70",
+      "67",
+      "65",
+      "60",
+      "55",
+      "50",
+      "45",
+      "40",
+      "35",
+      "30",
+      "25",
+      "20",
+      "15",
+      "10",
+      "5",
+      "0"
+    ],
+    "default": "10"
   },
   {
     "id": "provider.tvsubtitles.enabled",
@@ -285,63 +303,15 @@
     "default": "true"
   },
   {
-    "id": "subtitles.search.minimumTVScore",
-    "label": "Minimum score for TV subtitles to download",
-    "type": "enum",
-    "values": [
-      "100",
-      "95",
-      "90",
-      "85",
-      "80",
-      "75",
-      "70",
-      "67",
-      "65",
-      "60",
-      "55",
-      "50",
-      "45",
-      "40",
-      "35",
-      "30",
-      "25",
-      "20",
-      "15",
-      "10",
-      "5",
-      "0"
-    ],
-    "default": "85"
+    "id": "subtitles.search.minimumTVScore1",
+    "label": "Minimum score for TV (min: 77, sane: 110; see http://v.ht/szscores)",
+    "type": "text",
+    "default": "110"
   },
   {
-    "id": "subtitles.search.minimumMovieScore",
-    "label": "Minimum score for movie subtitles to download",
-    "type": "enum",
-    "values": [
-      "100",
-      "95",
-      "90",
-      "85",
-      "80",
-      "75",
-      "70",
-      "65",
-      "60",
-      "55",
-      "50",
-      "45",
-      "40",
-      "35",
-      "30",
-      "25",
-      "23",
-      "20",
-      "15",
-      "10",
-      "5",
-      "0"
-    ],
+    "id": "subtitles.search.minimumMovieScore1",
+    "label": "Minimum score for movies (def: 23, sane: 33; see http://v.ht/szscores)",
+    "type": "text",
     "default": "23"
   },
   {
@@ -362,6 +332,12 @@
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "subtitles.save.chmod",
+      "label": "Set file permissions to (integer, e.g.: 0775)",
+      "type": "text",
+      "default": ""
+  },
   {
     "id": "subtitles.save.subFolder",
     "label": "Subtitle Folder (\"current folder\" is the folder the current media file lives in)",
@@ -412,7 +388,7 @@
       "default": ""
   },
   {
-    "id": "scheduler.tasks.searchAllRecentlyAddedMissing",
+    "id": "scheduler.tasks.SearchAllRecentlyAddedMissing.frequency",
     "label": "Scheduler: Periodically search for recent items with missing subtitles",
     "type": "enum",
     "values": [
@@ -447,7 +423,44 @@
     "id": "scheduler.max_recent_items_per_library",
     "label": "Scheduler: Recent items to consider per library",
     "type": "text",
-    "default": "200"
+    "default": "2000"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.frequency",
+    "label": "Scheduler: Periodically search for better subtitles",
+    "type": "enum",
+    "values": [
+      "never",
+      "every 6 hours",
+      "every 12 hours",
+      "every 24 hours"
+    ],
+    "default": "every 12 hours"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.max_days_after_added",
+    "label": "Scheduler: Days to search for better subtitles (max: 30 days)",
+    "type": "text",
+    "default": "7"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected",
+    "label": "Scheduler: Overwrite manually selected subtitles when better found",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "history_size",
+    "label": "History: amount of items to store historical data for",
+    "type": "enum",
+    "values": [
+      "50",
+      "100",
+      "150",
+      "250",
+      "500"
+    ],
+    "default": "100"
   },
   {
     "id": "check_permissions",
@@ -473,5 +486,11 @@
     "label": "Log to console (for development/debugging)",
     "type": "bool",
     "default": "false"
+  },
+  {
+    "id": "track_usage",
+    "label": "Collect anonymous usage statistics",
+    "type": "bool",
+    "default": "true"
   }
 ]

Contents/Info.plist

@@ -9,11 +9,11 @@
         <key>CFBundleInfoDictionaryVersion</key>
         <string>6.0</string>
         <key>CFBundleShortVersionString</key>
-        <string>1.3.31</string>
+        <string>1.4.11</string>
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>1.3.33.522</string>
+        <string>1.4.11.781</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -32,13 +32,15 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 1.3.33.522
+Version 1.4.11.781
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 
 If you like this, buy me a beer: &lt;a href=&quot;https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&amp;hosted_button_id=G9VKR2B8PMNKG&quot; target=&quot;_blank&quot; title=&quot;donate&quot;&gt;&lt;img src=&quot;https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif&quot; alt=&quot;donate&quot; title=&quot;donate&quot; /&gt;&lt;/a&gt;
 
 &lt;strong&gt;Need help?&lt;/strong&gt;
+Wiki: &lt;a href=&quot;http://v.ht/szwiki&quot;&gt;http://v.ht/szwiki&lt;/a&gt;
+Score info: &lt;a href=&quot;http://v.ht/szscores&quot;&gt;http://v.ht/szscores&lt;/a&gt;
 Plex thread: &lt;a href=&quot;https://forums.plex.tv/discussion/186575&quot;>https://forums.plex.tv/discussion/186575&lt;/a&gt;
 Github: &lt;a href=&quot;https://github.com/pannal/Sub-Zero.bundle&quot;&gt;https://github.com/pannal/Sub-Zero&lt;/a&gt;
 

Contents/Libraries/Shared/pyga/__init__.py

@@ -0,0 +1,8 @@
+from pyga.requests import Q
+
+def shutdown():
+    '''
+    Fire all stored GIF requests One by One.
+    You should call this if you set Config.queue_requests = True
+    '''
+    map(lambda func: func(), Q.REQ_ARRAY)

Contents/Libraries/Shared/pyga/entities.py

@@ -0,0 +1,506 @@
+# -*- coding: utf-8 -*-
+
+from datetime import datetime
+from operator import itemgetter
+import six
+from pyga import utils
+from pyga import exceptions
+
+__author__ = "Arun KR (kra3) <the1.arun@gmail.com>"
+__license__ = "Simplified BSD"
+
+
+class Campaign(object):
+    '''
+    A representation of Campaign
+
+    Properties:
+    _type -- See TYPE_* constants, will be mapped to "__utmz" parameter.
+    creation_time --  Time of the creation of this campaign, will be mapped to "__utmz" parameter.
+    response_count -- Response Count, will be mapped to "__utmz" parameter.
+        Is also used to determine whether the campaign is new or repeated,
+        which will be mapped to "utmcn" and "utmcr" parameters.
+    id -- Campaign ID, a.k.a. "utm_id" query parameter for ga.js
+           Will be mapped to "__utmz" parameter.
+    source -- Source, a.k.a. "utm_source" query parameter for ga.js.
+              Will be mapped to "utmcsr" key in "__utmz" parameter.
+    g_click_id -- Google AdWords Click ID, a.k.a. "gclid" query parameter for ga.js.
+                  Will be mapped to "utmgclid" key in "__utmz" parameter.
+    d_click_id -- DoubleClick (?) Click ID. Will be mapped to "utmdclid" key in "__utmz" parameter.
+    name --  Name, a.k.a. "utm_campaign" query parameter for ga.js.
+             Will be mapped to "utmccn" key in "__utmz" parameter.
+    medium -- Medium, a.k.a. "utm_medium" query parameter for ga.js.
+              Will be mapped to "utmcmd" key in "__utmz" parameter.
+    term -- Terms/Keywords, a.k.a. "utm_term" query parameter for ga.js.
+            Will be mapped to "utmctr" key in "__utmz" parameter.
+    content -- Ad Content Description, a.k.a. "utm_content" query parameter for ga.js.
+               Will be mapped to "utmcct" key in "__utmz" parameter.
+
+    '''
+
+    TYPE_DIRECT = 'direct'
+    TYPE_ORGANIC = 'organic'
+    TYPE_REFERRAL = 'referral'
+
+    CAMPAIGN_DELIMITER = '|'
+
+    UTMZ_PARAM_MAP = {
+        'utmcid': 'id',
+        'utmcsr': 'source',
+        'utmgclid': 'g_click_id',
+        'utmdclid': 'd_click_id',
+        'utmccn': 'name',
+        'utmcmd': 'medium',
+        'utmctr': 'term',
+        'utmcct': 'content',
+    }
+
+    def __init__(self, typ):
+        self._type = None
+        self.creation_time = None
+        self.response_count = 0
+        self.id = None
+        self.source = None
+        self.g_click_id = None
+        self.d_click_id = None
+        self.name = None
+        self.medium = None
+        self.term = None
+        self.content = None
+
+        if typ:
+            if typ not in ('direct', 'organic', 'referral'):
+                raise ValueError('Campaign type has to be one of the Campaign::TYPE_* constant values.')
+
+            self._type = typ
+            if typ == Campaign.TYPE_DIRECT:
+                self.name = '(direct)'
+                self.source = '(direct)'
+                self.medium = '(none)'
+            elif typ == Campaign.TYPE_REFERRAL:
+                self.name = '(referral)'
+                self.medium = 'referral'
+            elif typ == Campaign.TYPE_ORGANIC:
+                self.name = '(organic)'
+                self.medium = 'organic'
+            else:
+                self._type = None
+
+        self.creation_time = datetime.utcnow()
+
+    def validate(self):
+        if not self.source:
+            raise exceptions.ValidationError('Campaigns need to have at least the "source" attribute defined.')
+
+    @staticmethod
+    def create_from_referrer(url):
+        obj = Campaign(Campaign.TYPE_REFERRAL)
+        parse_rslt = six.moves.urllib.parse.urlparse(url)
+        obj.source = parse_rslt.netloc
+        obj.content = parse_rslt.path
+        return obj
+
+    def extract_from_utmz(self, utmz):
+        parts = utmz.split('.', 4)
+
+        if len(parts) != 5:
+            raise ValueError('The given "__utmz" cookie value is invalid.')
+
+        self.creation_time = utils.convert_ga_timestamp(parts[1])
+        self.response_count = int(parts[3])
+        params = parts[4].split(Campaign.CAMPAIGN_DELIMITER)
+
+        for param in params:
+            key, val = param.split('=')
+
+            try:
+                setattr(self, self.UTMZ_PARAM_MAP[key], six.moves.urllib.parse.unquote_plus(val))
+            except KeyError:
+                continue
+
+        return self
+
+
+class CustomVariable(object):
+    '''
+    Represent a Custom Variable
+
+    Properties:
+    index -- Is the slot, you have 5 slots
+    name -- Name given to custom variable
+    value -- Value for the variable
+    scope -- Scope can be any one of 1, 2 or 3.
+
+    WATCH OUT: It's a known issue that GA will not decode URL-encoded
+    characters in custom variable names and values properly, so spaces
+    will show up as "%20" in the interface etc. (applicable to name & value)
+    http://www.google.com/support/forum/p/Google%20Analytics/thread?tid=2cdb3ec0be32e078
+
+    '''
+
+    SCOPE_VISITOR = 1
+    SCOPE_SESSION = 2
+    SCOPE_PAGE = 3
+
+    def __init__(self, index=None, name=None, value=None, scope=3):
+        self.index = index
+        self.name = name
+        self.value = value
+        self.scope = CustomVariable.SCOPE_PAGE
+        if scope:
+            self.scope = scope
+
+    def __setattr__(self, name, value):
+        if name == 'scope':
+            if value and value not in range(1, 4):
+                raise ValueError('Custom Variable scope has to be one of the 1,2 or 3')
+
+        if name == 'index':
+            # Custom Variables are limited to five slots officially, but there seems to be a
+            # trick to allow for more of them which we could investigate at a later time (see
+            # http://analyticsimpact.com/2010/05/24/get-more-than-5-custom-variables-in-google-analytics/
+            if value and (value < 0 or value > 5):
+                raise ValueError('Custom Variable index has to be between 1 and 5.')
+
+        object.__setattr__(self, name, value)
+
+    def validate(self):
+        '''
+        According to the GA documentation, there is a limit to the combined size of
+        name and value of 64 bytes after URL encoding,
+        see http://code.google.com/apis/analytics/docs/tracking/gaTrackingCustomVariables.html#varTypes
+        and http://xahlee.org/js/google_analytics_tracker_2010-07-01_expanded.js line 563
+        This limit was increased to 128 bytes BEFORE encoding with the 2012-01 release of ga.js however,
+        see http://code.google.com/apis/analytics/community/gajs_changelog.html
+        '''
+        if len('%s%s' % (self.name, self.value)) > 128:
+            raise exceptions.ValidationError('Custom Variable combined name and value length must not be larger than 128 bytes.')
+
+
+class Event(object):
+    '''
+    Represents an Event
+    https://developers.google.com/analytics/devguides/collection/gajs/eventTrackerGuide
+
+    Properties:
+    category -- The general event category
+    action -- The action for the event
+    label -- An optional descriptor for the event
+    value -- An optional value associated with the event. You can see your
+             event values in the Overview, Categories, and Actions reports,
+             where they are listed by event or aggregated across events,
+             depending upon your report view.
+    noninteraction -- By default, event hits will impact a visitor's bounce rate.
+                      By setting this parameter to true, this event hit
+                      will not be used in bounce rate calculations.
+                      (default False)
+    '''
+
+    def __init__(self, category=None, action=None, label=None, value=None, noninteraction=False):
+        self.category = category
+        self.action = action
+        self.label = label
+        self.value = value
+        self.noninteraction = bool(noninteraction)
+
+        if self.noninteraction and not self.value:
+            self.value = 0
+
+    def validate(self):
+        if not(self.category and self.action):
+            raise exceptions.ValidationError('Events, at least need to have a category and action defined.')
+
+
+class Item(object):
+    '''
+    Represents an Item in Transaction
+
+    Properties:
+    order_id -- Order ID, will be mapped to "utmtid" parameter
+    sku -- Product Code. This is the sku code for a given product, will be mapped to "utmipc" parameter
+    name -- Product Name, will be mapped to "utmipn" parameter
+    variation -- Variations on an item, will be mapped to "utmiva" parameter
+    price -- Unit Price. Value is set to numbers only, will be mapped to "utmipr" parameter
+    quantity -- Unit Quantity, will be mapped to "utmiqt" parameter
+
+    '''
+
+    def __init__(self):
+        self.order_id = None
+        self.sku = None
+        self.name = None
+        self.variation = None
+        self.price = None
+        self.quantity = 1
+
+    def validate(self):
+        if not self.sku:
+            raise exceptions.ValidationError('sku/product is a required parameter')
+
+
+class Page(object):
+    '''
+    Contains all parameters needed for tracking a page
+
+    Properties:
+    path -- Page request URI, will be mapped to "utmp" parameter
+    title -- Page title, will be mapped to "utmdt" parameter
+    charset -- Charset encoding, will be mapped to "utmcs" parameter
+    referrer -- Referer URL, will be mapped to "utmr" parameter
+    load_time -- Page load time in milliseconds, will be encoded into "utme" parameter.
+
+    '''
+    REFERRER_INTERNAL = '0'
+
+    def __init__(self, path):
+        self.path = None
+        self.title = None
+        self.charset = None
+        self.referrer = None
+        self.load_time = None
+
+        if path:
+            self.path = path
+
+    def __setattr__(self, name, value):
+        if name == 'path':
+            if value and value != '':
+                if value[0] != '/':
+                    raise ValueError('The page path should always start with a slash ("/").')
+        elif name == 'load_time':
+            if value and not isinstance(value, int):
+                raise ValueError('Page load time must be specified in integer milliseconds.')
+
+        object.__setattr__(self, name, value)
+
+
+class Session(object):
+    '''
+    You should serialize this object and store it in the user session to keep it
+    persistent between requests (similar to the "__umtb" cookie of the GA Javascript client).
+
+    Properties:
+    session_id -- A unique per-session ID, will be mapped to "utmhid" parameter
+    track_count -- The amount of pageviews that were tracked within this session so far,
+                   will be part of the "__utmb" cookie parameter.
+                   Will get incremented automatically upon each request
+    start_time -- Timestamp of the start of this new session, will be part of the "__utmb" cookie parameter
+
+    '''
+    def __init__(self):
+        self.session_id = utils.get_32bit_random_num()
+        self.track_count = 0
+        self.start_time = datetime.utcnow()
+
+    @staticmethod
+    def generate_session_id():
+        return utils.get_32bit_random_num()
+
+    def extract_from_utmb(self, utmb):
+        '''
+        Will extract information for the "trackCount" and "startTime"
+        properties from the given "__utmb" cookie value.
+        '''
+        parts = utmb.split('.')
+        if len(parts) != 4:
+            raise ValueError('The given "__utmb" cookie value is invalid.')
+
+        self.track_count = int(parts[1])
+        self.start_time = utils.convert_ga_timestamp(parts[3])
+
+        return self
+
+
+class SocialInteraction(object):
+    '''
+
+    Properties:
+    action -- Required. A string representing the social action being tracked,
+              will be mapped to "utmsa" parameter
+    network -- Required. A string representing the social network being tracked,
+               will be mapped to "utmsn" parameter
+    target -- Optional. A string representing the URL (or resource) which receives the action.
+
+    '''
+
+    def __init__(self, action=None, network=None, target=None):
+        self.action = action
+        self.network = network
+        self.target = target
+
+    def validate(self):
+        if not(self.action and self.network):
+            raise exceptions.ValidationError('Social interactions need to have at least the "network" and "action" attributes defined.')
+
+
+class Transaction(object):
+    '''
+    Represents parameters for a Transaction call
+
+    Properties:
+    order_id -- Order ID, will be mapped to "utmtid" parameter
+    affiliation -- Affiliation, Will be mapped to "utmtst" parameter
+    total -- Total Cost, will be mapped to "utmtto" parameter
+    tax -- Tax Cost, will be mapped to "utmttx" parameter
+    shipping -- Shipping Cost, values as for unit and price, will be mapped to "utmtsp" parameter
+    city -- Billing City, will be mapped to "utmtci" parameter
+    state -- Billing Region, will be mapped to "utmtrg" parameter
+    country -- Billing Country, will be mapped to "utmtco" parameter
+    items -- @entity.Items in a transaction
+
+    '''
+    def __init__(self):
+        self.items = []
+        self.order_id = None
+        self.affiliation = None
+        self.total = None
+        self.tax = None
+        self.shipping = None
+        self.city = None
+        self.state = None
+        self.country = None
+
+    def __setattr__(self, name, value):
+        if name == 'order_id':
+            for itm in self.items:
+                itm.order_id = value
+        object.__setattr__(self, name, value)
+
+    def validate(self):
+        if len(self.items) == 0:
+            raise exceptions.ValidationError('Transaction need to consist of at least one item')
+
+    def add_item(self, item):
+        ''' item of type entities.Item '''
+        if isinstance(item, Item):
+            item.order_id = self.order_id
+            self.items.append(item)
+
+
+class Visitor(object):
+    '''
+    You should serialize this object and store it in the user database to keep it
+    persistent for the same user permanently (similar to the "__umta" cookie of
+    the GA Javascript client).
+
+    Properties:
+    unique_id -- Unique user ID, will be part of the "__utma" cookie parameter
+    first_visit_time -- Time of the very first visit of this user, will be part of the "__utma" cookie parameter
+    previous_visit_time -- Time of the previous visit of this user, will be part of the "__utma" cookie parameter
+    current_visit_time -- Time of the current visit of this user, will be part of the "__utma" cookie parameter
+    visit_count -- Amount of total visits by this user, will be part of the "__utma" cookie parameter
+    ip_address -- IP Address of the end user, will be mapped to "utmip" parameter and "X-Forwarded-For" request header
+    user_agent -- User agent string of the end user, will be mapped to "User-Agent" request header
+    locale -- Locale string (country part optional) will be mapped to "utmul" parameter
+    flash_version -- Visitor's Flash version, will be maped to "utmfl" parameter
+    java_enabled -- Visitor's Java support, will be mapped to "utmje" parameter
+    screen_colour_depth -- Visitor's screen color depth, will be mapped to "utmsc" parameter
+    screen_resolution -- Visitor's screen resolution, will be mapped to "utmsr" parameter
+    '''
+    def __init__(self):
+        now = datetime.utcnow()
+
+        self.unique_id = None
+        self.first_visit_time = now
+        self.previous_visit_time = now
+        self.current_visit_time = now
+        self.visit_count = 1
+        self.ip_address = None
+        self.user_agent = None
+        self.locale = None
+        self.flash_version = None
+        self.java_enabled = None
+        self.screen_colour_depth = None
+        self.screen_resolution = None
+
+    def __setattr__(self, name, value):
+        if name == 'unique_id':
+            if value and (value < 0 or value > 0x7fffffff):
+                raise ValueError('Visitor unique ID has to be a 32-bit integer between 0 and 0x7fffffff')
+        object.__setattr__(self, name, value)
+
+    def __getattribute__(self, name):
+        if name == 'unique_id':
+            tmp = object.__getattribute__(self, name)
+            if tmp is None:
+                self.unique_id = self.generate_unique_id()
+        return object.__getattribute__(self, name)
+
+    def __getstate__(self):
+        state = self.__dict__
+        if state.get('user_agent') is None:
+            state['unique_id'] = self.generate_unique_id()
+
+        return state
+
+    def extract_from_utma(self, utma):
+        '''
+        Will extract information for the "unique_id", "first_visit_time", "previous_visit_time",
+        "current_visit_time" and "visit_count" properties from the given "__utma" cookie value.
+        '''
+        parts = utma.split('.')
+        if len(parts) != 6:
+            raise ValueError('The given "__utma" cookie value is invalid.')
+
+        self.unique_id = int(parts[1])
+        self.first_visit_time = utils.convert_ga_timestamp(parts[2])
+        self.previous_visit_time = utils.convert_ga_timestamp(parts[3])
+        self.current_visit_time = utils.convert_ga_timestamp(parts[4])
+        self.visit_count = int(parts[5])
+
+        return self
+
+    def extract_from_server_meta(self, meta):
+        '''
+        Will extract information for the "ip_address", "user_agent" and "locale"
+        properties from the given WSGI REQUEST META variable or equivalent.
+        '''
+        if 'REMOTE_ADDR' in meta and meta['REMOTE_ADDR']:
+            ip = None
+            for key in ('HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR'):
+                if key in meta and not ip:
+                    ips = meta.get(key, '').split(',')
+                    ip = ips[-1].strip()
+                    if not utils.is_valid_ip(ip):
+                        ip = ''
+                    if utils.is_private_ip(ip):
+                        ip = ''
+            if ip:
+                self.ip_address = ip
+
+        if 'HTTP_USER_AGENT' in meta and meta['HTTP_USER_AGENT']:
+            self.user_agent = meta['HTTP_USER_AGENT']
+
+        if 'HTTP_ACCEPT_LANGUAGE' in meta and meta['HTTP_ACCEPT_LANGUAGE']:
+            user_locals = []
+            matched_locales = utils.validate_locale(meta['HTTP_ACCEPT_LANGUAGE'])
+            if matched_locales:
+                lang_lst = map((lambda x: x.replace('-', '_')), (i[1] for i in matched_locales))
+                quality_lst = map((lambda x: x and x or 1), (float(i[4] and i[4] or '0') for i in matched_locales))
+                lang_quality_map = map((lambda x, y: (x, y)), lang_lst, quality_lst)
+                user_locals = [x[0] for x in sorted(lang_quality_map, key=itemgetter(1), reverse=True)]
+
+            if user_locals:
+                self.locale = user_locals[0]
+
+        return self
+
+    def generate_hash(self):
+        '''Generates a hashed value from user-specific properties.'''
+        tmpstr = "%s%s%s" % (self.user_agent, self.screen_resolution, self.screen_colour_depth)
+        return utils.generate_hash(tmpstr)
+
+    def generate_unique_id(self):
+        '''Generates a unique user ID from the current user-specific properties.'''
+        return ((utils.get_32bit_random_num() ^ self.generate_hash()) & 0x7fffffff)
+
+    def add_session(self, session):
+        '''
+        Updates the "previousVisitTime", "currentVisitTime" and "visitCount"
+        fields based on the given session object.
+        '''
+        start_time = session.start_time
+        if start_time != self.current_visit_time:
+            self.previous_visit_time = self.current_visit_time
+            self.current_visit_time = start_time
+            self.visit_count = self.visit_count + 1

Contents/Libraries/Shared/pyga/exceptions.py

@@ -0,0 +1,2 @@
+class ValidationError(Exception):
+    pass

Contents/Libraries/Shared/pyga/utils.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+
+import logging
+from random import randint
+import re
+import six
+import os
+from datetime import datetime
+
+__author__ = "Arun KR (kra3) <the1.arun@gmail.com>"
+__license__ = "Simplified BSD"
+
+RE_IP = re.compile(r'^[\d+]{1,3}\.[\d+]{1,3}\.[\d+]{1,3}\.[\d+]{1,3}$', re.I)
+RE_PRIV_IP = re.compile(r'^(?:127\.0\.0\.1|10\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[0-1])\.)')
+RE_LOCALE = re.compile(r'(^|\s*,\s*)([a-zA-Z]{1,8}(-[a-zA-Z]{1,8})*)\s*(;\s*q\s*=\s*(1(\.0{0,3})?|0(\.[0-9]{0,3})))?', re.I)
+RE_GA_ACCOUNT_ID = re.compile(r'^(UA|MO)-[0-9]*-[0-9]*$')
+RE_FIRST_THREE_OCTETS_OF_IP = re.compile(r'^((\d{1,3}\.){3})\d{1,3}$')
+
+def convert_ga_timestamp(timestamp_string):
+    timestamp = float(timestamp_string)
+    if timestamp > ((2 ** 31) - 1):
+        timestamp /= 1000
+    return datetime.utcfromtimestamp(timestamp)
+
+def get_32bit_random_num():
+    return randint(0, 0x7fffffff)
+
+def is_valid_ip(ip):
+    return True if RE_IP.match(str(ip)) else False
+
+def is_private_ip(ip):
+    return True if RE_PRIV_IP.match(str(ip)) else False
+
+def validate_locale(locale):
+    return RE_LOCALE.findall(str(locale))
+
+def is_valid_google_account(account):
+    return True if RE_GA_ACCOUNT_ID.match(str(account)) else False
+
+def generate_hash(tmpstr):
+    hash_val = 1
+
+    if tmpstr:
+        hash_val = 0
+        for ordinal in map(ord, tmpstr[::-1]):
+            hash_val = ((hash_val << 6) & 0xfffffff) + ordinal + (ordinal << 14)
+            left_most_7 = hash_val & 0xfe00000
+            if left_most_7 != 0:
+                hash_val ^= left_most_7 >> 21
+
+    return hash_val
+
+def anonymize_ip(ip):
+    if ip:
+        match = RE_FIRST_THREE_OCTETS_OF_IP.findall(str(ip))
+        if match:
+            return '%s%s' % (match[0][0], '0')
+
+    return ''
+
+def encode_uri_components(value):
+    '''Mimics Javascript's encodeURIComponent() function for consistency with the GA Javascript client.'''
+    return convert_to_uri_component_encoding(six.moves.urllib.parse.quote(value))
+
+def convert_to_uri_component_encoding(value):
+    return value.replace('%21', '!').replace('%2A', '*').replace('%27', "'").replace('%28', '(').replace('%29', ')')
+
+# Taken from expicient.com BJs repo.
+def stringify(s, stype=None, fn=None):
+    ''' Converts elements of a complex data structure to strings
+
+    The data structure can be a multi-tiered one - with tuples and lists etc
+    This method will loop through each and convert everything to string.
+    For example - it can be -
+    [[{'a1': {'a2': {'a3': ('a4', timedelta(0, 563)), 'a5': {'a6': datetime()}}}}]]
+    which will be converted to -
+    [[{'a1': {'a2': {'a3': ('a4', '0:09:23'), 'a5': {'a6': '2009-05-27 16:19:52.401500' }}}}]]
+
+    @param stype: If only one type of data element needs to be converted to
+        string without affecting others, stype can be used.
+        In the earlier example, if it is called with stringify(s, stype=datetime.timedelta)
+        the result would be
+        [[{'a1': {'a2': {'a3': ('a4', '0:09:23'), 'a5': {'a6': datetime() }}}}]]
+
+    Also, even though the name is stringify, any function can be run on it, based on
+    parameter fn. If fn is None, it will be stringified.
+
+    '''
+
+    if type(s) in [list, set, dict, tuple]:
+        if isinstance(s, dict):
+            for k in s:
+                s[k] = stringify(s[k], stype, fn)
+        elif type(s) in [list, set]:
+            for i, k in enumerate(s):
+                s[i] = stringify(k, stype, fn)
+        else: #tuple
+            tmp = []
+            for k in s:
+                tmp.append(stringify(k, stype, fn))
+            s = tuple(tmp)
+    else:
+        if fn:
+            if not stype or (stype == type(s)):
+                return fn(s)
+        else:
+            # To do str(s). But, str() can fail on unicode. So, use .encode instead
+            if not stype or (stype == type(s)):
+                try:
+                    return six.text_type(s)
+                    #return s.encode('ascii', 'replace')
+                except AttributeError:
+                    return str(s)
+                except UnicodeDecodeError:
+                    return s.decode('ascii', 'replace')
+    return s

Contents/Libraries/Shared/subliminal_patch/__init__.py

@@ -4,13 +4,15 @@ import subliminal
 import babelfish
 import logging
 
-# patch subliminal's subtitle encoding detection
+# patch subliminal's subtitle and provider base
 from .patch_subtitle import PatchedSubtitle
 subliminal.subtitle.Subtitle = PatchedSubtitle
 from subliminal.providers.addic7ed import Addic7edSubtitle
 from subliminal.providers.podnapisi import PodnapisiSubtitle
 from subliminal.providers.tvsubtitles import TVsubtitlesSubtitle
 from subliminal.providers.opensubtitles import OpenSubtitlesSubtitle
+
+# add our patched subtitle base classes
 setattr(Addic7edSubtitle, "__bases__", (PatchedSubtitle,))
 setattr(PodnapisiSubtitle, "__bases__", (PatchedSubtitle,))
 setattr(TVsubtitlesSubtitle, "__bases__", (PatchedSubtitle,))
@@ -19,13 +21,15 @@ setattr(OpenSubtitlesSubtitle, "__bases__", (PatchedSubtitle,))
 from .patch_provider_pool import PatchedProviderPool
 from .patch_video import patched_search_external_subtitles, scan_video
 from .patch_providers import addic7ed, podnapisi, tvsubtitles, opensubtitles
-from .patch_api import save_subtitles
+from .patch_api import save_subtitles, list_all_subtitles, download_subtitles
 
 # patch subliminal's ProviderPool
 subliminal.api.ProviderPool = PatchedProviderPool
 
-# patch subliminal's save_subtitles function
+# patch subliminal's functions
 subliminal.api.save_subtitles = save_subtitles
+subliminal.api.list_all_subtitles = list_all_subtitles
+subliminal.api.download_subtitles = download_subtitles
 
 # patch subliminal's subtitle classes
 def subtitleRepr(self):
@@ -55,6 +59,4 @@ subliminal.video.search_external_subtitles = patched_search_external_subtitles
 # patch subliminal's scan_video function
 subliminal.video.scan_video = scan_video
 
-subliminal.video.Episode.scores["boost"] = 40
-
 subliminal.video.Episode.scores["title"] = 0

Contents/Libraries/Shared/subliminal_patch/patch_api.py

@@ -2,13 +2,60 @@
 import os
 import logging
 from bs4 import UnicodeDammit
-from subliminal.api import get_subtitle_path, io
-from subzero.lib.io import get_viable_encoding
+from subliminal.api import get_subtitle_path, io, defaultdict
+from subliminal_patch.patch_provider_pool import PatchedProviderPool
 
 logger = logging.getLogger(__name__)
 
 
-def save_subtitles(video, subtitles, single=False, directory=None, encoding=None, encode_with=None):
+def download_subtitles(subtitles, **kwargs):
+    """Download :attr:`~subliminal.subtitle.Subtitle.content` of `subtitles`.
+
+    All other parameters are passed onwards to the :class:`ProviderPool` constructor.
+
+    :param subtitles: subtitles to download.
+    :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
+
+    """
+    with PatchedProviderPool(**kwargs) as pool:
+        for subtitle in subtitles:
+            logger.info('Downloading subtitle %r', subtitle)
+            pool.download_subtitle(subtitle)
+
+
+def list_all_subtitles(videos, languages, **kwargs):
+    """List all available subtitles.
+
+    The `videos` must pass the `languages` check of :func:`check_video`.
+
+    All other parameters are passed onwards to the :class:`ProviderPool` constructor.
+
+    :param videos: videos to list subtitles for.
+    :type videos: set of :class:`~subliminal.video.Video`
+    :param languages: languages to search for.
+    :type languages: set of :class:`~babelfish.language.Language`
+    :return: found subtitles per video.
+    :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
+
+    """
+    listed_subtitles = defaultdict(list)
+
+    # return immediatly if no video passed the checks
+    if not videos:
+        return listed_subtitles
+
+    # list subtitles
+    with PatchedProviderPool(**kwargs) as pool:
+        for video in videos:
+            logger.info('Listing subtitles for %r', video)
+            subtitles = pool.list_subtitles(video, languages - video.subtitle_languages)
+            listed_subtitles[video].extend(subtitles)
+            logger.info('Found %d subtitle(s)', len(subtitles))
+
+    return listed_subtitles
+
+
+def save_subtitles(video, subtitles, single=False, directory=None, encoding=None, encode_with=None, chmod=None):
     """Save subtitles on filesystem.
 
     Subtitles are saved in the order of the list. If a subtitle with a language has already been saved, other subtitles
@@ -64,6 +111,10 @@ def save_subtitles(video, subtitles, single=False, directory=None, encoding=None
             with io.open(subtitle_path, 'wb') as f:
                 f.write(content)
 
+            # change chmod if requested
+            if chmod:
+                os.chmod(subtitle_path, chmod)
+
             if single:
                 break
             continue
@@ -73,6 +124,10 @@ def save_subtitles(video, subtitles, single=False, directory=None, encoding=None
             with io.open(subtitle_path, 'w', encoding=encoding) as f:
                 f.write(subtitle.text)
 
+        # change chmod if requested
+        if chmod:
+            os.chmod(subtitle_path, chmod)
+
         saved_subtitles.append(subtitle)
 
         # check single

Contents/Libraries/Shared/subliminal_patch/patch_providers/addic7ed.py

@@ -2,10 +2,11 @@
 
 import logging
 import re
+import subliminal
 from random import randint
 from subliminal.providers.addic7ed import Addic7edProvider, Addic7edSubtitle, ParserBeautifulSoup, Language
 from subliminal.cache import SHOW_EXPIRATION_TIME, region
-from .mixins import PunctuationMixin
+from .mixins import PunctuationMixin, ProviderRetryMixin
 
 logger = logging.getLogger(__name__)
 
@@ -16,21 +17,24 @@ USE_BOOST = False
 
 
 class PatchedAddic7edSubtitle(Addic7edSubtitle):
-    def __init__(self, *args, **kwargs):
-        super(PatchedAddic7edSubtitle, self).__init__(*args, **kwargs)
+    def __init__(self, language, hearing_impaired, page_link, series, season, episode, title, year, version,
+                 download_link):
+        super(PatchedAddic7edSubtitle, self).__init__(language, hearing_impaired, page_link, series, season, episode,
+                                                      title, year, version, download_link)
+        self.release_info = version
 
     def get_matches(self, video, hearing_impaired=False):
         matches = super(PatchedAddic7edSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
-        if not USE_BOOST:
+        if not subliminal.video.Episode.scores["addic7ed_boost"]:
             return matches
 
         if {"series", "season", "episode", "year"}.issubset(matches) and "format" in matches:
-            matches.add("boost")
-            logger.info("Boosting Addic7ed subtitle")
+            matches.add("addic7ed_boost")
+            logger.info("Boosting Addic7ed subtitle by %s" % subliminal.video.Episode.scores["addic7ed_boost"])
         return matches
 
 
-class PatchedAddic7edProvider(PunctuationMixin, Addic7edProvider):
+class PatchedAddic7edProvider(PunctuationMixin, ProviderRetryMixin, Addic7edProvider):
     USE_ADDICTED_RANDOM_AGENTS = False
 
     def __init__(self, username=None, password=None, use_random_agents=False):
@@ -58,7 +62,7 @@ class PatchedAddic7edProvider(PunctuationMixin, Addic7edProvider):
         """
         # get the show page
         logger.info('Getting show ids')
-        r = self.session.get(self.server_url + 'shows.php', timeout=10)
+        r = self.retry(lambda: self.session.get(self.server_url + 'shows.php', timeout=10))
         r.raise_for_status()
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 
@@ -140,7 +144,7 @@ class PatchedAddic7edProvider(PunctuationMixin, Addic7edProvider):
 
         # make the search
         logger.info('Searching show ids with %r', params)
-        r = self.session.get(self.server_url + 'search.php', params=params, timeout=10)
+        r = self.retry(lambda: self.session.get(self.server_url + 'search.php', params=params, timeout=10))
         r.raise_for_status()
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 
@@ -167,7 +171,8 @@ class PatchedAddic7edProvider(PunctuationMixin, Addic7edProvider):
 
         # get the page of the season of the show
         logger.info('Getting the page of show id %d, season %d', show_id, season)
-        r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10)
+        r = self.retry(lambda: self.session.get(self.server_url + 'show/%d' % show_id,
+                                                params={'season': season}, timeout=10))
         r.raise_for_status()
         soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
 

Contents/Libraries/Shared/subliminal_patch/patch_providers/mixins.py

@@ -1,6 +1,10 @@
 # coding=utf-8
 
 import re
+import time
+import logging
+
+logger = logging.getLogger(__name__)
 
 clean_whitespace_re = re.compile(r'\s+')
 
@@ -20,3 +24,18 @@ class PunctuationMixin(object):
 
     def full_clean(self, s):
         return self.clean_whitespace(self.clean_punctuation(s))
+
+
+class ProviderRetryMixin(object):
+    def retry(self, f, amount=3, exc=Exception, retry_timeout=1):
+        i = 0
+        while i <= amount:
+            try:
+                return f()
+            except exc, e:
+                i += 1
+                if i == amount:
+                    raise
+
+            logger.debug(u"Retrying %s, try: %i/%i, exception: %s" % (self.__class__.__name__, i, amount, e))
+            time.sleep(retry_timeout)

Contents/Libraries/Shared/subliminal_patch/patch_providers/opensubtitles.py

@@ -5,7 +5,10 @@ import os
 
 from babelfish import Language
 from subliminal.exceptions import ConfigurationError
-from subliminal.providers.opensubtitles import OpenSubtitlesProvider, checked, get_version, __version__, OpenSubtitlesSubtitle, Episode
+from subliminal.providers.opensubtitles import OpenSubtitlesProvider, checked, get_version, __version__, \
+    OpenSubtitlesSubtitle, Episode, ServerProxy
+from mixins import ProviderRetryMixin
+from six.moves.xmlrpc_client import Transport
 
 logger = logging.getLogger(__name__)
 
@@ -18,6 +21,7 @@ class PatchedOpenSubtitlesSubtitle(OpenSubtitlesSubtitle):
                                                            movie_release_name, movie_year, movie_imdb_id, series_season, series_episode)
         self.query_parameters = query_parameters or {}
         self.fps = fps
+        self.release_info = movie_release_name
 
     def get_matches(self, video, hearing_impaired=False):
         matches = super(PatchedOpenSubtitlesSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
@@ -39,10 +43,24 @@ class PatchedOpenSubtitlesSubtitle(OpenSubtitlesSubtitle):
             # treat a tag match equally to a hash match
             logger.debug("Subtitle matched by tag, treating it as a hash-match. Tag: '%s'", self.query_parameters.get("tag", None))
             matches.add("hash")
+
         return matches
 
 
-class PatchedOpenSubtitlesProvider(OpenSubtitlesProvider):
+class TimeoutTransport(Transport):
+    """Timeout support for ``xmlrpc.client.SafeTransport``."""
+    def __init__(self, timeout, *args, **kwargs):
+        Transport.__init__(self, *args, **kwargs)
+        self.timeout = timeout
+
+    def make_connection(self, host):
+        c = Transport.make_connection(self, host)
+        c.timeout = self.timeout
+
+        return c
+
+
+class PatchedOpenSubtitlesProvider(ProviderRetryMixin, OpenSubtitlesProvider):
     def __init__(self, username=None, password=None, use_tag_search=False):
         if username is not None and password is None or username is None and password is not None:
             raise ConfigurationError('Username and password must be specified')
@@ -55,10 +73,16 @@ class PatchedOpenSubtitlesProvider(OpenSubtitlesProvider):
             logger.info("Using tag/exact filename search")
 
         super(PatchedOpenSubtitlesProvider, self).__init__()
+        self.server = ServerProxy('http://api.opensubtitles.org/xml-rpc', TimeoutTransport(10))
 
     def initialize(self):
         logger.info('Logging in')
-        response = checked(self.server.LogIn(self.username, self.password, 'eng', 'subliminal v%s' % get_version(__version__)))
+        # fixme: retry on SSLError
+        response = self.retry(
+            lambda: checked(
+                self.server.LogIn(self.username, self.password, 'eng', 'subliminal v%s' % get_version(__version__))
+            )
+        )
         self.token = response['token']
         logger.debug('Logged in with token %r', self.token)
 
@@ -70,6 +94,7 @@ class PatchedOpenSubtitlesProvider(OpenSubtitlesProvider):
 
          patch: query movies even if hash is known; add tag parameter
         """
+
         season = episode = None
         if isinstance(video, Episode):
             query = video.series
@@ -105,7 +130,7 @@ class PatchedOpenSubtitlesProvider(OpenSubtitlesProvider):
 
         # query the server
         logger.info('Searching subtitles %r', criteria)
-        response = checked(self.server.SearchSubtitles(self.token, criteria))
+        response = self.retry(lambda: checked(self.server.SearchSubtitles(self.token, criteria)))
         subtitles = []
 
         # exit if no data

Contents/Libraries/Shared/subliminal_patch/patch_providers/podnapisi.py

@@ -2,17 +2,38 @@
 
 import logging
 import io
+import re
+try:
+    from lxml import etree
+except ImportError:
+    try:
+        import xml.etree.cElementTree as etree
+    except ImportError:
+        import xml.etree.ElementTree as etree
+from babelfish import Language
 from zipfile import ZipFile
-from subliminal.providers.podnapisi import PodnapisiProvider, fix_line_ending, ProviderError
+from subliminal.providers.podnapisi import PodnapisiProvider, PodnapisiSubtitle, fix_line_ending, ProviderError
+from mixins import ProviderRetryMixin
 
 logger = logging.getLogger(__name__)
 
 
-class PatchedPodnapisiProvider(PodnapisiProvider):
+class PatchedPodnapisiSubtitle(PodnapisiSubtitle):
+    provider_name = 'podnapisi'
+
+    def __init__(self, language, hearing_impaired, page_link, pid, releases, title, season=None, episode=None,
+                 year=None):
+        super(PatchedPodnapisiSubtitle, self).__init__(language, hearing_impaired, page_link, pid, releases, title,
+                                                       season=season, episode=episode, year=year)
+        self.release_info = u", ".join(releases)
+
+
+class PatchedPodnapisiProvider(ProviderRetryMixin, PodnapisiProvider):
     def download_subtitle(self, subtitle):
         # download as a zip
         logger.info('Downloading subtitle %r', subtitle)
-        r = self.session.get(self.server_url + subtitle.pid + '/download', params={'container': 'zip'}, timeout=10)
+        r = self.retry(lambda: self.session.get(self.server_url + subtitle.pid + '/download',
+                                                params={'container': 'zip'}, timeout=10))
         r.raise_for_status()
 
         # open the zip
@@ -21,3 +42,69 @@ class PatchedPodnapisiProvider(PodnapisiProvider):
                 raise ProviderError('More than one file to unzip')
 
             subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
+
+    def query(self, language, keyword, season=None, episode=None, year=None):
+        # set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
+        params = {'sXML': 1, 'sL': str(language), 'sK': keyword}
+        is_episode = False
+        if season and episode:
+            is_episode = True
+            params['sTS'] = season
+            params['sTE'] = episode
+        if year:
+            params['sY'] = year
+
+        # loop over paginated results
+        logger.info('Searching subtitles %r', params)
+        subtitles = []
+        pids = set()
+        while True:
+            # query the server
+            xml = etree.fromstring(self.retry(lambda: self.session.get(self.server_url + 'search/old',
+                                                                       params=params, timeout=10).content))
+
+            # exit if no results
+            if not int(xml.find('pagination/results').text):
+                logger.debug('No subtitles found')
+                break
+
+            # loop over subtitles
+            for subtitle_xml in xml.findall('subtitle'):
+                # read xml elements
+                language = Language.fromietf(subtitle_xml.find('language').text)
+                hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
+                page_link = subtitle_xml.find('url').text
+                pid = subtitle_xml.find('pid').text
+                releases = []
+                if subtitle_xml.find('release').text:
+                    for release in subtitle_xml.find('release').text.split():
+                        releases.append(re.sub(r'\.+$', '', release))  # remove trailing dots
+                title = subtitle_xml.find('title').text
+                season = int(subtitle_xml.find('tvSeason').text)
+                episode = int(subtitle_xml.find('tvEpisode').text)
+                year = int(subtitle_xml.find('year').text)
+
+                if is_episode:
+                    subtitle = PatchedPodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+                                                 season=season, episode=episode, year=year)
+                else:
+                    subtitle = PatchedPodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+                                                 year=year)
+
+                # ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
+                if pid in pids:
+                    continue
+
+                logger.debug('Found subtitle %r', subtitle)
+                subtitles.append(subtitle)
+                pids.add(pid)
+
+            # stop on last page
+            if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
+                break
+
+            # increment current page
+            params['page'] = int(xml.find('pagination/current').text) + 1
+            logger.debug('Getting page %d', params['page'])
+
+        return subtitles

Contents/Libraries/Shared/subliminal_patch/patch_providers/tvsubtitles.py

@@ -2,10 +2,11 @@
 
 import re
 import logging
+from babelfish import Language
 from subliminal.providers import ParserBeautifulSoup
 from subliminal.cache import SHOW_EXPIRATION_TIME, region
-from subliminal.providers.tvsubtitles import TVsubtitlesProvider
-from .mixins import PunctuationMixin
+from subliminal.providers.tvsubtitles import TVsubtitlesProvider, TVsubtitlesSubtitle
+from .mixins import PunctuationMixin, ProviderRetryMixin
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +15,14 @@ logger = logging.getLogger(__name__)
 link_re = re.compile('^(?P<series>.+)(?: \(?\d{4}\)?| \((?:US|UK)\))? \((?P<first_year>\d{4})\d{4}\)$')
 
 
-class PatchedTVsubtitlesProvider(PunctuationMixin, TVsubtitlesProvider):
+class PatchedTVsubtitlesSubtitle(TVsubtitlesSubtitle):
+    def __init__(self, language, page_link, subtitle_id, series, season, episode, year, rip, release):
+        super(PatchedTVsubtitlesSubtitle, self).__init__(language, page_link, subtitle_id, series, season, episode,
+                                                         year, rip, release)
+        self.release_info = u"%s, %s" % (rip, release)
+
+
+class PatchedTVsubtitlesProvider(PunctuationMixin, ProviderRetryMixin, TVsubtitlesProvider):
     @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
     def search_show_id(self, series, year=None):
         """Search the show id from the `series` and `year`.
@@ -27,7 +35,7 @@ class PatchedTVsubtitlesProvider(PunctuationMixin, TVsubtitlesProvider):
         # make the search
         series_clean = self.clean_punctuation(series).lower()
         logger.info('Searching show id for %r', series_clean)
-        r = self.session.post(self.server_url + 'search.php', data={'q': series_clean}, timeout=10)
+        r = self.retry(lambda: self.session.post(self.server_url + 'search.php', data={'q': series_clean}, timeout=10))
         r.raise_for_status()
 
         # get the series out of the suggestions
@@ -48,3 +56,38 @@ class PatchedTVsubtitlesProvider(PunctuationMixin, TVsubtitlesProvider):
                 break
 
         return show_id
+
+    def query(self, series, season, episode, year=None):
+        # search the show id
+        show_id = self.search_show_id(series, year)
+        if show_id is None:
+            logger.error('No show id found for %r (%r)', series, {'year': year})
+            return []
+
+        # get the episode ids
+        episode_ids = self.retry(lambda: self.get_episode_ids(show_id, season))
+        if episode not in episode_ids:
+            logger.error('Episode %d not found', episode)
+            return []
+
+        # get the episode page
+        logger.info('Getting the page for episode %d', episode_ids[episode])
+        r = self.retry(lambda: self.session.get(self.server_url + 'episode-%d.html' % episode_ids[episode], timeout=10))
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # loop over subtitles rows
+        subtitles = []
+        for row in soup.select('.subtitlen'):
+            # read the item
+            language = Language.fromtvsubtitles(row.h5.img['src'][13:-4])
+            subtitle_id = int(row.parent['href'][10:-5])
+            page_link = self.server_url + 'subtitle-%d.html' % subtitle_id
+            rip = row.find('p', title='rip').text.strip() or None
+            release = row.find('p', title='release').text.strip() or None
+
+            subtitle = PatchedTVsubtitlesSubtitle(language, page_link, subtitle_id, series, season, episode, year, rip,
+                                           release)
+            logger.info('Found subtitle %s', subtitle)
+            subtitles.append(subtitle)
+
+        return subtitles

Contents/Libraries/Shared/subliminal_patch/patch_subtitle.py

@@ -35,8 +35,8 @@ def compute_score(matches, video, scores=None):
 
     is_episode = isinstance(video, Episode)
 
-    episode_hash_valid_if = {"series", "season", "episode"}
-    movie_hash_valid_if = {"title", "video_codec"}
+    episode_hash_valid_if = {"series", "season", "episode", "format"}
+    movie_hash_valid_if = {"video_codec", "format"}
 
     # remove equivalent match combinations
     if 'hash' in final_matches:
@@ -68,6 +68,8 @@ def compute_score(matches, video, scores=None):
 
 class PatchedSubtitle(Subtitle):
     storage_path = None
+    release_info = None
+    matches = None
 
     def guess_encoding(self):
         """Guess encoding using the language, falling back on chardet.
@@ -76,9 +78,8 @@ class PatchedSubtitle(Subtitle):
         :rtype: str
 
         """
-        logger.info('Guessing encoding for language %s', self.language)
+        logger.info('Guessing encoding for language %s', self.language.alpha3)
 
-        # always try utf-8 first
         encodings = ['utf-8']
 
         # add language-specific encodings
@@ -86,23 +87,32 @@ class PatchedSubtitle(Subtitle):
             encodings.extend(['gb18030', 'big5'])
         elif self.language.alpha3 == 'jpn':
             encodings.append('shift-jis')
-        elif self.language.alpha3 == 'ara':
+        elif self.language.alpha3 == 'tha':
+            encodings.append('tis-620')
+
+        # arabian/farsi
+        elif self.language.alpha3 in ('ara', 'fas', 'per'):
             encodings.append('windows-1256')
         elif self.language.alpha3 == 'heb':
             encodings.append('windows-1255')
         elif self.language.alpha3 == 'tur':
             encodings.extend(['iso-8859-9', 'windows-1254'])
 
+        # Greek
+        elif self.language.alpha3 in ('grc', 'gre', 'ell'):
+            encodings.extend(['windows-1253', 'cp1253', 'cp737', 'iso8859_7', 'cp875', 'cp869', 'iso2022_jp_2',
+                              'mac_greek'])
+
         # Polish, Czech, Slovak, Hungarian, Slovene, Bosnian, Croatian, Serbian (Latin script),
         # Romanian (before 1993 spelling reform) and Albanian
         elif self.language.alpha3 in ('pol', 'cze', 'svk', 'hun', 'svn', 'bih', 'hrv', 'srb', 'rou', 'alb'):
             # Eastern European Group 1
-            encodings.extend(['windows-1250'])
+            encodings.append('windows-1250')
 
         # Bulgarian, Serbian and Macedonian
         elif self.language.alpha3 in ('bul', 'srb', 'mkd'):
             # Eastern European Group 2
-            encodings.extend(['windows-1251'])
+            encodings.append('windows-1251')
         else:
             # Western European (windows-1252)
             encodings.append('latin-1')

Contents/Libraries/Shared/subliminal_patch/patch_video.py

@@ -5,8 +5,8 @@ import logging
 import traceback
 
 from babelfish import Error as BabelfishError
-from subliminal.video import SUBTITLE_EXTENSIONS, VIDEO_EXTENSIONS, Language, Video, EnzymeError, MKV, guess_file_info, hash_opensubtitles, \
-    hash_thesubdb
+from subliminal.video import SUBTITLE_EXTENSIONS, VIDEO_EXTENSIONS, Language, Video, EnzymeError, MKV, \
+    guess_file_info, hash_opensubtitles, hash_thesubdb
 
 logger = logging.getLogger(__name__)
 
@@ -80,6 +80,7 @@ def scan_video(path, subtitles=True, embedded_subtitles=True, hints=None, video_
     # patch: suggest video type to guessit beforehand
     """
     hints = hints or {}
+    video_type = hints.get("type")
 
     # check for non-existing path
     if not dont_use_actual_file and not os.path.exists(path):
@@ -92,34 +93,39 @@ def scan_video(path, subtitles=True, embedded_subtitles=True, hints=None, video_
     dirpath, filename = os.path.split(path)
 
     # hint guessit the filename itself and its 2 parent directories if we're an episode (most likely Series name/Season/filename), else only one
-    guess_from = os.path.join(*os.path.normpath(path).split(os.path.sep)[-3 if hints.get("type") == "episode" else -2:])
+    guess_from = os.path.join(*os.path.normpath(path).split(os.path.sep)[-3 if video_type == "episode" else -2:])
     hints = hints or {}
     logger.info('Scanning video (hints: %s) %r', hints, guess_from)
 
     # guess
-    try:
-        video = Video.fromguess(path, guess_file_info(guess_from, options=hints))
-        video.fps = video_fps
+    video = Video.fromguess(path, guess_file_info(guess_from, options=hints))
+    video.fps = video_fps
 
-        if dont_use_actual_file:
-            return video
+    # trust plex's series name
+    if video_type == "episode" and hints.get("expected_series"):
+        video.series = hints.get("expected_series")[0]
 
-        # size and hashes
-        video.size = os.path.getsize(path)
-        if video.size > 10485760:
-            logger.debug('Size is %d', video.size)
-            video.hashes['opensubtitles'] = hash_opensubtitles(path)
-            video.hashes['thesubdb'] = hash_thesubdb(path)
-            logger.debug('Computed hashes %r', video.hashes)
-        else:
-            logger.warning('Size is lower than 10MB: hashes not computed')
+    # trust plex's movie name
+    if video_type == "movie" and hints.get("expected_title"):
+        video.title = hints.get("expected_title")[0]
+
+    if dont_use_actual_file:
+        return video
+
+    # size and hashes
+    video.size = os.path.getsize(path)
+    if video.size > 10485760:
+        logger.debug('Size is %d', video.size)
+        video.hashes['opensubtitles'] = hash_opensubtitles(path)
+        video.hashes['thesubdb'] = hash_thesubdb(path)
+        logger.debug('Computed hashes %r', video.hashes)
+    else:
+        logger.warning('Size is lower than 10MB: hashes not computed')
+
+    # external subtitles
+    if subtitles:
+        video.subtitle_languages |= set(patched_search_external_subtitles(path).values())
 
-        # external subtitles
-        if subtitles:
-            video.subtitle_languages |= set(patched_search_external_subtitles(path).values())
-    except Exception:
-        logger.error("Something went wrong when running guessit: %s", traceback.format_exc())
-        return
 
     # video metadata with enzyme
     try:

Contents/Libraries/Shared/subzero/__init__.py

@@ -1,7 +1,5 @@
 # coding=utf-8
 
-from intent import intent
-
 OS_PLEX_USERAGENT = 'plexapp.com v9.0'
 
 DEPENDENCY_MODULE_NAMES = ['subliminal', 'subliminal_patch', 'enzyme', 'guessit', 'requests']

Contents/Libraries/Shared/subzero/analytics.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+
+import struct
+import binascii
+
+from pyga.requests import Event, Page, Tracker, Session, Visitor, Config
+
+
+def track_event(category=None, action=None, label=None, value=None, identifier=None, first_use=None, add=None,
+                noninteraction=True):
+    anonymousConfig = Config()
+    anonymousConfig.anonimize_ip_address = True
+
+    tracker = Tracker('UA-86466078-1', 'none', conf=anonymousConfig)
+    visitor = Visitor()
+
+    # convert the last 8 bytes of the machine identifier to an integer to get a "unique" user
+    visitor.unique_id = struct.unpack("!I", binascii.unhexlify(identifier[32:]))[0]/2
+
+    if add:
+        # add visitor's ip address (will be anonymized)
+        visitor.ip_address = add
+
+    if first_use:
+        visitor.first_visit_time = first_use
+
+    session = Session()
+    event = Event(category=category, action=action, label=label, value=value, noninteraction=noninteraction)
+    path = u"/" + u"/".join([category, action, label])
+    page = Page(path.lower())
+
+    tracker.track_event(event, session, visitor)
+    tracker.track_pageview(page, session, visitor)

Contents/Libraries/Shared/subzero/func.py

@@ -1,24 +0,0 @@
-# coding=utf-8
-import threading
-
-lock = threading.Lock()
-
-
-class Debouncer(object):
-    call_history = set()
-
-    def get_lookup_key(self, args, kwargs):
-        func_name = list(args).pop(0).__name__
-        return tuple([func_name] + [(key, value) for key, value in kwargs.iteritems()])
-
-    def __contains__(self, item):
-        args, kwargs = item
-        lookup = self.get_lookup_key(args, kwargs)
-        with lock:
-            return lookup in self.call_history
-
-    def add(self, args, kwargs):
-        with lock:
-            self.call_history.add(self.get_lookup_key(args, kwargs))
-
-debouncer = Debouncer()

Contents/Libraries/Shared/subzero/history_storage.py

@@ -0,0 +1,95 @@
+# coding=utf-8
+
+import datetime
+
+from subzero.lib.dict import DictProxy
+
+
+mode_map = {
+    "a": "auto",
+    "m": "manual",
+    "b": "auto-better"
+}
+
+
+class SubtitleHistoryItem(object):
+    item_title = None
+    section_title = None
+    rating_key = None
+    subtitle = None
+    time = None
+    mode = "a"
+
+    def __init__(self, item_title, rating_key, section_title=None, subtitle=subtitle, mode="a"):
+        self.item_title = item_title
+        self.section_title = section_title
+        self.rating_key = str(rating_key)
+        self.subtitle = subtitle
+        self.time = datetime.datetime.now()
+        self.mode = mode
+
+    @property
+    def title(self):
+        return u"%s: %s" % (self.section_title, self.item_title)
+
+    @property
+    def score(self):
+        return self.subtitle.score
+
+    @property
+    def provider_name(self):
+        return self.subtitle.provider_name
+
+    @property
+    def lang_name(self):
+        return self.subtitle.language.name
+
+    @property
+    def mode_verbose(self):
+        return mode_map.get(self.mode, "Unknown")
+
+    def __repr__(self):
+        return unicode(self)
+
+    def __unicode__(self):
+        return u"%s (Score: %s)" % (unicode(self.item_title), self.score)
+
+    def __str__(self):
+        return str(self.rating_key)
+
+    def __hash__(self):
+        return hash((self.rating_key, self.score))
+
+    def __eq__(self, other):
+        return (self.rating_key, self.score) == (other.rating_key, other.score)
+
+    def __ne__(self, other):
+        # Not strictly necessary, but to avoid having both x==y and x!=y
+        # True at the same time
+        return not (self == other)
+
+
+class SubtitleHistory(DictProxy):
+    store = "history"
+    size = 100
+
+    def __init__(self, storage, size=100):
+        super(SubtitleHistory, self).__init__(storage)
+        self.size = size
+
+    def setup_defaults(self):
+        return {"history_items": []}
+
+    def add(self, item_title, rating_key, section_title=None, subtitle=None, mode="a"):
+        # create copy
+        items = self.history_items[:]
+        item = SubtitleHistoryItem(item_title, rating_key, section_title=section_title, subtitle=subtitle, mode=mode)
+
+        # insert item
+        items.insert(0, item)
+
+        # clamp item amount
+        items = items[:self.size]
+
+        # store items
+        self.history_items = items

Contents/Libraries/Shared/subzero/intent.py

@@ -6,25 +6,16 @@ import threading
 lock = threading.Lock()
 
 
-class TempIntent(dict):
+class TempIntent(object):
     timeout = 1000  # milliseconds
     store = None
 
-    def __init__(self, timeout=1000):
+    def __init__(self, timeout=1000, store=None):
         self.timeout = timeout
-        with lock:
-            self.store = {}
+        if store is None:
+            raise NotImplementedError
 
-    def __getattr__(self, name):
-        if name in self:
-            return self[name]
-
-    def __setattr__(self, name, value):
-        self[name] = value
-
-    def __delattr__(self, name):
-        if name in self:
-            del self[name]
+        self.store = store
 
     def get(self, kind, *keys):
         with lock:
@@ -37,13 +28,15 @@ class TempIntent(dict):
                     continue
 
                 # valid kind?
-                if kind in self["store"]:
+                if kind in self.store:
                     now = datetime.datetime.now()
+                    key = str(key)
 
                     # iter all known kinds (previously created)
-                    for known_key in self["store"][kind].keys():
+                    for known_key in self.store[kind].keys():
                         # may need locking, for now just play it safe
-                        ends = self["store"][kind].get(known_key, None)
+                        data = self.store[kind].get(known_key, {})
+                        ends = data.get("timeout")
                         if not ends:
                             continue
 
@@ -57,7 +50,7 @@ class TempIntent(dict):
 
                         if timed_out:
                             try:
-                                del self["store"][kind][key]
+                                del self.store[kind][key]
                             except:
                                 continue
 
@@ -67,22 +60,42 @@ class TempIntent(dict):
 
     def resolve(self, kind, key):
         with lock:
-            if kind in self["store"] and key in self["store"][kind]:
-                del self["store"][kind][key]
+            if kind in self.store and key in self.store[kind]:
+                del self.store[kind][key]
                 return True
             return False
 
-    def set(self, kind, key, timeout=None):
+    def set(self, kind, key, data=None, timeout=None):
         with lock:
-            if kind not in self["store"]:
-                self["store"][kind] = {}
-            self["store"][kind][key] = datetime.datetime.now() + datetime.timedelta(milliseconds=timeout or self.timeout)
+            if kind not in self.store:
+                self.store[kind] = {}
+
+            key = str(key)
+            self.store[kind][key] = {
+                "data": data,
+                "timeout": datetime.datetime.now() + datetime.timedelta(milliseconds=timeout or self.timeout)
+            }
 
     def has(self, kind, key):
         with lock:
-            if kind not in self["store"]:
+            if kind not in self.store:
                 return False
-            return key in self["store"][kind]
+            return key in self.store[kind]
 
+    def cleanup(self):
+        now = datetime.datetime.now()
+        clear_all = False
+        for kind, info in self.store.items():
+            for key, intent_data in info.items():
+                # legacy intent data, clear everything
+                if not isinstance(intent_data, dict):
+                    clear_all = True
+                    continue
+
+                if now > intent_data["timeout"]:
+                    del self.store[kind][key]
+        if clear_all:
+            self.store.clear()
+
+        self.store.save()
 
-intent = TempIntent()

Contents/Libraries/Shared/subzero/lib/dict.py

@@ -10,11 +10,27 @@ class DictProxy(object):
 
         if self.store not in self.Dict or not self.Dict[self.store]:
             self.Dict[self.store] = self.setup_defaults()
+        self.save()
+        self.__initialized = True
 
     def __getattr__(self, name):
         if name in self.Dict[self.store]:
             return self.Dict[self.store][name]
-        return getattr(super(self.DictProxy, self), name)
+        return getattr(super(DictProxy, self), name)
+
+    def __setattr__(self, name, value):
+        if not self.__dict__.has_key(
+                '_DictProxy__initialized'):  # this test allows attributes to be set in the __init__ method
+            return object.__setattr__(self, name, value)
+
+        elif self.__dict__.has_key(name):  # any normal attributes are handled normally
+            object.__setattr__(self, name, value)
+
+        else:
+            if name in self.Dict[self.store]:
+                self.Dict[self.store][name] = value
+                return
+        object.__setattr__(self, name, value)
 
     def __cmp__(self, d):
         return cmp(self.Dict[self.store], d)
@@ -45,6 +61,9 @@ class DictProxy(object):
     def __delitem__(self, key):
         del self.Dict[self.store][key]
 
+    def save(self):
+        self.Dict.Save()
+
     def clear(self):
         del self.Dict[self.store]
         return None

Contents/Libraries/Shared/subzero/score_range.py

@@ -0,0 +1,22 @@
+# coding=utf-8
+
+import sys
+from itertools import chain, combinations, permutations
+
+from subliminal.video import Episode
+
+
+def permute(x):
+    return [base_score + i + j for i in x for j in x]
+
+if __name__ == "__main__":
+    scores = Episode.scores
+    base_score_keys = ["series", "season", "episode"]
+    leftover_keys = list(set(scores.keys()) - set(base_score_keys))
+    base_score = sum([val for key, val in scores.items() if key in base_score_keys])
+    leftover_scores = set([score for key, score in scores.items() if key in leftover_keys])
+    print "base score:", base_score
+    print "leftover:", sorted(set(leftover_scores))
+    # print sum_possible_scores(base_score, leftover_scores)
+    # print list(permutations(leftover_scores))
+    print ",\n".join(map(lambda x: '"%s"' % x, ["66"] + sorted(set(permute(leftover_scores)))))

README.md

@@ -1,6 +1,6 @@
 #Sub-Zero for Plex
 [![](https://img.shields.io/github/release/pannal/Sub-Zero.bundle.svg?style=flat)](https://github.com/pannal/Sub-Zero.bundle/releases)
-[![master](https://img.shields.io/badge/master-unstable-red.svg?maxAge=2592000)]()
+[![master](https://img.shields.io/badge/master-stable-green.svg?maxAge=2592000)]()
 [![Maintenance](https://img.shields.io/maintenance/yes/2016.svg?maxAge=2592000)]()
 
 ![logo](https://raw.githubusercontent.com/pannal/Sub-Zero.bundle/master/Contents/Resources/subzero.gif)
@@ -12,13 +12,24 @@ I've been receiving great support by [@ukdtom](https://github.com/ukdtom) recent
 He has created **[the Sub-Zero Wiki](https://github.com/pannal/Sub-Zero.bundle/wiki)**. Please have a look in case of any questions.
 
 ## Changelog
+1.4.11.781
+- core: cleanup, logging
+- core/menu: fix addic7ed display in manual subtitle list
+- core: use HTTP for OpenSubtitles instead of HTTPS because of current certificate errors
+- core: find better subtitles should now run smoothly even with replaced files (newer parts)
 
-1.3.33.522
 
-- core: fix library permission detection on windows; fixes #151
-- core: "Restrict to one language" now behaves like it should (one found subtitle of any language is treated as sufficient); fixes #149
-- core: add support for other subtitle formats such as ssa/ass/microdvd, convert to srt; fixes #138
-- core: hopefully more consistent force-refresh handling (intent); fixes #118
+1.4.10.769
+- core: hotfix for legacy intent storage regression
+
+1.4.10.768
+- core: automatically find better subtitles (configurable)
+- menu: display how the subtitle was downloaded (auto, manual, auto-better), in history menu
+- menu/core: correctly handle subtitle list for multiple languages
+- core: lower minimum series score to list subtitles for to 66
+- core: better matching of garbage filenames; we trust Plex now for the series name/movie title fully
+- core: add setting to specifically set the file permissions (chmod)
+
 
 [older changes](CHANGELOG.md)