update maintenance state in readme

core: refiners: tvdb: don't fail on bad firstAired date

CHANGELOG.md

@@ -1,3 +1,38 @@
+2.6.5.3183
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: don't fall back to default providers if none enabled
+- core: don't process any further if stream info is missing
+- core: support using mediainfo for retrieving MP4 MOV_TEXT subtitle stream titles (PMS bug)
+- core: fix embedded subtitle extraction in some cases (#681, #680)
+- core: scanning: add additional INFO logging for undetected languages
+- core: bazarr-backport: remove existing subtitle file, to support MergerFS
+- core: bazarr-backport: generic 10 minute throttling if uncaught exception occurs
+- providers: addic7ed: fix recaptcha solving; fix show ID retrieval (#681)
+- providers: addic7ed: add timeout on authentication error
+- providers: addic7ed: fix shows with dots in them (Mayans M.C.)
+- providers: addic7ed: fix detection of completed subtitle for non-english users (#686)
+- providers: addic7ed: add more timeouts in the login process
+- providers: argenteam: bazarr-backport: use new url; fixes
+
+
+2.6.5.3152
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: fix core issue possibly impacting results on OpenSubtitles in certain conditions
+- core: fix default values of opensubtitles-skip-wrong-fps, use_https; fix #676
+- core: fix for determining whether to search under certain circumstances; fixes #666
+- core: #664 fix missing language processing of multiple videos refreshed at once
+- core: #661 fix match strictness when determining preexisting external subtitles
+- providers: titlovi: New implementation of Titlovi using API (thanks @viking1304)
+
+
 2.6.5.3124
 
 subscene, addic7ed and titlovi

Contents/Code/__init__.py

@@ -21,20 +21,21 @@ import support
 import interface
 sys.modules["interface"] = interface
 
-from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
+from subzero.constants import OS_PLEX_USERAGENT
 from interface.menu import *
 from support.plex_media import media_to_videos, get_media_item_ids
+from support.extract import agent_extract_embedded
 from support.scanning import scan_videos
-from support.storage import save_subtitles, store_subtitle_info, get_subtitle_storage
+from support.storage import save_subtitles, store_subtitle_info
 from support.items import is_wanted
 from support.config import config
 from support.lib import get_intent
-from support.helpers import track_usage, get_title_for_video_metadata, get_identifier, cast_bool, \
-    audio_streams_match_languages
+from support.helpers import track_usage, get_title_for_video_metadata, get_identifier, cast_bool
 from support.history import get_history
 from support.data import dispatch_migrate
 from support.activities import activity
 from support.download import download_best_subtitles
+from support.localmedia import find_subtitles
 
 
 def Start():
@@ -96,61 +97,7 @@ def Start():
 
 def update_local_media(videos, ignore_parts_cleanup=None):
     for video in videos:
-        support.localmedia.find_subtitles(video["plex_part"], ignore_parts_cleanup=ignore_parts_cleanup)
-
-
-def agent_extract_embedded(video_part_map):
-    try:
-        subtitle_storage = get_subtitle_storage()
-
-        to_extract = []
-        item_count = 0
-
-        for scanned_video, part_info in video_part_map.iteritems():
-            plexapi_item = scanned_video.plexapi_metadata["item"]
-            stored_subs = subtitle_storage.load_or_new(plexapi_item)
-            valid_langs_in_media = audio_streams_match_languages(scanned_video, config.get_lang_list(ordered=True))
-
-            if not config.lang_list.difference(valid_langs_in_media):
-                Log.Debug("Skipping embedded subtitle extraction for %s, audio streams are in correct language(s)",
-                          plexapi_item.rating_key)
-                continue
-
-            for plexapi_part in get_all_parts(plexapi_item):
-                item_count = item_count + 1
-                used_one_unknown_stream = False
-                used_one_known_stream = False
-                for requested_language in config.lang_list:
-                    skip_unknown = used_one_unknown_stream or used_one_known_stream
-                    embedded_subs = stored_subs.get_by_provider(plexapi_part.id, requested_language, "embedded")
-                    current = stored_subs.get_any(plexapi_part.id, requested_language) or \
-                        requested_language in scanned_video.external_subtitle_languages
-
-                    if not embedded_subs:
-                        stream_data = get_embedded_subtitle_streams(plexapi_part, requested_language=requested_language,
-                                                                    skip_unknown=skip_unknown)
-
-                        if stream_data:
-                            stream = stream_data[0]["stream"]
-                            if stream_data[0]["is_unknown"]:
-                                used_one_unknown_stream = True
-                            else:
-                                used_one_known_stream = True
-
-                            to_extract.append(({scanned_video: part_info}, plexapi_part, str(stream.index),
-                                               str(requested_language), not current))
-
-                            if not cast_bool(Prefs["subtitles.search_after_autoextract"]):
-                                scanned_video.subtitle_languages.update({requested_language})
-                    else:
-                        Log.Debug("Skipping embedded subtitle extraction for %s, already got %r from %s",
-                                  plexapi_item.rating_key, requested_language, embedded_subs[0].id)
-        if to_extract:
-            Log.Info("Triggering extraction of %d embedded subtitles of %d items", len(to_extract), item_count)
-            Thread.Create(multi_extract_embedded, stream_list=to_extract, refresh=True, with_mods=True,
-                          single_thread=not config.advanced.auto_extract_multithread)
-    except:
-        Log.Error("Something went wrong when auto-extracting subtitles, continuing: %s", traceback.format_exc())
+        find_subtitles(video["plex_part"], ignore_parts_cleanup=ignore_parts_cleanup)
 
 
 class SubZeroAgent(object):

Contents/Code/interface/item_details.py

@@ -7,14 +7,16 @@ from subzero.language import Language
 
 from sub_mod import SubtitleModificationsMenu
 from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_incl_excl_options, get_item_task_data, \
-    set_refresh_menu_state, route, extract_embedded_sub
+    set_refresh_menu_state, route
+from support.extract import extract_embedded_sub
 
 from refresh_item import RefreshItem
 from subzero.constants import PREFIX
 from support.config import config, TEXT_SUBTITLE_EXTS
-from support.helpers import timestamp, df, get_language, display_language, get_language_from_stream, is_stream_forced
+from support.helpers import timestamp, df, get_language, display_language, get_language_from_stream
 from support.items import get_item_kind_from_rating_key, get_item, get_current_sub, get_item_title, save_stored_sub
-from support.plex_media import get_plex_metadata, get_part, get_embedded_subtitle_streams
+from support.plex_media import get_plex_metadata, get_part, get_embedded_subtitle_streams, is_stream_forced, \
+    update_stream_info
 from support.scanning import scan_videos
 from support.scheduler import scheduler
 from support.storage import get_subtitle_storage
@@ -118,6 +120,8 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
             if not os.path.exists(part.file):
                 continue
 
+            update_stream_info(part)
+
             part_id = str(part.id)
             part_index += 1
 
@@ -670,29 +674,28 @@ def ListEmbeddedSubsForItemMenu(**kwargs):
             stream = stream_data["stream"]
             is_forced = stream_data["is_forced"]
 
-            if language:
-                oc.add(DirectoryObject(
-                    key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(),
-                                 stream_index=str(stream.index), language=language, with_mods=True, **kwargs),
-                    title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s"
-                            u"%(stream_title)s with default mods",
-                            stream_index=stream.index,
-                            language=display_language(language),
-                            unknown_state=_(" (unknown)") if is_unknown else "",
-                            forced_state=_(" (forced)") if is_forced else "",
-                            stream_title=" (\"%s\")" % stream.title if stream.title else ""),
-                ))
-                oc.add(DirectoryObject(
-                    key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(),
-                                 stream_index=str(stream.index), language=language, **kwargs),
-                    title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s"
-                            u"%(stream_title)s",
-                            stream_index=stream.index,
-                            language=display_language(language),
-                            unknown_state=_(" (unknown)") if is_unknown else "",
-                            forced_state=_(" (forced)") if is_forced else "",
-                            stream_title=" (\"%s\")" % stream.title if stream.title else ""),
-                ))
+            oc.add(DirectoryObject(
+                key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(),
+                             stream_index=str(stream.index), language=language, with_mods=True, **kwargs),
+                title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s"
+                        u"%(stream_title)s with default mods",
+                        stream_index=stream.index,
+                        language=display_language(language),
+                        unknown_state=_(" (unknown)") if is_unknown else "",
+                        forced_state=_(" (forced)") if is_forced else "",
+                        stream_title=" (\"%s\")" % stream.title if stream.title else ""),
+            ))
+            oc.add(DirectoryObject(
+                key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(),
+                             stream_index=str(stream.index), language=language, **kwargs),
+                title=_(u"Extract stream %(stream_index)s, %(language)s%(unknown_state)s%(forced_state)s"
+                        u"%(stream_title)s",
+                        stream_index=stream.index,
+                        language=display_language(language),
+                        unknown_state=_(" (unknown)") if is_unknown else "",
+                        forced_state=_(" (forced)") if is_forced else "",
+                        stream_title=" (\"%s\")" % stream.title if stream.title else ""),
+            ))
     return oc
 
 

Contents/Code/interface/menu.py

@@ -12,19 +12,16 @@ from requests import HTTPError
 from item_details import ItemDetailsMenu
 from refresh_item import RefreshItem
 from menu_helpers import add_incl_excl_options, dig_tree, set_refresh_menu_state, \
-    default_thumb, debounce, ObjectContainer, SubFolderObjectContainer, route, \
-    extract_embedded_sub
+    default_thumb, debounce, ObjectContainer, SubFolderObjectContainer, route
 from main import fatality, InclExclMenu
 from advanced import DispatchRestart
 from subzero.constants import ART, PREFIX, DEPENDENCY_MODULE_NAMES
-from support.plex_media import get_all_parts, get_embedded_subtitle_streams
+from support.extract import season_extract_embedded
 from support.scheduler import scheduler
 from support.config import config
 from support.helpers import timestamp, df, display_language
 from support.ignore import get_decision_list
-from support.items import get_all_items, get_items_info, get_item_kind_from_rating_key, get_item, MI_KEY, \
-    get_item_title, get_item_thumb
-from support.storage import get_subtitle_storage
+from support.items import get_all_items, get_items_info, get_item_kind_from_rating_key, get_item, get_item_title
 from support.i18n import _
 
 # init GUI
@@ -174,53 +171,6 @@ def SeasonExtractEmbedded(**kwargs):
     return MetadataMenu(randomize=timestamp(), title=item_title, **kwargs)
 
 
-def multi_extract_embedded(stream_list, refresh=False, with_mods=False, single_thread=True, extract_mode="a",
-                           history_storage=None):
-    def execute():
-        for video_part_map, plexapi_part, stream_index, language, set_current in stream_list:
-            plexapi_item = video_part_map.keys()[0].plexapi_metadata["item"]
-
-            extract_embedded_sub(rating_key=plexapi_item.rating_key, part_id=plexapi_part.id,
-                                 plex_item=plexapi_item, part=plexapi_part, scanned_videos=video_part_map,
-                                 stream_index=stream_index, set_current=set_current,
-                                 language=language, with_mods=with_mods, refresh=refresh, extract_mode=extract_mode,
-                                 history_storage=history_storage)
-
-    if single_thread:
-        with Thread.Lock(key="extract_embedded"):
-            execute()
-    else:
-        execute()
-
-
-def season_extract_embedded(rating_key, requested_language, with_mods=False, force=False):
-    # get stored subtitle info for item id
-    subtitle_storage = get_subtitle_storage()
-
-    try:
-        for data in get_all_items(key="children", value=rating_key, base="library/metadata"):
-            item = get_item(data[MI_KEY])
-            if item:
-                stored_subs = subtitle_storage.load_or_new(item)
-                for part in get_all_parts(item):
-                    embedded_subs = stored_subs.get_by_provider(part.id, requested_language, "embedded")
-                    current = stored_subs.get_any(part.id, requested_language)
-                    if not embedded_subs or force:
-                        stream_data = get_embedded_subtitle_streams(part, requested_language=requested_language)
-                        if stream_data:
-                            stream = stream_data[0]["stream"]
-
-                            set_current = not current or force
-                            refresh = not current
-
-                            extract_embedded_sub(rating_key=item.rating_key, part_id=part.id,
-                                                 stream_index=str(stream.index), set_current=set_current,
-                                                 refresh=refresh, language=requested_language, with_mods=with_mods,
-                                                 extract_mode="m")
-    finally:
-        subtitle_storage.destroy()
-
-
 @route(PREFIX + '/ignore_list')
 def IgnoreListMenu():
     ref_list = get_decision_list()
@@ -368,7 +318,7 @@ def ValidatePrefs():
             "subtitle_destination_folder", "include", "include_exclude_paths", "include_exclude_sz_files",
             "new_style_cache", "dbm_supported", "lang_list", "providers", "normal_subs", "forced_only", "forced_also",
             "plex_transcoder", "refiner_settings", "unrar", "adv_cfg_path", "use_custom_dns",
-            "has_anticaptcha", "anticaptcha_cls"]:
+            "has_anticaptcha", "anticaptcha_cls", "mediainfo_bin"]:
 
         value = getattr(config, attr)
         if isinstance(value, dict):

Contents/Code/interface/menu_helpers.py

@@ -1,29 +1,16 @@
 # coding=utf-8
-import traceback
 import types
 import datetime
-import subprocess
-import os
-import operator
 
-from func import enable_channel_wrapper, route_wrapper, register_route_function
-from subzero.lib.io import get_viable_encoding
-from subzero.language import Language
+from func import enable_channel_wrapper, route_wrapper
 from support.i18n import is_localized_string, _
-from support.items import get_kind, get_item_thumb, get_item, get_item_kind_from_item, refresh_item
-from support.helpers import get_video_display_title, pad_title, display_language, quote_args, is_stream_forced, \
-    get_title_for_video_metadata, mswindows
-from support.history import get_history
+from support.items import get_item_thumb
+from support.helpers import get_video_display_title, pad_title
 from support.ignore import get_decision_list
 from support.lib import get_intent
 from support.config import config
 from subzero.constants import ICON_SUB, ICON
-from support.plex_media import get_part, get_plex_metadata
 from support.scheduler import scheduler
-from support.scanning import scan_videos
-from support.storage import save_subtitles
-
-from subliminal_patch.subtitle import ModifiedSubtitle
 
 default_thumb = R(ICON_SUB)
 main_icon = ICON if not config.is_development else "icon-dev.jpg"
@@ -156,89 +143,6 @@ def debounce(func):
     return func
 
 
-def extract_embedded_sub(**kwargs):
-    rating_key = kwargs["rating_key"]
-    part_id = kwargs.pop("part_id")
-    stream_index = kwargs.pop("stream_index")
-    with_mods = kwargs.pop("with_mods", False)
-    language = Language.fromietf(kwargs.pop("language"))
-    refresh = kwargs.pop("refresh", True)
-    set_current = kwargs.pop("set_current", True)
-
-    plex_item = kwargs.pop("plex_item", get_item(rating_key))
-    item_type = get_item_kind_from_item(plex_item)
-    part = kwargs.pop("part", get_part(plex_item, part_id))
-    scanned_videos = kwargs.pop("scanned_videos", None)
-    extract_mode = kwargs.pop("extract_mode", "a")
-
-    any_successful = False
-
-    if part:
-        if not scanned_videos:
-            metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item)
-            scanned_videos = scan_videos([metadata], ignore_all=True, skip_hashing=True)
-
-        for stream in part.streams:
-            # subtitle stream
-            if str(stream.index) == stream_index:
-                is_forced = is_stream_forced(stream)
-                bn = os.path.basename(part.file)
-
-                set_refresh_menu_state(_(u"Extracting subtitle %(stream_index)s of %(filename)s",
-                                         stream_index=stream_index,
-                                         filename=bn))
-                Log.Info(u"Extracting stream %s (%s) of %s", stream_index, str(language), bn)
-
-                out_codec = stream.codec if stream.codec != "mov_text" else "srt"
-
-                args = [
-                    config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", out_codec, "-"
-                ]
-
-                cmdline = quote_args(args)
-                Log.Debug(u"Calling: %s", cmdline)
-                if mswindows:
-                    Log.Debug("MSWindows: Fixing encoding")
-                    cmdline = cmdline.encode("mbcs")
-
-                output = None
-                try:
-                    output = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True)
-                except:
-                    Log.Error("Extraction failed: %s", traceback.format_exc())
-
-                if output:
-                    subtitle = ModifiedSubtitle(language, mods=config.default_mods if with_mods else None)
-                    subtitle.content = output
-                    subtitle.provider_name = "embedded"
-                    subtitle.id = "stream_%s" % stream_index
-                    subtitle.score = 0
-                    subtitle.set_encoding("utf-8")
-
-                    # fixme: speedup video; only video.name is needed
-                    video = scanned_videos.keys()[0]
-                    save_successful = save_subtitles(scanned_videos, {video: [subtitle]}, mode="m",
-                                                     set_current=set_current)
-                    set_refresh_menu_state(None)
-
-                    if save_successful and refresh:
-                        refresh_item(rating_key)
-
-                    # add item to history
-                    item_title = get_title_for_video_metadata(video.plexapi_metadata,
-                                                              add_section_title=False, add_episode_title=True)
-
-                    history = get_history()
-                    history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
-                                thumb=video.plexapi_metadata["super_thumb"],
-                                subtitle=subtitle, mode=extract_mode)
-                    history.destroy()
-
-                    any_successful = True
-
-    return any_successful
-
-
 class SZObjectContainer(ObjectContainer):
     def __init__(self, *args, **kwargs):
         skip_pin_lock = kwargs.pop("skip_pin_lock", False)

Contents/Code/support/__init__.py

@@ -19,6 +19,10 @@ sys.modules["support.i18n"] = i18n
 
 helpers._ = i18n._
 
+import history
+
+sys.modules["support.history"] = history
+
 import plex_media
 sys.modules["support.plex_media"] = plex_media
 
@@ -49,6 +53,10 @@ import missing_subtitles
 
 sys.modules["support.missing_subtitles"] = missing_subtitles
 
+import extract
+
+sys.modules["support.extract"] = extract
+
 import tasks
 
 sys.modules["support.tasks"] = tasks
@@ -57,10 +65,6 @@ import ignore
 
 sys.modules["support.ignore"] = ignore
 
-import history
-
-sys.modules["support.history"] = history
-
 import data
 
 sys.modules["support.data"] = data

Contents/Code/support/config.py

@@ -8,6 +8,8 @@ import sys
 import rarfile
 import jstyleson
 import datetime
+import stat
+import traceback
 
 import subliminal
 import subliminal_patch
@@ -22,6 +24,7 @@ from subzero.language import Language
 from subliminal.cli import MutexLock
 from subzero.lib.io import FileIO, get_viable_encoding
 from subzero.lib.dict import Dicked
+from subzero.lib.which import find_executable
 from subzero.util import get_root_path
 from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW, MEDIA_TYPE_TO_STRING
 from subzero.prefs import get_user_prefs, update_user_prefs
@@ -66,6 +69,7 @@ PROVIDER_THROTTLE_MAP = {
         DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
         ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
         APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
+        AuthenticationError: (datetime.timedelta(hours=2), "2 hours"),
     },
     "opensubtitles": {
         TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
@@ -75,6 +79,7 @@ PROVIDER_THROTTLE_MAP = {
     "addic7ed": {
         DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
         TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
+        AuthenticationError: (datetime.timedelta(hours=24), "24 hours"),
     }
 }
 
@@ -153,6 +158,7 @@ class Config(object):
     anticaptcha_token = None
     anticaptcha_cls = None
     has_anticaptcha = False
+    mediainfo_bin = None
 
     store_recently_played_amount = 40
 
@@ -239,6 +245,8 @@ class Config(object):
         self.embedded_auto_extract = cast_bool(Prefs["subtitles.embedded.autoextract"])
         self.ietf_as_alpha3 = cast_bool(Prefs["subtitles.language.ietf_normalize"])
         self.use_custom_dns = self.parse_custom_dns()
+        if not self.advanced.dont_use_mediainfo_mp4:
+            self.mediainfo_bin = self.advanced.mediainfo_bin or find_executable("mediainfo")
         self.initialized = True
 
     def migrate_prefs(self):
@@ -323,6 +331,19 @@ class Config(object):
         for exe in try_executables:
             rarfile.UNRAR_TOOL = exe
             rarfile.ORIG_UNRAR_TOOL = exe
+            if os.path.isfile(exe) and not os.access(exe, os.X_OK):
+                st = os.stat(exe)
+                try:
+                    Log.Debug("setting generic executable permissions for %s", exe)
+                    # fixme: too broad?
+                    os.chmod(exe, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+                except:
+                    Log.Debug("failed setting generic executable permissions for %s: %s", exe, traceback.format_exc())
+                    try:
+                        Log.Debug("setting executable permissions for %s", exe)
+                        os.chmod(exe, st.st_mode | stat.S_IEXEC)
+                    except:
+                        Log.Debug("failed setting executable permissions for %s: %s", exe, traceback.format_exc())
             try:
                 rarfile.custom_check([rarfile.UNRAR_TOOL], True)
             except:
@@ -774,7 +795,9 @@ class Config(object):
                      'argenteam': cast_bool(Prefs['provider.argenteam.enabled']),
                      'subscenter': False,
                      'assrt': cast_bool(Prefs['provider.assrt.enabled']),
-                     }
+                     'bsplayer': cast_bool(Prefs['provider.bsplayer.enabled']),
+                     'screwzira': cast_bool(Prefs['provider.screwzira.enabled']),
+                }
 
     @property
     def providers_enabled(self):
@@ -803,6 +826,8 @@ class Config(object):
             providers["assrt"] = False
             providers["subscene"] = False
             providers["napisy24"] = False
+            providers["bsplayer"] = False
+            providers["screwzira"] = False
             providers_forced_off = dict(providers)
 
         if not self.unrar and providers["legendastv"]:
@@ -903,10 +928,10 @@ class Config(object):
         throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
             PROVIDER_THROTTLE_MAP["default"].get(cls, None)
 
-        if not throttle_data:
-            return
-
-        throttle_delta, throttle_description = throttle_data
+        if throttle_data:
+            throttle_delta, throttle_description = throttle_data
+        else:
+            throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes"
 
         if "provider_throttle" not in Dict:
             Dict["provider_throttle"] = {}

Contents/Code/support/extract.py

@@ -0,0 +1,208 @@
+# coding=utf-8
+import os
+import subprocess
+import traceback
+
+from support.helpers import quote_args, mswindows, get_title_for_video_metadata, cast_bool, \
+    audio_streams_match_languages
+from support.i18n import _
+from support.items import get_item_kind_from_item, refresh_item, get_all_items, get_item, MI_KEY
+from support.storage import get_subtitle_storage, save_subtitles
+from support.config import config
+from support.history import get_history
+from support.plex_media import get_all_parts, get_embedded_subtitle_streams, get_part, get_plex_metadata, \
+    update_stream_info, is_stream_forced
+from support.scanning import scan_videos
+from subzero.language import Language
+from subliminal_patch.subtitle import ModifiedSubtitle
+
+
+def agent_extract_embedded(video_part_map, set_as_existing=False):
+    try:
+        subtitle_storage = get_subtitle_storage()
+
+        to_extract = []
+        item_count = 0
+
+        threads = []
+
+        for scanned_video, part_info in video_part_map.iteritems():
+            plexapi_item = scanned_video.plexapi_metadata["item"]
+            stored_subs = subtitle_storage.load_or_new(plexapi_item)
+            valid_langs_in_media = \
+                audio_streams_match_languages(scanned_video, config.get_lang_list(ordered=True))
+
+            if not config.lang_list.difference(valid_langs_in_media):
+                Log.Debug("Skipping embedded subtitle extraction for %s, audio streams are in correct language(s)",
+                          plexapi_item.rating_key)
+                continue
+
+            for plexapi_part in get_all_parts(plexapi_item):
+                item_count = item_count + 1
+                used_one_unknown_stream = False
+                used_one_known_stream = False
+                for requested_language in config.lang_list:
+                    skip_unknown = used_one_unknown_stream or used_one_known_stream
+                    embedded_subs = stored_subs.get_by_provider(plexapi_part.id, requested_language, "embedded")
+                    current = stored_subs.get_any(plexapi_part.id, requested_language) or \
+                        requested_language in scanned_video.external_subtitle_languages
+
+                    if not embedded_subs:
+                        stream_data = get_embedded_subtitle_streams(plexapi_part, requested_language=requested_language,
+                                                                    skip_unknown=skip_unknown)
+
+                        if stream_data and stream_data[0]["language"]:
+                            stream = stream_data[0]["stream"]
+                            if stream_data[0]["is_unknown"]:
+                                used_one_unknown_stream = True
+                            else:
+                                used_one_known_stream = True
+
+                            to_extract.append(({scanned_video: part_info}, plexapi_part, str(stream.index),
+                                               str(requested_language), not current))
+
+                            if not cast_bool(Prefs["subtitles.search_after_autoextract"]) or set_as_existing:
+                                scanned_video.subtitle_languages.update({requested_language})
+                    else:
+                        Log.Debug("Skipping embedded subtitle extraction for %s, already got %r from %s",
+                                  plexapi_item.rating_key, requested_language, embedded_subs[0].id)
+        if to_extract:
+            Log.Info("Triggering extraction of %d embedded subtitles of %d items", len(to_extract), item_count)
+            threads.append(Thread.Create(multi_extract_embedded, stream_list=to_extract, refresh=True, with_mods=True,
+                                         single_thread=not config.advanced.auto_extract_multithread))
+            return threads
+    except:
+        Log.Error("Something went wrong when auto-extracting subtitles, continuing: %s", traceback.format_exc())
+
+
+def multi_extract_embedded(stream_list, refresh=False, with_mods=False, single_thread=True, extract_mode="a",
+                           history_storage=None):
+    def execute():
+        for video_part_map, plexapi_part, stream_index, language, set_current in stream_list:
+            plexapi_item = video_part_map.keys()[0].plexapi_metadata["item"]
+
+            extract_embedded_sub(rating_key=plexapi_item.rating_key, part_id=plexapi_part.id,
+                                 plex_item=plexapi_item, part=plexapi_part, scanned_videos=video_part_map,
+                                 stream_index=stream_index, set_current=set_current,
+                                 language=language, with_mods=with_mods, refresh=refresh, extract_mode=extract_mode,
+                                 history_storage=history_storage)
+
+    if single_thread:
+        with Thread.Lock(key="extract_embedded"):
+            execute()
+    else:
+        execute()
+
+
+def season_extract_embedded(rating_key, requested_language, with_mods=False, force=False):
+    # get stored subtitle info for item id
+    subtitle_storage = get_subtitle_storage()
+
+    try:
+        for data in get_all_items(key="children", value=rating_key, base="library/metadata"):
+            item = get_item(data[MI_KEY])
+            if item:
+                stored_subs = subtitle_storage.load_or_new(item)
+                for part in get_all_parts(item):
+                    embedded_subs = stored_subs.get_by_provider(part.id, requested_language, "embedded")
+                    current = stored_subs.get_any(part.id, requested_language)
+                    if not embedded_subs or force:
+                        stream_data = get_embedded_subtitle_streams(part, requested_language=requested_language)
+                        if stream_data:
+                            stream = stream_data[0]["stream"]
+
+                            set_current = not current or force
+                            refresh = not current
+
+                            extract_embedded_sub(rating_key=item.rating_key, part_id=part.id,
+                                                 stream_index=str(stream.index), set_current=set_current,
+                                                 refresh=refresh, language=requested_language, with_mods=with_mods,
+                                                 extract_mode="m")
+    finally:
+        subtitle_storage.destroy()
+
+
+def extract_embedded_sub(**kwargs):
+    rating_key = kwargs["rating_key"]
+    part_id = kwargs.pop("part_id")
+    stream_index = kwargs.pop("stream_index")
+    with_mods = kwargs.pop("with_mods", False)
+    language = Language.fromietf(kwargs.pop("language"))
+    refresh = kwargs.pop("refresh", True)
+    set_current = kwargs.pop("set_current", True)
+
+    plex_item = kwargs.pop("plex_item", get_item(rating_key))
+    item_type = get_item_kind_from_item(plex_item)
+    part = kwargs.pop("part", get_part(plex_item, part_id))
+    scanned_videos = kwargs.pop("scanned_videos", None)
+    extract_mode = kwargs.pop("extract_mode", "a")
+
+    any_successful = False
+
+    from interface.menu_helpers import set_refresh_menu_state
+
+    if part:
+        if not scanned_videos:
+            metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item)
+            scanned_videos = scan_videos([metadata], ignore_all=True, skip_hashing=True)
+
+        update_stream_info(part)
+        for stream in part.streams:
+            # subtitle stream
+            if str(stream.index) == stream_index:
+                is_forced = is_stream_forced(stream)
+                bn = os.path.basename(part.file)
+
+                set_refresh_menu_state(_(u"Extracting subtitle %(stream_index)s of %(filename)s",
+                                         stream_index=stream_index,
+                                         filename=bn))
+                Log.Info(u"Extracting stream %s (%s) of %s", stream_index, str(language), bn)
+
+                out_codec = stream.codec if stream.codec != "mov_text" else "srt"
+
+                args = [
+                    config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", out_codec, "-"
+                ]
+
+                cmdline = quote_args(args)
+                Log.Debug(u"Calling: %s", cmdline)
+                if mswindows:
+                    Log.Debug("MSWindows: Fixing encoding")
+                    cmdline = cmdline.encode("mbcs")
+
+                output = None
+                try:
+                    output = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True)
+                except:
+                    Log.Error("Extraction failed: %s", traceback.format_exc())
+
+                if output:
+                    subtitle = ModifiedSubtitle(language, mods=config.default_mods if with_mods else None)
+                    subtitle.content = output
+                    subtitle.provider_name = "embedded"
+                    subtitle.id = "stream_%s" % stream_index
+                    subtitle.score = 0
+                    subtitle.set_encoding("utf-8")
+
+                    # fixme: speedup video; only video.name is needed
+                    video = scanned_videos.keys()[0]
+                    save_successful = save_subtitles(scanned_videos, {video: [subtitle]}, mode="m",
+                                                     set_current=set_current)
+                    set_refresh_menu_state(None)
+
+                    if save_successful and refresh:
+                        refresh_item(rating_key)
+
+                    # add item to history
+                    item_title = get_title_for_video_metadata(video.plexapi_metadata,
+                                                              add_section_title=False, add_episode_title=True)
+
+                    history = get_history()
+                    history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
+                                thumb=video.plexapi_metadata["super_thumb"],
+                                subtitle=subtitle, mode=extract_mode)
+                    history.destroy()
+
+                    any_successful = True
+
+    return any_successful

Contents/Code/support/helpers.py

@@ -437,17 +437,10 @@ def get_language(lang_short):
 
 
 def display_language(l):
+    if not l:
+        return "Unknown"
     return _(str(l.basename).lower()) + ((u" (%s)" % _("forced")) if l.forced else "")
 
 
-def is_stream_forced(stream):
-    stream_title = getattr(stream, "title", "") or ""
-    forced = getattr(stream, "forced", False)
-    if not forced and stream_title and "forced" in stream_title.strip().lower():
-        forced = True
-
-    return forced
-
-
 class PartUnknownException(Exception):
     pass

Contents/Code/support/missing_subtitles.py

@@ -7,7 +7,8 @@ import os
 from babelfish import LanguageReverseError
 
 from support.config import config, TEXT_SUBTITLE_EXTS
-from support.helpers import get_plex_item_display_title, cast_bool, get_language_from_stream, is_stream_forced
+from support.helpers import get_plex_item_display_title, cast_bool, get_language_from_stream
+from support.plex_media import is_stream_forced, update_stream_info
 from support.items import get_item
 from support.lib import Plex
 from support.storage import get_subtitle_storage
@@ -35,7 +36,7 @@ def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_t
     for media in item.media:
         existing_subs = {"internal": [], "external": [], "own_external": [], "count": 0}
         for part in media.parts:
-
+            update_stream_info(part)
             # did we already download an external subtitle before?
             if subtitle_target_dir and stored_subs:
                 for language in languages_set:

Contents/Code/support/plex_media.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 
 import os
+import subprocess
 
 import helpers
 from items import get_item
@@ -26,6 +27,9 @@ tvdb_guid_identifier = "com.plexapp.agents.thetvdb://"
 
 
 def get_plexapi_stream_info(plex_item, part_id=None):
+    if not plex_item:
+        return
+
     d = {"stream": {}}
     data = d["stream"]
 
@@ -100,6 +104,9 @@ def media_to_videos(media, kind="series"):
                 plex_episode = get_item(ep.id)
                 stream_info = get_plexapi_stream_info(plex_episode)
 
+                if not stream_info:
+                    continue
+
                 for item in media.seasons[season].episodes[episode].items:
                     for part in item.parts:
                         videos.append(
@@ -121,22 +128,24 @@ def media_to_videos(media, kind="series"):
                         )
     else:
         stream_info = get_plexapi_stream_info(plex_item)
-        imdb_id = None
-        if imdb_guid_identifier in media.guid:
-            imdb_id = media.guid[len(imdb_guid_identifier):].split("?")[0]
-        for item in media.items:
-            for part in item.parts:
-                videos.append(
-                    get_metadata_dict(plex_item, part, dict(stream_info, **{"plex_part": part, "type": "movie",
-                                                                             "title": media.title, "id": media.id,
-                                                                             "super_thumb": plex_item.thumb,
-                                                                             "series_id": None, "year": year,
-                                                                             "season_id": None, "imdb_id": imdb_id,
-                                                                             "original_title": original_title,
-                                                                             "series_tvdb_id": None, "tvdb_id": None,
-                                                                             "section": plex_item.section.title})
-                                      )
-                )
+
+        if stream_info:
+            imdb_id = None
+            if imdb_guid_identifier in media.guid:
+                imdb_id = media.guid[len(imdb_guid_identifier):].split("?")[0]
+            for item in media.items:
+                for part in item.parts:
+                    videos.append(
+                        get_metadata_dict(plex_item, part, dict(stream_info, **{"plex_part": part, "type": "movie",
+                                                                                 "title": media.title, "id": media.id,
+                                                                                 "super_thumb": plex_item.thumb,
+                                                                                 "series_id": None, "year": year,
+                                                                                 "season_id": None, "imdb_id": imdb_id,
+                                                                                 "original_title": original_title,
+                                                                                 "series_tvdb_id": None, "tvdb_id": None,
+                                                                                 "section": plex_item.section.title})
+                                          )
+                    )
     return videos
 
 
@@ -174,16 +183,59 @@ def get_all_parts(plex_item):
     return parts
 
 
+def update_stream_info(part):
+    try:
+        return update_stream_info_(part)
+    except:
+        Log.Exception("Getting Mediainfo failed for: %s", part.file)
+
+
+def update_stream_info_(part):
+    if config.mediainfo_bin and part.container == "mp4":
+        cmdline = '%s --Inform="Text;-%%ID%%_%%Title%%" %s' % (config.mediainfo_bin, helpers.quote(part.file))
+        result = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True)
+        if result:
+            try:
+                stream_titles = {}
+                for pair in result[1:].split("-"):
+                    sid, title = pair.split("_")
+                    stream_titles[int(sid.strip())] = title.strip()
+            except:
+                pass
+            else:
+                filled = []
+                for stream in part.streams:
+                    if stream.index is None:
+                        Log.Debug("Found stream with no index: %r", stream)
+
+                    index = stream.index+1 if stream.index is not None else 1
+                    if index in stream_titles:
+                        stream.title = stream_titles[index]
+                        filled.append(index-1)
+                if filled:
+                    Log.Debug("Filled missing MP4 stream title info for streams: %s", filled)
+
+
+def is_stream_forced(stream):
+    stream_title = getattr(stream, "title", "") or ""
+    forced = getattr(stream, "forced", False)
+    if not forced and stream_title and "forced" in stream_title.strip().lower():
+        forced = True
+
+    return forced
+
+
 def get_embedded_subtitle_streams(part, requested_language=None, skip_duplicate_unknown=True, skip_unknown=False):
     streams = []
     streams_unknown = []
     all_streams = []
     has_unknown = False
     found_requested_language = False
+    update_stream_info(part)
     for stream in part.streams:
         # subtitle stream
         if stream.stream_type == 3 and not stream.stream_key and stream.codec in TEXT_SUBTITLE_EXTS:
-            is_forced = helpers.is_stream_forced(stream)
+            is_forced = is_stream_forced(stream)
             language = helpers.get_language_from_stream(stream.language_code)
             if language:
                 language = Language.rebuild(language, forced=is_forced)
@@ -194,14 +246,14 @@ def get_embedded_subtitle_streams(part, requested_language=None, skip_duplicate_
 
             if not language:
                 # only consider first unknown subtitle stream
-                if requested_language and config.treat_und_as_first:
+                if config.treat_und_as_first:
                     if has_unknown and skip_duplicate_unknown:
                         Log.Debug("skipping duplicate unknown")
                         continue
 
                     language = Language.rebuild(list(config.lang_list)[0], forced=is_forced)
                 else:
-                    language = Language("unk")
+                    language = None
                 is_unknown = True
                 has_unknown = True
                 stream_data = {"stream": stream, "is_unknown": is_unknown, "language": language,
@@ -259,6 +311,9 @@ def get_plex_metadata(rating_key, part_id, item_type, plex_item=None):
 
     stream_info = get_plexapi_stream_info(plex_item, part_id)
 
+    if not stream_info:
+        return
+
     # get normalized metadata
     # fixme: duplicated logic of media_to_videos
     if item_type == "episode":
@@ -380,3 +435,4 @@ class PMSMediaProxy(object):
 
             m = m.children[0]
         return parts
+

Contents/Code/support/scanning.py

@@ -4,7 +4,7 @@ import helpers
 from babelfish.exceptions import LanguageError
 
 from support.lib import Plex, get_intent
-from support.plex_media import get_stream_fps
+from support.plex_media import get_stream_fps, is_stream_forced, update_stream_info
 from support.storage import get_subtitle_storage
 from support.config import config, TEXT_SUBTITLE_EXTS
 from support.subtitlehelpers import get_subtitles_from_metadata
@@ -29,7 +29,7 @@ def prepare_video(pms_video_info, ignore_all=False, hints=None, rating_key=None,
     if ignore_all:
         Log.Debug("Force refresh intended.")
 
-    Log.Debug("Detecting streams: %s, external_subtitles=%s, embedded_subtitles=%s" % (
+    Log.Debug("Detecting streams: %s, account_for_external_subtitles=%s, account_for_embedded_subtitles=%s" % (
         plex_part.file, external_subtitles, embedded_subtitles))
 
     known_embedded = []
@@ -46,23 +46,25 @@ def prepare_video(pms_video_info, ignore_all=False, hints=None, rating_key=None,
     # fixme: skip the whole scanning process if known_embedded == wanted languages?
     audio_languages = []
     if plexpy_part:
+        update_stream_info(plexpy_part)
         for stream in plexpy_part.streams:
             if stream.stream_type == 2:
                 lang = None
                 try:
                     lang = language_from_stream(stream.language_code)
                 except LanguageError:
-                    Log.Debug("Couldn't detect embedded audio stream language: %s", stream.language_code)
+                    Log.Info("Couldn't detect embedded audio stream language: %s", stream.language_code)
 
                 # treat unknown language as lang1?
                 if not lang and config.treat_und_as_first:
                     lang = Language.rebuild(list(config.lang_list)[0])
+                    Log.Info("Assuming language %s for audio stream: %s", lang, getattr(stream, "index", None))
 
                 audio_languages.append(lang)
 
             # subtitle stream
             elif stream.stream_type == 3 and embedded_subtitles:
-                is_forced = helpers.is_stream_forced(stream)
+                is_forced = is_stream_forced(stream)
 
                 if ((config.forced_only or config.forced_also) and is_forced) or not is_forced:
                     # embedded subtitle
@@ -73,11 +75,13 @@ def prepare_video(pms_video_info, ignore_all=False, hints=None, rating_key=None,
                             try:
                                 lang = language_from_stream(stream.language_code)
                             except LanguageError:
-                                Log.Debug("Couldn't detect embedded subtitle stream language: %s", stream.language_code)
+                                Log.Info("Couldn't detect embedded subtitle stream language: %s", stream.language_code)
 
                             # treat unknown language as lang1?
                             if not lang and config.treat_und_as_first:
                                 lang = Language.rebuild(list(config.lang_list)[0])
+                                Log.Info("Assuming language %s for subtitle stream: %s", lang,
+                                         getattr(stream, "index", None))
 
                             if lang:
                                 if is_forced:

Contents/Code/support/tasks.py

@@ -19,6 +19,7 @@ from support.config import config
 from support.items import get_recent_items, get_item, is_wanted, get_item_title
 from support.helpers import track_usage, get_title_for_video_metadata, cast_bool, PartUnknownException
 from support.plex_media import get_plex_metadata
+from support.extract import agent_extract_embedded
 from support.scanning import scan_videos
 from support.i18n import _
 from download import download_best_subtitles, pre_download_hook, post_download_hook, language_hook
@@ -449,6 +450,17 @@ class SearchAllRecentlyAddedMissing(Task):
                     Log.Debug(u"%s: Looking for missing subtitles: %s", self.name, get_item_title(plex_item))
                     scanned_parts = scan_videos([metadata], providers=providers)
 
+                    # auto extract embedded
+                    if config.embedded_auto_extract:
+                        if config.plex_transcoder:
+                            ts = agent_extract_embedded(scanned_parts, set_as_existing=True)
+                            if ts:
+                                Log.Debug("Waiting for %i extraction threads to finish" % len(ts))
+                                for t in ts:
+                                    t.join()
+                        else:
+                            Log.Warn("Plex Transcoder not found, can't auto extract")
+
                     downloaded_subtitles = download_best_subtitles(scanned_parts, min_score=min_score,
                                                                    providers=providers)
                     hit_providers = downloaded_subtitles is not None

Contents/DefaultPrefs.json

@@ -509,6 +509,18 @@
     "type": "text",
     "default": ""
   },
+  {
+    "id": "provider.bsplayer.enabled",
+    "label": "Provider: Enable BSPlayer Subtitles",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "provider.screwzira.enabled",
+    "label": "Provider: Enable ScrewZira (Hebrew)",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "providers.multithreading",
     "label": "Search enabled providers simultaneously (multithreading)",
@@ -794,6 +806,29 @@
     "type": "bool",
     "default": "false"
   },
+  {
+    "id": "scheduler.tasks.SubtitleStorageMaintenance.frequency",
+    "label": "Scheduler: Periodically run subtitle storage maintenance (SZ internal)",
+    "type": "enum",
+    "values": [
+      "never",
+      "every 6 hours",
+      "every 12 hours",
+      "every 24 hours",
+      "every 1 days",
+      "every 2 days",
+      "every 3 days",
+      "every 4 days",
+      "every 1 weeks",
+      "every 2 weeks",
+      "every 3 weeks",
+      "every 4 weeks",
+      "every 5 weeks",
+      "every 6 weeks",
+      "every 12 weeks"
+    ],
+    "default": "every 1 weeks"
+  },
   {
     "id": "history_size",
     "label": "History: amount of items to store historical data for",

Contents/Info.plist

@@ -13,7 +13,7 @@
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.6.5.3152</string>
+        <string>2.6.5.3217</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.6.5.3152
+Version 2.6.5.3217
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 

Contents/Libraries/Shared/subliminal/extensions.py

@@ -94,7 +94,8 @@ provider_manager = RegistrableExtensionManager('subliminal.providers', [
     'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
     'shooter = subliminal.providers.shooter:ShooterProvider',
     'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
-    'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'
+    'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider',
+    'screwzira = subliminal.providers.screwzira:ScrewZiraProvider'
 ])
 
 #: Refiner manager

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -30,7 +30,7 @@ from subliminal.core import guessit, ProviderPool, io, is_windows_special_path,
     ThreadPoolExecutor, check_video
 from subliminal_patch.exceptions import TooManyRequests, APIThrottled
 
-from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE
+from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE, FULL_LANGUAGE_LIST
 from scandir import scandir, scandir_generic as _scandir_generic
 
 logger = logging.getLogger(__name__)
@@ -62,7 +62,7 @@ class SZProviderPool(ProviderPool):
     def __init__(self, providers=None, provider_configs=None, blacklist=None, throttle_callback=None,
                  pre_download_hook=None, post_download_hook=None, language_hook=None):
         #: Name of providers to use
-        self.providers = providers or provider_registry.names()
+        self.providers = providers
 
         #: Provider configuration
         self.provider_configs = provider_configs or {}
@@ -186,12 +186,9 @@ class SZProviderPool(ProviderPool):
         except (requests.Timeout, socket.timeout):
             logger.error('Provider %r timed out', provider)
 
-        except (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled), e:
-            self.throttle_callback(provider, e)
-            return
-
-        except:
+        except Exception as e:
             logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
+            self.throttle_callback(provider, e)
 
     def list_subtitles(self, video, languages):
         """List subtitles.
@@ -283,14 +280,10 @@ class SZProviderPool(ProviderPool):
                 logger.debug("RAR Traceback: %s", traceback.format_exc())
                 return False
 
-            except (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled), e:
-                self.throttle_callback(subtitle.provider_name, e)
-                self.discarded_providers.add(subtitle.provider_name)
-                return False
-
-            except:
+            except Exception as e:
                 logger.exception('Unexpected error in provider %r, Traceback: %s', subtitle.provider_name,
                                  traceback.format_exc())
+                self.throttle_callback(subtitle.provider_name, e)
                 self.discarded_providers.add(subtitle.provider_name)
                 return False
 
@@ -548,8 +541,12 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski
         if video.size > 10485760:
             logger.debug('Size is %d', video.size)
             osub_hash = None
+
+            if "bsplayer" in providers:
+                video.hashes['bsplayer'] = osub_hash = hash_opensubtitles(hash_path)
+
             if "opensubtitles" in providers:
-                video.hashes['opensubtitles'] = osub_hash = hash_opensubtitles(hash_path)
+                video.hashes['opensubtitles'] = osub_hash = osub_hash or hash_opensubtitles(hash_path)
 
             if "shooter" in providers:
                 video.hashes['shooter'] = hash_shooter(hash_path)
@@ -614,11 +611,9 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen
         if adv_tag:
             forced = "forced" in adv_tag
 
-        # extract the potential language code
-        language_code = p_root.rsplit(".", 1)[1].replace('_', '-')
-
         # remove possible language code for matching
-        p_root_bare = ENDSWITH_LANGUAGECODE_RE.sub("", p_root)
+        p_root_bare = ENDSWITH_LANGUAGECODE_RE.sub(
+            lambda m: "" if str(m.group(1)).lower() in FULL_LANGUAGE_LIST else m.group(0), p_root)
 
         p_root_lower = p_root_bare.lower()
 
@@ -629,19 +624,21 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen
             if match_strictness == "strict" or (match_strictness == "loose" and not filename_contains):
                 continue
 
-        # default language is undefined
-        language = Language('und')
+        language = None
 
-        # attempt to parse
-        if language_code:
+        # extract the potential language code
+        try:
+            language_code = p_root.rsplit(".", 1)[1].replace('_', '-')
             try:
                 language = Language.fromietf(language_code)
                 language.forced = forced
-            except ValueError:
+            except (ValueError, LanguageReverseError):
                 logger.error('Cannot parse language code %r', language_code)
-                language = None
+                language_code = None
+        except IndexError:
+            language_code = None
 
-        elif not language_code and only_one:
+        if not language and not language_code and only_one:
             language = Language.rebuild(list(languages)[0], forced=forced)
 
         subtitles[p] = language
@@ -869,6 +866,9 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non
             logger.debug(u"Saving %r to %r", subtitle, subtitle_path)
             content = subtitle.get_modified_content(format=format, debug=debug_mods)
             if content:
+                if os.path.exists(subtitle_path):
+                    os.remove(subtitle_path)
+
                 with open(subtitle_path, 'w') as f:
                     f.write(content)
                 subtitle.storage_path = subtitle_path

Contents/Libraries/Shared/subliminal_patch/exceptions.py

@@ -9,3 +9,8 @@ class TooManyRequests(ProviderError):
 
 class APIThrottled(ProviderError):
     pass
+
+
+class ParseResponseError(ProviderError):
+    """Exception raised by providers when they are not able to parse the response."""
+    pass

Contents/Libraries/Shared/subliminal_patch/providers/addic7ed.py

@@ -6,11 +6,13 @@ import subliminal
 import time
 
 from random import randint
+
+from dogpile.cache.api import NO_VALUE
 from requests import Session
 from subliminal.cache import region
-from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError
+from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
 from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
-    Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup, show_cells_re
+    Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
 from subliminal.subtitle import fix_line_ending
 from subliminal_patch.utils import sanitize
 from subliminal_patch.exceptions import TooManyRequests
@@ -19,6 +21,8 @@ from subzero.language import Language
 
 logger = logging.getLogger(__name__)
 
+show_cells_re = re.compile(b'<td class="(?:version|vr)">.*?</td>', re.DOTALL)
+
 #: Series header parsing regex
 series_year_re = re.compile(r'^(?P<series>[ \w\'.:(),*&!?-]+?)(?: \((?P<year>\d{4})\))?$')
 
@@ -66,11 +70,15 @@ class Addic7edProvider(_Addic7edProvider):
     server_url = 'https://www.addic7ed.com/'
 
     sanitize_characters = {'-', ':', '(', ')', '.', '/'}
+    last_show_ids_fetch_key = "addic7ed_last_id_fetch"
 
     def __init__(self, username=None, password=None, use_random_agents=False):
         super(Addic7edProvider, self).__init__(username=username, password=password)
         self.USE_ADDICTED_RANDOM_AGENTS = use_random_agents
 
+        if not all((username, password)):
+            raise ConfigurationError('Username and password must be specified')
+
     def initialize(self):
         self.session = Session()
         self.session.headers['User-Agent'] = 'Subliminal/%s' % subliminal.__short_version__
@@ -101,13 +109,18 @@ class Addic7edProvider(_Addic7edProvider):
                     'remember': 'true'}
 
             tries = 0
-            while tries < 3:
+            while tries <= 3:
+                tries += 1
                 r = self.session.get(self.server_url + 'login.php', timeout=10, headers={"Referer": self.server_url})
-                if "grecaptcha" in r.content:
+                if "g-recaptcha" in r.content or "grecaptcha" in r.content:
                     logger.info('Addic7ed: Solving captcha. This might take a couple of minutes, but should only '
                                 'happen once every so often')
 
-                    site_key = re.search(r'grecaptcha.execute\(\'(.+?)\',', r.content).group(1)
+                    for g, s in (("g-recaptcha-response", r'g-recaptcha.+?data-sitekey=\"(.+?)\"'),
+                                 ("recaptcha_response", r'grecaptcha.execute\(\'(.+?)\',')):
+                        site_key = re.search(s, r.content).group(1)
+                        if site_key:
+                            break
                     if not site_key:
                         logger.error("Addic7ed: Captcha site-key not found!")
                         return
@@ -119,23 +132,31 @@ class Addic7edProvider(_Addic7edProvider):
 
                     result = pitcher.throw()
                     if not result:
-                        raise Exception("Addic7ed: Couldn't solve captcha!")
+                        if tries >= 3:
+                            raise Exception("Addic7ed: Couldn't solve captcha!")
+                        logger.info("Addic7ed: Couldn't solve captcha! Retrying")
+                        time.sleep(4)
+                        continue
 
-                    data["recaptcha_response"] = result
+                    data[g] = result
 
+                time.sleep(1)
                 r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10,
                                       headers={"Referer": self.server_url + "login.php"})
 
                 if "relax, slow down" in r.content:
                     raise TooManyRequests(self.username)
 
-                if r.status_code != 302:
-                    if "User <b></b> doesn't exist" in r.content and tries <= 2:
-                        logger.info("Addic7ed: Error, trying again. (%s/%s)", tries+1, 3)
-                        tries += 1
-                        continue
-
+                if "Wrong password" in r.content or "doesn't exist" in r.content:
                     raise AuthenticationError(self.username)
+
+                if r.status_code != 302:
+                    if tries >= 3:
+                        logger.error("Addic7ed: Something went wrong when logging in")
+                        raise AuthenticationError(self.username)
+                    logger.info("Addic7ed: Something went wrong when logging in; retrying")
+                    time.sleep(4)
+                    continue
                 break
 
             store_verification("addic7ed", self.session)
@@ -143,10 +164,12 @@ class Addic7edProvider(_Addic7edProvider):
             logger.debug('Addic7ed: Logged in')
             self.logged_in = True
 
+            time.sleep(2)
+
     def terminate(self):
         self.session.close()
 
-    def get_show_id(self, series, year=None, country_code=None):
+    def get_show_id(self, series, year=None, country_code=None, ignore_cache=False):
         """Get the best matching show id for `series`, `year` and `country_code`.
 
         First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id`.
@@ -158,32 +181,45 @@ class Addic7edProvider(_Addic7edProvider):
         :type country_code: str
         :return: the show id, if found.
         :rtype: int
-
         """
-        series_sanitized = sanitize(series).lower()
-        show_ids = self._get_show_ids()
         show_id = None
+        ids_to_look_for = {sanitize(series).lower(), sanitize(series.replace(".", "")).lower()}
+        show_ids = self._get_show_ids()
+        if ignore_cache or not show_ids:
+            show_ids = self._get_show_ids.refresh(self)
 
-        # attempt with country
-        if not show_id and country_code:
-            logger.debug('Getting show id with country')
-            show_id = show_ids.get('%s %s' % (series_sanitized, country_code.lower()))
+        logger.debug("Trying show ids: %s", ids_to_look_for)
+        for series_sanitized in ids_to_look_for:
+            # attempt with country
+            if not show_id and country_code:
+                logger.debug('Getting show id with country')
+                show_id = show_ids.get('%s %s' % (series_sanitized, country_code.lower()))
 
-        # attempt with year
-        if not show_id and year:
-            logger.debug('Getting show id with year')
-            show_id = show_ids.get('%s %d' % (series_sanitized, year))
+            # attempt with year
+            if not show_id and year:
+                logger.debug('Getting show id with year')
+                show_id = show_ids.get('%s %d' % (series_sanitized, year))
 
-        # attempt clean
-        if not show_id:
-            logger.debug('Getting show id')
-            show_id = show_ids.get(series_sanitized)
+            # attempt clean
+            if not show_id:
+                logger.debug('Getting show id')
+                show_id = show_ids.get(series_sanitized)
 
-        # search as last resort
-        # broken right now
-        # if not show_id:
-        #     logger.warning('Series %s not found in show ids', series)
-        #     show_id = self._search_show_id(series)
+                if not show_id:
+                    now = datetime.datetime.now()
+                    last_fetch = region.get(self.last_show_ids_fetch_key)
+
+                    # re-fetch show ids once per day if any show ID not found
+                    if not ignore_cache and last_fetch != NO_VALUE and last_fetch + datetime.timedelta(days=1) < now:
+                        logger.info("Show id not found; re-fetching show ids")
+                        return self.get_show_id(series, year=year, country_code=country_code, ignore_cache=True)
+                    logger.debug("Not refreshing show ids, as the last fetch has been too recent")
+
+            # search as last resort
+            # broken right now
+            # if not show_id:
+            #     logger.warning('Series %s not found in show ids', series)
+            #     show_id = self._search_show_id(series)
 
         return show_id
 
@@ -197,6 +233,8 @@ class Addic7edProvider(_Addic7edProvider):
         """
         # get the show page
         logger.info('Getting show ids')
+        region.set(self.last_show_ids_fetch_key, datetime.datetime.now())
+
         r = self.session.get(self.server_url + 'shows.php', timeout=10)
         r.raise_for_status()
 
@@ -205,14 +243,15 @@ class Addic7edProvider(_Addic7edProvider):
         # Assuming the site's markup is bad, and stripping it down to only contain what's needed.
         show_cells = re.findall(show_cells_re, r.content)
         if show_cells:
-            soup = ParserBeautifulSoup(b''.join(show_cells), ['lxml', 'html.parser'])
+            soup = ParserBeautifulSoup(b''.join(show_cells).decode('utf-8', 'ignore'), ['lxml', 'html.parser'])
         else:
             # If RegEx fails, fall back to original r.content and use 'html.parser'
             soup = ParserBeautifulSoup(r.content, ['html.parser'])
 
         # populate the show ids
         show_ids = {}
-        for show in soup.select('td > h3 > a[href^="/show/"]'):
+        shows = soup.select('td > h3 > a[href^="/show/"]')
+        for show in shows:
             show_clean = sanitize(show.text, default_characters=self.sanitize_characters)
             try:
                 show_id = int(show['href'][6:])
@@ -230,6 +269,9 @@ class Addic7edProvider(_Addic7edProvider):
 
         logger.debug('Found %d show ids', len(show_ids))
 
+        if not show_ids:
+            raise Exception("Addic7ed: No show IDs found!")
+
         return show_ids
 
     @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
@@ -329,7 +371,7 @@ class Addic7edProvider(_Addic7edProvider):
 
             # ignore incomplete subtitles
             status = cells[5].text
-            if status != 'Completed':
+            if "%" in status:
                 logger.debug('Ignoring subtitle with status %s', status)
                 continue
 

Contents/Libraries/Shared/subliminal_patch/providers/argenteam.py

@@ -136,7 +136,7 @@ class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
     provider_name = 'argenteam'
     languages = {Language.fromalpha2(l) for l in ['es']}
     video_types = (Episode, Movie)
-    BASE_URL = "http://www.argenteam.net/"
+    BASE_URL = "https://argenteam.net/"
     API_URL = BASE_URL + "api/v1/"
     subtitle_class = ArgenteamSubtitle
     hearing_impaired_verifiable = False
@@ -244,7 +244,9 @@ class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
                 for s in r['subtitles']:
                     movie_kind = "episode" if is_episode else "movie"
                     page_link = self.BASE_URL + movie_kind + "/" + str(aid)
-                    sub = ArgenteamSubtitle(language, page_link, s['uri'], movie_kind, returned_title,
+                    # use https and new domain
+                    download_link = s['uri'].replace('http://www.argenteam.net/', self.BASE_URL)
+                    sub = ArgenteamSubtitle(language, page_link, download_link, movie_kind, returned_title,
                                             season, episode, year, r.get('team'), r.get('tags'),
                                             r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id,
                                             asked_for_release_group=video.release_group,

Contents/Libraries/Shared/subliminal_patch/providers/bsplayer.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+import logging
+import io
+import os
+
+from requests import Session
+from guessit import guessit
+from subliminal_patch.providers import Provider
+from subliminal_patch.subtitle import Subtitle
+from subliminal.utils import sanitize_release_group
+from subliminal.subtitle import guess_matches
+from subzero.language import Language
+
+import gzip
+import random
+from time import sleep
+from xml.etree import ElementTree
+
+logger = logging.getLogger(__name__)
+
+class BSPlayerSubtitle(Subtitle):
+    """BSPlayer Subtitle."""
+    provider_name = 'bsplayer'
+
+    def __init__(self, language, filename, subtype, video, link):
+        super(BSPlayerSubtitle, self).__init__(language)
+        self.language = language
+        self.filename = filename
+        self.page_link = link
+        self.subtype = subtype
+        self.video = video
+
+    @property
+    def id(self):
+        return self.page_link
+
+    @property
+    def release_info(self):
+        return self.filename
+
+    def get_matches(self, video):
+        matches = set()
+
+        video_filename = video.name
+        video_filename = os.path.basename(video_filename)
+        video_filename, _ = os.path.splitext(video_filename)
+        video_filename = sanitize_release_group(video_filename)
+
+        subtitle_filename = self.filename
+        subtitle_filename = os.path.basename(subtitle_filename)
+        subtitle_filename, _ = os.path.splitext(subtitle_filename)
+        subtitle_filename = sanitize_release_group(subtitle_filename)
+
+
+        matches |= guess_matches(video, guessit(self.filename))
+
+        matches.add(id(self))
+        matches.add('hash')
+
+        return matches
+
+
+
+class BSPlayerProvider(Provider):
+    """BSPlayer Provider."""
+    languages = {Language('por', 'BR')} | {Language(l) for l in [
+        'ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor', 'nld', 'pol', 'por',
+        'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho'
+    ]}
+    SEARCH_THROTTLE = 8
+
+    # batantly based on kodi's bsplayer plugin
+    # also took from BSPlayer-Subtitles-Downloader
+    def __init__(self):
+        self.initialize()
+
+    def initialize(self):
+        self.session = Session()
+        self.search_url = self.get_sub_domain()
+        self.token = None
+        self.login()
+
+    def terminate(self):
+        self.session.close()
+        self.logout()
+
+    def api_request(self, func_name='logIn', params='', tries=5):
+        headers = {
+            'User-Agent': 'BSPlayer/2.x (1022.12360)',
+            'Content-Type': 'text/xml; charset=utf-8',
+            'Connection': 'close',
+            'SOAPAction': '"http://api.bsplayer-subtitles.com/v1.php#{func_name}"'.format(func_name=func_name)
+        }
+        data = (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            '<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" '
+            'xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" '
+            'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
+            'xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:ns1="{search_url}">'
+            '<SOAP-ENV:Body SOAP-ENV:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">'
+            '<ns1:{func_name}>{params}</ns1:{func_name}></SOAP-ENV:Body></SOAP-ENV:Envelope>'
+        ).format(search_url=self.search_url, func_name=func_name, params=params)
+        logger.info('Sending request: %s.' % func_name)
+        for i in iter(range(tries)):
+            try:
+                self.session.headers.update(headers.items())
+                res = self.session.post(self.search_url, data)
+                return ElementTree.fromstring(res.text)
+
+                ### with requests
+                # res = requests.post(
+                #     url=self.search_url,
+                #     data=data,
+                #     headers=headers
+                # ) 
+                # return ElementTree.fromstring(res.text)
+               
+            except Exception as ex:
+                logger.info("ERROR: %s." % ex)
+                if func_name == 'logIn':
+                    self.search_url = self.get_sub_domain()
+                sleep(1)
+        logger.info('ERROR: Too many tries (%d)...' % tries)
+        raise Exception('Too many tries...')
+
+    def login(self):
+        # If already logged in
+        if self.token:
+            return True
+
+        root = self.api_request(
+            func_name='logIn',
+            params=('<username></username>'
+                    '<password></password>'
+                    '<AppID>BSPlayer v2.67</AppID>')
+        )
+        res = root.find('.//return')
+        if res.find('status').text == 'OK':
+            self.token = res.find('data').text
+            logger.info("Logged In Successfully.")
+            return True
+        return False
+
+    def logout(self):
+        # If already logged out / not logged in
+        if not self.token:
+            return True
+
+        root = self.api_request(
+            func_name='logOut',
+            params='<handle>{token}</handle>'.format(token=self.token)
+        )
+        res = root.find('.//return')
+        self.token = None
+        if res.find('status').text == 'OK':
+            logger.info("Logged Out Successfully.")
+            return True
+        return False
+
+    def query(self, video, video_hash, language):
+        if not self.login():
+            return []
+
+        if isinstance(language, (tuple, list, set)):
+            # language_ids = ",".join(language)
+            # language_ids = 'spa'
+            language_ids = ','.join(sorted(l.opensubtitles for l in language))
+
+
+        if video.imdb_id is None:
+            imdbId = '*'
+        else:
+            imdbId = video.imdb_id
+        sleep(self.SEARCH_THROTTLE)
+        root = self.api_request(
+            func_name='searchSubtitles',
+            params=(
+                '<handle>{token}</handle>'
+                '<movieHash>{movie_hash}</movieHash>'
+                '<movieSize>{movie_size}</movieSize>'
+                '<languageId>{language_ids}</languageId>'
+                '<imdbId>{imdbId}</imdbId>'
+            ).format(token=self.token, movie_hash=video_hash,
+                     movie_size=video.size, language_ids=language_ids, imdbId=imdbId)
+        )
+        res = root.find('.//return/result')
+        if res.find('status').text != 'OK':
+            return []
+
+        items = root.findall('.//return/data/item')
+        subtitles = []
+        if items:
+            logger.info("Subtitles Found.")
+            for item in items:
+                subID=item.find('subID').text
+                subDownloadLink=item.find('subDownloadLink').text
+                subLang= Language.fromopensubtitles(item.find('subLang').text)
+                subName=item.find('subName').text
+                subFormat=item.find('subFormat').text
+                subtitles.append(
+                    BSPlayerSubtitle(subLang,subName, subFormat, video, subDownloadLink)
+                )
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return self.query(video, video.hashes['bsplayer'], languages)
+
+    def get_sub_domain(self):
+    # s1-9, s101-109
+        SUB_DOMAINS = ['s1', 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9',
+                's101', 's102', 's103', 's104', 's105', 's106', 's107', 's108', 's109']
+        API_URL_TEMPLATE = "http://{sub_domain}.api.bsplayer-subtitles.com/v1.php"
+        sub_domains_end = len(SUB_DOMAINS) - 1
+        return API_URL_TEMPLATE.format(sub_domain=SUB_DOMAINS[random.randint(0, sub_domains_end)])
+
+    def download_subtitle(self, subtitle):
+        session = Session()
+        _addheaders = {
+            'User-Agent': 'Mozilla/4.0 (compatible; Synapse)'
+        }
+        session.headers.update(_addheaders)
+        res = session.get(subtitle.page_link)
+        if res:
+            if res.text == '500':
+                raise ValueError('Error 500 on server')
+
+            with gzip.GzipFile(fileobj=io.BytesIO(res.content)) as gf:
+                subtitle.content = gf.read() 
+                subtitle.normalize()
+
+            return subtitle
+        raise ValueError('Problems conecting to the server')
+
+

Contents/Libraries/Shared/subliminal_patch/providers/screwzira.py

@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+from requests import Session
+import json
+import logging
+from subzero.language import Language
+from bs4 import BeautifulSoup
+from guessit import guessit
+
+from subliminal_patch.providers import Provider
+from subliminal.providers import Episode, Movie
+from subliminal_patch.utils import sanitize
+from subliminal_patch.subtitle import Subtitle, guess_matches
+from subliminal.subtitle import fix_line_ending
+
+__author__ = "Dor Nizar"
+
+logger = logging.getLogger(__name__)
+
+
+class ScrewZiraSubtitle(Subtitle):
+    provider_name = 'screwzira'
+
+    def __init__(self, language, title_id, subtitle_id, series, season, episode, release, year):
+        super(ScrewZiraSubtitle, self).__init__(language, subtitle_id)
+        self.title_id = title_id
+        self.subtitle_id = subtitle_id
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.release = release
+        self.year = year
+
+    def get_matches(self, video):
+        matches = set()
+        logger.debug("--ScrewZiraSubtitle--\n{}".format(self.__dict__))
+
+        # episode
+        if isinstance(video, Episode):
+            # series
+            if video.series and sanitize(self.series) == sanitize(video.series):
+                matches.add('series')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+            # guess
+            matches |= guess_matches(video, guessit(self.release, {'type': 'episode'}))
+        # movie
+        elif isinstance(video, Movie):
+            # title
+            if video.title and (sanitize(self.series) in (
+                    sanitize(name) for name in [video.title] + video.alternative_titles)):
+                matches.add('title')
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+            # guess
+            matches |= guess_matches(video, guessit(self.release, {'type': 'movie'}))
+
+        logger.debug("ScrewZira subtitle criteria match:\n{}".format(matches))
+        return matches
+
+    @property
+    def id(self):
+        return self.subtitle_id
+
+
+class ScrewZiraProvider(Provider):
+    subtitle_class = ScrewZiraSubtitle
+    languages = {Language.fromalpha2(l) for l in ['he']}
+    URL_SERVER = 'https://www.screwzira.com/'
+
+    URI_SEARCH_TITLE = 'Services/ContentProvider.svc/GetSearchForecast'
+    URI_SEARCH_SERIES_SUBTITLE = 'Services/GetModuleAjax.ashx'
+    URI_SEARCH_MOVIE_SUBTITLE = "MovieInfo.aspx"
+    URI_REQ_SUBTITLE_ID = "Services/ContentProvider.svc/RequestSubtitleDownload"
+    URI_DOWNLOAD_SUBTITLE = "Services/DownloadFile.ashx"
+
+    def initialize(self):
+        logger.debug("ScrewZira initialize")
+        self.session = Session()
+        self.session.headers[
+            'User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; ' \
+                            'Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
+
+    def terminate(self):
+        logger.debug("ScrewZira terminate")
+        self.session.close()
+
+    def __init__(self):
+        self.session = None
+
+    def _search_series(self, title):
+        logger.debug("Searching '{}'".format(title))
+        title_request = {
+            "request": {
+                "SearchString": title,
+                "SearchType": "Film"
+            }
+        }
+        r = self.session.post(self.URL_SERVER + self.URI_SEARCH_TITLE, json=title_request, allow_redirects=False,
+                              timeout=10)
+        r.raise_for_status()
+        series_found = r.json()
+        if 'd' in series_found:
+            try:
+                series_found = json.loads(series_found['d'])
+            except ValueError:
+                series_found = None
+        if 'Items' in series_found:
+            return series_found['Items']
+        return []
+
+    def _search_subtitles(self, title_id, season=None, episode=None):
+        if season and episode:
+            params = {
+                'moduleName': 'SubtitlesList',
+                'SeriesID': title_id,
+                'Season': season,
+                'Episode': episode
+            }
+            r = self.session.get(url=self.URL_SERVER + self.URI_SEARCH_SERIES_SUBTITLE, params=params)
+        else:
+            params = {
+                'ID': title_id,
+            }
+            r = self.session.get(url=self.URL_SERVER + self.URI_SEARCH_MOVIE_SUBTITLE, params=params)
+
+        r.raise_for_status()
+        results = r.content
+        if not results:
+            return []
+        subtitles = BeautifulSoup(results, 'html.parser').select('a.fa')
+        logger.debug("[BS4] Elements found:\n{}".format(subtitles))
+        subtitle_list = []
+        for i in subtitles:
+            subtitle_id = i.attrs['data-subtitle-id']
+            release = i.findParent().findParent().text.strip().split('\n')[0]
+            subtitle_list.append((subtitle_id, release))
+
+        return subtitle_list  # [(Subtitle ID, name), (....)]
+
+    def _req_download_identifier(self, title_id, subtitle_id):
+        logger.debug("Request subtitle identifier for: title id: {}, subtitle id: {}".format(title_id, subtitle_id))
+        data = {
+            'request': {
+                'FilmID': title_id,
+                'SubtitleID': subtitle_id,
+                'FontSize': 0,
+                'FontColor': "",
+                'PredefinedLayout': -1
+            }
+        }
+
+        r = self.session.post(self.URL_SERVER + self.URI_REQ_SUBTITLE_ID, json=data, allow_redirects=False,
+                              timeout=10)
+        r.raise_for_status()
+        try:
+            r = json.loads(r.json()['d'])
+        except ValueError:
+            r = {}
+
+        if 'DownloadIdentifier' not in r:
+            logger.error("Download Identifier not found")
+            return None
+        return r['DownloadIdentifier']
+
+    def _download_subtitles(self, download_id):
+        logger.debug("Downloading subtitles by download identifier - {}".format(download_id))
+        data = {'DownloadIdentifier': download_id}
+        r = self.session.get(self.URL_SERVER + self.URI_DOWNLOAD_SUBTITLE, params=data,
+                             timeout=10)
+        r.raise_for_status()
+        if not r.content:
+            logger.debug("Download subtitle failed")
+            return None
+
+        logger.debug("Download subtitle success")
+        return r.content
+
+    def query(self, title, season=None, episode=None, year=None):
+        subtitles = []
+        titles = self._search_series(title)
+        if season and episode:
+            logger.debug("Searching for:\nTitle: {}\nSeason: {}\nEpisode: {}\nYear: {}".format(title, season,
+                                                                                               episode, year))
+        else:
+            logger.debug("Searching for:\nTitle: {}\nYear: {}\n".format(title, year))
+        for title in titles:
+            logger.debug("Title Candidate: {}".format(title))
+            title_id = title['ID']
+            if season and episode:
+                result = self._search_subtitles(title_id, season, episode)
+            else:
+                result = self._search_subtitles(title_id)
+
+            if not result:
+                continue
+
+            for subtitle_id, release in result:
+                subtitles.append(self.subtitle_class(next(iter(self.languages)), title_id, subtitle_id,
+                                                     title['EngName'], season, episode, release, year))
+
+        if subtitles:
+            logger.debug("Found Subtitle Candidates: {}".format(subtitles))
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        season = episode = year = title = None
+
+        if isinstance(video, Episode):
+            logger.info("list_subtitles Series: {}, season: {}, episode: {}".format(video.series,
+                                                                                    video.season,
+                                                                                    video.episode))
+            title = video.series
+            season = video.season
+            episode = video.episode
+        elif isinstance(video, Movie):
+            logger.info("list_subtitles Movie: {}, year: {}".format(video.title, video.year))
+            title = video.title
+            year = video.year
+
+        return [s for s in self.query(title, season, episode, year) if s.language in languages]
+
+    def download_subtitle(self, subtitle):
+        # type: (ScrewZiraSubtitle) -> None
+
+        logger.info('Downloading subtitle from ScrewZira: %r', subtitle)
+        downloadID = self._req_download_identifier(subtitle.title_id, subtitle.subtitle_id)
+        if not downloadID:
+            logger.debug('Unable to retrieve download identifier')
+            return None
+
+        content = self._download_subtitles(downloadID)
+        if not content:
+            logger.debug('Unable to download subtitle')
+            return None
+
+        subtitle.content = fix_line_ending(content)

Contents/Libraries/Shared/subliminal_patch/refiners/tvdb.py

@@ -87,7 +87,10 @@ def refine(video, **kwargs):
         # parse series year
         series_year = None
         if result['firstAired']:
-            series_year = datetime.datetime.strptime(result['firstAired'], '%Y-%m-%d').year
+            try:
+                series_year = datetime.datetime.strptime(result['firstAired'], '%Y-%m-%d').year
+            except ValueError:
+                continue
 
         # discard mismatches on year
         if video.year and series_year and video.year != series_year:

Contents/Libraries/Shared/subliminal_patch/subtitle.py

@@ -19,6 +19,15 @@ from subliminal import Subtitle as Subtitle_
 from subliminal.subtitle import Episode, Movie, sanitize_release_group, get_equivalent_release_groups
 from subliminal_patch.utils import sanitize
 from ftfy import fix_text
+from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
+
+BOMS = (
+    (BOM_UTF8, "UTF-8"),
+    (BOM_UTF32_BE, "UTF-32-BE"),
+    (BOM_UTF32_LE, "UTF-32-LE"),
+    (BOM_UTF16_BE, "UTF-16-BE"),
+    (BOM_UTF16_LE, "UTF-16-LE"),
+)
 
 logger = logging.getLogger(__name__)
 
@@ -105,6 +114,9 @@ class Subtitle(Subtitle_):
         # normalize line endings
         self.content = self.content.replace("\r\n", "\n").replace('\r', '\n')
 
+    def _check_bom(self, data):
+        return [encoding for bom, encoding in BOMS if data.startswith(bom)]
+
     def guess_encoding(self):
         """Guess encoding using the language, falling back on chardet.
 
@@ -119,6 +131,11 @@ class Subtitle(Subtitle_):
 
         encodings = ['utf-8']
 
+        # check UTF BOMs
+        bom_encodings = self._check_bom(self.content)
+        if bom_encodings:
+            encodings = list(set(enc.lower() for enc in bom_encodings + encodings))
+
         # add language-specific encodings
         # http://scratchpad.wikia.com/wiki/Character_Encoding_Recommendation_for_Languages
 
@@ -340,6 +357,15 @@ class ModifiedSubtitle(Subtitle):
     id = None
 
 
+MERGED_FORMATS = {
+    "TV": ("HDTV", "SDTV", "AHDTV", "UHDTV"),
+    "Air": ("SATRip", "DVB", "PPV"),
+    "Disk": ("DVD", "HD-DVD", "BluRay")
+}
+
+MERGED_FORMATS_REV = dict((v.lower(), k.lower()) for k in MERGED_FORMATS for v in MERGED_FORMATS[k])
+
+
 def guess_matches(video, guess, partial=False):
     """Get matches between a `video` and a `guess`.
 
@@ -422,21 +448,25 @@ def guess_matches(video, guess, partial=False):
             formats = [formats]
 
         if video.format:
-            video_format = video.format
-            if video_format in ("HDTV", "SDTV", "TV"):
-                video_format = "TV"
-                logger.debug("Treating HDTV/SDTV the same")
+            video_format = video.format.lower()
+            _video_gen_format = MERGED_FORMATS_REV.get(video_format)
+            if _video_gen_format:
+                logger.debug("Treating %s as %s the same", video_format, _video_gen_format)
 
             for frmt in formats:
-                if frmt in ("HDTV", "SDTV"):
-                    frmt = "TV"
+                _guess_gen_frmt = MERGED_FORMATS_REV.get(frmt.lower())
 
-                if frmt.lower() == video_format.lower():
+                if _guess_gen_frmt == _video_gen_format:
                     matches.add('format')
                     break
+        if "release_group" in matches and "format" not in matches:
+            logger.info("Release group matched but format didn't. Remnoving release group match.")
+            matches.remove("release_group")
+
     # video_codec
     if video.video_codec and 'video_codec' in guess and guess['video_codec'] == video.video_codec:
         matches.add('video_codec')
+
     # audio_codec
     if video.audio_codec and 'audio_codec' in guess and guess['audio_codec'] == video.audio_codec:
         matches.add('audio_codec')

Contents/Libraries/Shared/subzero/language.py

@@ -3,7 +3,7 @@ import types
 import re
 
 from babelfish.exceptions import LanguageError
-from babelfish import Language as Language_, basestr
+from babelfish import Language as Language_, basestr, LANGUAGE_MATRIX
 
 repl_map = {
     "dk": "da",
@@ -31,6 +31,12 @@ repl_map = {
 }
 
 
+ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)))) + list(repl_map.values())
+ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)))) + \
+               list(set(filter(lambda x: len(x) == 3, list(repl_map.keys()))))
+FULL_LANGUAGE_LIST = ALPHA2_LIST + ALPHA3b_LIST
+
+
 def language_from_stream(l):
     if not l:
         raise LanguageError()

Contents/advanced_settings.json.template

@@ -24,6 +24,13 @@ Don't expect support if you mess this up.
     "find_better_as_extracted_tv_score": 352,
     "find_better_as_extracted_movie_score": 82,
 
+    // SZ can use mediainfo if present to detect titles/forced state of MP4 MOV_TEXT, because the PMS currently doesn't
+    // set the title attribute
+    "dont_use_mediainfo_mp4": False,
+
+    // specific mediainfo binary path
+    "mediainfo_bin": null,
+
     "debug_i18n": false,
 
     // per-provider-config

README.md

@@ -1,7 +1,7 @@
 # Sub-Zero for Plex
 [![](https://img.shields.io/github/release/pannal/Sub-Zero.bundle.svg?style=flat&label=stable)](https://github.com/pannal/Sub-Zero.bundle/releases/latest) 
 [![master](https://img.shields.io/badge/master-stable-green.svg?maxAge=2592000)]()
-[![Maintenance](https://img.shields.io/maintenance/yes/2019.svg)]()
+[![Maintenance](https://img.shields.io/maintenance/yes/2020.svg)]()
 [![Slack Status](https://szslack.fragstore.net/badge.svg)](https://szslack.fragstore.net)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fpannal%2FSub-Zero.bundle.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fpannal%2FSub-Zero.bundle?ref=badge_shield)
 
@@ -95,19 +95,22 @@ the.vbm, mmgoodnow, Vertig0ne, thliu78, tattoomees, ostman, count_confucius, ehe
 ## Changelog
 
 
-2.6.5.3152
+2.6.5.3217
 
 subscene, addic7ed
 - either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
 
 Changelog
-- core: fix core issue possibly impacting results on OpenSubtitles in certain conditions
-- core: fix default values of opensubtitles-skip-wrong-fps, use_https; fix #676
-- core: fix for determining whether to search under certain circumstances; fixes #666
-- core: #664 fix missing language processing of multiple videos refreshed at once
-- core: #661 fix match strictness when determining preexisting external subtitles
-- providers: titlovi: New implementation of Titlovi using API (thanks @viking1304)
-
+- core: also extract (missing) embedded subtitles when SearchAllRecentlyAddedMissing is running
+- core: core: clarify detecting streams (in logs)
+- core: UnRAR: set binary to executable, even if not checked out from git; might fix #693
+- core: bazarr-backport: morpheus65535/bazarr#703: use proper language code detection instead of a wild guess; should fix bad existing subtitle detection
+- core: bazarr-backport: morpheus65535/bazarr#660: fix BOM encoding stuff
+- core: bazarr-backport: morpheus65535/bazarr#656 further generalize formats; skip release group match if format match failed
+- core: fix stream detection when using mediainfo (#711)
+- config/core: make periodic SZ-internal subtitle maintenance interval configurable
+- providers: add BSPlayer Subtitles
+- providers: add ScrewZira (Hebrew)
 
 [older changes](CHANGELOG.md)