incremental tmp

Contents/Code/support/config.py

@@ -10,6 +10,8 @@ import jstyleson
 import datetime
 import stat
 import traceback
+import socket
+import requests
 
 import subliminal
 import subliminal_patch
@@ -63,7 +65,9 @@ def int_or_default(s, default):
 
 
 VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, DownloadLimitPerDayExceeded,
-                             ServiceUnavailable, APIThrottled)
+                             ServiceUnavailable, APIThrottled, requests.Timeout, requests.ReadTimeout, socket.timeout)
+
+def_timeout = (datetime.timedelta(minutes=20), "20 minutes")
 
 PROVIDER_THROTTLE_MAP = {
     "default": {
@@ -73,6 +77,9 @@ PROVIDER_THROTTLE_MAP = {
         ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
         APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
         AuthenticationError: (datetime.timedelta(hours=2), "2 hours"),
+        requests.Timeout: def_timeout,
+        socket.timeout: def_timeout,
+        requests.ReadTimeout: def_timeout,
     },
     "opensubtitles": {
         TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
@@ -666,12 +673,18 @@ class Config(object):
             if not agent.primary:
                 continue
 
-            for t in list(agent.media_types):
-                if t.media_type in (MOVIE, SHOW):
-                    related_agents = Plex.primary_agent(agent.identifier, t.media_type)
+            media_types = [t.media_type for t in list(agent.media_types)]
+
+            # the new movie agent doesn't populate its media types, workaround
+            if not media_types and agent.identifier == "tv.plex.agents.movie":
+                media_types = [MOVIE]
+
+            for media_type in media_types:
+                if media_type in (MOVIE, SHOW):
+                    related_agents = Plex.primary_agent(agent.identifier, media_type)
                     for a in related_agents:
                         if a.identifier == PLUGIN_IDENTIFIER and a.enabled:
-                            enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[t.media_type]].append(agent.identifier)
+                            enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[media_type]].append(agent.identifier)
 
         # find the libraries that use them
         for library in self.sections:

Contents/Code/support/tasks.py

@@ -670,7 +670,7 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
         min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
         min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
         min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
-        min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 82
+        min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 112
         overwrite_manually_modified = cast_bool(
             Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
         overwrite_manually_selected = cast_bool(

Contents/Info.plist

@@ -13,7 +13,7 @@
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.6.5.3241</string>
+        <string>2.6.5.3247</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -23,7 +23,7 @@
         <key>PlexPluginConsoleLogging</key>
         <string>0</string>
         <key>PlexPluginDevMode</key>
-        <string>0</string>
+        <string>1</string>
          <key>PlexPluginCodePolicy</key>
             <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API -->
             <string>Elevated</string>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.6.5.3241
+Version 2.6.5.3247 DEV
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -108,10 +108,12 @@ class SZProviderPool(ProviderPool):
         try:
             logger.info('Terminating provider %s', name)
             self.initialized_providers[name].terminate()
-        except (requests.Timeout, socket.timeout):
+        except (requests.Timeout, socket.timeout) as e:
             logger.error('Provider %r timed out, improperly terminated', name)
-        except:
+            self.throttle_callback(name, e)
+        except Exception as e:
             logger.exception('Provider %r terminated unexpectedly', name)
+            self.throttle_callback(name, e)
 
         del self.initialized_providers[name]
 
@@ -183,8 +185,9 @@ class SZProviderPool(ProviderPool):
 
             return out
 
-        except (requests.Timeout, socket.timeout):
-            logger.error('Provider %r timed out', provider)
+        except (requests.Timeout, socket.timeout) as e:
+            logger.exception('Provider %r timed out', provider)
+            self.throttle_callback(provider, e)
 
         except Exception as e:
             logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
@@ -263,10 +266,11 @@ class SZProviderPool(ProviderPool):
                     requests.exceptions.ProxyError,
                     requests.exceptions.SSLError,
                     requests.Timeout,
-                    socket.timeout):
+                    socket.timeout) as e:
                 logger.exception('Provider %r connection error', subtitle.provider_name)
+                self.throttle_callback(subtitle.provider_name, e)
 
-            except ResponseNotReady:
+            except ResponseNotReady as e:
                 logger.error('Provider %r response error, reinitializing', subtitle.provider_name)
                 try:
                     self[subtitle.provider_name].terminate()
@@ -274,6 +278,7 @@ class SZProviderPool(ProviderPool):
                 except:
                     logger.error('Provider %r reinitialization error: %s', subtitle.provider_name,
                                  traceback.format_exc())
+                    self.throttle_callback(subtitle.provider_name, e)
 
             except rarfile.BadRarFile:
                 logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)

Contents/Libraries/Shared/submod_test.py

@@ -21,7 +21,7 @@ if debug:
     logging.basicConfig(level=logging.DEBUG)
 
 #sub = Subtitle(Language.fromietf("eng:forced"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=-500)", "shift_offset(ms=500)", "shift_offset(s=2,ms=800)"])
-sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=0,s=1)"])
+sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=0,s=1)", "fix_incremental", "fix_short"])
 sub.content = open(fn).read()
 sub.normalize()
 sub.is_valid()

Contents/Libraries/Shared/subzero/modification/mods/common.py

@@ -1,6 +1,8 @@
 # coding=utf-8
 
 import re
+import logging
+from collections import OrderedDict
 
 from subzero.language import Language
 from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, SubtitleModification
@@ -9,7 +11,7 @@ from subzero.modification.processors.re_processor import NReProcessor
 from subzero.modification import registry
 from tld import get_tld
 
-
+logger = logging.getLogger(__name__)
 ENGLISH = Language("eng")
 
 
@@ -181,7 +183,104 @@ class FixUppercase(SubtitleModification):
             entry.plaintext = self.capitalize(entry.plaintext)
 
 
+class FixIncremental(SubtitleModification):
+    identifier = "fix_incremental"
+    description = "Fixes inremental-repeating subtitles"
+    modifies_whole_file = True
+    exclusive = True
+
+    long_description = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+    def modify(self, content, debug=False, parent=None, **kwargs):
+        prev_entry = None
+        for entry in parent.f:
+            subs = []
+            for sub in entry.text.split("\N"):
+                if prev_entry and prev_entry.text and prev_entry.text.lower().endswith(sub.lower()):
+                    if debug:
+                        logger.debug(u"Skipping incremental/dup: %s" % sub)
+                    continue
+                subs.append(sub)
+
+            if subs:
+                entry.text = "\N".join(subs)
+
+            prev_entry = entry
+
+
+
+class FixShort(SubtitleModification):
+    identifier = "fix_short"
+    description = "ASDasdasdasdasdd"
+    modifies_whole_file = True
+    exclusive = True
+
+    long_description = "adsadsdasdsadsa"
+
+    def modify(self, content, debug=False, parent=None, **kwargs):
+        prev_entry = None
+        prev_entry_dur = None
+        max_duration = 500
+        max_line_len = 200
+        max_lines = 3
+        entries = []
+        last_lines = []
+        for index, entry in enumerate(parent.f):
+            current_new_lines = []
+            if not last_lines and parent.f[index-1]:
+                print "YOO"
+                # find last lines
+                last_lines = parent.f[index-1].text.split("\N")
+            has_space = len(last_lines) < max_lines
+            last_line = ""
+            # go through each line and pack them
+            for line in entry.text.split("\N"):
+                new_line = ""
+                if line:
+                    if last_line != line and last_line and len(last_line + line) <= max_line_len:
+                        # new line plus line fits
+                        if re.match(".+\W$", line):
+                            if last_line.endswith(" "):
+                                new_line = last_line + line
+                            else:
+                                new_line = last_line + " " + line
+                            logger.debug("MERGING '%s' with '%s' to '%s'", last_line, line, new_line)
+                    else:
+                        new_line = line
+                last_line = new_line
+                current_new_lines.append(new_line)
+
+            # merge entries
+            if prev_entry:
+                #print prev_entry.duration, max_duration, len(new_lines), max_lines
+                if prev_entry.duration < max_duration and len(current_new_lines) < max_lines:
+                    #len(prev_entry.text) < max_len
+                    print "HIT", prev_entry.text, " + ", entry.text
+                    entry_text = prev_entry.text + "\N" + "\N".join(current_new_lines)
+
+                else:
+                    entry_text = "\N".join(current_new_lines)
+            else:
+                entry_text = "\N".join(current_new_lines)
+
+            #prev_entry = entry.copy()
+            if not prev_entry:
+                prev_entry = entry.copy()
+                continue
+            new_entry = prev_entry.copy()
+            new_entry.text = entry_text
+            prev_entry = new_entry.copy()
+            entries.append(new_entry)
+            #new_entries.append(entry.copy())
+
+        parent.f.entries = entries
+
+
+
+
 registry.register(CommonFixes)
 registry.register(RemoveTags)
 registry.register(ReverseRTL)
 registry.register(FixUppercase)
+registry.register(FixIncremental)
+registry.register(FixShort)

Contents/advanced_settings.json.template

@@ -22,7 +22,7 @@ Don't expect support if you mess this up.
     // when the find better subtitles task finds a subtitle for an item that has an active extracted embedded subtitle
     // set, what should be the minimum score that subtitle has to have in order to be considered better?
     "find_better_as_extracted_tv_score": 352,
-    "find_better_as_extracted_movie_score": 82,
+    "find_better_as_extracted_movie_score": 112,
 
     // SZ can use mediainfo if present to detect titles/forced state of MP4 MOV_TEXT, because the PMS currently doesn't
     // set the title attribute

README.md

@@ -94,6 +94,16 @@ the.vbm, mmgoodnow, Vertig0ne, thliu78, tattoomees, ostman, count_confucius, ehe
 
 ## Changelog
 
+2.6.5.3247
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+core: fix for tv.plex.agents.movie not populating its media types
+core: tasks: findBetterSubtitles: increase minimum score for better subtitles for movies with extracted embedded subs from 82 to 112
+
+
 2.6.5.3241
 
 subscene, addic7ed