Update README.md

CHANGELOG.md

@@ -1,3 +1,42 @@
+2.6.5.3099
+
+subscene, addic7ed and titlovi
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: allow system DNS again by putting "system" as the DNS
+- providers: subscene: fix again (subscene, contact us please, so we can end this)
+
+
+2.6.5.3092
+
+subscene, addic7ed and titlovi
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- providers: subscene: fix endpoint (hopefully for longer now)
+- providers: subscene: don't search for season packs (broken for now; relieves 50% of server load on provider)
+- providers: subscene: don't calculate video fn for now
+- providers: argenteam: backport fixes from bazarr
+- subtitle: try decoding with utf-16 by default as well (zho/farsi)
+- submod: HI: remove music tags by default
+- core: compat (bazarr): add env var SZ_KEEP_ENCODING to keep encoding of subtitles
+
+
+2.6.5.3074
+
+Changelog
+- core: cf: bypass cf 95% of the time without captchas
+- core: fix breaking line endings of certain languages (chinese, UTF-16); fixes #646
+- core: update pysubs2 to 0.2.3
+
+
+2.6.5.3062
+
+Changelog
+- core: cf: optimize
+- core: http: don't query DNS with IPs. thanks @fgump (fixes sonarr/radarr)
+
 
 2.6.5.3041
 

Contents/Code/support/config.py

@@ -761,6 +761,7 @@ class Config(object):
         return {'opensubtitles': cast_bool(Prefs['provider.opensubtitles.enabled']),
                      # 'thesubdb': Prefs['provider.thesubdb.enabled'],
                      'podnapisi': cast_bool(Prefs['provider.podnapisi.enabled']),
+                     'napisy24': cast_bool(Prefs['provider.napisy24.enabled']),
                      'titlovi': cast_bool(Prefs['provider.titlovi.enabled']),
                      'addic7ed': cast_bool(Prefs['provider.addic7ed.enabled']) and self.has_anticaptcha,
                      'tvsubtitles': cast_bool(Prefs['provider.tvsubtitles.enabled']),
@@ -801,6 +802,7 @@ class Config(object):
             providers["argenteam"] = False
             providers["assrt"] = False
             providers["subscene"] = False
+            providers["napisy24"] = False
             providers_forced_off = dict(providers)
 
         if not self.unrar and providers["legendastv"]:
@@ -864,8 +866,14 @@ class Config(object):
                                  'only_foreign': self.forced_only,
                                  'also_foreign': self.forced_also,
                              },
+                             'napisy24': {
+                                 'username': Prefs['provider.napisy24.username'],
+                                 'password': Prefs['provider.napisy24.password'],
+                             },
                              'subscene': {
                                  'only_foreign': self.forced_only,
+                                 'username': Prefs['provider.subscene.username'],
+                                 'password': Prefs['provider.subscene.password'],
                              },
                              'legendastv': {'username': Prefs['provider.legendastv.username'],
                                             'password': Prefs['provider.legendastv.password'],
@@ -1084,11 +1092,12 @@ class Config(object):
     def parse_custom_dns(self):
         custom_dns = Prefs['use_custom_dns2'].strip()
         os.environ["dns_resolvers"] = ""
-        if custom_dns:
+
+        if custom_dns and custom_dns != "system":
             ips = filter(lambda x: x, [d.strip() for d in custom_dns.split(",")])
             if ips:
                 os.environ["dns_resolvers"] = json.dumps(ips)
-                return os.environ["dns_resolvers"]
+            return os.environ["dns_resolvers"]
 
     def init_subliminal_patches(self):
         # configure custom subtitle destination folders for scanning pre-existing subs

Contents/DefaultPrefs.json

@@ -335,6 +335,26 @@
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "provider.napisy24.enabled",
+    "label": "Provider: Enable Napisy24 (pl)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.napisy24.username",
+    "label": "Napisy24 Username",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "provider.napisy24.password",
+    "label": "Napisy24 Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
+  },
   {
     "id": "provider.addic7ed.enabled",
     "label": "Provider: Enable Addic7ed (needs AntiCaptcha)",
@@ -431,6 +451,20 @@
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "provider.subscene.username",
+    "label": "SubScene Username",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "provider.subscene.password",
+    "label": "SubScene Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
+  },
   {
     "id": "provider.supersubtitles.enabled",
     "label": "Provider: Enable feliratok.info (Hungarian)",
@@ -861,7 +895,7 @@
   },
   {
     "id": "use_custom_dns2",
-    "label": "Use custom DNS (IPs, comma-separated, leave empty for system DNS. Default: Google/CF)",
+    "label": "Use custom DNS (IPs, comma-separated, set to 'system' for system DNS. Default: Google/CF)",
     "type": "text",
     "default": "1.1.1.1, 8.8.8.8"
   },

Contents/Info.plist

@@ -13,7 +13,7 @@
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.6.5.3074</string>
+        <string>2.6.5.3109</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -23,7 +23,7 @@
         <key>PlexPluginConsoleLogging</key>
         <string>0</string>
         <key>PlexPluginDevMode</key>
-        <string>1</string>
+        <string>0</string>
          <key>PlexPluginCodePolicy</key>
             <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API -->
             <string>Elevated</string>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.6.5.3074 DEV
+Version 2.6.5.3109
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -309,7 +309,8 @@ class SZProviderPool(ProviderPool):
             logger.error('Invalid subtitle')
             return False
 
-        subtitle.normalize()
+        if not os.environ.get("SZ_KEEP_ENCODING", False):
+            subtitle.normalize()
 
         return True
 
@@ -472,7 +473,7 @@ if is_windows_special_path:
     SZAsyncProviderPool = SZProviderPool
 
 
-def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, skip_hashing=False):
+def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, skip_hashing=False, hash_from=None):
     """Scan a video from a `path`.
 
     patch:
@@ -537,28 +538,34 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski
             video.alternative_titles.append(alt_guess["title"])
         logger.debug("Adding alternative title: %s", alt_guess["title"])
 
-    if dont_use_actual_file:
+    if dont_use_actual_file and not hash_from:
         return video
 
     # size and hashes
     if not skip_hashing:
-        video.size = os.path.getsize(path)
+        hash_path = hash_from or path
+        video.size = os.path.getsize(hash_path)
         if video.size > 10485760:
             logger.debug('Size is %d', video.size)
+            osub_hash = None
             if "opensubtitles" in providers:
-                video.hashes['opensubtitles'] = hash_opensubtitles(path)
+                video.hashes['opensubtitles'] = osub_hash = hash_opensubtitles(hash_path)
 
             if "shooter" in providers:
-                video.hashes['shooter'] = hash_shooter(path)
+                video.hashes['shooter'] = hash_shooter(hash_path)
 
             if "thesubdb" in providers:
-                video.hashes['thesubdb'] = hash_thesubdb(path)
+                video.hashes['thesubdb'] = hash_thesubdb(hash_path)
 
             if "napiprojekt" in providers:
                 try:
-                    video.hashes['napiprojekt'] = hash_napiprojekt(path)
+                    video.hashes['napiprojekt'] = hash_napiprojekt(hash_path)
                 except MemoryError:
-                    logger.warning(u"Couldn't compute napiprojekt hash for %s", path)
+                    logger.warning(u"Couldn't compute napiprojekt hash for %s", hash_path)
+
+            if "napisy24" in providers:
+                # Napisy24 uses the same hash as opensubtitles
+                video.hashes['napisy24'] = osub_hash or hash_opensubtitles(hash_path)
 
             logger.debug('Computed hashes %r', video.hashes)
         else:

Contents/Libraries/Shared/subliminal_patch/providers/argenteam.py

@@ -23,9 +23,10 @@ class ArgenteamSubtitle(Subtitle):
     hearing_impaired_verifiable = False
     _release_info = None
 
-    def __init__(self, language, download_link, movie_kind, title, season, episode, year, release, version, source,
+    def __init__(self, language, page_link, download_link, movie_kind, title, season, episode, year, release, version, source,
                  video_codec, tvdb_id, imdb_id, asked_for_episode=None, asked_for_release_group=None, *args, **kwargs):
-        super(ArgenteamSubtitle, self).__init__(language, download_link, *args, **kwargs)
+        super(ArgenteamSubtitle, self).__init__(language, page_link=page_link, *args, **kwargs)
+        self.page_link = page_link
         self.download_link = download_link
         self.movie_kind = movie_kind
         self.title = title
@@ -135,7 +136,8 @@ class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
     provider_name = 'argenteam'
     languages = {Language.fromalpha2(l) for l in ['es']}
     video_types = (Episode, Movie)
-    API_URL = "http://argenteam.net/api/v1/"
+    BASE_URL = "http://www.argenteam.net/"
+    API_URL = BASE_URL + "api/v1/"
     subtitle_class = ArgenteamSubtitle
     hearing_impaired_verifiable = False
     language_list = list(languages)
@@ -240,12 +242,13 @@ class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
 
             for r in content['releases']:
                 for s in r['subtitles']:
-                    sub = ArgenteamSubtitle(language, s['uri'], "episode" if is_episode else "movie", returned_title,
+                    movie_kind = "episode" if is_episode else "movie"
+                    page_link = self.BASE_URL + movie_kind + "/" + str(aid)
+                    sub = ArgenteamSubtitle(language, page_link, s['uri'], movie_kind, returned_title,
                                             season, episode, year, r.get('team'), r.get('tags'),
                                             r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id,
                                             asked_for_release_group=video.release_group,
-                                            asked_for_episode=episode
-                                            )
+                                            asked_for_episode=episode)
                     subtitles.append(sub)
 
             if has_multiple_ids:

Contents/Libraries/Shared/subliminal_patch/providers/napisy24.py

@@ -0,0 +1,124 @@
+import logging
+import os
+from io import BytesIO
+from zipfile import ZipFile
+
+from requests import Session
+
+from subliminal_patch.subtitle import Subtitle
+from subliminal_patch.providers import Provider
+from subliminal import __short_version__
+from subliminal.exceptions import AuthenticationError, ConfigurationError
+from subliminal.subtitle import fix_line_ending
+from subzero.language import Language
+
+logger = logging.getLogger(__name__)
+
+
+class Napisy24Subtitle(Subtitle):
+    '''Napisy24 Subtitle.'''
+    provider_name = 'napisy24'
+
+    def __init__(self, language, hash, imdb_id, napis_id):
+        super(Napisy24Subtitle, self).__init__(language)
+        self.hash = hash
+        self.imdb_id = imdb_id
+        self.napis_id = napis_id
+
+    @property
+    def id(self):
+        return self.hash
+
+    def get_matches(self, video):
+        matches = set()
+
+        # hash
+        if 'napisy24' in video.hashes and video.hashes['napisy24'] == self.hash:
+            matches.add('hash')
+
+        # imdb_id
+        if video.imdb_id and self.imdb_id == video.imdb_id:
+            matches.add('imdb_id')
+
+        return matches
+
+
+class Napisy24Provider(Provider):
+    '''Napisy24 Provider.'''
+    languages = {Language(l) for l in ['pol']}
+    required_hash = 'napisy24'
+    api_url = 'http://napisy24.pl/run/CheckSubAgent.php'
+
+    def __init__(self, username=None, password=None):
+        if all((username, password)):
+            self.username = username
+            self.password = password
+        else:
+            self.username = 'subliminal'
+            self.password = 'lanimilbus'
+
+        self.session = None
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+        self.session.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+    def terminate(self):
+        self.session.close()
+
+    def query(self, language, size, name, hash):
+        params = {
+            'postAction': 'CheckSub',
+            'ua': self.username,
+            'ap': self.password,
+            'fs': size,
+            'fh': hash,
+            'fn': os.path.basename(name),
+            'n24pref': 1
+        }
+
+        response = self.session.post(self.api_url, data=params, timeout=10)
+        response.raise_for_status()
+
+        response_content = response.content.split(b'||', 1)
+        n24_data = response_content[0].decode()
+
+        if n24_data[:2] != 'OK':
+            if n24_data[:11] == 'login error':
+                raise AuthenticationError('Login failed')
+            logger.error('Unknown response: %s', response.content)
+            return None
+
+        n24_status = n24_data[:4]
+        if n24_status == 'OK-0':
+            logger.info('No subtitles found')
+            return None
+
+        subtitle_info = dict(p.split(':', 1) for p in n24_data.split('|')[1:])
+        logger.debug('Subtitle info: %s', subtitle_info)
+
+        if n24_status == 'OK-1':
+            logger.info('No subtitles found but got video info')
+            return None
+        elif n24_status == 'OK-2':
+            logger.info('Found subtitles')
+        elif n24_status == 'OK-3':
+            logger.info('Found subtitles but not from Napisy24 database')
+            return None
+        
+        subtitle_content = response_content[1]
+
+        subtitle = Napisy24Subtitle(language, hash, 'tt%s' % subtitle_info['imdb'].zfill(7), subtitle_info['napisId'])
+        with ZipFile(BytesIO(subtitle_content)) as zf:
+            subtitle.content = fix_line_ending(zf.open(zf.namelist()[0]).read())
+
+        return subtitle
+
+    def list_subtitles(self, video, languages):
+        subtitles = [self.query(l, video.size, video.name, video.hashes['napisy24']) for l in languages]
+        return [s for s in subtitles if s is not None]
+
+    def download_subtitle(self, subtitle):
+        # there is no download step, content is already filled from listing subtitles
+        pass

Contents/Libraries/Shared/subliminal_patch/providers/subscene.py

@@ -4,24 +4,34 @@ import io
 import logging
 import os
 import time
+import traceback
+
+import requests
+
 import inflect
+import re
+import json
+import HTMLParser
+import urlparse
 
 from zipfile import ZipFile
 from babelfish import language_converters
 from guessit import guessit
+from dogpile.cache.api import NO_VALUE
 from subliminal import Episode, ProviderError
+from subliminal.exceptions import ConfigurationError, ServiceUnavailable
 from subliminal.utils import sanitize_release_group
+from subliminal.cache import region
 from subliminal_patch.http import RetryingCFSession
 from subliminal_patch.providers import Provider
 from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
 from subliminal_patch.subtitle import Subtitle, guess_matches
 from subliminal_patch.converters.subscene import language_ids, supported_languages
-from subscene_api.subscene import search, Subtitle as APISubtitle
+from subscene_api.subscene import search, Subtitle as APISubtitle, SITE_DOMAIN
 from subzero.language import Language
 
 p = inflect.engine()
 
-
 language_converters.register('subscene = subliminal_patch.converters.subscene:SubsceneConverter')
 logger = logging.getLogger(__name__)
 
@@ -112,16 +122,71 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
     skip_wrong_fps = False
     hearing_impaired_verifiable = True
     only_foreign = False
+    username = None
+    password = None
 
-    search_throttle = 2  # seconds
+    search_throttle = 5  # seconds
+
+    def __init__(self, only_foreign=False, username=None, password=None):
+        if not all((username, password)):
+            raise ConfigurationError('Username and password must be specified')
 
-    def __init__(self, only_foreign=False):
         self.only_foreign = only_foreign
+        self.username = username
+        self.password = password
 
     def initialize(self):
         logger.info("Creating session")
         self.session = RetryingCFSession()
 
+    def login(self):
+        r = self.session.get("https://subscene.com/account/login")
+        if "Server Error" in r.content:
+            logger.error("Login unavailable; Maintenance?")
+            raise ServiceUnavailable("Login unavailable; Maintenance?")
+
+        match = re.search(r"<script id='modelJson' type='application/json'>\s*(.+)\s*</script>", r.content)
+
+        if match:
+            h = HTMLParser.HTMLParser()
+            data = json.loads(h.unescape(match.group(1)))
+            login_url = urlparse.urljoin(data["siteUrl"], data["loginUrl"])
+            time.sleep(1.0)
+
+            r = self.session.post(login_url,
+                                  {
+                                      "username": self.username,
+                                      "password": self.password,
+                                      data["antiForgery"]["name"]: data["antiForgery"]["value"]
+                                  })
+            pep_content = re.search(r"<form method=\"post\" action=\"https://subscene\.com/\">"
+                                    r".+name=\"id_token\".+?value=\"(?P<id_token>.+?)\".*?"
+                                    r"access_token\".+?value=\"(?P<access_token>.+?)\".+?"
+                                    r"token_type.+?value=\"(?P<token_type>.+?)\".+?"
+                                    r"expires_in.+?value=\"(?P<expires_in>.+?)\".+?"
+                                    r"scope.+?value=\"(?P<scope>.+?)\".+?"
+                                    r"state.+?value=\"(?P<state>.+?)\".+?"
+                                    r"session_state.+?value=\"(?P<session_state>.+?)\"",
+                                    r.content, re.MULTILINE | re.DOTALL)
+
+            if pep_content:
+                r = self.session.post(SITE_DOMAIN, pep_content.groupdict())
+                try:
+                    r.raise_for_status()
+                except Exception:
+                    raise ProviderError("Something went wrong when trying to log in: %s", traceback.format_exc())
+                else:
+                    cj = self.session.cookies.copy()
+                    store_cks = ("scene", "idsrv", "idsrv.xsrf", "idsvr.clients", "idsvr.session", "idsvr.username")
+                    for cn in self.session.cookies.iterkeys():
+                        if cn not in store_cks:
+                            del cj[cn]
+
+                    logger.debug("Storing cookies: %r", cj)
+                    region.set("subscene_cookies2", cj)
+                    return
+        raise ProviderError("Something went wrong when trying to log in #1")
+
     def terminate(self):
         logger.info("Closing session")
         self.session.close()
@@ -176,7 +241,11 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
     def parse_results(self, video, film):
         subtitles = []
         for s in film.subtitles:
-            subtitle = SubsceneSubtitle.from_api(s)
+            try:
+                subtitle = SubsceneSubtitle.from_api(s)
+            except NotImplementedError, e:
+                logger.info(e)
+                continue
             subtitle.asked_for_release_group = video.release_group
             if isinstance(video, Episode):
                 subtitle.asked_for_episode = video.episode
@@ -189,10 +258,16 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
 
         return subtitles
 
+    def do_search(self, *args, **kwargs):
+        try:
+            return search(*args, **kwargs)
+        except requests.HTTPError:
+            region.delete("subscene_cookies2")
+
     def query(self, video):
-        vfn = get_video_filename(video)
+        # vfn = get_video_filename(video)
         subtitles = []
-        #logger.debug(u"Searching for: %s", vfn)
+        # logger.debug(u"Searching for: %s", vfn)
         # film = search(vfn, session=self.session)
         #
         # if film and film.subtitles:
@@ -201,16 +276,24 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
         # else:
         #     logger.debug('No release results found')
 
-        #time.sleep(self.search_throttle)
+        # time.sleep(self.search_throttle)
+        prev_cookies = region.get("subscene_cookies2")
+        if prev_cookies != NO_VALUE:
+            logger.debug("Re-using old subscene cookies: %r", prev_cookies)
+            self.session.cookies.update(prev_cookies)
+
+        else:
+            logger.debug("Logging in")
+            self.login()
 
         # re-search for episodes without explicit release name
         if isinstance(video, Episode):
-            #term = u"%s S%02iE%02i" % (video.series, video.season, video.episode)
+            # term = u"%s S%02iE%02i" % (video.series, video.season, video.episode)
             more_than_one = len([video.series] + video.alternative_series) > 1
-            for series in [video.series] + video.alternative_series:
+            for series in set([video.series] + video.alternative_series):
                 term = u"%s - %s Season" % (series, p.number_to_words("%sth" % video.season).capitalize())
                 logger.debug('Searching for alternative results: %s', term)
-                film = search(term, session=self.session, release=False, throttle=self.search_throttle)
+                film = self.do_search(term, session=self.session, release=False, throttle=self.search_throttle)
                 if film and film.subtitles:
                     logger.debug('Alternative results found: %s', len(film.subtitles))
                     subtitles += self.parse_results(video, film)
@@ -218,26 +301,26 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
                     logger.debug('No alternative results found')
 
                 # packs
-                if video.season_fully_aired:
-                    term = u"%s S%02i" % (series, video.season)
-                    logger.debug('Searching for packs: %s', term)
-                    time.sleep(self.search_throttle)
-                    film = search(term, session=self.session, throttle=self.search_throttle)
-                    if film and film.subtitles:
-                        logger.debug('Pack results found: %s', len(film.subtitles))
-                        subtitles += self.parse_results(video, film)
-                    else:
-                        logger.debug('No pack results found')
-                else:
-                    logger.debug("Not searching for packs, because the season hasn't fully aired")
+                # if video.season_fully_aired:
+                #     term = u"%s S%02i" % (series, video.season)
+                #     logger.debug('Searching for packs: %s', term)
+                #     time.sleep(self.search_throttle)
+                #     film = search(term, session=self.session, throttle=self.search_throttle)
+                #     if film and film.subtitles:
+                #         logger.debug('Pack results found: %s', len(film.subtitles))
+                #         subtitles += self.parse_results(video, film)
+                #     else:
+                #         logger.debug('No pack results found')
+                # else:
+                #     logger.debug("Not searching for packs, because the season hasn't fully aired")
                 if more_than_one:
                     time.sleep(self.search_throttle)
         else:
             more_than_one = len([video.title] + video.alternative_titles) > 1
-            for title in [video.title] + video.alternative_titles:
-                logger.debug('Searching for movie results: %s', title)
-                film = search(title, year=video.year, session=self.session, limit_to=None, release=False,
-                              throttle=self.search_throttle)
+            for title in set([video.title] + video.alternative_titles):
+                logger.debug('Searching for movie results: %r', title)
+                film = self.do_search(title, year=video.year, session=self.session, limit_to=None, release=False,
+                                      throttle=self.search_throttle)
                 if film and film.subtitles:
                     subtitles += self.parse_results(video, film)
                 if more_than_one:

Contents/Libraries/Shared/subliminal_patch/subtitle.py

@@ -117,14 +117,14 @@ class Subtitle(Subtitle_):
 
         logger.info('Guessing encoding for language %s', self.language)
 
-        encodings = ['utf-8', 'utf-16']
+        encodings = ['utf-8']
 
         # add language-specific encodings
         # http://scratchpad.wikia.com/wiki/Character_Encoding_Recommendation_for_Languages
 
         if self.language.alpha3 == 'zho':
             encodings.extend(['cp936', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022_jp_2', 'cp950', 'gb18030', 'big5',
-                              'big5hkscs'])
+                              'big5hkscs', 'utf-16'])
         elif self.language.alpha3 == 'jpn':
             encodings.extend(['shift-jis', 'cp932', 'euc_jp', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2',
                               'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext', ])
@@ -133,7 +133,7 @@ class Subtitle(Subtitle_):
 
         # arabian/farsi
         elif self.language.alpha3 in ('ara', 'fas', 'per'):
-            encodings.append('windows-1256')
+            encodings.extend(['windows-1256', 'utf-16'])
         elif self.language.alpha3 == 'heb':
             encodings.extend(['windows-1255', 'iso-8859-8'])
         elif self.language.alpha3 == 'tur':
@@ -251,8 +251,7 @@ class Subtitle(Subtitle_):
                 subs = pysubs2.SSAFile.from_string(text, fps=self.plex_media_fps)
 
             unicontent = self.pysubs2_to_unicode(subs)
-            self.content = unicontent.encode("utf-8")
-            self._guessed_encoding = "utf-8"
+            self.content = unicontent.encode(self._guessed_encoding)
         except:
             logger.exception("Couldn't convert subtitle %s to .srt format: %s", self, traceback.format_exc())
             return False
@@ -320,7 +319,8 @@ class Subtitle(Subtitle_):
         :return: string 
         """
         if not self.mods:
-            return fix_text(self.content.decode("utf-8"), **ftfy_defaults).encode(encoding="utf-8")
+            return fix_text(self.content.decode(encoding=self._guessed_encoding), **ftfy_defaults).encode(
+                encoding=self._guessed_encoding)
 
         submods = SubtitleModifications(debug=debug)
         if submods.load(content=self.text, language=self.language):
@@ -329,7 +329,7 @@ class Subtitle(Subtitle_):
             self.mods = submods.mods_used
 
             content = fix_text(self.pysubs2_to_unicode(submods.f, format=format), **ftfy_defaults)\
-                .encode(encoding="utf-8")
+                .encode(encoding=self._guessed_encoding)
             submods.f = None
             del submods
             return content

Contents/Libraries/Shared/subscene_api/subscene.py

@@ -30,6 +30,7 @@ import enum
 import sys
 import requests
 import time
+import logging
 
 is_PY2 = sys.version_info[0] < 3
 if is_PY2:
@@ -39,8 +40,13 @@ else:
     from contextlib import suppress
     from urllib2.request import Request, urlopen
 
+from dogpile.cache.api import NO_VALUE
+from subliminal.cache import region
 from bs4 import BeautifulSoup, NavigableString
 
+
+logger = logging.getLogger(__name__)
+
 # constants
 HEADERS = {
 }
@@ -50,14 +56,21 @@ DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWeb"\
                      "Kit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36"
 
 
+ENDPOINT_RE = re.compile(ur'(?uis)<form.+?action="/subtitles/(.+)">.*?<input type="text"')
+
+
+class NewEndpoint(Exception):
+    pass
+
+
 # utils
-def soup_for(url, session=None, user_agent=DEFAULT_USER_AGENT):
+def soup_for(url, data=None, session=None, user_agent=DEFAULT_USER_AGENT):
     url = re.sub("\s", "+", url)
     if not session:
         r = Request(url, data=None, headers=dict(HEADERS, **{"User-Agent": user_agent}))
         html = urlopen(r).read().decode("utf-8")
     else:
-        ret = session.get(url)
+        ret = session.post(url, data=data)
         ret.raise_for_status()
         html = ret.text
     return BeautifulSoup(html, "html.parser")
@@ -242,29 +255,42 @@ def get_first_film(soup, section, year=None, session=None):
                 url = SITE_DOMAIN + t.div.a.get("href")
                 break
         if not url:
-            return
+            # fallback to non-year results
+            logger.info("Falling back to non-year results as year wasn't found (%s)", year)
+            url = SITE_DOMAIN + tag.findNext("ul").find("li").div.a.get("href")
 
     return Film.from_url(url, session=session)
 
 
+def find_endpoint(session, content=None):
+    endpoint = region.get("subscene_endpoint2")
+    if endpoint is NO_VALUE:
+        if not content:
+            content = session.get(SITE_DOMAIN).text
+
+        m = ENDPOINT_RE.search(content)
+        if m:
+            endpoint = m.group(1).strip()
+            logger.debug("Switching main endpoint to %s", endpoint)
+            region.set("subscene_endpoint2", endpoint)
+    return endpoint
+
+
 def search(term, release=True, session=None, year=None, limit_to=SearchTypes.Exact, throttle=0):
     # note to subscene: if you actually start to randomize the endpoint, we'll have to query your server even more
-    endpoints = ["searching", "search", "srch", "find"]
-    if release:
-        endpoints = ["release"]
 
-    soup = None
-    for endpoint in endpoints:
-        try:
-            soup = soup_for("%s/subtitles/%s?q=%s" % (SITE_DOMAIN, endpoint, term),
-                            session=session)
-        except requests.HTTPError, e:
-            if e.response.status_code == 404:
-                time.sleep(throttle)
-                # fixme: detect endpoint from html
-                continue
-            raise
-        break
+    if release:
+        endpoint = "release"
+    else:
+        endpoint = find_endpoint(session)
+        time.sleep(throttle)
+
+    if not endpoint:
+        logger.error("Couldn't find endpoint, exiting")
+        return
+
+    soup = soup_for("%s/subtitles/%s" % (SITE_DOMAIN, endpoint), data={"query": term},
+                    session=session)
 
     if soup:
         if "Subtitle search by" in str(soup):

Contents/Libraries/Shared/subzero/video.py

@@ -52,10 +52,10 @@ def set_existing_languages(video, video_info, external_subtitles=False, embedded
             video.subtitle_languages.add(language)
 
 
-def parse_video(fn, hints, skip_hashing=False, dry_run=False, providers=None):
+def parse_video(fn, hints, skip_hashing=False, dry_run=False, providers=None, hash_from=None):
     logger.debug("Parsing video: %s, hints: %s", os.path.basename(fn), hints)
     return scan_video(fn, hints=hints, dont_use_actual_file=dry_run, providers=providers,
-                      skip_hashing=skip_hashing)
+                      skip_hashing=skip_hashing, hash_from=hash_from)
 
 
 def refine_video(video, no_refining=False, refiner_settings=None):

README.md

@@ -90,29 +90,22 @@ the.vbm, mmgoodnow, Vertig0ne, thliu78, tattoomees, ostman, count_confucius, ehe
 
 ## Changelog
 
-
-2.6.5.3074
+2.6.5.3109
 
 subscene, addic7ed and titlovi
 - either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
 
 Changelog
-- core: cf: bypass cf 95% of the time without captchas
-- core: fix breaking line endings of certain languages (chinese, UTF-16); fixes #646
-- core: update pysubs2 to 0.2.3
-
-
-2.6.5.3062
-
-Changelog
-- core: cf: optimize
-- core: http: don't query DNS with IPs. thanks @fgump (fixes sonarr/radarr)
+- providers: add Napisy24 (polish)
+- providers: subscene: reduce provider load by possibly half
+- providers: subscene: support logging in (username/password are now required)
+- providers: subscene: fallback to non year results if none found with year
 
 
 [older changes](CHANGELOG.md)
 
 
-Subtitles provided by [OpenSubtitles.org](http://www.opensubtitles.org/), [Podnapisi.NET](https://www.podnapisi.net/), [TVSubtitles.net](http://www.tvsubtitles.net/), [Addic7ed.com](http://www.addic7ed.com/), [Legendas TV](http://legendas.tv/), [Napi Projekt](http://www.napiprojekt.pl/), [Shooter](http://shooter.cn/), [Titlovi](http://titlovi.com), [aRGENTeaM](http://argenteam.net), [SubScene](https://subscene.com/), [Hosszupuska](http://hosszupuskasub.com/)
+Subtitles provided by [OpenSubtitles.org](http://www.opensubtitles.org/), [Podnapisi.NET](https://www.podnapisi.net/), [TVSubtitles.net](http://www.tvsubtitles.net/), [Addic7ed.com](http://www.addic7ed.com/), [Legendas TV](http://legendas.tv/), [Napi Projekt](http://www.napiprojekt.pl/), [Shooter](http://shooter.cn/), [Titlovi](http://titlovi.com), [aRGENTeaM](http://argenteam.net), [SubScene](https://subscene.com/), [Hosszupuska](http://hosszupuskasub.com/), [Napisy24](https://napisy24.pl/)
 
 [3rd party licenses](https://github.com/pannal/Sub-Zero.bundle/tree/master/Licenses)