incremental tmp

core: update certifi to 2020.6.20

CHANGELOG.md

@@ -1,3 +1,41 @@
+2.6.5.3217
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: also extract (missing) embedded subtitles when SearchAllRecentlyAddedMissing is running
+- core: core: clarify detecting streams (in logs)
+- core: UnRAR: set binary to executable, even if not checked out from git; might fix #693
+- core: bazarr-backport: morpheus65535/bazarr#703: use proper language code detection instead of a wild guess; should fix bad existing subtitle detection
+- core: bazarr-backport: morpheus65535/bazarr#660: fix BOM encoding stuff
+- core: bazarr-backport: morpheus65535/bazarr#656 further generalize formats; skip release group match if format match failed
+- core: fix stream detection when using mediainfo (#711)
+- config/core: make periodic SZ-internal subtitle maintenance interval configurable
+- providers: add BSPlayer Subtitles
+- providers: add ScrewZira (Hebrew)
+
+2.6.5.3183
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: don't fall back to default providers if none enabled
+- core: don't process any further if stream info is missing
+- core: support using mediainfo for retrieving MP4 MOV_TEXT subtitle stream titles (PMS bug)
+- core: fix embedded subtitle extraction in some cases (#681, #680)
+- core: scanning: add additional INFO logging for undetected languages
+- core: bazarr-backport: remove existing subtitle file, to support MergerFS
+- core: bazarr-backport: generic 10 minute throttling if uncaught exception occurs
+- providers: addic7ed: fix recaptcha solving; fix show ID retrieval (#681)
+- providers: addic7ed: add timeout on authentication error
+- providers: addic7ed: fix shows with dots in them (Mayans M.C.)
+- providers: addic7ed: fix detection of completed subtitle for non-english users (#686)
+- providers: addic7ed: add more timeouts in the login process
+- providers: argenteam: bazarr-backport: use new url; fixes
+
+
 2.6.5.3152
 
 subscene, addic7ed

Contents/Code/__init__.py

@@ -262,7 +262,8 @@ class SubZeroAgent(object):
 
 
 class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies):
-    contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', 'com.plexapp.agents.hama']
+    contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb',
+                      'com.plexapp.agents.hama',  'tv.plex.agents.movie']
     score_prefs_key = "subtitles.search.minimumMovieScore2"
     agent_type_verbose = "Movies"
 

Contents/Code/support/config.py

@@ -10,12 +10,15 @@ import jstyleson
 import datetime
 import stat
 import traceback
+import socket
+import requests
 
 import subliminal
 import subliminal_patch
 import subzero.constants
 import lib
-from subliminal.exceptions import ServiceUnavailable, DownloadLimitExceeded, AuthenticationError
+from subliminal.exceptions import ServiceUnavailable, DownloadLimitExceeded, AuthenticationError, \
+    DownloadLimitPerDayExceeded
 from subliminal_patch.core import is_windows_special_path
 from whichdb import whichdb
 
@@ -61,15 +64,22 @@ def int_or_default(s, default):
         return default
 
 
-VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled)
+VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, DownloadLimitPerDayExceeded,
+                             ServiceUnavailable, APIThrottled, requests.Timeout, requests.ReadTimeout, socket.timeout)
+
+def_timeout = (datetime.timedelta(minutes=20), "20 minutes")
 
 PROVIDER_THROTTLE_MAP = {
     "default": {
         TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
         DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
+        DownloadLimitPerDayExceeded: (datetime.timedelta(hours=4), "4 hours"),
         ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
         APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
         AuthenticationError: (datetime.timedelta(hours=2), "2 hours"),
+        requests.Timeout: def_timeout,
+        socket.timeout: def_timeout,
+        requests.ReadTimeout: def_timeout,
     },
     "opensubtitles": {
         TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
@@ -663,12 +673,18 @@ class Config(object):
             if not agent.primary:
                 continue
 
-            for t in list(agent.media_types):
-                if t.media_type in (MOVIE, SHOW):
-                    related_agents = Plex.primary_agent(agent.identifier, t.media_type)
+            media_types = [t.media_type for t in list(agent.media_types)]
+
+            # the new movie agent doesn't populate its media types, workaround
+            if not media_types and agent.identifier == "tv.plex.agents.movie":
+                media_types = [MOVIE]
+
+            for media_type in media_types:
+                if media_type in (MOVIE, SHOW):
+                    related_agents = Plex.primary_agent(agent.identifier, media_type)
                     for a in related_agents:
                         if a.identifier == PLUGIN_IDENTIFIER and a.enabled:
-                            enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[t.media_type]].append(agent.identifier)
+                            enabled_for_primary_agents[MEDIA_TYPE_TO_STRING[media_type]].append(agent.identifier)
 
         # find the libraries that use them
         for library in self.sections:
@@ -795,7 +811,9 @@ class Config(object):
                      'argenteam': cast_bool(Prefs['provider.argenteam.enabled']),
                      'subscenter': False,
                      'assrt': cast_bool(Prefs['provider.assrt.enabled']),
-                     }
+                     'bsplayer': cast_bool(Prefs['provider.bsplayer.enabled']),
+                     'screwzira': cast_bool(Prefs['provider.screwzira.enabled']),
+                }
 
     @property
     def providers_enabled(self):
@@ -824,6 +842,8 @@ class Config(object):
             providers["assrt"] = False
             providers["subscene"] = False
             providers["napisy24"] = False
+            providers["bsplayer"] = False
+            providers["screwzira"] = False
             providers_forced_off = dict(providers)
 
         if not self.unrar and providers["legendastv"]:
@@ -869,6 +889,7 @@ class Config(object):
 
         provider_settings = {'addic7ed': {'username': Prefs['provider.addic7ed.username'],
                                           'password': Prefs['provider.addic7ed.password'],
+                                          'is_vip': cast_bool(Prefs['provider.addic7ed.is_vip']),
                                           },
                              'opensubtitles': {'username': Prefs['provider.opensubtitles.username'],
                                                'password': Prefs['provider.opensubtitles.password'],

Contents/Code/support/plex_media.py

@@ -184,6 +184,13 @@ def get_all_parts(plex_item):
 
 
 def update_stream_info(part):
+    try:
+        return update_stream_info_(part)
+    except:
+        Log.Exception("Getting Mediainfo failed for: %s", part.file)
+
+
+def update_stream_info_(part):
     if config.mediainfo_bin and part.container == "mp4":
         cmdline = '%s --Inform="Text;-%%ID%%_%%Title%%" %s' % (config.mediainfo_bin, helpers.quote(part.file))
         result = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True)
@@ -198,7 +205,10 @@ def update_stream_info(part):
             else:
                 filled = []
                 for stream in part.streams:
-                    index = stream.index+1
+                    if stream.index is None:
+                        Log.Debug("Found stream with no index: %r", stream)
+
+                    index = stream.index+1 if stream.index is not None else 1
                     if index in stream_titles:
                         stream.title = stream_titles[index]
                         filled.append(index-1)

Contents/Code/support/tasks.py

@@ -171,12 +171,15 @@ class SubtitleListingMixin(object):
                     else:
                         s.wrong_season_ep = True
 
+            orig_matches = matches.copy()
+            score, score_without_hash = compute_score(matches, s, video, hearing_impaired=use_hearing_impaired)
+
             unsorted_subtitles.append(
-                (s, compute_score(matches, s, video, hearing_impaired=use_hearing_impaired), matches))
-        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1), reverse=True)
+                (s, score, score_without_hash, matches, orig_matches))
+        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1, 2), reverse=True)
 
         subtitles = []
-        for subtitle, score, matches in scored_subtitles:
+        for subtitle, score, score_without_hash, matches, orig_matches in scored_subtitles:
             # check score
             if score < min_score and not subtitle.wrong_series:
                 Log.Info(u'%s: Score %d is below min_score (%d)', self.name, score, min_score)
@@ -667,7 +670,7 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
         min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
         min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
         min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
-        min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 82
+        min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 112
         overwrite_manually_modified = cast_bool(
             Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
         overwrite_manually_selected = cast_bool(

Contents/DefaultPrefs.json

@@ -375,6 +375,12 @@
     "default": "",
     "secure": "true"
   },
+  {
+    "id": "provider.addic7ed.is_vip",
+    "label": "Addic7ed VIP? (80 vs 40 downloads per day)",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "provider.addic7ed.boost_by2",
     "label": "Addic7ed: boost score (if requirements met)",
@@ -509,6 +515,18 @@
     "type": "text",
     "default": ""
   },
+  {
+    "id": "provider.bsplayer.enabled",
+    "label": "Provider: Enable BSPlayer Subtitles",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "provider.screwzira.enabled",
+    "label": "Provider: Enable ScrewZira (Hebrew)",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "providers.multithreading",
     "label": "Search enabled providers simultaneously (multithreading)",
@@ -794,6 +812,29 @@
     "type": "bool",
     "default": "false"
   },
+  {
+    "id": "scheduler.tasks.SubtitleStorageMaintenance.frequency",
+    "label": "Scheduler: Periodically run subtitle storage maintenance (SZ internal)",
+    "type": "enum",
+    "values": [
+      "never",
+      "every 6 hours",
+      "every 12 hours",
+      "every 24 hours",
+      "every 1 days",
+      "every 2 days",
+      "every 3 days",
+      "every 4 days",
+      "every 1 weeks",
+      "every 2 weeks",
+      "every 3 weeks",
+      "every 4 weeks",
+      "every 5 weeks",
+      "every 6 weeks",
+      "every 12 weeks"
+    ],
+    "default": "every 1 weeks"
+  },
   {
     "id": "history_size",
     "label": "History: amount of items to store historical data for",

Contents/Info.plist

@@ -13,7 +13,7 @@
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.6.5.3195</string>
+        <string>2.6.5.3247</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.6.5.3195 DEV
+Version 2.6.5.3247 DEV
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 

Contents/Libraries/Shared/backports/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

Contents/Libraries/Shared/backports/functools_lru_cache.py

@@ -0,0 +1,196 @@
+from __future__ import absolute_import
+
+import functools
+from collections import namedtuple
+from threading import RLock
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+
+@functools.wraps(functools.update_wrapper)
+def update_wrapper(
+    wrapper,
+    wrapped,
+    assigned=functools.WRAPPER_ASSIGNMENTS,
+    updated=functools.WRAPPER_UPDATES,
+):
+    """
+    Patch two bugs in functools.update_wrapper.
+    """
+    # workaround for http://bugs.python.org/issue3445
+    assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr))
+    wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated)
+    # workaround for https://bugs.python.org/issue17482
+    wrapper.__wrapped__ = wrapped
+    return wrapper
+
+
+class _HashedSeq(list):
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+
+def _make_key(
+    args,
+    kwds,
+    typed,
+    kwd_mark=(object(),),
+    fasttypes=set([int, str, frozenset, type(None)]),
+    sorted=sorted,
+    tuple=tuple,
+    type=type,
+    len=len,
+):
+    'Make a cache key from optionally typed positional and keyword arguments'
+    key = args
+    if kwds:
+        sorted_items = sorted(kwds.items())
+        key += kwd_mark
+        for item in sorted_items:
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for k, v in sorted_items)
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+
+def lru_cache(maxsize=100, typed=False):
+    """Least-recently-used cache decorator.
+
+    If *maxsize* is set to None, the LRU features are disabled and the cache
+    can grow without bound.
+
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
+    Arguments to the cached function must be hashable.
+
+    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
+    f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    Access the underlying function with f.__wrapped__.
+
+    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+
+    """
+
+    # Users should only access the lru_cache through its public API:
+    #       cache_info, cache_clear, and f.__wrapped__
+    # The internals of the lru_cache are encapsulated for thread safety and
+    # to allow the implementation to change (including a possible C version).
+
+    def decorating_function(user_function):
+
+        cache = dict()
+        stats = [0, 0]  # make statistics updateable non-locally
+        HITS, MISSES = 0, 1  # names for the stats fields
+        make_key = _make_key
+        cache_get = cache.get  # bound method to lookup key or return None
+        _len = len  # localize the global len() function
+        lock = RLock()  # because linkedlist updates aren't threadsafe
+        root = []  # root of the circular doubly linked list
+        root[:] = [root, root, None, None]  # initialize by pointing to self
+        nonlocal_root = [root]  # make updateable non-locally
+        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3  # names for the link fields
+
+        if maxsize == 0:
+
+            def wrapper(*args, **kwds):
+                # no caching, just do a statistics update after a successful call
+                result = user_function(*args, **kwds)
+                stats[MISSES] += 1
+                return result
+
+        elif maxsize is None:
+
+            def wrapper(*args, **kwds):
+                # simple caching without ordering or size limit
+                key = make_key(args, kwds, typed)
+                result = cache_get(
+                    key, root
+                )  # root used here as a unique not-found sentinel
+                if result is not root:
+                    stats[HITS] += 1
+                    return result
+                result = user_function(*args, **kwds)
+                cache[key] = result
+                stats[MISSES] += 1
+                return result
+
+        else:
+
+            def wrapper(*args, **kwds):
+                # size limited caching that tracks accesses by recency
+                key = make_key(args, kwds, typed) if kwds or typed else args
+                with lock:
+                    link = cache_get(key)
+                    if link is not None:
+                        # record recent use of the key by moving it
+                        # to the front of the list
+                        root, = nonlocal_root
+                        link_prev, link_next, key, result = link
+                        link_prev[NEXT] = link_next
+                        link_next[PREV] = link_prev
+                        last = root[PREV]
+                        last[NEXT] = root[PREV] = link
+                        link[PREV] = last
+                        link[NEXT] = root
+                        stats[HITS] += 1
+                        return result
+                result = user_function(*args, **kwds)
+                with lock:
+                    root, = nonlocal_root
+                    if key in cache:
+                        # getting here means that this same key was added to the
+                        # cache while the lock was released.  since the link
+                        # update is already done, we need only return the
+                        # computed result and update the count of misses.
+                        pass
+                    elif _len(cache) >= maxsize:
+                        # use the old root to store the new key and result
+                        oldroot = root
+                        oldroot[KEY] = key
+                        oldroot[RESULT] = result
+                        # empty the oldest link and make it the new root
+                        root = nonlocal_root[0] = oldroot[NEXT]
+                        oldkey = root[KEY]
+                        root[KEY] = root[RESULT] = None
+                        # now update the cache dictionary for the new links
+                        del cache[oldkey]
+                        cache[key] = oldroot
+                    else:
+                        # put result in a new link at the front of the list
+                        last = root[PREV]
+                        link = [last, root, key, result]
+                        last[NEXT] = root[PREV] = cache[key] = link
+                    stats[MISSES] += 1
+                return result
+
+        def cache_info():
+            """Report cache statistics"""
+            with lock:
+                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))
+
+        def cache_clear():
+            """Clear the cache and cache statistics"""
+            with lock:
+                cache.clear()
+                root = nonlocal_root[0]
+                root[:] = [root, root, None, None]
+                stats[:] = [0, 0]
+
+        wrapper.__wrapped__ = user_function
+        wrapper.cache_info = cache_info
+        wrapper.cache_clear = cache_clear
+        return update_wrapper(wrapper, user_function)
+
+    return decorating_function

Contents/Libraries/Shared/certifi/__init__.py

@@ -1,3 +1,3 @@
-from .core import where
+from .core import contents, where
 
-__version__ = "2019.03.09"
+__version__ = "2020.06.20"

Contents/Libraries/Shared/certifi/__main__.py

@@ -1,2 +1,12 @@
-from certifi import where
-print(where())
+import argparse
+
+from certifi import contents, where
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-c", "--contents", action="store_true")
+args = parser.parse_args()
+
+if args.contents:
+    print(contents())
+else:
+    print(where())

Contents/Libraries/Shared/certifi/cacert.pem

@@ -58,38 +58,6 @@ AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7
 TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg==
 -----END CERTIFICATE-----
 
-# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only
-# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only
-# Label: "Verisign Class 3 Public Primary Certification Authority - G3"
-# Serial: 206684696279472310254277870180966723415
-# MD5 Fingerprint: cd:68:b6:a7:c7:c4:ce:75:e0:1d:4f:57:44:61:92:09
-# SHA1 Fingerprint: 13:2d:0d:45:53:4b:69:97:cd:b2:d5:c3:39:e2:55:76:60:9b:5c:c6
-# SHA256 Fingerprint: eb:04:cf:5e:b1:f3:9a:fa:76:2f:2b:b1:20:f2:96:cb:a5:20:c1:b9:7d:b1:58:95:65:b8:1c:b9:a1:7b:72:44
------BEGIN CERTIFICATE-----
-MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw
-CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl
-cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu
-LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT
-aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp
-dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD
-VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT
-aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ
-bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu
-IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg
-LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b
-N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t
-KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu
-kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm
-CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ
-Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu
-imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te
-2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe
-DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC
-/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p
-F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt
-TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ==
------END CERTIFICATE-----
-
 # Issuer: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited
 # Subject: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited
 # Label: "Entrust.net Premium 2048 Secure Server CA"
@@ -152,39 +120,6 @@ ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS
 R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp
 -----END CERTIFICATE-----
 
-# Issuer: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network
-# Subject: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network
-# Label: "AddTrust External Root"
-# Serial: 1
-# MD5 Fingerprint: 1d:35:54:04:85:78:b0:3f:42:42:4d:bf:20:73:0a:3f
-# SHA1 Fingerprint: 02:fa:f3:e2:91:43:54:68:60:78:57:69:4d:f5:e4:5b:68:85:18:68
-# SHA256 Fingerprint: 68:7f:a4:51:38:22:78:ff:f0:c8:b1:1f:8d:43:d5:76:67:1c:6e:b2:bc:ea:b4:13:fb:83:d9:65:d0:6d:2f:f2
------BEGIN CERTIFICATE-----
-MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEU
-MBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFs
-IFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290
-MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFowbzELMAkGA1UEBhMCU0Ux
-FDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRUcnVzdCBFeHRlcm5h
-bCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0EgUm9v
-dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvt
-H7xsD821+iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9
-uMq/NzgtHj6RQa1wVsfwTz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzX
-mk6vBbOmcZSccbNQYArHE504B4YCqOmoaSYYkKtMsE8jqzpPhNjfzp/haW+710LX
-a0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy2xSoRcRdKn23tNbE7qzN
-E0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv77+ldU9U0
-WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYD
-VR0PBAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0
-Jvf6xCZU7wO94CTLVBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRU
-cnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsx
-IjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENBIFJvb3SCAQEwDQYJKoZIhvcN
-AQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZlj7DYd7usQWxH
-YINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5
-6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvC
-Nr4TDea9Y355e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEX
-c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a
-mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ=
------END CERTIFICATE-----
-
 # Issuer: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc.
 # Subject: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc.
 # Label: "Entrust Root Certification Authority"
@@ -771,36 +706,6 @@ vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
 +OkuE6N36B9K
 -----END CERTIFICATE-----
 
-# Issuer: CN=Class 2 Primary CA O=Certplus
-# Subject: CN=Class 2 Primary CA O=Certplus
-# Label: "Certplus Class 2 Primary CA"
-# Serial: 177770208045934040241468760488327595043
-# MD5 Fingerprint: 88:2c:8c:52:b8:a2:3c:f3:f7:bb:03:ea:ae:ac:42:0b
-# SHA1 Fingerprint: 74:20:74:41:72:9c:dd:92:ec:79:31:d8:23:10:8d:c2:81:92:e2:bb
-# SHA256 Fingerprint: 0f:99:3c:8a:ef:97:ba:af:56:87:14:0e:d5:9a:d1:82:1b:b4:af:ac:f0:aa:9a:58:b5:d5:7a:33:8a:3a:fb:cb
------BEGIN CERTIFICATE-----
-MIIDkjCCAnqgAwIBAgIRAIW9S/PY2uNp9pTXX8OlRCMwDQYJKoZIhvcNAQEFBQAw
-PTELMAkGA1UEBhMCRlIxETAPBgNVBAoTCENlcnRwbHVzMRswGQYDVQQDExJDbGFz
-cyAyIFByaW1hcnkgQ0EwHhcNOTkwNzA3MTcwNTAwWhcNMTkwNzA2MjM1OTU5WjA9
-MQswCQYDVQQGEwJGUjERMA8GA1UEChMIQ2VydHBsdXMxGzAZBgNVBAMTEkNsYXNz
-IDIgUHJpbWFyeSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANxQ
-ltAS+DXSCHh6tlJw/W/uz7kRy1134ezpfgSN1sxvc0NXYKwzCkTsA18cgCSR5aiR
-VhKC9+Ar9NuuYS6JEI1rbLqzAr3VNsVINyPi8Fo3UjMXEuLRYE2+L0ER4/YXJQyL
-kcAbmXuZVg2v7tK8R1fjeUl7NIknJITesezpWE7+Tt9avkGtrAjFGA7v0lPubNCd
-EgETjdyAYveVqUSISnFOYFWe2yMZeVYHDD9jC1yw4r5+FfyUM1hBOHTE4Y+L3yas
-H7WLO7dDWWuwJKZtkIvEcupdM5i3y95ee++U8Rs+yskhwcWYAqqi9lt3m/V+llU0
-HGdpwPFC40es/CgcZlUCAwEAAaOBjDCBiTAPBgNVHRMECDAGAQH/AgEKMAsGA1Ud
-DwQEAwIBBjAdBgNVHQ4EFgQU43Mt38sOKAze3bOkynm4jrvoMIkwEQYJYIZIAYb4
-QgEBBAQDAgEGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHA6Ly93d3cuY2VydHBsdXMu
-Y29tL0NSTC9jbGFzczIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCnVM+IRBnL39R/
-AN9WM2K191EBkOvDP9GIROkkXe/nFL0gt5o8AP5tn9uQ3Nf0YtaLcF3n5QRIqWh8
-yfFC82x/xXp8HVGIutIKPidd3i1RTtMTZGnkLuPT55sJmabglZvOGtd/vjzOUrMR
-FcEPF80Du5wlFbqidon8BvEY0JNLDnyCt6X09l/+7UCmnYR0ObncHoUW2ikbhiMA
-ybuJfm6AiB4vFLQDJKgybwOaRywwvlbGp0ICcBvqQNi6BQNwB6SW//1IMwrh3KWB
-kJtN3X3n57LNXMhqlfil9o3EXXgIvnsG1knPGTZQIy4I5p4FTUcY1Rbpsda2ENW7
-l7+ijrRU
------END CERTIFICATE-----
-
 # Issuer: CN=DST Root CA X3 O=Digital Signature Trust Co.
 # Subject: CN=DST Root CA X3 O=Digital Signature Trust Co.
 # Label: "DST Root CA X3"
@@ -1219,36 +1124,6 @@ t0QmwCbAr1UwnjvVNioZBPRcHv/PLLf/0P2HQBHVESO7SMAhqaQoLf0V+LBOK/Qw
 WyH8EZE0vkHve52Xdf+XlcCWWC/qu0bXu+TZLg==
 -----END CERTIFICATE-----
 
-# Issuer: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center
-# Subject: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center
-# Label: "Deutsche Telekom Root CA 2"
-# Serial: 38
-# MD5 Fingerprint: 74:01:4a:91:b1:08:c4:58:ce:47:cd:f0:dd:11:53:08
-# SHA1 Fingerprint: 85:a4:08:c0:9c:19:3e:5d:51:58:7d:cd:d6:13:30:fd:8c:de:37:bf
-# SHA256 Fingerprint: b6:19:1a:50:d0:c3:97:7f:7d:a9:9b:cd:aa:c8:6a:22:7d:ae:b9:67:9e:c7:0b:a3:b0:c9:d9:22:71:c1:70:d3
------BEGIN CERTIFICATE-----
-MIIDnzCCAoegAwIBAgIBJjANBgkqhkiG9w0BAQUFADBxMQswCQYDVQQGEwJERTEc
-MBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxlU2Vj
-IFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290IENB
-IDIwHhcNOTkwNzA5MTIxMTAwWhcNMTkwNzA5MjM1OTAwWjBxMQswCQYDVQQGEwJE
-RTEcMBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxl
-U2VjIFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290
-IENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrC6M14IspFLEU
-ha88EOQ5bzVdSq7d6mGNlUn0b2SjGmBmpKlAIoTZ1KXleJMOaAGtuU1cOs7TuKhC
-QN/Po7qCWWqSG6wcmtoIKyUn+WkjR/Hg6yx6m/UTAtB+NHzCnjwAWav12gz1Mjwr
-rFDa1sPeg5TKqAyZMg4ISFZbavva4VhYAUlfckE8FQYBjl2tqriTtM2e66foai1S
-NNs671x1Udrb8zH57nGYMsRUFUQM+ZtV7a3fGAigo4aKSe5TBY8ZTNXeWHmb0moc
-QqvF1afPaA+W5OFhmHZhyJF81j4A4pFQh+GdCuatl9Idxjp9y7zaAzTVjlsB9WoH
-txa2bkp/AgMBAAGjQjBAMB0GA1UdDgQWBBQxw3kbuvVT1xfgiXotF2wKsyudMzAP
-BgNVHRMECDAGAQH/AgEFMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOC
-AQEAlGRZrTlk5ynrE/5aw4sTV8gEJPB0d8Bg42f76Ymmg7+Wgnxu1MM9756Abrsp
-tJh6sTtU6zkXR34ajgv8HzFZMQSyzhfzLMdiNlXiItiJVbSYSKpk+tYcNthEeFpa
-IzpXl/V6ME+un2pMSyuOoAPjPuCp1NJ70rOo4nI8rZ7/gFnkm0W09juwzTkZmDLl
-6iFhkOQxIY40sfcvNUqFENrnijchvllj4PKFiDFT1FQUhXB59C4Gdyd1Lx+4ivn+
-xbrYNuSD7Odlt79jWvNGr4GUN9RBjNYj1h7P9WgbRGOiWrqnNVmh5XAFmw4jV5mU
-Cm26OWMohpLzGITY+9HPBVZkVw==
------END CERTIFICATE-----
-
 # Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc
 # Subject: CN=Cybertrust Global Root O=Cybertrust, Inc
 # Label: "Cybertrust Global Root"
@@ -1559,47 +1434,6 @@ uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2
 XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E=
 -----END CERTIFICATE-----
 
-# Issuer: CN=Staat der Nederlanden Root CA - G2 O=Staat der Nederlanden
-# Subject: CN=Staat der Nederlanden Root CA - G2 O=Staat der Nederlanden
-# Label: "Staat der Nederlanden Root CA - G2"
-# Serial: 10000012
-# MD5 Fingerprint: 7c:a5:0f:f8:5b:9a:7d:6d:30:ae:54:5a:e3:42:a2:8a
-# SHA1 Fingerprint: 59:af:82:79:91:86:c7:b4:75:07:cb:cf:03:57:46:eb:04:dd:b7:16
-# SHA256 Fingerprint: 66:8c:83:94:7d:a6:3b:72:4b:ec:e1:74:3c:31:a0:e6:ae:d0:db:8e:c5:b3:1b:e3:77:bb:78:4f:91:b6:71:6f
------BEGIN CERTIFICATE-----
-MIIFyjCCA7KgAwIBAgIEAJiWjDANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJO
-TDEeMBwGA1UECgwVU3RhYXQgZGVyIE5lZGVybGFuZGVuMSswKQYDVQQDDCJTdGFh
-dCBkZXIgTmVkZXJsYW5kZW4gUm9vdCBDQSAtIEcyMB4XDTA4MDMyNjExMTgxN1oX
-DTIwMDMyNTExMDMxMFowWjELMAkGA1UEBhMCTkwxHjAcBgNVBAoMFVN0YWF0IGRl
-ciBOZWRlcmxhbmRlbjErMCkGA1UEAwwiU3RhYXQgZGVyIE5lZGVybGFuZGVuIFJv
-b3QgQ0EgLSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMVZ5291
-qj5LnLW4rJ4L5PnZyqtdj7U5EILXr1HgO+EASGrP2uEGQxGZqhQlEq0i6ABtQ8Sp
-uOUfiUtnvWFI7/3S4GCI5bkYYCjDdyutsDeqN95kWSpGV+RLufg3fNU254DBtvPU
-Z5uW6M7XxgpT0GtJlvOjCwV3SPcl5XCsMBQgJeN/dVrlSPhOewMHBPqCYYdu8DvE
-pMfQ9XQ+pV0aCPKbJdL2rAQmPlU6Yiile7Iwr/g3wtG61jj99O9JMDeZJiFIhQGp
-5Rbn3JBV3w/oOM2ZNyFPXfUib2rFEhZgF1XyZWampzCROME4HYYEhLoaJXhena/M
-UGDWE4dS7WMfbWV9whUYdMrhfmQpjHLYFhN9C0lK8SgbIHRrxT3dsKpICT0ugpTN
-GmXZK4iambwYfp/ufWZ8Pr2UuIHOzZgweMFvZ9C+X+Bo7d7iscksWXiSqt8rYGPy
-5V6548r6f1CGPqI0GAwJaCgRHOThuVw+R7oyPxjMW4T182t0xHJ04eOLoEq9jWYv
-6q012iDTiIJh8BIitrzQ1aTsr1SIJSQ8p22xcik/Plemf1WvbibG/ufMQFxRRIEK
-eN5KzlW/HdXZt1bv8Hb/C3m1r737qWmRRpdogBQ2HbN/uymYNqUg+oJgYjOk7Na6
-B6duxc8UpufWkjTYgfX8HV2qXB72o007uPc5AgMBAAGjgZcwgZQwDwYDVR0TAQH/
-BAUwAwEB/zBSBgNVHSAESzBJMEcGBFUdIAAwPzA9BggrBgEFBQcCARYxaHR0cDov
-L3d3dy5wa2lvdmVyaGVpZC5ubC9wb2xpY2llcy9yb290LXBvbGljeS1HMjAOBgNV
-HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJFoMocVHYnitfGsNig0jQt8YojrMA0GCSqG
-SIb3DQEBCwUAA4ICAQCoQUpnKpKBglBu4dfYszk78wIVCVBR7y29JHuIhjv5tLyS
-CZa59sCrI2AGeYwRTlHSeYAz+51IvuxBQ4EffkdAHOV6CMqqi3WtFMTC6GY8ggen
-5ieCWxjmD27ZUD6KQhgpxrRW/FYQoAUXvQwjf/ST7ZwaUb7dRUG/kSS0H4zpX897
-IZmflZ85OkYcbPnNe5yQzSipx6lVu6xiNGI1E0sUOlWDuYaNkqbG9AclVMwWVxJK
-gnjIFNkXgiYtXSAfea7+1HAWFpWD2DU5/1JddRwWxRNVz0fMdWVSSt7wsKfkCpYL
-+63C4iWEst3kvX5ZbJvw8NjnyvLplzh+ib7M+zkXYT9y2zqR2GUBGR2tUKRXCnxL
-vJxxcypFURmFzI79R6d0lR2o0a9OF7FpJsKqeFdbxU2n5Z4FF5TKsl+gSRiNNOkm
-bEgeqmiSBeGCc1qb3AdbCG19ndeNIdn8FCCqwkXfP+cAslHkwvgFuXkajDTznlvk
-N1trSt8sV4pAWja63XVECDdCcAz+3F4hoKOKwJCcaNpQ5kUQR3i2TtJlycM33+FC
-Y7BXN0Ute4qcvwXqZVUz9zkQxSgqIXobisQk+T8VyJoVIPVVYpbtbZNQvOSqeK3Z
-ywplh6ZmwcSBo3c6WB4L7oOLnR7SUqTMHW+wmG2UMbX4cQrcufx9MmDm66+KAQ==
------END CERTIFICATE-----
-
 # Issuer: CN=Hongkong Post Root CA 1 O=Hongkong Post
 # Subject: CN=Hongkong Post Root CA 1 O=Hongkong Post
 # Label: "Hongkong Post Root CA 1"
@@ -2200,6 +2034,45 @@ t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy
 SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03
 -----END CERTIFICATE-----
 
+# Issuer: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes
+# Subject: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes
+# Label: "EC-ACC"
+# Serial: -23701579247955709139626555126524820479
+# MD5 Fingerprint: eb:f5:9d:29:0d:61:f9:42:1f:7c:c2:ba:6d:e3:15:09
+# SHA1 Fingerprint: 28:90:3a:63:5b:52:80:fa:e6:77:4c:0b:6d:a7:d6:ba:a6:4a:f2:e8
+# SHA256 Fingerprint: 88:49:7f:01:60:2f:31:54:24:6a:e2:8c:4d:5a:ef:10:f1:d8:7e:bb:76:62:6f:4a:e0:b7:f9:5b:a7:96:87:99
+-----BEGIN CERTIFICATE-----
+MIIFVjCCBD6gAwIBAgIQ7is969Qh3hSoYqwE893EATANBgkqhkiG9w0BAQUFADCB
+8zELMAkGA1UEBhMCRVMxOzA5BgNVBAoTMkFnZW5jaWEgQ2F0YWxhbmEgZGUgQ2Vy
+dGlmaWNhY2lvIChOSUYgUS0wODAxMTc2LUkpMSgwJgYDVQQLEx9TZXJ2ZWlzIFB1
+YmxpY3MgZGUgQ2VydGlmaWNhY2lvMTUwMwYDVQQLEyxWZWdldSBodHRwczovL3d3
+dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAoYykwMzE1MDMGA1UECxMsSmVyYXJxdWlh
+IEVudGl0YXRzIGRlIENlcnRpZmljYWNpbyBDYXRhbGFuZXMxDzANBgNVBAMTBkVD
+LUFDQzAeFw0wMzAxMDcyMzAwMDBaFw0zMTAxMDcyMjU5NTlaMIHzMQswCQYDVQQG
+EwJFUzE7MDkGA1UEChMyQWdlbmNpYSBDYXRhbGFuYSBkZSBDZXJ0aWZpY2FjaW8g
+KE5JRiBRLTA4MDExNzYtSSkxKDAmBgNVBAsTH1NlcnZlaXMgUHVibGljcyBkZSBD
+ZXJ0aWZpY2FjaW8xNTAzBgNVBAsTLFZlZ2V1IGh0dHBzOi8vd3d3LmNhdGNlcnQu
+bmV0L3ZlcmFycmVsIChjKTAzMTUwMwYDVQQLEyxKZXJhcnF1aWEgRW50aXRhdHMg
+ZGUgQ2VydGlmaWNhY2lvIENhdGFsYW5lczEPMA0GA1UEAxMGRUMtQUNDMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsyLHT+KXQpWIR4NA9h0X84NzJB5R
+85iKw5K4/0CQBXCHYMkAqbWUZRkiFRfCQ2xmRJoNBD45b6VLeqpjt4pEndljkYRm
+4CgPukLjbo73FCeTae6RDqNfDrHrZqJyTxIThmV6PttPB/SnCWDaOkKZx7J/sxaV
+HMf5NLWUhdWZXqBIoH7nF2W4onW4HvPlQn2v7fOKSGRdghST2MDk/7NQcvJ29rNd
+QlB50JQ+awwAvthrDk4q7D7SzIKiGGUzE3eeml0aE9jD2z3Il3rucO2n5nzbcc8t
+lGLfbdb1OL4/pYUKGbio2Al1QnDE6u/LDsg0qBIimAy4E5S2S+zw0JDnJwIDAQAB
+o4HjMIHgMB0GA1UdEQQWMBSBEmVjX2FjY0BjYXRjZXJ0Lm5ldDAPBgNVHRMBAf8E
+BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUoMOLRKo3pUW/l4Ba0fF4
+opvpXY0wfwYDVR0gBHgwdjB0BgsrBgEEAfV4AQMBCjBlMCwGCCsGAQUFBwIBFiBo
+dHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbDA1BggrBgEFBQcCAjApGidW
+ZWdldSBodHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAwDQYJKoZIhvcN
+AQEFBQADggEBAKBIW4IB9k1IuDlVNZyAelOZ1Vr/sXE7zDkJlF7W2u++AVtd0x7Y
+/X1PzaBB4DSTv8vihpw3kpBWHNzrKQXlxJ7HNd+KDM3FIUPpqojlNcAZQmNaAl6k
+SBg6hW/cnbw/nZzBh7h6YQjpdwt/cKt63dmXLGQehb+8dJahw3oS7AwaboMMPOhy
+Rp/7SNVel+axofjk70YllJyJ22k4vuxcDlbHZVHlUIiIv0LVKz3l+bqeLrPK9HOS
+Agu+TGbrIP65y7WZf+a2E/rKS03Z7lNGBjvGTq2TWoF+bCpLagVFjPIhpDGQh2xl
+nJ2lYJU6Un/10asIbvPuW/mIPX64b24D5EI=
+-----END CERTIFICATE-----
+
 # Issuer: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority
 # Subject: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority
 # Label: "Hellenic Academic and Research Institutions RootCA 2011"
@@ -3453,46 +3326,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ
 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su
 -----END CERTIFICATE-----
 
-# Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903
-# Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903
-# Label: "Certinomis - Root CA"
-# Serial: 1
-# MD5 Fingerprint: 14:0a:fd:8d:a8:28:b5:38:69:db:56:7e:61:22:03:3f
-# SHA1 Fingerprint: 9d:70:bb:01:a5:a4:a0:18:11:2e:f7:1c:01:b9:32:c5:34:e7:88:a8
-# SHA256 Fingerprint: 2a:99:f5:bc:11:74:b7:3c:bb:1d:62:08:84:e0:1c:34:e5:1c:cb:39:78:da:12:5f:0e:33:26:88:83:bf:41:58
------BEGIN CERTIFICATE-----
-MIIFkjCCA3qgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJGUjET
-MBEGA1UEChMKQ2VydGlub21pczEXMBUGA1UECxMOMDAwMiA0MzM5OTg5MDMxHTAb
-BgNVBAMTFENlcnRpbm9taXMgLSBSb290IENBMB4XDTEzMTAyMTA5MTcxOFoXDTMz
-MTAyMTA5MTcxOFowWjELMAkGA1UEBhMCRlIxEzARBgNVBAoTCkNlcnRpbm9taXMx
-FzAVBgNVBAsTDjAwMDIgNDMzOTk4OTAzMR0wGwYDVQQDExRDZXJ0aW5vbWlzIC0g
-Um9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANTMCQosP5L2
-fxSeC5yaah1AMGT9qt8OHgZbn1CF6s2Nq0Nn3rD6foCWnoR4kkjW4znuzuRZWJfl
-LieY6pOod5tK8O90gC3rMB+12ceAnGInkYjwSond3IjmFPnVAy//ldu9n+ws+hQV
-WZUKxkd8aRi5pwP5ynapz8dvtF4F/u7BUrJ1Mofs7SlmO/NKFoL21prbcpjp3vDF
-TKWrteoB4owuZH9kb/2jJZOLyKIOSY008B/sWEUuNKqEUL3nskoTuLAPrjhdsKkb
-5nPJWqHZZkCqqU2mNAKthH6yI8H7KsZn9DS2sJVqM09xRLWtwHkziOC/7aOgFLSc
-CbAK42C++PhmiM1b8XcF4LVzbsF9Ri6OSyemzTUK/eVNfaoqoynHWmgE6OXWk6Ri
-wsXm9E/G+Z8ajYJJGYrKWUM66A0ywfRMEwNvbqY/kXPLynNvEiCL7sCCeN5LLsJJ
-wx3tFvYk9CcbXFcx3FXuqB5vbKziRcxXV4p1VxngtViZSTYxPDMBbRZKzbgqg4SG
-m/lg0h9tkQPTYKbVPZrdd5A9NaSfD171UkRpucC63M9933zZxKyGIjK8e2uR73r4
-F2iw4lNVYC2vPsKD2NkJK/DAZNuHi5HMkesE/Xa0lZrmFAYb1TQdvtj/dBxThZng
-WVJKYe2InmtJiUZ+IFrZ50rlau7SZRFDAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIB
-BjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTvkUz1pcMw6C8I6tNxIqSSaHh0
-2TAfBgNVHSMEGDAWgBTvkUz1pcMw6C8I6tNxIqSSaHh02TANBgkqhkiG9w0BAQsF
-AAOCAgEAfj1U2iJdGlg+O1QnurrMyOMaauo++RLrVl89UM7g6kgmJs95Vn6RHJk/
-0KGRHCwPT5iVWVO90CLYiF2cN/z7ZMF4jIuaYAnq1fohX9B0ZedQxb8uuQsLrbWw
-F6YSjNRieOpWauwK0kDDPAUwPk2Ut59KA9N9J0u2/kTO+hkzGm2kQtHdzMjI1xZS
-g081lLMSVX3l4kLr5JyTCcBMWwerx20RoFAXlCOotQqSD7J6wWAsOMwaplv/8gzj
-qh8c3LigkyfeY+N/IZ865Z764BNqdeuWXGKRlI5nU7aJ+BIJy29SWwNyhlCVCNSN
-h4YVH5Uk2KRvms6knZtt0rJ2BobGVgjF6wnaNsIbW0G+YSrjcOa4pvi2WsS9Iff/
-ql+hbHY5ZtbqTFXhADObE5hjyW/QASAJN1LnDE8+zbz1X5YnpyACleAu6AdBBR8V
-btaw5BngDwKTACdyxYvRVB9dSsNAl35VpnzBMwQUAR1JIGkLGZOdblgi90AMRgwj
-Y/M50n92Uaf0yKHxDHYiI0ZSKS3io0EHVmmY0gUJvGnHWmHNj4FgFU2A3ZDifcRQ
-8ow7bkrHxuaAKzyBvBGAFhAn1/DNP3nMcyrDflOR1m749fPH0FFNjkulW+YZFzvW
-gQncItzujrnEj1PhZ7szuIgVRs/taTX/dQ1G885x4cVrhkIGuUE=
------END CERTIFICATE-----
-
 # Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed
 # Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed
 # Label: "OISTE WISeKey Global Root GB CA"
@@ -3849,47 +3682,6 @@ CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW
 1KyLa2tJElMzrdfkviT8tQp21KW8EA==
 -----END CERTIFICATE-----
 
-# Issuer: CN=LuxTrust Global Root 2 O=LuxTrust S.A.
-# Subject: CN=LuxTrust Global Root 2 O=LuxTrust S.A.
-# Label: "LuxTrust Global Root 2"
-# Serial: 59914338225734147123941058376788110305822489521
-# MD5 Fingerprint: b2:e1:09:00:61:af:f7:f1:91:6f:c4:ad:8d:5e:3b:7c
-# SHA1 Fingerprint: 1e:0e:56:19:0a:d1:8b:25:98:b2:04:44:ff:66:8a:04:17:99:5f:3f
-# SHA256 Fingerprint: 54:45:5f:71:29:c2:0b:14:47:c4:18:f9:97:16:8f:24:c5:8f:c5:02:3b:f5:da:5b:e2:eb:6e:1d:d8:90:2e:d5
------BEGIN CERTIFICATE-----
-MIIFwzCCA6ugAwIBAgIUCn6m30tEntpqJIWe5rgV0xZ/u7EwDQYJKoZIhvcNAQEL
-BQAwRjELMAkGA1UEBhMCTFUxFjAUBgNVBAoMDUx1eFRydXN0IFMuQS4xHzAdBgNV
-BAMMFkx1eFRydXN0IEdsb2JhbCBSb290IDIwHhcNMTUwMzA1MTMyMTU3WhcNMzUw
-MzA1MTMyMTU3WjBGMQswCQYDVQQGEwJMVTEWMBQGA1UECgwNTHV4VHJ1c3QgUy5B
-LjEfMB0GA1UEAwwWTHV4VHJ1c3QgR2xvYmFsIFJvb3QgMjCCAiIwDQYJKoZIhvcN
-AQEBBQADggIPADCCAgoCggIBANeFl78RmOnwYoNMPIf5U2o3C/IPPIfOb9wmKb3F
-ibrJgz337spbxm1Jc7TJRqMbNBM/wYlFV/TZsfs2ZUv7COJIcRHIbjuend+JZTem
-hfY7RBi2xjcwYkSSl2l9QjAk5A0MiWtj3sXh306pFGxT4GHO9hcvHTy95iJMHZP1
-EMShduxq3sVs35a0VkBCwGKSMKEtFZSg0iAGCW5qbeXrt77U8PEVfIvmTroTzEsn
-Xpk8F12PgX8zPU/TPxvsXD/wPEx1bvKm1Z3aLQdjAsZy6ZS8TEmVT4hSyNvoaYL4
-zDRbIvCGp4m9SAptZoFtyMhk+wHh9OHe2Z7d21vUKpkmFRseTJIpgp7VkoGSQXAZ
-96Tlk0u8d2cx3Rz9MXANF5kM+Qw5GSoXtTBxVdUPrljhPS80m8+f9niFwpN6cj5m
-j5wWEWCPnolvZ77gR1o7DJpni89Gxq44o/KnvObWhWszJHAiS8sIm7vI+AIpHb4g
-DEa/a4ebsypmQjVGbKq6rfmYe+lQVRQxv7HaLe2ArWgk+2mr2HETMOZns4dA/Yl+
-8kPREd8vZS9kzl8UubG/Mb2HeFpZZYiq/FkySIbWTLkpS5XTdvN3JW1CHDiDTf2j
-X5t/Lax5Gw5CMZdjpPuKadUiDTSQMC6otOBttpSsvItO13D8xTiOZCXhTTmQzsmH
-hFhxAgMBAAGjgagwgaUwDwYDVR0TAQH/BAUwAwEB/zBCBgNVHSAEOzA5MDcGByuB
-KwEBAQowLDAqBggrBgEFBQcCARYeaHR0cHM6Ly9yZXBvc2l0b3J5Lmx1eHRydXN0
-Lmx1MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBT/GCh2+UgFLKGu8SsbK7JT
-+Et8szAdBgNVHQ4EFgQU/xgodvlIBSyhrvErGyuyU/hLfLMwDQYJKoZIhvcNAQEL
-BQADggIBAGoZFO1uecEsh9QNcH7X9njJCwROxLHOk3D+sFTAMs2ZMGQXvw/l4jP9
-BzZAcg4atmpZ1gDlaCDdLnINH2pkMSCEfUmmWjfrRcmF9dTHF5kH5ptV5AzoqbTO
-jFu1EVzPig4N1qx3gf4ynCSecs5U89BvolbW7MM3LGVYvlcAGvI1+ut7MV3CwRI9
-loGIlonBWVx65n9wNOeD4rHh4bhY79SV5GCc8JaXcozrhAIuZY+kt9J/Z93I055c
-qqmkoCUUBpvsT34tC38ddfEz2O3OuHVtPlu5mB0xDVbYQw8wkbIEa91WvpWAVWe+
-2M2D2RjuLg+GLZKecBPs3lHJQ3gCpU3I+V/EkVhGFndadKpAvAefMLmx9xIX3eP/
-JEAdemrRTxgKqpAd60Ae36EeRJIQmvKN4dFLRp7oRUKX6kWZ8+xm1QL68qZKJKre
-zrnK+T+Tb/mjuuqlPpmt/f97mfVl7vBZKGfXkJWkE4SphMHozs51k2MavDzq1WQf
-LSoSOcbDWjLtR5EWDrw4wVDej8oqkDQc7kGUnF4ZLvhFSZl0kbAEb+MEWrGrKqv+
-x9CWttrhSmQGbmBNvUJO/3jaJMobtNeWOWyu8Q6qp31IiyBMz2TWuJdGsE7RKlY6
-oJO9r4Ak4Ap+58rVyuiFVdw2KuGUaJPHZnJED4AhMmwlxyOAgwrr
------END CERTIFICATE-----
-
 # Issuer: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM
 # Subject: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM
 # Label: "TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1"
@@ -4656,3 +4448,173 @@ L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa
 LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG
 mpv0
 -----END CERTIFICATE-----
+
+# Issuer: CN=Entrust Root Certification Authority - G4 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2015 Entrust, Inc. - for authorized use only
+# Subject: CN=Entrust Root Certification Authority - G4 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2015 Entrust, Inc. - for authorized use only
+# Label: "Entrust Root Certification Authority - G4"
+# Serial: 289383649854506086828220374796556676440
+# MD5 Fingerprint: 89:53:f1:83:23:b7:7c:8e:05:f1:8c:71:38:4e:1f:88
+# SHA1 Fingerprint: 14:88:4e:86:26:37:b0:26:af:59:62:5c:40:77:ec:35:29:ba:96:01
+# SHA256 Fingerprint: db:35:17:d1:f6:73:2a:2d:5a:b9:7c:53:3e:c7:07:79:ee:32:70:a6:2f:b4:ac:42:38:37:24:60:e6:f0:1e:88
+-----BEGIN CERTIFICATE-----
+MIIGSzCCBDOgAwIBAgIRANm1Q3+vqTkPAAAAAFVlrVgwDQYJKoZIhvcNAQELBQAw
+gb4xCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQL
+Ex9TZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykg
+MjAxNSBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAw
+BgNVBAMTKUVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0
+MB4XDTE1MDUyNzExMTExNloXDTM3MTIyNzExNDExNlowgb4xCzAJBgNVBAYTAlVT
+MRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1
+c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNSBFbnRydXN0LCBJ
+bmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAwBgNVBAMTKUVudHJ1c3Qg
+Um9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0MIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAsewsQu7i0TD/pZJH4i3DumSXbcr3DbVZwbPLqGgZ
+2K+EbTBwXX7zLtJTmeH+H17ZSK9dE43b/2MzTdMAArzE+NEGCJR5WIoV3imz/f3E
+T+iq4qA7ec2/a0My3dl0ELn39GjUu9CH1apLiipvKgS1sqbHoHrmSKvS0VnM1n4j
+5pds8ELl3FFLFUHtSUrJ3hCX1nbB76W1NhSXNdh4IjVS70O92yfbYVaCNNzLiGAM
+C1rlLAHGVK/XqsEQe9IFWrhAnoanw5CGAlZSCXqc0ieCU0plUmr1POeo8pyvi73T
+DtTUXm6Hnmo9RR3RXRv06QqsYJn7ibT/mCzPfB3pAqoEmh643IhuJbNsZvc8kPNX
+wbMv9W3y+8qh+CmdRouzavbmZwe+LGcKKh9asj5XxNMhIWNlUpEbsZmOeX7m640A
+2Vqq6nPopIICR5b+W45UYaPrL0swsIsjdXJ8ITzI9vF01Bx7owVV7rtNOzK+mndm
+nqxpkCIHH2E6lr7lmk/MBTwoWdPBDFSoWWG9yHJM6Nyfh3+9nEg2XpWjDrk4JFX8
+dWbrAuMINClKxuMrLzOg2qOGpRKX/YAr2hRC45K9PvJdXmd0LhyIRyk0X+IyqJwl
+N4y6mACXi0mWHv0liqzc2thddG5msP9E36EYxr5ILzeUePiVSj9/E15dWf10hkNj
+c0kCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD
+VR0OBBYEFJ84xFYjwznooHFs6FRM5Og6sb9nMA0GCSqGSIb3DQEBCwUAA4ICAQAS
+5UKme4sPDORGpbZgQIeMJX6tuGguW8ZAdjwD+MlZ9POrYs4QjbRaZIxowLByQzTS
+Gwv2LFPSypBLhmb8qoMi9IsabyZIrHZ3CL/FmFz0Jomee8O5ZDIBf9PD3Vht7LGr
+hFV0d4QEJ1JrhkzO3bll/9bGXp+aEJlLdWr+aumXIOTkdnrG0CSqkM0gkLpHZPt/
+B7NTeLUKYvJzQ85BK4FqLoUWlFPUa19yIqtRLULVAJyZv967lDtX/Zr1hstWO1uI
+AeV8KEsD+UmDfLJ/fOPtjqF/YFOOVZ1QNBIPt5d7bIdKROf1beyAN/BYGW5KaHbw
+H5Lk6rWS02FREAutp9lfx1/cH6NcjKF+m7ee01ZvZl4HliDtC3T7Zk6LERXpgUl+
+b7DUUH8i119lAg2m9IUe2K4GS0qn0jFmwvjO5QimpAKWRGhXxNUzzxkvFMSUHHuk
+2fCfDrGA4tGeEWSpiBE6doLlYsKA2KSD7ZPvfC+QsDJMlhVoSFLUmQjAJOgc47Ol
+IQ6SwJAfzyBfyjs4x7dtOvPmRLgOMWuIjnDrnBdSqEGULoe256YSxXXfW8AKbnuk
+5F6G+TaU33fD6Q3AOfF5u0aOq0NZJ7cguyPpVkAh7DE9ZapD8j3fcEThuk0mEDuY
+n/PIjhs4ViFqUZPTkcpG2om3PVODLAgfi49T3f+sHw==
+-----END CERTIFICATE-----
+
+# Issuer: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation
+# Subject: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation
+# Label: "Microsoft ECC Root Certificate Authority 2017"
+# Serial: 136839042543790627607696632466672567020
+# MD5 Fingerprint: dd:a1:03:e6:4a:93:10:d1:bf:f0:19:42:cb:fe:ed:67
+# SHA1 Fingerprint: 99:9a:64:c3:7f:f4:7d:9f:ab:95:f1:47:69:89:14:60:ee:c4:c3:c5
+# SHA256 Fingerprint: 35:8d:f3:9d:76:4a:f9:e1:b7:66:e9:c9:72:df:35:2e:e1:5c:fa:c2:27:af:6a:d1:d7:0e:8e:4a:6e:dc:ba:02
+-----BEGIN CERTIFICATE-----
+MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw
+CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD
+VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw
+MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV
+UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy
+b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq
+hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR
+ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb
+hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3
+FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV
+L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB
+iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M=
+-----END CERTIFICATE-----
+
+# Issuer: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation
+# Subject: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation
+# Label: "Microsoft RSA Root Certificate Authority 2017"
+# Serial: 40975477897264996090493496164228220339
+# MD5 Fingerprint: 10:ff:00:ff:cf:c9:f8:c7:7a:c0:ee:35:8e:c9:0f:47
+# SHA1 Fingerprint: 73:a5:e6:4a:3b:ff:83:16:ff:0e:dc:cc:61:8a:90:6e:4e:ae:4d:74
+# SHA256 Fingerprint: c7:41:f7:0f:4b:2a:8d:88:bf:2e:71:c1:41:22:ef:53:ef:10:eb:a0:cf:a5:e6:4c:fa:20:f4:18:85:30:73:e0
+-----BEGIN CERTIFICATE-----
+MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl
+MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw
+NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
+IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG
+EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N
+aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi
+MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ
+Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0
+ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1
+HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm
+gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ
+jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc
+aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG
+YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6
+W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K
+UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH
++FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q
+W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/
+BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC
+NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC
+LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC
+gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6
+tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh
+SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2
+TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3
+pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR
+xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp
+GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9
+dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN
+AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB
+RA+GsCyRxj3qrg+E
+-----END CERTIFICATE-----
+
+# Issuer: CN=e-Szigno Root CA 2017 O=Microsec Ltd.
+# Subject: CN=e-Szigno Root CA 2017 O=Microsec Ltd.
+# Label: "e-Szigno Root CA 2017"
+# Serial: 411379200276854331539784714
+# MD5 Fingerprint: de:1f:f6:9e:84:ae:a7:b4:21:ce:1e:58:7d:d1:84:98
+# SHA1 Fingerprint: 89:d4:83:03:4f:9e:9a:48:80:5f:72:37:d4:a9:a6:ef:cb:7c:1f:d1
+# SHA256 Fingerprint: be:b0:0b:30:83:9b:9b:c3:2c:32:e4:44:79:05:95:06:41:f2:64:21:b1:5e:d0:89:19:8b:51:8a:e2:ea:1b:99
+-----BEGIN CERTIFICATE-----
+MIICQDCCAeWgAwIBAgIMAVRI7yH9l1kN9QQKMAoGCCqGSM49BAMCMHExCzAJBgNV
+BAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMgTHRk
+LjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25vIFJv
+b3QgQ0EgMjAxNzAeFw0xNzA4MjIxMjA3MDZaFw00MjA4MjIxMjA3MDZaMHExCzAJ
+BgNVBAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMg
+THRkLjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25v
+IFJvb3QgQ0EgMjAxNzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJbcPYrYsHtv
+xie+RJCxs1YVe45DJH0ahFnuY2iyxl6H0BVIHqiQrb1TotreOpCmYF9oMrWGQd+H
+Wyx7xf58etqjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G
+A1UdDgQWBBSHERUI0arBeAyxr87GyZDvvzAEwDAfBgNVHSMEGDAWgBSHERUI0arB
+eAyxr87GyZDvvzAEwDAKBggqhkjOPQQDAgNJADBGAiEAtVfd14pVCzbhhkT61Nlo
+jbjcI4qKDdQvfepz7L9NbKgCIQDLpbQS+ue16M9+k/zzNY9vTlp8tLxOsvxyqltZ
++efcMQ==
+-----END CERTIFICATE-----
+
+# Issuer: O=CERTSIGN SA OU=certSIGN ROOT CA G2
+# Subject: O=CERTSIGN SA OU=certSIGN ROOT CA G2
+# Label: "certSIGN Root CA G2"
+# Serial: 313609486401300475190
+# MD5 Fingerprint: 8c:f1:75:8a:c6:19:cf:94:b7:f7:65:20:87:c3:97:c7
+# SHA1 Fingerprint: 26:f9:93:b4:ed:3d:28:27:b0:b9:4b:a7:e9:15:1d:a3:8d:92:e5:32
+# SHA256 Fingerprint: 65:7c:fe:2f:a7:3f:aa:38:46:25:71:f3:32:a2:36:3a:46:fc:e7:02:09:51:71:07:02:cd:fb:b6:ee:da:33:05
+-----BEGIN CERTIFICATE-----
+MIIFRzCCAy+gAwIBAgIJEQA0tk7GNi02MA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV
+BAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJR04g
+Uk9PVCBDQSBHMjAeFw0xNzAyMDYwOTI3MzVaFw00MjAyMDYwOTI3MzVaMEExCzAJ
+BgNVBAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJ
+R04gUk9PVCBDQSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMDF
+dRmRfUR0dIf+DjuW3NgBFszuY5HnC2/OOwppGnzC46+CjobXXo9X69MhWf05N0Iw
+vlDqtg+piNguLWkh59E3GE59kdUWX2tbAMI5Qw02hVK5U2UPHULlj88F0+7cDBrZ
+uIt4ImfkabBoxTzkbFpG583H+u/E7Eu9aqSs/cwoUe+StCmrqzWaTOTECMYmzPhp
+n+Sc8CnTXPnGFiWeI8MgwT0PPzhAsP6CRDiqWhqKa2NYOLQV07YRaXseVO6MGiKs
+cpc/I1mbySKEwQdPzH/iV8oScLumZfNpdWO9lfsbl83kqK/20U6o2YpxJM02PbyW
+xPFsqa7lzw1uKA2wDrXKUXt4FMMgL3/7FFXhEZn91QqhngLjYl/rNUssuHLoPj1P
+rCy7Lobio3aP5ZMqz6WryFyNSwb/EkaseMsUBzXgqd+L6a8VTxaJW732jcZZroiF
+DsGJ6x9nxUWO/203Nit4ZoORUSs9/1F3dmKh7Gc+PoGD4FapUB8fepmrY7+EF3fx
+DTvf95xhszWYijqy7DwaNz9+j5LP2RIUZNoQAhVB/0/E6xyjyfqZ90bp4RjZsbgy
+LcsUDFDYg2WD7rlcz8sFWkz6GZdr1l0T08JcVLwyc6B49fFtHsufpaafItzRUZ6C
+eWRgKRM+o/1Pcmqr4tTluCRVLERLiohEnMqE0yo7AgMBAAGjQjBAMA8GA1UdEwEB
+/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSCIS1mxteg4BXrzkwJ
+d8RgnlRuAzANBgkqhkiG9w0BAQsFAAOCAgEAYN4auOfyYILVAzOBywaK8SJJ6ejq
+kX/GM15oGQOGO0MBzwdw5AgeZYWR5hEit/UCI46uuR59H35s5r0l1ZUa8gWmr4UC
+b6741jH/JclKyMeKqdmfS0mbEVeZkkMR3rYzpMzXjWR91M08KCy0mpbqTfXERMQl
+qiCA2ClV9+BB/AYm/7k29UMUA2Z44RGx2iBfRgB4ACGlHgAoYXhvqAEBj500mv/0
+OJD7uNGzcgbJceaBxXntC6Z58hMLnPddDnskk7RI24Zf3lCGeOdA5jGokHZwYa+c
+NywRtYK3qq4kNFtyDGkNzVmf9nGvnAvRCjj5BiKDUyUM/FHE5r7iOZULJK2v0ZXk
+ltd0ZGtxTgI8qoXzIKNDOXZbbFD+mpwUHmUUihW9o4JFWklWatKcsWMy5WHgUyIO
+pwpJ6st+H6jiYoD2EEVSmAYY3qXNL3+q1Ok+CHLsIwMCPKaq2LxndD0UF/tUSxfj
+03k9bWtJySgOLnRQvwzZRjoQhsmnP+mg7H/rpXdYaXHmgwo38oZJar55CJD2AhZk
+PuXaTH4MNMn5X7azKFGnpyuqSfqNZSlO42sTp5SjLVFteAxEy9/eCG/Oo2Sr05WE
+1LlSVHJ7liXMvGnjSG4N0MedJ5qq+BOS3R7fY581qRY27Iy4g/Q9iY/NtBde17MX
+QRBdJ3NghVdJIgc=
+-----END CERTIFICATE-----

Contents/Libraries/Shared/certifi/core.py

@@ -4,12 +4,57 @@
 certifi.py
 ~~~~~~~~~~
 
-This module returns the installation location of cacert.pem.
+This module returns the installation location of cacert.pem or its contents.
 """
 import os
 
+try:
+    from importlib.resources import path as get_path, read_text
 
-def where():
-    f = os.path.dirname(__file__)
+    _CACERT_CTX = None
+    _CACERT_PATH = None
 
-    return os.path.join(f, 'cacert.pem')
+    def where():
+        # This is slightly terrible, but we want to delay extracting the file
+        # in cases where we're inside of a zipimport situation until someone
+        # actually calls where(), but we don't want to re-extract the file
+        # on every call of where(), so we'll do it once then store it in a
+        # global variable.
+        global _CACERT_CTX
+        global _CACERT_PATH
+        if _CACERT_PATH is None:
+            # This is slightly janky, the importlib.resources API wants you to
+            # manage the cleanup of this file, so it doesn't actually return a
+            # path, it returns a context manager that will give you the path
+            # when you enter it and will do any cleanup when you leave it. In
+            # the common case of not needing a temporary file, it will just
+            # return the file system location and the __exit__() is a no-op.
+            #
+            # We also have to hold onto the actual context manager, because
+            # it will do the cleanup whenever it gets garbage collected, so
+            # we will also store that at the global level as well.
+            _CACERT_CTX = get_path("certifi", "cacert.pem")
+            _CACERT_PATH = str(_CACERT_CTX.__enter__())
+
+        return _CACERT_PATH
+
+
+except ImportError:
+    # This fallback will work for Python versions prior to 3.7 that lack the
+    # importlib.resources module but relies on the existing `where` function
+    # so won't address issues with environments like PyOxidizer that don't set
+    # __file__ on modules.
+    def read_text(_module, _path, encoding="ascii"):
+        with open(where(), "r", encoding=encoding) as data:
+            return data.read()
+
+    # If we don't have importlib.resources, then we will just do the old logic
+    # of assuming we're on the filesystem and munge the path directly.
+    def where():
+        f = os.path.dirname(__file__)
+
+        return os.path.join(f, "cacert.pem")
+
+
+def contents():
+    return read_text("certifi", "cacert.pem", encoding="ascii")

Contents/Libraries/Shared/provider_test.sh

@@ -43,6 +43,8 @@ python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.get
 # subscenter:list
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subliminal.core import scan_video; print SZProviderPool(providers=['subscenter'], )['subscenter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('heb')])"
 
+# subscene:list
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from subzero.language import Language; from subzero.video import parse_video; SZProviderPool(providers=['subscene'], provider_configs={'subscene': {'username': 'USERNAME', 'password': 'PASSWORD'}})['subscene'].list_subtitles(parse_video('FILENAME', {}, {'type': 'episode'}, dry_run=True), languages=[Language('eng')])"
 
 # refining
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video, refine_video; video = parse_video('FILE_NAME', {'type': 'episode'}, dry_run=True); print refine_video(video)"

Contents/Libraries/Shared/pysubs2/exceptions.py

@@ -12,3 +12,6 @@ class UnknownFormatIdentifierError(Pysubs2Error):
 
 class FormatAutodetectionError(Pysubs2Error):
     """Subtitle format is ambiguous or unknown."""
+
+class ContentNotUsable(Pysubs2Error):
+    """Current content not usable for specified format"""

Contents/Libraries/Shared/pysubs2/ssastyle.py

@@ -41,6 +41,7 @@ class SSAStyle(object):
         self.italic = False #: Italic
         self.underline = False #: Underline (ASS only)
         self.strikeout = False #: Strikeout (ASS only)
+        self.drawing = False #: Drawing (ASS only, see http://docs.aegisub.org/3.1/ASS_Tags/#drawing-tags
         self.scalex = 100.0 #: Horizontal scaling (ASS only)
         self.scaley = 100.0 #: Vertical scaling (ASS only)
         self.spacing = 0.0 #: Letter spacing (ASS only)

Contents/Libraries/Shared/pysubs2/subrip.py

@@ -5,6 +5,7 @@ from .formatbase import FormatBase
 from .ssaevent import SSAEvent
 from .ssastyle import SSAStyle
 from .substation import parse_tags
+from .exceptions import ContentNotUsable
 from .time import ms_to_times, make_time, TIMESTAMP, timestamp_to_ms
 
 #: Largest timestamp allowed in SubRip, ie. 99:59:59,999.
@@ -81,6 +82,7 @@ class SubripFormat(FormatBase):
                 if sty.italic: fragment = "<i>%s</i>" % fragment
                 if sty.underline: fragment = "<u>%s</u>" % fragment
                 if sty.strikeout: fragment = "<s>%s</s>" % fragment
+                if sty.drawing: raise ContentNotUsable
                 body.append(fragment)
 
             return re.sub("\n+", "\n", "".join(body).strip())
@@ -90,7 +92,10 @@ class SubripFormat(FormatBase):
         for i, line in enumerate(visible_lines, 1):
             start = ms_to_timestamp(line.start)
             end = ms_to_timestamp(line.end)
-            text = prepare_text(line.text, subs.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            try:
+                text = prepare_text(line.text, subs.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            except ContentNotUsable:
+                continue
 
             print("%d" % i, file=fp) # Python 2.7 compat
             print(start, "-->", end, file=fp)

Contents/Libraries/Shared/pysubs2/substation.py

@@ -110,7 +110,7 @@ def parse_tags(text, style=SSAStyle.DEFAULT_STYLE, styles={}):
     
     def apply_overrides(all_overrides):
         s = style.copy()
-        for tag in re.findall(r"\\[ibus][10]|\\r[a-zA-Z_0-9 ]*", all_overrides):
+        for tag in re.findall(r"\\[ibusp][0-9]|\\r[a-zA-Z_0-9 ]*", all_overrides):
             if tag == r"\r":
                 s = style.copy() # reset to original line style
             elif tag.startswith(r"\r"):
@@ -122,6 +122,13 @@ def parse_tags(text, style=SSAStyle.DEFAULT_STYLE, styles={}):
                 elif "b" in tag: s.bold = "1" in tag
                 elif "u" in tag: s.underline = "1" in tag
                 elif "s" in tag: s.strikeout = "1" in tag
+                elif "p" in tag:
+                    try:
+                        scale = int(tag[2:])
+                    except (ValueError, IndexError):
+                        continue
+
+                    s.drawing = scale > 0
         return s
     
     overrides = SSAEvent.OVERRIDE_SEQUENCE.findall(text)

Contents/Libraries/Shared/subliminal/exceptions.py

@@ -27,3 +27,8 @@ class ServiceUnavailable(ProviderError):
 class DownloadLimitExceeded(ProviderError):
     """Exception raised by providers when download limit is exceeded."""
     pass
+
+
+class DownloadLimitPerDayExceeded(ProviderError):
+    """Exception raised by providers when download limit is exceeded."""
+    pass

Contents/Libraries/Shared/subliminal/extensions.py

@@ -94,7 +94,8 @@ provider_manager = RegistrableExtensionManager('subliminal.providers', [
     'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
     'shooter = subliminal.providers.shooter:ShooterProvider',
     'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
-    'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'
+    'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider',
+    'screwzira = subliminal.providers.screwzira:ScrewZiraProvider'
 ])
 
 #: Refiner manager

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -108,10 +108,12 @@ class SZProviderPool(ProviderPool):
         try:
             logger.info('Terminating provider %s', name)
             self.initialized_providers[name].terminate()
-        except (requests.Timeout, socket.timeout):
+        except (requests.Timeout, socket.timeout) as e:
             logger.error('Provider %r timed out, improperly terminated', name)
-        except:
+            self.throttle_callback(name, e)
+        except Exception as e:
             logger.exception('Provider %r terminated unexpectedly', name)
+            self.throttle_callback(name, e)
 
         del self.initialized_providers[name]
 
@@ -183,8 +185,9 @@ class SZProviderPool(ProviderPool):
 
             return out
 
-        except (requests.Timeout, socket.timeout):
-            logger.error('Provider %r timed out', provider)
+        except (requests.Timeout, socket.timeout) as e:
+            logger.exception('Provider %r timed out', provider)
+            self.throttle_callback(provider, e)
 
         except Exception as e:
             logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
@@ -263,10 +266,11 @@ class SZProviderPool(ProviderPool):
                     requests.exceptions.ProxyError,
                     requests.exceptions.SSLError,
                     requests.Timeout,
-                    socket.timeout):
-                logger.error('Provider %r connection error', subtitle.provider_name)
+                    socket.timeout) as e:
+                logger.exception('Provider %r connection error', subtitle.provider_name)
+                self.throttle_callback(subtitle.provider_name, e)
 
-            except ResponseNotReady:
+            except ResponseNotReady as e:
                 logger.error('Provider %r response error, reinitializing', subtitle.provider_name)
                 try:
                     self[subtitle.provider_name].terminate()
@@ -274,6 +278,7 @@ class SZProviderPool(ProviderPool):
                 except:
                     logger.error('Provider %r reinitialization error: %s', subtitle.provider_name,
                                  traceback.format_exc())
+                    self.throttle_callback(subtitle.provider_name, e)
 
             except rarfile.BadRarFile:
                 logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)
@@ -354,15 +359,16 @@ class SZProviderPool(ProviderPool):
             orig_matches = matches.copy()
 
             logger.debug('%r: Found matches %r', s, matches)
+            score, score_without_hash = compute_score(matches, s, video, hearing_impaired=use_hearing_impaired)
             unsorted_subtitles.append(
-                (s, compute_score(matches, s, video, hearing_impaired=use_hearing_impaired), matches, orig_matches))
+                (s, score, score_without_hash, matches, orig_matches))
 
         # sort subtitles by score
-        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1), reverse=True)
+        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1, 2), reverse=True)
 
         # download best subtitles, falling back on the next on error
         downloaded_subtitles = []
-        for subtitle, score, matches, orig_matches in scored_subtitles:
+        for subtitle, score, score_without_hash, matches, orig_matches in scored_subtitles:
             # check score
             if score < min_score:
                 logger.info('%r: Score %d is below min_score (%d)', subtitle, score, min_score)
@@ -541,8 +547,12 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski
         if video.size > 10485760:
             logger.debug('Size is %d', video.size)
             osub_hash = None
+
+            if "bsplayer" in providers:
+                video.hashes['bsplayer'] = osub_hash = hash_opensubtitles(hash_path)
+
             if "opensubtitles" in providers:
-                video.hashes['opensubtitles'] = osub_hash = hash_opensubtitles(hash_path)
+                video.hashes['opensubtitles'] = osub_hash = osub_hash or hash_opensubtitles(hash_path)
 
             if "shooter" in providers:
                 video.hashes['shooter'] = hash_shooter(hash_path)

Contents/Libraries/Shared/subliminal_patch/http.py

@@ -35,12 +35,8 @@ except ImportError:
 from subzero.lib.io import get_viable_encoding
 
 logger = logging.getLogger(__name__)
-pem_file = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(unicode(__file__, get_viable_encoding()))), "..", certifi.where()))
-try:
-    default_ssl_context = ssl.create_default_context(cafile=pem_file)
-except AttributeError:
-    # < Python 2.7.9
-    default_ssl_context = None
+pem_file = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(unicode(__file__, get_viable_encoding()))),
+                                         "..", certifi.where()))
 
 
 class TimeoutSession(requests.Session):
@@ -58,9 +54,9 @@ class TimeoutSession(requests.Session):
 
 
 class CertifiSession(TimeoutSession):
-    def __init__(self):
+    def __init__(self, verify=None):
         super(CertifiSession, self).__init__()
-        self.verify = pem_file
+        self.verify = verify or pem_file
 
 
 class NeedsCaptchaException(Exception):
@@ -241,12 +237,20 @@ class SubZeroRequestsTransport(xmlrpclib.SafeTransport):
     # change our user agent to reflect Requests
     user_agent = "Python XMLRPC with Requests (python-requests.org)"
     proxies = None
+    xm_ver = 1
+    session_var = "PHPSESSID"
 
     def __init__(self, use_https=True, verify=None, user_agent=None, timeout=10, *args, **kwargs):
         self.verify = pem_file if verify is None else verify
         self.use_https = use_https
         self.user_agent = user_agent if user_agent is not None else self.user_agent
         self.timeout = timeout
+        self.session = requests.Session()
+        self.session.headers['User-Agent'] = self.user_agent
+        # if 'requests' in self.session.headers['User-Agent']:
+        #     # Set a random User-Agent if no custom User-Agent has been set
+        #     self.session.headers = User_Agent(allow_brotli=False).headers
+
         proxy = os.environ.get('SZ_HTTP_PROXY')
         if proxy:
             self.proxies = {
@@ -260,18 +264,40 @@ class SubZeroRequestsTransport(xmlrpclib.SafeTransport):
         """
         Make an xmlrpc request.
         """
-        headers = {'User-Agent': self.user_agent}
         url = self._build_url(host, handler)
+        cache_key = "xm%s_%s" % (self.xm_ver, host)
+
+        old_sessvar = self.session.cookies.get(self.session_var, "")
+        if not old_sessvar:
+            data = region.get(cache_key)
+            if data is not NO_VALUE:
+                logger.debug("Trying to re-use headers/cookies for %s" % host)
+                self.session.cookies, self.session.headers = data
+                old_sessvar = self.session.cookies.get(self.session_var, "")
+
         try:
-            resp = requests.post(url, data=request_body, headers=headers,
-                                 stream=True, timeout=self.timeout, proxies=self.proxies,
-                                 verify=self.verify)
+            resp = self.session.post(url, data=request_body,
+                                     stream=True, timeout=self.timeout, proxies=self.proxies,
+                                     verify=self.verify)
+
+            if self.session_var in resp.cookies and resp.cookies[self.session_var] != old_sessvar:
+                logger.debug("Storing %s cookies" % host)
+                region.set(cache_key, [self.session.cookies, self.session.headers])
         except ValueError:
+            logger.debug("Wiping cookies/headers cache (VE) for %s" % host)
+            region.delete(cache_key)
             raise
         except Exception:
+            logger.debug("Wiping cookies/headers cache (EX) for %s" % host)
+            region.delete(cache_key)
             raise  # something went wrong
         else:
-            resp.raise_for_status()
+            try:
+                resp.raise_for_status()
+            except requests.exceptions.HTTPError:
+                logger.debug("Wiping cookies/headers cache (RE) for %s" % host)
+                region.delete(cache_key)
+                raise
 
             try:
                 if 'x-ratelimit-remaining' in resp.headers and int(resp.headers['x-ratelimit-remaining']) <= 2:

Contents/Libraries/Shared/subliminal_patch/providers/addic7ed.py

@@ -2,6 +2,8 @@
 import logging
 import re
 import datetime
+import types
+
 import subliminal
 import time
 
@@ -10,7 +12,8 @@ from random import randint
 from dogpile.cache.api import NO_VALUE
 from requests import Session
 from subliminal.cache import region
-from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
+from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError, \
+    DownloadLimitPerDayExceeded
 from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
     Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
 from subliminal.subtitle import fix_line_ending
@@ -64,6 +67,7 @@ class Addic7edProvider(_Addic7edProvider):
         'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'zho'
     ]} | {Language.fromietf(l) for l in ["sr-Latn", "sr-Cyrl"]}
 
+    vip = False
     USE_ADDICTED_RANDOM_AGENTS = False
     hearing_impaired_verifiable = True
     subtitle_class = Addic7edSubtitle
@@ -72,9 +76,10 @@ class Addic7edProvider(_Addic7edProvider):
     sanitize_characters = {'-', ':', '(', ')', '.', '/'}
     last_show_ids_fetch_key = "addic7ed_last_id_fetch"
 
-    def __init__(self, username=None, password=None, use_random_agents=False):
+    def __init__(self, username=None, password=None, use_random_agents=False, is_vip=False):
         super(Addic7edProvider, self).__init__(username=username, password=password)
         self.USE_ADDICTED_RANDOM_AGENTS = use_random_agents
+        self.vip = is_vip
 
         if not all((username, password)):
             raise ConfigurationError('Username and password must be specified')
@@ -397,6 +402,27 @@ class Addic7edProvider(_Addic7edProvider):
         return subtitles
 
     def download_subtitle(self, subtitle):
+        last_dls = region.get("addic7ed_dls")
+        now = datetime.datetime.now()
+        one_day = datetime.timedelta(hours=24)
+
+        def raise_limit():
+            logger.info("Addic7ed: Downloads per day exceeded (%s)", cap)
+            raise DownloadLimitPerDayExceeded
+
+        if not isinstance(last_dls, types.ListType):
+            last_dls = []
+        else:
+            # filter all non-expired DLs
+            last_dls = filter(lambda t: t + one_day > now, last_dls)
+            region.set("addic7ed_dls", last_dls)
+
+        cap = self.vip and 80 or 40
+        amount = len(last_dls)
+
+        if amount >= cap:
+            raise_limit()
+
         # download the subtitle
         r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link},
                              timeout=10)
@@ -408,7 +434,7 @@ class Addic7edProvider(_Addic7edProvider):
         if not r.content:
             # Provider wrongful return a status of 304 Not Modified with an empty content
             # raise_for_status won't raise exception for that status code
-            logger.error('Unable to download subtitle. No data returned from provider')
+            logger.error('Addic7ed: Unable to download subtitle. No data returned from provider')
             return
 
         # detect download limit exceeded
@@ -416,3 +442,10 @@ class Addic7edProvider(_Addic7edProvider):
             raise DownloadLimitExceeded
 
         subtitle.content = fix_line_ending(r.content)
+        last_dls.append(datetime.datetime.now())
+        region.set("addic7ed_dls", last_dls)
+        logger.info("Addic7ed: Used %s/%s downloads", amount + 1, cap)
+
+        if amount + 1 >= cap:
+            raise_limit()
+

Contents/Libraries/Shared/subliminal_patch/providers/bsplayer.py

@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+import logging
+import io
+import os
+
+from requests import Session
+from guessit import guessit
+from subliminal_patch.providers import Provider
+from subliminal_patch.subtitle import Subtitle
+from subliminal.utils import sanitize_release_group
+from subliminal.subtitle import guess_matches
+from subzero.language import Language
+
+import gzip
+import random
+from time import sleep
+from xml.etree import ElementTree
+
+logger = logging.getLogger(__name__)
+
+
+class BSPlayerSubtitle(Subtitle):
+    """BSPlayer Subtitle."""
+    provider_name = 'bsplayer'
+    hash_verifiable = True
+
+    def __init__(self, language, filename, subtype, video, link):
+        super(BSPlayerSubtitle, self).__init__(language)
+        self.language = language
+        self.filename = filename
+        self.page_link = link
+        self.subtype = subtype
+        self.video = video
+
+    @property
+    def id(self):
+        return self.page_link
+
+    @property
+    def release_info(self):
+        return self.filename
+
+    def get_matches(self, video):
+        matches = set()
+        matches |= guess_matches(video, guessit(self.filename))
+        matches.add('hash')
+
+        return matches
+
+
+class BSPlayerProvider(Provider):
+    """BSPlayer Provider."""
+    languages = {Language('por', 'BR')} | {Language(l) for l in [
+        'ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor', 'nld', 'pol', 'por',
+        'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho'
+    ]}
+    SEARCH_THROTTLE = 8
+    hash_verifiable = True
+
+    # batantly based on kodi's bsplayer plugin
+    # also took from BSPlayer-Subtitles-Downloader
+    def __init__(self):
+        self.initialize()
+
+    def initialize(self):
+        self.session = Session()
+        self.search_url = self.get_sub_domain()
+        self.token = None
+        self.login()
+
+    def terminate(self):
+        self.session.close()
+        self.logout()
+
+    def api_request(self, func_name='logIn', params='', tries=5):
+        headers = {
+            'User-Agent': 'BSPlayer/2.x (1022.12360)',
+            'Content-Type': 'text/xml; charset=utf-8',
+            'Connection': 'close',
+            'SOAPAction': '"http://api.bsplayer-subtitles.com/v1.php#{func_name}"'.format(func_name=func_name)
+        }
+        data = (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            '<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" '
+            'xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" '
+            'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
+            'xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:ns1="{search_url}">'
+            '<SOAP-ENV:Body SOAP-ENV:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">'
+            '<ns1:{func_name}>{params}</ns1:{func_name}></SOAP-ENV:Body></SOAP-ENV:Envelope>'
+        ).format(search_url=self.search_url, func_name=func_name, params=params)
+        logger.info('Sending request: %s.' % func_name)
+        for i in iter(range(tries)):
+            try:
+                self.session.headers.update(headers.items())
+                res = self.session.post(self.search_url, data)
+                return ElementTree.fromstring(res.text)
+
+            except Exception as ex:
+                logger.info("ERROR: %s." % ex)
+                if func_name == 'logIn':
+                    self.search_url = self.get_sub_domain()
+
+                sleep(1)
+        logger.info('ERROR: Too many tries (%d)...' % tries)
+        raise Exception('Too many tries...')
+
+    def login(self):
+        # If already logged in
+        if self.token:
+            return True
+
+        root = self.api_request(
+            func_name='logIn',
+            params=('<username></username>'
+                    '<password></password>'
+                    '<AppID>BSPlayer v2.67</AppID>')
+        )
+        res = root.find('.//return')
+        if res.find('status').text == 'OK':
+            self.token = res.find('data').text
+            logger.info("Logged In Successfully.")
+            return True
+        return False
+
+    def logout(self):
+        # If already logged out / not logged in
+        if not self.token:
+            return True
+
+        root = self.api_request(
+            func_name='logOut',
+            params='<handle>{token}</handle>'.format(token=self.token)
+        )
+        res = root.find('.//return')
+        self.token = None
+        if res.find('status').text == 'OK':
+            logger.info("Logged Out Successfully.")
+            return True
+        return False
+
+    def query(self, video, video_hash, language):
+        if not self.login():
+            return []
+
+        if isinstance(language, (tuple, list, set)):
+            # language_ids = ",".join(language)
+            # language_ids = 'spa'
+            language_ids = ','.join(sorted(l.opensubtitles for l in language))
+
+        if video.imdb_id is None:
+            imdbId = '*'
+        else:
+            imdbId = video.imdb_id
+        sleep(self.SEARCH_THROTTLE)
+        root = self.api_request(
+            func_name='searchSubtitles',
+            params=(
+                '<handle>{token}</handle>'
+                '<movieHash>{movie_hash}</movieHash>'
+                '<movieSize>{movie_size}</movieSize>'
+                '<languageId>{language_ids}</languageId>'
+                '<imdbId>{imdbId}</imdbId>'
+            ).format(token=self.token, movie_hash=video_hash,
+                     movie_size=video.size, language_ids=language_ids, imdbId=imdbId)
+        )
+        res = root.find('.//return/result')
+        if res.find('status').text != 'OK':
+            return []
+
+        items = root.findall('.//return/data/item')
+        subtitles = []
+        if items:
+            logger.info("Subtitles Found.")
+            for item in items:
+                subID = item.find('subID').text
+                subDownloadLink = item.find('subDownloadLink').text
+                subLang = Language.fromopensubtitles(item.find('subLang').text)
+                subName = item.find('subName').text
+                subFormat = item.find('subFormat').text
+                subtitles.append(
+                    BSPlayerSubtitle(subLang, subName, subFormat, video, subDownloadLink)
+                )
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return self.query(video, video.hashes['bsplayer'], languages)
+
+    def get_sub_domain(self):
+        # s1-9, s101-109
+        SUB_DOMAINS = ['s1', 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9',
+                       's101', 's102', 's103', 's104', 's105', 's106', 's107', 's108', 's109']
+        API_URL_TEMPLATE = "http://{sub_domain}.api.bsplayer-subtitles.com/v1.php"
+        sub_domains_end = len(SUB_DOMAINS) - 1
+        return API_URL_TEMPLATE.format(sub_domain=SUB_DOMAINS[random.randint(0, sub_domains_end)])
+
+    def download_subtitle(self, subtitle):
+        session = Session()
+        _addheaders = {
+            'User-Agent': 'Mozilla/4.0 (compatible; Synapse)'
+        }
+        session.headers.update(_addheaders)
+        res = session.get(subtitle.page_link)
+        if res:
+            if res.text == '500':
+                raise ValueError('Error 500 on server')
+
+            with gzip.GzipFile(fileobj=io.BytesIO(res.content)) as gf:
+                subtitle.content = gf.read()
+                subtitle.normalize()
+
+            return subtitle
+        raise ValueError('Problems conecting to the server')

Contents/Libraries/Shared/subliminal_patch/providers/opensubtitles.py

@@ -105,7 +105,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
 
     def __init__(self, username=None, password=None, use_tag_search=False, only_foreign=False, also_foreign=False,
                  skip_wrong_fps=True, is_vip=False, use_ssl=True, timeout=15):
-        if any((username, password)) and not all((username, password)):
+        if not all((username, password)):
             raise ConfigurationError('Username and password must be specified')
 
         self.username = username or ''
@@ -154,6 +154,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
         logger.debug('Logged in with token %r', self.token[:10]+"X"*(len(self.token)-10))
 
         region.set("os_token", self.token)
+        time.sleep(1)
 
     def use_token_or_login(self, func):
         if not self.token:
@@ -162,6 +163,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
         try:
             return func()
         except Unauthorized:
+            logger.debug("Token not valid, logging in again")
             self.log_in()
             return func()
 
@@ -197,16 +199,11 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
                     return
 
             logger.error("Login failed, please check your credentials")
+            raise
                 
     def terminate(self):
-        if self.token:
-            try:
-                checked(lambda: self.server.LogOut(self.token))
-            except:
-                logger.error("Logout failed: %s", traceback.format_exc())
-
         self.server = None
-        self.token = None
+        #self.token = None
 
     def list_subtitles(self, video, languages):
         """

Contents/Libraries/Shared/subliminal_patch/providers/screwzira.py

@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+from requests import Session
+import json
+import logging
+from subzero.language import Language
+from bs4 import BeautifulSoup
+from guessit import guessit
+
+from subliminal_patch.providers import Provider
+from subliminal.providers import Episode, Movie
+from subliminal_patch.utils import sanitize
+from subliminal_patch.subtitle import Subtitle, guess_matches
+from subliminal.subtitle import fix_line_ending
+
+__author__ = "Dor Nizar"
+
+logger = logging.getLogger(__name__)
+
+
+class ScrewZiraSubtitle(Subtitle):
+    provider_name = 'screwzira'
+
+    def __init__(self, language, title_id, subtitle_id, series, season, episode, release, year):
+        super(ScrewZiraSubtitle, self).__init__(language, subtitle_id)
+        self.title_id = title_id
+        self.subtitle_id = subtitle_id
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.release = release
+        self.year = year
+
+    def get_matches(self, video):
+        matches = set()
+        logger.debug("--ScrewZiraSubtitle--\n{}".format(self.__dict__))
+
+        # episode
+        if isinstance(video, Episode):
+            # series
+            if video.series and sanitize(self.series) == sanitize(video.series):
+                matches.add('series')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+            # guess
+            matches |= guess_matches(video, guessit(self.release, {'type': 'episode'}))
+        # movie
+        elif isinstance(video, Movie):
+            # title
+            if video.title and (sanitize(self.series) in (
+                    sanitize(name) for name in [video.title] + video.alternative_titles)):
+                matches.add('title')
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+            # guess
+            matches |= guess_matches(video, guessit(self.release, {'type': 'movie'}))
+
+        logger.debug("ScrewZira subtitle criteria match:\n{}".format(matches))
+        return matches
+
+    @property
+    def id(self):
+        return self.subtitle_id
+
+
+class ScrewZiraProvider(Provider):
+    subtitle_class = ScrewZiraSubtitle
+    languages = {Language.fromalpha2(l) for l in ['he']}
+    URL_SERVER = 'https://www.screwzira.com/'
+
+    URI_SEARCH_TITLE = 'Services/ContentProvider.svc/GetSearchForecast'
+    URI_SEARCH_SERIES_SUBTITLE = 'Services/GetModuleAjax.ashx'
+    URI_SEARCH_MOVIE_SUBTITLE = "MovieInfo.aspx"
+    URI_REQ_SUBTITLE_ID = "Services/ContentProvider.svc/RequestSubtitleDownload"
+    URI_DOWNLOAD_SUBTITLE = "Services/DownloadFile.ashx"
+
+    def initialize(self):
+        logger.debug("ScrewZira initialize")
+        self.session = Session()
+        self.session.headers[
+            'User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; ' \
+                            'Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
+
+    def terminate(self):
+        logger.debug("ScrewZira terminate")
+        self.session.close()
+
+    def __init__(self):
+        self.session = None
+
+    def _search_series(self, title):
+        logger.debug("Searching '{}'".format(title))
+        title_request = {
+            "request": {
+                "SearchString": title,
+                "SearchType": "Film"
+            }
+        }
+        r = self.session.post(self.URL_SERVER + self.URI_SEARCH_TITLE, json=title_request, allow_redirects=False,
+                              timeout=10)
+        r.raise_for_status()
+        series_found = r.json()
+        if 'd' in series_found:
+            try:
+                series_found = json.loads(series_found['d'])
+            except ValueError:
+                series_found = None
+        if 'Items' in series_found:
+            return series_found['Items']
+        return []
+
+    def _search_subtitles(self, title_id, season=None, episode=None):
+        if season and episode:
+            params = {
+                'moduleName': 'SubtitlesList',
+                'SeriesID': title_id,
+                'Season': season,
+                'Episode': episode
+            }
+            r = self.session.get(url=self.URL_SERVER + self.URI_SEARCH_SERIES_SUBTITLE, params=params)
+        else:
+            params = {
+                'ID': title_id,
+            }
+            r = self.session.get(url=self.URL_SERVER + self.URI_SEARCH_MOVIE_SUBTITLE, params=params)
+
+        r.raise_for_status()
+        results = r.content
+        if not results:
+            return []
+        subtitles = BeautifulSoup(results, 'html.parser').select('a.fa')
+        logger.debug("[BS4] Elements found:\n{}".format(subtitles))
+        subtitle_list = []
+        for i in subtitles:
+            subtitle_id = i.attrs['data-subtitle-id']
+            release = i.findParent().findParent().text.strip().split('\n')[0]
+            subtitle_list.append((subtitle_id, release))
+
+        return subtitle_list  # [(Subtitle ID, name), (....)]
+
+    def _req_download_identifier(self, title_id, subtitle_id):
+        logger.debug("Request subtitle identifier for: title id: {}, subtitle id: {}".format(title_id, subtitle_id))
+        data = {
+            'request': {
+                'FilmID': title_id,
+                'SubtitleID': subtitle_id,
+                'FontSize': 0,
+                'FontColor': "",
+                'PredefinedLayout': -1
+            }
+        }
+
+        r = self.session.post(self.URL_SERVER + self.URI_REQ_SUBTITLE_ID, json=data, allow_redirects=False,
+                              timeout=10)
+        r.raise_for_status()
+        try:
+            r = json.loads(r.json()['d'])
+        except ValueError:
+            r = {}
+
+        if 'DownloadIdentifier' not in r:
+            logger.error("Download Identifier not found")
+            return None
+        return r['DownloadIdentifier']
+
+    def _download_subtitles(self, download_id):
+        logger.debug("Downloading subtitles by download identifier - {}".format(download_id))
+        data = {'DownloadIdentifier': download_id}
+        r = self.session.get(self.URL_SERVER + self.URI_DOWNLOAD_SUBTITLE, params=data,
+                             timeout=10)
+        r.raise_for_status()
+        if not r.content:
+            logger.debug("Download subtitle failed")
+            return None
+
+        logger.debug("Download subtitle success")
+        return r.content
+
+    def query(self, title, season=None, episode=None, year=None):
+        subtitles = []
+        titles = self._search_series(title)
+        if season and episode:
+            logger.debug("Searching for:\nTitle: {}\nSeason: {}\nEpisode: {}\nYear: {}".format(title, season,
+                                                                                               episode, year))
+        else:
+            logger.debug("Searching for:\nTitle: {}\nYear: {}\n".format(title, year))
+        for title in titles:
+            logger.debug("Title Candidate: {}".format(title))
+            title_id = title['ID']
+            if season and episode:
+                result = self._search_subtitles(title_id, season, episode)
+            else:
+                result = self._search_subtitles(title_id)
+
+            if not result:
+                continue
+
+            for subtitle_id, release in result:
+                subtitles.append(self.subtitle_class(next(iter(self.languages)), title_id, subtitle_id,
+                                                     title['EngName'], season, episode, release, year))
+
+        if subtitles:
+            logger.debug("Found Subtitle Candidates: {}".format(subtitles))
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        season = episode = year = title = None
+
+        if isinstance(video, Episode):
+            logger.info("list_subtitles Series: {}, season: {}, episode: {}".format(video.series,
+                                                                                    video.season,
+                                                                                    video.episode))
+            title = video.series
+            season = video.season
+            episode = video.episode
+        elif isinstance(video, Movie):
+            logger.info("list_subtitles Movie: {}, year: {}".format(video.title, video.year))
+            title = video.title
+            year = video.year
+
+        return [s for s in self.query(title, season, episode, year) if s.language in languages]
+
+    def download_subtitle(self, subtitle):
+        # type: (ScrewZiraSubtitle) -> None
+
+        logger.info('Downloading subtitle from ScrewZira: %r', subtitle)
+        downloadID = self._req_download_identifier(subtitle.title_id, subtitle.subtitle_id)
+        if not downloadID:
+            logger.debug('Unable to retrieve download identifier')
+            return None
+
+        content = self._download_subtitles(downloadID)
+        if not content:
+            logger.debug('Unable to download subtitle')
+            return None
+
+        subtitle.content = fix_line_ending(content)

Contents/Libraries/Shared/subliminal_patch/refiners/drone.py

@@ -19,11 +19,11 @@ class DroneAPIClient(object):
     _fill_attrs = None
 
     def __init__(self, version=1, session=None, headers=None, timeout=10, base_url=None, api_key=None,
-                 ssl_no_verify=False):
+                 ssl_no_verify=False, pem_file=None):
         headers = dict(headers or {}, **{"X-Api-Key": api_key})
 
         #: Session for the requests
-        self.session = session or CertifiSession()
+        self.session = session or CertifiSession(verify=pem_file)
         if ssl_no_verify:
             self.session.verify = False
 

Contents/Libraries/Shared/subliminal_patch/refiners/tvdb.py

@@ -87,7 +87,10 @@ def refine(video, **kwargs):
         # parse series year
         series_year = None
         if result['firstAired']:
-            series_year = datetime.datetime.strptime(result['firstAired'], '%Y-%m-%d').year
+            try:
+                series_year = datetime.datetime.strptime(result['firstAired'], '%Y-%m-%d').year
+            except ValueError:
+                continue
 
         # discard mismatches on year
         if video.year and series_year and video.year != series_year:

Contents/Libraries/Shared/subliminal_patch/score.py

@@ -60,6 +60,8 @@ def compute_score(matches, subtitle, video, hearing_impaired=None):
     episode_hash_valid_if = {"series", "season", "episode", "format"}
     movie_hash_valid_if = {"video_codec", "format"}
 
+    orig_matches = matches.copy()
+
     # on hash match, discard everything else
     if subtitle.hash_verifiable:
         if 'hash' in matches:
@@ -83,41 +85,47 @@ def compute_score(matches, subtitle, video, hearing_impaired=None):
         matches &= {'hash'}
 
     # handle equivalent matches
+    eq_matches = set()
     if is_episode:
         if 'title' in matches:
             logger.debug('Adding title match equivalent')
-            matches.add('episode')
+            eq_matches.add('episode')
         if 'series_imdb_id' in matches:
             logger.debug('Adding series_imdb_id match equivalent')
-            matches |= {'series', 'year'}
+            eq_matches |= {'series', 'year'}
         if 'imdb_id' in matches:
             logger.debug('Adding imdb_id match equivalents')
-            matches |= {'series', 'year', 'season', 'episode'}
+            eq_matches |= {'series', 'year', 'season', 'episode'}
         if 'tvdb_id' in matches:
             logger.debug('Adding tvdb_id match equivalents')
-            matches |= {'series', 'year', 'season', 'episode', 'title'}
+            eq_matches |= {'series', 'year', 'season', 'episode', 'title'}
         if 'series_tvdb_id' in matches:
             logger.debug('Adding series_tvdb_id match equivalents')
-            matches |= {'series', 'year'}
+            eq_matches |= {'series', 'year'}
 
         # specials
         if video.is_special and 'title' in matches and 'series' in matches \
                 and 'year' in matches:
             logger.debug('Adding special title match equivalent')
-            matches |= {'season', 'episode'}
+            eq_matches |= {'season', 'episode'}
 
     elif is_movie:
         if 'imdb_id' in matches:
             logger.debug('Adding imdb_id match equivalents')
-            matches |= {'title', 'year'}
+            eq_matches |= {'title', 'year'}
+
+    matches |= eq_matches
 
     # handle hearing impaired
     if hearing_impaired is not None and subtitle.hearing_impaired == hearing_impaired:
         logger.debug('Matched hearing_impaired')
         matches.add('hearing_impaired')
+        orig_matches.add('hearing_impaired')
 
     # compute the score
     score = sum((scores.get(match, 0) for match in matches))
     logger.info('%r: Computed score %r with final matches %r', subtitle, score, matches)
 
-    return score
+    score_without_hash = sum((scores.get(match, 0) for match in orig_matches | eq_matches if match != "hash"))
+
+    return score, score_without_hash

Contents/Libraries/Shared/subliminal_patch/subtitle.py

@@ -278,6 +278,12 @@ class Subtitle(Subtitle_):
 
     @classmethod
     def pysubs2_to_unicode(cls, sub, format="srt"):
+        """
+        this is a modified version of pysubs2.SubripFormat.to_file with special handling for drawing tags in ASS
+        :param sub:
+        :param format:
+        :return:
+        """
         def ms_to_timestamp(ms, mssep=","):
             """Convert ms to 'HH:MM:SS,mmm'"""
             # XXX throw on overflow/underflow?
@@ -289,9 +295,12 @@ class Subtitle(Subtitle_):
         def prepare_text(text, style):
             body = []
             for fragment, sty in parse_tags(text, style, sub.styles):
-                fragment = fragment.replace(ur"\h", u" ")
-                fragment = fragment.replace(ur"\n", u"\n")
-                fragment = fragment.replace(ur"\N", u"\n")
+                fragment = fragment.replace(r"\h", u" ")
+                fragment = fragment.replace(r"\n", u"\n")
+                fragment = fragment.replace(r"\N", u"\n")
+                if sty.drawing:
+                    raise pysubs2.ContentNotUsable
+
                 if format == "srt":
                     if sty.italic:
                         fragment = u"<i>%s</i>" % fragment
@@ -323,7 +332,10 @@ class Subtitle(Subtitle_):
         for i, line in enumerate(visible_lines, 1):
             start = ms_to_timestamp(line.start, mssep=mssep)
             end = ms_to_timestamp(line.end, mssep=mssep)
-            text = prepare_text(line.text, sub.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            try:
+                text = prepare_text(line.text, sub.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            except pysubs2.ContentNotUsable:
+                continue
 
             out.append(u"%d\n" % i)
             out.append(u"%s --> %s\n" % (start, end))
@@ -357,6 +369,15 @@ class ModifiedSubtitle(Subtitle):
     id = None
 
 
+MERGED_FORMATS = {
+    "TV": ("HDTV", "SDTV", "AHDTV", "UHDTV"),
+    "Air": ("SATRip", "DVB", "PPV"),
+    "Disk": ("DVD", "HD-DVD", "BluRay")
+}
+
+MERGED_FORMATS_REV = dict((v.lower(), k.lower()) for k in MERGED_FORMATS for v in MERGED_FORMATS[k])
+
+
 def guess_matches(video, guess, partial=False):
     """Get matches between a `video` and a `guess`.
 
@@ -439,21 +460,25 @@ def guess_matches(video, guess, partial=False):
             formats = [formats]
 
         if video.format:
-            video_format = video.format
-            if video_format in ("HDTV", "SDTV", "TV"):
-                video_format = "TV"
-                logger.debug("Treating HDTV/SDTV the same")
+            video_format = video.format.lower()
+            _video_gen_format = MERGED_FORMATS_REV.get(video_format)
+            if _video_gen_format:
+                logger.debug("Treating %s as %s the same", video_format, _video_gen_format)
 
             for frmt in formats:
-                if frmt in ("HDTV", "SDTV"):
-                    frmt = "TV"
+                _guess_gen_frmt = MERGED_FORMATS_REV.get(frmt.lower())
 
-                if frmt.lower() == video_format.lower():
+                if _guess_gen_frmt == _video_gen_format:
                     matches.add('format')
                     break
+        if "release_group" in matches and "format" not in matches:
+            logger.info("Release group matched but format didn't. Remnoving release group match.")
+            matches.remove("release_group")
+
     # video_codec
     if video.video_codec and 'video_codec' in guess and guess['video_codec'] == video.video_codec:
         matches.add('video_codec')
+
     # audio_codec
     if video.audio_codec and 'audio_codec' in guess and guess['audio_codec'] == video.audio_codec:
         matches.add('audio_codec')

Contents/Libraries/Shared/submod_test.py

@@ -21,9 +21,10 @@ if debug:
     logging.basicConfig(level=logging.DEBUG)
 
 #sub = Subtitle(Language.fromietf("eng:forced"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=-500)", "shift_offset(ms=500)", "shift_offset(s=2,ms=800)"])
-sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=0,s=1)"])
+sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=0,s=1)", "fix_incremental", "fix_short"])
 sub.content = open(fn).read()
 sub.normalize()
+sub.is_valid()
 content = sub.get_modified_content(debug=True)
 
 #submod = SubMod(debug=debug)

Contents/Libraries/Shared/subzero/language.py

@@ -31,9 +31,9 @@ repl_map = {
 }
 
 
-ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)) + repl_map.values()))
-ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)) +
-                        filter(lambda x: len(x) == 3, repl_map.keys())))
+ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)))) + list(repl_map.values())
+ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)))) + \
+               list(set(filter(lambda x: len(x) == 3, list(repl_map.keys()))))
 FULL_LANGUAGE_LIST = ALPHA2_LIST + ALPHA3b_LIST
 
 

Contents/Libraries/Shared/subzero/modification/exc.py

@@ -0,0 +1,7 @@
+# coding=utf-8
+class EmptyEntryError(Exception):
+    pass
+
+
+class EmptyLineError(Exception):
+    pass

Contents/Libraries/Shared/subzero/modification/main.py

@@ -6,7 +6,8 @@ import pysubs2
 import logging
 import time
 
-from mods import EMPTY_TAG_PROCESSOR, EmptyEntryError
+from mods import EMPTY_TAG_PROCESSOR
+from exc import EmptyEntryError
 from registry import registry
 from subzero.language import Language
 
@@ -300,11 +301,11 @@ class SubtitleModifications(object):
                     mod = self.initialized_mods[identifier]
 
                     try:
-                        line = mod.modify(line.strip(), entry=entry.text, debug=self.debug, parent=self, index=index,
+                        line = mod.modify(line.strip(), entry=t, debug=self.debug, parent=self, index=index,
                                           **args)
                     except EmptyEntryError:
                         if self.debug:
-                            logger.debug(u"%d: %s: %r -> ''", index, identifier, entry.text)
+                            logger.debug(u"%d: %s: %r -> ''", index, identifier, t)
                         skip_entry = True
                         break
 
@@ -329,11 +330,11 @@ class SubtitleModifications(object):
                     mod = self.initialized_mods[identifier]
 
                     try:
-                        line = mod.modify(line.strip(), entry=entry.text, debug=self.debug, parent=self, index=index,
+                        line = mod.modify(line.strip(), entry=t, debug=self.debug, parent=self, index=index,
                                           procs=["last_process"], **args)
                     except EmptyEntryError:
                         if self.debug:
-                            logger.debug(u"%d: %s: %r -> ''", index, identifier, entry.text)
+                            logger.debug(u"%d: %s: %r -> ''", index, identifier, t)
                         skip_entry = True
                         break
 

Contents/Libraries/Shared/subzero/modification/mods/__init__.py

@@ -107,9 +107,3 @@ empty_line_post_processors = [
 ]
 
 
-class EmptyEntryError(Exception):
-    pass
-
-
-class EmptyLineError(Exception):
-    pass

Contents/Libraries/Shared/subzero/modification/mods/common.py

@@ -1,14 +1,17 @@
 # coding=utf-8
 
 import re
+import logging
+from collections import OrderedDict
 
 from subzero.language import Language
 from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, SubtitleModification
 from subzero.modification.processors import FuncProcessor
 from subzero.modification.processors.re_processor import NReProcessor
 from subzero.modification import registry
+from tld import get_tld
 
-
+logger = logging.getLogger(__name__)
 ENGLISH = Language("eng")
 
 
@@ -28,7 +31,7 @@ class CommonFixes(SubtitleTextModification):
         NReProcessor(re.compile(r'(?u)(\w|\b|\s|^)(-\s?-{1,2})'), ur"\1—", name="CM_multidash"),
 
         # line = _/-/\s
-        NReProcessor(re.compile(r'(?u)(^\W*[-_.:>~]+\W*$)'), "", name="<CM_non_word_only"),
+        NReProcessor(re.compile(r'(?u)(^\W*[-_.:<>~"\']+\W*$)'), "", name="CM_non_word_only"),
 
         # remove >>
         NReProcessor(re.compile(r'(?u)^\s?>>\s*'), "", name="CM_leading_crocodiles"),
@@ -113,7 +116,9 @@ class CommonFixes(SubtitleTextModification):
         NReProcessor(re.compile(r'(?u)(?:(?<=^)|(?<=\w)) +([!?.,](?![!?.,]| \.))'), r"\1", name="CM_punctuation_space"),
 
         # add space after punctuation
-        NReProcessor(re.compile(r'(?u)([!?.,:])([A-zÀ-ž]{2,})'), r"\1 \2", name="CM_punctuation_space2"),
+        NReProcessor(re.compile(r'(?u)(([^\s]*)([!?.,:])([A-zÀ-ž]{2,}))'),
+                     lambda match: u"%s%s %s" % (match.group(2), match.group(3), match.group(4)) if not get_tld(match.group(1), fail_silently=True, fix_protocol=True) else match.group(1),
+                     name="CM_punctuation_space2"),
 
         # fix lowercase I in english
         NReProcessor(re.compile(r'(?u)(\b)i(\b)'), r"\1I\2", name="CM_EN_lowercase_i",
@@ -178,7 +183,104 @@ class FixUppercase(SubtitleModification):
             entry.plaintext = self.capitalize(entry.plaintext)
 
 
+class FixIncremental(SubtitleModification):
+    identifier = "fix_incremental"
+    description = "Fixes inremental-repeating subtitles"
+    modifies_whole_file = True
+    exclusive = True
+
+    long_description = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+    def modify(self, content, debug=False, parent=None, **kwargs):
+        prev_entry = None
+        for entry in parent.f:
+            subs = []
+            for sub in entry.text.split("\N"):
+                if prev_entry and prev_entry.text and prev_entry.text.lower().endswith(sub.lower()):
+                    if debug:
+                        logger.debug(u"Skipping incremental/dup: %s" % sub)
+                    continue
+                subs.append(sub)
+
+            if subs:
+                entry.text = "\N".join(subs)
+
+            prev_entry = entry
+
+
+
+class FixShort(SubtitleModification):
+    identifier = "fix_short"
+    description = "ASDasdasdasdasdd"
+    modifies_whole_file = True
+    exclusive = True
+
+    long_description = "adsadsdasdsadsa"
+
+    def modify(self, content, debug=False, parent=None, **kwargs):
+        prev_entry = None
+        prev_entry_dur = None
+        max_duration = 500
+        max_line_len = 200
+        max_lines = 3
+        entries = []
+        last_lines = []
+        for index, entry in enumerate(parent.f):
+            current_new_lines = []
+            if not last_lines and parent.f[index-1]:
+                print "YOO"
+                # find last lines
+                last_lines = parent.f[index-1].text.split("\N")
+            has_space = len(last_lines) < max_lines
+            last_line = ""
+            # go through each line and pack them
+            for line in entry.text.split("\N"):
+                new_line = ""
+                if line:
+                    if last_line != line and last_line and len(last_line + line) <= max_line_len:
+                        # new line plus line fits
+                        if re.match(".+\W$", line):
+                            if last_line.endswith(" "):
+                                new_line = last_line + line
+                            else:
+                                new_line = last_line + " " + line
+                            logger.debug("MERGING '%s' with '%s' to '%s'", last_line, line, new_line)
+                    else:
+                        new_line = line
+                last_line = new_line
+                current_new_lines.append(new_line)
+
+            # merge entries
+            if prev_entry:
+                #print prev_entry.duration, max_duration, len(new_lines), max_lines
+                if prev_entry.duration < max_duration and len(current_new_lines) < max_lines:
+                    #len(prev_entry.text) < max_len
+                    print "HIT", prev_entry.text, " + ", entry.text
+                    entry_text = prev_entry.text + "\N" + "\N".join(current_new_lines)
+
+                else:
+                    entry_text = "\N".join(current_new_lines)
+            else:
+                entry_text = "\N".join(current_new_lines)
+
+            #prev_entry = entry.copy()
+            if not prev_entry:
+                prev_entry = entry.copy()
+                continue
+            new_entry = prev_entry.copy()
+            new_entry.text = entry_text
+            prev_entry = new_entry.copy()
+            entries.append(new_entry)
+            #new_entries.append(entry.copy())
+
+        parent.f.entries = entries
+
+
+
+
 registry.register(CommonFixes)
 registry.register(RemoveTags)
 registry.register(ReverseRTL)
 registry.register(FixUppercase)
+registry.register(FixIncremental)
+registry.register(FixShort)

Contents/Libraries/Shared/subzero/modification/mods/hearing_impaired.py

@@ -1,7 +1,8 @@
 # coding=utf-8
 import re
 
-from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, EmptyEntryError, TAG
+from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, TAG
+from subzero.modification.exc import EmptyEntryError
 from subzero.modification.processors.re_processor import NReProcessor
 from subzero.modification import registry
 
@@ -46,7 +47,7 @@ class HearingImpaired(SubtitleTextModification):
                      name="HI_before_colon_noncaps"),
 
         # brackets (only remove if at least 3 chars in brackets)
-        NReProcessor(re.compile(ur'(?sux)-?%(t)s[([][^([)\]]+?(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+[)\]][\s:]*%(t)s' %
+        NReProcessor(re.compile(ur'(?sux)-?%(t)s["\']*[([][^([)\]]+?(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+[)\]]["\']*[\s:]*%(t)s' %
                                 {"t": TAG}), "", name="HI_brackets"),
 
         #NReProcessor(re.compile(ur'(?sux)-?%(t)s[([]%(t)s(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+%(t)s$' % {"t": TAG}),
@@ -90,8 +91,8 @@ class HearingImpaired(SubtitleTextModification):
                      "", name="HI_music_symbols_only"),
 
         # remove music entries
-        NReProcessor(re.compile(ur'(?ums)(^[-\s>~]*[♫♪]+\s*.+|.+\s*[♫♪]+\s*$)'),
-                     "", name="HI_music"),
+        NReProcessor(re.compile(ur'(?ums)(^[-\s>~]*[*#¶♫♪]+\s*.+|.+\s*[*#¶♫♪]+\s*$)'),
+                     "", name="HI_music", entry=True),
     ]
 
 

Contents/Libraries/Shared/subzero/modification/processors/__init__.py

@@ -10,7 +10,7 @@ class Processor(object):
     supported = None
     enabled = True
 
-    def __init__(self, name=None, parent=None, supported=None):
+    def __init__(self, name=None, parent=None, supported=None, **kwargs):
         self.name = name
         self.parent = parent
         self.supported = supported if supported else lambda parent: True
@@ -35,7 +35,7 @@ class Processor(object):
 class FuncProcessor(Processor):
     func = None
 
-    def __init__(self, func, name=None, parent=None, supported=None):
+    def __init__(self, func, name=None, parent=None, supported=None, **kwargs):
         super(FuncProcessor, self).__init__(name=name, supported=supported)
         self.func = func
 

Contents/Libraries/Shared/subzero/modification/processors/re_processor.py

@@ -2,6 +2,7 @@
 import re
 import logging
 
+from subzero.modification.exc import EmptyEntryError
 from subzero.modification.processors import Processor
 
 logger = logging.getLogger(__name__)
@@ -14,13 +15,22 @@ class ReProcessor(Processor):
     pattern = None
     replace_with = None
 
-    def __init__(self, pattern, replace_with, name=None, supported=None):
+    def __init__(self, pattern, replace_with, name=None, supported=None, entry=False, **kwargs):
         super(ReProcessor, self).__init__(name=name, supported=supported)
         self.pattern = pattern
         self.replace_with = replace_with
+        self.use_entry = entry
 
-    def process(self, content, debug=False, **kwargs):
-        return self.pattern.sub(self.replace_with, content)
+    def process(self, content, debug=False, entry=None, **kwargs):
+        if not self.use_entry:
+            return self.pattern.sub(self.replace_with, content)
+
+        ret = self.pattern.sub(self.replace_with, entry)
+        if not ret:
+            raise EmptyEntryError()
+        elif ret != entry:
+            return ret
+        return content
 
 
 class NReProcessor(ReProcessor):
@@ -36,7 +46,7 @@ class MultipleWordReProcessor(ReProcessor):
     }
     replaces found key in pattern with the corresponding value in data
     """
-    def __init__(self, snr_dict, name=None, parent=None, supported=None):
+    def __init__(self, snr_dict, name=None, parent=None, supported=None, **kwargs):
         super(ReProcessor, self).__init__(name=name, supported=supported)
         self.snr_dict = snr_dict
 

Contents/Libraries/Shared/subzero/modification/processors/string_processor.py

@@ -12,7 +12,7 @@ class StringProcessor(Processor):
     String replacement processor base
     """
 
-    def __init__(self, search, replace, name=None, parent=None, supported=None):
+    def __init__(self, search, replace, name=None, parent=None, supported=None, **kwargs):
         super(StringProcessor, self).__init__(name=name, supported=supported)
         self.search = search
         self.replace = replace
@@ -31,7 +31,7 @@ class MultipleLineProcessor(Processor):
         "data": {"old_value": "new_value"}
     }
     """
-    def __init__(self, snr_dict, name=None, parent=None, supported=None):
+    def __init__(self, snr_dict, name=None, parent=None, supported=None, **kwargs):
         super(MultipleLineProcessor, self).__init__(name=name, supported=supported)
         self.snr_dict = snr_dict
 

Contents/Libraries/Shared/test.srt

@@ -19,6 +19,8 @@ I can't keep running. L can't!
 <b>i don't know. Some kind of wrong "1 00" number---
 of signal, drawing the Tardis off.... course.</b>
 # I'm singing in the rain
+www.website.com
+www.nowebsite.badlol
 
 4
 00:00:16,099 --> 00:00:17,224

Contents/Libraries/Shared/tld/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from .utils import get_fld, get_tld, get_tld_names, is_tld, parse_tld, Result, update_tld_names
+__title__ = u'tld'
+__version__ = u'0.11.10'
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'get_fld', u'get_tld', u'get_tld_names', u'is_tld',
+           u'parse_tld', u'Result', u'update_tld_names')

Contents/Libraries/Shared/tld/base.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from six import with_metaclass as _py_backwards_six_withmetaclass
+from codecs import open as codecs_open
+try:
+    from urllib.request import urlopen
+except ImportError:
+    from six.moves.urllib.request import urlopen as urlopen
+from .exceptions import TldIOError, TldImproperlyConfigured
+from .helpers import project_dir
+from .registry import Registry
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'BaseTLDSourceParser',)
+
+
+class BaseTLDSourceParser(
+        _py_backwards_six_withmetaclass(Registry, *[object])):
+    u'Base TLD source parser.'
+    uid = None
+    source_url = None
+    local_path = None
+
+    @classmethod
+    def validate(cls):
+        u'Constructor.'
+        if (not cls.uid):
+            raise TldImproperlyConfigured(
+                u'The `uid` property of the TLD source parser shall be defined.')
+
+    @classmethod
+    def get_tld_names(cls, fail_silently=False, retry_count=0):
+        u'Get tld names.\n\n        :param fail_silently:\n        :param retry_count:\n        :return:\n        '
+        cls.validate()
+        raise NotImplementedError(
+            u'Your TLD source parser shall implement `get_tld_names` method.')
+
+    @classmethod
+    def update_tld_names(cls, fail_silently=False):
+        u'Update the local copy of the TLD file.\n\n        :param fail_silently:\n        :return:\n        '
+        try:
+            remote_file = urlopen(cls.source_url)
+            local_file = codecs_open(project_dir(
+                cls.local_path), u'wb', encoding='utf8')
+            local_file.write(remote_file.read().decode(u'utf8'))
+            local_file.close()
+            remote_file.close()
+        except Exception as err:
+            if fail_silently:
+                return False
+            raise TldIOError(err)
+        return True

Contents/Libraries/Shared/tld/conf.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from typing import Any
+from . import defaults
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'get_setting', u'reset_settings', u'set_setting', u'settings')
+
+
+class Settings(object):
+    u'Settings registry.'
+
+    def __init__(self):
+        self._settings = {
+
+        }
+        self._settings_get = self._settings.get
+
+    def set(self, name, value):
+        u'\n        Override default settings.\n\n        :param str name:\n        :param mixed value:\n        '
+        self._settings[name] = value
+
+    def get(self, name, default=None):
+        u'\n        Gets a variable from local settings.\n\n        :param str name:\n        :param mixed default: Default value.\n        :return mixed:\n        '
+        if (name in self._settings):
+            return self._settings_get(name, default)
+        elif hasattr(defaults, name):
+            return getattr(defaults, name, default)
+        return default
+
+    def reset(self):
+        u'Reset settings.'
+        for name in defaults.__all__:
+            self.set(name, getattr(defaults, name))
+
+
+settings = Settings()
+get_setting = settings.get
+set_setting = settings.set
+reset_settings = settings.reset

Contents/Libraries/Shared/tld/defaults.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from os.path import dirname
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'DEBUG', u'NAMES_LOCAL_PATH_PARENT')
+NAMES_LOCAL_PATH_PARENT = dirname(__file__)
+DEBUG = False

Contents/Libraries/Shared/tld/exceptions.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from .conf import get_setting
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'TldBadUrl', u'TldDomainNotFound',
+           u'TldImproperlyConfigured', u'TldIOError')
+
+
+class TldIOError(IOError):
+    u'TldIOError.\n\n    Supposed to be thrown when problems with reading/writing occur.\n    '
+
+    def __init__(self, msg=None):
+        tld_names_local_path = get_setting(u'NAMES_LOCAL_PATH')
+        if (msg is None):
+            msg = (u"Can't read from or write to the %s file!" %
+                   tld_names_local_path)
+        super(TldIOError, self).__init__(msg)
+
+
+class TldDomainNotFound(ValueError):
+    u"TldDomainNotFound.\n\n    Supposed to be thrown when domain name is not found (didn't match) the\n    local TLD policy.\n    "
+
+    def __init__(self, domain_name):
+        super(TldDomainNotFound, self).__init__(
+            (u"Domain %s didn't match any existing TLD name!" % domain_name))
+
+
+class TldBadUrl(ValueError):
+    u'TldBadUrl.\n\n    Supposed to be thrown when bad URL is given.\n    '
+
+    def __init__(self, url):
+        super(TldBadUrl, self).__init__((u'Is not a valid URL %s!' % url))
+
+
+class TldImproperlyConfigured(Exception):
+    u'TldImproperlyConfigured.\n\n    Supposed to be thrown when code is improperly configured. Typical use-case\n    is when user tries to use `get_tld` function with both `search_public` and\n    `search_private` set to False.\n    '
+
+    def __init__(self, msg=None):
+        if (msg is None):
+            msg = u'Improperly configured.'
+        else:
+            msg = (u'Improperly configured. %s' % msg)
+        super(TldImproperlyConfigured, self).__init__(msg)

Contents/Libraries/Shared/tld/helpers.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from os.path import abspath, join
+from .conf import get_setting
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'project_dir', u'PROJECT_DIR')
+
+
+def project_dir(base):
+    u'Project dir.'
+    tld_names_local_path_parent = get_setting(u'NAMES_LOCAL_PATH_PARENT')
+    return abspath(join(tld_names_local_path_parent, base).replace(u'\\', u'/'))
+
+
+PROJECT_DIR = project_dir

Contents/Libraries/Shared/tld/parsers/__init__.py

@@ -0,0 +1,5 @@
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals

Contents/Libraries/Shared/tld/parsers/mozilla.py

@@ -0,0 +1,5 @@
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals

Contents/Libraries/Shared/tld/registry.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from typing import Type, Dict
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'Registry',)
+
+
+class Registry(type):
+    REGISTRY = {
+
+    }
+
+    def __new__(cls, name, bases, attrs):
+        new_cls = type.__new__(cls, name, bases, attrs)
+        if getattr(new_cls, u'_uid', None):
+            cls.REGISTRY[new_cls._uid] = new_cls
+        return new_cls
+
+    @property
+    def _uid(cls):
+        return getattr(cls, 'uid', cls.__name__)
+
+    @classmethod
+    def reset(cls):
+        cls.REGISTRY = {
+
+        }
+
+    @classmethod
+    def get(cls, key, default=None):
+        return cls.REGISTRY.get(key, default)
+
+    @classmethod
+    def items(cls):
+        return cls.REGISTRY.items()

Contents/Libraries/Shared/tld/result.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from typing import Dict
+try:
+    from urllib.parse import SplitResult
+except ImportError:
+    from six.moves.urllib_parse import SplitResult
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'Result',)
+
+
+class Result(object):
+    u'Container.'
+    __slots__ = (u'subdomain', u'domain', u'tld', u'__fld', u'parsed_url')
+
+    def __init__(self, tld, domain, subdomain, parsed_url):
+        self.tld = tld
+        self.domain = (domain if (domain != u'') else tld)
+        self.subdomain = subdomain
+        self.parsed_url = parsed_url
+        if domain:
+            self.__fld = u''.join(
+                [u'{}'.format(self.domain), u'.', u'{}'.format(self.tld)])
+        else:
+            self.__fld = self.tld
+
+    @property
+    def extension(self):
+        u'Alias of ``tld``.\n\n        :return str:\n        '
+        return self.tld
+    suffix = extension
+
+    @property
+    def fld(self):
+        u'First level domain.\n\n        :return:\n        :rtype: str\n        '
+        return self.__fld
+
+    def __str__(self):
+        return self.tld
+    __repr__ = __str__
+
+    @property
+    def __dict__(self):
+        u'Mimic __dict__ functionality.\n\n        :return:\n        :rtype: dict\n        '
+        return {
+            u'tld': self.tld,
+            u'domain': self.domain,
+            u'subdomain': self.subdomain,
+            u'fld': self.fld,
+            u'parsed_url': self.parsed_url,
+        }

Contents/Libraries/Shared/tld/tests/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import unittest
+from .test_core import *
+from .test_commands import *
+if (__name__ == u'__main__'):
+    unittest.main()

Contents/Libraries/Shared/tld/tests/base.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from backports.functools_lru_cache import lru_cache
+import logging
+import socket
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'internet_available_only', u'log_info')
+LOG_INFO = True
+LOGGER = logging.getLogger(__name__)
+
+
+def log_info(func):
+    u'Log some useful info.'
+    if (not LOG_INFO):
+        return func
+
+    def inner(self, *args, **kwargs):
+        u'Inner.'
+        result = func(*([self] + list(args)), **kwargs)
+        LOGGER.debug(u'\n\n%s', func.__name__)
+        LOGGER.debug(u'============================')
+        if func.__doc__:
+            LOGGER.debug(u'""" %s """', func.__doc__.strip())
+        LOGGER.debug(u'----------------------------')
+        if (result is not None):
+            LOGGER.debug(result)
+        LOGGER.debug(u'\n++++++++++++++++++++++++++++')
+        return result
+    return inner
+
+
+@lru_cache(maxsize=32)
+def is_internet_available(host='8.8.8.8', port=53, timeout=3):
+    u'Check if internet is available.\n\n    Host: 8.8.8.8 (google-public-dns-a.google.com)\n    OpenPort: 53/tcp\n    Service: domain (DNS/TCP)\n    '
+    try:
+        socket.setdefaulttimeout(timeout)
+        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+        return True
+    except socket.error as ex:
+        print(ex)
+        return False
+
+
+def internet_available_only(func):
+
+    def inner(self, *args, **kwargs):
+        u'Inner.'
+        if (not is_internet_available()):
+            LOGGER.debug(u'\n\n%s', func.__name__)
+            LOGGER.debug(u'============================')
+            if func.__doc__:
+                LOGGER.debug(u'""" %s """', func.__doc__.strip())
+            LOGGER.debug(u'----------------------------')
+            LOGGER.debug(u'Skipping because no Internet connection available.')
+            LOGGER.debug(u'\n++++++++++++++++++++++++++++')
+            return None
+        result = func(*([self] + list(args)), **kwargs)
+        return result
+    return inner

Contents/Libraries/Shared/tld/tests/test_commands.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import logging
+import unittest
+import subprocess
+from .base import log_info, internet_available_only
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'GPL 2.0/LGPL 2.1'
+__all__ = (u'TestCommands',)
+LOGGER = logging.getLogger(__name__)
+
+
+class TestCommands(unittest.TestCase):
+    u'Tld commands tests.'
+
+    def setUp(self):
+        u'Set up.'
+
+    @internet_available_only
+    @log_info
+    def test_1_update_tld_names_command(self):
+        u'Test updating the tld names (re-fetch mozilla source).'
+        res = subprocess.check_output([u'update-tld-names']).strip()
+        self.assertEqual(res, b'')
+        return res
+
+    @internet_available_only
+    @log_info
+    def test_1_update_tld_names_mozilla_command(self):
+        u'Test updating the tld names (re-fetch mozilla source).'
+        res = subprocess.check_output(
+            [u'update-tld-names', u'mozilla']).strip()
+        self.assertEqual(res, b'')
+        return res
+
+
+if (__name__ == u'__main__'):
+    unittest.main()

Contents/Libraries/Shared/tld/tests/test_core.py

@@ -0,0 +1,708 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import logging
+from os.path import abspath, join
+import unittest
+from tempfile import gettempdir
+from typing import Type
+try:
+    from urllib.parse import urlsplit
+except ImportError:
+    from six.moves.urllib_parse import urlsplit
+from faker import Faker
+from .. import defaults
+from ..base import BaseTLDSourceParser
+from ..conf import get_setting, reset_settings, set_setting
+from ..exceptions import TldBadUrl, TldDomainNotFound, TldImproperlyConfigured, TldIOError
+from ..helpers import project_dir
+from ..registry import Registry
+from ..utils import get_fld, get_tld, get_tld_names, get_tld_names_container, is_tld, MozillaTLDSourceParser, BaseMozillaTLDSourceParser, parse_tld, reset_tld_names, update_tld_names, update_tld_names_cli
+from .base import internet_available_only, log_info
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'TestCore',)
+LOGGER = logging.getLogger(__name__)
+
+
+class TestCore(unittest.TestCase):
+    u'Core tld functionality tests.'
+
+    @classmethod
+    def setUpClass(cls):
+        cls.faker = Faker()
+        cls.temp_dir = gettempdir()
+
+    def setUp(self):
+        u'Set up.'
+        self.good_patterns = [{
+            u'url': u'http://www.google.co.uk',
+            u'fld': u'google.co.uk',
+            u'subdomain': u'www',
+            u'domain': u'google',
+            u'suffix': u'co.uk',
+            u'tld': u'co.uk',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.v2.google.co.uk',
+            u'fld': u'google.co.uk',
+            u'subdomain': u'www.v2',
+            u'domain': u'google',
+            u'suffix': u'co.uk',
+            u'tld': u'co.uk',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://хром.гугл.рф',
+            u'fld': u'гугл.рф',
+            u'subdomain': u'хром',
+            u'domain': u'гугл',
+            u'suffix': u'рф',
+            u'tld': u'рф',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.google.co.uk:8001/lorem-ipsum/',
+            u'fld': u'google.co.uk',
+            u'subdomain': u'www',
+            u'domain': u'google',
+            u'suffix': u'co.uk',
+            u'tld': u'co.uk',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.me.cloudfront.net',
+            u'fld': u'me.cloudfront.net',
+            u'subdomain': u'www',
+            u'domain': u'me',
+            u'suffix': u'cloudfront.net',
+            u'tld': u'cloudfront.net',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.v2.forum.tech.google.co.uk:8001/lorem-ipsum/',
+            u'fld': u'google.co.uk',
+            u'subdomain': u'www.v2.forum.tech',
+            u'domain': u'google',
+            u'suffix': u'co.uk',
+            u'tld': u'co.uk',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'https://pantheon.io/',
+            u'fld': u'pantheon.io',
+            u'subdomain': u'',
+            u'domain': u'pantheon',
+            u'suffix': u'io',
+            u'tld': u'io',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'v2.www.google.com',
+            u'fld': u'google.com',
+            u'subdomain': u'v2.www',
+            u'domain': u'google',
+            u'suffix': u'com',
+            u'tld': u'com',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'//v2.www.google.com',
+            u'fld': u'google.com',
+            u'subdomain': u'v2.www',
+            u'domain': u'google',
+            u'suffix': u'com',
+            u'tld': u'com',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'http://foo@bar.com',
+            u'fld': u'bar.com',
+            u'subdomain': u'',
+            u'domain': u'bar',
+            u'suffix': u'com',
+            u'tld': u'com',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://user:foo@bar.com',
+            u'fld': u'bar.com',
+            u'subdomain': u'',
+            u'domain': u'bar',
+            u'suffix': u'com',
+            u'tld': u'com',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'https://faguoren.xn--fiqs8s',
+            u'fld': u'faguoren.xn--fiqs8s',
+            u'subdomain': u'',
+            u'domain': u'faguoren',
+            u'suffix': u'xn--fiqs8s',
+            u'tld': u'xn--fiqs8s',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'blogs.lemonde.paris',
+            u'fld': u'lemonde.paris',
+            u'subdomain': u'blogs',
+            u'domain': u'lemonde',
+            u'suffix': u'paris',
+            u'tld': u'paris',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'axel.brighton.ac.uk',
+            u'fld': u'brighton.ac.uk',
+            u'subdomain': u'axel',
+            u'domain': u'brighton',
+            u'suffix': u'ac.uk',
+            u'tld': u'ac.uk',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'm.fr.blogspot.com.au',
+            u'fld': u'fr.blogspot.com.au',
+            u'subdomain': u'm',
+            u'domain': u'fr',
+            u'suffix': u'blogspot.com.au',
+            u'tld': u'blogspot.com.au',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'help.www.福岡.jp',
+            u'fld': u'www.福岡.jp',
+            u'subdomain': u'help',
+            u'domain': u'www',
+            u'suffix': u'福岡.jp',
+            u'tld': u'福岡.jp',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'syria.arabic.variant.سوريا',
+            u'fld': u'variant.سوريا',
+            u'subdomain': u'syria.arabic',
+            u'domain': u'variant',
+            u'suffix': u'سوريا',
+            u'tld': u'سوريا',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': u'http://www.help.kawasaki.jp',
+            u'fld': u'www.help.kawasaki.jp',
+            u'subdomain': u'',
+            u'domain': u'www',
+            u'suffix': u'help.kawasaki.jp',
+            u'tld': u'help.kawasaki.jp',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.city.kawasaki.jp',
+            u'fld': u'city.kawasaki.jp',
+            u'subdomain': u'www',
+            u'domain': u'city',
+            u'suffix': u'kawasaki.jp',
+            u'tld': u'kawasaki.jp',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://fedoraproject.org',
+            u'fld': u'fedoraproject.org',
+            u'subdomain': u'',
+            u'domain': u'fedoraproject',
+            u'suffix': u'org',
+            u'tld': u'org',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.cloud.fedoraproject.org',
+            u'fld': u'www.cloud.fedoraproject.org',
+            u'subdomain': u'',
+            u'domain': u'www',
+            u'suffix': u'cloud.fedoraproject.org',
+            u'tld': u'cloud.fedoraproject.org',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'https://www.john.app.os.fedoraproject.org',
+            u'fld': u'john.app.os.fedoraproject.org',
+            u'subdomain': u'www',
+            u'domain': u'john',
+            u'suffix': u'app.os.fedoraproject.org',
+            u'tld': u'app.os.fedoraproject.org',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'ftp://www.xn--mxail5aa.xn--11b4c3d',
+            u'fld': u'xn--mxail5aa.xn--11b4c3d',
+            u'subdomain': u'www',
+            u'domain': u'xn--mxail5aa',
+            u'suffix': u'xn--11b4c3d',
+            u'tld': u'xn--11b4c3d',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://cloud.fedoraproject.org',
+            u'fld': u'cloud.fedoraproject.org',
+            u'subdomain': u'',
+            u'domain': u'cloud.fedoraproject.org',
+            u'suffix': u'cloud.fedoraproject.org',
+            u'tld': u'cloud.fedoraproject.org',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'github.io',
+            u'fld': u'github.io',
+            u'subdomain': u'',
+            u'domain': u'github.io',
+            u'suffix': u'github.io',
+            u'tld': u'github.io',
+            u'kwargs': {
+                u'fail_silently': True,
+                u'fix_protocol': True,
+            },
+        }, {
+            u'url': urlsplit(u'http://lemonde.fr/article.html'),
+            u'fld': u'lemonde.fr',
+            u'subdomain': u'',
+            u'domain': u'lemonde',
+            u'suffix': u'fr',
+            u'tld': u'fr',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }]
+        self.bad_patterns = {
+            u'v2.www.google.com': {
+                u'exception': TldBadUrl,
+            },
+            u'/index.php?a=1&b=2': {
+                u'exception': TldBadUrl,
+            },
+            u'http://www.tld.doesnotexist': {
+                u'exception': TldDomainNotFound,
+            },
+            u'https://2001:0db8:0000:85a3:0000:0000:ac1f:8001': {
+                u'exception': TldDomainNotFound,
+            },
+            u'http://192.169.1.1': {
+                u'exception': TldDomainNotFound,
+            },
+            u'http://localhost:8080': {
+                u'exception': TldDomainNotFound,
+            },
+            u'https://localhost': {
+                u'exception': TldDomainNotFound,
+            },
+            u'https://localhost2': {
+                u'exception': TldImproperlyConfigured,
+                u'kwargs': {
+                    u'search_public': False,
+                    u'search_private': False,
+                },
+            },
+        }
+        self.invalid_tlds = {u'v2.www.google.com', u'tld.doesnotexist',
+                             u'2001:0db8:0000:85a3:0000:0000:ac1f', u'192.169.1.1', 'localhost', u'google.com'}
+        self.tld_names_local_path_custom = project_dir(
+            join(u'tests', u'res', u'effective_tld_names_custom.dat.txt'))
+        self.good_patterns_custom_parser = [{
+            u'url': u'http://www.foreverchild',
+            u'fld': u'www.foreverchild',
+            u'subdomain': u'',
+            u'domain': u'www',
+            u'suffix': u'foreverchild',
+            u'tld': u'foreverchild',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }, {
+            u'url': u'http://www.v2.foreverchild',
+            u'fld': u'v2.foreverchild',
+            u'subdomain': u'www',
+            u'domain': u'v2',
+            u'suffix': u'foreverchild',
+            u'tld': u'foreverchild',
+            u'kwargs': {
+                u'fail_silently': True,
+            },
+        }]
+        reset_settings()
+
+    def tearDown(self):
+        u'Tear down.'
+        reset_settings()
+        Registry.reset()
+
+    @property
+    def good_url(self):
+        return self.good_patterns[0][u'url']
+
+    @property
+    def bad_url(self):
+        return list(self.bad_patterns.keys())[0]
+
+    def get_custom_parser_class(self, uid='custom_mozilla', source_url=None, local_path='tests/res/effective_tld_names_custom.dat.txt'):
+        parser_class = type('CustomMozillaTLDSourceParser', (BaseMozillaTLDSourceParser,), {
+            'uid': uid,
+            'source_url': source_url,
+            'local_path': local_path,
+        })
+        return parser_class
+
+    @log_info
+    def test_0_tld_names_loaded(self):
+        u'Test if tld names are loaded.'
+        get_fld(u'http://www.google.co.uk')
+        from ..utils import tld_names
+        res = (len(tld_names) > 0)
+        self.assertTrue(res)
+        return res
+
+    @internet_available_only
+    @log_info
+    def test_1_update_tld_names(self):
+        u'Test updating the tld names (re-fetch mozilla source).'
+        res = update_tld_names(fail_silently=False)
+        self.assertTrue(res)
+        return res
+
+    @log_info
+    def test_2_fld_good_patterns_pass(self):
+        u'Test good URL patterns.'
+        res = []
+        for data in self.good_patterns:
+            _res = get_fld(data[u'url'], **data[u'kwargs'])
+            self.assertEqual(_res, data[u'fld'])
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_3_fld_bad_patterns_pass(self):
+        u'Test bad URL patterns.'
+        res = []
+        for (url, params) in self.bad_patterns.items():
+            _res = get_fld(url, fail_silently=True)
+            self.assertEqual(_res, None)
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_4_override_settings(self):
+        u'Testing settings override.'
+
+        def override_settings():
+            u'Override settings.'
+            return get_setting(u'DEBUG')
+        self.assertEqual(defaults.DEBUG, override_settings())
+        set_setting(u'DEBUG', True)
+        self.assertEqual(True, override_settings())
+        return override_settings()
+
+    @log_info
+    def test_5_tld_good_patterns_pass_parsed_object(self):
+        u'Test good URL patterns.'
+        res = []
+        for data in self.good_patterns:
+            kwargs = copy.copy(data[u'kwargs'])
+            kwargs.update({
+                u'as_object': True,
+            })
+            _res = get_tld(data[u'url'], **kwargs)
+            self.assertEqual(_res.tld, data[u'tld'])
+            self.assertEqual(_res.subdomain, data[u'subdomain'])
+            self.assertEqual(_res.domain, data[u'domain'])
+            self.assertEqual(_res.suffix, data[u'suffix'])
+            self.assertEqual(_res.fld, data[u'fld'])
+            self.assertEqual(unicode(_res).encode(u'utf8'),
+                             data[u'tld'].encode(u'utf8'))
+            self.assertEqual(_res.__dict__, {
+                u'tld': _res.tld,
+                u'domain': _res.domain,
+                u'subdomain': _res.subdomain,
+                u'fld': _res.fld,
+                u'parsed_url': _res.parsed_url,
+            })
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_6_override_full_names_path(self):
+        default = project_dir(u'dummy.txt')
+        override_base = u'/tmp/test'
+        set_setting(u'NAMES_LOCAL_PATH_PARENT', override_base)
+        modified = project_dir(u'dummy.txt')
+        self.assertNotEqual(default, modified)
+        self.assertEqual(modified, abspath(u'/tmp/test/dummy.txt'))
+
+    @log_info
+    def test_7_public_private(self):
+        res = get_fld(u'http://silly.cc.ua',
+                      fail_silently=True, search_private=False)
+        self.assertEqual(res, None)
+        res = get_fld(u'http://silly.cc.ua',
+                      fail_silently=True, search_private=True)
+        self.assertEqual(res, u'silly.cc.ua')
+        res = get_fld(u'mercy.compute.amazonaws.com',
+                      fail_silently=True, search_private=False, fix_protocol=True)
+        self.assertEqual(res, None)
+        res = get_fld(u'http://whatever.com',
+                      fail_silently=True, search_public=False)
+        self.assertEqual(res, None)
+
+    @log_info
+    def test_8_fld_bad_patterns_exceptions(self):
+        u'Test exceptions.'
+        res = []
+        for (url, params) in self.bad_patterns.items():
+            kwargs = (params[u'kwargs'] if (u'kwargs' in params) else {
+
+            })
+            kwargs.update({
+                u'fail_silently': False,
+            })
+            with self.assertRaises(params[u'exception']):
+                _res = get_fld(url, **kwargs)
+                res.append(_res)
+        return res
+
+    @log_info
+    def test_9_tld_good_patterns_pass(self):
+        u'Test `get_tld` good URL patterns.'
+        res = []
+        for data in self.good_patterns:
+            _res = get_tld(data[u'url'], **data[u'kwargs'])
+            self.assertEqual(_res, data[u'tld'])
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_10_tld_bad_patterns_pass(self):
+        u'Test `get_tld` bad URL patterns.'
+        res = []
+        for (url, params) in self.bad_patterns.items():
+            _res = get_tld(url, fail_silently=True)
+            self.assertEqual(_res, None)
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_11_parse_tld_good_patterns(self):
+        u'Test `parse_tld` good URL patterns.'
+        res = []
+        for data in self.good_patterns:
+            _res = parse_tld(data[u'url'], **data[u'kwargs'])
+            self.assertEqual(
+                _res, (data[u'tld'], data[u'domain'], data[u'subdomain']))
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_12_is_tld_good_patterns(self):
+        u'Test `is_tld` good URL patterns.'
+        for data in self.good_patterns:
+            self.assertTrue(is_tld(data[u'tld']))
+
+    @log_info
+    def test_13_is_tld_bad_patterns(self):
+        u'Test `is_tld` bad URL patterns.'
+        for _tld in self.invalid_tlds:
+            self.assertFalse(is_tld(_tld))
+
+    @log_info
+    def test_14_fail_update_tld_names(self):
+        u'Test fail `update_tld_names`.'
+        parser_class = self.get_custom_parser_class(
+            uid='custom_mozilla_2', source_url='i-do-not-exist')
+        with self.assertRaises(TldIOError):
+            update_tld_names(fail_silently=False, parser_uid=parser_class.uid)
+        self.assertFalse(update_tld_names(
+            fail_silently=True, parser_uid=parser_class.uid))
+
+    @log_info
+    def test_15_fail_get_fld_wrong_kwargs(self):
+        u'Test fail `get_fld` with wrong kwargs.'
+        with self.assertRaises(TldImproperlyConfigured):
+            get_fld(self.good_url, as_object=True)
+
+    @log_info
+    def test_16_fail_parse_tld(self):
+        u'Test fail `parse_tld`.\n\n        Assert raise TldIOError on wrong `NAMES_SOURCE_URL` for `parse_tld`.\n        '
+        parser_class = self.get_custom_parser_class(
+            source_url='i-do-not-exist')
+        parsed_tld = parse_tld(
+            self.bad_url, fail_silently=False, parser_class=parser_class)
+        self.assertEqual(parsed_tld, (None, None, None))
+
+    @log_info
+    def test_17_get_tld_names_and_reset_tld_names(self):
+        u'Test fail `get_tld_names` and repair using `reset_tld_names`.'
+        tmp_filename = join(gettempdir(), u''.join(
+            [u'{}'.format(self.faker.uuid4()), u'.dat.txt']))
+        parser_class = self.get_custom_parser_class(
+            source_url='i-do-not-exist', local_path=tmp_filename)
+        reset_tld_names()
+        if True:
+            with self.assertRaises(TldIOError):
+                get_tld_names(fail_silently=False, parser_class=parser_class)
+        tmp_filename = join(gettempdir(), u''.join(
+            [u'{}'.format(self.faker.uuid4()), u'.dat.txt']))
+        parser_class_2 = self.get_custom_parser_class(
+            source_url='i-do-not-exist-2', local_path=tmp_filename)
+        reset_tld_names()
+        if True:
+            self.assertIsNone(get_tld_names(
+                fail_silently=True, parser_class=parser_class_2))
+
+    @internet_available_only
+    @log_info
+    def test_18_update_tld_names_cli(self):
+        u'Test the return code of the CLI version of `update_tld_names`.'
+        reset_tld_names()
+        res = update_tld_names_cli()
+        self.assertEqual(res, 0)
+
+    @log_info
+    def test_19_parse_tld_custom_tld_names_good_patterns(self):
+        u'Test `parse_tld` good URL patterns for custom tld names.'
+        res = []
+        for data in self.good_patterns_custom_parser:
+            kwargs = copy.copy(data[u'kwargs'])
+            kwargs.update({
+                u'parser_class': self.get_custom_parser_class(),
+            })
+            _res = parse_tld(data[u'url'], **kwargs)
+            self.assertEqual(
+                _res, (data[u'tld'], data[u'domain'], data[u'subdomain']))
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_20_tld_custom_tld_names_good_patterns_pass_parsed_object(self):
+        u'Test `get_tld` good URL patterns for custom tld names.'
+        res = []
+        for data in self.good_patterns_custom_parser:
+            kwargs = copy.copy(data[u'kwargs'])
+            kwargs.update({
+                u'as_object': True,
+                u'parser_class': self.get_custom_parser_class(),
+            })
+            _res = get_tld(data[u'url'], **kwargs)
+            self.assertEqual(_res.tld, data[u'tld'])
+            self.assertEqual(_res.subdomain, data[u'subdomain'])
+            self.assertEqual(_res.domain, data[u'domain'])
+            self.assertEqual(_res.suffix, data[u'suffix'])
+            self.assertEqual(_res.fld, data[u'fld'])
+            self.assertEqual(unicode(_res).encode(u'utf8'),
+                             data[u'tld'].encode(u'utf8'))
+            self.assertEqual(_res.__dict__, {
+                u'tld': _res.tld,
+                u'domain': _res.domain,
+                u'subdomain': _res.subdomain,
+                u'fld': _res.fld,
+                u'parsed_url': _res.parsed_url,
+            })
+            res.append(_res)
+        return res
+
+    @log_info
+    def test_21_reset_tld_names_for_custom_parser(self):
+        u'Test `reset_tld_names` for `tld_names_local_path`.'
+        res = []
+        parser_class = self.get_custom_parser_class()
+        for data in self.good_patterns_custom_parser:
+            kwargs = copy.copy(data[u'kwargs'])
+            kwargs.update({
+                u'as_object': True,
+                u'parser_class': self.get_custom_parser_class(),
+            })
+            _res = get_tld(data[u'url'], **kwargs)
+            self.assertEqual(_res.tld, data[u'tld'])
+            self.assertEqual(_res.subdomain, data[u'subdomain'])
+            self.assertEqual(_res.domain, data[u'domain'])
+            self.assertEqual(_res.suffix, data[u'suffix'])
+            self.assertEqual(_res.fld, data[u'fld'])
+            self.assertEqual(unicode(_res).encode(u'utf8'),
+                             data[u'tld'].encode(u'utf8'))
+            self.assertEqual(_res.__dict__, {
+                u'tld': _res.tld,
+                u'domain': _res.domain,
+                u'subdomain': _res.subdomain,
+                u'fld': _res.fld,
+                u'parsed_url': _res.parsed_url,
+            })
+            res.append(_res)
+        tld_names = get_tld_names_container()
+        self.assertIn(parser_class.local_path, tld_names)
+        reset_tld_names(parser_class.local_path)
+        self.assertNotIn(parser_class.local_path, tld_names)
+        return res
+
+    @log_info
+    def test_22_fail_define_custom_parser_class_without_uid(self):
+        u'Test fail define custom parser class without `uid`.'
+
+        class CustomParser(BaseTLDSourceParser):
+            pass
+
+        class AnotherCustomParser(BaseTLDSourceParser):
+            uid = u'another-custom-parser'
+        with self.assertRaises(TldImproperlyConfigured):
+            CustomParser.get_tld_names()
+        with self.assertRaises(NotImplementedError):
+            AnotherCustomParser.get_tld_names()
+
+    @log_info
+    def test_23_len_trie_nodes(self):
+        u'Test len of the trie nodes.'
+        get_tld(u'http://delusionalinsanity.com')
+        tld_names = get_tld_names_container()
+        self.assertGreater(
+            len(tld_names[MozillaTLDSourceParser.local_path]), 0)
+
+    @log_info
+    def test_24_get_tld_names_no_arguments(self):
+        u'Test len of the trie nodes.'
+        tld_names = get_tld_names()
+        self.assertGreater(len(tld_names), 0)
+
+
+if (__name__ == u'__main__'):
+    unittest.main()

Contents/Libraries/Shared/tld/trie.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'Trie', u'TrieNode')
+
+
+class TrieNode(object):
+    u'Class representing a single Trie node.'
+    __slots__ = (u'children', u'exception', u'leaf', u'private')
+
+    def __init__(self):
+        self.children = None
+        self.exception = None
+        self.leaf = False
+        self.private = False
+
+
+class Trie(object):
+    u'An adhoc Trie data structure to store tlds in reverse notation order.'
+
+    def __init__(self):
+        self.root = TrieNode()
+        self.__nodes = 0
+
+    def __len__(self):
+        return self.__nodes
+
+    def add(self, tld, private=False):
+        node = self.root
+        for part in reversed(tld.split(u'.')):
+            if part.startswith(u'!'):
+                node.exception = part[1:]
+                break
+            if (node.children is None):
+                node.children = {
+
+                }
+                child = TrieNode()
+            else:
+                child = node.children.get(part)
+                if (child is None):
+                    child = TrieNode()
+            node.children[part] = child
+            node = child
+        node.leaf = True
+        if private:
+            node.private = True
+        self.__nodes += 1

Contents/Libraries/Shared/tld/utils.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import unicode_literals
+import argparse
+from codecs import open as codecs_open
+from backports.functools_lru_cache import lru_cache
+from os.path import isabs
+import sys
+from typing import Dict, Type, Union, Tuple, List
+try:
+    from urllib.parse import urlsplit, SplitResult
+except ImportError:
+    from six.moves.urllib_parse import urlsplit, SplitResult
+from .base import BaseTLDSourceParser
+from .exceptions import TldBadUrl, TldDomainNotFound, TldImproperlyConfigured, TldIOError
+from .helpers import project_dir
+from .trie import Trie
+from .registry import Registry
+from .result import Result
+__author__ = u'Artur Barseghyan'
+__copyright__ = u'2013-2019 Artur Barseghyan'
+__license__ = u'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
+__all__ = (u'BaseMozillaTLDSourceParser', u'get_fld', u'get_tld', u'get_tld_names', u'get_tld_names_container', u'is_tld', u'MozillaTLDSourceParser', u'parse_tld',
+           u'pop_tld_names_container', u'process_url', u'reset_tld_names', u'Result', u'update_tld_names', u'update_tld_names_cli', u'update_tld_names_container')
+tld_names = {
+
+}
+
+
+def get_tld_names_container():
+    u'Get container of all tld names.\n\n    :return:\n    :rtype dict:\n    '
+    global tld_names
+    return tld_names
+
+
+def update_tld_names_container(tld_names_local_path, trie_obj):
+    u'Update TLD Names container item.\n\n    :param tld_names_local_path:\n    :param trie_obj:\n    :return:\n    '
+    global tld_names
+    tld_names.update({
+        tld_names_local_path: trie_obj,
+    })
+
+
+def pop_tld_names_container(tld_names_local_path):
+    u'Remove TLD names container item.\n\n    :param tld_names_local_path:\n    :return:\n    '
+    global tld_names
+    tld_names.pop(tld_names_local_path, None)
+
+
+@lru_cache(maxsize=128, typed=True)
+def update_tld_names(fail_silently=False, parser_uid=None):
+    u'Update TLD names.\n\n    :param fail_silently:\n    :param parser_uid:\n    :return:\n    '
+    results = []
+    results_append = results.append
+    if parser_uid:
+        parser_cls = Registry.get(parser_uid, None)
+        if (parser_cls and parser_cls.source_url):
+            results_append(parser_cls.update_tld_names(
+                fail_silently=fail_silently))
+    else:
+        for (parser_uid, parser_cls) in Registry.items():
+            if (parser_cls and parser_cls.source_url):
+                results_append(parser_cls.update_tld_names(
+                    fail_silently=fail_silently))
+    return all(results)
+
+
+def update_tld_names_cli():
+    u'CLI wrapper for update_tld_names.\n\n    Since update_tld_names returns True on success, we need to negate the\n    result to match CLI semantics.\n    '
+    parser = argparse.ArgumentParser(description='Update TLD names')
+    parser.add_argument(u'parser_uid', nargs='?', default=None,
+                        help='UID of the parser to update TLD names for.')
+    parser.add_argument(u'--fail-silently', dest='fail_silently',
+                        default=False, action='store_true', help='Fail silently')
+    args = parser.parse_args(sys.argv[1:])
+    parser_uid = args.parser_uid
+    fail_silently = args.fail_silently
+    return int((not update_tld_names(parser_uid=parser_uid, fail_silently=fail_silently)))
+
+
+def get_tld_names(fail_silently=False, retry_count=0, parser_class=None):
+    u'Build the ``tlds`` list if empty. Recursive.\n\n    :param fail_silently: If set to True, no exceptions are raised and None\n        is returned on failure.\n    :param retry_count: If greater than 1, we raise an exception in order\n        to avoid infinite loops.\n    :param parser_class:\n    :type fail_silently: bool\n    :type retry_count: int\n    :type parser_class: BaseTLDSourceParser\n    :return: List of TLD names\n    :rtype: obj:`tld.utils.Trie`\n    '
+    if (not parser_class):
+        parser_class = MozillaTLDSourceParser
+    return parser_class.get_tld_names(fail_silently=fail_silently, retry_count=retry_count)
+
+
+class BaseMozillaTLDSourceParser(BaseTLDSourceParser):
+
+    @classmethod
+    def get_tld_names(cls, fail_silently=False, retry_count=0):
+        u'Parse.\n\n        :param fail_silently:\n        :param retry_count:\n        :return:\n        '
+        if (retry_count > 1):
+            if fail_silently:
+                return None
+            else:
+                raise TldIOError
+        global tld_names
+        _tld_names = tld_names
+        if ((cls.local_path in _tld_names) and (_tld_names[cls.local_path] is not None)):
+            return _tld_names
+        local_file = None
+        try:
+            if isabs(cls.local_path):
+                local_path = cls.local_path
+            else:
+                local_path = project_dir(cls.local_path)
+            local_file = codecs_open(local_path, u'r', encoding='utf8')
+            trie = Trie()
+            trie_add = trie.add
+            private_section = False
+            for line in local_file:
+                if (u'===BEGIN PRIVATE DOMAINS===' in line):
+                    private_section = True
+                if (u'// xn--' in line):
+                    line = line.split()[1]
+                if (line[0] in (u'/', u'\n')):
+                    continue
+                trie_add(u''.join([u'{}'.format(line.strip())]),
+                         private=private_section)
+            update_tld_names_container(cls.local_path, trie)
+            local_file.close()
+        except IOError as err:
+            cls.update_tld_names(fail_silently=fail_silently)
+            retry_count += 1
+            return cls.get_tld_names(fail_silently=fail_silently, retry_count=retry_count)
+        except Exception as err:
+            if fail_silently:
+                return None
+            else:
+                raise err
+        finally:
+            try:
+                local_file.close()
+            except Exception:
+                pass
+        return _tld_names
+
+
+class MozillaTLDSourceParser(BaseMozillaTLDSourceParser):
+    u'Mozilla TLD source.'
+    uid = u'mozilla'
+    source_url = u'http://mxr.mozilla.org/mozilla/source/netwerk/dns/src/effective_tld_names.dat?raw=1'
+    local_path = u'res/effective_tld_names.dat.txt'
+
+
+def process_url(url, fail_silently=False, fix_protocol=False, search_public=True, search_private=True, parser_class=MozillaTLDSourceParser):
+    u'Process URL.\n\n    :param parser_class:\n    :param url:\n    :param fail_silently:\n    :param fix_protocol:\n    :param search_public:\n    :param search_private:\n    :return:\n    '
+    if (not (search_public or search_private)):
+        raise TldImproperlyConfigured(
+            u'Either `search_public` or `search_private` (or both) shall be set to True.')
+    _tld_names = get_tld_names(
+        fail_silently=fail_silently, parser_class=parser_class)
+    if (not isinstance(url, SplitResult)):
+        url = url.lower()
+        if (fix_protocol and (not url.startswith((u'//', u'http://', u'https://')))):
+            url = u''.join([u'https://', u'{}'.format(url)])
+        parsed_url = urlsplit(url)
+    else:
+        parsed_url = url
+    domain_name = parsed_url.hostname
+    if (not domain_name):
+        if fail_silently:
+            return (None, None, parsed_url)
+        else:
+            raise TldBadUrl(url=url)
+    domain_parts = domain_name.split(u'.')
+    tld_names_local_path = parser_class.local_path
+    node = _tld_names[tld_names_local_path].root
+    current_length = 0
+    tld_length = 0
+    match = None
+    len_domain_parts = len(domain_parts)
+    for i in reversed(range(len_domain_parts)):
+        part = domain_parts[i]
+        if (node.children is None):
+            break
+        if (part == node.exception):
+            break
+        child = node.children.get(part)
+        if (child is None):
+            child = node.children.get(u'*')
+        if (child is None):
+            break
+        current_length += 1
+        node = child
+        if node.leaf:
+            tld_length = current_length
+            match = node
+    if ((match is None) or (not match.leaf) or ((not search_public) and (not match.private)) or ((not search_private) and match.private)):
+        if fail_silently:
+            return (None, None, parsed_url)
+        else:
+            raise TldDomainNotFound(domain_name=domain_name)
+    if (len_domain_parts == tld_length):
+        non_zero_i = (- 1)
+    else:
+        non_zero_i = max(1, (len_domain_parts - tld_length))
+    return (domain_parts, non_zero_i, parsed_url)
+
+
+def get_fld(url, fail_silently=False, fix_protocol=False, search_public=True, search_private=True, parser_class=MozillaTLDSourceParser, **kwargs):
+    u"Extract the first level domain.\n\n    Extract the top level domain based on the mozilla's effective TLD names\n    dat file. Returns a string. May throw ``TldBadUrl`` or\n    ``TldDomainNotFound`` exceptions if there's bad URL provided or no TLD\n    match found respectively.\n\n    :param url: URL to get top level domain from.\n    :param fail_silently: If set to True, no exceptions are raised and None\n        is returned on failure.\n    :param fix_protocol: If set to True, missing or wrong protocol is\n        ignored (https is appended instead).\n    :param search_public: If set to True, search in public domains.\n    :param search_private: If set to True, search in private domains.\n    :param parser_class:\n    :type url: str\n    :type fail_silently: bool\n    :type fix_protocol: bool\n    :type search_public: bool\n    :type search_private: bool\n    :return: String with top level domain (if ``as_object`` argument\n        is set to False) or a ``tld.utils.Result`` object (if ``as_object``\n        argument is set to True); returns None on failure.\n    :rtype: str\n    "
+    if (u'as_object' in kwargs):
+        raise TldImproperlyConfigured(
+            u'`as_object` argument is deprecated for `get_fld`. Use `get_tld` instead.')
+    (domain_parts, non_zero_i, parsed_url) = process_url(url=url, fail_silently=fail_silently,
+                                                         fix_protocol=fix_protocol, search_public=search_public, search_private=search_private, parser_class=parser_class)
+    if (domain_parts is None):
+        return None
+    if (non_zero_i < 0):
+        return parsed_url.hostname
+    return u'.'.join(domain_parts[(non_zero_i - 1):])
+
+
+def get_tld(url, fail_silently=False, as_object=False, fix_protocol=False, search_public=True, search_private=True, parser_class=MozillaTLDSourceParser):
+    u"Extract the top level domain.\n\n    Extract the top level domain based on the mozilla's effective TLD names\n    dat file. Returns a string. May throw ``TldBadUrl`` or\n    ``TldDomainNotFound`` exceptions if there's bad URL provided or no TLD\n    match found respectively.\n\n    :param url: URL to get top level domain from.\n    :param fail_silently: If set to True, no exceptions are raised and None\n        is returned on failure.\n    :param as_object: If set to True, ``tld.utils.Result`` object is returned,\n        ``domain``, ``suffix`` and ``tld`` properties.\n    :param fix_protocol: If set to True, missing or wrong protocol is\n        ignored (https is appended instead).\n    :param search_public: If set to True, search in public domains.\n    :param search_private: If set to True, search in private domains.\n    :param parser_class:\n    :type url: str\n    :type fail_silently: bool\n    :type as_object: bool\n    :type fix_protocol: bool\n    :type search_public: bool\n    :type search_private: bool\n    :return: String with top level domain (if ``as_object`` argument\n        is set to False) or a ``tld.utils.Result`` object (if ``as_object``\n        argument is set to True); returns None on failure.\n    :rtype: str\n    "
+    (domain_parts, non_zero_i, parsed_url) = process_url(url=url, fail_silently=fail_silently,
+                                                         fix_protocol=fix_protocol, search_public=search_public, search_private=search_private, parser_class=parser_class)
+    if (domain_parts is None):
+        return None
+    if (not as_object):
+        if (non_zero_i < 0):
+            return parsed_url.hostname
+        return u'.'.join(domain_parts[non_zero_i:])
+    if (non_zero_i < 0):
+        subdomain = u''
+        domain = u''
+        _tld = parsed_url.hostname
+    else:
+        subdomain = u'.'.join(domain_parts[:(non_zero_i - 1)])
+        domain = u'.'.join(domain_parts[(non_zero_i - 1):non_zero_i])
+        _tld = u'.'.join(domain_parts[non_zero_i:])
+    return Result(subdomain=subdomain, domain=domain, tld=_tld, parsed_url=parsed_url)
+
+
+def parse_tld(url, fail_silently=False, fix_protocol=False, search_public=True, search_private=True, parser_class=MozillaTLDSourceParser):
+    u'Parse TLD into parts.\n\n    :param url:\n    :param fail_silently:\n    :param fix_protocol:\n    :param search_public:\n    :param search_private:\n    :param parser_class:\n    :return:\n    :rtype: tuple\n    '
+    try:
+        obj = get_tld(url, fail_silently=fail_silently, as_object=True, fix_protocol=fix_protocol,
+                      search_public=search_public, search_private=search_private, parser_class=parser_class)
+        _tld = obj.tld
+        domain = obj.domain
+        subdomain = obj.subdomain
+    except (TldBadUrl, TldDomainNotFound, TldImproperlyConfigured, TldIOError):
+        _tld = None
+        domain = None
+        subdomain = None
+    return (_tld, domain, subdomain)
+
+
+def is_tld(value, search_public=True, search_private=True, parser_class=MozillaTLDSourceParser):
+    u'Check if given URL is tld.\n\n    :param value: URL to get top level domain from.\n    :param search_public: If set to True, search in public domains.\n    :param search_private: If set to True, search in private domains.\n    :param parser_class:\n    :type value: str\n    :type search_public: bool\n    :type search_private: bool\n    :return:\n    :rtype: bool\n    '
+    _tld = get_tld(url=value, fail_silently=True, fix_protocol=True,
+                   search_public=search_public, search_private=search_private, parser_class=parser_class)
+    return (value == _tld)
+
+
+def reset_tld_names(tld_names_local_path=None):
+    u'Reset the ``tld_names`` to empty value.\n\n    If ``tld_names_local_path`` is given, removes specified\n    entry from ``tld_names`` instead.\n\n    :param tld_names_local_path:\n    :type tld_names_local_path: str\n    :return:\n    '
+    if tld_names_local_path:
+        pop_tld_names_container(tld_names_local_path)
+    else:
+        global tld_names
+        tld_names = {
+
+        }

Contents/advanced_settings.json.template

@@ -22,7 +22,7 @@ Don't expect support if you mess this up.
     // when the find better subtitles task finds a subtitle for an item that has an active extracted embedded subtitle
     // set, what should be the minimum score that subtitle has to have in order to be considered better?
     "find_better_as_extracted_tv_score": 352,
-    "find_better_as_extracted_movie_score": 82,
+    "find_better_as_extracted_movie_score": 112,
 
     // SZ can use mediainfo if present to detect titles/forced state of MP4 MOV_TEXT, because the PMS currently doesn't
     // set the title attribute
@@ -85,9 +85,12 @@ Don't expect support if you mess this up.
         "sonarr": {
             // don't verify HTTPS certificates? Set to True for self-signed certificates
             "ssl_no_verify": false,
+            // custom path to certificate pem file
+            "pem_file": None,
         },
         "radarr": {
             "ssl_no_verify": false,
+            "pem_file": None,
         },
     }
 }

README.md

@@ -1,7 +1,7 @@
 # Sub-Zero for Plex
 [![](https://img.shields.io/github/release/pannal/Sub-Zero.bundle.svg?style=flat&label=stable)](https://github.com/pannal/Sub-Zero.bundle/releases/latest) 
 [![master](https://img.shields.io/badge/master-stable-green.svg?maxAge=2592000)]()
-[![Maintenance](https://img.shields.io/maintenance/yes/2019.svg)]()
+[![Maintenance](https://img.shields.io/maintenance/yes/2020.svg)]()
 [![Slack Status](https://szslack.fragstore.net/badge.svg)](https://szslack.fragstore.net)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fpannal%2FSub-Zero.bundle.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fpannal%2FSub-Zero.bundle?ref=badge_shield)
 
@@ -94,26 +94,43 @@ the.vbm, mmgoodnow, Vertig0ne, thliu78, tattoomees, ostman, count_confucius, ehe
 
 ## Changelog
 
-
-2.6.5.3183
+2.6.5.3247
 
 subscene, addic7ed
 - either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
 
 Changelog
-- core: don't fall back to default providers if none enabled
-- core: don't process any further if stream info is missing
-- core: support using mediainfo for retrieving MP4 MOV_TEXT subtitle stream titles (PMS bug)
-- core: fix embedded subtitle extraction in some cases (#681, #680)
-- core: scanning: add additional INFO logging for undetected languages
-- core: bazarr-backport: remove existing subtitle file, to support MergerFS
-- core: bazarr-backport: generic 10 minute throttling if uncaught exception occurs
-- providers: addic7ed: fix recaptcha solving; fix show ID retrieval (#681)
-- providers: addic7ed: add timeout on authentication error
-- providers: addic7ed: fix shows with dots in them (Mayans M.C.)
-- providers: addic7ed: fix detection of completed subtitle for non-english users (#686)
-- providers: addic7ed: add more timeouts in the login process
-- providers: argenteam: bazarr-backport: use new url; fixes
+core: fix for tv.plex.agents.movie not populating its media types
+core: tasks: findBetterSubtitles: increase minimum score for better subtitles for movies with extracted embedded subs from 82 to 112
+
+
+2.6.5.3241
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: add support for new Plex Movie agent
+- core: remove py3 compat breaking unnecessary change
+- core: skip drawing tags for SRT
+- core: advanced_settings: refiners: drone: add custom pem_file support; fixes #735
+- providers: core: set DownloadLimitPerDayExceeded timeout to 4 hours (was one day);
+- providers: addic7ed: limit downloads per day; add vip setting
+- providers: addic7ed: properly compare last_dl, add last_reset tracking info to log #723
+- providers: addic7ed: properly implement limits
+- submod: HI: remove more music tags
+- submod: common CM_punctuation_space2: detect AND don't try changing domain/url/host when fixing punctuation
+
+
+2.6.5.3223
+
+subscene, addic7ed
+- either of those providers might impose a reCAPTCHA verification. In order to use those providers, please create an account at an AntiCaptcha service ([anti-captcha.com](http://getcaptchasolution.com/kkvviom7nh) or [deathbycaptcha.com](http://deathbycaptcha.com)), add funds, then supply your credentials/apikey in the configuration
+
+Changelog
+- core: scoring: reorder subtitles based on second non-hash-score if main hash score is the same; morpheus65535/bazarr#821
+- providers: bsplayer: verify hash; clean up
+
 
 [older changes](CHANGELOG.md)