update version to 2.5.3.2414

Resized images

CHANGELOG.md

@@ -1,5 +1,33 @@
 
 
+2.5.0.2287
+
+- core: reduce main icon size
+- core: fix usage on NVIDIA SHIELD (hopefully, please report back), #441
+- core: add scandir fallback to listdir in case of badly configured locale in environment, #441, #440
+- core: get subtitles from archive: don't assume an episode match
+- core: get subtitles from archive: don't assume any attributes in guess
+- core: improve release group detection for drone/filebot/file_info refiners
+- core: fix language detection for embedded subtitle streams
+- core: support extraction of embedded mov_text subtitles in mp4 video files
+- refiners: drone: add http:// to url if not given
+- providers: opensubtitles: retry/reinitialize request when encountering ResponseNotReady
+- config: clarify subscene being only enabled for TV series by default
+- menu: when encountering permission errors when scanning media files, warn in the menu about them
+- submod: common: don't break -- addic7ed --
+- submod: common: remove lines that consist only of dash, underscore
+- submod: OCR: fix Ls = Is
+- submod: OCR: fix bad HI colons (ANNOUNCER; instead of ANNOUNCER:)
+- submod: common: fix lines consisting only of bad music symbols (*#¶ = ♪)
+- submod: HI: remove music-symbol-only-lines
+- submod: HI: be less aggressive about lines ending with a colon; please re-apply all your mods via advanced menu
+- submod: OCR: fix it'sjust, isn'tjust, Iam, Ican
+
+
+2.5.0.2247
+- fix ignoring by-hash-matched episodes
+
+
 2.5.0.2241
 
 - fix issue when removing crap from filenames to not accidentally remove release group #436

Contents/Code/__init__.py

@@ -114,32 +114,41 @@ def update_local_media(metadata, media, media_type="movies"):
             pass
 
 
-def agent_extract_embedded(videos):
+def agent_extract_embedded(video_part_map):
     try:
         subtitle_storage = get_subtitle_storage()
 
-        for video in videos:
-            item = video["item"]
-            stored_subs = subtitle_storage.load_or_new(item)
+        to_extract = []
+        item_count = 0
 
-            for part in get_all_parts(item):
+        for scanned_video, part_info in video_part_map.iteritems():
+            plexapi_item = scanned_video.plexapi_metadata["item"]
+            stored_subs = subtitle_storage.load_or_new(plexapi_item)
+
+            for plexapi_part in get_all_parts(plexapi_item):
+                item_count = item_count + 1
                 for requested_language in config.lang_list:
-                    embedded_subs = stored_subs.get_by_provider(part.id, requested_language, "embedded")
-                    current = stored_subs.get_any(part.id, requested_language)
+                    embedded_subs = stored_subs.get_by_provider(plexapi_part.id, requested_language, "embedded")
+                    current = stored_subs.get_any(plexapi_part.id, requested_language)
                     if not embedded_subs:
-                        stream_data = get_embedded_subtitle_streams(part, requested_language=requested_language,
+                        stream_data = get_embedded_subtitle_streams(plexapi_part, requested_language=requested_language,
                                                                     get_forced=config.forced_only)
 
                         if stream_data:
                             stream = stream_data[0]["stream"]
 
-                            extract_embedded_sub(rating_key=item.rating_key, part_id=part.id,
-                                                 stream_index=str(stream.index),
-                                                 language=str(requested_language), with_mods=True, refresh=False,
-                                                 set_current=not current)
+                            to_extract.append(({scanned_video: part_info}, plexapi_part, str(stream.index),
+                                               str(requested_language), not current))
+
+                            if not cast_bool(Prefs["subtitles.search_after_autoextract"]):
+                                scanned_video.subtitle_languages.update({requested_language})
                     else:
                         Log.Debug("Skipping embedded subtitle extraction for %s, already got %r from %s",
-                                  item.rating_key, requested_language, embedded_subs[0].id)
+                                  plexapi_item.rating_key, requested_language, embedded_subs[0].id)
+        if to_extract:
+            Log.Info("Triggering extraction of %d embedded subtitles of %d items", len(to_extract), item_count)
+            Thread.Create(multi_extract_embedded, stream_list=to_extract, refresh=True, with_mods=True,
+                          single_thread=not config.advanced.auto_extract_multithread)
     except:
         Log.Error("Something went wrong when auto-extracting subtitles, continuing: %s", traceback.format_exc())
 
@@ -217,7 +226,10 @@ class SubZeroAgent(object):
 
             # auto extract embedded
             if config.embedded_auto_extract:
-                agent_extract_embedded(videos)
+                if config.plex_transcoder:
+                    agent_extract_embedded(scanned_video_part_map)
+                else:
+                    Log.Warning("Plex Transcoder not found, can't auto extract")
 
             # clear missing subtitles menu data
             if not scheduler.is_task_running("MissingSubtitles"):

Contents/Code/interface/item_details.py

@@ -1,7 +1,5 @@
 # coding=utf-8
 import os
-import subprocess
-import traceback
 
 from subzero.language import Language
 
@@ -12,7 +10,7 @@ from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_
 from refresh_item import RefreshItem
 from subzero.constants import PREFIX
 from support.config import config, TEXT_SUBTITLE_EXTS
-from support.helpers import timestamp, df, get_language, display_language, quote_args, get_language_from_stream
+from support.helpers import timestamp, df, get_language, display_language, get_language_from_stream
 from support.items import get_item_kind_from_rating_key, get_item, get_current_sub, get_item_title, save_stored_sub
 from support.plex_media import get_plex_metadata, get_part, get_embedded_subtitle_streams
 from support.scanning import scan_videos
@@ -24,7 +22,8 @@ from support.storage import get_subtitle_storage
 
 @route(PREFIX + '/item/{rating_key}/actions')
 @debounce
-def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None, header=None):
+def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None, header=None,
+                    message=None):
     """
     displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
     :param rating_key:
@@ -42,7 +41,7 @@ def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, ra
 
     timeout = 30
 
-    oc = SubFolderObjectContainer(title2=title, replace_parent=True, header=header)
+    oc = SubFolderObjectContainer(title2=title, replace_parent=True, header=header, message=message)
 
     if not item:
         oc.add(DirectoryObject(
@@ -251,7 +250,7 @@ def ListStoredSubsForItemMenu(**kwargs):
     all_subs = stored_subs.get_all(part_id, language)
     kwargs.pop("randomize")
 
-    for key, subtitle in sorted(filter(lambda x: x[0] != "current", all_subs.items()),
+    for key, subtitle in sorted(filter(lambda x: x[0] not in ("current", "blacklist"), all_subs.items()),
                                 key=lambda x: x[1].date_added, reverse=True):
         is_current = key == all_subs["current"]
 
@@ -593,16 +592,16 @@ def ListEmbeddedSubsForItemMenu(**kwargs):
             language = stream_data["language"]
             is_unknown = stream_data["is_unknown"]
             stream = stream_data["stream"]
+            is_forced = stream_data["is_forced"]
 
             if language:
-                forced = stream.forced
                 oc.add(DirectoryObject(
                     key=Callback(TriggerExtractEmbeddedSubForItemMenu, randomize=timestamp(),
                                  stream_index=str(stream.index), language=language, with_mods=True, **kwargs),
                     title=u"Extract stream %s, "
                           u"%s%s%s%s with default mods" % (stream.index, display_language(language),
                                                            " (unknown)" if is_unknown else "",
-                                                           " (forced)" if forced else "",
+                                                           " (forced)" if is_forced else "",
                                                            " (\"%s\")" % stream.title if stream.title else ""),
                 ))
                 oc.add(DirectoryObject(
@@ -610,7 +609,7 @@ def ListEmbeddedSubsForItemMenu(**kwargs):
                                  stream_index=str(stream.index), language=language, **kwargs),
                     title=u"Extract stream %s, %s%s%s%s" % (stream.index, display_language(language),
                                                             " (unknown)" if is_unknown else "",
-                                                            " (forced)" if forced else "",
+                                                            " (forced)" if is_forced else "",
                                                             " (\"%s\")" % stream.title if stream.title else ""),
                 ))
     return oc
@@ -634,6 +633,7 @@ def TriggerExtractEmbeddedSubForItemMenu(**kwargs):
     kwargs.pop("language")
     kwargs["title"] = kwargs["item_title"]
     kwargs["header"] = header
+    kwargs["message"] = header
 
     return ItemDetailsMenu(randomize=timestamp(), **kwargs)
 

Contents/Code/interface/menu.py

@@ -58,7 +58,7 @@ def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, displa
 
 @route(PREFIX + '/section/contents', display_items=bool)
 def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None,
-                 previous_rating_key=None, header=None, randomize=None):
+                 previous_rating_key=None, message=None, header=None, randomize=None):
     """
     displays the contents of a section based on whether it has a deeper tree or not (movies->movie (item) list; series->series list)
     :param rating_key:
@@ -72,7 +72,7 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
     title = unicode(title)
     item_title = title
     title = base_title + " > " + title
-    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True, header=header,
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True, header=header, message=message,
                                   view_group="full_details")
 
     current_kind = get_item_kind_from_rating_key(rating_key)
@@ -117,18 +117,19 @@ def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, p
                                  title=title,
                                  previous_item_type=previous_item_type, with_mods=True,
                                  previous_rating_key=previous_rating_key, randomize=timestamp()),
-                    title=u"Extract missing %s embedded subtitles with default mods" % display_language(lang),
+                    title=u"Extract missing %s embedded subtitles" % display_language(lang),
                     summary="Extracts the not yet extracted embedded subtitles of all episodes for the current season "
                             "with all configured default modifications"
                 ))
                 oc.add(DirectoryObject(
                     key=Callback(SeasonExtractEmbedded, rating_key=rating_key, language=lang,
                                  base_title=show.section.title, display_items=display_items, item_title=item_title,
-                                 title=title,
-                                 previous_item_type=previous_item_type, with_mods=False,
+                                 title=title, force=True,
+                                 previous_item_type=previous_item_type, with_mods=True,
                                  previous_rating_key=previous_rating_key, randomize=timestamp()),
-                    title=u"Extract missing %s embedded subtitles" % display_language(lang),
-                    summary="Extracts the not yet extracted embedded subtitles of all episodes for the current season"
+                    title=u"Extract and activate %s embedded subtitles" % display_language(lang),
+                    summary="Extracts embedded subtitles of all episodes for the current season "
+                            "with all configured default modifications"
                 ))
 
         # add refresh
@@ -158,9 +159,10 @@ def SeasonExtractEmbedded(**kwargs):
     with_mods = kwargs.pop("with_mods")
     item_title = kwargs.pop("item_title")
     title = kwargs.pop("title")
+    force = kwargs.pop("force", False)
 
     Thread.Create(season_extract_embedded, **{"rating_key": rating_key, "requested_language": requested_language,
-                                              "with_mods": with_mods})
+                                              "with_mods": with_mods, "force": force})
 
     kwargs["header"] = 'Success'
     kwargs["message"] = u"Extracting of embedded subtitles for %s triggered" % title
@@ -169,7 +171,24 @@ def SeasonExtractEmbedded(**kwargs):
     return MetadataMenu(randomize=timestamp(), title=item_title, **kwargs)
 
 
-def season_extract_embedded(rating_key, requested_language, with_mods=False):
+def multi_extract_embedded(stream_list, refresh=False, with_mods=False, single_thread=True):
+    def execute():
+        for video_part_map, plexapi_part, stream_index, language, set_current in stream_list:
+            plexapi_item = video_part_map.keys()[0].plexapi_metadata["item"]
+
+            extract_embedded_sub(rating_key=plexapi_item.rating_key, part_id=plexapi_part.id,
+                                 plex_item=plexapi_item, part=plexapi_part, scanned_videos=video_part_map,
+                                 stream_index=stream_index, set_current=set_current,
+                                 language=language, with_mods=with_mods, refresh=refresh)
+
+    if single_thread:
+        with Thread.Lock(key="extract_embedded"):
+            execute()
+    else:
+        execute()
+
+
+def season_extract_embedded(rating_key, requested_language, with_mods=False, force=False):
     # get stored subtitle info for item id
     subtitle_storage = get_subtitle_storage()
 
@@ -180,15 +199,19 @@ def season_extract_embedded(rating_key, requested_language, with_mods=False):
                 stored_subs = subtitle_storage.load_or_new(item)
                 for part in get_all_parts(item):
                     embedded_subs = stored_subs.get_by_provider(part.id, requested_language, "embedded")
-                    if not embedded_subs:
+                    current = stored_subs.get_any(part.id, requested_language)
+                    if not embedded_subs or force:
                         stream_data = get_embedded_subtitle_streams(part, requested_language=requested_language,
                                                                     get_forced=config.forced_only)
                         if stream_data:
                             stream = stream_data[0]["stream"]
 
+                            set_current = not current or force
+                            refresh = not current
+
                             extract_embedded_sub(rating_key=item.rating_key, part_id=part.id,
-                                                 stream_index=str(stream.index),
-                                                 language=requested_language, with_mods=with_mods)
+                                                 stream_index=str(stream.index), set_current=set_current,
+                                                 refresh=refresh, language=requested_language, with_mods=with_mods)
     finally:
         subtitle_storage.destroy()
 

Contents/Code/interface/menu_helpers.py

@@ -8,7 +8,7 @@ import os
 from func import enable_channel_wrapper
 from subzero.language import Language
 from support.items import get_kind, get_item_thumb, get_item, get_item_kind_from_item, refresh_item
-from support.helpers import get_video_display_title, pad_title, display_language, quote_args
+from support.helpers import get_video_display_title, pad_title, display_language, quote_args, is_stream_forced
 from support.ignore import ignore_list
 from support.lib import get_intent
 from support.config import config
@@ -160,24 +160,31 @@ def extract_embedded_sub(**kwargs):
     refresh = kwargs.pop("refresh", True)
     set_current = kwargs.pop("set_current", True)
 
-    plex_item = get_item(rating_key)
+    plex_item = kwargs.pop("plex_item", get_item(rating_key))
     item_type = get_item_kind_from_item(plex_item)
-    part = get_part(plex_item, part_id)
+    part = kwargs.pop("part", get_part(plex_item, part_id))
+    scanned_videos = kwargs.pop("scanned_videos", None)
+
+    any_successful = False
 
     if part:
-        metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item)
-        scanned_parts = scan_videos([metadata], ignore_all=True, skip_hashing=True)
+        if not scanned_videos:
+            metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item)
+            scanned_videos = scan_videos([metadata], ignore_all=True, skip_hashing=True)
+
         for stream in part.streams:
             # subtitle stream
             if str(stream.index) == stream_index:
-                forced = stream.forced
+                is_forced = is_stream_forced(stream)
                 bn = os.path.basename(part.file)
 
                 set_refresh_menu_state(u"Extracting subtitle %s of %s" % (stream_index, bn))
                 Log.Info(u"Extracting stream %s (%s) of %s", stream_index, display_language(language), bn)
 
+                out_codec = stream.codec if stream.codec != "mov_text" else "srt"
+
                 args = [
-                    config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", "srt", "-"
+                    config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", out_codec, "-"
                 ]
                 output = None
                 try:
@@ -194,13 +201,17 @@ def extract_embedded_sub(**kwargs):
                     subtitle.set_encoding("utf-8")
 
                     # fixme: speedup video; only video.name is needed
-                    save_successful = save_subtitles(scanned_parts, {scanned_parts.keys()[0]: [subtitle]}, mode="m",
-                                                     set_current=set_current)
+                    save_successful = save_subtitles(scanned_videos, {scanned_videos.keys()[0]: [subtitle]}, mode="m",
+                                                     set_current=set_current, is_forced=is_forced)
                     set_refresh_menu_state(None)
 
                     if save_successful and refresh:
                         refresh_item(rating_key)
 
+                    any_successful = True
+
+    return any_successful
+
 
 class SZObjectContainer(ObjectContainer):
     def __init__(self, *args, **kwargs):

Contents/Code/interface/sub_mod.py

@@ -20,6 +20,7 @@ def SubtitleModificationsMenu(**kwargs):
     rating_key = kwargs["rating_key"]
     part_id = kwargs["part_id"]
     language = kwargs["language"]
+    lang_instance = Language.fromietf(language)
     current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
     kwargs.pop("randomize")
 
@@ -42,6 +43,9 @@ def SubtitleModificationsMenu(**kwargs):
         if mod.exclusive and identifier in current_mods:
             continue
 
+        if mod.languages and lang_instance not in mod.languages:
+            continue
+
         oc.add(DirectoryObject(
             key=Callback(SubtitleSetMods, mods=identifier, mode="add", randomize=timestamp(), **kwargs),
             title=pad_title(mod.description), summary=mod.long_description or ""
@@ -172,11 +176,11 @@ def SubtitleShiftModMenu(unit=None, **kwargs):
 
     rng = []
     if unit == "h":
-        rng = range(-10, 11)
+        rng = list(reversed(range(-10, 0))) + list(reversed(range(1, 11)))
     elif unit in ("m", "s"):
-        rng = range(-15, 15)
+        rng = list(reversed(range(-15, 0))) + list(reversed(range(1, 16)))
     elif unit == "ms":
-        rng = range(-900, 1000, 100)
+        rng = list(reversed(range(-900, 0, 100))) + list(reversed(range(100, 1000, 100)))
 
     for i in rng:
         if i == 0:

Contents/Code/support/config.py

@@ -116,6 +116,7 @@ class Config(object):
     remove_tags = False
     fix_ocr = False
     fix_common = False
+    reverse_rtl = False
     colors = ""
     chmod = None
     forced_only = False
@@ -188,6 +189,7 @@ class Config(object):
         self.remove_tags = cast_bool(Prefs['subtitles.remove_tags'])
         self.fix_ocr = cast_bool(Prefs['subtitles.fix_ocr'])
         self.fix_common = cast_bool(Prefs['subtitles.fix_common'])
+        self.reverse_rtl = cast_bool(Prefs['subtitles.reverse_rtl'])
         self.colors = Prefs['subtitles.colors'] if Prefs['subtitles.colors'] != "don't change" else None
         self.chmod = self.check_chmod()
         self.exotic_ext = cast_bool(Prefs["subtitles.scan.exotic_ext"])
@@ -576,8 +578,10 @@ class Config(object):
                      'tvsubtitles': cast_bool(Prefs['provider.tvsubtitles.enabled']),
                      'legendastv': cast_bool(Prefs['provider.legendastv.enabled']),
                      'napiprojekt': cast_bool(Prefs['provider.napiprojekt.enabled']),
+                     'hosszupuska': cast_bool(Prefs['provider.hosszupuska.enabled']),
                      'shooter': False,
                      'subscene': cast_bool(Prefs['provider.subscene.enabled']),
+                     'argenteam': cast_bool(Prefs['provider.argenteam.enabled']),
                      'subscenter': False,
                      }
 
@@ -594,7 +598,9 @@ class Config(object):
             providers["legendastv"] = False
             providers["napiprojekt"] = False
             providers["shooter"] = False
+            providers["hosszupuska"] = False
             providers["titlovi"] = False
+            providers["argenteam"] = False
 
         # advanced settings
         if media_type and self.advanced.providers:
@@ -632,7 +638,7 @@ class Config(object):
     def get_provider_settings(self):
         provider_settings = {'addic7ed': {'username': Prefs['provider.addic7ed.username'],
                                           'password': Prefs['provider.addic7ed.password'],
-                                          'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents']),
+                                          'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents1']),
                                           },
                              'opensubtitles': {'username': Prefs['provider.opensubtitles.username'],
                                                'password': Prefs['provider.opensubtitles.password'],
@@ -743,6 +749,8 @@ class Config(object):
             mods.append("common")
         if self.colors:
             mods.append("color(name=%s)" % self.colors)
+        if self.reverse_rtl:
+            mods.append("reverse_rtl")
 
         return mods
 
@@ -786,6 +794,11 @@ class Config(object):
         if os.path.isfile(fn):
             return fn
 
+        # look inside Resources folder as fallback, as well
+        fn = os.path.join(base_path, "Resources", "Plex Transcoder")
+        if os.path.isfile(fn):
+            return fn
+
     def parse_rename_mode(self):
         # fixme: exact_filenames should be determined via callback combined with info about the current video
         # (original_name)
@@ -827,6 +840,10 @@ class Config(object):
                 }
                 self.exact_filenames = True
 
+    @property
+    def text_based_formats(self):
+        return self.advanced.text_subtitle_formats or TEXT_SUBTITLE_EXTS
+
     def init_subliminal_patches(self):
         # configure custom subtitle destination folders for scanning pre-existing subs
         Log.Debug("Patching subliminal ...")

Contents/Code/support/helpers.py

@@ -292,7 +292,7 @@ def notify_executable(exe_info, videos, subtitles, storage):
             prepared_arguments = [arg % prepared_data for arg in arguments]
 
             Log.Debug(u"Calling %s with arguments: %s" % (exe, prepared_arguments))
-            env = os.environ
+            env = dict(os.environ)
             if not mswindows:
                 env_path = {"PATH": os.pathsep.join(
                                         [
@@ -304,6 +304,8 @@ def notify_executable(exe_info, videos, subtitles, storage):
                             }
                 env = dict(os.environ, **env_path)
 
+            env.pop("LD_LIBRARY_PATH", None)
+
             try:
                 output = subprocess.check_output(quote_args([exe] + prepared_arguments),
                                                  stderr=subprocess.STDOUT, shell=True, env=env)
@@ -377,5 +379,14 @@ def display_language(l):
     return l.name if not addons else "%s (%s)" % (l.name, ", ".join(addons))
 
 
+def is_stream_forced(stream):
+    stream_title = getattr(stream, "title", "") or ""
+    forced = getattr(stream, "forced", False)
+    if not forced and stream_title and "forced" in stream_title.strip().lower():
+        forced = True
+
+    return forced
+
+
 class PartUnknownException(Exception):
     pass

Contents/Code/support/plex_media.py

@@ -180,9 +180,10 @@ def get_embedded_subtitle_streams(part, requested_language=None, skip_duplicate_
             language = helpers.get_language_from_stream(stream.language_code)
             is_unknown = False
             found_requested_language = requested_language and requested_language == language
+            is_forced = helpers.is_stream_forced(stream)
 
             if get_forced is not None:
-                if (get_forced and not stream.forced) or (not get_forced and stream.forced):
+                if (get_forced and not is_forced) or (not get_forced and is_forced):
                     continue
 
             if not language and config.treat_und_as_first:
@@ -194,8 +195,9 @@ def get_embedded_subtitle_streams(part, requested_language=None, skip_duplicate_
                 is_unknown = True
                 has_unknown = True
 
-            if not requested_language or found_requested_language:
-                streams.append({"stream": stream, "is_unknown": is_unknown, "language": language})
+            if not requested_language or found_requested_language or has_unknown:
+                streams.append({"stream": stream, "is_unknown": is_unknown, "language": language,
+                                "is_forced": is_forced})
 
                 if found_requested_language:
                     break

Contents/Code/support/scanning.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 import traceback
 import helpers
+from babelfish.exceptions import LanguageError
 
 from support.lib import Plex, get_intent
 from support.plex_media import get_stream_fps
@@ -8,6 +9,7 @@ from support.storage import get_subtitle_storage
 from support.config import config, TEXT_SUBTITLE_EXTS
 
 from subzero.video import parse_video, set_existing_languages
+from subzero.language import language_from_stream
 
 
 def scan_video(pms_video_info, ignore_all=False, hints=None, rating_key=None, providers=None, skip_hashing=False):
@@ -41,39 +43,44 @@ def scan_video(pms_video_info, ignore_all=False, hints=None, rating_key=None, pr
             plexpy_part = part
 
     # embedded subtitles
+    # fixme: skip the whole scanning process if known_embedded == wanted languages?
     if plexpy_part:
-        for stream in plexpy_part.streams:
-            # subtitle stream
-            if stream.stream_type == 3:
-                if (config.forced_only and getattr(stream, "forced")) or \
-                        (not config.forced_only and not getattr(stream, "forced")):
+        if embedded_subtitles:
+            for stream in plexpy_part.streams:
+                # subtitle stream
+                if stream.stream_type == 3:
+                    is_forced = helpers.is_stream_forced(stream)
 
-                    # embedded subtitle
-                    # fixme: tap into external subtitles here instead of scanning for ourselves later?
-                    if not stream.stream_key and stream.codec:
-                        if config.exotic_ext or stream.codec.lower() in TEXT_SUBTITLE_EXTS:
-                            lang = helpers.get_language_from_stream(stream.language_code)
+                    if (config.forced_only and is_forced) or \
+                            (not config.forced_only and not is_forced):
 
-                            # treat unknown language as lang1?
-                            if not lang and config.treat_und_as_first:
-                                lang = list(config.lang_list)[0]
+                        # embedded subtitle
+                        # fixme: tap into external subtitles here instead of scanning for ourselves later?
+                        if stream.codec and getattr(stream, "index", None):
+                            if config.exotic_ext or stream.codec.lower() in config.text_based_formats:
+                                lang = None
+                                try:
+                                    lang = language_from_stream(stream.language_code)
+                                except LanguageError:
+                                    Log.Debug("Couldn't detect embedded subtitle stream language: %s", stream.language_code)
 
-                            if lang:
-                                known_embedded.append(lang.alpha3)
+                                # treat unknown language as lang1?
+                                if not lang and config.treat_und_as_first:
+                                    lang = list(config.lang_list)[0]
+
+                                if lang:
+                                    known_embedded.append(lang.alpha3)
     else:
         Log.Warn("Part %s missing of %s, not able to scan internal streams", plex_part.id, rating_key)
 
+    Log.Debug("Known embedded: %r", known_embedded)
+
     subtitle_storage = get_subtitle_storage()
     stored_subs = subtitle_storage.load(rating_key)
     subtitle_storage.destroy()
 
     try:
         # get basic video info scan (filename)
-        # video = parse_video(plex_part.file, pms_video_info, hints, external_subtitles=external_subtitles,
-        #                     embedded_subtitles=embedded_subtitles, known_embedded=known_embedded,
-        #                     forced_only=config.forced_only, no_refining=no_refining, ignore_all=ignore_all,
-        #                     stored_subs=stored_subs, refiner_settings=config.refiner_settings, providers=providers,
-        #                     skip_hashing=config.low_impact_mode)
         video = parse_video(plex_part.file, hints, skip_hashing=config.low_impact_mode or skip_hashing,
                             providers=providers)
 

Contents/Code/support/storage.py

@@ -32,6 +32,10 @@ def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_ty
         part_id = str(part.id)
         video_id = str(video.id)
         plex_item = get_item(video_id)
+        if not plex_item:
+            Log.Warning("Plex item not found: %s", video_id)
+            continue
+
         metadata = video.plexapi_metadata
         title = get_title_for_video_metadata(metadata)
 
@@ -127,7 +131,7 @@ def save_subtitles_to_file(subtitles, tags=None, forced_tag=None):
     return True
 
 
-def save_subtitles_to_metadata(videos, subtitles):
+def save_subtitles_to_metadata(videos, subtitles, is_forced=False):
     for video, video_subtitles in subtitles.items():
         mediaPart = videos[video]
         for subtitle in video_subtitles:
@@ -139,12 +143,13 @@ def save_subtitles_to_metadata(videos, subtitles):
                 mp = PMSMediaProxy(video.id).get_part(mediaPart.id)
             else:
                 mp = mediaPart
-            mp.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.id] = Proxy.Media(content, ext="srt")
+            pm = Proxy.Media(content, ext="srt", forced="1" if is_forced else None)
+            mp.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.id] = pm
     return True
 
 
 def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_save=False, mods=None,
-                   set_current=True):
+                   set_current=True, is_forced=False):
     """
      
     :param set_current: save the subtitle as the current one
@@ -179,7 +184,7 @@ def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_
         if save_to_fs:
             try:
                 Log.Debug("Using filesystem as subtitle storage")
-                save_subtitles_to_file(downloaded_subtitles)
+                save_subtitles_to_file(downloaded_subtitles, forced_tag=is_forced)
             except OSError:
                 if cast_bool(Prefs["subtitles.save.metadata_fallback"]):
                     meta_fallback = True
@@ -194,7 +199,8 @@ def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_
                 Log.Debug("Using metadata as subtitle storage, because filesystem storage failed")
             else:
                 Log.Debug("Using metadata as subtitle storage")
-            save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles)
+            save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles,
+                                                         is_forced=is_forced)
 
         if not bare_save and save_successful and config.notify_executable:
             notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage)

Contents/Code/support/tasks.py

@@ -385,12 +385,13 @@ class SearchAllRecentlyAddedMissing(Task):
         try:
             for fn in recent_files:
                 stored_subs = subtitle_storage.load(filename=fn)
-                video_id = stored_subs.video_id
                 if not stored_subs:
-                    Log.Debug("Skipping item %s because storage is empty", video_id)
+                    Log.Debug("Skipping item %s because storage is empty", fn)
                     skip_item()
                     continue
 
+                video_id = stored_subs.video_id
+
                 # added_date <= max_search_days?
                 if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
                     Log.Debug("Skipping item %s because it's too old", video_id)
@@ -641,6 +642,8 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
         now = datetime.datetime.now()
         min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
         min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
+        min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
+        min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 82
         overwrite_manually_modified = cast_bool(
             Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
         overwrite_manually_selected = cast_bool(
@@ -666,9 +669,11 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
                 if stored_subs.item_type == "episode":
                     cutoff = self.series_cutoff
                     min_score = min_score_series
+                    min_score_extracted = min_score_extracted_series
                 else:
                     cutoff = self.movies_cutoff
                     min_score = min_score_movies
+                    min_score_extracted = min_score_extracted_movies
 
                 # don't search for better subtitles until at least 30 minutes have passed
                 if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
@@ -735,6 +740,13 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
                             better_visited = 0
                             for sub in subs:
                                 if sub.score > current_score and sub.score > min_score:
+                                    if current.provider_name == "embedded" and sub.score < min_score_extracted:
+                                        Log.Debug(u"%s: Not downloading subtitle for %s, we've got an active extracted "
+                                                  u"embedded sub and the min score %s isn't met (%s).",
+                                                  self.name, video_id, min_score_extracted, sub.score)
+                                        better_visited += 1
+                                        break
+
                                     Log.Debug(u"%s: Better subtitle found for %s, downloading", self.name, video_id)
                                     better_tried_download += 1
                                     ret = self.download_subtitle(sub, video_id, mode="b")

Contents/DefaultPrefs.json

@@ -337,10 +337,10 @@
     "default": "19"
   },
   {
-    "id": "provider.addic7ed.use_random_agents",
+    "id": "provider.addic7ed.use_random_agents1",
     "label": "Addic7ed: Use random user agents",
     "type": "bool",
-    "default": "false"
+    "default": "true"
   },
   {
     "id": "provider.legendastv.enabled",
@@ -378,6 +378,18 @@
     "id": "provider.subscene.enabled",
     "label": "Provider: Enable SubScene (TV shows)",
     "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "provider.hosszupuska.enabled",
+    "label": "Provider: Enable hosszupuskasub.com (Hungarian)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.argenteam.enabled",
+    "label": "Provider: Enable aRGENTeaM (Spanish)",
+    "type": "bool",
     "default": "false"
   },
   {
@@ -392,6 +404,12 @@
     "type": "bool",
     "default": "false"
   },
+  {
+    "id": "subtitles.search_after_autoextract",
+    "label": "After automatic extraction of embedded subtitles, also immediately search for available subtitles?",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "subtitles.scan.embedded",
     "label": "Don't search for subtitles of a language if there are embedded subtitles inside the media file (MKV/MP4)?",
@@ -459,7 +477,7 @@
   },
   {
     "id": "subtitles.fix_common",
-    "label": "Fix common whitespace/punctuation issues in subtitles",
+    "label": "Fix common issues in subtitles",
     "type": "bool",
     "default": "true"
   },
@@ -469,6 +487,12 @@
     "type": "bool",
     "default": "true"
   },
+  {
+    "id": "subtitles.reverse_rtl",
+    "label": "Reverse punctuation in RTL languages (heb)",
+    "type": "bool",
+    "default": "false"
+  },
   {
     "id": "subtitles.colors",
     "label": "Change colors of subtitles to",

Contents/Info.plist

@@ -9,11 +9,11 @@
         <key>CFBundleInfoDictionaryVersion</key>
         <string>6.0</string>
         <key>CFBundleShortVersionString</key>
-        <string>2.5.0</string>
+        <string>2.5.3</string>
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>2.5.0.2287</string>
+        <string>2.5.3.2414</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -32,7 +32,7 @@
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 2.5.0.2287
+Version 2.5.3.2414
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 
@@ -44,7 +44,7 @@ Score info: &lt;a href=&quot;http://v.ht/szscores&quot;&gt;http://v.ht/szscores&
 Plex thread: &lt;a href=&quot;https://forums.plex.tv/discussion/186575&quot;>https://forums.plex.tv/discussion/186575&lt;/a&gt;
 Github: &lt;a href=&quot;https://github.com/pannal/Sub-Zero.bundle&quot;&gt;https://github.com/pannal/Sub-Zero&lt;/a&gt;
 
-panni, 2017
+panni, 2018
 &lt;/div&gt;
 	</string>
     </dict>

Contents/Libraries/Shared/libfilebot/main.py

@@ -23,7 +23,10 @@ def quote_args(seq):
 
 def win32_xattr(fn):
     handler = ADS(fn)
-    return handler.get_stream_content("net.filebot.filename")
+    try:
+        return handler.get_stream_content("net.filebot.filename")
+    except IOError:
+        pass
 
 
 def default_xattr(fn):

Contents/Libraries/Shared/provider_test.sh

@@ -1,5 +1,8 @@
 # addic7ed
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; ProviderPool(providers=['addic7ed'], provider_configs={'addic7ed': {'use_random_agents': True}})['addic7ed'].query('Game of Thrones', 2)"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; from subliminal_patch.core import SZProviderPool; from babelfish import Language; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'episode'}, dry_run=True); provider = SZProviderPool(providers=['addic7ed'], provider_configs={'addic7ed': {'use_random_agents': True}} )['addic7ed']; subtitle = provider.list_subtitles(video, languages=[Language('fra')]); provider.download_subtitle(subtitle[0]);"
+
+# addic7ed download_best
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; from subliminal_patch.core import SZProviderPool, download_best_subtitles; from subliminal_patch.score import compute_score; from babelfish import Language; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'episode'}, dry_run=True); download_best_subtitles([video], pool_class=SZProviderPool, compute_score=compute_score, providers=['addic7ed'], provider_configs={'addic7ed': {'use_random_agents': True}}, languages={Language('fra')} )"
 
 # opensubtitles
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subzero.video import parse_video; SZProviderPool(providers=['opensubtitles'], )['opensubtitles'].list_subtitles(parse_video('FULL_PATH', {}, {'type': 'episode'}), languages=[Language('eng')])"
@@ -16,6 +19,17 @@ python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.get
 # napiprojekt:download
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from subliminal_patch.score import compute_score; from subliminal import download_best_subtitles; from babelfish import Language; from subliminal.core import scan_video; subs = download_best_subtitles([scan_video('FULL_PATH')], languages={Language('eng')}, providers=['napiprojekt'], pool_class=SZProviderPool, compute_score=compute_score); print subs.values()[0][0].is_valid()"
 
+# argenteam:list
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'movie'}, dry_run=True); print SZProviderPool(providers=['argenteam'], )['argenteam'].list_subtitles(video, languages=[Language('spa')])"
+
+# argenteam:download
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool, download_best_subtitles; from subliminal_patch.score import compute_score; from babelfish import Language; from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'movie'}, dry_run=True); download_best_subtitles([video], pool_class=SZProviderPool, compute_score=compute_score, providers=['argenteam'], languages={Language('spa')} )"
+
+# hosszupuska:list
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'episode'}, dry_run=True); print SZProviderPool(providers=['hosszupuska'], )['hosszupuska'].list_subtitles(video, languages=[Language('hun')])"
+
+# hosszupuska:download
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool, download_best_subtitles; from subliminal_patch.score import compute_score; from babelfish import Language; from subzero.video import parse_video; video = parse_video('FILE_INFO', hints={'type': 'episode'}, dry_run=True); download_best_subtitles([video], pool_class=SZProviderPool, compute_score=compute_score, providers=['hosszupuska'], languages={Language('hun')} )"
 
 # shooter:list
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subliminal_patch.core import SZProviderPool; from babelfish import Language; from subliminal.core import scan_video; print SZProviderPool(providers=['shooter'], )['shooter'].list_subtitles(scan_video('FULL_PATH'), languages=[Language('zho')])"
@@ -28,4 +42,4 @@ python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.get
 python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video, refine_video; video = parse_video('FILE_NAME', {'type': 'episode'}, dry_run=True); print refine_video(video)"
 
 # full test with download
-python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; from subliminal_patch.core import SZProviderPool; from babelfish import Language; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; video = parse_video('FILE_NAME', {}, hints={'type': 'episode'}, dry_run=True); subtitle = SZProviderPool(providers=['titlovi'], )['titlovi'].list_subtitles(video, languages=[Language('hrv')]); SZProviderPool(providers=['titlovi'], )['titlovi'].download_subtitle(subtitle[0]);"
+python -c "import logging; logging.basicConfig(level=logging.DEBUG); logging.getLogger('rebulk').setLevel(logging.WARNING); import os; os.environ['U1pfT01EQl9LRVk'] = '789CF30DAC2C8B0AF433F5C9AD34290A712DF30D7135F12D0FB3E502006FDE081E'; import subliminal_patch, subliminal; from subliminal_patch.core import SZProviderPool; from babelfish import Language; subliminal.region.configure('dogpile.cache.memory'); from subzero.video import parse_video; video = parse_video('FILE_NAME', hints={'type': 'episode'}, dry_run=True); subtitle = SZProviderPool(providers=['titlovi'], )['titlovi'].list_subtitles(video, languages=[Language('hrv')]); SZProviderPool(providers=['titlovi'], )['titlovi'].download_subtitle(subtitle[0]);"

Contents/Libraries/Shared/subliminal/core.py

@@ -15,6 +15,7 @@ if not is_windows_special_path:
 else:
     ThreadPoolExecutor = object
 
+
 from datetime import datetime
 import io
 import itertools

Contents/Libraries/Shared/subliminal/providers/addic7ed.py

@@ -231,18 +231,12 @@ class Addic7edProvider(Provider):
 
         # search as last resort
         if not show_id:
-            logger.warning('Series not found in show ids')
+            logger.warning('Series %s not found in show ids', series)
             show_id = self._search_show_id(series)
 
         return show_id
 
-    def query(self, series, season, year=None, country=None):
-        # get the show id
-        show_id = self.get_show_id(series, year, country)
-        if show_id is None:
-            logger.error('No show id found for %r (%r)', series, {'year': year, 'country': country})
-            return []
-
+    def query(self, show_id, series, season, year=None, country=None):
         # get the page of the season of the show
         logger.info('Getting the page of show id %d, season %d', show_id, season)
         r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10)
@@ -288,12 +282,22 @@ class Addic7edProvider(Provider):
         return subtitles
 
     def list_subtitles(self, video, languages):
+        # lookup show_id
         titles = [video.series] + video.alternative_series
+        show_id = None
         for title in titles:
-            subtitles = [s for s in self.query(title, video.season, video.year)
+            show_id = self.get_show_id(title, video.year)
+            if show_id is not None:
+                break
+
+        # query for subtitles with the show_id
+        if show_id is not None:
+            subtitles = [s for s in self.query(show_id, title, video.season, video.year)
                          if s.language in languages and s.episode == video.episode]
             if subtitles:
                 return subtitles
+        else:
+            logger.error('No show id found for %r (%r)', video.series, {'year': video.year})
 
         return []
 

Contents/Libraries/Shared/subliminal/providers/opensubtitles.py

@@ -220,7 +220,7 @@ class OpenSubtitlesProvider(Provider):
         if isinstance(video, Episode):
             query = video.series
             season = video.season
-            episode = min(video.episode) if isinstance(video.episode, list) else video.episode
+            episode = video.episode
         else:
             query = video.title
 

Contents/Libraries/Shared/subliminal/providers/tvsubtitles.py

@@ -163,18 +163,9 @@ class TVsubtitlesProvider(Provider):
 
         return episode_ids
 
-    def query(self, series, season, episode, year=None):
-        # search the show id
-        show_id = self.search_show_id(series, year)
-        if show_id is None:
-            logger.error('No show id found for %r (%r)', series, {'year': year})
-            return []
-
+    def query(self, show_id, series, season, episode, year=None):
         # get the episode ids
         episode_ids = self.get_episode_ids(show_id, season)
-        # Provider doesn't store multi episode information
-        episode = min(episode) if episode and isinstance(episode, list) else episode
-
         if episode not in episode_ids:
             logger.error('Episode %d not found', episode)
             return []
@@ -202,12 +193,22 @@ class TVsubtitlesProvider(Provider):
         return subtitles
 
     def list_subtitles(self, video, languages):
+        # lookup show_id
         titles = [video.series] + video.alternative_series
+        show_id = None
         for title in titles:
-            subtitles = [s for s in self.query(title, video.season, video.episode, video.year)
-                         if s.language in languages]
+            show_id = self.search_show_id(title, video.year)
+            if show_id is not None:
+                break
+
+        # query for subtitles with the show_id
+        if show_id is not None:
+            subtitles = [s for s in self.query(show_id, title, video.season, video.episode, video.year)
+                         if s.language in languages and s.episode == video.episode]
             if subtitles:
                 return subtitles
+        else:
+            logger.error('No show id found for %r (%r)', video.series, {'year': video.year})
 
         return []
 

Contents/Libraries/Shared/subliminal/subtitle.py

@@ -208,8 +208,14 @@ def guess_matches(video, guess, partial=False):
         if video.season and 'season' in guess and guess['season'] == video.season:
             matches.add('season')
         # episode
-        if video.episode and 'episode' in guess and guess['episode'] == video.episode:
-            matches.add('episode')
+        # Currently we only have single-ep support (guessit returns a multi-ep as a list with int values)
+        # Most providers only support single-ep, so make sure it contains only 1 episode
+        # In case of multi-ep, take the lowest episode (subtitles will normally be available on lowest episode number)
+        if video.episode and 'episode' in guess:
+            episode_guess = guess['episode']
+            episode = min(episode_guess) if episode_guess and isinstance(episode_guess, list) else episode_guess
+            if episode == video.episode:
+                matches.add('episode')
         # year
         if video.year and 'year' in guess and guess['year'] == video.year:
             matches.add('year')

Contents/Libraries/Shared/subliminal/video.py

@@ -169,7 +169,13 @@ class Episode(Video):
         if 'title' not in guess or 'episode' not in guess:
             raise ValueError('Insufficient data to process the guess')
 
-        return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'),
+        # Currently we only have single-ep support (guessit returns a multi-ep as a list with int values)
+        # Most providers only support single-ep, so make sure it contains only 1 episode
+        # In case of multi-ep, take the lowest episode (subtitles will normally be available on lowest episode number)
+        episode_guess = guess.get('episode')
+        episode = min(episode_guess) if episode_guess and isinstance(episode_guess, list) else episode_guess
+
+        return cls(name, guess['title'], guess.get('season', 1), episode, title=guess.get('episode_title'),
                    year=guess.get('year'), format=guess.get('format'), original_series='year' not in guess,
                    release_group=guess.get('release_group'), resolution=guess.get('screen_size'),
                    video_codec=guess.get('video_codec'), audio_codec=guess.get('audio_codec'))

Contents/Libraries/Shared/subliminal_patch/converters/hosszupuska.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from babelfish import LanguageReverseConverter, language_converters
+
+
+class HosszupuskaConverter(LanguageReverseConverter):
+    def __init__(self):
+        self.alpha2_converter = language_converters['alpha2']
+        self.from_hosszupuska = {'hu': ('hun', ), 'en': ('eng',)}
+        self.to_hosszupuska = {v: k for k, v in self.from_hosszupuska.items()}
+        self.codes = self.alpha2_converter.codes | set(self.from_hosszupuska.keys())
+
+    def convert(self, alpha3, country=None, script=None):
+        if (alpha3, country) in self.to_hosszupuska:
+            return self.to_hosszupuska[(alpha3, country)]
+        if (alpha3,) in self.to_hosszupuska:
+            return self.to_hosszupuska[(alpha3,)]
+
+        return self.alpha2_converter.convert(alpha3, country, script)
+
+    def reverse(self, hosszupuska):
+        if hosszupuska in self.from_hosszupuska:
+            return self.from_hosszupuska[hosszupuska]
+
+        return self.alpha2_converter.reverse(hosszupuska)

Contents/Libraries/Shared/subliminal_patch/core.py

@@ -131,7 +131,10 @@ class SZProviderPool(ProviderPool):
         :rtype: list of :class:`~subliminal.subtitle.Subtitle` or None
 
         """
-        languages_search_base = self.language_hook(provider)
+        if self.language_hook:
+            languages_search_base = self.language_hook(provider)
+        else:
+            languages_search_base = languages
 
         # check video validity
         if not provider_registry[provider].check(video):
@@ -797,7 +800,7 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non
                 subtitle_path = os.path.splitext(subtitle_path)[0] + (u".%s" % format)
 
             logger.debug(u"Saving %r to %r", subtitle, subtitle_path)
-            content = subtitle.get_modified_content(format=format)
+            content = subtitle.get_modified_content(format=format, debug=debug_mods)
             if content:
                 with open(subtitle_path, 'w') as f:
                     f.write(content)

Contents/Libraries/Shared/subliminal_patch/providers/addic7ed.py

@@ -167,13 +167,8 @@ class Addic7edProvider(_Addic7edProvider):
             soup.decompose()
             soup = None
 
-    def query(self, series, season, year=None, country=None):
+    def query(self, show_id, series, season, year=None, country=None):
         # patch: fix logging
-        # get the show id
-        show_id = self.get_show_id(series, year, country)
-        if show_id is None:
-            logger.info('No show id found for %r (%r)', series, {'year': year, 'country': country})
-            return []
 
         # get the page of the season of the show
         logger.info('Getting the page of show id %d, season %d', show_id, season)

Contents/Libraries/Shared/subliminal_patch/providers/argenteam.py

@@ -0,0 +1,279 @@
+# coding=utf-8
+import logging
+import os
+import io
+import time
+
+from zipfile import ZipFile
+from guessit import guessit
+from requests import Session
+from subliminal import Episode, Movie
+from subliminal.score import get_equivalent_release_groups
+from subliminal.utils import sanitize_release_group, sanitize
+from subliminal_patch.providers import Provider
+from subliminal_patch.subtitle import Subtitle, guess_matches
+from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
+from subzero.language import Language
+
+logger = logging.getLogger(__name__)
+
+
+class ArgenteamSubtitle(Subtitle):
+    provider_name = 'argenteam'
+    hearing_impaired_verifiable = False
+    _release_info = None
+
+    def __init__(self, language, download_link, movie_kind, title, season, episode, year, release, version, source,
+                 video_codec, tvdb_id, imdb_id, asked_for_episode=None, asked_for_release_group=None, *args, **kwargs):
+        super(ArgenteamSubtitle, self).__init__(language, download_link, *args, **kwargs)
+        self.download_link = download_link
+        self.movie_kind = movie_kind
+        self.title = title
+        self.year = year
+        self.season = season
+        self.episode = episode
+        self.release = release
+        self.version = version
+        self.asked_for_release_group = asked_for_release_group
+        self.asked_for_episode = asked_for_episode
+        self.matches = None
+        self.format = source
+        self.video_codec = video_codec
+        self.tvdb_id = tvdb_id
+        self.imdb_id = "tt" + imdb_id if imdb_id else None
+        self.releases = self.release_info
+
+    @property
+    def id(self):
+        return self.download_link
+
+    @property
+    def release_info(self):
+        if self._release_info:
+            return self._release_info
+
+        combine = []
+        for attr in ("format", "version", "video_codec"):
+            value = getattr(self, attr)
+            if value:
+                combine.append(value)
+
+        self._release_info = u".".join(combine) + (u"-"+self.release if self.release else "")
+        return self._release_info
+
+    def __repr__(self):
+        ep_addon = (" S%02dE%02d" % (self.season, self.episode)) if self.episode else ""
+        return '<%s %r [%s]>' % (
+            self.__class__.__name__, u"%s%s%s." % (self.title, " (%s)" % self.year if self.year else "", ep_addon) +
+            self.release_info, self.language)
+
+    def get_matches(self, video):
+        matches = set()
+        # series
+        if isinstance(video, Episode) and self.movie_kind == 'episode':
+            if video.series and (sanitize(self.title) in (
+                     sanitize(name) for name in [video.series] + video.alternative_series)):
+                matches.add('series')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+
+            # tvdb_id
+            if video.tvdb_id and str(self.tvdb_id) == str(video.tvdb_id):
+                matches.add('tvdb_id')
+
+        elif isinstance(video, Movie) and self.movie_kind == 'movie':
+            # title
+            if video.title and (sanitize(self.title) in (
+                     sanitize(name) for name in [video.title] + video.alternative_titles)):
+                matches.add('title')
+
+            # imdb_id
+            if video.imdb_id and self.imdb_id and str(self.imdb_id) == str(video.imdb_id):
+                matches.add('imdb_id')
+
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+        else:
+            logger.info('%r is not a valid movie_kind', self.movie_kind)
+            return matches
+
+        # release_group
+        if video.release_group and self.release:
+            rg = sanitize_release_group(video.release_group)
+            if any(r in sanitize_release_group(self.release) for r in get_equivalent_release_groups(rg)):
+                matches.add('release_group')
+
+                # blatantly assume we've got a matching format if the release group matches
+                # fixme: smart?
+                #matches.add('format')
+
+        # resolution
+        if video.resolution and self.version and str(video.resolution) in self.version.lower():
+            matches.add('resolution')
+        # format
+        if video.format and self.format:
+            formats = [video.format]
+            if video.format == "WEB-DL":
+                formats.append("WEB")
+
+            for fmt in formats:
+                if fmt.lower() in self.format.lower():
+                    matches.add('format')
+                    break
+
+        matches |= guess_matches(video, guessit(self.release_info), partial=True)
+        self.matches = matches
+        return matches
+
+
+class ArgenteamProvider(Provider, ProviderSubtitleArchiveMixin):
+    provider_name = 'argenteam'
+    languages = {Language.fromalpha2(l) for l in ['es']}
+    video_types = (Episode, Movie)
+    API_URL = "http://argenteam.net/api/v1/"
+    subtitle_class = ArgenteamSubtitle
+    hearing_impaired_verifiable = False
+    language_list = list(languages)
+
+    multi_result_throttle = 2  # seconds
+
+    def __init__(self):
+        self.session = None
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers = {'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")}
+
+    def terminate(self):
+        self.session.close()
+
+    def search_ids(self, title, year=None, imdb_id=None, season=None, episode=None, titles=None):
+        """Search movie or episode id from the `title`, `season` and `episode`.
+
+        :param imdb_id: imdb id of the given movie
+        :param titles: all titles of the given series or movie
+        :param year: release year of the given movie
+        :param str title: series of the episode or movie name
+        :param int season: season of the episode.
+        :param int episode: episode number.
+        :return: list of ids
+        :rtype: list
+
+        """
+        # make the search
+        query = title
+        titles = titles or []
+
+        is_episode = False
+        if season and episode:
+            is_episode = True
+            query = '%s S%#02dE%#02d' % (title, season, episode)
+
+        logger.info(u'Searching %s ID for %r', "episode" if is_episode else "movie", query)
+        r = self.session.get(self.API_URL + 'search', params={'q': query}, timeout=10)
+        r.raise_for_status()
+        results = r.json()
+        match_ids = []
+        if results['total'] >= 1:
+            for result in results["results"]:
+                if (result['type'] == "episode" and not is_episode) or (result['type'] == "movie" and is_episode):
+                    continue
+
+                # shortcut in case of matching imdb id
+                if not is_episode and imdb_id and "imdb" in result and "tt%s" % result["imdb"] == str(imdb_id):
+                    logger.debug("Movie matched by IMDB ID %s, taking shortcut", imdb_id)
+                    match_ids = [result['id']]
+                    break
+
+                # advanced title check in case of multiple movie results
+                if results['total'] > 1:
+                    if not is_episode and year:
+                        if result["title"] and not (sanitize(result["title"]) in (u"%s %s" % (sanitize(name), year)
+                                                                                  for name in titles)):
+                            continue
+
+                match_ids.append(result['id'])
+        else:
+            logger.error(u'No episode ID found for %r', query)
+
+        if match_ids:
+            logger.debug(u"Found matching IDs: %s", ", ".join(str(id) for id in match_ids))
+
+        return match_ids
+
+    def query(self, title, video, titles=None):
+        is_episode = isinstance(video, Episode)
+        season = episode = None
+        url = self.API_URL + 'movie'
+        if is_episode:
+            season = video.season
+            episode = video.episode
+            url = self.API_URL + 'episode'
+            argenteam_ids = self.search_ids(title, season=season, episode=episode, titles=titles)
+
+        else:
+            argenteam_ids = self.search_ids(title, year=video.year, imdb_id=video.imdb_id, titles=titles)
+
+        if not argenteam_ids:
+            return []
+
+        language = self.language_list[0]
+        subtitles = []
+        has_multiple_ids = len(argenteam_ids) > 1
+        for aid in argenteam_ids:
+            response = self.session.get(url, params={'id': aid}, timeout=10)
+
+            response.raise_for_status()
+            content = response.json()
+
+            imdb_id = year = None
+            returned_title = title
+            if not is_episode and "info" in content:
+                imdb_id = content["info"].get("imdb")
+                year = content["info"].get("year")
+                returned_title = content["info"].get("title", title)
+
+            for r in content['releases']:
+                for s in r['subtitles']:
+                    sub = ArgenteamSubtitle(language, s['uri'], "episode" if is_episode else "movie", returned_title,
+                                            season, episode, year, r.get('team'), r.get('tags'),
+                                            r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id,
+                                            asked_for_release_group=video.release_group,
+                                            asked_for_episode=episode
+                                            )
+                    subtitles.append(sub)
+
+            if has_multiple_ids:
+                time.sleep(self.multi_result_throttle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        if isinstance(video, Episode):
+            titles = [video.series] + video.alternative_series
+        else:
+            titles = [video.title] + video.alternative_titles
+
+        for title in titles:
+            subs = self.query(title, video, titles=titles)
+            if subs:
+                return subs
+
+            time.sleep(self.multi_result_throttle)
+
+        return []
+
+    def download_subtitle(self, subtitle):
+        # download as a zip
+        logger.info('Downloading subtitle %r', subtitle)
+        r = self.session.get(subtitle.download_link, timeout=10)
+        r.raise_for_status()
+
+        # open the zip
+        with ZipFile(io.BytesIO(r.content)) as zf:
+            subtitle.content = self.get_subtitle_from_archive(subtitle, zf)

Contents/Libraries/Shared/subliminal_patch/providers/hosszupuska.py

@@ -0,0 +1,244 @@
+# coding: utf-8
+
+import io
+import six
+import logging
+import re
+import os
+import time
+
+from babelfish import Language, language_converters
+from requests import Session
+
+from subliminal_patch.providers import Provider
+from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
+from subliminal.providers import ParserBeautifulSoup
+from subliminal_patch.exceptions import ProviderError
+from subliminal.score import get_equivalent_release_groups
+from subliminal_patch.subtitle import Subtitle, guess_matches
+from subliminal.utils import sanitize, sanitize_release_group
+from subliminal.video import Episode
+from zipfile import ZipFile, is_zipfile
+from rarfile import RarFile, is_rarfile
+from subliminal_patch.utils import sanitize, fix_inconsistent_naming as _fix_inconsistent_naming
+from guessit import guessit
+
+
+def fix_inconsistent_naming(title):
+    """Fix titles with inconsistent naming using dictionary and sanitize them.
+
+    :param str title: original title.
+    :return: new title.
+    :rtype: str
+
+    """
+    return _fix_inconsistent_naming(title, {"DC's Legends of Tomorrow": "Legends of Tomorrow",
+                                            "Marvel's Jessica Jones": "Jessica Jones"})
+
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('hosszupuska = subliminal_patch.converters.hosszupuska:HosszupuskaConverter')
+
+
+class HosszupuskaSubtitle(Subtitle):
+    """Hosszupuska Subtitle."""
+    provider_name = 'hosszupuska'
+
+    def __str__(self):
+        subtit = "Subtitle id: " + str(self.subtitle_id) \
+               + " Series: " + self.series \
+               + " Season: " + str(self.season) \
+               + " Episode: " + str(self.episode) \
+               + " Releases: " + str(self.releases)
+        if self.year:
+            subtit = subtit + " Year: " + str(self.year)
+        if six.PY3:
+            return subtit
+        return subtit.encode('utf-8')
+
+    def __init__(self, language, page_link, subtitle_id, series, season, episode, version,
+                 releases, year, asked_for_release_group=None, asked_for_episode=None):
+        super(HosszupuskaSubtitle, self).__init__(language, page_link=page_link)
+        self.subtitle_id = subtitle_id
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.version = version
+        self.releases = releases
+        self.year = year
+        if year:
+            self.year = int(year)
+
+        self.release_info = u", ".join(releases)
+        self.page_link = page_link
+        self.asked_for_release_group = asked_for_release_group
+        self.asked_for_episode = asked_for_episode
+
+    def __repr__(self):
+        ep_addon = (" S%02dE%02d" % (self.season, self.episode)) if self.episode else ""
+        return '<%s %r [%s]>' % (
+            self.__class__.__name__, u"%s%s%s [%s]" % (self.series, " (%s)" % self.year if self.year else "", ep_addon,
+                                                       self.release_info), self.language)
+
+    @property
+    def id(self):
+        return str(self.subtitle_id)
+
+    def get_matches(self, video):
+        matches = set()
+        # series
+        if video.series and sanitize(self.series) == sanitize(video.series):
+            matches.add('series')
+        # season
+        if video.season and self.season == video.season:
+            matches.add('season')
+        # episode
+        if video.episode and self.episode == video.episode:
+            matches.add('episode')
+        # year
+        if ('series' in matches and video.original_series and self.year is None or
+           video.year and video.year == self.year):
+            matches.add('year')
+
+        # release_group
+        if (video.release_group and self.version and
+                any(r in sanitize_release_group(self.version)
+                    for r in get_equivalent_release_groups(sanitize_release_group(video.release_group)))):
+            matches.add('release_group')
+        # resolution
+        if video.resolution and self.version and video.resolution in self.version.lower():
+            matches.add('resolution')
+        # format
+        if video.format and self.version and video.format.lower() in self.version.lower():
+            matches.add('format')
+        # other properties
+        matches |= guess_matches(video, guessit(self.release_info.encode("utf-8")))
+
+        return matches
+
+
+class HosszupuskaProvider(Provider, ProviderSubtitleArchiveMixin):
+    """Hosszupuska Provider."""
+    languages = {Language('hun', 'HU')} | {Language(l) for l in [
+        'hun', 'eng'
+    ]}
+    video_types = (Episode,)
+    server_url = 'http://hosszupuskasub.com/'
+    subtitle_class = HosszupuskaSubtitle
+    hearing_impaired_verifiable = False
+    multi_result_throttle = 2  # seconds
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers = {'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")}
+
+    def terminate(self):
+        self.session.close()
+
+    def get_language(self, text):
+        if text == '1.gif':
+            return Language.fromhosszupuska('hu')
+        if text == '2.gif':
+            return Language.fromhosszupuska('en')
+        return None
+
+    def query(self, series, season, episode, year=None, video=None):
+
+        # Search for s01e03 instead of s1e3
+        seasona = "%02d" % season
+        episodea = "%02d" % episode
+        series = fix_inconsistent_naming(series)
+        seriesa = series.replace(' ', '+').replace('\'', '')
+
+        # get the episode page
+        logger.info('Getting the page for episode %s', episode)
+        url = self.server_url + "sorozatok.php?cim=" + seriesa + "&evad="+str(seasona) + \
+            "&resz="+str(episodea)+"&nyelvtipus=%25&x=24&y=8"
+        logger.info('Url %s', url)
+
+        r = self.session.get(url, timeout=10).content
+
+        i = 0
+        soup = ParserBeautifulSoup(r, ['lxml'])
+
+        table = soup.find_all("table")[9]
+
+        subtitles = []
+        # loop over subtitles rows
+        for row in table.find_all("tr"):
+            i = i + 1
+            if "this.style.backgroundImage='url(css/over2.jpg)" in str(row) and i > 5:
+                datas = row.find_all("td")
+
+                # Currently subliminal not use these params, but maybe later will come in handy
+                # hunagrian_name = re.split('s(\d{1,2})', datas[1].find_all('b')[0].getText())[0]
+                # Translator of subtitle
+                # sub_translator = datas[3].getText()
+                # Posting date of subtitle
+                # sub_date = datas[4].getText()
+
+                sub_year = sub_english_name = sub_version = None
+                # Handle the case when '(' in subtitle
+                if datas[1].getText().count('(') == 2:
+                    sub_english_name = re.split('s(\d{1,2})e(\d{1,2})', datas[1].getText())[3]
+                if datas[1].getText().count('(') == 3:
+                    sub_year = re.findall(r"(?<=\()(\d{4})(?=\))", datas[1].getText().strip())[0]
+                    sub_english_name = re.split('s(\d{1,2})e(\d{1,2})', datas[1].getText().split('(')[0])[0]
+
+                if not sub_english_name:
+                    continue
+
+                sub_season = int((re.findall('s(\d{1,2})', datas[1].find_all('b')[0].getText(), re.VERBOSE)[0])
+                                 .lstrip('0'))
+                sub_episode = int((re.findall('e(\d{1,2})', datas[1].find_all('b')[0].getText(), re.VERBOSE)[0])
+                                  .lstrip('0'))
+
+                if sub_season == season and sub_episode == episode:
+                    sub_language = self.get_language(datas[2].find_all('img')[0]['src'].split('/')[1])
+                    sub_downloadlink = datas[6].find_all('a')[1]['href']
+                    sub_id = sub_downloadlink.split('=')[1].split('.')[0]
+
+                    if datas[1].getText().count('(') == 2:
+                        sub_version = datas[1].getText().split('(')[1].split(')')[0]
+                    if datas[1].getText().count('(') == 3:
+                        sub_version = datas[1].getText().split('(')[2].split(')')[0]
+
+                    # One subtitle can be used for several releases
+                    sub_releases = [s.strip() for s in sub_version.split(',')]
+                    subtitle = self.subtitle_class(sub_language, sub_downloadlink, sub_id, sub_english_name.strip(),
+                                                   sub_season, sub_episode, sub_version, sub_releases, sub_year,
+                                                   asked_for_release_group=video.release_group,
+                                                   asked_for_episode=episode)
+
+                    logger.debug('Found subtitle: %r', subtitle)
+                    subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        titles = [video.series] + video.alternative_series
+
+        for title in titles:
+            subs = self.query(title, video.season, video.episode, video.year, video=video)
+            if subs:
+                return subs
+
+            time.sleep(self.multi_result_throttle)
+
+    def download_subtitle(self, subtitle):
+        r = self.session.get(subtitle.page_link, timeout=10)
+        r.raise_for_status()
+
+        # open the archive
+        archive_stream = io.BytesIO(r.content)
+        if is_rarfile(archive_stream):
+            logger.debug('Archive identified as rar')
+            archive = RarFile(archive_stream)
+        elif is_zipfile(archive_stream):
+            logger.debug('Archive identified as zip')
+            archive = ZipFile(archive_stream)
+        else:
+            raise ProviderError('Unidentified archive type')
+
+        subtitle.content = self.get_subtitle_from_archive(subtitle, archive)

Contents/Libraries/Shared/subliminal_patch/providers/mixins.py

@@ -122,12 +122,13 @@ class ProviderSubtitleArchiveMixin(object):
                                             ("release_group" in subtitle.matches or
                                              "hash" in subtitle.matches)):
 
-                        asked_for_rlsgrp = subtitle.asked_for_release_group.lower()
+                        if subtitle.asked_for_release_group:
+                            asked_for_rlsgrp = subtitle.asked_for_release_group.lower()
 
-                        if asked_for_rlsgrp:
-                            release_group_matches = False
-                            if asked_for_rlsgrp in sub_name_lower:
-                                release_group_matches = True
+                            if asked_for_rlsgrp:
+                                release_group_matches = False
+                                if asked_for_rlsgrp in sub_name_lower:
+                                    release_group_matches = True
 
                     if release_group_matches and format_matches:
                         matching_sub = sub_name

Contents/Libraries/Shared/subliminal_patch/providers/opensubtitles.py

@@ -239,24 +239,32 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
 
         # loop over subtitle items
         for subtitle_item in response['data']:
+            _subtitle_item = subtitle_item
+            if not isinstance(_subtitle_item, dict):
+                _subtitle_item = response["data"][subtitle_item]
+
+                if not isinstance(_subtitle_item, dict):
+                    logger.error("Malformed data returned from API")
+                    continue
+
             # read the item
-            language = Language.fromopensubtitles(subtitle_item['SubLanguageID'])
-            hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
-            page_link = subtitle_item['SubtitlesLink']
-            subtitle_id = int(subtitle_item['IDSubtitleFile'])
-            matched_by = subtitle_item['MatchedBy']
-            movie_kind = subtitle_item['MovieKind']
-            hash = subtitle_item['MovieHash']
-            movie_name = subtitle_item['MovieName']
-            movie_release_name = subtitle_item['MovieReleaseName']
-            movie_year = int(subtitle_item['MovieYear']) if subtitle_item['MovieYear'] else None
-            movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb']
-            movie_fps = subtitle_item.get('MovieFPS')
-            series_season = int(subtitle_item['SeriesSeason']) if subtitle_item['SeriesSeason'] else None
-            series_episode = int(subtitle_item['SeriesEpisode']) if subtitle_item['SeriesEpisode'] else None
-            filename = subtitle_item['SubFileName']
-            encoding = subtitle_item.get('SubEncoding') or None
-            foreign_parts_only = bool(int(subtitle_item.get('SubForeignPartsOnly', 0)))
+            language = Language.fromopensubtitles(_subtitle_item['SubLanguageID'])
+            hearing_impaired = bool(int(_subtitle_item['SubHearingImpaired']))
+            page_link = _subtitle_item['SubtitlesLink']
+            subtitle_id = int(_subtitle_item['IDSubtitleFile'])
+            matched_by = _subtitle_item['MatchedBy']
+            movie_kind = _subtitle_item['MovieKind']
+            hash = _subtitle_item['MovieHash']
+            movie_name = _subtitle_item['MovieName']
+            movie_release_name = _subtitle_item['MovieReleaseName']
+            movie_year = int(_subtitle_item['MovieYear']) if _subtitle_item['MovieYear'] else None
+            movie_imdb_id = 'tt' + _subtitle_item['IDMovieImdb']
+            movie_fps = _subtitle_item.get('MovieFPS')
+            series_season = int(_subtitle_item['SeriesSeason']) if _subtitle_item['SeriesSeason'] else None
+            series_episode = int(_subtitle_item['SeriesEpisode']) if _subtitle_item['SeriesEpisode'] else None
+            filename = _subtitle_item['SubFileName']
+            encoding = _subtitle_item.get('SubEncoding') or None
+            foreign_parts_only = bool(int(_subtitle_item.get('SubForeignPartsOnly', 0)))
 
             # foreign/forced subtitles only wanted
             if only_foreign and not foreign_parts_only:
@@ -266,7 +274,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
             if not only_foreign and foreign_parts_only:
                 continue
 
-            query_parameters = subtitle_item.get("QueryParameters")
+            query_parameters = _subtitle_item.get("QueryParameters")
 
             subtitle = self.subtitle_class(language, hearing_impaired, page_link, subtitle_id, matched_by,
                                            movie_kind,

Contents/Libraries/Shared/subliminal_patch/providers/tvsubtitles.py

@@ -97,13 +97,7 @@ class TVsubtitlesProvider(_TVsubtitlesProvider):
 
         return episode_ids
 
-    def query(self, series, season, episode, year=None):
-        # search the show id
-        show_id = self.search_show_id(series, year)
-        if show_id is None:
-            logger.info('No show id found for %r (%r)', series, {'year': year})
-            return []
-
+    def query(self, show_id, series, season, episode, year=None):
         # get the episode ids
         episode_ids = self.get_episode_ids(show_id, season)
         # Provider doesn't store multi episode information

Contents/Libraries/Shared/subliminal_patch/providers/utils.py

@@ -82,4 +82,702 @@ FIRST_THOUSAND_OR_SO_USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/527 (KHTML, like Gecko, Safari/419.3) Arora/0.6 (Change: )",
+    "Avant Browser/1.2.789rel1 (http://www.avantbrowser.com)",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
+    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0",
+    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
+    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)",
+    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110622 Firefox/6.0a2",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
+    "Mozilla/5.0 (Windows NT 6.1; rv:12.0) Gecko/20120403211507 Firefox/12.0",
+    "Mozilla/5.0 (Windows NT 6.0; rv:14.0) Gecko/20100101 Firefox/14.0.1",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20120427 Firefox/15.0a1",
+    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0",
+    "Mozilla/5.0 (Windows NT 6.2; rv:19.0) Gecko/20121129 Firefox/19.0",
+    "Mozilla/5.0 (Windows NT 6.2; rv:20.0) Gecko/20121202 Firefox/20.0",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Maxthon 2.0)",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre",
+    "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0 )",
+    "Opera/9.80 (Windows NT 5.2; U; en) Presto/2.2.15 Version/10.10",
+    "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10",
+    "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.7.62 Version/11.01",
+    "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
+    "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
+    "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.2b) Gecko/20021001 Phoenix/0.2",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.23) Gecko/20090825 SeaMonkey/1.1.18",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
+    "Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
+    "Mozilla/5.0 (Windows; U; ; en-NZ) AppleWebKit/527 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Beamrise/17.2.0.9 Chrome/17.0.939.0 Safari/535.8",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2869.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 AOL/11.0 AOLBUILD/11.0.1305 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3191.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0",
+    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240",
+    "Mozilla/5.0 (MSIE 9.0; Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14931",
+    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063",
+    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0",
+    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0",
+    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
+    "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
+    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:35.0) Gecko/20100101 Firefox/35.0",
+    "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
+    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0",
+    "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0",
+    "Mozilla/5.0 (compatible; Konqueror/4.5; Windows) KHTML/4.5.4 (like Gecko)",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML like Gecko) Maxthon/4.0.0.2000 Chrome/22.0.1229.79 Safari/537.1",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.6.1000 Chrome/30.0.1599.101 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/5.0.4.3000 Chrome/47.0.2526.73 Safari/537.36",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
+    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
+    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/4.0)",
+    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/5.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; Trident/5.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; WOW64; Trident/5.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/6.0)",
+    "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
+    "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)",
+    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) MxBrowser/4.5.10.7000 Chrome/30.0.1551.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; MATBJS; rv:11.0) like Gecko",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; MALNJS; rv:11.0) like Gecko",
+    "Opera/9.80 (Windows NT 6.1; WOW64) Presto/2.12.388 Version/12.16",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.12 Safari/537.36 OPR/14.0.1116.4",
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.29 Safari/537.36 OPR/15.0.1147.24 (Edition Next)",
+    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.56",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36 OPR/20.0.1387.91",
+    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36 OPR/28.0.1750.40",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36 OPR/31.0.1889.174",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36 OPR/36.0.2130.46",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36 OPR/47.0.2631.55",
+    "Mozilla/5.0 (Windows NT 10.0; rv:45.9) Gecko/20100101 Goanna/3.2 Firefox/45.9 PaleMoon/27.4.0",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.2; es-US ) AppleWebKit/540.0 (KHTML like Gecko) Version/6.0 Safari/8900.00",
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.71 (KHTML like Gecko) WebVideo/1.0.1.10 Version/7.0 Safari/537.71",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20120422 Firefox/12.0 SeaMonkey/2.9",
+    "Mozilla/5.0 (Windows NT 6.0; rv:36.0) Gecko/20100101 Firefox/36.0 SeaMonkey/2.33.1",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.13705.206 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Vivaldi/1.0.94.2 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.90 Safari/537.36 Vivaldi/1.4.589.11",
+    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.91 Safari/537.36 Vivaldi/1.92.917.39",
+    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 YaBrowser/17.3.0.1785 Yowser/2.5 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Camino/2.2.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre Camino/2.2a1pre",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",
+    "Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0) Gecko/20100101 Firefox/5.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0",
+    "Mozilla/4.0 (compatible; MSIE 5.15; Mac_PowerPC)",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/125.4 (KHTML, like Gecko, Safari) OmniWeb/v563.15",
+    "Opera/9.0 (Macintosh; PPC Mac OS X; U; en)",
+    "Opera/9.20 (Macintosh; Intel Mac OS X; U; en)",
+    "Opera/9.64 (Macintosh; PPC Mac OS X; U; en) Presto/2.1.1",
+    "Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61",
+    "Opera/9.80 (Macintosh; Intel Mac OS X 10.4.11; U; en) Presto/2.7.62 Version/11.00",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.2 (KHTML, like Gecko) Safari/85.8",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.8 (KHTML, like Gecko) Safari/419.3",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.31 (KHTML like Gecko) Chrome/26.0.1410.63 Safari/537.31",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 1083) AppleWebKit/537.36 (KHTML like Gecko) Chrome/28.0.1469.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2859.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.49 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:20.0) Gecko/20100101 Firefox/20.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:35.0) Gecko/20100101 Firefox/35.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:40.0) Gecko/20100101 Firefox/40.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:47.0) Gecko/20100101 Firefox/47.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:55.0) Gecko/20100101 Firefox/55.0",
+    "iTunes/4.2 (Macintosh; U; PPC Mac OS X 10.2)",
+    "iTunes/9.0.3 (Macintosh; U; Intel Mac OS X 10_6_2; en-ca)",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Maxthon/4.5.2",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0.112941",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0",
+    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 OPR/28.0.1750.51",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.82 Safari/537.36 OPR/29.0.1795.41",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7; en-us) AppleWebKit/534.20.8 (KHTML, like Gecko) Version/5.1 Safari/534.20.8",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/536.26.17 (KHTML like Gecko) Version/6.0.2 Safari/536.26.17",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.78.1 (KHTML like Gecko) Version/7.0.6 Safari/537.78.1",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) AppleWebKit/601.1.56 (KHTML, like Gecko) Version/9.0 Safari/601.1.56",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.8 (KHTML, like Gecko) Version/10.1 Safari/603.1.30",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us; Silk/1.0.13.81_10003810) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16 Silk-Accelerated=true",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.105 Safari/537.36 Vivaldi/1.0.162.9",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.4 (KHTML, like Gecko) Chrome/4.0.237.0 Safari/532.4 Debian",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.277.0 Safari/532.8",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.309.0 Safari/532.9",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML, like Gecko) Ubuntu/10.10 Chrome/9.1.0.0 Safari/540.0",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Chrome/10.0.613.0 Safari/534.15",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.613.0 Chrome/10.0.613.0 Safari/534.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.703.0 Chrome/12.0.703.0 Safari/534.24",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.20 Safari/535.1",
+    "Mozilla/5.0 Slackware/13.37 (X11; U; Linux x86_64; en-US) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.825.0 Chrome/14.0.825.0 Safari/535.1",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/11.10 Chromium/15.0.874.120 Chrome/15.0.874.120 Safari/535.2",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
+    "Mozilla/5.0 (X11; U; Linux; i686; en-US; rv:1.6) Gecko Epiphany/1.2.5",
+    "Mozilla/5.0 (X11; U; Linux i586; en-US; rv:1.7.3) Gecko/20040924 Epiphany/1.4.4 (Ubuntu)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040614 Firefox/0.8",
+    "Mozilla/5.0 (X11; U; Linux x86_64; sv-SE; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.11) Gecko/2009060309 Ubuntu/9.10 (karmic) Firefox/3.0.11",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.2) Gecko/20090803 Ubuntu/9.04 (jaunty) Shiretoko/3.5.2",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.5) Gecko/20091107 Firefox/3.5.5",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20091020 Linux Mint/8 (Helena) Firefox/3.5.3",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.9) Gecko/20100915 Gentoo Firefox/3.6.9",
+    "Mozilla/5.0 (X11; U; Linux i686; pl-PL; rv:1.9.0.2) Gecko/20121223 Ubuntu/9.25 (jaunty) Firefox/3.8",
+    "Mozilla/5.0 (X11; Linux i686; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre",
+    "Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (X11; Linux i686; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:2.2a1pre) Gecko/20100101 Firefox/4.2a1pre",
+    "Mozilla/5.0 (X11; Linux i686; rv:5.0) Gecko/20100101 Firefox/5.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:7.0a1) Gecko/20110623 Firefox/7.0a1",
+    "Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.16) Gecko/20120421 Gecko Firefox/11.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:12.0) Gecko/20100101 Firefox/12.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1",
+    "Mozilla/5.0 (X11; U; Linux; i686; en-US; rv:1.6) Gecko Galeon/1.3.14",
+    "Mozilla/5.0 (X11; U; Linux ppc; en-US; rv:1.8.1.13) Gecko/20080313 Iceape/1.1.9 (Debian-1.1.9-5)",
+    "Mozilla/5.0 (X11; U; Linux i686; pt-PT; rv:1.9.2.3) Gecko/20100402 Iceweasel/3.6.3 (like Firefox/3.6.3) GTB7.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Iceweasel/5.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:6.0a2) Gecko/20110615 Firefox/6.0a2 Iceweasel/6.0a2",
+    "Mozilla/5.0 (X11; Linux i686; rv:14.0) Gecko/20100101 Firefox/14.0.1 Iceweasel/14.0.1",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:15.0) Gecko/20120724 Debian Iceweasel/15.02",
+    "Konqueror/3.0-rc4; (Konqueror/3.0-rc4; i686 Linux;;datecode)",
+    "Mozilla/5.0 (compatible; Konqueror/3.3; Linux 2.6.8-gentoo-r3; X11;",
+    "Mozilla/5.0 (compatible; Konqueror/3.5; Linux 2.6.30-7.dmz.1-liquorix-686; X11) KHTML/3.5.10 (like Gecko) (Debian package 4:3.5.10.dfsg.1-1 b1)",
+    "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; en_US) KHTML/3.5.6 (like Gecko) (Kubuntu)",
+    "Mozilla/5.0 (X11; Linux x86_64; en-US; rv:2.0b2pre) Gecko/20100712 Minefield/4.0b2pre",
+    "Mozilla/5.0 (X11; U; Linux; i686; en-US; rv:1.6) Gecko Debian/1.6-7",
+    "MSIE (MSIE 6.0; X11; Linux; i686) Opera 7.23",
+    "Opera/9.64 (X11; Linux i686; U; Linux Mint; nb) Presto/2.1.1",
+    "Opera/9.80 (X11; Linux i686; U; en) Presto/2.2.15 Version/10.10",
+    "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.34 (KHTML, like Gecko) QupZilla/1.2.0 Safari/534.34",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.17) Gecko/20110123 SeaMonkey/2.0.12",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061024 Firefox/2.0 (Swiftfox)",
+    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527 (KHTML, like Gecko, Safari/419.3) Arora/0.10.1",
+    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.56 Safari/537.4",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1478.0 Safari/537.36",
+    "Mozilla/5.0 (X11; CrOS x86_64 5841.83.0) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.138 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.125 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2166.2 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36",
+    "Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2876.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3187.0 Safari/537.366",
+    "Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3178.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/33.0.1750.152 Chrome/33.0.1750.152 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/51.0.2704.79 Chrome/51.0.2704.79 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/60.0.3112.78 Chrome/60.0.3112.78 Safari/537.36",
+    "Mozilla/4.0 (compatible; Dillo 3.0)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-us) AppleWebKit/528.5 (KHTML, like Gecko, Safari/528.5 ) lt-GtkLauncher",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.32 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.32 Epiphany/3.8.2",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/604.1 (KHTML, like Gecko) Version/11.0 Safari/604.1 Ubuntu/17.04 (3.24.1-0ubuntu1) Epiphany/3.24.1",
+    "Mozilla/5.0 (X11; Linux i686; rv:16.0) Gecko/20100101 Firefox/16.0",
+    "Mozilla/5.0 (X11; U; Linux i686; rv:19.0) Gecko/20100101 Slackware/13 Firefox/19.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:20.0) Gecko/20100101 Firefox/20.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:20.0) Gecko/20100101 Firefox/20.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:28.0) Gecko/20100101 Firefox/28.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:32.0) Gecko/20100101 Firefox/32.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0",
+    "Mozilla/5.0 (X11; CentOS; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:40.0) Gecko/20100101 Firefox/40.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:43.0) Gecko/20100101 Firefox/43.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:46.0) Gecko/20100101 Firefox/46.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0",
+    "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Galeon/2.0.6 (Ubuntu 2.0.6-2)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.16) Gecko/20080716 (Gentoo) Galeon/2.0.6",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.13) Gecko/20100916 Iceape/2.0.8",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:19.0) Gecko/20100101 Firefox/19.0 Iceweasel/19.0.2",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.2.1",
+    "Mozilla/5.0 (compatible; Konqueror/4.2; Linux) KHTML/4.2.4 (like Gecko) Slackware/13.0",
+    "Mozilla/5.0 (compatible; Konqueror/4.3; Linux) KHTML/4.3.1 (like Gecko) Fedora/4.3.1-3.fc11",
+    "Mozilla/5.0 (compatible; Konqueror/4.4; Linux) KHTML/4.4.1 (like Gecko) Fedora/4.4.1-1.fc12",
+    "Mozilla/5.0 (compatible; Konqueror/4.4; Linux 2.6.32-22-generic; X11; en_US) KHTML/4.4.3 (like Gecko) Kubuntu",
+    "Mozilla/5.0 (compatible; Konqueror/4.4; Linux 2.6.32-22-generic; X11; en_US) KHTML/4.4.3 (like Gecko) Kubuntu",
+    "Mozilla/5.0 (X11; Linux 3.8-6.dmz.1-liquorix-686) KHTML/4.8.4 (like Gecko) Konqueror/4.8",
+    "Mozilla/5.0 (X11; Linux) KHTML/4.9.1 (like Gecko) Konqueror/4.9",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) konqueror/4.14.10 Safari/537.21",
+    "Midori/0.1.10 (X11; Linux i686; U; en-us) WebKit/(531).(2)",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.3) Gecko/2008092814 (Debian-3.0.1-1)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330",
+    "Opera/9.80 (X11; Linux i686) Presto/2.12.388 Version/12.16",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.166 Safari/537.36 OPR/20.0.1396.73172",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.166 Safari/537.36 OPR/20.0.1396.73172",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36 OPR/32.0.1948.25",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36 OPR/40.0.2308.62",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.114 Safari/537.36 Puffin/4.8.0.2965AT",
+    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.8.6 Safari/538.1",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.9.0 Safari/538.1",
+    "Mozilla/5.0 (X11; Linux i686; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
+    "Mozilla/5.0 (X11; Linux i686; rv:12.0) Gecko/20120502 Firefox/12.0 SeaMonkey/2.9.1",
+    "Mozilla/5.0 (Windows NT 5.1; rv:38.0) Gecko/20100101 Firefox/38.0 SeaMonkey/2.35",
+    "Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0 SeaMonkey/2.46",
+    "Mozilla/5.0 (X11; U; Linux x86_64; us; rv:1.9.1.19) Gecko/20110430 shadowfox/7.0 (like Firefox/7.0",
+    "Mozilla/5.0 (X11; U; Linux i686; it; rv:1.9.2.3) Gecko/20100406 Firefox/3.6.3 (Swiftfox)",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Vivaldi/1.0.344.37",
+    "Mozilla/5.0 (X11; U; FreeBSD i386; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0",
+    "Mozilla/5.0 (X11; U; OpenBSD i386; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.359.0 Safari/533.3",
+    "Mozilla/5.0 (X11; U; FreeBSD x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16",
+    "Mozilla/5.0 (X11; U; SunOS sun4m; en-US; rv:1.4b) Gecko/20030517 Mozilla Firebird/0.6",
+    "Mozilla/5.0 (X11; U; SunOS i86pc; en-US; rv:1.9.1b3) Gecko/20090429 Firefox/3.1b3",
+    "Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.9.1) Gecko/20090702 Firefox/3.5",
+    "Mozilla/5.0 (X11; U; FreeBSD i386; de-CH; rv:1.9.2.8) Gecko/20100729 Firefox/3.6.8",
+    "Mozilla/5.0 (X11; FreeBSD amd64; rv:5.0) Gecko/20100101 Firefox/5.0",
+    "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.6) Gecko/20040406 Galeon/1.3.15",
+    "Mozilla/5.0 (compatible; Konqueror/3.5; NetBSD 4.0_RC3; X11) KHTML/3.5.7 (like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/3.5; SunOS) KHTML/3.5.1 (like Gecko)",
+    "Mozilla/5.0 (X11; U; FreeBSD; i386; en-US; rv:1.7) Gecko",
+    "Mozilla/4.77 [en] (X11; I; IRIX;64 6.5 IP30)",
+    "Mozilla/4.8 [en] (X11; U; SunOS; 5.7 sun4u)",
+    "Mozilla/5.0 (Unknown; U; UNIX BSD/SYSV system; C -) AppleWebKit/527 (KHTML, like Gecko, Safari/419.3) Arora/0.10.2",
+    "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/536.5",
+    "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",
+    "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
+    "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
+    "Mozilla/5.0 (X11; NetBSD x86; en-us) AppleWebKit/666.6+ (KHTML, like Gecko) Chromium/20.0.0000.00 Chrome/20.0.0000.00 Safari/666.6+",
+    "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/535.22+ (KHTML, like Gecko) Chromium/17.0.963.56 Chrome/17.0.963.56 Safari/535.22+ Epiphany/2.30.6",
+    "Mozilla/5.0 (X11; U; OpenBSD arm; en-us) AppleWebKit/531.2 (KHTML, like Gecko) Safari/531.2 Epiphany/2.30.0",
+    "Mozilla/5.0 (X11; NetBSD amd64; rv:16.0) Gecko/20121102 Firefox/16.0",
+    "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
+    "Mozilla/5.0 (X11; NetBSD amd64; rv:30.0) Gecko/20100101 Firefox/30.0",
+    "Mozilla/5.0 (X11; OpenBSD amd64; rv:30.0) Gecko/20100101 Firefox/30.0",
+    "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
+    "Mozilla/5.0 (X11; FreeBSD amd64; rv:54.0) Gecko/20100101 Firefox/54.0",
+    "Mozilla/5.0 (compatible; Konqueror/4.1; DragonFly) KHTML/4.1.4 (like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/4.1; OpenBSD) KHTML/4.1.4 (like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/4.5; NetBSD 5.0.2; X11; amd64; en_US) KHTML/4.5.4 (like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)",
+    "Mozilla/5.0 (X11; U; NetBSD amd64; en-US; rv:1.9.2.15) Gecko/20110308 Namoroka/3.6.15",
+    "NetSurf/1.2 (NetBSD; amd64)",
+    "Opera/9.80 (X11; FreeBSD 8.1-RELEASE i386; Edition Next) Presto/2.12.388 Version/12.10",
+    "Mozilla/5.0 (Unknown; UNIX BSD/SYSV system) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.7.0 Safari/538.1",
+    "Mozilla/5.0 (X11; U; SunOS i86pc; en-US; rv:1.8.1.12) Gecko/20080303 SeaMonkey/1.1.8",
+    "Mozilla/5.0 (X11; FreeBSD i386; rv:28.0) Gecko/20100101 Firefox/28.0 SeaMonkey/2.25",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; BOLT/2.800) AppleWebKit/534.6 (KHTML, like Gecko) Version/5.0 Safari/534.6.3",
+    "Mozilla/5.0 (Linux; Android 4.4.2; SAMSUNG-SM-T537A Build/KOT49H) AppleWebKit/537.36 (KHTML like Gecko) Chrome/35.0.1916.141 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; DEVICE INFO) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Mobile Safari/537.36 Edge/12.0",
+    "Mozilla/5.0 (Android; Mobile; rv:35.0) Gecko/35.0 Firefox/35.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 6.12; Microsoft ZuneHD 4.3)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0) Asus;Galaxy6",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch)",
+    "Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2.1",
+    "Mozilla/5.0 (WindowsCE 6.0; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
+    "Mozilla/5.0 (X11; U; Linux armv61; en-US; rv:1.9.1b2pre) Gecko/20081015 Fennec/1.0a1",
+    "Mozilla/5.0 (Maemo; Linux armv7l; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1",
+    "Mozilla/5.0 (Maemo; Linux armv7l; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 Fennec/10.0.1",
+    "Mozilla/5.0 (Android 6.0.1; Mobile; rv:48.0) Gecko/48.0 Firefox/48.0",
+    "Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016",
+    "Mozilla/5.0 (X11; U; Linux armv6l; rv 1.8.1.5pre) Gecko/20070619 Minimo/0.020",
+    "Mozilla/5.0 (X11; U; Linux arm7tdmi; rv:1.8.1.11) Gecko/20071130 Minimo/0.025",
+    "Mozilla/4.0 (PDA; PalmOS/sony/model prmr/Revision:1.1.54 (en)) NetFront/3.0",
+    "Opera/9.51 Beta (Microsoft Windows; PPC; Opera Mobi/1718; U; en)",
+    "Opera/9.60 (J2ME/MIDP; Opera Mini/4.1.11320/608; U; en) Presto/2.2.0",
+    "Opera/9.60 (J2ME/MIDP; Opera Mini/4.2.14320/554; U; cs) Presto/2.2.0",
+    "Opera/9.80 (S60; SymbOS; Opera Mobi/499; U; ru) Presto/2.4.18 Version/10.00",
+    "Opera/10.61 (J2ME/MIDP; Opera Mini/5.1.21219/19.999; en-US; rv:1.9.3a5) WebKit/534.5 Presto/2.6.30",
+    "Opera/9.80 (Android; Opera Mini/7.5.33361/31.1543; U; en) Presto/2.8.119 Version/11.1010",
+    "Opera/9.80 (J2ME/MIDP; Opera Mini/8.0.35626/37.8918; U; en) Presto/2.12.423 Version/12.16",
+    "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Safari/537.36 OPR/30.0.1856.93524",
+    "Opera/9.80 (Android; Opera Mini/9.0.1829/66.318; U; en) Presto/2.12.423 Version/12.16",
+    "Opera/9.80 (Linux i686; Opera Mobi/1040; U; en) Presto/2.5.24 Version/10.00",
+    "POLARIS/6.01 (BREW 3.1.5; U; en-us; LG; LX265; POLARIS/6.01/WAP) MMP/2.0 profile/MIDP-2.1 Configuration/CLDC-1.1",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-gb) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/2.9174AP",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/2.9174AT",
+    "Mozilla/5.0 (iPod; U; CPU iPhone OS 6_1 like Mac OS X; en-HK) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/3.9174IP Mobile",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-AU) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/3.9174IT",
+    "Mozilla/5.0 (X11; U; Linux i686; en-gb) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/2.0.5603M",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.114 Safari/537.36 Puffin/4.5.0IT",
+    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; da-dk) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; XBLWP7; ZuneWP7) UCBrowser/2.9.0.263",
+    "Mozilla/5.0 (Linux; U; Android 2.3.3; en-us ; LS670 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1/UCBrowser/8.6.1.262/145/355",
+    "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
+    "Mozilla/5.0 (Linux; U; Android 4.1; en-us; sdk Build/MR1) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.1 Safari/534.30",
+    "Mozilla/5.0 (Linux; U; Android 4.2; en-us; sdk Build/MR1) AppleWebKit/535.19 (KHTML, like Gecko) Version/4.2 Safari/535.19",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/2.9174AT",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-AU) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.35 Puffin/3.9174IT",
+    "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3",
+    "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
+    "Mozilla/5.0 (iPad; CPU OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML like Gecko) Mobile/12A405 Version/7.0 Safari/9537.53",
+    "Mozilla/5.0 (iPad; CPU OS 8_4_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12H321 Safari/600.1.4",
+    "Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1",
+    "Mozilla/5.0 (iPad; CPU OS 10_0 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/49.0.2623.109 Mobile/14A5335b Safari/601.1.46",
+    "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.114 Safari/537.36 Puffin/4.5.0IT",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7;en-us) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Safari/530.17",
+    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; en-us; KFTT Build/IML74K) AppleWebKit/535.19 (KHTML, like Gecko) Silk/2.1 Mobile Safari/535.19 Silk-Accelerated=true",
+    "Mozilla/5.0 (Linux; Android 4.4.2; LG-V410 Build/KOT49I.V41010d) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.103 Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
+    "Mozilla/5.0 (Linux; Android 4.0.4; BNTV400 Build/IMM76L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Safari/537.36",
+    "Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML like Gecko) Version/7.2.1.0 Safari/536.2+",
+    "Mozilla/5.0 (Linux; U; Android 1.5; de-de; Galaxy Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; SCH-I800 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; GT-P5210 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30",
+    "Mozilla/5.0 (Linux; U; Android 3.0.1; en-us; GT-P7100 Build/HRI83) AppleWebkit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
+    "Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T530NU Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
+    "Mozilla/4.0 (compatible; Linux 2.6.22) NetFront/3.4 Kindle/2.0 (screen 600x800)",
+    "Mozilla/5.0 (Linux U; en-US) AppleWebKit/528.5 (KHTML, like Gecko, Safari/528.5 ) Version/4.0 Kindle/3.0 (screen 600x800; rotate)",
+    "Mozilla/5.0 (X11; U; Linux armv7l like Android; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/533.2+ Kindle/3.0+",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; en-us; KFTT Build/IML74K) AppleWebKit/535.19 (KHTML, like Gecko) Silk/2.1 Mobile Safari/535.19 Silk-Accelerated=true",
+    "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; U; CPU iPad OS 5_0_1 like Mac OS X; en-us) AppleWebKit/535.1+ (KHTML like Gecko) Version/7.2.0.0 Safari/6533.18.5",
+    "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
+    "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) CriOS/30.0.1599.12 Mobile/11A465 Safari/8536.25 (3B92C18B-D9DE-4CB7-A02A-22FD2AF17C8F)",
+    "Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53",
+    "Mozilla/5.0 (iPad; CPU OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML like Gecko) Mobile/12A405 Version/7.0 Safari/9537.53",
+    "Mozilla/5.0 (iPad; CPU OS 8_4_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12H321 Safari/600.1.4",
+    "Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.114 Safari/537.36 Puffin/4.5.0IT",
+    "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420 (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 2_0 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5A347 Safari/525.200",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/531.22.7",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; da-dk) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; da-dk) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/8.0.57838 Mobile/12H321 Safari/600.1.4",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13C75 Safari/601.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A346 Safari/602.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/18.0.130791545 Mobile/14A5345a Safari/600.1.4",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1",
+    "Mozilla/5.0 (iPod; U; CPU iPhone OS 2_2_1 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5H11a Safari/525.20",
+    "Mozilla/5.0 (iPod; U; CPU iPhone OS 3_1_1 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7C145",
+    "Mozilla/5.0 (iPod touch; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML like Gecko) Version/7.0 Mobile/11D167 Safari/123E71C",
+    "Mozilla/5.0 (iPod; CPU iPhone OS 8_4 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) CriOS/44.0.2403.67 Mobile/12H143 Safari/600.1.4",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7;en-us) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; BNTV250 Build/GINGERBREAD) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Safari/533.1",
+    "Mozilla/5.0 (Linux; Android 4.0.4; BNTV400 Build/IMM76L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Safari/537.36",
+    "BlackBerry7100i/4.1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/103",
+    "BlackBerry8300/4.2.2 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/107 UP.Link/6.2.3.15.0",
+    "BlackBerry8320/4.2.2 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/100",
+    "BlackBerry8330/4.3.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/105",
+    "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102",
+    "BlackBerry9530/4.7.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 UP.Link/6.3.1.20.0",
+    "BlackBerry9700/5.0.0.351 Profile/MIDP-2.1 Configuration/CLDC-1.1 VendorID/123",
+    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1 (KHTML, Like Gecko) Version/6.0.0.141 Mobile Safari/534.1",
+    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9930; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.267 Mobile Safari/534.11+",
+    "Mozilla/5.0 (Linux; Android 7.1.1; BBB100-1 Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36",
+    "Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML like Gecko) Version/7.2.1.0 Safari/536.2+",
+    "Mozilla/5.0 (BB10; Touch) AppleWebKit/537.10+ (KHTML, like Gecko) Version/10.1.0.2342 Mobile Safari/537.10+",
+    "Mozilla/5.0 (Linux; Android 5.1.1; Coolpad 3622A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.1.1; Coolpad 3632A Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/BuildID) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 6.0; Nexus 5X Build/MDB08L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.1.2; Nexus 6P Build/N2G48C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.4; Nexus 7 Build/KTU84P) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.135 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Safari/537.36 OPR/30.0.1856.93524",
+    "Mozilla/5.0 (Linux; Android 7.0; Nexus 9 Build/NRD90R) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.1.2; Pixel Build/NHG47N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
+    "Mozilla/5.0 (Linux; webOS/2.2.4; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) webOSBrowser/221.56 Safari/534.6 Pre/3.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11) Sprint:PPC6800",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11) XV6800",
+    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3",
+    "Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+    "HTC-ST7377/1.59.502.3 (67150) Opera/9.50 (Windows NT 5.1; U; en) UP.Link/6.3.1.17.0",
+    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; Android 6.0; ALE-L21 Build/HuaweiALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.89 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.1; C6740N Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 4.1.2; en-us; LG-P870/P87020d Build/JZO54K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+    "LG-LX550 AU-MIC-LX550/2.0 MMP/2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Mozilla/5.0 (Linux; Android 6.0; LG-D850 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.97 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; LG-H918 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; LGL84VL Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; LGUS997 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.2; LGMS323 Build/KOT49I.MS32310b) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.103 Mobile Safari/537.36",
+    "POLARIS/6.01(BREW 3.1.5;U;en-us;LG;LX265;POLARIS/6.01/WAP;)MMP/2.0 profile/MIDP-201 Configuration /CLDC-1.1",
+    "LG-GC900/V10a Obigo/WAP2.0 Profile/MIDP-2.1 Configuration/CLDC-1.1",
+    "Mozilla/5.0 (Linux; Android 4.4.2; LG-V410 Build/KOT49I.V41010d) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.103 Safari/537.36",
+    "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; MDA Pro/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1)",
+    "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
+    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "MOT-L7v/08.B7.5DR MIB/2.2.1 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.0.0.0",
+    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; Android 7.0; Moto G (5) Plus Build/NPNS25.137-35-5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.1.1; XT1710-02 Build/NDS26.74-36) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36",
+    "MOT-V9mm/00.62 UP.Browser/6.2.3.4.c.1.123 (GUI) MMP/2.0",
+    "MOTORIZR-Z8/46.00.00 Mozilla/4.0 (compatible; MSIE 6.0; Symbian OS; 356) Opera 8.65 [it] UP.Link/6.3.0.0.0",
+    "MOT-V177/0.1.75 UP.Browser/6.2.3.9.c.12 (GUI) MMP/2.0 UP.Link/6.3.1.13.0",
+    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
+    "Mozilla/5.0 (Linux; Android 4.4.4; XT1032 Build/KXB21.14-L1.61) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.94 Mobile Safari/537.36",
+    "portalmmm/2.0 N410i(c20;TB)",
+    "Nokia3230/2.0 (5.0614.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
+    "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 Nokia5700/3.27; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
+    "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 Nokia6120c/3.70; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
+    "Nokia6230/2.0 (04.44) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Mozilla/4.1 (compatible; MSIE 5.0; Symbian OS; Nokia 6600;452) Opera 6.20 [en-US]",
+    "Nokia6630/1.0 (2.39.15) SymbianOS/8.0 Series60/2.6 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia7250/1.0 (3.14) Profile/MIDP-1.0 Configuration/CLDC-1.0",
+    "Mozilla/4.0 (compatible; MSIE 5.0; Series80/2.0 Nokia9500/4.51 Profile/MIDP-2.0 Configuration/CLDC-1.1)",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaC6-01/011.010; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 BrowserNG/7.2.7.2 3gpp-gba",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaC7-00/012.003; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 BrowserNG/7.2.7.3 3gpp-gba",
+    "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413 es50",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaE6-00/021.002; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/533.4 (KHTML, like Gecko) NokiaBrowser/7.3.1.16 Mobile Safari/533.4 3gpp-gba",
+    "UCWEB/8.8 (SymbianOS/9.2; U; en-US; NokiaE63) AppleWebKit/534.1 UCBrowser/8.8.0.245 Mobile",
+    "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413 es65",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaE7-00/010.016; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 BrowserNG/7.2.7.3 3gpp-gba",
+    "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413 es70",
+    "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaE90-1/07.24.0.3; Profile/MIDP-2.0 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413 UP.Link/6.2.3.18.0",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 630) like Gecko",
+    "Mozilla/5.0 (Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop; NOKIA; Lumia 635) like Gecko",
+    "Mozilla/5.0 (Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop; NOKIA; Lumia 920) like Geckoo",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920) like Gecko",
+    "NokiaN70-1/5.0609.2.0.1 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0",
+    "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
+    "NokiaN73-1/3.0649.0.0.1 Series60/3.0 Profile/MIDP2.0 Configuration/CLDC-1.1",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaN8-00/014.002; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 BrowserNG/7.2.6.4 3gpp-gba",
+    "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
+    "Mozilla/5.0 (MeeGo; NokiaN9) AppleWebKit/534.13 (KHTML, like Gecko) NokiaBrowser/8.5.0 Mobile Safari/534.13",
+    "Mozilla/5.0 (SymbianOS/9.1; U; de) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
+    "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413 UP.Link/6.3.0.0.0",
+    "Mozilla/5.0 (MeeGo; NokiaN950-00/00) AppleWebKit/534.13 (KHTML, like Gecko) NokiaBrowser/8.5.0 Mobile Safari/534.13",
+    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/10.0.012; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) WicKed/7.1.12344",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaX7-00/021.004; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/533.4 (KHTML, like Gecko) NokiaBrowser/7.3.1.21 Mobile Safari/533.4 3gpp-gba",
+    "Mozilla/5.0 (webOS/1.3; U; en-US) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/1.0 Safari/525.27.1 Desktop/1.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/hspr-H102; Blazer/4.0) 16;320x320",
+    "SEC-SGHE900/1.0 NetFront/3.2 Profile/MIDP-2.0 Configuration/CLDC-1.1 Opera/8.01 (J2ME/MIDP; Opera Mini/2.0.4509/1378; nl; U; ssr)",
+    "Mozilla/5.0 (Linux; U; Android 1.5; de-de; Galaxy Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; SCH-I800 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; de-de; Galaxy S II Build/GRJ22) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+    "Mozilla/5.0 (Linux; Android 4.3; SPH-L710 Build/JSS15J) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.99 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.0.1; SCH-R970 Build/LRX22C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.2; SAMSUNG-SM-G900A Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.94 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; GT-P5210 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30",
+    "Mozilla/5.0 (Linux; U; Android 3.0.1; en-us; GT-P7100 Build/HRI83) AppleWebkit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
+    "SAMSUNG-S8000/S8000XXIF3 SHP/VPP/R5 Jasmine/1.0 Nextreaming SMM-MMS/1.2.0 profile/MIDP-2.1 configuration/CLDC-1.1 FirePHP/0.3",
+    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; SPH-M900 Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "SAMSUNG-SGH-A867/A867UCHJ3 SHP/VPP/R5 NetFront/35 SMM-MMS/1.2.0 profile/MIDP-2.0 configuration/CLDC-1.1 UP.Link/6.3.0.0.0",
+    "SEC-SGHX210/1.0 UP.Link/6.3.1.13.0",
+    "Mozilla/5.0 (Linux; Android 6.0.1; SM-G900H Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G925R6 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.2; SAMSUNG-SM-T537A Build/KOT49H) AppleWebKit/537.36 (KHTML like Gecko) Chrome/35.0.1916.141 Safari/537.36",
+    "Mozilla/5.0 (Linux; U; Android 1.5; fr-fr; GT-I5700 Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "SEC-SGHX820/1.0 NetFront/3.2 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0",
+    "SonyEricssonK550i/R1JD Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonK610i/R1CB Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.16823/1428; U; en) Presto/2.2.0",
+    "SonyEricssonK800i/R1CB Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.0.0.0",
+    "SonyEricssonK810i/R1KG Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Opera/8.01 (J2ME/MIDP; Opera Mini/1.0.1479/HiFi; SonyEricsson P900; no; U; ssr)",
+    "SonyEricssonS500i/R6BC Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 SonyEricssonP100/01; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 Safari/525",
+    "SonyEricssonT68/R201A",
+    "SonyEricssonT100/R101",
+    "SonyEricssonT610/R201 Profile/MIDP-1.0 Configuration/CLDC-1.0",
+    "SonyEricssonT650i/R7AA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonW580i/R6BC Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonW660i/R6AD Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonW810i/R4EA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.0.0.0",
+    "SonyEricssonW850i/R1ED Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonW950i/R100 Mozilla/4.0 (compatible; MSIE 6.0; Symbian OS; 323) Opera 8.60 [en-US]",
+    "SonyEricssonW995/R1EA Profile/MIDP-2.1 Configuration/CLDC-1.1 UNTRUSTED/1.0",
+    "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Opera/9.5 (Microsoft Windows; PPC; Opera Mobi; U) SonyEricssonX1i/R2AA Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "SonyEricssonZ800/R1Y Browser/SEMC-Browser/4.1 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.0.0.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 6.12; Microsoft ZuneHD 4.3)",
+    "Opera/9.80 (Android; Opera Mini/7.5.33361/31.1543; U; en) Presto/2.8.119 Version/11.1010",
+    "Mozilla/5.0 (Android; Mobile; rv:35.0) Gecko/35.0 Firefox/35.0",
+    "Mozilla/5.0 (Android 6.0.1; Mobile; rv:48.0) Gecko/48.0 Firefox/48.0",
+    "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3",
+    "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
+    "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
+    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+    "Mozilla/5.0 (Android; Linux armv7l; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1",
+    "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
+    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+    "Mozilla/5.0 (Linux; U; Android 4.0.3; de-de; Galaxy S II Build/GRJ22) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+    "Opera/9.80 (Android 4.0.4; Linux; Opera Mobi/ADR-1205181138; U; pl) Presto/2.10.254 Version/12.00",
+    "Mozilla/5.0 (Android; Linux armv7l; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 Fennec/10.0.1",
+    "Mozilla/5.0 (Linux; Android 4.1.2; SHV-E250S Build/JZO54K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.82 Mobile Safari/537.36",
+    "Mozilla/5.0 (Android 4.2; rv:19.0) Gecko/20121129 Firefox/19.0",
+    "Mozilla/5.0 (Linux; U; Android 4.3; en-us; sdk Build/MR1) AppleWebKit/536.23 (KHTML, like Gecko) Version/4.3 Mobile Safari/536.23",
+    "Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/BuildID) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.2; SAMSUNG-SM-T537A Build/KOT49H) AppleWebKit/537.36 (KHTML like Gecko) Chrome/35.0.1916.141 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 4.4.2; SM-T230NU Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.81 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.0.1; SCH-R970 Build/LRX22C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T530NU Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Safari/537.36 OPR/30.0.1856.93524",
+    "Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 6.0; LG-D850 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.97 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 6.0; Nexus 5X Build/MDB08L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 6.0.1; SM-G900H Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; Nexus 9 Build/NRD90R) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 7.0; LG-H918 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36",
+    "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420 (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 2_0 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5A347 Safari/525.200",
+    "Mozilla/5.0 (iPod; U; CPU iPhone OS 2_2_1 like Mac OS X; en-us) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5H11a Safari/525.20",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16",
+    "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
+    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; da-dk) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; de-de) AppleWebKit/533.17.9 (KHTML, like Gecko) Mobile/8F190",
+    "MobileSafari/600.1.4 CFNetwork/711.1.12 Darwin/14.0.0",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; da-dk) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13C75 Safari/601.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1",
+    "Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1",
+    "Mozilla/5.0 (Maemo; Linux armv7l; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1",
+    "Mozilla/5.0 (webOS/1.3; U; en-US) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/1.0 Safari/525.27.1 Desktop/1.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/hspr-H102; Blazer/4.0) 16;320x320",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaN8-00/014.002; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) Version/3.0 BrowserNG/7.2.6.4 3gpp-gba",
+    "Mozilla/5.0 (Symbian/3; Series60/5.2 NokiaX7-00/021.004; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/533.4 (KHTML, like Gecko) NokiaBrowser/7.3.1.21 Mobile Safari/533.4 3gpp-gba",
+    "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaE90-1/07.24.0.3; Profile/MIDP-2.0 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413 UP.Link/6.2.3.18.0",
+    "Mozilla/5.0 (SymbianOS 9.4; Series60/5.0 NokiaN97-1/10.0.012; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) WicKed/7.1.12344",
+    "Opera/9.80 (S60; SymbOS; Opera Mobi/499; U; ru) Presto/2.4.18 Version/10.00",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 6.12; Microsoft ZuneHD 4.3)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 7.11) Sprint:PPC6800",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows CE; IEMobile 8.12; MSIEMobile6.0)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0) Asus;Galaxy6",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch)",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko",
+    "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 630) like Gecko",
+    "Mozilla/5.0 (Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop; NOKIA; Lumia 635) like Gecko",
+    "Mozilla/5.0 (Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop; NOKIA; Lumia 920) like Geckoo",
+    "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920) like Gecko",
+    "Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 929) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; DEVICE INFO) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Mobile Safari/537.36 Edge/12.0",
+    "Mozilla/5.0 (Windows NT 10.0; ARM; Lumia 950 Dual SIM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393",
+
 ]

Contents/Libraries/Shared/subliminal_patch/score.py

@@ -8,10 +8,9 @@ from subliminal.score import get_scores
 logger = logging.getLogger(__name__)
 
 
-FPS_EQUALITY = {
-    23.976: 23.98
-}
-FPS_EQUALITY.update({value: key for key, value in FPS_EQUALITY.iteritems()})
+FPS_EQUALITY = (
+    (23.976, 23.98, 24.0),
+)
 
 
 def framerate_equal(source, check):
@@ -23,8 +22,9 @@ def framerate_equal(source, check):
     if source == check:
         return True
 
-    if check in FPS_EQUALITY and FPS_EQUALITY[check] == source:
-        return True
+    for equal_fps_tuple in FPS_EQUALITY:
+        if check in equal_fps_tuple and source in equal_fps_tuple:
+            return True
 
     return False
 
@@ -95,7 +95,7 @@ def compute_score(matches, subtitle, video, hearing_impaired=None):
             matches |= {'series', 'year', 'season', 'episode'}
         if 'tvdb_id' in matches:
             logger.debug('Adding tvdb_id match equivalents')
-            matches |= {'series', 'year', 'season', 'episode'}
+            matches |= {'series', 'year', 'season', 'episode', 'title'}
         if 'series_tvdb_id' in matches:
             logger.debug('Adding series_tvdb_id match equivalents')
             matches |= {'series', 'year'}

Contents/Libraries/Shared/subliminal_patch/subtitle.py

@@ -82,12 +82,11 @@ class Subtitle(Subtitle_):
 
     def set_encoding(self, encoding):
         ge = self.guess_encoding()
-        logger.debug("Encoding change requested: to %s, from %s", encoding, ge)
         if encoding == ge:
-            logger.debug("Encoding already is %s", encoding)
             return
 
         unicontent = self.text
+        logger.debug("Changing encoding: to %s, from %s", encoding, ge)
         self.content = unicontent.encode(encoding)
         self._guessed_encoding = encoding
 
@@ -109,7 +108,6 @@ class Subtitle(Subtitle_):
 
         """
         if self._guessed_encoding:
-            logger.debug('Encoding already guessed: %s', self._guessed_encoding)
             return self._guessed_encoding
 
         logger.info('Guessing encoding for language %s', self.language)
@@ -311,7 +309,7 @@ class Subtitle(Subtitle_):
         :return: string 
         """
         if not self.mods:
-            return fix_text(self.content.decode("utf-8"), **ftfy_defaults)
+            return fix_text(self.content.decode("utf-8"), **ftfy_defaults).encode(encoding="utf-8")
 
         submods = SubtitleModifications(debug=debug)
         if submods.load(content=self.text, language=self.language):
@@ -359,8 +357,14 @@ def guess_matches(video, guess, partial=False):
         if video.season and 'season' in guess and guess['season'] == video.season:
             matches.add('season')
         # episode
-        if video.episode and 'episode' in guess and guess['episode'] == video.episode:
-            matches.add('episode')
+        # Currently we only have single-ep support (guessit returns a multi-ep as a list with int values)
+        # Most providers only support single-ep, so make sure it contains only 1 episode
+        # In case of multi-ep, take the lowest episode (subtitles will normally be available on lowest episode number)
+        if video.episode and 'episode' in guess:
+            episode_guess = guess['episode']
+            episode = min(episode_guess) if episode_guess and isinstance(episode_guess, list) else episode_guess
+            if episode == video.episode:
+                matches.add('episode')
         # year
         if video.year and 'year' in guess and guess['year'] == video.year:
             matches.add('year')

Contents/Libraries/Shared/submod_test.py

@@ -20,7 +20,7 @@ debug = "--debug" in sys.argv
 if debug:
     logging.basicConfig(level=logging.DEBUG)
 
-sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "remove_tags", "OCR_fixes"])
+sub = Subtitle(Language.fromietf("eng"), mods=["common", "remove_HI", "OCR_fixes"])
 sub.content = open(fn).read()
 sub.normalize()
 content = sub.get_modified_content(debug=True)

Contents/Libraries/Shared/subzero/language.py

@@ -14,7 +14,7 @@ def language_from_stream(l):
     for method in ("fromietf", "fromalpha3t", "fromalpha3b"):
         try:
             return getattr(Language, method)(l)
-        except LanguageError:
+        except (LanguageError, ValueError):
             pass
     raise LanguageError()
 

Contents/Libraries/Shared/subzero/modification/dictionaries/make_data.py

@@ -85,6 +85,7 @@ SZ_FIX_DATA = {
             u"L haven't": u"I haven't",
             u"L couldn't": u"I couldn't",
             u"L won't": u"I won't",
+            u"H i": u"Hi",
         },
         "BeginLines": {
             u"l ": u"I ",

Contents/Libraries/Shared/subzero/modification/main.py

@@ -88,6 +88,13 @@ class SubtitleModifications(object):
             if identifier in final_mods and mod_cls.exclusive:
                 final_mods.pop(identifier)
 
+            # language-specific mod, check validity
+            if mod_cls.languages and self.language not in mod_cls.languages:
+                if self.debug:
+                    logger.debug("Skipping %s, because %r is not a valid language for this mod",
+                                 identifier, self.language)
+                continue
+
             # merge args of duplicate mods if possible
             elif identifier in final_mods and mod_cls.args_mergeable:
                 final_mods[identifier] = mod_cls.merge_args(final_mods[identifier], args)
@@ -132,7 +139,8 @@ class SubtitleModifications(object):
             if self.debug:
                 logger.debug("Line mods took %ss", time.time() - line_mods_start)
 
-        self.f.events = new_entries
+            if new_entries:
+                self.f.events = new_entries
 
         # apply last file mods
         if non_line_mods:
@@ -194,7 +202,8 @@ class SubtitleModifications(object):
                     mod = self.initialized_mods[identifier]
 
                     try:
-                        line = mod.modify(line.strip(), entry=entry.text, debug=self.debug, parent=self, **args)
+                        line = mod.modify(line.strip(), entry=entry.text, debug=self.debug, parent=self, index=index,
+                                          **args)
                     except EmptyEntryError:
                         if self.debug:
                             logger.debug(u"%d: %s: %r -> ''", index, identifier, entry.text)
@@ -263,6 +272,8 @@ class SubtitleModifications(object):
                     if self.debug:
                         logger.debug(u"Fixing tags: %s (%r -> %r)", str(add_start_tags+add_end_tags), new_text,
                                      entry.text)
+                else:
+                    entry.text = new_text
             else:
                 entry.text = new_text
 

Contents/Libraries/Shared/subzero/modification/mods/__init__.py

@@ -20,11 +20,12 @@ class SubtitleModification(object):
     pre_processors = []
     processors = []
     post_processors = []
+    languages = []
 
     def __init__(self, parent):
         return
 
-    def _process(self, content, processors, debug=False, parent=None, **kwargs):
+    def _process(self, content, processors, debug=False, parent=None, index=None, **kwargs):
         if not content:
             return
 
@@ -41,12 +42,13 @@ class SubtitleModification(object):
             new_content = processor.process(new_content, debug=debug, **kwargs)
             if not new_content:
                 if debug:
-                    logger.debug("Processor returned empty line: %s", processor)
+                    logger.debug("Processor returned empty line: %s", processor.name)
                 break
             if debug:
                 if old_content == new_content:
                     continue
-                logger.debug("%s: %s -> %s", processor, repr(old_content), repr(new_content))
+                logger.debug("%d: %s: %s -> %s", index, processor.name, repr(old_content), repr(new_content))
+
         return new_content
 
     def pre_process(self, content, debug=False, parent=None, **kwargs):
@@ -84,6 +86,7 @@ class SubtitleTextModification(SubtitleModification):
     pass
 
 
+TAG = ur"(?:\s*{\\[iusb][0-1]}\s*)*"
 EMPTY_TAG_PROCESSOR = ReProcessor(re.compile(r'({\\\w1})[\s.,-_!?]*({\\\w0})'), "", name="empty_tag")
 
 empty_line_post_processors = [

Contents/Libraries/Shared/subzero/modification/mods/common.py

@@ -2,6 +2,7 @@
 
 import re
 
+from subzero.language import Language
 from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, SubtitleModification
 from subzero.modification.processors.string_processor import StringProcessor
 from subzero.modification.processors.re_processor import NReProcessor
@@ -19,11 +20,11 @@ class CommonFixes(SubtitleTextModification):
     """
 
     processors = [
-        # -- = ...
-        NReProcessor(re.compile(r'(?u)(^-\s?-[-\s]*)(?!.+\s?-\s?-[-\s]*)'), "", name="CM_doubledash"),
+        # -- = em dash
+        NReProcessor(re.compile(r'(?u)(\w|\b|\s|^)(-\s?-{1,2})'), ur"\1—", name="CM_multidash"),
 
         # line = _/-/\s
-        NReProcessor(re.compile(r'(?u)(^[-_\s]*[-_\s]+[-_\s]*$)'), "", name="CM_non_word_only"),
+        NReProcessor(re.compile(r'(?u)(^[-_\s.]*[-_\s.]+[-_\s.]*$)'), "", name="CM_non_word_only"),
 
         # fix music symbols
         NReProcessor(re.compile(ur'(?u)(^[*#¶\s]*[*#¶]+[*#¶\s]*$)'), u"♪", name="CM_music_symbols"),
@@ -46,6 +47,9 @@ class CommonFixes(SubtitleTextModification):
         # multiple spaces
         NReProcessor(re.compile(r'(?u)[\s]{2,}'), " ", name="CM_multiple_spaces"),
 
+        # more than 3 dots
+        NReProcessor(re.compile(r'(?u)\.{3,}'), "...", name="CM_dots"),
+
         # no space after starting dash
         NReProcessor(re.compile(r'(?u)^-(?![\s-])'), "- ", name="CM_dash_space"),
 
@@ -73,11 +77,11 @@ class CommonFixes(SubtitleTextModification):
         # countdowns otherwise); don't break up ellipses
         NReProcessor(
             re.compile(r'(?u)(\b[0-9]+[0-9:\']*(?<!\.\.)\s+(?!\.\.)[0-9,.:\'\s]*(?=[0-9]+)[0-9,.:\'])'),
-            lambda match: match.group(1).replace(" ", ""),
+            lambda match: match.group(1).replace(" ", "") if match.group(1).count(" ") == 1 else match.group(1),
             name="CM_spaces_in_numbers"),
 
         # uppercase after dot
-        NReProcessor(re.compile(ur'(?u)((?:[^.\s])+\.\s+)([a-zà-ž])'),
+        NReProcessor(re.compile(ur'(?u)((?<!(?=\s*[A-ZÀ-Ž-_0-9.]\s*))(?:[^.\s])+\.\s+)([a-zà-ž])'),
                      lambda match: ur'%s%s' % (match.group(1), match.group(2).upper()), name="CM_uppercase_after_dot"),
 
         # remove double interpunction
@@ -112,12 +116,23 @@ class ReverseRTL(SubtitleModification):
     identifier = "reverse_rtl"
     description = "Reverse punctuation in RTL languages"
     exclusive = True
+    order = 50
+    languages = [Language("heb")]
+
+    long_description = """\
+    Some playback devices don't properly handle right-to-left markers for punctuation. Physically swap punctuation.
+    Applicable to languages: hebrew
+    """
 
     processors = [
-        NReProcessor(re.compile(ur"(?u)((?=(?<=\b|^)|(?<=\s))([.!?-]+)([^.!?-]+)(?=\b|$|\s))"), r"\3\2",
+        # new? (?u)(^([\s.!?]*)(.+?)(\s*)(-?\s*)$); \5\4\3\2
+        #NReProcessor(re.compile(ur"(?u)((?=(?<=\b|^)|(?<=\s))([.!?-]+)([^.!?-]+)(?=\b|$|\s))"), r"\3\2",
+        #             name="CM_RTL_reverse")
+        NReProcessor(re.compile(ur"(?u)(^([\s.!?]*)(.+?)(\s*)(-?\s*)$)"), r"\5\4\3\2",
                      name="CM_RTL_reverse")
     ]
 
 
 registry.register(CommonFixes)
 registry.register(RemoveTags)
+registry.register(ReverseRTL)

Contents/Libraries/Shared/subzero/modification/mods/hearing_impaired.py

@@ -1,7 +1,7 @@
 # coding=utf-8
 import re
 
-from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, EmptyEntryError
+from subzero.modification.mods import SubtitleTextModification, empty_line_post_processors, EmptyEntryError, TAG
 from subzero.modification.processors.re_processor import NReProcessor
 from subzero.modification import registry
 
@@ -28,16 +28,17 @@ class HearingImpaired(SubtitleTextModification):
 
     processors = [
         # full bracket entry, single or multiline; starting with brackets and ending with brackets
-        FullBracketEntryProcessor(re.compile(ur'(?sux)^-?\s?[([].+(?=[^)\]]{3,}).+[)\]]$'), "",
-                                  name="HI_brackets_full"),
+        FullBracketEntryProcessor(re.compile(ur'(?sux)^-?%(t)s[([].+(?=[^)\]]{3,}).+[)\]]%(t)s$' % {"t": TAG}),
+                                  "", name="HI_brackets_full"),
 
         # brackets (only remove if at least 3 chars in brackets)
-        NReProcessor(re.compile(ur'(?sux)-?\s*[([][^([)\]]+?(?=[A-zÀ-ž"\']{3,})[^([)\]]+[)\]][\s:]*'), "",
-                     name="HI_brackets"),
+        NReProcessor(re.compile(ur'(?sux)-?%(t)s[([][^([)\]]+?(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+[)\]][\s:]*%(t)s' %
+                                {"t": TAG}), "", name="HI_brackets"),
 
-        NReProcessor(re.compile(ur'(?sux)-?\s*[([]\s*(?=[A-zÀ-ž"\']{3,})[^([)\]]+$'), "", name="HI_bracket_open_start"),
+        NReProcessor(re.compile(ur'(?sux)-?%(t)s[([]%(t)s(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+%(t)s$' % {"t": TAG}),
+                     "", name="HI_bracket_open_start"),
 
-        NReProcessor(re.compile(ur'(?sux)-?\s*(?=[A-zÀ-ž"\']{3,})[^([)\]]+[)\]][\s:]*'), "",
+        NReProcessor(re.compile(ur'(?sux)-?%(t)s(?=[A-zÀ-ž"\'.]{3,})[^([)\]]+[)\]][\s:]*%(t)s' % {"t": TAG}), "",
                      name="HI_bracket_open_end"),
 
         # text before colon (and possible dash in front), max 11 chars after the first whitespace (if any)
@@ -48,15 +49,16 @@ class HearingImpaired(SubtitleTextModification):
         #             name="HI_before_colon"),
 
         # uppercase text before colon (at least 3 uppercase chars); at start or after a sentence,
-        # possibly with a dash in front
+        # possibly with a dash in front; ignore anything ending with a quote
         NReProcessor(re.compile(ur'(?u)(?:(?<=^)|(?<=[.\-!?\"\']))([\s-]*(?=[A-ZÀ-Ž]\s*[A-ZÀ-Ž]\s*[A-ZÀ-Ž])'
-                                ur'[A-ZÀ-Ž-_0-9\s\"\']+:\s*)(?![0-9])'), "", name="HI_before_colon_caps"),
+                                ur'[A-ZÀ-Ž-_0-9\s\"\']+:(?![\"\'’ʼ❜‘‛”“‟„])\s*)(?![0-9])'), "",
+                     name="HI_before_colon_caps"),
 
-        # any text before colon (at least 3 uppercase chars); at start or after a sentence,
+        # any text before colon (at least 3 chars); at start or after a sentence,
         # possibly with a dash in front; try not breaking actual sentences with a colon at the end by not matching if
-        # more than one space is inside the text
-        NReProcessor(re.compile(ur'(?u)(?:(?<=^)|(?<=[.\-!?\"\']))([\s-]*(?=[A-zÀ-ž]\s*[A-zÀ-ž]\s*[A-zÀ-ž])'
-                                ur'[A-zÀ-ž-_0-9\s\"\']+:\s*)(?![0-9])'),
+        # more than one space is inside the text; ignore anything ending with a quote
+        NReProcessor(re.compile(ur'(?u)(?:(?<=^)|(?<=[.\-!?\"]))([\s-]*(?=[A-zÀ-ž]\s*[A-zÀ-ž]\s*[A-zÀ-ž])'
+                                ur'[A-zÀ-ž-_0-9\s\"\']+:(?![\"’ʼ❜‘‛”“‟„])\s*)(?![0-9])'),
                      lambda match: match.group(1) if (match.group(1).count(" ") > 1
                                                       or match.group(1).count("-") > 1) else "",
                      name="HI_before_colon_noncaps"),
@@ -77,7 +79,8 @@ class HearingImpaired(SubtitleTextModification):
                      name="HI_starting_upper_then_sentence"),
 
         # remove music symbols
-        NReProcessor(re.compile(ur'(?u)(^[*#¶♫♪\s]*[*#¶♫♪\s]+[*#¶♫♪\s]*$)'), "", name="HI_music_symbols_only"),
+        NReProcessor(re.compile(ur'(?u)(^%(t)s[*#¶♫♪\s]*%(t)s[*#¶♫♪\s]+%(t)s[*#¶♫♪\s]*%(t)s$)' % {"t": TAG}),
+                     "", name="HI_music_symbols_only"),
     ]
 
     post_processors = empty_line_post_processors

Contents/Libraries/Shared/subzero/modification/mods/ocr_fixes.py

@@ -39,7 +39,9 @@ class FixOCR(SubtitleTextModification):
 
         return [
             # remove broken HI tag colons (ANNOUNCER'., ". instead of :) after at least 3 uppercase chars
-            NReProcessor(re.compile(ur'(?u)(^.*(?<=[A-ZÀ-Ž]{3})[A-ZÀ-Ž-_\s0-9]+)(["\'’ʼ❜‘‛”“‟„]*[.,‚،⹁、;]+)(\s*)'),
+            # don't modify stuff inside quotes
+            NReProcessor(re.compile(ur'(?u)(^[^"\'’ʼ❜‘‛”“‟„]*(?<=[A-ZÀ-Ž]{3})[A-ZÀ-Ž-_\s0-9]+)'
+                                    ur'(["\'’ʼ❜‘‛”“‟„]*[.,‚،⹁、;]+)(\s*)(?!["\'’ʼ❜‘‛”“‟„])'),
                          r"\1:\3", name="OCR_fix_HI_colons"),
             # fix F'bla
             NReProcessor(re.compile(ur'(?u)(\bF)(\')([A-zÀ-ž]*\b)'), r"\1\3", name="OCR_fix_F"),

Contents/Libraries/Shared/subzero/subtitle_storage.py

@@ -212,7 +212,7 @@ class JSONStoredVideoSubtitles(object):
             return out
 
         for key, subtitle in all_subs.iteritems():
-            if key == "current":
+            if key in ("current", "blacklist"):
                 continue
 
             if subtitle.provider_name == provider_name:
@@ -226,7 +226,7 @@ class JSONStoredVideoSubtitles(object):
             return 0
 
         subs = part.get(str(lang))
-        return len(filter(lambda key: key != "current", subs.keys()))
+        return len(filter(lambda key: key not in ("current", "blacklist"), subs.keys()))
 
     def get_sub_key(self, provider_name, id):
         return provider_name, str(id)

Contents/Libraries/Shared/test.srt

@@ -1,16 +1,19 @@
 1
 00:00:01,250 --> 00:00:04,419
 (WHOOSHING)
+This is a phone number: 123 4325 123
+H i. But not MATCH hindrance.
 
 2
 00:00:10,759 --> 00:00:12,678
 ROSE: (Help us. Please. . .help us.)
 What's "wrong"? over 9, 000!
+I can't keep running. L can't!
 
 3
 00:00:12,679 --> 00:00:16,097
-<b>I don't know. Some kind of wrong "1 00" number
-of signal, drawing the Tardis off course.</b>
+<b>I don't know. Some kind of wrong "1 00" number---
+of signal, drawing the Tardis off.... course.</b>
 
 4
 00:00:16,099 --> 00:00:17,224
@@ -34,24 +37,32 @@ But fix this: 81 ,00
 00:00:21,103 --> 00:00:23,603
 <i>(laughing): lrn gonna And when are we? (chuckles)
 lrn gonna And when are we?</i>
+"I luv butts".
 
 8
 00:00:24,274 --> 00:00:26,649
 ...2012.  weII it's 1 2:00 o'clock
+like "MAMASBMW,"
+"LEXUS4LIZ," or "BOOBGUY"?
 
 9
 00:00:26,650 --> 00:00:29,370
 (BIG BROTHER
 oh god multiline! THEME MUSIC)
+takeaway from today, it's this:
 
 10
 00:00:30,612 --> 00:00:33,112
-(SWITCH CLICKING)
+<i> ( intercom buzzes )</i>
+- woman:<i> Ken's here</i>
+<i> for the meeting.</i>
 
 11
 00:00:33,658 --> 00:00:34,783
 (WHOO
 SHING) >>geil
+A.K.A. being nice
+[J.J.] alte. nice. being nice
 
 12
 00:00:34,783 --> 00:00:36,826

Contents/advanced_settings.json.template

@@ -10,6 +10,20 @@ Don't expect support if you mess this up.
     // true: in case of a configured subtitle sub-folder, also clean up the folder containing the media item
     "thorough_cleaning": false,
 
+    // list of subtitle formats that should be considered as text-based. useful if your media player doesn't natively
+    // support ASS/SSA for example (remove them from this list then)
+    "text_subtitle_formats": ["srt", "ass", "ssa", "vtt", "mov_text"],
+
+    // allow multiple automatic subtitle extractions at the same time; by default, to not clog up the CPU, SZ only
+    // executes one automatic extraction at a time. This does not affect manually started extractions and only affect
+    // multiple agent/metadata update calls. Refreshing a season or a full TV show will still only use one thread.
+    "auto_extract_multithread": false,
+
+    // when the find better subtitles task finds a subtitle for an item that has an active extracted embedded subtitle
+    // set, what should be the minimum score that subtitle has to have in order to be considered better?
+    "find_better_as_extracted_tv_score": 352,
+    "find_better_as_extracted_movie_score": 82,
+
     // per-provider-config
     "providers": {
         // enabled_for: specifies which media type to enable the provider for. does not override global/throttled
@@ -45,5 +59,11 @@ Don't expect support if you mess this up.
         "tvsubtitles": {
             "enabled_for": ["series"],
         },
+        "hosszupuska": {
+            "enabled_for": ["series"],
+        },
+        "argenteam": {
+            "enabled_for": ["series", "movies"],
+        }
     }
 }

README.md

@@ -26,7 +26,7 @@ This is just a tiny peek at the full feature-set of Sub-Zero.
 It searches up to 8 individual subtitle provider sites and APIs, selects the best matching subtitle and downloads it for you.
 
 The matching is done by looking at the filename of your media files, as well as media information inside the container.
-Every subtitle gets a score assigned, based on the matching algorithm. The one with the highest score gets picked automatically. The more information your media filenames have, the better. `Moviename.mkv` has a higher chance of getting bad subtitles than `Moviename.2015.720p.BluRay-RLSGRP`.
+Every subtitle gets a score assigned, based on the matching algorithm. The one with the highest score gets picked automatically. The more information your media filenames have, the better. `Moviename.mkv` has a higher chance of getting bad subtitles than `Moviename.2015.720p.BluRay-RLSGRP`. If you like renaming your media files, you want to have a look at [SZ refiners](https://github.com/pannal/Sub-Zero.bundle/wiki/Refiners).
 
 #### Storage-Options
 *You* can decide where Sub-Zero stores its downloaded subtitles. By default it saves the subtitles externally, as "sidecars", besides the actual media file.
@@ -66,7 +66,7 @@ In addition to that Sub-Zero also fixes problems introduced by the subtitle crea
 Ever had broken music icons in a subtitle? Nordic characters like `Å` which turned into `Ã¥`? Not anymore.
 
 ## Installation
-Simply go to the Plex Channels in your Plex Media Server, search for Sub-Zero and install it.
+Simply go to the Plex Plugins in your Plex Media Server, search for Sub-Zero and install it.
 For further help or manual installation, [please go to the wiki](https://github.com/pannal/Sub-Zero.bundle/wiki).
 
 ## Big thanks to the beta testing team (in no particular order)!
@@ -77,34 +77,54 @@ Jacob K, Ninjouz, chopeta, fvb
 
 ## Changelog
 
-2.5.0.2287
+2.5.3.2414
 
-- core: reduce main icon size
-- core: fix usage on NVIDIA SHIELD (hopefully, please report back), #441
-- core: add scandir fallback to listdir in case of badly configured locale in environment, #441, #440
-- core: get subtitles from archive: don't assume an episode match
-- core: get subtitles from archive: don't assume any attributes in guess
-- core: improve release group detection for drone/filebot/file_info refiners
-- core: fix language detection for embedded subtitle streams
-- core: support extraction of embedded mov_text subtitles in mp4 video files
-- refiners: drone: add http:// to url if not given
-- providers: opensubtitles: retry/reinitialize request when encountering ResponseNotReady
-- config: clarify subscene being only enabled for TV series by default
-- menu: when encountering permission errors when scanning media files, warn in the menu about them
-- submod: common: don't break -- addic7ed --
-- submod: common: remove lines that consist only of dash, underscore
-- submod: OCR: fix Ls = Is
-- submod: OCR: fix bad HI colons (ANNOUNCER; instead of ANNOUNCER:)
-- submod: common: fix lines consisting only of bad music symbols (*#¶ = ♪)
-- submod: HI: remove music-symbol-only-lines
-- submod: HI: be less aggressive about lines ending with a colon; please re-apply all your mods via advanced menu
-- submod: OCR: fix it'sjust, isn'tjust, Iam, Ican
+- core: expand user agent list
+- core: update subliminal to 4ad5d31
+- core: treat 23.976, 23.98, 24.0 fps as equal
+- core: correctly skip blacklist entries when iterating through currently known subs
+- core: fix unpacking of packs without asked-for-release-group
+- core: fix embedded subtitle language detection; add debug log
+- core: treat embedded subtitle containing "forced" in its title as forced
+- core: improve embedded subtitles detection
+- core: store extracted embedded forced subtitles with the "forced" suffix (e.g.: video.en.forced.srt)
+- core: don't bother trying to extract embedded subtitle if transcoder wasn't found
+- core: fix automatic extraction of unknown embedded subtitle streams
+- core: skip immediately searching for new subtitle after successfully extracting embedded
+- core: extract embedded ASS: don't transcode to SRT using ffmpeg (Plex Transcoder), do the transcoding later using pysubs2; fixes offset issues
+- core: extract embedded: let ffmpeg auto convert mov_text/tx3g to srt
+- core: fix transcoder detection; add fallback #460
+- core: remove LD_LIBRARY_PATH from environment before calling notification executable
+- core: auto extract embedded subtitles in a separate thread
+- core: reduce encoding change log spam
+- core: only allow one automatic extraction at a time; add optional advanced settings "auto_extract_multithread"
+- core: add minimum score a subtitle has to have when considered by the find better subtitles task, when the current subtitle is an extracted embedded one; add advanced_settings entries
+- core/config: automatic extraction: add config setting to indicate whether there should be an immediate search for available subtitles after extraction or not (default: off)
+- core/menu/submod: add reverse_rtl modification for Hebrew; fixes #409
+- core: scoring: assume title match on tvdb_id match
+- tasks: search all recently added missing: fix attribute access on missing stored subtitle info
+- providers: add hosszupuska (hungarian, thanks morpheus133 for the basic implementation)
+- providers: add argenteam (spanish, thanks mmiraglia for the basic implementation)
+- providers: addic7ed: use random user agent by default (enforce for existing configs)
+- providers: enable subscene by default
+- providers: opensubtitles: add fallback for dict based query response in contrast to list/array based
+- advanced settings: make text-based-subtitle-formats configurable
+- menu: submod: inverse-reverse subtitle timing time-choices for better accessibility
+- submod: reduce log spam in case of debug logs enabled
+- submod: style tags could result in no output at all
+- submod: fix empty content if only non-line-mods were used, no line-mods; fixes #449
+- submod: HI: correctly handle style tags when checking for brackets
+- submod: HI: don't remove anything that's surrounded by quotes
+- submod: HI: double or triple dash is em dash
+- submod: HI: HI_before_colon_noncaps, don't assume single quotes are sentence enders
+- submod: common: don't uppercase after abbreviations
+- submod: common: don't break phone numbers (more than one spaced number pair found)
+- submod: common: also count lines only consisting of dots as removable
+- submod: common: replace more than 3 consecutive dots with 3 dots
+- submod: OCR: "H i." = "Hi."
 
 
 
-2.5.0.2247
-- fix ignoring by-hash-matched episodes
-
 
 [older changes](CHANGELOG.md)