add log warning for dogpile cache region setup; bump to 2.0.0.9

# Conflicts:
#	Contents/Code/interface/menu.py

.gitignore

@@ -54,3 +54,5 @@ docs/_build/
 
 # pycharm
 .idea
+
+icon.psd

CHANGELOG.md

@@ -1,3 +1,166 @@
+1.4.27.957
+- core: correctly fall back to the next best subtitle if the current one couldn't be downloaded; hopefully fixes #231
+- core: add "Scan: which external subtitles should be picked up?"-setting
+- core: add optional on_playing activities. refresh currently playing movie, refresh next episode in season, both or none; fixes #259 #33
+- core: skip to next best subtitle if findbettersubtitles failed
+- core: add setting to treat undefined-language embedded subtitle as configured language1 #239
+- core: fix handling of inexistant addic7ed show id
+- core: fix regression issue breaking relative custom subtitle folder handling
+- core: fix loading of stored subtitle info data of now-non-existant items
+- core: re-add separate global subtitle folder handling
+- menu: remove obsolete actions from the advanced menu
+
+
+1.4.23.920
+- core: handle undecodable paths better #255
+- core: don't fail on unrecoverable data #257
+- core: increase default scores from 110 (series) and 23 (movies) to 116 and 33
+- core: fix global subtitle folder handling #234
+- core: better invoking of configured executable after subtitle addition #247
+
+
+1.4.22.908
+- core: hotfix for more robust migrations
+
+
+1.4.22.898
+- core: migrate history and subtitle storage to a better implementation, making it far more stable. subtitle storage now also stores the downloaded subtitle data for future usage, so it will be possible to switch between them
+- core/menu: manual subtitle download and the FindBetterSubtitles-task now also work with metadata storage (hi @ shield users)
+- core: optimize FindBetterSubtitles-task
+
+
+1.4.19.882
+- core: fix tasks for new users
+- core: double check pin correctness/existance when pin is enabled
+
+
+1.4.19.878
+- core/menu: fix a task's last runtime display
+- core: task optimizations
+- core: fix leftover subtitles cleanup handling in case of a custom subtitle folder #234
+- core: run the scheduler even if permissions for libraries are wrong ("fixes" #236)
+- core: store subtitle history data in a different data format; reduce used storage size drastically (#233)
+
+
+1.4.19.866
+- core: fix wrong usage of LogKit
+
+
+1.4.19.857
+
+- core: add option to enable/disable channel and/or agent modes (fixes #220)
+- core: skip inexistent internal streams when scanning for internal subtitles (fixes #222)
+- core: fix filename encoding (fixes #223)
+- core: storage optimizations
+- menu: add pin-based channel menu locking (the whole channel or only the advanced menu)
+
+
+1.4.17.836
+- core: support for any media file that PMS supports (internal subtitles on mp4 for example)
+- core: fix broken ignore folders containing "subzero.ignore/.subzero.ignore/.nosz"
+- core: fix duplicate subtitles (lowercase/default case)
+- core: fix broken tasks queue due to oversight
+
+
+1.4.16.822
+- menu: add per-section recently added menu
+- menu: fix accidentally double-triggering a just triggered force-refresh
+- core: reorder settings in a more logical, grouped way
+- core: add simple automatic filesystem/external leftover subtitle cleaning (#133, #152)
+- core: fix force-refresh for big seasons/series
+- core: add setting to look for forced/foreign-only subtitles only (only works for opensubtitles and podnapisi)
+- core: fix custom subtitle folder was being ignored (#211)
+- core: only trust PMS for its movie name, not the series title (fixes #210)
+- core: full support (in filesystem/external mode) for forced/default/normal subtitle tags
+- core: ignore "non-standard" external subtitle files when scanning by default (everything but .srt, .ass, .ssa, fixes #192)
+- core: lower default max_recent_items_per_library to 500
+- core: skip forced/foreign-only subtitles if not specifically wanted
+- core: modify the task queue, hopefully helping #206
+- core: update anonymous usage collection
+
+
+1.4.11.781
+- core: cleanup, logging
+- core/menu: fix addic7ed display in manual subtitle list
+- core: use HTTP for OpenSubtitles instead of HTTPS because of current certificate errors
+- core: find better subtitles should now run smoothly even with replaced files (newer parts)
+
+
+1.4.10.769
+- core: hotfix for legacy intent storage regression
+
+1.4.10.768
+- core: automatically find better subtitles (configurable)
+- menu: display how the subtitle was downloaded (auto, manual, auto-better), in history menu
+- menu/core: correctly handle subtitle list for multiple languages
+- core: lower minimum series score to list subtitles for to 66
+- core: better matching of garbage filenames; we trust Plex now for the series name/movie title fully
+- core: add setting to specifically set the file permissions (chmod)
+
+
+1.4.5.742
+- core: fix force-refresh in certain situations
+- menu: add history
+- menu: add manual subtitle selection
+- menu: run Items with missing subtitles in separate thread for big libraries
+- settings: add history list size option (default: 100)
+- settings: add new default scores (TV: 110); use input instead of dropdown
+- settings: increase default missing subtitles amount per library to 2000
+- core: generic rewrites and optimizations
+- core: better hash verification
+- core: add anonymous usage data (opt-out in settings)
+- core: fix pt-BR display (IETF) again
+- wiki: update (thanks @dane22!) - quick URL: http://v.ht/szwiki
+- wiki: add score explanation - quick URL: http://v.ht/szscores
+- core: add persian/farsi encoding support
+
+
+1.3.49.636
+- core/menu: fix force refreshing (again)
+- core/menu: fix redundant route calls
+
+
+1.3.49.630 (backported some changes of the develop-1.4 branch to 1.3)
+- core/menu: make addic7ed boost configurable; lower the default boost value massively (to 10)
+- core: fix force refreshing (hopefully)
+- core: add (thai) tis-620 subtitle encoding support
+- menu: lower letter based menu browsing from 200 to 80 items
+- core: support greek encodings (windows-1253, cp1253, cp737, iso8859_7, cp875, cp869, iso2022_jp_2, mac_greek); hopefully fixes badly saved greek subs
+- menu: add generic back-to-home button to the top of every container view
+- menu: warn the user when SZ isn't enabled for any sections/libraries
+- menu: always re-check permissions status and enabled sections when opening the main menu; no server restart necessary anymore
+
+
+1.3.46.606
+- core: hotfix for new users (who've never downloaded a subtitle with SZ before); fixes #169
+
+
+1.3.46.605
+
+- add wiki (thanks @ukdtom / @dane22)
+- core: remove necessity of Plex credentials; fixes #148
+- core: fix non-SRT subtitle support; fixes #138
+- core: generic source overhaul in preparation for release 1.4
+- core: better filesystem encoding detection; may fix #159
+- core: add encoding handling for windows-1250 and windows-1251 encoding (eastern europe); fixes #162
+- core: overhaul ignore handling; fixes #164
+- core: implement ignore by path setting; fixes #134
+- core: add setting for optional fallback to metadata storage, if filesystem storage failed; fixes #100
+- core: add setting for notifying an executable after a subtitle has been downloaded (see Wiki); fixes #65
+- core: only handle sections for which Sub-Zero is enabled (in PMS agent settings); fixes #167
+- menu: add series/season force-refresh
+- menu: show item thumbnail/art where applicable
+- menu: mitigate PlexWeb behaviour of calling our handlers twice; fixes #168
+
+
+1.3.33.522
+
+- core: fix library permission detection on windows; fixes #151
+- core: "Restrict to one language" now behaves like it should (one found subtitle of any language is treated as sufficient); fixes #149
+- core: add support for other subtitle formats such as ssa/ass/microdvd, convert to srt; fixes #138
+- core: hopefully more consistent force-refresh handling (intent); fixes #118
+
+
 1.3.31.513
 
 - core: add option to only download one language again (and skip the addition of .lang to the subtitle filename) (default: off); fixes #126 

Contents/Code/__init__.py

@@ -1,8 +1,9 @@
 # coding=utf-8
-import os
 import sys
+import datetime
+import os
 
-# just some slight modifications to support sum and iter again
+from subliminal_patch import compute_score
 from subzero.sandbox import restore_builtins
 
 module = sys.modules['__main__']
@@ -17,174 +18,87 @@ import logger
 
 sys.modules["logger"] = logger
 
-from subzero import intent
 import subliminal
-import subliminal_patch
 import support
 
 import interface
-
 sys.modules["interface"] = interface
 
+from subliminal.cli import MutexLock
 from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
-from subzero import intent
 from interface.menu import *
-from support.subtitlehelpers import getSubtitlesFromMetadata, force_utf8
-from support.storage import storeSubtitleInfo
-from support.config import config, IGNORE_FN
+from support.plex_media import media_to_videos, get_media_item_ids, scan_videos
+from support.subtitlehelpers import get_subtitles_from_metadata
+from support.storage import whack_missing_parts, save_subtitles, get_subtitle_storage
+from support.items import is_ignored
+from support.config import config
+from support.lib import get_intent
+from support.helpers import track_usage, get_title_for_video_metadata, get_identifier, cast_bool
+from support.history import get_history
+from support.data import dispatch_migrate
+from support.activities import activity
 
 
 def Start():
     HTTP.CacheTime = 0
     HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
 
-    # configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
-    subliminal.region.configure('dogpile.cache.memory')
+    try:
+        subliminal.region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
+                                    arguments={'filename': os.path.join(config.data_items_path, 'subzero.dbm'),
+                                               'lock_factory': MutexLock})
+    except:
+        Log.Warn("Not using file based cache!")
+        subliminal.region.configure('dogpile.cache.memory')
+
+    # clear expired intents
+    intent = get_intent()
+    intent.cleanup()
+
+    # clear expired menu history items
+    now = datetime.datetime.now()
+    if "menu_history" in Dict:
+        for key, timeout in Dict["menu_history"].items():
+            if now > timeout:
+                del Dict["menu_history"][key]
+
+    # run migrations
+    if "subs" in Dict or "history" in Dict:
+        Thread.Create(dispatch_migrate)
+
+    # clear old task data
+    scheduler.clear_task_data()
 
     # init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
     ValidatePrefs()
     Log.Debug(config.full_version)
 
-    if not config.plex_api_working:
-        Log.Error(lib_unaccessible_error)
-        return
-
     if not config.permissions_ok:
         Log.Error("Insufficient permissions on library folders:")
         for title, path in config.missing_permissions:
             Log.Error("Insufficient permissions on library %s, folder: %s" % (title, path))
-        return
 
+    # run task scheduler
     scheduler.run()
 
+    # bind activities
+    Thread.Create(activity.start)
 
-def initSubliminalPatches():
-    # configure custom subtitle destination folders for scanning pre-existing subs
-    dest_folder = config.subtitleDestinationFolder
-    subliminal_patch.patch_video.CUSTOM_PATHS = [dest_folder] if dest_folder else []
-    subliminal_patch.patch_provider_pool.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
-    subliminal_patch.patch_providers.addic7ed.USE_BOOST = bool(Prefs['provider.addic7ed.boost'])
+    if "anon_id" not in Dict:
+        Dict["anon_id"] = get_identifier()
+
+    # track usage
+    if cast_bool(Prefs["track_usage"]):
+        if "first_use" not in Dict:
+            Dict["first_use"] = datetime.datetime.utcnow()
+            Dict.Save()
+            track_usage("General", "plugin", "first_start", config.version)
+        track_usage("General", "plugin", "start", config.version)
 
 
-def flattenToParts(media, kind="series"):
-    """
-    iterates through media and returns the associated parts (videos)
-    :param media:
-    :param kind:
-    :return:
-    """
-    parts = []
-    if kind == "series":
-        for season in media.seasons:
-            for episode in media.seasons[season].episodes:
-                ep = media.seasons[season].episodes[episode]
-                for item in media.seasons[season].episodes[episode].items:
-                    for part in item.parts:
-                        parts.append({"video": part, "type": "episode", "title": ep.title, "series": media.title, "id": ep.id})
-    else:
-        for item in media.items:
-            for part in item.parts:
-                parts.append({"video": part, "type": "movie", "title": media.title, "id": media.id})
-    return parts
-
-
-def parseMediaToParts(media, kind="series"):
-    """
-    returns a list of parts to be used later on; ignores folders with an existing "subzero.ignore" file
-    :param media:
-    :param kind:
-    :return:
-    """
-    parts = flattenToParts(media, kind=kind)
-    if not Prefs["subtitles.ignore_fs"]:
-        return parts
-
-    use_parts = []
-    check_ignore_paths = [".", "../"]
-    if kind == "series":
-        check_ignore_paths.append("../../")
-
-    for part in parts:
-        base_folder, fn = os.path.split(part["video"].file)
-
-        ignore = False
-        for rel_path in check_ignore_paths:
-            fld = os.path.abspath(os.path.join(base_folder, rel_path))
-            for ifn in IGNORE_FN:
-                if os.path.isfile(os.path.join(fld, ifn)):
-                    Log.Info(u'Ignoring "%s" because "%s" exists in "%s"', fn, ifn, fld)
-                    ignore = True
-                    break
-            if ignore:
-                break
-
-        if not ignore:
-            use_parts.append(part)
-    return use_parts
-
-
-def getFPS(streams):
-    for stream in streams:
-        # video
-        if stream.type == 1:
-            return stream.frameRate
-    return "25.000"
-
-
-def scanParts(parts, kind="series"):
-    """
-    receives a list of parts containing dictionaries returned by flattenToParts
-    :param parts:
-    :param kind: series or movies
-    :return: dictionary of scanned videos of subliminal.video.scan_video
-    """
-    ret = {}
-    for part in parts:
-        force_refresh = intent.get("force", part["id"])
-        hints = {"expected_title": [part["title"]]}
-        hints.update({"type": "episode", "expected_series": [part["series"]]} if kind == "series" else {"type": "movie"})
-        part["video"].fps = getFPS(part["video"].streams)
-        scanned_video = scanVideo(part["video"], ignore_all=force_refresh, hints=hints)
-        if not scanned_video:
-            continue
-
-        scanned_video.id = part["id"]
-        ret[scanned_video] = part["video"]
-    return ret
-
-
-def getItemIDs(media, kind="series"):
-    ids = []
-    if kind == "movies":
-        ids.append(media.id)
-    else:
-        for season in media.seasons:
-            for episode in media.seasons[season].episodes:
-                ids.append(media.seasons[season].episodes[episode].id)
-
-    return ids
-
-
-def scanVideo(part, ignore_all=False, hints=None):
-    embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
-    external_subtitles = not ignore_all and Prefs['subtitles.scan.external']
-
-    if ignore_all:
-        Log.Debug("Force refresh intended.")
-
-    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (part.file, external_subtitles, embedded_subtitles))
-
-    try:
-        return subliminal.video.scan_video(part.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles, hints=hints or {},
-                                           video_fps=part.fps)
-
-    except ValueError:
-        Log.Warn("File could not be guessed by subliminal")
-
-
-def downloadBestSubtitles(video_part_map, min_score=0):
+def download_best_subtitles(video_part_map, min_score=0):
     hearing_impaired = Prefs['subtitles.search.hearingImpaired']
-    languages = config.langList
+    languages = config.lang_list
     if not languages:
         return
 
@@ -192,7 +106,7 @@ def downloadBestSubtitles(video_part_map, min_score=0):
     for video, part in video_part_map.iteritems():
         if not Prefs['subtitles.save.filesystem']:
             # scan for existing metadata subtitles
-            meta_subs = getSubtitlesFromMetadata(part)
+            meta_subs = get_subtitles_from_metadata(part)
             for language, subList in meta_subs.iteritems():
                 if subList:
                     video.subtitle_languages.add(language)
@@ -215,63 +129,18 @@ def downloadBestSubtitles(video_part_map, min_score=0):
     if missing_languages:
         Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" % (min_score, hearing_impaired))
 
-        return subliminal.api.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
-                                                      provider_configs=config.providerSettings)
+        return subliminal.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
+                                                  provider_configs=config.provider_settings, pool_class=config.provider_pool,
+                                                  compute_score=compute_score)
     Log.Debug("All languages for all requested videos exist. Doing nothing.")
 
 
-def saveSubtitles(videos, subtitles):
-    if Prefs['subtitles.save.filesystem']:
-        Log.Debug("Using filesystem as subtitle storage")
-        saveSubtitlesToFile(subtitles)
-        storage = "filesystem"
-    else:
-        Log.Debug("Using metadata as subtitle storage")
-        saveSubtitlesToMetadata(videos, subtitles)
-        storage = "metadata"
-
-    storeSubtitleInfo(videos, subtitles, storage)
-
-
-def saveSubtitlesToFile(subtitles):
-    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
-
-    for video, video_subtitles in subtitles.items():
-        if not video_subtitles:
-            continue
-
-        fld = None
-        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
-            # specific subFolder requested, create it if it doesn't exist
-            fld_base = os.path.split(video.name)[0]
-            if fld_custom:
-                if fld_custom.startswith("/"):
-                    # absolute folder
-                    fld = fld_custom
-                else:
-                    fld = os.path.join(fld_base, fld_custom)
-            else:
-                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
-            if not os.path.exists(fld):
-                os.makedirs(fld)
-        subliminal.api.save_subtitles(video, video_subtitles, directory=fld, single=Prefs['subtitles.only_one'],
-                                      encode_with=force_utf8 if Prefs['subtitles.enforce_encoding'] else None)
-
-
-def saveSubtitlesToMetadata(videos, subtitles):
-    for video, video_subtitles in subtitles.items():
-        mediaPart = videos[video]
-        for subtitle in video_subtitles:
-            content = force_utf8(subtitle.text) if Prefs['subtitles.enforce_encoding'] else subtitle.content
-            mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.page_link] = Proxy.Media(content, ext="srt")
-
-
-def updateLocalMedia(metadata, media, media_type="movies"):
+def update_local_media(metadata, media, media_type="movies"):
     # Look for subtitles
     if media_type == "movies":
         for item in media.items:
             for part in item.parts:
-                support.localmedia.findSubtitles(part)
+                support.localmedia.find_subtitles(part)
         return
 
     # Look for subtitles for each episode.
@@ -284,7 +153,7 @@ def updateLocalMedia(metadata, media, media_type="movies"):
 
                     # Look for subtitles.
                     for part in i.parts:
-                        support.localmedia.findSubtitles(part)
+                        support.localmedia.find_subtitles(part)
         else:
             pass
 
@@ -299,34 +168,74 @@ class SubZeroAgent(object):
     def __init__(self, *args, **kwargs):
         super(SubZeroAgent, self).__init__(*args, **kwargs)
         self.agent_type = "movies" if isinstance(self, Agent.Movies) else "series"
-        self.name = "Sub-Zero Subtitles (%s, %s)" % (self.agent_type_verbose, config.getVersion())
+        self.name = "Sub-Zero Subtitles (%s, %s)" % (self.agent_type_verbose, config.get_version())
 
     def search(self, results, media, lang):
         Log.Debug("Sub-Zero %s, %s search" % (config.version, self.agent_type))
         results.Append(MetadataSearchResult(id='null', score=100))
 
     def update(self, metadata, media, lang):
+        if not config.enable_agent:
+            Log.Debug("Skipping Sub-Zero agent(s)")
+            return
+
         Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))
+        intent = get_intent()
 
         if not media:
             Log.Error("Called with empty media, something is really wrong with your setup!")
             return
 
-        set_refresh_menu_state(media, media_type=self.agent_type)
-
         item_ids = []
         try:
-            initSubliminalPatches()
-            parts = parseMediaToParts(media, kind=self.agent_type)
-            use_score = Prefs[self.score_prefs_key]
-            scanned_parts = scanParts(parts, kind=self.agent_type)
-            subtitles = downloadBestSubtitles(scanned_parts, min_score=int(use_score))
-            item_ids = getItemIDs(media, kind=self.agent_type)
+            config.init_subliminal_patches()
+            videos = media_to_videos(media, kind=self.agent_type)
 
-            if subtitles:
-                saveSubtitles(scanned_parts, subtitles)
+            # media ignored?
+            use_any_parts = False
+            for video in videos:
+                if is_ignored(video["id"]):
+                    Log.Debug(u"Ignoring %s" % video)
+                    continue
+                use_any_parts = True
 
-            updateLocalMedia(metadata, media, media_type=self.agent_type)
+            if not use_any_parts:
+                Log.Debug(u"Nothing to do.")
+                return
+
+            try:
+                use_score = int(Prefs[self.score_prefs_key].strip())
+            except ValueError:
+                Log.Error("Please only put numbers into the scores setting. Exiting")
+                return
+
+            set_refresh_menu_state(media, media_type=self.agent_type)
+
+            # find local media
+            update_local_media(metadata, media, media_type=self.agent_type)
+
+            # scanned_video_part_map = {subliminal.Video: plex_part, ...}
+            scanned_video_part_map = scan_videos(videos, kind=self.agent_type)
+
+            # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
+            downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score)
+            item_ids = get_media_item_ids(media, kind=self.agent_type)
+
+            whack_missing_parts(scanned_video_part_map)
+
+            if downloaded_subtitles:
+                save_subtitles(scanned_video_part_map, downloaded_subtitles)
+                track_usage("Subtitle", "refreshed", "download", 1)
+
+                for video, video_subtitles in downloaded_subtitles.items():
+                    # store item(s) in history
+                    for subtitle in video_subtitles:
+                        item_title = get_title_for_video_metadata(video.plexapi_metadata, add_section_title=False)
+                        history = get_history()
+                        history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
+                                    subtitle=subtitle)
+
+            update_local_media(metadata, media, media_type=self.agent_type)
 
         finally:
             # update the menu state
@@ -338,17 +247,18 @@ class SubZeroAgent(object):
 
                 # resolve existing intent for that id
                 intent.resolve("force", item_id)
+
             Dict.Save()
 
 
 class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies):
     contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', 'com.plexapp.agents.hama']
-    score_prefs_key = "subtitles.search.minimumMovieScore"
+    score_prefs_key = "subtitles.search.minimumMovieScore2"
     agent_type_verbose = "Movies"
 
 
 class SubZeroSubtitlesAgentTvShows(SubZeroAgent, Agent.TV_Shows):
-    contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv',
-                      'com.plexapp.agents.hama']
-    score_prefs_key = "subtitles.search.minimumTVScore"
+    contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.themoviedb',
+                      'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv', 'com.plexapp.agents.hama']
+    score_prefs_key = "subtitles.search.minimumTVScore2"
     agent_type_verbose = "TV"

Contents/Code/interface/__init__.py

@@ -2,6 +2,19 @@ import sys
 
 import menu
 sys.modules["interface.menu"] = menu
+sys.modules["menu"] = menu
 
 import menu_helpers
-sys.modules["interface.menu_helpers"] = menu_helpers
+sys.modules["interface.menu_helpers"] = menu_helpers
+
+import advanced
+sys.modules["interface.advanced"] = advanced
+
+import main
+sys.modules["interface.main"] = main
+
+import refresh_item
+sys.modules["interface.refresh_item"] = refresh_item
+
+import item_details
+sys.modules["interface.item_details"] = item_details

Contents/Code/interface/advanced.py

@@ -0,0 +1,231 @@
+# coding=utf-8
+import datetime
+import StringIO
+import glob
+import os
+import urlparse
+
+from zipfile import ZipFile, ZIP_DEFLATED
+
+from subzero.lib.io import FileIO
+from subzero.constants import PREFIX, PLUGIN_IDENTIFIER
+from menu_helpers import SubFolderObjectContainer, debounce, set_refresh_menu_state, ZipObject
+from main import fatality
+from support.helpers import timestamp, pad_title
+from support.config import config
+from support.lib import Plex
+from support.storage import reset_storage, log_storage
+from support.background import scheduler
+
+
+@route(PREFIX + '/advanced')
+def AdvancedMenu(randomize=None, header=None, message=None):
+    oc = SubFolderObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True,
+                                  no_history=True,
+                                  replace_parent=False, title2="Advanced")
+
+    if config.lock_advanced_menu and not config.pin_correct:
+        oc.add(DirectoryObject(
+            key=Callback(PinMenu, randomize=timestamp(), success_go_to="advanced"),
+            title=pad_title("Enter PIN"),
+            summary="The owner has restricted the access to this menu. Please enter the correct pin",
+        ))
+        return oc
+
+    oc.add(DirectoryObject(
+        key=Callback(TriggerRestart, randomize=timestamp()),
+        title=pad_title("Restart the plugin"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(GetLogsLink),
+        title="Get my logs (copy the appearing link and open it in your browser, please)",
+        summary="Copy the appearing link and open it in your browser, please",
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(TriggerBetterSubtitles, randomize=timestamp()),
+        title=pad_title("Trigger find better subtitles"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(TriggerStorageMaintenance, randomize=timestamp()),
+        title=pad_title("Trigger subtitle storage maintenance"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(LogStorage, key="tasks", randomize=timestamp()),
+        title=pad_title("Log the plugin's scheduled tasks state storage"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(LogStorage, key="ignore", randomize=timestamp()),
+        title=pad_title("Log the plugin's internal ignorelist storage"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
+        title=pad_title("Reset the plugin's scheduled tasks state storage"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(ResetStorage, key="ignore", randomize=timestamp()),
+        title=pad_title("Reset the plugin's internal ignorelist storage"),
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(InvalidateCache, randomize=timestamp()),
+        title=pad_title("Invalidate Sub-Zero metadata caches (subliminal)"),
+    ))
+    return oc
+
+
+def DispatchRestart():
+    Thread.CreateTimer(1.0, Restart)
+
+
+@route(PREFIX + '/advanced/restart/trigger')
+@debounce
+def TriggerRestart(randomize=None):
+    set_refresh_menu_state("Restarting the plugin")
+    DispatchRestart()
+    return fatality(header="Restart triggered, please wait about 5 seconds", force_title=" ", only_refresh=True,
+                    replace_parent=True,
+                    no_history=True, randomize=timestamp())
+
+
+@route(PREFIX + '/advanced/restart/execute')
+def Restart():
+    Plex[":/plugins"].restart(PLUGIN_IDENTIFIER)
+
+
+@route(PREFIX + '/storage/reset', sure=bool)
+def ResetStorage(key, randomize=None, sure=False):
+    if not sure:
+        oc = SubFolderObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
+        oc.add(DirectoryObject(
+            key=Callback(ResetStorage, key=key, sure=True, randomize=timestamp()),
+            title=pad_title("Are you really sure?"),
+
+        ))
+        return oc
+
+    reset_storage(key)
+
+    if key == "tasks":
+        # reinitialize the scheduler
+        scheduler.init_storage()
+        scheduler.setup_tasks()
+
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='Information Storage (%s) reset' % key
+    )
+
+
+@route(PREFIX + '/storage/log')
+def LogStorage(key, randomize=None):
+    log_storage(key)
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='Information Storage (%s) logged' % key
+    )
+
+
+@route(PREFIX + '/triggerbetter')
+def TriggerBetterSubtitles(randomize=None):
+    scheduler.dispatch_task("FindBetterSubtitles")
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='FindBetterSubtitles triggered'
+    )
+
+
+@route(PREFIX + '/triggermaintenance')
+def TriggerStorageMaintenance(randomize=None):
+    scheduler.dispatch_task("SubtitleStorageMaintenance")
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='SubtitleStorageMaintenance triggered'
+    )
+
+
+@route(PREFIX + '/get_logs_link')
+def GetLogsLink():
+    # try getting the link base via the request in context, first, otherwise use the public ip
+    req_headers = Core.sandbox.context.request.headers
+
+    if "Origin" in req_headers:
+        link_base = req_headers["Origin"]
+        Log.Debug("Using origin-based link_base")
+
+    elif "Referer" in req_headers:
+        parsed = urlparse.urlparse(req_headers["Referer"])
+        link_base = "%s://%s:%s" % (parsed.scheme, parsed.hostname, parsed.port)
+        Log.Debug("Using referer-based link_base")
+
+    else:
+        ip = Core.networking.http_request("http://www.plexapp.com/ip.php", cacheTime=7200).content.strip()
+        link_base = "https://%s:32400" % ip
+        Log.Debug("Using ip-based fallback link_base")
+
+    logs_link = "%s%s?X-Plex-Token=%s" % (link_base, PREFIX + '/logs', config.universal_plex_token)
+    oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
+                         header="Copy this link and open this in your browser, please",
+                         message=logs_link)
+    return oc
+
+
+@route(PREFIX + '/logs')
+def DownloadLogs():
+    buff = StringIO.StringIO()
+    zip_archive = ZipFile(buff, mode='w', compression=ZIP_DEFLATED)
+
+    logs = sorted(glob.glob(config.plugin_log_path + '*')) + [config.server_log_path]
+    for path in logs:
+        data = StringIO.StringIO()
+        data.write(FileIO.read(path))
+        zip_archive.writestr(os.path.basename(path), data.getvalue())
+
+    zip_archive.close()
+
+    return ZipObject(buff.getvalue())
+
+
+@route(PREFIX + '/invalidatecache')
+def InvalidateCache(randomize=None):
+    from subliminal.cache import region
+    region.invalidate()
+    return AdvancedMenu(
+        randomize=timestamp(),
+        header='Success',
+        message='Cache invalidated'
+    )
+
+
+@route(PREFIX + '/pin')
+def PinMenu(pin="", randomize=None, success_go_to="channel"):
+    oc = ObjectContainer(title2="Enter PIN number %s" % (len(pin) + 1), no_cache=True, no_history=True,
+                         skip_pin_lock=True)
+
+    if pin == config.pin:
+        Dict["pin_correct_time"] = datetime.datetime.now()
+        config.locked = False
+        if success_go_to == "channel":
+            return fatality(force_title="PIN correct", header="PIN correct", no_history=True)
+        elif success_go_to == "advanced":
+            return AdvancedMenu(randomize=timestamp())
+
+    for i in range(10):
+        oc.add(DirectoryObject(
+            key=Callback(PinMenu, randomize=timestamp(), pin=pin + str(i), success_go_to=success_go_to),
+            title=pad_title(str(i)),
+        ))
+    oc.add(DirectoryObject(
+        key=Callback(PinMenu, randomize=timestamp(), success_go_to=success_go_to),
+        title=pad_title("Reset"),
+    ))
+    return oc
+
+
+@route(PREFIX + '/pin_lock')
+def ClearPin(randomize=None):
+    Dict["pin_correct_time"] = None
+    config.locked = True
+    return fatality(force_title="Menu locked", header=" ", no_history=True)

Contents/Code/interface/item_details.py

@@ -0,0 +1,201 @@
+# coding=utf-8
+import os
+
+from subzero.constants import PREFIX
+from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_ignore_options, get_item_task_data, \
+    set_refresh_menu_state
+from refresh_item import RefreshItem
+from support.helpers import timestamp, cast_bool, df, get_language
+from support.items import get_item_kind_from_rating_key, get_item
+from support.plex_media import get_plex_metadata, scan_videos
+from support.lib import Plex
+from support.storage import get_subtitle_storage
+from support.config import config
+from support.background import scheduler
+
+
+@route(PREFIX + '/item/{rating_key}/actions')
+@debounce
+def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None):
+    """
+    displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
+    :param rating_key:
+    :param title:
+    :param base_title:
+    :param item_title:
+    :param randomize:
+    :return:
+    """
+    from interface.main import IgnoreMenu
+
+    title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title)
+    item = get_item(rating_key)
+    current_kind = get_item_kind_from_rating_key(rating_key)
+
+    timeout = 30
+
+    oc = SubFolderObjectContainer(title2=title, replace_parent=True)
+    oc.add(DirectoryObject(
+        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp(),
+                     timeout=timeout * 1000),
+        title=u"Refresh: %s" % item_title,
+        summary="Refreshes the %s, possibly searching for missing and picking up new subtitles on disk" % current_kind,
+        thumb=item.thumb or default_thumb
+    ))
+    oc.add(DirectoryObject(
+        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(),
+                     timeout=timeout * 1000),
+        title=u"Auto-search: %s" % item_title,
+        summary="Issues a forced refresh, ignoring known subtitles and searching for new ones",
+        thumb=item.thumb or default_thumb
+    ))
+
+    # get stored subtitle info for item id
+    subtitle_storage = get_subtitle_storage()
+    stored_subs = subtitle_storage.load_or_new(item)
+
+    # get the plex item
+    plex_item = list(Plex["library"].metadata(rating_key))[0]
+
+    # get current media info for that item
+    media = plex_item.media
+
+    # look for subtitles for all available media parts and all of their languages
+    for part in media.parts:
+        filename = os.path.basename(part.file)
+        part_id = str(part.id)
+
+        # iterate through all configured languages
+        for lang in config.lang_list:
+            lang_a2 = lang.alpha2
+            # ietf lang?
+            if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
+                lang_a2 = lang_a2.split("-")[0]
+
+            # get corresponding stored subtitle data for that media part (physical media item), for language
+            current_sub = stored_subs.get_any(part_id, lang_a2)
+            current_sub_id = None
+            current_sub_provider_name = None
+
+            summary = u"No current subtitle in storage"
+            current_score = None
+            if current_sub:
+                current_sub_id = current_sub.id
+                current_sub_provider_name = current_sub.provider_name
+                current_score = current_sub.score
+
+                summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
+                          (current_sub.provider_name, df(current_sub.date_added), current_sub.mode_verbose, lang,
+                           current_sub.score, current_sub.storage_type)
+
+            oc.add(DirectoryObject(
+                key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title,
+                             item_title=item_title, language=lang, current_id=current_sub_id,
+                             item_type=plex_item.type, filename=filename, current_data=summary,
+                             randomize=timestamp(), current_provider=current_sub_provider_name,
+                             current_score=current_score),
+                title=u"List %s subtitles" % lang.name,
+                summary=summary
+            ))
+
+    add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
+
+    return oc
+
+
+@route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool)
+@debounce
+def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None,
+                                 item_type="episode", language=None, force=False, current_id=None, current_data=None,
+                                 current_provider=None, current_score=None, randomize=None):
+    assert rating_key, part_id
+
+    running = scheduler.is_task_running("AvailableSubsForItem")
+    search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
+
+    if (search_results is None or force) and not running:
+        scheduler.dispatch_task("AvailableSubsForItem", rating_key=rating_key, item_type=item_type, part_id=part_id,
+                                language=language)
+        running = True
+
+    oc = SubFolderObjectContainer(title2=unicode(title), replace_parent=True)
+    oc.add(DirectoryObject(
+        key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()),
+        title=u"Back to: %s" % title,
+        summary=current_data,
+        thumb=default_thumb
+    ))
+
+    metadata = get_plex_metadata(rating_key, part_id, item_type)
+    scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+
+    if not scanned_parts:
+        Log.Error("Couldn't list available subtitles for %s", rating_key)
+        return oc
+
+    video, plex_part = scanned_parts.items()[0]
+
+    video_display_data = [video.format] if video.format else []
+    if video.release_group:
+        video_display_data.append(u"by %s" % video.release_group)
+    video_display_data = " ".join(video_display_data)
+
+    current_display = (u"Current: %s (%s) " % (current_provider, current_score) if current_provider else "")
+    if not running:
+        oc.add(DirectoryObject(
+            key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, language=language,
+                         filename=filename, part_id=part_id, title=title, current_id=current_id, force=True,
+                         current_provider=current_provider, current_score=current_score,
+                         current_data=current_data, item_type=item_type, randomize=timestamp()),
+            title=u"Search for %s subs (%s)" % (get_language(language).name, video_display_data),
+            summary=u"%sFilename: %s" % (current_display, filename),
+            thumb=default_thumb
+        ))
+    else:
+        oc.add(DirectoryObject(
+            key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
+                         language=language, filename=filename, current_data=current_data,
+                         part_id=part_id, title=title, current_id=current_id, item_type=item_type,
+                         current_provider=current_provider, current_score=current_score,
+                         randomize=timestamp()),
+            title=u"Searching for %s subs (%s), refresh here ..." % (get_language(language).name, video_display_data),
+            summary=u"%sFilename: %s" % (current_display, filename),
+            thumb=default_thumb
+        ))
+
+    if not search_results:
+        return oc
+
+    for subtitle in search_results:
+        oc.add(DirectoryObject(
+            key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title,
+                         subtitle_id=str(subtitle.id), language=language),
+            title=u"%s: %s, score: %s" % ("Available" if current_id != subtitle.id else "Current",
+                                          subtitle.provider_name, subtitle.score),
+            summary=u"Release: %s, Matches: %s" % (subtitle.release_info, ", ".join(subtitle.matches)),
+            thumb=default_thumb
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/download_subtitle/{rating_key}')
+@debounce
+def TriggerDownloadSubtitle(rating_key=None, subtitle_id=None, item_title=None, language=None, randomize=None):
+    from interface.main import fatality
+
+    set_refresh_menu_state("Downloading subtitle for %s" % item_title or rating_key)
+    search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
+
+    download_subtitle = None
+    for subtitle in search_results:
+        if str(subtitle.id) == subtitle_id:
+            download_subtitle = subtitle
+            break
+    if not download_subtitle:
+        Log.Error(u"Something went horribly wrong")
+
+    else:
+        scheduler.dispatch_task("DownloadSubtitleForItem", rating_key=rating_key, subtitle=download_subtitle)
+
+    return fatality(randomize=timestamp(), header=" ", replace_parent=True)

Contents/Code/interface/main.py

@@ -0,0 +1,387 @@
+# coding=utf-8
+
+from subzero.constants import PREFIX, TITLE, ART
+from support.config import config
+from support.helpers import pad_title, timestamp, df
+from support.background import scheduler
+from support.ignore import ignore_list
+from support.items import get_item_thumb, get_on_deck_items, get_all_items, get_items_info
+from menu_helpers import main_icon, debounce, SubFolderObjectContainer, default_thumb, dig_tree, add_ignore_options
+from item_details import ItemDetailsMenu
+
+
+@handler(PREFIX, TITLE if not config.is_development else TITLE + " DEV", art=ART, thumb=main_icon)
+@route(PREFIX)
+def fatality(randomize=None, force_title=None, header=None, message=None, only_refresh=False, no_history=False,
+             replace_parent=False):
+    """
+    subzero main menu
+    """
+    from interface.advanced import PinMenu, ClearPin, AdvancedMenu
+    from interface.menu import RefreshMissing, IgnoreListMenu, HistoryMenu
+
+    title = config.full_version  # force_title if force_title is not None else config.full_version
+    oc = ObjectContainer(title1=title, title2=title, header=unicode(header) if header else title, message=message,
+                         no_history=no_history,
+                         replace_parent=replace_parent, no_cache=True)
+
+    # always re-check permissions
+    config.refresh_permissions_status()
+
+    # always re-check enabled sections
+    config.refresh_enabled_sections()
+
+    if config.lock_menu and not config.pin_correct:
+        oc.add(DirectoryObject(
+            key=Callback(PinMenu, randomize=timestamp()),
+            title=pad_title("Enter PIN"),
+            summary="The owner has restricted the access to this menu. Please enter the correct pin",
+        ))
+        return oc
+
+    if not config.permissions_ok and config.missing_permissions:
+        for title, path in config.missing_permissions:
+            oc.add(DirectoryObject(
+                key=Callback(fatality, randomize=timestamp()),
+                title=pad_title("Insufficient permissions"),
+                summary="Insufficient permissions on library %s, folder: %s" % (title, path),
+            ))
+        return oc
+
+    if not config.enabled_sections:
+        oc.add(DirectoryObject(
+            key=Callback(fatality, randomize=timestamp()),
+            title=pad_title("I'm not enabled!"),
+            summary="Please enable me for some of your libraries in your server settings; currently I do nothing",
+        ))
+        return oc
+
+    if not only_refresh:
+        if Dict["current_refresh_state"]:
+            oc.add(DirectoryObject(
+                key=Callback(fatality, force_title=" ", randomize=timestamp()),
+                title=pad_title("Working ... refresh here"),
+                summary="Current state: %s; Last state: %s" % (
+                    (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
+                    (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
+                )
+            ))
+
+        oc.add(DirectoryObject(
+            key=Callback(OnDeckMenu),
+            title="On Deck items",
+            summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/"
+                    "subtitles.",
+            thumb=R("icon-ondeck.jpg")
+        ))
+        oc.add(DirectoryObject(
+            key=Callback(RecentlyAddedMenu),
+            title="Recently Added items",
+            summary="Shows the recently added items per section.",
+            thumb=R("icon-recent.jpg")
+        ))
+        oc.add(DirectoryObject(
+            key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()),
+            title="Items with missing subtitles",
+            summary="Shows the items honoring the configured 'Item age to be considered recent'-setting (%s)"
+                    " and allowing you to individually (force-) refresh their metadata/subtitles. " %
+                    Prefs["scheduler.item_is_recent_age"],
+            thumb=R("icon-missing.jpg")
+        ))
+        oc.add(DirectoryObject(
+            key=Callback(SectionsMenu),
+            title="Browse all items",
+            summary="Go through your whole library and manage your ignore list. You can also "
+                    "(force-) refresh the metadata/subtitles of individual items.",
+            thumb=R("icon-browse.jpg")
+        ))
+
+        task_name = "SearchAllRecentlyAddedMissing"
+        task = scheduler.task(task_name)
+
+        if task.ready_for_display:
+            task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
+        else:
+            task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (
+                df(scheduler.last_run(task_name)) or "never",
+                df(scheduler.next_run(task_name)) or "never",
+                str(task.last_run_time).split(".")[0])
+
+        oc.add(DirectoryObject(
+            key=Callback(RefreshMissing, randomize=timestamp()),
+            title="Search for missing subtitles (in recently-added items, max-age: %s)" % Prefs[
+                "scheduler.item_is_recent_age"],
+            summary="Automatically run periodically by the scheduler, if configured. %s" % task_state,
+            thumb=R("icon-search.jpg")
+        ))
+
+        oc.add(DirectoryObject(
+            key=Callback(IgnoreListMenu),
+            title="Display ignore list (%d)" % len(ignore_list),
+            summary="Show the current ignore list (mainly used for the automatic tasks)",
+            thumb=R("icon-ignore.jpg")
+        ))
+
+        oc.add(DirectoryObject(
+            key=Callback(HistoryMenu),
+            title="History",
+            summary="Show the last %i downloaded subtitles" % int(Prefs["history_size"]),
+            thumb=R("icon-history.jpg")
+        ))
+
+    oc.add(DirectoryObject(
+        key=Callback(fatality, force_title=" ", randomize=timestamp()),
+        title=pad_title("Refresh"),
+        summary="Current state: %s; Last state: %s" % (
+            (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
+            (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
+        ),
+        thumb=R("icon-refresh.jpg")
+    ))
+
+    # add re-lock after pin unlock
+    if config.pin:
+        oc.add(DirectoryObject(
+            key=Callback(ClearPin, randomize=timestamp()),
+            title=pad_title("Re-lock menu(s)"),
+            summary="Enabled the PIN again for menu(s)"
+        ))
+
+    if not only_refresh:
+        oc.add(DirectoryObject(
+            key=Callback(AdvancedMenu),
+            title=pad_title("Advanced functions"),
+            summary="Use at your own risk",
+            thumb=R("icon-advanced.jpg")
+        ))
+
+    return oc
+
+
+@route(PREFIX + '/on_deck')
+def OnDeckMenu(message=None):
+    """
+    displays the items on deck
+    :param message:
+    :return:
+    """
+    return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=get_on_deck_items)
+
+
+@route(PREFIX + '/recently_added')
+def RecentlyAddedMenu(message=None):
+    """
+    displays the items recently added per section
+    :param message:
+    :return:
+    """
+    return SectionsMenu(base_title="Recently added", section_items_key="recently_added", ignore_options=False)
+
+
+@route(PREFIX + '/recent', force=bool)
+@debounce
+def RecentMissingSubtitlesMenu(force=False, randomize=None):
+    title = "Items with missing subtitles"
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
+
+    running = scheduler.is_task_running("MissingSubtitles")
+    task_data = scheduler.get_task_data("MissingSubtitles")
+    missing_items = task_data["missing_subtitles"] if task_data else None
+
+    if ((missing_items is None) or force) and not running:
+        scheduler.dispatch_task("MissingSubtitles")
+        running = True
+
+    if not running:
+        oc.add(DirectoryObject(
+            key=Callback(RecentMissingSubtitlesMenu, force=True, randomize=timestamp()),
+            title=u"Get items with missing subtitles",
+            thumb=default_thumb
+        ))
+    else:
+        oc.add(DirectoryObject(
+            key=Callback(RecentMissingSubtitlesMenu, force=False, randomize=timestamp()),
+            title=u"Updating, refresh here ...",
+            thumb=default_thumb
+        ))
+
+    if missing_items is not None:
+        for added_at, item_id, item_title, item, missing_languages in missing_items:
+            oc.add(DirectoryObject(
+                key=Callback(ItemDetailsMenu, title=title + " > " + item_title, item_title=item_title,
+                             rating_key=item_id),
+                title=item_title,
+                summary="Missing: %s" % ", ".join(l.name for l in missing_languages),
+                thumb=get_item_thumb(item) or default_thumb
+            ))
+
+        scheduler.clear_task_data("MissingSubtitles")
+
+    return oc
+
+
+def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *args, **kwargs):
+    """
+    displays an item list of dynamic kinds of items
+    :param title:
+    :param itemGetter:
+    :param itemGetterKwArgs:
+    :param base_title:
+    :param args:
+    :param kwargs:
+    :return:
+    """
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
+    items = itemGetter(*args, **kwargs)
+
+    for kind, title, item_id, deeper, item in items:
+        oc.add(DirectoryObject(
+            title=title,
+            key=Callback(ItemDetailsMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id),
+            thumb=get_item_thumb(item) or default_thumb
+        ))
+
+    return oc
+
+
+def determine_section_display(kind, item, pass_kwargs=None):
+    """
+    returns the menu function for a section based on the size of it (amount of items)
+    :param kind:
+    :param item:
+    :return:
+    """
+    if pass_kwargs and pass_kwargs.get("section_items_key", "all") != "all":
+        return SectionMenu
+    if item.size > 80:
+        return SectionFirstLetterMenu
+    return SectionMenu
+
+
+@route(PREFIX + '/ignore/set/{kind}/{rating_key}/{todo}/sure={sure}', kind=str, rating_key=str, todo=str, sure=bool)
+def IgnoreMenu(kind, rating_key, title=None, sure=False, todo="not_set"):
+    """
+    displays the ignore options for a menu
+    :param kind:
+    :param rating_key:
+    :param title:
+    :param sure:
+    :param todo:
+    :return:
+    """
+    is_ignored = rating_key in ignore_list[kind]
+    if not sure:
+        oc = SubFolderObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
+            "Add" if not is_ignored else "Remove", ignore_list.verbose(kind), title,
+            "to" if not is_ignored else "from"), title2="Are you sure?")
+        oc.add(DirectoryObject(
+            key=Callback(IgnoreMenu, kind=kind, rating_key=rating_key, title=title, sure=True,
+                         todo="add" if not is_ignored else "remove"),
+            title=pad_title("Are you sure?"),
+        ))
+        return oc
+
+    rel = ignore_list[kind]
+    dont_change = False
+    if todo == "remove":
+        if not is_ignored:
+            dont_change = True
+        else:
+            rel.remove(rating_key)
+            Log.Info("Removed %s (%s) from the ignore list", title, rating_key)
+            ignore_list.remove_title(kind, rating_key)
+            ignore_list.save()
+            state = "removed from"
+    elif todo == "add":
+        if is_ignored:
+            dont_change = True
+        else:
+            rel.append(rating_key)
+            Log.Info("Added %s (%s) to the ignore list", title, rating_key)
+            ignore_list.add_title(kind, rating_key, title)
+            ignore_list.save()
+            state = "added to"
+    else:
+        dont_change = True
+
+    if dont_change:
+        return fatality(force_title=" ", header="Didn't change the ignore list", no_history=True)
+
+    return fatality(force_title=" ", header="%s %s the ignore list" % (title, state), no_history=True)
+
+
+@route(PREFIX + '/sections')
+def SectionsMenu(base_title="Sections", section_items_key="all", ignore_options=True):
+    """
+    displays the menu for all sections
+    :return:
+    """
+    items = get_all_items("sections")
+
+    return dig_tree(SubFolderObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
+                    menu_determination_callback=determine_section_display, pass_kwargs={"base_title": base_title,
+                                                                                        "section_items_key": section_items_key,
+                                                                                        "ignore_options": ignore_options},
+                    fill_args={"title": "section_title"})
+
+
+@route(PREFIX + '/section', ignore_options=bool)
+def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
+                section_items_key="all"):
+    """
+    displays the contents of a section
+    :param section_items_key:
+    :param rating_key:
+    :param title:
+    :param base_title:
+    :param section_title:
+    :param ignore_options:
+    :return:
+    """
+    from menu import MetadataMenu
+    items = get_all_items(key=section_items_key, value=rating_key, base="library/sections")
+
+    kind, deeper = get_items_info(items)
+    title = unicode(title)
+
+    section_title = title
+    title = base_title + " > " + title
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
+    if ignore_options:
+        add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
+
+    return dig_tree(oc, items, MetadataMenu,
+                    pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": "section",
+                                 "previous_rating_key": rating_key})
+
+
+@route(PREFIX + '/section/firstLetter', deeper=bool)
+def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
+                           section_items_key="all"):
+    """
+    displays the contents of a section indexed by its first char (A-Z, 0-9...)
+    :param ignore_options: ignored
+    :param section_items_key: ignored
+    :param rating_key:
+    :param title:
+    :param base_title:
+    :param section_title:
+    :return:
+    """
+    from menu import FirstLetterMetadataMenu
+    items = get_all_items(key="first_character", value=rating_key, base="library/sections")
+
+    kind, deeper = get_items_info(items)
+
+    title = unicode(title)
+    oc = SubFolderObjectContainer(title2=section_title, no_cache=True, no_history=True)
+    title = base_title + " > " + title
+    add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
+
+    oc.add(DirectoryObject(
+        key=Callback(SectionMenu, title="All", base_title=title, rating_key=rating_key, ignore_options=False),
+        title="All"
+    )
+    )
+    return dig_tree(oc, items, FirstLetterMetadataMenu, force_rating_key=rating_key, fill_args={"key": "key"},
+                    pass_kwargs={"base_title": title, "display_items": deeper, "previous_rating_key": rating_key})

Contents/Code/interface/menu.py

@@ -1,263 +1,39 @@
 # coding=utf-8
 import logging
-
 import logger
-from menu_helpers import add_ignore_options, dig_tree, set_refresh_menu_state, should_display_ignore, enable_channel_wrapper
-from subzero.constants import TITLE, ART, ICON, PREFIX, PLUGIN_IDENTIFIER, DEPENDENCY_MODULE_NAMES
-from support.auth import refresh_plex_token
+
+from item_details import ItemDetailsMenu
+from refresh_item import RefreshItem
+from menu_helpers import add_ignore_options, dig_tree, set_refresh_menu_state, \
+    should_display_ignore, enable_channel_wrapper, default_thumb, debounce, ObjectContainer, SubFolderObjectContainer
+from main import fatality, IgnoreMenu
+from advanced import DispatchRestart
+from subzero.constants import ART, PREFIX, DEPENDENCY_MODULE_NAMES
 from support.background import scheduler
 from support.config import config
-from support.helpers import pad_title, timestamp
+from support.helpers import timestamp,  df
 from support.ignore import ignore_list
-from support.items import getOnDeckItems, refreshItem, getAllItems
-from support.items import getRecentItems, get_items_info
-from support.lib import Plex, lib_unaccessible_error
-from support.missing_subtitles import getAllMissing
-from support.storage import resetStorage, logStorage
+from support.items import get_all_items, get_items_info, \
+    get_item_kind_from_rating_key
 
 # init GUI
 ObjectContainer.art = R(ART)
 ObjectContainer.no_cache = True
 
+# default thumb for DirectoryObjects
+DirectoryObject.thumb = default_thumb
+
 # noinspection PyUnboundLocalVariable
 route = enable_channel_wrapper(route)
 # noinspection PyUnboundLocalVariable
 handler = enable_channel_wrapper(handler)
 
 
-@handler(PREFIX, TITLE, art=ART, thumb=ICON)
-@route(PREFIX)
-def fatality(randomize=None, force_title=None, header=None, message=None, only_refresh=False, no_history=False, replace_parent=False):
-    """
-    subzero main menu
-    """
-    title = force_title if force_title is not None else config.full_version
-    oc = ObjectContainer(title1=title, title2=None, header=unicode(header) if header else header, message=message, no_history=no_history,
-                         replace_parent=replace_parent)
-
-    if not config.plex_api_working:
-        oc.add(DirectoryObject(
-            key=Callback(fatality, randomize=timestamp()),
-            title=pad_title("PMS API ERROR"),
-            summary=lib_unaccessible_error
-        ))
-        return oc
-
-    if not config.permissions_ok:
-        for title, path in config.missing_permissions:
-            oc.add(DirectoryObject(
-                key=Callback(fatality, randomize=timestamp()),
-                title=pad_title("Insufficient permissions"),
-                summary="Insufficient permissions on library %s, folder: %s" % (title, path)
-            ))
-        return oc
-
-    if not only_refresh:
-        if Dict["current_refresh_state"]:
-            oc.add(DirectoryObject(
-                key=Callback(fatality, force_title=" ", randomize=timestamp()),
-                title=pad_title("Working ... refresh here"),
-                summary="Current state: %s; Last state: %s" % (
-                    (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
-                    (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
-                )
-            ))
-
-        oc.add(DirectoryObject(
-            key=Callback(OnDeckMenu),
-            title=pad_title("On Deck items"),
-            summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/subtitles."
-        ))
-        oc.add(DirectoryObject(
-            key=Callback(RecentlyAddedMenu),
-            title="Items with missing subtitles",
-            summary="Shows the items honoring the configured 'Item age to be considered recent'-setting (%s)"
-                    " and allowing you to individually (force-) refresh their metadata/subtitles. " % Prefs["scheduler.item_is_recent_age"]
-        ))
-        oc.add(DirectoryObject(
-            key=Callback(SectionsMenu),
-            title="Browse all items",
-            summary="Go through your whole library and manage your ignore list. You can also "
-                    "(force-) refresh the metadata/subtitles of individual items."
-        ))
-
-        task_name = "searchAllRecentlyAddedMissing"
-        task = scheduler.task(task_name)
-
-        if task.ready_for_display:
-            task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
-        else:
-            task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (scheduler.last_run(task_name) or "never",
-                                                                                               scheduler.next_run(task_name) or "never",
-                                                                                               str(task.last_run_time).split(".")[0])
-
-        oc.add(DirectoryObject(
-            key=Callback(RefreshMissing),
-            title="Search for missing subtitles (in recently-added items, max-age: %s)" % Prefs["scheduler.item_is_recent_age"],
-            summary="Automatically run periodically by the scheduler, if configured. %s" % task_state
-        ))
-
-        oc.add(DirectoryObject(
-            key=Callback(IgnoreListMenu),
-            title="Display ignore list (%d)" % len(ignore_list),
-            summary="Show the current ignore list (mainly used for the automatic tasks)"
-        ))
-
-    oc.add(DirectoryObject(
-        key=Callback(fatality, force_title=" ", randomize=timestamp()),
-        title=pad_title("Refresh"),
-        summary="Current state: %s; Last state: %s" % (
-            (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
-            (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
-        )
-    ))
-
-    if not only_refresh:
-        oc.add(DirectoryObject(
-            key=Callback(AdvancedMenu),
-            title=pad_title("Advanced functions"),
-            summary="Use at your own risk"
-        ))
-
-    return oc
-
-
-@route(PREFIX + '/on_deck')
-def OnDeckMenu(message=None):
-    return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=getOnDeckItems)
-
-
-@route(PREFIX + '/recent')
-def RecentlyAddedMenu(message=None):
-    return recentItemsMenu(title="Missing Subtitles", base_title="Missing Subtitles")
-
-
-def recentItemsMenu(title, base_title=None):
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
-    recent_items = getRecentItems()
-    if recent_items:
-        missing_items = getAllMissing(recent_items)
-        if missing_items:
-            for added_at, item_id, title in missing_items:
-                oc.add(DirectoryObject(
-                    key=Callback(RefreshItemMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id), title=title
-                ))
-
-    return oc
-
-
-def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *args, **kwargs):
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
-    items = itemGetter(*args, **kwargs)
-
-    for kind, title, item_id, deeper, item in items:
-        oc.add(DirectoryObject(
-            title=title,
-            key=Callback(RefreshItemMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id)
-        ))
-
-    return oc
-
-
-def determine_section_display(kind, item):
-    if item.size > 200:
-        return SectionFirstLetterMenu
-    return SectionMenu
-
-
-@route(PREFIX + '/ignore/set/{kind}/{rating_key}/{todo}/sure={sure}', kind=str, rating_key=str, todo=str, sure=bool)
-def IgnoreMenu(kind, rating_key, title=None, sure=False, todo="not_set"):
-    is_ignored = rating_key in ignore_list[kind]
-    if not sure:
-        oc = ObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
-            "Add" if not is_ignored else "Remove", ignore_list.verbose(kind), title, "to" if not is_ignored else "from"), title2="Are you sure?")
-        oc.add(DirectoryObject(
-            key=Callback(IgnoreMenu, kind=kind, rating_key=rating_key, title=title, sure=True, todo="add" if not is_ignored else "remove"),
-            title=pad_title("Are you sure?")
-        ))
-        return oc
-
-    rel = ignore_list[kind]
-    dont_change = False
-    if todo == "remove":
-        if not is_ignored:
-            dont_change = True
-        else:
-            rel.remove(rating_key)
-            Log.Info("Removed %s (%s) from the ignore list", title, rating_key)
-            ignore_list.remove_title(kind, rating_key)
-            ignore_list.save()
-            state = "removed from"
-    elif todo == "add":
-        if is_ignored:
-            dont_change = True
-        else:
-            rel.append(rating_key)
-            Log.Info("Added %s (%s) to the ignore list", title, rating_key)
-            ignore_list.add_title(kind, rating_key, title)
-            ignore_list.save()
-            state = "added to"
-    else:
-        dont_change = True
-
-    if dont_change:
-        return fatality(force_title=" ", header="Didn't change the ignore list", no_history=True)
-
-    return fatality(force_title=" ", header="%s %s the ignore list" % (title, state), no_history=True)
-
-
-@route(PREFIX + '/sections')
-def SectionsMenu():
-    items = getAllItems("sections")
-
-    return dig_tree(ObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
-                    menu_determination_callback=determine_section_display, pass_kwargs={"base_title": "Sections"},
-                    fill_args={"title": "section_title"})
-
-
-@route(PREFIX + '/section', ignore_options=bool)
-def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True):
-    items = getAllItems(key="all", value=rating_key, base="library/sections")
-
-    kind, deeper = get_items_info(items)
-    title = unicode(title)
-
-    section_title = title
-    title = base_title + " > " + title
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
-    if ignore_options:
-        add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
-
-    return dig_tree(oc, items, MetadataMenu,
-                    pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": "section",
-                                 "previous_rating_key": rating_key})
-
-
-@route(PREFIX + '/section/firstLetter', deeper=bool)
-def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_title=None):
-    items = getAllItems(key="first_character", value=rating_key, base="library/sections")
-
-    kind, deeper = get_items_info(items)
-
-    title = unicode(title)
-    oc = ObjectContainer(title2=section_title, no_cache=True, no_history=True)
-    title = base_title + " > " + title
-    add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
-
-    oc.add(DirectoryObject(
-        key=Callback(SectionMenu, title="All", base_title=title, rating_key=rating_key, ignore_options=False),
-        title="All"
-    )
-    )
-    return dig_tree(oc, items, FirstLetterMetadataMenu, force_rating_key=rating_key, fill_args={"key": "key"},
-                    pass_kwargs={"base_title": title, "display_items": deeper, "previous_rating_key": rating_key})
-
-
 @route(PREFIX + '/section/firstLetter/key', deeper=bool)
 def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, display_items=False, previous_item_type=None,
                             previous_rating_key=None):
     """
-
+    displays the contents of a section filtered by the first letter
     :param rating_key: actually is the section's key
     :param key: the firstLetter wanted
     :param title: the first letter, or #
@@ -265,118 +41,107 @@ def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, displa
     :return:
     """
     title = base_title + " > " + unicode(title)
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
 
-    items = getAllItems(key="first_character", value=[rating_key, key], base="library/sections", flat=False)
+    items = get_all_items(key="first_character", value=[rating_key, key], base="library/sections", flat=False)
     kind, deeper = get_items_info(items)
     dig_tree(oc, items, MetadataMenu,
-             pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind, "previous_rating_key": rating_key})
+             pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind,
+                          "previous_rating_key": rating_key})
     return oc
 
 
 @route(PREFIX + '/section/contents', display_items=bool)
-def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None, previous_rating_key=None):
+def MetadataMenu(rating_key, title=None, base_title=None, display_items=False, previous_item_type=None,
+                 previous_rating_key=None):
+    """
+    displays the contents of a section based on whether it has a deeper tree or not (movies->movie (item) list; series->series list)
+    :param rating_key:
+    :param title:
+    :param base_title:
+    :param display_items:
+    :param previous_item_type:
+    :param previous_rating_key:
+    :return:
+    """
     title = unicode(title)
     item_title = title
     title = base_title + " > " + title
-    oc = ObjectContainer(title2=title, no_cache=True, no_history=True)
+    oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
+
+    current_kind = get_item_kind_from_rating_key(rating_key)
 
     if display_items:
-        items = getAllItems(key="children", value=rating_key, base="library/metadata")
+        items = get_all_items(key="children", value=rating_key, base="library/metadata")
         kind, deeper = get_items_info(items)
         dig_tree(oc, items, MetadataMenu,
-                 pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind, "previous_rating_key": rating_key})
+                 pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": kind,
+                              "previous_rating_key": rating_key})
         # we don't know exactly where we are here, only add ignore option to series
         if should_display_ignore(items, previous=previous_item_type):
             add_ignore_options(oc, "series", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
+
+        timeout = 30
+        if current_kind == "season":
+            timeout = 360
+        elif current_kind == "series":
+            timeout = 1800
+
+        # add refresh
+        oc.add(DirectoryObject(
+            key=Callback(RefreshItem, rating_key=rating_key, item_title=title, refresh_kind=current_kind,
+                         previous_rating_key=previous_rating_key, timeout=timeout * 1000, randomize=timestamp()),
+            title=u"Refresh: %s" % item_title,
+            summary="Refreshes the %s, possibly searching for missing and picking up new subtitles on disk" % current_kind
+        ))
+        oc.add(DirectoryObject(
+            key=Callback(RefreshItem, rating_key=rating_key, item_title=title, force=True,
+                         refresh_kind=current_kind, previous_rating_key=previous_rating_key, timeout=timeout * 1000,
+                         randomize=timestamp()),
+            title=u"Auto-Find subtitles: %s" % item_title,
+            summary="Issues a forced refresh, ignoring known subtitles and searching for new ones"
+        ))
     else:
-        return RefreshItemMenu(rating_key=rating_key, title=title, item_title=item_title)
+        return ItemDetailsMenu(rating_key=rating_key, title=title, item_title=item_title)
 
     return oc
 
 
 @route(PREFIX + '/ignore_list')
 def IgnoreListMenu():
-    oc = ObjectContainer(title2="Ignore list", replace_parent=True)
+    oc = SubFolderObjectContainer(title2="Ignore list", replace_parent=True)
     for key in ignore_list.key_order:
         values = ignore_list[key]
         for value in values:
-            add_ignore_options(oc, key, title=ignore_list.get_title(key, value), rating_key=value, callback_menu=IgnoreMenu)
+            add_ignore_options(oc, key, title=ignore_list.get_title(key, value), rating_key=value,
+                               callback_menu=IgnoreMenu)
     return oc
 
 
-@route(PREFIX + '/item/{rating_key}/actions')
-def RefreshItemMenu(rating_key, title=None, base_title=None, item_title=None, came_from="/recent"):
-    title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title)
-    oc = ObjectContainer(title2=title, replace_parent=True)
-    oc.add(DirectoryObject(
-        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title),
-        title=u"Refresh: %s" % item_title,
-        summary="Refreshes the item, possibly picking up new subtitles on disk"
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True),
-        title=u"Force-Refresh: %s" % item_title,
-        summary="Issues a forced refresh, ignoring known subtitles and searching for new ones"
-    ))
-    add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
+@route(PREFIX + '/history')
+def HistoryMenu():
+    from support.history import get_history
+    history = get_history()
+    oc = SubFolderObjectContainer(title2="History", replace_parent=True)
+
+    for item in history.history_items:
+        oc.add(DirectoryObject(
+            key=Callback(ItemDetailsMenu, title=item.title, item_title=item.item_title,
+                         rating_key=item.rating_key),
+            title=u"%s (%s)" % (item.item_title, item.mode_verbose),
+            summary=u"%s in %s (%s, score: %s), %s" % (item.lang_name, item.section_title,
+                                                       item.provider_name, item.score, df(item.time))
+        ))
 
     return oc
 
 
-@route(PREFIX + '/item/{rating_key}')
-def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False):
-    assert rating_key
-    set_refresh_menu_state(u"Triggering %sRefresh for %s" % ("Force-" if force else "", item_title))
-    Thread.Create(refreshItem, rating_key=rating_key, force=force)
-    return fatality(randomize=timestamp(), header=u"%s of item %s triggered" % ("Refresh" if not force else "Forced-refresh", rating_key),
-                    replace_parent=True)
-
-
 @route(PREFIX + '/missing/refresh')
+@debounce
 def RefreshMissing(randomize=None):
-    Thread.CreateTimer(1.0, lambda: scheduler.run_task("searchAllRecentlyAddedMissing"))
-    return fatality(header="Refresh of recently added items with missing subtitles triggered", replace_parent=True)
-
-
-@route(PREFIX + '/advanced')
-def AdvancedMenu(randomize=None, header=None, message=None):
-    oc = ObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True, no_history=True,
-                         replace_parent=True, title2="Advanced")
-
-    oc.add(DirectoryObject(
-        key=Callback(TriggerRestart),
-        title=pad_title("Restart the plugin")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(RefreshToken, randomize=timestamp()),
-        title=pad_title("Re-request the API token from plex.tv")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(LogStorage, key="tasks", randomize=timestamp()),
-        title=pad_title("Log the plugin's scheduled tasks state storage")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(LogStorage, key="subs", randomize=timestamp()),
-        title=pad_title("Log the plugin's internal subtitle information storage")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(LogStorage, key="ignore", randomize=timestamp()),
-        title=pad_title("Log the plugin's internal ignorelist storage")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
-        title=pad_title("Reset the plugin's scheduled tasks state storage")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(ResetStorage, key="subs", randomize=timestamp()),
-        title=pad_title("Reset the plugin's internal subtitle information storage")
-    ))
-    oc.add(DirectoryObject(
-        key=Callback(ResetStorage, key="ignore", randomize=timestamp()),
-        title=pad_title("Reset the plugin's internal ignorelist storage")
-    ))
-    return oc
+    scheduler.dispatch_task("SearchAllRecentlyAddedMissing")
+    header = "Refresh of recently added items with missing subtitles triggered"
+    return fatality(header=header, replace_parent=True)
 
 
 @route(PREFIX + '/ValidatePrefs', enforce_route=True)
@@ -387,22 +152,26 @@ def ValidatePrefs():
     # cache the channel state
     update_dict = False
     restart = False
+
+    # reset pin
+    Dict["pin_correct_time"] = None
+
+    config.initialize()
     if "channel_enabled" not in Dict:
         update_dict = True
 
-    elif Dict["channel_enabled"] != Prefs["enable_channel"]:
-        Log.Debug("Channel features %s, restarting plugin", "enabled" if Prefs["enable_channel"] else "disabled")
+    elif Dict["channel_enabled"] != config.enable_channel:
+        Log.Debug("Channel features %s, restarting plugin", "enabled" if config.enable_channel else "disabled")
         update_dict = True
         restart = True
 
     if update_dict:
-        Dict["channel_enabled"] = Prefs["enable_channel"]
+        Dict["channel_enabled"] = config.enable_channel
         Dict.Save()
 
     if restart:
         DispatchRestart()
 
-    config.initialize()
     scheduler.setup_tasks()
     set_refresh_menu_state(None)
 
@@ -414,69 +183,8 @@ def ValidatePrefs():
         Log.Debug("Stop logging to console")
 
     Log.Debug("Setting log-level to %s", Prefs["log_level"])
-    logger.registerLoggingHander(DEPENDENCY_MODULE_NAMES, level=Prefs["log_level"])
+    logger.register_logging_handler(DEPENDENCY_MODULE_NAMES, level=Prefs["log_level"])
     Core.log.setLevel(logging.getLevelName(Prefs["log_level"]))
 
     return
 
-
-def DispatchRestart():
-    Thread.CreateTimer(1.0, Restart)
-
-
-@route(PREFIX + '/advanced/restart/trigger')
-def TriggerRestart(randomize=None):
-    set_refresh_menu_state("Restarting the plugin")
-    DispatchRestart()
-    return fatality(header="Restart triggered, please wait about 5 seconds", force_title=" ", only_refresh=True, replace_parent=True,
-                    no_history=True)
-
-
-@route(PREFIX + '/advanced/restart/execute')
-def Restart():
-    Plex[":/plugins"].restart(PLUGIN_IDENTIFIER)
-
-
-@route(PREFIX + '/storage/reset', sure=bool)
-def ResetStorage(key, randomize=None, sure=False):
-    if not sure:
-        oc = ObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
-        oc.add(DirectoryObject(
-            key=Callback(ResetStorage, key=key, sure=True, randomize=timestamp()),
-            title=pad_title("Are you really sure?")
-        ))
-        return oc
-
-    resetStorage(key)
-
-    if key == "tasks":
-        # reinitialize the scheduler
-        scheduler.init_storage()
-        scheduler.setup_tasks()
-
-    return AdvancedMenu(
-        randomize=timestamp(),
-        header='Success',
-        message='Information Storage (%s) reset' % key
-    )
-
-
-@route(PREFIX + '/storage/log')
-def LogStorage(key, randomize=None):
-    logStorage(key)
-    return AdvancedMenu(
-        randomize=timestamp(),
-        header='Success',
-        message='Information Storage (%s) logged' % key
-    )
-
-
-@route(PREFIX + '/refresh_token')
-def RefreshToken(randomize=None):
-    result = refresh_plex_token()
-    if result:
-        msg = "Token successfully refreshed."
-    else:
-        msg = "Couldn't refresh the token, please check your credentials"
-
-    return AdvancedMenu(header=msg)

Contents/Code/interface/menu_helpers.py

@@ -1,10 +1,17 @@
 # coding=utf-8
 import types
+import datetime
 
-from support.items import get_kind
-from subzero import intent
-from support.helpers import format_video
+from support.items import get_kind, get_item_thumb
+from support.helpers import get_video_display_title
 from support.ignore import ignore_list
+from support.lib import get_intent
+from support.config import config
+from subzero.constants import ICON_SUB, ICON
+from support.background import scheduler
+
+default_thumb = R(ICON_SUB)
+main_icon = ICON if not config.is_development else "icon-dev.jpg"
 
 
 def should_display_ignore(items, previous=None):
@@ -42,18 +49,24 @@ def add_ignore_options(oc, kind, callback_menu=None, title=None, rating_key=None
     )
 
 
-def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_rating_key=None, fill_args=None, pass_kwargs=None):
+def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_rating_key=None, fill_args=None,
+             pass_kwargs=None, thumb=default_thumb):
     for kind, title, key, dig_deeper, item in items:
+        thumb = get_item_thumb(item) or thumb
+
         add_kwargs = {}
         if fill_args:
             add_kwargs = dict((name, getattr(item, k)) for k, name in fill_args.iteritems() if item and hasattr(item, k))
         if pass_kwargs:
             add_kwargs.update(pass_kwargs)
 
+        # force details view for show/season
+        summary = " " if kind in ("show", "season") else None
+
         oc.add(DirectoryObject(
-            key=Callback(menu_callback or menu_determination_callback(kind, item), title=title, rating_key=force_rating_key or key,
-                         **add_kwargs),
-            title=title
+            key=Callback(menu_callback or menu_determination_callback(kind, item, pass_kwargs=pass_kwargs), title=title,
+                         rating_key=force_rating_key or key, **add_kwargs),
+            title=title, thumb=thumb, summary=summary
         ))
     return oc
 
@@ -83,14 +96,22 @@ def set_refresh_menu_state(state_or_media, media_type="movies"):
             for episode in media.seasons[season].episodes:
                 ep = media.seasons[season].episodes[episode]
                 media_id = ep.id
-                title = format_video("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
+                title = get_video_display_title("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
     else:
-        title = format_video("movie", media.title)
+        title = get_video_display_title("movie", media.title)
+
+    intent = get_intent()
     force_refresh = intent.get("force", media_id)
 
     Dict["current_refresh_state"] = u"%sRefreshing %s" % ("Force-" if force_refresh else "", unicode(title))
 
 
+def get_item_task_data(task_name, rating_key, language):
+    task_data = scheduler.get_task_data(task_name)
+    search_results = task_data.get(rating_key, {}) if task_data else {}
+    return search_results.get(language)
+
+
 def enable_channel_wrapper(func):
     """
     returns the original wrapper :func: (route or handler) if applicable, else the plain to-be-wrapped function
@@ -110,6 +131,88 @@ def enable_channel_wrapper(func):
 
     def wrap(*args, **kwargs):
         enforce_route = kwargs.pop("enforce_route", None)
-        return (func if Prefs["enable_channel"] or enforce_route else noop)(*args, **kwargs)
+        return (func if config.enable_channel or enforce_route else noop)(*args, **kwargs)
 
     return wrap
+
+
+def debounce(func):
+    """
+    prevent func from being called twice with the same arguments
+    :param func:
+    :return:
+    """
+    def get_lookup_key(args, kwargs):
+        func_name = list(args).pop(0).__name__
+        return tuple([func_name] + [(key, value) for key, value in kwargs.iteritems()])
+
+    def wrap(*args, **kwargs):
+        if "randomize" in kwargs:
+            if not "menu_history" in Dict:
+                Dict["menu_history"] = {}
+
+            key = get_lookup_key([func] + list(args), kwargs)
+            if key in Dict["menu_history"]:
+                Log.Debug("not triggering %s twice with %s, %s" % (func, args, kwargs))
+                return ObjectContainer()
+            else:
+                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(days=1)
+                Dict.Save()
+        return func(*args, **kwargs)
+
+    return wrap
+
+
+class SZObjectContainer(ObjectContainer):
+    def __init__(self, *args, **kwargs):
+        skip_pin_lock = kwargs.pop("skip_pin_lock", False)
+
+        super(SZObjectContainer, self).__init__(*args, **kwargs)
+
+        if (config.lock_menu or config.lock_advanced_menu) and not config.pin_correct and not skip_pin_lock:
+            config.locked = True
+
+    def add(self, *args, **kwargs):
+        # disable self.add if we're in lockdown
+        container = args[0]
+        current_menu_target = container.key.split("?")[0]
+        is_pin_menu = current_menu_target.endswith("/pin")
+
+        if config.locked and config.lock_menu and not is_pin_menu:
+            return
+        return super(SZObjectContainer, self).add(*args, **kwargs)
+
+
+OriginalObjectContainer = ObjectContainer
+ObjectContainer = SZObjectContainer
+
+
+class SubFolderObjectContainer(ObjectContainer):
+    def __init__(self, *args, **kwargs):
+        super(SubFolderObjectContainer, self).__init__(*args, **kwargs)
+        from interface.menu import fatality
+        from support.helpers import pad_title, timestamp
+        self.add(DirectoryObject(
+            key=Callback(fatality, force_title=" ", randomize=timestamp()),
+            title=pad_title("<< Back to home"),
+            summary="Current state: %s; Last state: %s" % (
+                (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
+                (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
+            )
+        ))
+
+
+ObjectClass = getattr(getattr(Redirect, "_object_class"), "__bases__")[0]
+
+
+class ZipObject(ObjectClass):
+    def __init__(self, data):
+        ObjectClass.__init__(self, "")
+        self.zipdata = data
+        self.SetHeader("Content-Type", "application/zip")
+
+    def Content(self):
+        self.SetHeader("Content-Disposition",
+                       'attachment; filename="' + datetime.datetime.now().strftime("Logs_%y%m%d_%H-%M-%S.zip")
+                       + '"')
+        return self.zipdata

Contents/Code/interface/refresh_item.py

@@ -0,0 +1,22 @@
+# coding=utf-8
+
+from subzero.constants import PREFIX
+from menu_helpers import debounce, set_refresh_menu_state
+from support.items import refresh_item
+from support.helpers import timestamp
+
+
+@route(PREFIX + '/item/{rating_key}')
+@debounce
+def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None,
+                previous_rating_key=None, timeout=8000, randomize=None, trigger=True):
+    assert rating_key
+    from interface.main import fatality
+    header = " "
+    if trigger:
+        set_refresh_menu_state(u"Triggering %sRefresh for %s" % ("Force-" if force else "", item_title))
+        Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind,
+                      parent_rating_key=previous_rating_key, timeout=int(timeout))
+
+        header = u"%s of item %s triggered" % ("Refresh" if not force else "Forced-refresh", rating_key)
+    return fatality(randomize=timestamp(), header=header, replace_parent=True)

Contents/Code/logger.py

@@ -1,8 +1,8 @@
 import logging
 
 
-def registerLoggingHander(dependencies, level="ERROR"):
-    plexHandler = PlexLoggerHandler()
+def register_logging_handler(dependencies, level="ERROR"):
+    plex_handler = PlexLoggerHandler()
     for dependency in dependencies:
         Log.Debug("Registering LoggerHandler for dependency: %s" % dependency)
         log = logging.getLogger(dependency)
@@ -13,7 +13,7 @@ def registerLoggingHander(dependencies, level="ERROR"):
                 log.removeHandler(handler)
 
         log.setLevel(level)
-        log.addHandler(plexHandler)
+        log.addHandler(plex_handler)
 
 
 class PlexLoggerHandler(logging.StreamHandler):

Contents/Code/support/__init__.py

@@ -13,6 +13,9 @@ import lib
 
 sys.modules["support.lib"] = lib
 
+import plex_media
+sys.modules["support.plex_media"] = plex_media
+
 import localmedia
 
 sys.modules["subzero.localmedia"] = localmedia
@@ -41,10 +44,17 @@ import storage
 
 sys.modules["support.storage"] = storage
 
-import auth
-
-sys.modules["support.auth"] = auth
-
 import ignore
 
 sys.modules["support.ignore"] = ignore
+
+import history
+
+sys.modules["support.history"] = history
+
+import data
+
+sys.modules["support.data"] = data
+
+import activities
+sys.modules["support.activities"] = activities

Contents/Code/support/activities.py

@@ -0,0 +1,111 @@
+# coding=utf-8
+from wraptor.decorators import throttle
+from config import config
+from items import get_item, get_item_kind_from_item, refresh_item
+
+from plex_activity import Activity
+from plex_activity.sources.s_logging.main import Logging as Activity_Logging
+
+
+class PlexActivityManager(object):
+    def start(self):
+        activity_sources_enabled = None
+
+        if config.universal_plex_token:
+            from plex import Plex
+            Plex.configuration.defaults.authentication(config.universal_plex_token)
+            activity_sources_enabled = ["websocket"]
+            Activity.on('websocket.playing', self.on_playing)
+
+        elif config.server_log_path:
+            Activity_Logging.add_hint(config.server_log_path, None)
+            activity_sources_enabled = ["logging"]
+            Activity.on('logging.playing', self.on_playing)
+
+        if activity_sources_enabled:
+            Activity.start(activity_sources_enabled)
+
+    @throttle(5, instance_method=True)
+    def on_playing(self, info):
+        if not config.use_activities:
+            return
+
+        # ignore non-playing states and anything too far in
+        if info["state"] != "playing" or info["viewOffset"] > 60000:
+            return
+
+        # don't trigger on the first hit ever
+        if "last_played_items" not in Dict:
+            Dict["last_played_items"] = []
+            Dict.Save()
+            return
+
+        rating_key = info["ratingKey"]
+        if rating_key not in Dict["last_played_items"]:
+            # new playing; store last 10 recently played items
+            Dict["last_played_items"].insert(0, rating_key)
+            Dict["last_played_items"] = Dict["last_played_items"][:10]
+
+            Dict.Save()
+
+            debug_msg = "Started playing %s. Refreshing it." % rating_key
+
+            key_to_refresh = None
+            if config.activity_mode in ["refresh", "next_episode", "hybrid"]:
+                # next episode or next episode and current movie
+                if config.activity_mode in ["next_episode", "hybrid"]:
+                    plex_item = get_item(rating_key)
+                    if not plex_item:
+                        Log.Warn("Can't determine media type of %s, skipping" % rating_key)
+                        return
+
+                    if get_item_kind_from_item(plex_item) == "episode":
+                        next_ep = self.get_next_episode(rating_key)
+                        if next_ep:
+                            key_to_refresh = next_ep.rating_key
+                            debug_msg = "Started playing %s. Refreshing next episode (%s, S%02iE%02i)." % \
+                                        (rating_key, next_ep.rating_key, int(next_ep.season.index), int(next_ep.index))
+
+                    else:
+                        if config.activity_mode == "hybrid":
+                            key_to_refresh = rating_key
+                elif config.activity_mode == "refresh":
+                    key_to_refresh = rating_key
+
+                if key_to_refresh:
+                    Log.Debug(debug_msg)
+                    refresh_item(key_to_refresh)
+
+    def get_next_episode(self, rating_key):
+        plex_item = get_item(rating_key)
+        if not plex_item:
+            return
+
+        if get_item_kind_from_item(plex_item) == "episode":
+            # get season
+            season = get_item(plex_item.season.rating_key)
+            if not season:
+                return
+
+            # determine next episode
+            # next episode is in the same season
+            if plex_item.index < season.episode_count:
+                # get next ep
+                for ep in season.children():
+                    if ep.index == plex_item.index + 1:
+                        return ep
+
+            # it's not, try getting the first episode of the next season
+            else:
+                # get show
+                show = get_item(plex_item.show.rating_key)
+                # is there a next season?
+                if season.index < show.season_count:
+                    for other_season in show.children():
+                        if other_season.index == season.index + 1:
+                            next_season = other_season
+                            for ep in next_season.children():
+                                if ep.index == 1:
+                                    return ep
+
+activity = PlexActivityManager()

Contents/Code/support/background.py

@@ -6,7 +6,7 @@ import traceback
 
 
 def parse_frequency(s):
-    if s == "never":
+    if s == "never" or s == None:
         return None, None
     kind, num, unit = s.split()
     return int(num), unit
@@ -27,9 +27,40 @@ class DefaultScheduler(object):
 
     def init_storage(self):
         if "tasks" not in Dict:
-            Dict["tasks"] = {}
+            Dict["tasks"] = {"queue": []}
             Dict.Save()
 
+        if "queue" not in Dict["tasks"]:
+            Dict["tasks"]["queue"] = []
+
+    def get_task_data(self, name):
+        if name not in Dict["tasks"]:
+            raise NotImplementedError("Task missing! %s" % name)
+
+        if "data" in Dict["tasks"][name]:
+            return Dict["tasks"][name]["data"]
+
+    def clear_task_data(self, name=None):
+        if name is None:
+            # full clean
+            Log.Debug("Clearing previous task data")
+            if Dict["tasks"]:
+                for task_name in Dict["tasks"].keys():
+                    if task_name == "queue":
+                        continue
+
+                    Dict["tasks"][task_name]["data"] = {}
+                    Dict["tasks"][task_name]["running"] = False
+                Dict.Save()
+            return
+
+        if name not in Dict["tasks"]:
+            raise NotImplementedError("Task missing! %s" % name)
+
+        Dict["tasks"][name]["data"] = {}
+        Dict.Save()
+        Log.Debug("Task data cleared: %s", name)
+
     def register(self, task):
         self.registry.append(task)
 
@@ -38,7 +69,12 @@ class DefaultScheduler(object):
         self.tasks = {}
         for cls in self.registry:
             task = cls(self)
-            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task.name])}
+            try:
+                task_frequency = Prefs["scheduler.tasks.%s.frequency" % task.name]
+            except KeyError:
+                task_frequency = getattr(task, "frequency", None)
+
+            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)}
 
     def run(self):
         self.running = True
@@ -52,13 +88,18 @@ class DefaultScheduler(object):
             return None
         return self.tasks[name]["task"]
 
+    def is_task_running(self, name):
+        task = self.task(name)
+        if task:
+            return task.running
+
     def last_run(self, task):
         if task not in self.tasks:
             return None
         return self.tasks[task]["task"].last_run
 
     def next_run(self, task):
-        if task not in self.tasks:
+        if task not in self.tasks or not self.tasks[task]["task"].periodic:
             return None
         frequency_num, frequency_key = self.tasks[task]["frequency"]
         if not frequency_num:
@@ -70,27 +111,41 @@ class DefaultScheduler(object):
             use_date = now
         return max(use_date + datetime.timedelta(**{frequency_key: frequency_num}), now)
 
-    def run_task(self, name):
+    def run_task(self, name, *args, **kwargs):
         task = self.tasks[name]["task"]
         if task.running:
             Log.Debug("Scheduler: Not running %s, as it's currently running.", name)
-            return
+            return False
 
         Log.Debug("Scheduler: Running task %s", name)
         try:
-            task.prepare()
+            task.prepare(*args, **kwargs)
             task.run()
         except Exception, e:
             Log.Error("Scheduler: Something went wrong when running %s: %s", name, traceback.format_exc())
         finally:
-            task.post_run()
+            task.post_run(Dict["tasks"][name]["data"])
+            Dict.Save()
+
+    def dispatch_task(self, *args, **kwargs):
+        if "queue" not in Dict["tasks"]:
+            Dict["tasks"]["queue"] = []
+
+        Dict["tasks"]["queue"].append((args, kwargs))
 
     def signal(self, name, *args, **kwargs):
         for task_name, info in self.tasks.iteritems():
             task = info["task"]
+            if not task.periodic:
+                continue
+
             if task.running:
                 Log.Debug("Scheduler: Sending signal %s to task %s (%s, %s)", name, task_name, args, kwargs)
-                status = task.signal(name, *args, **kwargs)
+                try:
+                    status = task.signal(name, *args, **kwargs)
+                except NotImplementedError:
+                    Log.Debug("Scheduler: Signal ignored by %s", task_name)
+                    continue
                 if status:
                     Log.Debug("Scheduler: Signal accepted by %s", task_name)
                 else:
@@ -104,11 +159,22 @@ class DefaultScheduler(object):
             if not self.running:
                 break
 
+            # single dispatch requested?
+            if Dict["tasks"]["queue"]:
+                # work queue off
+                queue = Dict["tasks"]["queue"][:]
+                Dict["tasks"]["queue"] = []
+                Dict.Save()
+                for args, kwargs in queue:
+                    Log.Debug("Dispatching single task: %s, %s", args, kwargs)
+                    Thread.Create(self.run_task, True, *args, **kwargs)
+
+            # scheduled tasks
             for name, info in self.tasks.iteritems():
                 now = datetime.datetime.now()
                 task = info["task"]
 
-                if name not in Dict["tasks"]:
+                if name not in Dict["tasks"] or not task.periodic:
                     continue
 
                 if task.running:
@@ -118,10 +184,10 @@ class DefaultScheduler(object):
                 if not frequency_num:
                     continue
 
-                if not task.last_run or task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now:
+                if not task.last_run or (task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now):
                     self.run_task(name)
 
-            Thread.Sleep(10.0)
+            Thread.Sleep(5.0)
 
 
 scheduler = DefaultScheduler()

Contents/Code/support/config.py

@@ -4,21 +4,28 @@ import os
 import re
 import inspect
 
+import datetime
+
+import subliminal
+import subliminal_patch
 from babelfish import Language
-from subzero.lib.io import FileIO
-from subzero.constants import PLUGIN_NAME
-from lib import configure_plex, Plex
-from helpers import check_write_permissions
+from subzero.lib.io import FileIO, get_viable_encoding
+from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW
+from lib import Plex
+from helpers import check_write_permissions, cast_bool
 
 SUBTITLE_EXTS = ['utf', 'utf8', 'utf-8', 'srt', 'smi', 'rt', 'ssa', 'aqt', 'jss', 'ass', 'idx', 'sub', 'txt', 'psb']
-VIDEO_EXTS = ['3g2', '3gp', 'asf', 'asx', 'avc', 'avi', 'avs', 'bivx', 'bup', 'divx', 'dv', 'dvr-ms', 'evo', 'fli', 'flv',
+VIDEO_EXTS = ['3g2', '3gp', 'asf', 'asx', 'avc', 'avi', 'avs', 'bivx', 'bup', 'divx', 'dv', 'dvr-ms', 'evo', 'fli',
+              'flv',
               'm2t', 'm2ts', 'm2v', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'mts', 'nsv', 'nuv', 'ogm', 'ogv', 'tp',
-              'pva', 'qt', 'rm', 'rmvb', 'sdp', 'svq3', 'strm', 'ts', 'ty', 'vdr', 'viv', 'vob', 'vp3', 'wmv', 'wpl', 'wtv', 'xsp', 'xvid',
+              'pva', 'qt', 'rm', 'rmvb', 'sdp', 'svq3', 'strm', 'ts', 'ty', 'vdr', 'viv', 'vob', 'vp3', 'wmv', 'wpl',
+              'wtv', 'xsp', 'xvid',
               'webm']
 
 IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")
 
 VERSION_RE = re.compile(ur'CFBundleVersion.+?<string>([0-9\.]+)</string>', re.DOTALL)
+DEV_RE = re.compile(ur'PlexPluginDevMode.+?<string>([01]+)</string>', re.DOTALL)
 
 
 def int_or_default(s, default):
@@ -29,80 +36,267 @@ def int_or_default(s, default):
 
 
 class Config(object):
+    plugin_info = ""
     version = None
-    langList = None
-    subtitleDestinationFolder = None
+    full_version = None
+    plugin_log_path = None
+    server_log_path = None
+    app_support_path = None
+    data_path = None
+    data_items_path = None
+    universal_plex_token = None
+    is_development = False
+
+    enable_channel = True
+    enable_agent = True
+    pin = None
+    lock_menu = False
+    lock_advanced_menu = False
+    locked = False
+    pin_valid_minutes = 10
+    lang_list = None
+    subtitle_destination_folder = None
     providers = None
-    providerSettings = None
+    provider_settings = None
     max_recent_items_per_library = 200
-    plex_api_working = False
     permissions_ok = False
     missing_permissions = None
+    ignore_sz_files = False
+    ignore_paths = None
+    fs_encoding = None
+    notify_executable = None
+    sections = None
+    enabled_sections = None
+    enforce_encoding = False
+    chmod = None
+    forced_only = False
+    exotic_ext = False
+    treat_und_as_first = False
+    ext_match_strictness = False
+    use_activities = False
+    activity_mode = None
 
     initialized = False
 
     def initialize(self):
-        self.version = self.getVersion()
+        self.fs_encoding = get_viable_encoding()
+        self.plugin_info = self.get_plugin_info()
+        self.is_development = self.get_dev_mode()
+        self.version = self.get_version()
         self.full_version = u"%s %s" % (PLUGIN_NAME, self.version)
-        self.langList = self.getLangList()
-        self.subtitleDestinationFolder = self.getSubtitleDestinationFolder()
-        self.providers = self.getProviders()
-        self.providerSettings = self.getProviderSettings()
-        self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 200)
-        self.initialized = True
-        configure_plex()
-        self.plex_api_working = self.checkPlexAPI()
+        self.set_log_paths()
+        self.app_support_path = Core.app_support_path
+        self.data_path = getattr(Data, "_core").storage.data_path
+        self.data_items_path = os.path.join(self.data_path, "DataItems")
+        self.universal_plex_token = self.get_universal_plex_token()
+
+        self.set_plugin_mode()
+        self.set_plugin_lock()
+        self.set_activity_modes()
+
+        self.lang_list = self.get_lang_list()
+        self.subtitle_destination_folder = self.get_subtitle_destination_folder()
+        self.providers = self.get_providers()
+        self.provider_settings = self.get_provider_settings()
+        self.max_recent_items_per_library = int_or_default(Prefs["scheduler.max_recent_items_per_library"], 2000)
+        self.sections = list(Plex["library"].sections())
         self.missing_permissions = []
-        self.permissions_ok = self.checkPermissions()
+        self.ignore_sz_files = cast_bool(Prefs["subtitles.ignore_fs"])
+        self.ignore_paths = self.parse_ignore_paths()
+        self.enabled_sections = self.check_enabled_sections()
+        self.permissions_ok = self.check_permissions()
+        self.notify_executable = self.check_notify_executable()
+        self.enforce_encoding = cast_bool(Prefs['subtitles.enforce_encoding'])
+        self.chmod = self.check_chmod()
+        self.forced_only = cast_bool(Prefs["subtitles.only_foreign"])
+        self.exotic_ext = cast_bool(Prefs["subtitles.scan.exotic_ext"])
+        self.treat_und_as_first = cast_bool(Prefs["subtitles.language.treat_und_as_first"])
+        self.ext_match_strictness = self.determine_ext_sub_strictness()
+        self.initialized = True
 
-    def checkPlexAPI(self):
-        return bool(Plex["library"].sections())
+    def set_log_paths(self):
+        # find log handler
+        for handler in Core.log.handlers:
+            if getattr(getattr(handler, "__class__"), "__name__") in (
+                    'FileHandler', 'RotatingFileHandler', 'TimedRotatingFileHandler'):
+                plugin_log_file = handler.baseFilename
+                if os.path.isfile(os.path.realpath(plugin_log_file)):
+                    self.plugin_log_path = plugin_log_file
 
-    def checkPermissions(self):
+                if plugin_log_file:
+                    server_log_file = os.path.realpath(os.path.join(plugin_log_file, "../../Plex Media Server.log"))
+                    if os.path.isfile(server_log_file):
+                        self.server_log_path = server_log_file
+
+    def get_universal_plex_token(self):
+        # thanks to: https://forums.plex.tv/discussion/247136/read-current-x-plex-token-in-an-agent-ensure-that-a-http-request-gets-executed-exactly-once#latest
+        pref_path = os.path.join(self.app_support_path, "Preferences.xml")
+        if os.path.exists(pref_path):
+            try:
+                global_prefs = Core.storage.load(pref_path)
+                return XML.ElementFromString(global_prefs).xpath('//Preferences/@PlexOnlineToken')[0]
+            except:
+                Log.Warn("Couldn't determine Plex Token")
+        else:
+            Log("Did NOT find Preferences file - please check logfile and hierarchy. Aborting!")
+
+    def set_plugin_mode(self):
+        if Prefs["plugin_mode"] == "only agent":
+            self.enable_channel = False
+        elif Prefs["plugin_mode"] == "only channel":
+            self.enable_agent = False
+
+    def set_plugin_lock(self):
+        if Prefs["plugin_pin_mode"] in ("channel menu", "advanced menu"):
+            # check pin
+            pin = Prefs["plugin_pin"]
+            if not pin or not len(pin):
+                Log.Warn("PIN enabled but not set, disabling PIN!")
+                return
+
+            pin = pin.strip()
+            try:
+                int(pin)
+            except ValueError:
+                Log.Warn("PIN has to be an integer (0-9)")
+            self.pin = pin
+            self.lock_advanced_menu = Prefs["plugin_pin_mode"] == "advanced menu"
+            self.lock_menu = Prefs["plugin_pin_mode"] == "channel menu"
+
+            try:
+                self.pin_valid_minutes = int(Prefs["plugin_pin_valid_for"].strip())
+            except ValueError:
+                pass
+
+    @property
+    def pin_correct(self):
+        if isinstance(Dict["pin_correct_time"], datetime.datetime) \
+                and Dict["pin_correct_time"] + datetime.timedelta(
+                    minutes=self.pin_valid_minutes) > datetime.datetime.now():
+            return True
+
+    def refresh_permissions_status(self):
+        self.permissions_ok = self.check_permissions()
+
+    def check_permissions(self):
         if not Prefs["subtitles.save.filesystem"] or not Prefs["check_permissions"]:
             return True
 
-        if not self.plex_api_working:
-            return
-
+        self.missing_permissions = []
         use_ignore_fs = Prefs["subtitles.ignore_fs"]
-        sections = Plex["library"].sections()
         all_permissions_ok = True
-        for section in list(sections):
+        for section in self.sections:
+            if section.key not in self.enabled_sections:
+                continue
+
             title = section.title
             for location in section:
+                path_str = location.path
+                if isinstance(path_str, unicode):
+                    path_str = path_str.encode(self.fs_encoding)
+
                 if use_ignore_fs:
-                    ignore = False
                     # check whether we've got an ignore file inside the section path
-                    for ifn in IGNORE_FN:
-                        if os.path.isfile(os.path.join(location.path, ifn)):
-                            ignore = True
-                    if ignore:
+                    if self.is_physically_ignored(path_str):
                         continue
 
+                if self.is_path_ignored(path_str):
+                    # is the path in our ignored paths setting?
+                    continue
+
                 # section not ignored, check for write permissions
-                if not check_write_permissions(location.path):
+                if not check_write_permissions(path_str):
                     # not enough permissions
                     self.missing_permissions.append((title, location.path))
                     all_permissions_ok = False
 
         return all_permissions_ok
 
-    def getVersion(self):
+    def get_version(self):
+        result = VERSION_RE.search(self.plugin_info)
+        add = "" if not self.is_development else " DEV"
+
+        if result:
+            return result.group(1) + add
+
+    def get_dev_mode(self):
+        dev = DEV_RE.search(self.plugin_info)
+        if dev and dev.group(1) == "1":
+            return True
+
+    def get_plugin_info(self):
         curDir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
         info_file_path = os.path.abspath(os.path.join(curDir, "..", "..", "Info.plist"))
-        data = FileIO.read(info_file_path)
-        result = VERSION_RE.search(data)
-        if result:
-            return result.group(1)
+        return FileIO.read(info_file_path)
 
-    def getBlacklist(self, key):
-        return map(lambda id: id.strip(), (Prefs[key] or "").split(","))
+    def parse_ignore_paths(self):
+        paths = Prefs["subtitles.ignore_paths"]
+        if paths:
+            try:
+                return [path.strip() for path in paths.split(",")]
+            except:
+                Log.Error("Couldn't parse your ignore paths settings: %s" % paths)
+        return []
+
+    def is_physically_ignored(self, folder):
+        # check whether we've got an ignore file inside the path
+        for ifn in IGNORE_FN:
+            if os.path.isfile(os.path.join(folder, ifn)):
+                Log.Info(u'Ignoring "%s" because "%s" exists', folder, ifn)
+                return True
+
+        return False
+
+    def is_path_ignored(self, fn):
+        for path in self.ignore_paths:
+            if fn.startswith(path):
+                return True
+        return False
+
+    def check_notify_executable(self):
+        fn = Prefs["notify_executable"]
+        if not fn:
+            return
+
+        splitted_fn = fn.split()
+        exe_fn = splitted_fn[0]
+        arguments = [arg.strip() for arg in splitted_fn[1:]]
+
+        if os.path.isfile(exe_fn) and os.access(exe_fn, os.X_OK):
+            return exe_fn, arguments
+        Log.Error("Notify executable not existing or not executable: %s" % exe_fn)
+
+    def refresh_enabled_sections(self):
+        self.enabled_sections = self.check_enabled_sections()
+
+    def check_enabled_sections(self):
+        enabled_for_primary_agents = []
+        enabled_sections = {}
+
+        # find which agents we're enabled for
+        for agent in Plex.agents():
+            if not agent.primary:
+                continue
+
+            for t in list(agent.media_types):
+                if t.media_type in (MOVIE, SHOW):
+                    related_agents = Plex.primary_agent(agent.identifier, t.media_type)
+                    for a in related_agents:
+                        if a.identifier == PLUGIN_IDENTIFIER and a.enabled:
+                            enabled_for_primary_agents.append(agent.identifier)
+
+        # find the libraries that use them
+        for library in self.sections:
+            if library.agent in enabled_for_primary_agents:
+                enabled_sections[library.key] = library
+
+        Log.Debug(u"I'm enabled for: %s" % [lib.title for key, lib in enabled_sections.iteritems()])
+        return enabled_sections
 
     # Prepare a list of languages we want subs for
-    def getLangList(self):
+    def get_lang_list(self):
         l = {Language.fromietf(Prefs["langPref1"])}
-        langCustom = Prefs["langPrefCustom"].strip()
+        lang_custom = Prefs["langPrefCustom"].strip()
 
         if Prefs['subtitles.only_one']:
             return l
@@ -113,8 +307,8 @@ class Config(object):
         if Prefs["langPref3"] != "None":
             l.update({Language.fromietf(Prefs["langPref3"])})
 
-        if len(langCustom) and langCustom != "None":
-            for lang in langCustom.split(u","):
+        if len(lang_custom) and lang_custom != "None":
+            for lang in lang_custom.split(u","):
                 lang = lang.strip()
                 try:
                     real_lang = Language.fromietf(lang)
@@ -127,34 +321,113 @@ class Config(object):
 
         return l
 
-    def getSubtitleDestinationFolder(self):
+    def get_subtitle_destination_folder(self):
         if not Prefs["subtitles.save.filesystem"]:
             return
 
-        fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
-        return fld_custom or (Prefs["subtitles.save.subFolder"] if Prefs["subtitles.save.subFolder"] != "current folder" else None)
+        fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if cast_bool(
+            Prefs["subtitles.save.subFolder.Custom"]) else None
+        return fld_custom or (
+            Prefs["subtitles.save.subFolder"] if Prefs["subtitles.save.subFolder"] != "current folder" else None)
 
-    def getProviders(self):
-        providers = {'opensubtitles': Prefs['provider.opensubtitles.enabled'],
-                     #'thesubdb': Prefs['provider.thesubdb.enabled'],
-                     'podnapisi': Prefs['provider.podnapisi.enabled'],
-                     'addic7ed': Prefs['provider.addic7ed.enabled'],
-                     'tvsubtitles': Prefs['provider.tvsubtitles.enabled']
+    def get_providers(self):
+        providers = {'opensubtitles': cast_bool(Prefs['provider.opensubtitles.enabled']),
+                     # 'thesubdb': Prefs['provider.thesubdb.enabled'],
+                     'podnapisi': cast_bool(Prefs['provider.podnapisi.enabled']),
+                     'addic7ed': cast_bool(Prefs['provider.addic7ed.enabled']),
+                     'tvsubtitles': cast_bool(Prefs['provider.tvsubtitles.enabled']),
+                     'legendastv': cast_bool(Prefs['provider.legendastv.enabled']),
+                     'napiprojekt': cast_bool(Prefs['provider.napiprojekt.enabled']),
+                     'shooter': cast_bool(Prefs['provider.shooter.enabled']),
+                     'subscenter': cast_bool(Prefs['provider.subscenter.enabled']),
                      }
+
+        # ditch non-forced-subtitles-reporting providers
+        if cast_bool(Prefs['subtitles.only_foreign']):
+            providers["addic7ed"] = False
+            providers["tvsubtitles"] = False
+            providers["legendastv"] = False
+
         return filter(lambda prov: providers[prov], providers)
 
-    def getProviderSettings(self):
+    def get_provider_settings(self):
         provider_settings = {'addic7ed': {'username': Prefs['provider.addic7ed.username'],
                                           'password': Prefs['provider.addic7ed.password'],
-                                          'use_random_agents': Prefs['provider.addic7ed.use_random_agents'],
+                                          'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents']),
                                           },
                              'opensubtitles': {'username': Prefs['provider.opensubtitles.username'],
                                                'password': Prefs['provider.opensubtitles.password'],
-                                               'use_tag_search': Prefs['provider.opensubtitles.use_tags']
+                                               'use_tag_search': cast_bool(Prefs['provider.opensubtitles.use_tags']),
+                                               'only_foreign': cast_bool(Prefs['subtitles.only_foreign'])
                                                },
+                             'podnapisi': {
+                                 'only_foreign': cast_bool(Prefs['subtitles.only_foreign'])
+                             },
+                             'legendastv': {'username': Prefs['provider.legendastv.username'],
+                                            'password': Prefs['provider.legendastv.password'],
+                                            },
+                             'subscenter': {'username': Prefs['provider.subscenter.username'],
+                                            'password': Prefs['provider.subscenter.password'],
+                                            },
                              }
 
         return provider_settings
 
+    @property
+    def provider_pool(self):
+        if cast_bool(Prefs['providers.multithreading']):
+            return subliminal_patch.core.SZAsyncProviderPool
+        return subliminal_patch.core.SZProviderPool
+
+    def check_chmod(self):
+        val = Prefs["subtitles.save.chmod"]
+        if not val or not len(val):
+            return
+
+        wrong_chmod = False
+        if len(val) != 4:
+            wrong_chmod = True
+
+        try:
+            return int(val, 8)
+        except ValueError:
+            wrong_chmod = True
+
+        if wrong_chmod:
+            Log.Warn("Chmod setting ignored, please use only 4-digit integers with leading 0 (e.g.: 775)")
+
+    def determine_ext_sub_strictness(self):
+        val = Prefs["subtitles.scan.filename_strictness"]
+        if val == "any":
+            return "any"
+        elif val.startswith("loose"):
+            return "loose"
+        return "strict"
+
+    def set_activity_modes(self):
+        val = Prefs["activity.on_playback"]
+        if val == "never":
+            self.use_activities = False
+            return
+
+        self.use_activities = True
+        if val == "current media item":
+            self.activity_mode = "refresh"
+        elif val == "hybrid: current item or next episode":
+            self.activity_mode = "hybrid"
+        else:
+            self.activity_mode = "next_episode"
+
+    def init_subliminal_patches(self):
+        # configure custom subtitle destination folders for scanning pre-existing subs
+        Log.Debug("Patching subliminal ...")
+        dest_folder = self.subtitle_destination_folder
+        subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else []
+        subliminal_patch.core.INCLUDE_EXOTIC_SUBS = self.exotic_ext
+
+        subliminal_patch.core.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
+        subliminal.score.episode_scores["addic7ed_boost"] = int(Prefs['provider.addic7ed.boost_by1'])
+
 
 config = Config()
+config.initialize()

Contents/Code/support/data.py

@@ -0,0 +1,84 @@
+# coding=utf-8
+
+
+def dispatch_migrate():
+    try:
+        migrate()
+    except:
+        Log.Error("Migration failed: %s" % traceback.format_exc())
+
+
+def migrate():
+    """
+    some Dict/Data migrations here, no need for a more in-depth migration path for now
+    :return:
+    """
+
+    # migrate subtitle history from Dict to Data
+    if "history" in Dict and Dict["history"].get("history_items"):
+        Log.Debug("Running migration for history data")
+        from support.history import get_history
+        history = get_history()
+
+        for item in reversed(Dict["history"]["history_items"]):
+            history.add(item.item_title, item.rating_key, item.section_title, subtitle=item.subtitle, mode=item.mode,
+                        time=item.time)
+
+        del Dict["history"]
+        Dict.Save()
+
+    # migrate subtitle storage from Dict to Data
+    if "subs" in Dict:
+        from support.storage import get_subtitle_storage
+        from subzero.subtitle_storage import StoredSubtitle
+        from support.plex_media import get_item
+
+        subtitle_storage = get_subtitle_storage()
+
+        for video_id, parts in Dict["subs"].iteritems():
+            try:
+                item = get_item(video_id)
+            except:
+                continue
+
+            if not item:
+                continue
+
+            stored_subs = subtitle_storage.load_or_new(item)
+            stored_subs.version = 1
+
+            Log.Debug(u"Migrating %s" % video_id)
+
+            stored_any = False
+            for part_id, lang_dict in parts.iteritems():
+                part_id = str(part_id)
+                Log.Debug(u"Migrating %s, %s" % (video_id, part_id))
+
+                for lang, subs in lang_dict.iteritems():
+                    lang = str(lang)
+                    if "current" in subs:
+                        current_key = subs["current"]
+                        provider_name, subtitle_id = current_key
+                        sub = subs.get(current_key)
+                        if sub and sub.get("title") and sub.get("mode"):  # ditch legacy data without sufficient info
+                            stored_subs.title = sub["title"]
+                            new_sub = StoredSubtitle(sub["score"], sub["storage"], sub["hash"], provider_name,
+                                                     subtitle_id, date_added=sub["date_added"], mode=sub["mode"])
+
+                            if part_id not in stored_subs.parts:
+                                stored_subs.parts[part_id] = {}
+
+                            if lang not in stored_subs.parts[part_id]:
+                                stored_subs.parts[part_id][lang] = {}
+
+                            Log.Debug(u"Migrating %s, %s, %s" % (video_id, part_id, current_key))
+
+                            stored_subs.parts[part_id][lang][current_key] = new_sub
+                            stored_subs.parts[part_id][lang]["current"] = current_key
+                            stored_any = True
+
+            if stored_any:
+                subtitle_storage.save(stored_subs)
+
+        del Dict["subs"]
+        Dict.Save()

Contents/Code/support/helpers.py

@@ -1,14 +1,24 @@
 # coding=utf-8
 import os
+import traceback
+import types
 import unicodedata
 import datetime
 import urllib
 import time
 import re
 import platform
+import subprocess
+
+from bs4 import UnicodeDammit
+
+import chardet
+
+from babelfish import Language
+
+from subzero.analytics import track_event
 
 # Unicode control characters can appear in ID3v2 tags but are not legal in XML.
-
 RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
                      u'|' + \
                      u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \
@@ -19,8 +29,12 @@ RE_UNICODE_CONTROL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])'
                      )
 
 
+def cast_bool(value):
+    return str(value) in ("true", "True")
+
+
 # A platform independent way to split paths which might come in with different separators.
-def splitPath(str):
+def split_path(str):
     if str.find('\\') != -1:
         return str.split('\\')
     else:
@@ -32,18 +46,32 @@ def unicodize(s):
     try:
         filename = unicodedata.normalize('NFC', unicode(s.decode('utf-8')))
     except:
-        Log('Failed to unicodize: ' + filename)
+        Log('Failed to unicodize: ' + repr(filename))
     try:
         filename = re.sub(RE_UNICODE_CONTROL, '', filename)
     except:
-        Log('Couldn\'t strip control characters: ' + filename)
+        Log('Couldn\'t strip control characters: ' + repr(filename))
     return filename
 
 
-def cleanFilename(filename):
+def force_unicode(s):
+    if not isinstance(s, types.UnicodeType):
+        try:
+            s = s.decode("utf-8")
+        except UnicodeDecodeError:
+            t = chardet.detect(s)
+            try:
+                s = s.decode(t["encoding"])
+            except UnicodeDecodeError:
+                s = UnicodeDammit(s).unicode_markup
+    return s
+
+
+def clean_filename(filename):
     # this will remove any whitespace and punctuation chars and replace them with spaces, strip and return as lowercase
     return string.translate(filename.encode('utf-8'), string.maketrans(string.punctuation + string.whitespace,
-                                                                       ' ' * len(string.punctuation + string.whitespace))).strip().lower()
+                                                                       ' ' * len(
+                                                                           string.punctuation + string.whitespace))).strip().lower()
 
 
 def is_recent(t):
@@ -84,10 +112,11 @@ def str_pad(s, length, align='left', pad_char=' ', trim=False):
 
 def pad_title(value):
     """Pad a title to 30 characters to force the 'details' view."""
-    return str_pad(value, 30, pad_char=' ')
+    return str_pad(value, 49, pad_char=' ')
 
 
-def format_item(item, kind, parent=None, parent_title=None, section_title=None, add_section_title=False):
+def get_plex_item_display_title(item, kind, parent=None, parent_title=None, section_title=None,
+                                add_section_title=False):
     """
     :param item: plex item
     :param kind: show or movie
@@ -95,26 +124,64 @@ def format_item(item, kind, parent=None, parent_title=None, section_title=None,
     :param parent_title: parentTitle or None
     :return:
     """
-    return format_video(kind, item.title,
-                        section_title=(section_title or (parent.section.title if parent and getattr(parent, "section") else None)),
-                        parent_title=(parent_title or (parent.show.title if parent else None)),
-                        season=parent.index if parent else None,
-                        episode=item.index if kind == "show" else None,
-                        add_section_title=add_section_title)
+    return get_video_display_title(kind, item.title,
+                                   section_title=(
+                                       section_title or (parent.section.title if parent and getattr(parent, "section")
+                                                         else None)),
+                                   parent_title=(parent_title or (parent.show.title if parent else None)),
+                                   season=parent.index if parent else None,
+                                   episode=item.index if kind == "show" else None,
+                                   add_section_title=add_section_title)
 
 
-def format_video(kind, title, section_title=None, parent_title=None, season=None, episode=None, add_section_title=False):
+def get_video_display_title(kind, title, section_title=None, parent_title=None, season=None, episode=None,
+                            add_section_title=False):
     section_add = ""
     if add_section_title:
         section_add = ("%s: " % section_title) if section_title else ""
 
     if kind == "show" and parent_title:
         if season and episode:
-            return '%s%s S%02dE%02d, %s' % (section_add, parent_title, season or 0, episode or 0, title)
-        return '%s%s, %s' % (section_add, parent_title, title)
+            return '%s%s S%02dE%02d%s' % (section_add, parent_title, season or 0, episode or 0,
+                                          (", %s" % title if title else ""))
+        return '%s%s%s' % (section_add, parent_title, (", %s" % title if title else ""))
     return "%s%s" % (section_add, title)
 
 
+def get_title_for_video_metadata(metadata, add_section_title=True, add_episode_title=False):
+    """
+
+    :param metadata:
+    :param add_section_title:
+    :param add_episode_title: add the episode's title if its an episode else always add title
+    :return:
+    """
+    # compute item title
+    add_title = (add_episode_title and metadata["series_id"]) or not metadata["series_id"]
+    return get_video_display_title(
+        "show" if metadata["series_id"] else "movie",
+        metadata["title"] if add_title else "",
+        parent_title=metadata.get("series", None),
+        season=metadata.get("season", None),
+        episode=metadata.get("episode", None),
+        section_title=metadata.get("section", None),
+        add_section_title=add_section_title
+    )
+
+
+def get_identifier():
+    identifier = None
+    try:
+        identifier = Platform.MachineIdentifier
+    except:
+        pass
+
+    if not identifier:
+        identifier = String.UUID()
+
+    return Hash.SHA1(identifier + "SUBZEROOOOOOOOOO")
+
+
 def encode_message(base, s):
     return "%s?message=%s" % (base, urllib.quote_plus(s))
 
@@ -127,6 +194,10 @@ def timestamp():
     return int(time.time())
 
 
+def df(d):
+    return d.strftime("%Y-%m-%d %H:%M:%S") if d else "legacy data"
+
+
 def query_plex(url, args):
     """
     simple http query to the plex API without parsing anything too complicated
@@ -135,8 +206,6 @@ def query_plex(url, args):
     :return:
     """
     use_args = args.copy()
-    if "token" in Dict and Dict["token"]:
-        use_args["X-Plex-Token"] = Dict["token"]
 
     computed_args = "&".join(["%s=%s" % (key, String.Quote(value)) for key, value in use_args.iteritems()])
 
@@ -160,3 +229,77 @@ def check_write_permissions(path):
         # os.access check
         return os.access(path, os.W_OK | os.X_OK)
     return False
+
+
+def get_item_hints(data):
+    """
+    :param data: video item dict of media_to_videos 
+    :return: 
+    """
+    hints = {"title": data["title"], "type": "movie"}
+    if data["type"] == "episode":
+        hints.update(
+            {
+                "type": "episode",
+                "episode_title": data["title"],
+                "title": data["series"],
+            }
+        )
+    return hints
+
+
+def notify_executable(exe_info, videos, subtitles, storage):
+    variables = (
+        "subtitle_language", "subtitle_path", "subtitle_filename", "provider", "score", "storage", "series_id",
+        "series", "title", "section", "filename", "path", "folder", "season_id", "type", "id", "season"
+    )
+    exe, arguments = exe_info
+    for video, video_subtitles in subtitles.items():
+        for subtitle in video_subtitles:
+            lang = Locale.Language.Match(subtitle.language.alpha2)
+            data = video.plexapi_metadata.copy()
+            data.update({
+                "subtitle_language": lang,
+                "provider": subtitle.provider_name,
+                "score": subtitle.score,
+                "storage": storage,
+                "subtitle_path": subtitle.storage_path,
+                "subtitle_filename": os.path.basename(subtitle.storage_path)
+            })
+
+            # fill missing data with None
+            prepared_data = dict((v, data.get(v)) for v in variables)
+
+            prepared_arguments = [arg % prepared_data for arg in arguments]
+
+            Log.Debug(u"Calling %s with arguments: %s" % (exe, prepared_arguments))
+            try:
+                output = subprocess.check_output(subprocess.list2cmdline([exe] + prepared_arguments),
+                                                 stderr=subprocess.STDOUT, shell=True)
+            except subprocess.CalledProcessError:
+                Log.Error(u"Calling %s failed: %s" % (exe, traceback.format_exc()))
+            else:
+                Log.Debug(u"Process output: %s" % output)
+
+
+def track_usage(category=None, action=None, label=None, value=None):
+    if not cast_bool(Prefs["track_usage"]):
+        return
+
+    Thread.Create(dispatch_track_usage, category, action, label, value,
+                  identifier=Dict["anon_id"], first_use=Dict["first_use"],
+                  add=Network.PublicAddress)
+
+
+def dispatch_track_usage(*args, **kwargs):
+    identifier = kwargs.pop("identifier")
+    first_use = kwargs.pop("first_use")
+    add = kwargs.pop("add")
+    try:
+        track_event(identifier=identifier, first_use=first_use, add=add, *[str(a) for a in args])
+    except:
+        Log.Debug("Something went wrong when reporting anonymous user statistics: %s", traceback.format_exc())
+
+
+def get_language(lang_short):
+    return Language.fromietf(lang_short)

Contents/Code/support/history.py

@@ -0,0 +1,4 @@
+# coding=utf-8
+from subzero.history_storage import SubtitleHistory
+
+get_history = lambda: SubtitleHistory(Data, int(Prefs["history_size"]))

Contents/Code/support/items.py

@@ -3,11 +3,11 @@
 import logging
 import re
 import types
+import os
 from ignore import ignore_list
-from helpers import is_recent, format_item, query_plex
-from subzero import intent
-from lib import Plex
-from config import config
+from helpers import is_recent, get_plex_item_display_title, query_plex
+from lib import Plex, get_intent
+from config import config, IGNORE_FN
 
 logger = logging.getLogger(__name__)
 
@@ -16,6 +16,46 @@ MI_KIND, MI_TITLE, MI_KEY, MI_DEEPER, MI_ITEM = 0, 1, 2, 3, 4
 container_size_re = re.compile(ur'totalSize="(\d+)"')
 
 
+def get_item(key):
+    item_id = int(key)
+    item_container = Plex["library"].metadata(item_id)
+
+    try:
+        return list(item_container)[0]
+    except IndexError:
+        pass
+
+
+def get_item_kind(item):
+    return type(item).__name__
+
+
+PLEX_API_TYPE_MAP = {
+    "Show": "series",
+    "Season": "season",
+    "Episode": "episode",
+    "Movie": "movie",
+}
+
+
+def get_item_kind_from_rating_key(key):
+    item = get_item(key)
+    return PLEX_API_TYPE_MAP[get_item_kind(item)]
+
+
+def get_item_kind_from_item(item):
+    return PLEX_API_TYPE_MAP[get_item_kind(item)]
+
+
+def get_item_thumb(item):
+    kind = get_item_kind(item)
+    if kind == "Episode":
+        return item.show.thumb
+    elif kind == "Section":
+        return item.art
+    return item.thumb
+
+
 def get_items_info(items):
     return items[0][MI_KIND], items[0][MI_DEEPER]
 
@@ -24,7 +64,7 @@ def get_kind(items):
     return items[0][MI_KIND]
 
 
-def getSectionSize(key):
+def get_section_size(key):
     """
     quick query to determine the section size
     :param key:
@@ -44,7 +84,7 @@ def getSectionSize(key):
     return size
 
 
-def getItems(key="recently_added", base="library", value=None, flat=False, add_section_title=False):
+def get_items(key="recently_added", base="library", value=None, flat=False, add_section_title=False):
     """
     try to handle all return types plex throws at us and return a generalized item tuple
     """
@@ -66,12 +106,23 @@ def getItems(key="recently_added", base="library", value=None, flat=False, add_s
         else:
             kind = item.type
 
+        # only return items for our enabled sections
+        section_key = None
+        if kind == "section":
+            section_key = item.key
+        else:
+            if hasattr(item, "section_key"):
+                section_key = getattr(item, "section_key")
+
+        if section_key and section_key not in config.enabled_sections:
+            continue
+
         if kind == "season":
             # fixme: i think this case is unused now
             if flat:
                 # return episodes
                 for child in item.children():
-                    items.append(("episode", format_item(child, "show", parent=item, add_section_title=add_section_title), int(item.rating_key),
+                    items.append(("episode", get_plex_item_display_title(child, "show", parent=item, add_section_title=add_section_title), int(item.rating_key),
                                   False, child))
             else:
                 # return seasons
@@ -81,32 +132,28 @@ def getItems(key="recently_added", base="library", value=None, flat=False, add_s
             items.append(("directory", item.title, item.key, True, item))
 
         elif kind == "section":
-            item.size = getSectionSize(item.key)
-            items.append(("section", item.title, int(item.key), True, item))
+            if item.type in ['movie', 'show']:
+                item.size = get_section_size(item.key)
+                items.append(("section", item.title, int(item.key), True, item))
 
         elif kind == "episode":
             items.append(
-                (kind, format_item(item, "show", parent=item.season, parent_title=item.show.title, section_title=item.section.title,
-                                   add_section_title=add_section_title), int(item.rating_key), False, item))
+                (kind, get_plex_item_display_title(item, "show", parent=item.season, parent_title=item.show.title, section_title=item.section.title,
+                                                   add_section_title=add_section_title), int(item.rating_key), False, item))
 
         elif kind in ("movie", "artist", "photo"):
-            items.append((kind, format_item(item, kind, section_title=item.section.title, add_section_title=add_section_title),
+            items.append((kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title),
                           int(item.rating_key), False, item))
 
         elif kind == "show":
             items.append((
-                kind, format_item(item, kind, section_title=item.section.title, add_section_title=add_section_title), int(item.rating_key), True,
+                kind, get_plex_item_display_title(item, kind, section_title=item.section.title, add_section_title=add_section_title), int(item.rating_key), True,
                 item))
 
     return items
 
 
-def getRecentlyAddedItems():
-    items = getItems(key="recently_added")
-    return filter(lambda x: is_recent(x[MI_ITEM].added_at), items)
-
-
-def getRecentItems():
+def get_recent_items():
     """
     actually get the recent items, not limited like /library/recentlyAdded
     :return:
@@ -128,7 +175,9 @@ def getRecentItems():
     recent = []
 
     for section in Plex["library"].sections():
-        if section.type not in ("movie", "show") or section.key in ignore_list.sections:
+        if section.type not in ("movie", "show") \
+                or section.key not in config.enabled_sections \
+                or section.key in ignore_list.sections:
             Log.Debug(u"Skipping section: %s" % section.title)
             continue
 
@@ -155,17 +204,82 @@ def getRecentItems():
     return recent
 
 
-def getOnDeckItems():
-    return getItems(key="on_deck", add_section_title=True)
+def get_on_deck_items():
+    return get_items(key="on_deck", add_section_title=True)
 
 
-def getAllItems(key, base="library", value=None, flat=False):
-    return getItems(key, base=base, value=value, flat=flat)
+def get_recently_added_items():
+    return get_items(key="recently_added", add_section_title=True, flat=False)
 
 
-def refreshItem(rating_key, force=False, timeout=8000):
+def get_all_items(key, base="library", value=None, flat=False):
+    return get_items(key, base=base, value=value, flat=flat)
+
+
+def is_ignored(rating_key, item=None):
+    """
+    check whether an item, its show/season/section is in the soft or the hard ignore list
+    :param rating_key:
+    :param item:
+    :return:
+    """
+    # item in soft ignore list
+    if rating_key in ignore_list["videos"]:
+        Log.Debug("Item %s is in the soft ignore list" % rating_key)
+        return True
+
+    item = item or get_item(rating_key)
+    kind = get_item_kind(item)
+
+    # show in soft ignore list
+    if kind == "Episode" and item.show.rating_key in ignore_list["series"]:
+        Log.Debug("Item %s's show is in the soft ignore list" % rating_key)
+        return True
+
+    # section in soft ignore list
+    if item.section.key in ignore_list["sections"]:
+        Log.Debug("Item %s's section is in the soft ignore list" % rating_key)
+        return True
+
+    # physical/path ignore
+    if config.ignore_sz_files or config.ignore_paths:
+        # normally check current item folder and the library
+        check_ignore_paths = [".", "../"]
+        if kind == "Episode":
+            # series/episode, we've got a season folder here, also
+            check_ignore_paths.append("../../")
+
+        for part in item.media.parts:
+            if config.ignore_paths and config.is_path_ignored(part.file):
+                Log.Debug("Item %s's path is manually ignored" % rating_key)
+                return True
+
+            if config.ignore_sz_files:
+                for sub_path in check_ignore_paths:
+                    if config.is_physically_ignored(os.path.abspath(os.path.join(os.path.dirname(part.file), sub_path))):
+                        Log.Debug("An ignore file exists in either the items or its parent folders")
+                        return True
+
+    return False
+
+
+def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, parent_rating_key=None):
+    intent = get_intent()
+
     # timeout actually is the time for which the intent will be valid
     if force:
+        Log.Debug("Setting intent for force-refresh of %s to timeout: %s", rating_key, timeout)
         intent.set("force", rating_key, timeout=timeout)
-    Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", rating_key)
-    Plex["library/metadata"].refresh(rating_key)
+
+        # force Dict.Save()
+        intent.store.save()
+
+    refresh = [rating_key]
+
+    if refresh_kind == "season":
+        # season refresh, needs explicit per-episode refresh
+        refresh = [item.rating_key for item in list(Plex["library/metadata"].children(int(rating_key)))]
+
+    for key in refresh:
+        Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", key)
+        Plex["library/metadata"].refresh(key)

Contents/Code/support/lib.py

@@ -1,18 +1,56 @@
 # coding=utf-8
 
-from plex import Plex
-from auth import refresh_plex_token
+import plex
+from subzero.intent import TempIntent
+from subzero.lib.dict import DictProxy
+from subzero.lib.httpfake import PlexPyNativeResponseProxy
+from subzero.constants import DEFAULT_TIMEOUT
 
 
-def configure_plex():
-    # this may be the only viable usage of global :O (correct me if i'm wrong)
-    global Plex
-    if not "token" in Dict or not (Prefs["plex_username"] and Prefs["plex_password"]):
-        refresh_plex_token()
+class PlexPyNativeRequestProxy(object):
+    """
+    A really dumb object that tries to mimic requests.Request in an incomplete way, so that plex.Plex
+    uses native plex HTTPRequests instead of the better requests.Request class.
 
-    # initialize Plex api
-    Plex.configuration.defaults.authentication(Dict["token"] if "token" in Dict else None)
+    This allows us to operate freely on 127.0.0.1's PMS.
+
+    To be used in conjunction with subzero.lib.httpfake.PlexPyNativeResponseProxy
+    """
+    url = None
+    data = None
+    headers = None
+    method = None
+
+    def prepare(self):
+        return self
+
+    def send(self):
+        # fixme: add self.data to HTTP.Request
+        data = None
+        status_code = 200
+        try:
+            data = HTTP.Request(self.url, headers=self.headers, immediate=True, method=self.method,
+                                timeout=DEFAULT_TIMEOUT)
+        except Ex.HTTPError as e:
+            status_code = e.code
+        return PlexPyNativeResponseProxy(data, status_code, self)
 
 
-lib_unaccessible_error = "\n\n\n!!!!!!!!!!!!!! ATTENTION !!!!!!!!!!!!! \nCan't access your Plex Media Servers' API.\nAre you using Plex Home?"" \
-""Please configure your Plex.tv credentials! Advanced features disabled!\n\n\n"
+plex.request.Request = PlexPyNativeRequestProxy
+
+Plex = plex.Plex
+
+
+class IntentDictStorage(DictProxy):
+    store = "intent"
+
+    def setup_defaults(self):
+        return {"force": {}}
+
+
+def get_intent():
+    """
+    use this to get an intent from inside a separate thread
+    :return:
+    """
+    return TempIntent(store=IntentDictStorage(Dict))

Contents/Code/support/localmedia.py

@@ -1,22 +1,25 @@
 # coding=utf-8
 
 import os
+
 import config
 import helpers
 import subtitlehelpers
 
-from subzero.lib.io import getViableEncoding
+from config import config as sz_config
 
 
-def findSubtitles(part):
+def find_subtitles(part):
     lang_sub_map = {}
     part_filename = helpers.unicodize(part.file)
     part_basename = os.path.splitext(os.path.basename(part_filename))[0]
-    use_filesystem = bool(Prefs["subtitles.save.filesystem"])
+    use_filesystem = helpers.cast_bool(Prefs["subtitles.save.filesystem"])
     paths = [os.path.dirname(part_filename)] if use_filesystem else []
 
     global_subtitle_folder = None
 
+    global_folders = []
+
     if use_filesystem:
         # Check for local subtitles subdirectory
         sub_dir_base = paths[0]
@@ -27,15 +30,20 @@ def findSubtitles(part):
             # got selected subfolder
             sub_dir_list.append(os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"]))
 
-        sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
+        sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \
+            if Prefs["subtitles.save.subFolder.Custom"] else None
+
         if sub_dir_custom:
             # got custom subfolder
-            if sub_dir_custom.startswith("/"):
+            sub_dir_custom = os.path.normpath(sub_dir_custom)
+            if os.path.isdir(sub_dir_custom) and os.path.isabs(sub_dir_custom):
                 # absolute folder
                 sub_dir_list.append(sub_dir_custom)
+                global_folders.append(sub_dir_custom)
             else:
                 # relative folder
-                sub_dir_list.append(os.path.join(sub_dir_base, sub_dir_custom))
+                fld = os.path.join(sub_dir_base, sub_dir_custom)
+                sub_dir_list.append(fld)
 
         for sub_dir in sub_dir_list:
             if os.path.isdir(sub_dir):
@@ -45,6 +53,10 @@ def findSubtitles(part):
         global_subtitle_folder = os.path.join(Core.app_support_path, 'Subtitles')
         if os.path.exists(global_subtitle_folder):
             paths.append(global_subtitle_folder)
+            global_folders.append(global_subtitle_folder)
+
+    # normalize all paths
+    paths = [os.path.normpath(helpers.unicodize(path)) for path in paths]
 
     # We start by building a dictionary of files to their absolute paths. We also need to know
     # the number of media files that are actually present, in case the found local media asset
@@ -52,16 +64,15 @@ def findSubtitles(part):
     #
     file_paths = {}
     total_media_files = 0
+    media_files = []
     for path in paths:
-        path = helpers.unicodize(path)
-        for file_path_listing in os.listdir(path):
-
+        for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)):
             # When using os.listdir with a unicode path, it will always return a string using the
             # NFD form. However, we internally are using the form NFC and therefore need to convert
             # it to allow correct regex / comparisons to be performed.
             #
             file_path_listing = helpers.unicodize(file_path_listing)
-            if os.path.isfile(os.path.join(path, file_path_listing).encode(getViableEncoding())):
+            if os.path.isfile(os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)):
                 file_paths[file_path_listing.lower()] = os.path.join(path, file_path_listing)
 
             # If we've found an actual media file, we should record it.
@@ -69,29 +80,94 @@ def findSubtitles(part):
             if ext.lower()[1:] in config.VIDEO_EXTS:
                 total_media_files += 1
 
+                # collect found media files
+                media_files.append(root)
+
+    # cleanup any leftover subtitle if no associated media file was found
+    if helpers.cast_bool(Prefs["subtitles.autoclean"]):
+        for path in paths:
+            # we can't housekeep the global subtitle folders as we don't know about *all* media files
+            # in a library; skip them
+            skip_path = False
+            for fld in global_folders:
+                if path.startswith(fld):
+                    Log.Info("Skipping housekeeping of folder: %s", path)
+                    skip_path = True
+                    break
+
+            if skip_path:
+                continue
+
+            for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)):
+                file_path_listing = helpers.unicodize(file_path_listing)
+                enc_fn = os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)
+
+                if os.path.isfile(enc_fn):
+                    (root, ext) = os.path.splitext(file_path_listing)
+                    # it's a subtitle file
+                    if ext.lower()[1:] in config.SUBTITLE_EXTS:
+                        # get fn without forced/default/normal tag
+                        split_tag = root.rsplit(".", 1)
+                        if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default']:
+                            root = split_tag[0]
+
+                        # get associated media file name without language
+                        sub_fn = subtitlehelpers.ENDSWITH_LANGUAGECODE_RE.sub("", root)
+
+                        # subtitle basename and basename without possible language tag  not found in collected
+                        # media files? kill.
+                        if root not in media_files and sub_fn not in media_files:
+                            Log.Info("Removing leftover subtitle: %s", os.path.join(path, file_path_listing))
+                            try:
+                                os.remove(enc_fn)
+                            except (OSError, IOError):
+                                Log.Error("Removing failed")
+
     Log('Looking for subtitle media in %d paths with %d media files.', len(paths), total_media_files)
     Log('Paths: %s', ", ".join([helpers.unicodize(p) for p in paths]))
 
     for file_path in file_paths.values():
+        local_filename = os.path.basename(file_path)
+        bn, ext = os.path.splitext(local_filename)
+        local_basename = helpers.unicodize(bn)
 
-        local_basename = helpers.unicodize(os.path.splitext(os.path.basename(file_path))[0])
+        # get fn without forced/default/normal tag
+        split_tag = local_basename.rsplit(".", 1)
+        has_additional_tag = False
+        if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default', 'embedded', 'custom']:
+            local_basename = split_tag[0]
+            has_additional_tag = True
+
+        # split off possible language tag
         local_basename2 = local_basename.rsplit('.', 1)[0]
         filename_matches_part = local_basename == part_basename or local_basename2 == part_basename
+        filename_contains_part = part_basename in local_basename
 
-        # If the file is located within the global subtitle folder and it's name doesn't match exactly
-        # then we should simply ignore it.
-        #
-        if global_subtitle_folder and file_path.count(global_subtitle_folder) and not filename_matches_part:
+        if not ext.lower()[1:] in config.SUBTITLE_EXTS:
             continue
 
-        # If we have more than one media file within the folder and located filename doesn't match
-        # exactly then we should simply ignore it.
-        #
-        if total_media_files > 1 and not filename_matches_part:
-            continue
+        # if the file is located within the global subtitle folders and its name doesn't match exactly, ignore it
+        if global_folders and not filename_matches_part:
+            skip_path = False
+            for fld in global_folders:
+                if file_path.startswith(fld):
+                    skip_path = True
+                    break
 
-        subtitle_helper = subtitlehelpers.SubtitleHelpers(file_path)
-        if subtitle_helper != None:
+            if skip_path:
+                continue
+
+        # determine whether to pick up the subtitle based on our match strictness
+        elif not filename_matches_part:
+            if sz_config.ext_match_strictness == "strict" or (
+                    sz_config.ext_match_strictness == "loose" and not filename_contains_part):
+
+                #Log.Debug("%s doesn't match %s, skipping" % (helpers.unicodize(local_filename),
+                #                                             helpers.unicodize(part_basename)))
+                continue
+
+        subtitle_helper = subtitlehelpers.subtitle_helpers(file_path)
+        if subtitle_helper is not None:
             local_lang_map = subtitle_helper.process_subtitles(part)
             for new_language, subtitles in local_lang_map.items():
 
@@ -104,7 +180,7 @@ def findSubtitles(part):
 
     # add known metadata subs to our sub list
     if not use_filesystem:
-        for language, sub_list in subtitlehelpers.getSubtitlesFromMetadata(part).iteritems():
+        for language, sub_list in subtitlehelpers.get_subtitles_from_metadata(part).iteritems():
             if sub_list:
                 if language not in lang_sub_map:
                     lang_sub_map[language] = []

Contents/Code/support/missing_subtitles.py

@@ -1,22 +1,22 @@
 # coding=utf-8
 import traceback
+
 from support.config import config
-from support.helpers import format_item
-from lib import Plex
+from support.helpers import get_plex_item_display_title, cast_bool
+from support.items import get_item
+from support.lib import Plex
 
 
-def itemDiscoverMissing(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
+def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
     existing_subs = {"internal": [], "external": [], "count": 0}
 
     item_id = int(rating_key)
-    item_container = Plex["library"].metadata(item_id)
-
-    item = list(item_container)[0]
+    item = get_item(rating_key)
 
     if kind == "show":
-        item_title = format_item(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
+        item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
     else:
-        item_title = format_item(item, kind, section_title=section_title)
+        item_title = get_plex_item_display_title(item, kind, section_title=section_title)
 
     video = item.media
 
@@ -44,34 +44,34 @@ def itemDiscoverMissing(rating_key, kind="show", added_at=None, section_title=No
         Log.Info(u"Subs still missing for '%s': %s", item_title, missing)
 
     if missing:
-        return added_at, item_id, item_title
+        return added_at, item_id, item_title, item, missing
 
 
-def getAllMissing(items):
+def items_get_all_missing_subs(items):
     missing = []
     for added_at, kind, section_title, key in items:
         try:
-            state = itemDiscoverMissing(
+            state = item_discover_missing_subs(
                 key,
                 kind=kind,
                 added_at=added_at,
                 section_title=section_title,
-                languages=config.langList,
-                internal=bool(Prefs["subtitles.scan.embedded"]),
-                external=bool(Prefs["subtitles.scan.external"])
+                languages=config.lang_list,
+                internal=cast_bool(Prefs["subtitles.scan.embedded"]),
+                external=cast_bool(Prefs["subtitles.scan.external"])
             )
             if state:
-                # (added_at, item_id, title)
+                # (added_at, item_id, title, item, missing_languages)
                 missing.append(state)
         except:
             Log.Error("Something went wrong when getting the state of item %s: %s", key, traceback.format_exc())
     return missing
 
 
-def refresh_item(item, title):
+def refresh_item(item):
     Plex["library/metadata"].refresh(item)
 
 
 def refresh_items(items):
     for item, title in items:
-        refresh_item(item, title)
+        refresh_item(item)

Contents/Code/support/plex_media.py

@@ -0,0 +1,259 @@
+# coding=utf-8
+
+import os
+
+import helpers
+
+from items import get_item
+from lib import get_intent, Plex
+from config import config
+from subzero.video import parse_video
+
+
+def get_metadata_dict(item, part, add):
+    data = {
+        "item": item,
+        "section": item.section.title,
+        "path": part.file,
+        "folder": os.path.dirname(part.file),
+        "filename": os.path.basename(part.file)
+    }
+    data.update(add)
+    return data
+
+
+def media_to_videos(media, kind="series"):
+    """
+    iterates through media and returns the associated parts (videos)
+    :param media:
+    :param kind:
+    :return:
+    """
+    videos = []
+
+    if kind == "series":
+        for season in media.seasons:
+            season_object = media.seasons[season]
+            for episode in media.seasons[season].episodes:
+                ep = media.seasons[season].episodes[episode]
+
+                # get plex item via API for additional metadata
+                plex_episode = get_item(ep.id)
+
+                for item in media.seasons[season].episodes[episode].items:
+                    for part in item.parts:
+                        videos.append(
+                            get_metadata_dict(plex_episode, part,
+                                              {"plex_part": part, "type": "episode", "title": ep.title,
+                                               "series": media.title, "id": ep.id,
+                                               "series_id": media.id, "season_id": season_object.id,
+                                               "episode": plex_episode.index, "season": plex_episode.season.index,
+                                               "section": plex_episode.section.title
+                                               })
+                        )
+    else:
+        plex_item = get_item(media.id)
+        for item in media.items:
+            for part in item.parts:
+                videos.append(
+                    get_metadata_dict(plex_item, part, {"plex_part": part, "type": "movie",
+                                                        "title": media.title, "id": media.id,
+                                                        "series_id": None,
+                                                        "season_id": None,
+                                                        "section": plex_item.section.title})
+                )
+    return videos
+
+
+IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")
+
+
+def get_stream_fps(streams):
+    """
+    accepts a list of plex streams or a list of the plex api streams
+    """
+    for stream in streams:
+        # video
+        stream_type = getattr(stream, "type", getattr(stream, "stream_type", None))
+        if stream_type == 1:
+            return getattr(stream, "frameRate", getattr(stream, "frame_rate", "25.000"))
+    return "25.000"
+
+
+def get_media_item_ids(media, kind="series"):
+    ids = []
+    if kind == "movies":
+        ids.append(media.id)
+    else:
+        for season in media.seasons:
+            for episode in media.seasons[season].episodes:
+                ids.append(media.seasons[season].episodes[episode].id)
+
+    return ids
+
+
+def scan_video(plex_part, ignore_all=False, hints=None, rating_key=None):
+    """
+    returnes a subliminal/guessit-refined parsed video
+    :param plex_part: 
+    :param ignore_all: 
+    :param hints: 
+    :param rating_key: 
+    :return: 
+    """
+    embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
+    external_subtitles = not ignore_all and Prefs['subtitles.scan.external']
+
+    if ignore_all:
+        Log.Debug("Force refresh intended.")
+
+    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (
+        plex_part.file, external_subtitles, embedded_subtitles))
+
+    known_embedded = []
+    parts = list(Plex["library"].metadata(rating_key))[0].media.parts
+    plexpy_part = None
+    for part in parts:
+        if int(part.id) == int(plex_part.id):
+            plexpy_part = part
+
+    # embedded subtitles
+    if plexpy_part:
+        for stream in plexpy_part.streams:
+            # subtitle stream
+            if stream.stream_type == 3:
+                if (config.forced_only and getattr(stream, "forced")) or \
+                        (not config.forced_only and not getattr(stream, "forced")):
+
+                    # embedded subtitle
+                    if not stream.stream_key:
+                        if config.exotic_ext or stream.codec in ("srt", "ass", "ssa"):
+                            lang_code = stream.language_code
+
+                            # treat unknown language as lang1?
+                            if not lang_code and config.treat_und_as_first:
+                                lang_code = list(config.lang_list)[0].alpha3
+                            known_embedded.append(lang_code)
+    else:
+        Log.Warn("Part %s missing of %s, not able to scan internal streams", plex_part.id, rating_key)
+
+    try:
+        # get basic video info scan (filename)
+        video = parse_video(plex_part.file, hints, external_subtitles=external_subtitles,
+                            embedded_subtitles=embedded_subtitles, known_embedded=known_embedded,
+                            forced_only=config.forced_only, video_fps=plex_part.fps)
+
+        return video
+
+    except ValueError:
+        Log.Warn("File could not be guessed by subliminal: %s" % plex_part.file)
+
+
+def scan_videos(videos, kind="series", ignore_all=False):
+    """
+    receives a list of videos containing dictionaries returned by media_to_videos
+    :param videos:
+    :param kind: series or movies
+    :return: dictionary of subliminal.video.scan_video, key=subliminal scanned video, value=plex file part
+    """
+    ret = {}
+    for video in videos:
+        intent = get_intent()
+        force_refresh = intent.get("force", video["id"], video["series_id"], video["season_id"])
+        Log.Debug("Determining force-refresh (video: %s, series: %s, season: %s), result: %s"
+                  % (video["id"], video["series_id"], video["season_id"], force_refresh))
+
+        hints = helpers.get_item_hints(video)
+        video["plex_part"].fps = get_stream_fps(video["plex_part"].streams)
+        scanned_video = scan_video(video["plex_part"], ignore_all=force_refresh or ignore_all, hints=hints,
+                                   rating_key=video["id"])
+
+        if not scanned_video:
+            continue
+
+        scanned_video.id = video["id"]
+        part_metadata = video.copy()
+        del part_metadata["plex_part"]
+        scanned_video.plexapi_metadata = part_metadata
+        ret[scanned_video] = video["plex_part"]
+    return ret
+
+
+class PartUnknownException(Exception):
+    pass
+
+
+def get_plex_metadata(rating_key, part_id, item_type):
+    """
+    uses the Plex 3rd party API accessor to get metadata information
+
+    :param rating_key:
+    :param part_id:
+    :param item_type:
+    :return:
+    """
+
+    plex_item = list(Plex["library"].metadata(rating_key))[0]
+
+    # find current part
+    current_part = None
+    for part in plex_item.media.parts:
+        if str(part.id) == part_id:
+            current_part = part
+
+    if not current_part:
+        raise PartUnknownException("Part unknown")
+
+    # get normalized metadata
+    if item_type == "episode":
+        metadata = get_metadata_dict(plex_item, current_part,
+                                     {"plex_part": current_part, "type": "episode", "title": plex_item.title,
+                                      "series": plex_item.show.title, "id": plex_item.rating_key,
+                                      "series_id": plex_item.show.rating_key,
+                                      "season_id": plex_item.season.rating_key,
+                                      "season": plex_item.season.index,
+                                      "episode": plex_item.index
+                                      })
+    else:
+        metadata = get_metadata_dict(plex_item, current_part, {"plex_part": current_part, "type": "movie",
+                                                               "title": plex_item.title, "id": plex_item.rating_key,
+                                                               "series_id": None,
+                                                               "season_id": None,
+                                                               "season": None,
+                                                               "episode": None,
+                                                               "section": plex_item.section.title})
+    return metadata
+
+
+class PMSMediaProxy(object):
+    """
+    Proxy object for getting data from a mediatree items "internally" via the PMS
+
+    note: this could be useful later on: Media.TV_Show(getattr(Metadata, "_access_point"), id=XXXXXX)
+    """
+
+    def __init__(self, media_id):
+        self.mediatree = Media.TreeForDatabaseID(media_id)
+
+    def get_part(self, part_id=None):
+        """
+        walk the mediatree until the given part was found; if no part was given, return the first one
+        :param part_id:
+        :return:
+        """
+        m = self.mediatree
+        while 1:
+            if m.items:
+                media_item = m.items[0]
+                if not part_id:
+                    return media_item.parts[0] if media_item.parts else None
+
+                for part in media_item.parts:
+                    if str(part.id) == str(part_id):
+                        return part
+                break
+
+            if not m.children:
+                break
+
+            m = m.children[0]

Contents/Code/support/storage.py

@@ -1,42 +1,100 @@
 # coding=utf-8
 
 import datetime
+import os
 import pprint
+import copy
+
+import subliminal
+from items import get_item
+from subzero.subtitle_storage import StoredSubtitlesManager
+
+from subtitlehelpers import force_utf8
+from config import config
+from helpers import notify_executable, get_title_for_video_metadata, cast_bool, force_unicode
+from plex_media import PMSMediaProxy
 
 
-def storeSubtitleInfo(videos, subtitles, storage_type):
+get_subtitle_storage = lambda: StoredSubtitlesManager(Data, get_item)
+
+
+def whack_missing_parts(scanned_video_part_map, existing_parts=None):
+    """
+    cleans out our internal storage's video parts (parts may get updated/deleted/whatever)
+    :param existing_parts: optional list of part ids known
+    :param scanned_video_part_map: videos to check for
+    :return:
+    """
+    # shortcut
+
+    if "subs" not in Dict:
+        return
+
+    if not existing_parts:
+        existing_parts = []
+        for part in scanned_video_part_map.viewvalues():
+            existing_parts.append(str(part.id))
+
+    whacked_parts = False
+    for video in scanned_video_part_map.keys():
+        video_id = str(video.id)
+        if video_id not in Dict["subs"]:
+            continue
+
+        parts = Dict["subs"][video_id].keys()
+
+        for part_id in parts:
+            part_id = str(part_id)
+            if part_id not in existing_parts:
+                Log.Info("Whacking part %s in internal storage of video %s (%s, %s)", part_id, video_id,
+                         repr(existing_parts), repr(parts))
+                del Dict["subs"][video_id][part_id]
+                whacked_parts = True
+
+    if whacked_parts:
+        Dict.Save()
+
+
+def store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage_type, mode="a"):
     """
     stores information about downloaded subtitles in plex's Dict()
     """
-    if not "subs" in Dict:
-        Dict["subs"] = {}
+    existing_parts = []
+    for video, video_subtitles in downloaded_subtitles.items():
+        part = scanned_video_part_map[video]
+        part_id = str(part.id)
+        video_id = str(video.id)
+        plex_item = get_item(video_id)
+        metadata = video.plexapi_metadata
+        title = get_title_for_video_metadata(metadata)
 
-    storage = Dict["subs"]
+        subtitle_storage = get_subtitle_storage()
+        stored_subs = subtitle_storage.load_or_new(plex_item)
 
-    for video, video_subtitles in subtitles.items():
-        part = videos[video]
+        existing_parts.append(part_id)
 
-        if not video.id in storage:
-            storage[video.id] = {}
-
-        video_dict = storage[video.id]
-        if not part.id in video_dict:
-            video_dict[part.id] = {}
-
-        part_dict = video_dict[part.id]
+        stored_any = False
         for subtitle in video_subtitles:
             lang = Locale.Language.Match(subtitle.language.alpha2)
-            if lang not in part_dict:
-                part_dict[lang] = {}
-            lang_dict = part_dict[lang]
-            sub_key = (subtitle.provider_name, subtitle.id)
-            lang_dict[sub_key] = dict(score=subtitle.score, link=subtitle.page_link, storage=storage_type, hash=Hash.MD5(subtitle.content),
-                                      date_added=datetime.datetime.now())
+            Log.Debug(u"Adding subtitle to storage: %s, %s, %s" % (video_id, part_id, title))
+            ret_val = stored_subs.add(part_id, lang, subtitle, storage_type, mode=mode)
 
-    Dict.Save()
+            if ret_val:
+                Log.Debug("Subtitle stored")
+                stored_any = True
+
+            else:
+                Log.Debug("Subtitle already existing in storage")
+
+        if stored_any:
+            Log.Debug("Saving subtitle storage for %s" % video_id)
+            subtitle_storage.save(stored_subs)
+
+    #if existing_parts:
+    #    whack_missing_parts(scanned_video_part_map, existing_parts=existing_parts)
 
 
-def resetStorage(key):
+def reset_storage(key):
     """
     resets the Dict[key] storage, thanks to https://docs.google.com/document/d/1hhLjV1pI-TA5y91TiJq64BdgKwdLnFt4hWgeOqpz1NA/edit#
     We can't use the nice Plex interface for this, as it calls get multiple times before set
@@ -48,6 +106,82 @@ def resetStorage(key):
     Dict.Save()
 
 
-def logStorage(key):
+def log_storage(key):
     if key in Dict:
         Log.Debug(pprint.pformat(Dict[key]))
+
+
+def save_subtitles_to_file(subtitles):
+    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \
+        if Prefs["subtitles.save.subFolder.Custom"] else None
+
+    for video, video_subtitles in subtitles.items():
+        if not video_subtitles:
+            continue
+
+        fld = None
+        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
+            # specific subFolder requested, create it if it doesn't exist
+            fld_base = os.path.split(video.name)[0]
+            if fld_custom:
+                if fld_custom.startswith("/"):
+                    # absolute folder
+                    fld = fld_custom
+                else:
+                    fld = os.path.join(fld_base, fld_custom)
+            else:
+                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
+            fld = force_unicode(fld)
+            if not os.path.exists(fld):
+                os.makedirs(fld)
+        subliminal.save_subtitles(video, video_subtitles, directory=fld, single=cast_bool(Prefs['subtitles.only_one']),
+                                      encode_with=force_utf8 if config.enforce_encoding else None,
+                                      chmod=config.chmod, forced_tag=config.forced_only, path_decoder=force_unicode)
+    return True
+
+
+def save_subtitles_to_metadata(videos, subtitles):
+    for video, video_subtitles in subtitles.items():
+        mediaPart = videos[video]
+        for subtitle in video_subtitles:
+            content = force_utf8(subtitle.text) if config.enforce_encoding else subtitle.content
+
+            if not isinstance(mediaPart, Framework.api.agentkit.MediaPart):
+                # we're being handed a Plex.py model instance here, not an internal PMS MediaPart object.
+                # get the correct one
+                mp = PMSMediaProxy(video.id).get_part(mediaPart.id)
+            else:
+                mp = mediaPart
+            mp.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.id] = Proxy.Media(content, ext="srt")
+    return True
+
+
+def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
+    meta_fallback = False
+    save_successful = False
+    storage = "metadata"
+    if Prefs['subtitles.save.filesystem']:
+        storage = "filesystem"
+        try:
+            Log.Debug("Using filesystem as subtitle storage")
+            save_subtitles_to_file(downloaded_subtitles)
+        except OSError:
+            if Prefs["subtitles.save.metadata_fallback"]:
+                meta_fallback = True
+            else:
+                raise
+        else:
+            save_successful = True
+
+    if not Prefs['subtitles.save.filesystem'] or meta_fallback:
+        if meta_fallback:
+            Log.Debug("Using metadata as subtitle storage, because filesystem storage failed")
+        else:
+            Log.Debug("Using metadata as subtitle storage")
+        save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles)
+
+    if save_successful and config.notify_executable:
+        notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage)
+
+    store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)
+

Contents/Code/support/subtitlehelpers.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-import re, unicodedata, os
+import re, os
 import config
 import helpers
 
@@ -12,9 +12,14 @@ class SubtitleHelper(object):
         self.filename = filename
 
 
-def SubtitleHelpers(filename):
+def subtitle_helpers(filename):
     filename = helpers.unicodize(filename)
-    for cls in [VobSubSubtitleHelper, DefaultSubtitleHelper]:
+    helper_classes = [DefaultSubtitleHelper]
+
+    if helpers.cast_bool(Prefs["subtitles.scan.exotic_ext"]):
+        helper_classes.insert(0, VobSubSubtitleHelper)
+
+    for cls in helper_classes:
         if cls.is_helper_for(filename):
             return cls(filename)
     return None
@@ -79,6 +84,20 @@ class VobSubSubtitleHelper(SubtitleHelper):
 
 #####################################################################################################################
 
+
+IETF_MATCH = ".+\.([^-.]+)(?:-[A-Za-z]+)?$"
+ENDSWITH_LANGUAGECODE_RE = re.compile("\.([^-.]{2,3})(?:-[A-Za-z]{2})?$")
+
+
+def match_ietf_language(s):
+    language_match = re.match(".+\.([^\.]+)$" if not helpers.cast_bool(Prefs["subtitles.language.ietf"])
+                              else IETF_MATCH, s)
+    if language_match and len(language_match.groups()) == 1:
+        language = language_match.groups()[0]
+        return language
+    return s
+
+
 class DefaultSubtitleHelper(SubtitleHelper):
     @classmethod
     def is_helper_for(cls, filename):
@@ -89,20 +108,35 @@ class DefaultSubtitleHelper(SubtitleHelper):
 
         lang_sub_map = {}
 
+        if not os.path.exists(self.filename):
+            return lang_sub_map
+
         basename = os.path.basename(self.filename)
         (file, ext) = os.path.splitext(self.filename)
 
         # Remove the initial '.' from the extension
         ext = ext[1:]
 
+        forced = ''
+        default = ''
+        split_tag = file.rsplit('.', 1)
+        if len(split_tag) > 1 and split_tag[1].lower() in ['forced', 'normal', 'default', 'embedded', 'custom']:
+            file = split_tag[0]
+            # don't do anything with 'normal', we don't need it
+            if 'forced' == split_tag[1].lower():
+                forced = '1'
+            if 'default' == split_tag[1].lower():
+                default = '1'
+
         # Attempt to extract the language from the filename (e.g. Avatar (2009).eng)
         language = ""
 
         # IETF support thanks to https://github.com/hpsbranco/LocalMedia.bundle/commit/4fad9aefedece78a1fa96401304351347f644369
-        language_match = re.match(".+\.([^\.]+)$" if not Prefs["subtitles.language.ietf"] else ".+\.([^-.]+)(?:-[A-Za-z]+)?$", file)
-        if language_match and len(language_match.groups()) == 1:
-            language = language_match.groups()[0]
-        language = Locale.Language.Match(language)
+        language = Locale.Language.Match(match_ietf_language(file))
+
+        # skip non-SRT if wanted
+        if not helpers.cast_bool(Prefs["subtitles.scan.exotic_ext"]) and ext not in ["srt", "ass", "ssa"]:
+            return lang_sub_map
 
         codec = None
         format = None
@@ -130,14 +164,16 @@ class DefaultSubtitleHelper(SubtitleHelper):
         if format is None:
             format = codec
 
-        Log('Found subtitle file: ' + self.filename + ' language: ' + language + ' codec: ' + str(codec) + ' format: ' + str(format))
-        part.subtitles[language][basename] = Proxy.LocalFile(self.filename, codec=codec, format=format)
+        Log('Found subtitle file: ' + self.filename + ' language: ' + language + ' codec: ' + str(
+            codec) + ' format: ' + str(format) + ' default: ' + default + ' forced: ' + forced)
+        part.subtitles[language][basename] = Proxy.LocalFile(self.filename, codec=codec, format=format, default=default,
+                                                             forced=forced)
 
         lang_sub_map[language] = [basename]
         return lang_sub_map
 
 
-def getSubtitlesFromMetadata(part):
+def get_subtitles_from_metadata(part):
     subs = {}
     for language in part.subtitles:
         subs[language] = []

Contents/Code/support/tasks.py

@@ -2,30 +2,49 @@
 
 import datetime
 import time
+import operator
+import traceback
 
-from missing_subtitles import getAllMissing, refresh_item
+from subliminal_patch.score import compute_score
+from subliminal_patch.core import download_subtitles
+from subliminal import list_subtitles as list_all_subtitles
+from babelfish import Language
+
+from missing_subtitles import items_get_all_missing_subs, refresh_item
 from background import scheduler
-from support.items import getRecentItems
+from storage import save_subtitles, whack_missing_parts, get_subtitle_storage
+from support.config import config
+from support.items import get_recent_items, is_ignored, get_item
+from support.lib import Plex
+from support.helpers import track_usage, get_title_for_video_metadata, cast_bool
+from support.plex_media import scan_videos, get_plex_metadata, PartUnknownException
 
 
 class Task(object):
     name = None
     scheduler = None
+    periodic = False
     running = False
     time_start = None
+    data = None
 
-    stored_attributes = ("last_run", "last_run_time")
+    stored_attributes = ("last_run", "last_run_time", "running")
+    default_data = {"last_run": None, "last_run_time": None, "running": False, "data": {}}
 
     # task ready for being status-displayed?
     ready_for_display = False
 
     def __init__(self, scheduler):
+        self.name = self.get_class_name()
         self.ready_for_display = False
-        self.running = False
         self.time_start = None
         self.scheduler = scheduler
-        if self.name not in Dict["tasks"]:
-            Dict["tasks"][self.name] = {"last_run": None, "last_run_time": None}
+        self.setup_defaults()
+
+        self.running = False
+
+    def get_class_name(self):
+        return getattr(getattr(self, "__class__"), "__name__")
 
     def __getattribute__(self, name):
         if name in object.__getattribute__(self, "stored_attributes"):
@@ -41,18 +60,38 @@ class Task(object):
 
         object.__setattr__(self, name, value)
 
+    def setup_defaults(self):
+        if self.name not in Dict["tasks"]:
+            Dict["tasks"][self.name] = self.default_data.copy()
+            return
+
+        sd = Dict["tasks"][self.name]
+
+        # forward-migration
+        for key, def_value in self.default_data.iteritems():
+            hasval = key in sd
+            if not hasval:
+                sd[key] = def_value
+
     def signal(self, *args, **kwargs):
         raise NotImplementedError
 
-    def prepare(self):
-        raise NotImplementedError
+    def prepare(self, *args, **kwargs):
+        return
 
     def run(self):
-        raise NotImplementedError
+        self.time_start = datetime.datetime.now()
+
+    def post_run(self, data_holder):
+        self.running = False
+        self.last_run = datetime.datetime.now()
+        if self.time_start:
+            self.last_run_time = self.last_run - self.time_start
+        self.time_start = None
 
 
 class SearchAllRecentlyAddedMissing(Task):
-    name = "searchAllRecentlyAddedMissing"
+    periodic = True
     items_done = None
     items_searching = None
     items_searching_ids = None
@@ -76,30 +115,30 @@ class SearchAllRecentlyAddedMissing(Task):
     def signal_updated_metadata(self, *args, **kwargs):
         item_id = int(args[0])
 
-        if item_id in self.items_searching_ids:
+        if self.items_searching_ids is not None and item_id in self.items_searching_ids:
             self.items_done.append(item_id)
             return True
 
-    def prepare(self):
+    def prepare(self, *args, **kwargs):
         self.items_done = []
-        recent_items = getRecentItems()
-        missing = getAllMissing(recent_items)
-        ids = set([id for added_at, id, title in missing])
+        recent_items = get_recent_items()
+        missing = items_get_all_missing_subs(recent_items)
+        ids = set([id for added_at, id, title, item, missing_languages in missing if not is_ignored(id, item=item)])
         self.items_searching = missing
         self.items_searching_ids = ids
         self.items_failed = []
         self.percentage = 0
-        self.time_start = datetime.datetime.now()
         self.ready_for_display = True
 
     def run(self):
+        super(SearchAllRecentlyAddedMissing, self).run()
         self.running = True
         missing_count = len(self.items_searching)
         items_done_count = 0
 
-        for added_at, item_id, title in self.items_searching:
+        for added_at, item_id, title, item, missing_languages in self.items_searching:
             Log.Debug(u"Task: %s, triggering refresh for %s (%s)", self.name, title, item_id)
-            refresh_item(item_id, title)
+            refresh_item(item_id)
             search_started = datetime.datetime.now()
             tries = 1
             while 1:
@@ -116,9 +155,10 @@ class SearchAllRecentlyAddedMissing(Task):
                         Log.Debug(u"Task: %s, item stalled for %s times: %s, skipping", self.name, tries, item_id)
                         break
 
-                    Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time, item_id)
+                    Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time,
+                              item_id)
                     tries += 1
-                    refresh_item(item_id, title)
+                    refresh_item(item_id)
                     search_started = datetime.datetime.now()
                     time.sleep(1)
                 time.sleep(0.1)
@@ -128,12 +168,9 @@ class SearchAllRecentlyAddedMissing(Task):
         Log.Debug("Task: %s, done. Failed items: %s", self.name, self.items_failed)
         self.running = False
 
-    def post_run(self):
+    def post_run(self, task_data):
+        super(SearchAllRecentlyAddedMissing, self).post_run(task_data)
         self.ready_for_display = False
-        self.last_run = datetime.datetime.now()
-        if self.time_start:
-            self.last_run_time = self.last_run - self.time_start
-        self.time_start = None
         self.percentage = 0
         self.items_done = None
         self.items_failed = None
@@ -141,4 +178,296 @@ class SearchAllRecentlyAddedMissing(Task):
         self.items_searching_ids = None
 
 
+class SubtitleListingMixin(object):
+    def list_subtitles(self, rating_key, item_type, part_id, language):
+        metadata = get_plex_metadata(rating_key, part_id, item_type)
+
+        if item_type == "episode":
+            min_score = 240
+        else:
+            min_score = 60
+
+        scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+        if not scanned_parts:
+            Log.Error("Couldn't list available subtitles for %s", rating_key)
+            return
+
+        video, plex_part = scanned_parts.items()[0]
+        config.init_subliminal_patches()
+
+        available_subs = list_all_subtitles(scanned_parts, {Language.fromietf(language)},
+                                            providers=config.providers,
+                                            provider_configs=config.provider_settings,
+                                            pool_class=config.provider_pool)
+
+        use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in ("prefer", "force HI")
+
+        # sort subtitles by score
+        unsorted_subtitles = []
+        for s in available_subs[video]:
+            Log.Debug("Starting score computation for %s", s)
+            try:
+                matches = s.get_matches(video)
+            except AttributeError:
+                Log.Error("Match computation failed for %s: %s", s, traceback.format_exc())
+                continue
+
+            unsorted_subtitles.append(
+                (s, compute_score(matches, s, video, hearing_impaired=use_hearing_impaired), matches))
+        scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1), reverse=True)
+
+        subtitles = []
+        for subtitle, score, matches in scored_subtitles:
+            # check score
+            if score < min_score:
+                Log.Info('Score %d is below min_score (%d)', score, min_score)
+                continue
+            subtitle.score = score
+            subtitle.matches = matches
+            subtitle.part_id = part_id
+            subtitle.item_type = item_type
+            subtitles.append(subtitle)
+        return subtitles
+
+
+class DownloadSubtitleMixin(object):
+    def download_subtitle(self, subtitle, rating_key, mode="m"):
+        from interface.menu_helpers import set_refresh_menu_state
+
+        item_type = subtitle.item_type
+        part_id = subtitle.part_id
+        metadata = get_plex_metadata(rating_key, part_id, item_type)
+        scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
+        video, plex_part = scanned_parts.items()[0]
+
+        # downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
+        download_subtitles([subtitle], providers=config.providers, provider_configs=config.provider_settings,
+                           pool_class=config.provider_pool)
+        download_successful = False
+
+        if subtitle.content:
+            try:
+                whack_missing_parts(scanned_parts)
+                save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode)
+                Log.Debug("Manually downloaded subtitle for: %s", rating_key)
+                download_successful = True
+                refresh_item(rating_key)
+                track_usage("Subtitle", "manual", "download", 1)
+            except:
+                Log.Error("Something went wrong when downloading specific subtitle: %s", traceback.format_exc())
+            finally:
+                set_refresh_menu_state(None)
+
+                if download_successful:
+                    # store item in history
+                    from support.history import get_history
+                    item_title = get_title_for_video_metadata(metadata, add_section_title=False)
+                    history = get_history()
+                    history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
+                                subtitle=subtitle,
+                                mode=mode)
+        return download_successful
+
+
+class AvailableSubsForItem(SubtitleListingMixin, Task):
+    item_type = None
+    part_id = None
+    language = None
+    rating_key = None
+
+    def prepare(self, *args, **kwargs):
+        self.item_type = kwargs.get("item_type")
+        self.part_id = kwargs.get("part_id")
+        self.language = kwargs.get("language")
+        self.rating_key = kwargs.get("rating_key")
+
+    def setup_defaults(self):
+        super(AvailableSubsForItem, self).setup_defaults()
+
+        # reset any previous data
+        Dict["tasks"][self.name]["data"] = {}
+
+    def run(self):
+        super(AvailableSubsForItem, self).run()
+        self.running = True
+        track_usage("Subtitle", "manual", "list", 1)
+        self.data = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language)
+
+    def post_run(self, task_data):
+        super(AvailableSubsForItem, self).post_run(task_data)
+        if self.rating_key not in task_data:
+            task_data[self.rating_key] = {}
+
+        task_data[self.rating_key][self.language] = self.data
+
+
+class DownloadSubtitleForItem(DownloadSubtitleMixin, Task):
+    subtitle = None
+    rating_key = None
+
+    def prepare(self, *args, **kwargs):
+        self.subtitle = kwargs["subtitle"]
+        self.rating_key = kwargs["rating_key"]
+
+    def run(self):
+        super(DownloadSubtitleForItem, self).run()
+        self.running = True
+        self.download_subtitle(self.subtitle, self.rating_key)
+        self.running = False
+
+
+class MissingSubtitles(Task):
+    rating_key = None
+    item_type = None
+    part_id = None
+    language = None
+
+    def run(self):
+        super(MissingSubtitles, self).run()
+        self.running = True
+        self.data = []
+        recent_items = get_recent_items()
+        if recent_items:
+            self.data = items_get_all_missing_subs(recent_items)
+
+    def post_run(self, task_data):
+        super(MissingSubtitles, self).post_run(task_data)
+        task_data["missing_subtitles"] = self.data
+
+
+class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
+    periodic = True
+
+    # TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired
+    series_cutoff = 355
+
+    # movies: format, title, release_group, year, video_codec, resolution, hearing_impaired
+    movies_cutoff = 117
+
+    def signal_updated_metadata(self, *args, **kwargs):
+        return True
+
+    def run(self):
+        super(FindBetterSubtitles, self).run()
+        self.running = True
+        better_found = 0
+        try:
+            max_search_days = int(Prefs["scheduler.tasks.FindBetterSubtitles.max_days_after_added"].strip())
+        except ValueError:
+            Log.Error("Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting")
+            return
+        else:
+            if max_search_days > 30:
+                Log.Error("FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.")
+                return
+
+        now = datetime.datetime.now()
+
+        subtitle_storage = get_subtitle_storage()
+        recent_subs = subtitle_storage.load_recent_files(age_days=max_search_days)
+
+        for fn, stored_subs in recent_subs.iteritems():
+            video_id = stored_subs.video_id
+            cutoff = self.series_cutoff if stored_subs.item_type == "episode" else self.movies_cutoff
+
+            # don't search for better subtitles until at least 30 minutes have passed
+            if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
+                Log.Debug("Item %s too new, skipping", video_id)
+                continue
+
+            # added_date <= max_search_days?
+            if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
+                continue
+
+            ditch_parts = []
+
+            # look through all stored subtitle data
+            for part_id, languages in stored_subs.parts.iteritems():
+                part_id = str(part_id)
+
+                # all languages
+                for language, current_subs in languages.iteritems():
+                    current_key = current_subs.get("current")
+                    current = current_subs.get(current_key)
+
+                    # currently got subtitle?
+                    if not current:
+                        continue
+                    current_score = current.score
+                    current_mode = current.mode
+
+                    # late cutoff met? skip
+                    if current_score >= cutoff:
+                        Log.Debug(u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s",
+                                  current_score, cutoff, stored_subs.title)
+                        continue
+
+                    # got manual subtitle but don't want to touch those?
+                    if current_mode == "m" and \
+                            not cast_bool(Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]):
+                        Log.Debug(u"Skipping finding better subs, had manual: %s", stored_subs.title)
+                        continue
+
+                    try:
+                        subs = self.list_subtitles(video_id, stored_subs.item_type, part_id, language)
+                    except PartUnknownException:
+                        Log.Info("Part %s unknown/gone; ditching subtitle info", part_id)
+                        ditch_parts.append(part_id)
+                        continue
+
+                    if subs:
+                        # subs are already sorted by score
+                        better_downloaded = False
+                        better_tried_download = 0
+                        for sub in subs:
+                            if sub.score > current_score:
+                                Log.Debug("Better subtitle found for %s, downloading", video_id)
+                                better_tried_download += 1
+                                ret = self.download_subtitle(sub, video_id, mode="b")
+                                if ret:
+                                    better_found += 1
+                                    better_downloaded = True
+                                    break
+                                else:
+                                    Log.Debug("Couldn't download/save subtitle. Continuing to the next one")
+                        if better_tried_download and not better_downloaded:
+                            Log.Debug("Tried downloading better subtitle for %s, but every try failed.", video_id)
+
+                        elif better_downloaded:
+                            Log.Debug("Better subtitle downloaded for %s", video_id)
+
+            if ditch_parts:
+                for part_id in ditch_parts:
+                    try:
+                        del stored_subs.parts[part_id]
+                    except KeyError:
+                        pass
+                subtitle_storage.save(stored_subs)
+
+        if better_found:
+            Log.Debug("Task: %s, done. Better subtitles found for %s items", self.name, better_found)
+
+
+class SubtitleStorageMaintenance(Task):
+    periodic = True
+    frequency = "every 7 days"
+
+    def run(self):
+        super(SubtitleStorageMaintenance, self).run()
+        self.running = True
+        Log.Info("Running subtitle storage maintenance")
+        storage = get_subtitle_storage()
+        deleted_items = storage.delete_missing_files()
+        if deleted_items:
+            Log.Info("Subtitle information for %d non-existant videos have been cleaned up" % len(deleted_items))
+            Log.Debug("Videos: %s" % deleted_items)
+        else:
+            Log.Info("Nothing to do")
+
+
 scheduler.register(SearchAllRecentlyAddedMissing)
+scheduler.register(AvailableSubsForItem)
+scheduler.register(DownloadSubtitleForItem)
+scheduler.register(MissingSubtitles)
+scheduler.register(FindBetterSubtitles)
+scheduler.register(SubtitleStorageMaintenance)

Contents/DefaultPrefs.json

@@ -1,70 +1,4 @@
 [
-  {
-    "id": "enable_channel",
-    "label": "Enable Sub-Zero channel (disabling doesn't affect the subtitle features)?",
-    "type": "bool",
-    "default": "true"
-  },
-  {
-    "id": "subtitles.try_downloads",
-    "label": "How many download tries per subtitle (on timeout or error)",
-    "type": "enum",
-    "values": [
-      "1",
-      "2",
-      "3",
-      "4"
-    ],
-    "default": "2"
-  },
-  {
-    "id": "plex_username",
-    "label": "Plex.tv Username (needed for Plex Home users)",
-    "type": "text",
-    "default": ""
-  },
-  {
-    "id": "plex_password",
-    "label": "Plex.tv Password",
-    "type": "text",
-    "option": "hidden",
-    "default": "",
-    "secure": "true"
-  },
-  {
-    "id": "provider.addic7ed.username",
-    "label": "Addic7ed Username",
-    "type": "text",
-    "default": ""
-  },
-  {
-    "id": "provider.addic7ed.password",
-    "label": "Addic7ed Password",
-    "type": "text",
-    "option": "hidden",
-    "default": "",
-    "secure": "true"
-  },
-  {
-    "id": "provider.opensubtitles.username",
-    "label": "Opensubtitles Username (VIP)",
-    "type": "text",
-    "default": ""
-  },
-  {
-    "id": "provider.opensubtitles.password",
-    "label": "Opensubtitles Password",
-    "type": "text",
-    "option": "hidden",
-    "default": "",
-    "secure": "true"
-  },
-  {
-    "id": "provider.addic7ed.use_random_agents",
-    "label": "Addic7ed: Use random user agents (should not be necessary)",
-    "type": "bool",
-    "default": "false"
-  },
   {
     "id": "langPref1",
     "label": "Subtitle Language (1)",
@@ -232,6 +166,18 @@
     "type": "text",
     "default": "None"
   },
+  {
+    "id": "subtitles.only_foreign",
+    "label": "Only download foreign/forced subtitles",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "subtitles.language.ietf",
+    "label": "Treat IETF language tags as ISO 639-1 (e.g. pt-BR = pt)",
+    "type": "bool",
+    "default": "true"
+  },
   {
     "id": "subtitles.only_one",
     "label": "Restrict to one language (skips adding \".lang.\" to the subtitle filename; only uses \"Subtitle Language (1)\")",
@@ -239,8 +185,8 @@
     "default": "false"
   },
   {
-    "id": "subtitles.enforce_encoding",
-    "label": "Normalize subtitle encoding to UTF-8",
+    "id": "subtitles.language.treat_und_as_first",
+    "label": "Embedded subtitles: Treat \"Undefined\" (und) as language 1",
     "type": "bool",
     "default": "true"
   },
@@ -251,10 +197,18 @@
     "default": "true"
   },
   {
-    "id": "provider.thesubdb.enabled",
-    "label": "Provider: Enable TheSubDB",
-    "type": "bool",
-    "default": "true"
+    "id": "provider.opensubtitles.username",
+    "label": "Opensubtitles Username (VIP)",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "provider.opensubtitles.password",
+    "label": "Opensubtitles Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
   },
   {
     "id": "provider.podnapisi.enabled",
@@ -269,38 +223,22 @@
     "default": "true"
   },
   {
-    "id": "provider.addic7ed.boost",
-    "label": "Addic7ed: prefer over other providers (if requirements met)",
-    "type": "bool",
-    "default": "false"
+    "id": "provider.addic7ed.username",
+    "label": "Addic7ed Username",
+    "type": "text",
+    "default": ""
   },
   {
-    "id": "provider.tvsubtitles.enabled",
-    "label": "Provider: Enable TVsubtitles.net",
-    "type": "bool",
-    "default": "true"
+    "id": "provider.addic7ed.password",
+    "label": "Addic7ed Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
   },
   {
-    "id": "provider.opensubtitles.use_tags",
-    "label": "I keep the exact (release-) filename of my media files",
-    "type": "bool",
-    "default": "true"
-  },
-  {
-    "id": "subtitles.scan.embedded",
-    "label": "Scan: include embedded subtitles (in the media file (MKV/MP4), don't download if existing)",
-    "type": "bool",
-    "default": "false"
-  },
-  {
-    "id": "subtitles.scan.external",
-    "label": "Scan: include external subtitles (metadata/filesystem, don't download if existing)",
-    "type": "bool",
-    "default": "true"
-  },
-  {
-    "id": "subtitles.search.minimumTVScore",
-    "label": "Minimum score for TV subtitles to download",
+    "id": "provider.addic7ed.boost_by1",
+    "label": "Addic7ed: boost score (if requirements met)",
     "type": "enum",
     "values": [
       "100",
@@ -326,37 +264,124 @@
       "5",
       "0"
     ],
-    "default": "85"
+    "default": "25"
   },
   {
-    "id": "subtitles.search.minimumMovieScore",
-    "label": "Minimum score for movie subtitles to download",
+    "id": "provider.addic7ed.use_random_agents",
+    "label": "Addic7ed: Use random user agents",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.legendastv.enabled",
+    "label": "Provider: Enable Legendas TV (mostly pt-BR)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.legendastv.username",
+    "label": "Legendas TV Username",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "provider.legendastv.password",
+    "label": "Legendas TV Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
+  },
+  {
+    "id": "provider.tvsubtitles.enabled",
+    "label": "Provider: Enable TVsubtitles.net",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "provider.napiprojekt.enabled",
+    "label": "Provider: Enable NapiProjekt.pl (Polish)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.shooter.enabled",
+    "label": "Provider: Enable Shooter.cn (Chinese)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.subscenter.enabled",
+    "label": "Provider: Enable SubsCenter (Hebrew)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "provider.subscenter.username",
+    "label": "SubsCenter Username",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "provider.subscenter.password",
+    "label": "SubsCenter Password",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
+  },
+  {
+    "id": "providers.multithreading",
+    "label": "Search enabled providers simuntaneously (multithreading)",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "provider.opensubtitles.use_tags",
+    "label": "I keep the exact (release-) filename of my media files",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "subtitles.scan.embedded",
+    "label": "Scan: include embedded subtitles (in the media file (MKV/MP4), don't download if existing)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "subtitles.scan.external",
+    "label": "Scan: include external subtitles (metadata/filesystem, don't download if existing)",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "subtitles.scan.exotic_ext",
+    "label": "Scan: include \"exotic\" external subtitle formats (anything else than .srt/.ssa/.ass)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "subtitles.scan.filename_strictness",
+    "label": "Scan: which external subtitles should be picked up?",
     "type": "enum",
     "values": [
-      "100",
-      "95",
-      "90",
-      "85",
-      "80",
-      "75",
-      "70",
-      "65",
-      "60",
-      "55",
-      "50",
-      "45",
-      "40",
-      "35",
-      "30",
-      "25",
-      "23",
-      "20",
-      "15",
-      "10",
-      "5",
-      "0"
+      "exact: media filename match",
+      "loose: filename contains media filename",
+      "any"
     ],
-    "default": "23"
+    "default": "loose: filename contains media filename"
+  },
+  {
+    "id": "subtitles.search.minimumTVScore2",
+    "label": "Minimum score for TV (min: 240, def/sane: 337, min-ideal: 352; see http://v.ht/szscores)",
+    "type": "text",
+    "default": "337"
+  },
+  {
+    "id": "subtitles.search.minimumMovieScore2",
+    "label": "Minimum score for movies (min: 60, def/sane: 69, min-ideal: 82; see http://v.ht/szscores)",
+    "type": "text",
+    "default": "69"
   },
   {
     "id": "subtitles.search.hearingImpaired",
@@ -370,6 +395,12 @@
     ],
     "default": "don't prefer"
   },
+  {
+    "id": "subtitles.enforce_encoding",
+    "label": "Normalize subtitle encoding to UTF-8",
+    "type": "bool",
+    "default": "true"
+  },
   {
     "id": "subtitles.save.filesystem",
     "label": "Store subtitles next to media files (instead of metadata)",
@@ -396,19 +427,37 @@
     "default": ""
   },
   {
-    "id": "subtitles.language.ietf",
-    "label": "Treat IETF language tags as ISO 639-1 (e.g. pt-BR = pt)",
-    "type": "bool",
-    "default": "true"
-  },
-  {
-    "id": "subtitles.ignore_fs",
-    "label": "Ignore folders (with \"subzero.ignore/.subzero.ignore/.nosz\" files in them)",
+    "id": "subtitles.save.metadata_fallback",
+    "label": "Fall back to metadata storage if filesystem storage failed",
     "type": "bool",
     "default": "false"
   },
   {
-    "id": "scheduler.tasks.searchAllRecentlyAddedMissing",
+    "id": "subtitles.save.chmod",
+      "label": "Set subtitle file permissions to (integer, e.g.: 0775)",
+      "type": "text",
+      "default": ""
+  },
+  {
+    "id": "subtitles.autoclean",
+    "label": "Automatically delete leftover/unused (externally saved) subtitles",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "activity.on_playback",
+    "label": "On media playback: search for missing subtitles (refresh item)",
+    "type": "enum",
+    "values": [
+      "never",
+      "current media item",
+      "next episode (series)",
+      "hybrid: current item or next episode"
+    ],
+    "default": "never"
+  },
+  {
+    "id": "scheduler.tasks.SearchAllRecentlyAddedMissing.frequency",
     "label": "Scheduler: Periodically search for recent items with missing subtitles",
     "type": "enum",
     "values": [
@@ -443,7 +492,110 @@
     "id": "scheduler.max_recent_items_per_library",
     "label": "Scheduler: Recent items to consider per library",
     "type": "text",
-    "default": "200"
+    "default": "500"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.frequency",
+    "label": "Scheduler: Periodically search for better subtitles",
+    "type": "enum",
+    "values": [
+      "never",
+      "every 6 hours",
+      "every 12 hours",
+      "every 24 hours"
+    ],
+    "default": "every 12 hours"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.max_days_after_added",
+    "label": "Scheduler: Days to search for better subtitles (max: 30 days)",
+    "type": "text",
+    "default": "7"
+  },
+  {
+    "id": "scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected",
+    "label": "Scheduler: Overwrite manually selected subtitles when better found",
+    "type": "bool",
+    "default": "true"
+  },
+  {
+    "id": "history_size",
+    "label": "History: amount of items to store historical data for",
+    "type": "enum",
+    "values": [
+      "50",
+      "100",
+      "150",
+      "250",
+      "500"
+    ],
+    "default": "100"
+  },
+  {
+    "id": "subtitles.try_downloads",
+    "label": "How many download tries per subtitle (on timeout or error)",
+    "type": "enum",
+    "values": [
+      "1",
+      "2",
+      "3",
+      "4"
+    ],
+    "default": "2"
+  },
+  {
+    "id": "subtitles.ignore_fs",
+    "label": "Ignore folders (with \"subzero.ignore/.subzero.ignore/.nosz\" files in them)",
+    "type": "bool",
+    "default": "false"
+  },
+  {
+    "id": "subtitles.ignore_paths",
+    "label": "Ignore anything in the following paths (comma-separated)",
+    "type": "text",
+    "default": ""
+  },
+  {
+    "id": "plugin_mode",
+    "label": "Sub-Zero mode",
+    "type": "enum",
+    "values": [
+      "agent + channel",
+      "only agent",
+      "only channel"
+    ],
+    "default": "agent + channel"
+  },
+  {
+    "id": "plugin_pin",
+    "label": "Access PIN (any amount of numbers, 0-9)",
+    "type": "text",
+    "option": "hidden",
+    "default": "",
+    "secure": "true"
+  },
+  {
+    "id": "plugin_pin_valid_for",
+    "label": "Access PIN valid for minutes",
+    "type": "text",
+    "default": "10"
+  },
+  {
+    "id": "plugin_pin_mode",
+    "label": "Use PIN to restrict access to (needs plugin or PMS restart)",
+    "type": "enum",
+    "values": [
+      "disabled",
+      "channel menu",
+      "advanced menu"
+    ],
+    "default": "disabled"
+  },
+  {
+    "id": "notify_executable",
+    "label": "Call this executable upon successful subtitle download",
+    "type": "text",
+    "default": ""
   },
   {
     "id": "check_permissions",
@@ -469,5 +621,11 @@
     "label": "Log to console (for development/debugging)",
     "type": "bool",
     "default": "false"
+  },
+  {
+    "id": "track_usage",
+    "label": "Collect anonymous usage statistics",
+    "type": "bool",
+    "default": "true"
   }
 ]

Contents/Info.plist

@@ -9,11 +9,11 @@
         <key>CFBundleInfoDictionaryVersion</key>
         <string>6.0</string>
         <key>CFBundleShortVersionString</key>
-        <string>1.3.31</string>
+        <string>2.0.0</string>
         <key>CFBundleSignature</key>
         <string>????</string>
         <key>CFBundleVersion</key>
-        <string>1.3.33.522</string>
+        <string>2.0.0.9</string>
         <key>PlexFrameworkVersion</key>
         <string>2</string>
         <key>PlexPluginClass</key>
@@ -23,26 +23,28 @@
         <key>PlexPluginConsoleLogging</key>
         <string>0</string>
         <key>PlexPluginDevMode</key>
-        <string>0</string>
+        <string>1</string>
          <key>PlexPluginCodePolicy</key>
-            <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API -->     
+            <!-- this allows channels to access some python methods which are otherwise blocked, as well as import external code libraries, and interact with the PMS HTTP API -->
             <string>Elevated</string>
 	<key>PlexAgentAttributionText</key>
 	<string>&lt;div style=&quot;white-space: pre;&quot;&gt;&lt;img src=&quot;https://raw.githubusercontent.com/pannal/Sub-Zero.bundle/master/Contents/Resources/subzero.gif&quot; /&gt;
 
 &lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
 
-Version 1.3.33.522
+Version 2.0.0.9 DEV
 
 Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
 
 If you like this, buy me a beer: &lt;a href=&quot;https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&amp;hosted_button_id=G9VKR2B8PMNKG&quot; target=&quot;_blank&quot; title=&quot;donate&quot;&gt;&lt;img src=&quot;https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif&quot; alt=&quot;donate&quot; title=&quot;donate&quot; /&gt;&lt;/a&gt;
 
 &lt;strong&gt;Need help?&lt;/strong&gt;
+Wiki: &lt;a href=&quot;http://v.ht/szwiki&quot;&gt;http://v.ht/szwiki&lt;/a&gt;
+Score info: &lt;a href=&quot;http://v.ht/szscores&quot;&gt;http://v.ht/szscores&lt;/a&gt;
 Plex thread: &lt;a href=&quot;https://forums.plex.tv/discussion/186575&quot;>https://forums.plex.tv/discussion/186575&lt;/a&gt;
 Github: &lt;a href=&quot;https://github.com/pannal/Sub-Zero.bundle&quot;&gt;https://github.com/pannal/Sub-Zero&lt;/a&gt;
 
-panni, 2016
+panni, 2017
 &lt;/div&gt;
 	</string>
     </dict>

Contents/Libraries/Shared/appdirs.py

@@ -0,0 +1,552 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2005-2010 ActiveState Software Inc.
+# Copyright (c) 2013 Eddy Petrișor
+
+"""Utilities for determining application-specific dirs.
+
+See <http://github.com/ActiveState/appdirs> for details and usage.
+"""
+# Dev Notes:
+# - MSDN on where to store app data files:
+#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
+# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
+# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+__version_info__ = (1, 4, 0)
+__version__ = '.'.join(map(str, __version_info__))
+
+
+import sys
+import os
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    unicode = str
+
+if sys.platform.startswith('java'):
+    import platform
+    os_name = platform.java_ver()[3][0]
+    if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
+        system = 'win32'
+    elif os_name.startswith('Mac'): # "Mac OS X", etc.
+        system = 'darwin'
+    else: # "Linux", "SunOS", "FreeBSD", etc.
+        # Setting this to "linux2" is not ideal, but only Windows or Mac
+        # are actually checked for and the rest of the module expects
+        # *sys.platform* style strings.
+        system = 'linux2'
+else:
+    system = sys.platform
+
+
+
+def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user data directories are:
+        Mac OS X:               ~/Library/Application Support/<AppName>
+        Unix:                   ~/.local/share/<AppName>    # or in $XDG_DATA_HOME, if defined
+        Win XP (not roaming):   C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName>
+        Win XP (roaming):       C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>
+        Win 7  (not roaming):   C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>
+        Win 7  (roaming):       C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName>
+
+    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
+    That means, by default "~/.local/share/<AppName>".
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
+        path = os.path.normpath(_get_win_folder(const))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Application Support/')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
+    """Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of data dirs should be
+            returned. By default, the first item from XDG_DATA_DIRS is
+            returned, or '/usr/local/share/<AppName>',
+            if XDG_DATA_DIRS is not set
+
+    Typical user data directories are:
+        Mac OS X:   /Library/Application Support/<AppName>
+        Unix:       /usr/local/share/<AppName> or /usr/share/<AppName>
+        Win XP:     C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName>
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+        Win 7:      C:\ProgramData\<AppAuthor>\<AppName>   # Hidden, but writeable on Win 7.
+
+    For Unix, this is using the $XDG_DATA_DIRS[0] default.
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('/Library/Application Support')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        # XDG default for $XDG_DATA_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_DATA_DIRS',
+                         os.pathsep.join(['/usr/local/share', '/usr/share']))
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+        return path
+
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific config dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user data directories are:
+        Mac OS X:               same as user_data_dir
+        Unix:                   ~/.config/<AppName>     # or in $XDG_CONFIG_HOME, if defined
+        Win *:                  same as user_data_dir
+
+    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
+    That means, by deafult "~/.config/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
+    """Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of config dirs should be
+            returned. By default, the first item from XDG_CONFIG_DIRS is
+            returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set
+
+    Typical user data directories are:
+        Mac OS X:   same as site_data_dir
+        Unix:       /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in
+                    $XDG_CONFIG_DIRS
+        Win *:      same as site_data_dir
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+
+    For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system in ["win32", "darwin"]:
+        path = site_data_dir(appname, appauthor)
+        if appname and version:
+            path = os.path.join(path, version)
+    else:
+        # XDG default for $XDG_CONFIG_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+    return path
+
+
+def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific cache dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Cache" to the base app data dir for Windows. See
+            discussion below.
+
+    Typical user cache directories are:
+        Mac OS X:   ~/Library/Caches/<AppName>
+        Unix:       ~/.cache/<AppName> (XDG default)
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache
+
+    On Windows the only suggestion in the MSDN docs is that local settings go in
+    the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
+    app data dir (the default returned by `user_data_dir` above). Apps typically
+    put cache data somewhere *under* the given dir here. Some examples:
+        ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
+        ...\Acme\SuperApp\Cache\1.0
+    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+            if opinion:
+                path = os.path.join(path, "Cache")
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Caches')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_log_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific log dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Logs" to the base app data dir for Windows, and "log" to the
+            base cache dir for Unix. See discussion below.
+
+    Typical user cache directories are:
+        Mac OS X:   ~/Library/Logs/<AppName>
+        Unix:       ~/.cache/<AppName>/log  # or under $XDG_CACHE_HOME if defined
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs
+
+    On Windows the only suggestion in the MSDN docs is that local settings
+    go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in
+    examples of what some windows apps use for a logs dir.)
+
+    OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA`
+    value for Windows and appends "log" to the user cache dir for Unix.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "darwin":
+        path = os.path.join(
+            os.path.expanduser('~/Library/Logs'),
+            appname)
+    elif system == "win32":
+        path = user_data_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "Logs")
+    else:
+        path = user_cache_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "log")
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+class AppDirs(object):
+    """Convenience wrapper for getting application dirs."""
+    def __init__(self, appname, appauthor=None, version=None, roaming=False,
+                 multipath=False):
+        self.appname = appname
+        self.appauthor = appauthor
+        self.version = version
+        self.roaming = roaming
+        self.multipath = multipath
+
+    @property
+    def user_data_dir(self):
+        return user_data_dir(self.appname, self.appauthor,
+                             version=self.version, roaming=self.roaming)
+
+    @property
+    def site_data_dir(self):
+        return site_data_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_config_dir(self):
+        return user_config_dir(self.appname, self.appauthor,
+                               version=self.version, roaming=self.roaming)
+
+    @property
+    def site_config_dir(self):
+        return site_config_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_cache_dir(self):
+        return user_cache_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_log_dir(self):
+        return user_log_dir(self.appname, self.appauthor,
+                            version=self.version)
+
+
+#---- internal support stuff
+
+def _get_win_folder_from_registry(csidl_name):
+    """This is a fallback technique at best. I'm not sure if using the
+    registry for this guarantees us the correct answer for all CSIDL_*
+    names.
+    """
+    import _winreg
+
+    shell_folder_name = {
+        "CSIDL_APPDATA": "AppData",
+        "CSIDL_COMMON_APPDATA": "Common AppData",
+        "CSIDL_LOCAL_APPDATA": "Local AppData",
+    }[csidl_name]
+
+    key = _winreg.OpenKey(
+        _winreg.HKEY_CURRENT_USER,
+        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+    )
+    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
+    return dir
+
+
+def _get_win_folder_with_pywin32(csidl_name):
+    from win32com.shell import shellcon, shell
+    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
+    # Try to make this a unicode path because SHGetFolderPath does
+    # not return unicode strings when there is unicode data in the
+    # path.
+    try:
+        dir = unicode(dir)
+
+        # Downgrade to short path name if have highbit chars. See
+        # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+        has_high_char = False
+        for c in dir:
+            if ord(c) > 255:
+                has_high_char = True
+                break
+        if has_high_char:
+            try:
+                import win32api
+                dir = win32api.GetShortPathName(dir)
+            except ImportError:
+                pass
+    except UnicodeError:
+        pass
+    return dir
+
+
+def _get_win_folder_with_ctypes(csidl_name):
+    import ctypes
+
+    csidl_const = {
+        "CSIDL_APPDATA": 26,
+        "CSIDL_COMMON_APPDATA": 35,
+        "CSIDL_LOCAL_APPDATA": 28,
+    }[csidl_name]
+
+    buf = ctypes.create_unicode_buffer(1024)
+    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in buf:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf2 = ctypes.create_unicode_buffer(1024)
+        if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+            buf = buf2
+
+    return buf.value
+
+def _get_win_folder_with_jna(csidl_name):
+    import array
+    from com.sun import jna
+    from com.sun.jna.platform import win32
+
+    buf_size = win32.WinDef.MAX_PATH * 2
+    buf = array.zeros('c', buf_size)
+    shell = win32.Shell32.INSTANCE
+    shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
+    dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in dir:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf = array.zeros('c', buf_size)
+        kernel = win32.Kernel32.INSTANCE
+        if kernal.GetShortPathName(dir, buf, buf_size):
+            dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    return dir
+
+if system == "win32":
+    try:
+        import win32com.shell
+        _get_win_folder = _get_win_folder_with_pywin32
+    except ImportError:
+        try:
+            from ctypes import windll
+            _get_win_folder = _get_win_folder_with_ctypes
+        except ImportError:
+            try:
+                import com.sun.jna
+                _get_win_folder = _get_win_folder_with_jna
+            except ImportError:
+                _get_win_folder = _get_win_folder_from_registry
+
+
+#---- self test code
+
+if __name__ == "__main__":
+    appname = "MyApp"
+    appauthor = "MyCompany"
+
+    props = ("user_data_dir", "site_data_dir",
+             "user_config_dir", "site_config_dir",
+             "user_cache_dir", "user_log_dir")
+
+    print("-- app dirs (with optional 'version')")
+    dirs = AppDirs(appname, appauthor, version="1.0")
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'version')")
+    dirs = AppDirs(appname, appauthor)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'appauthor')")
+    dirs = AppDirs(appname)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (with disabled 'appauthor')")
+    dirs = AppDirs(appname, appauthor=False)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))

Contents/Libraries/Shared/asio/__init__.py

@@ -0,0 +1,61 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from asio.file import SEEK_ORIGIN_CURRENT
+from asio.file_opener import FileOpener
+from asio.open_parameters import OpenParameters
+from asio.interfaces.posix import PosixInterface
+from asio.interfaces.windows import WindowsInterface
+
+import os
+
+
+class ASIO(object):
+    platform_handler = None
+
+    @classmethod
+    def get_handler(cls):
+        if cls.platform_handler:
+            return cls.platform_handler
+
+        if os.name == 'nt':
+            cls.platform_handler = WindowsInterface
+        elif os.name == 'posix':
+            cls.platform_handler = PosixInterface
+        else:
+            raise NotImplementedError()
+
+        return cls.platform_handler
+
+    @classmethod
+    def open(cls, file_path, opener=True, parameters=None):
+        """Open file
+
+        :type file_path: str
+
+        :param opener: Use FileOpener, for use with the 'with' statement
+        :type opener: bool
+
+        :rtype: asio.file.File
+        """
+        if not parameters:
+            parameters = OpenParameters()
+
+        if opener:
+            return FileOpener(file_path, parameters)
+
+        return ASIO.get_handler().open(
+            file_path,
+            parameters=parameters.handlers.get(ASIO.get_handler())
+        )

Contents/Libraries/Shared/asio/file.py

@@ -0,0 +1,92 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from io import RawIOBase
+import time
+
+DEFAULT_BUFFER_SIZE = 4096
+
+SEEK_ORIGIN_BEGIN = 0
+SEEK_ORIGIN_CURRENT = 1
+SEEK_ORIGIN_END = 2
+
+
+class ReadTimeoutError(Exception):
+    pass
+
+
+class File(RawIOBase):
+    platform_handler = None
+
+    def __init__(self, *args, **kwargs):
+        super(File, self).__init__(*args, **kwargs)
+
+    def get_handler(self):
+        """
+        :rtype: asio.interfaces.base.Interface
+        """
+        if not self.platform_handler:
+            raise ValueError()
+
+        return self.platform_handler
+
+    def get_size(self):
+        """Get the current file size
+
+        :rtype: int
+        """
+        return self.get_handler().get_size(self)
+
+    def get_path(self):
+        """Get the path of this file
+
+        :rtype: str
+        """
+        return self.get_handler().get_path(self)
+
+    def seek(self, offset, origin):
+        """Sets a reference point of a file to the given value.
+
+        :param offset: The point relative to origin to move
+        :type offset: int
+
+        :param origin: Reference point to seek (SEEK_ORIGIN_BEGIN, SEEK_ORIGIN_CURRENT, SEEK_ORIGIN_END)
+        :type origin: int
+        """
+        return self.get_handler().seek(self, offset, origin)
+
+    def read(self, n=-1):
+        """Read up to n bytes from the object and return them.
+
+        :type n: int
+        :rtype: str
+        """
+        return self.get_handler().read(self, n)
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b and return the number of bytes read."""
+        data = self.read(len(b))
+
+        if data is None:
+            return None
+
+        b[:len(data)] = data
+        return len(data)
+
+    def close(self):
+        """Close the file handle"""
+        return self.get_handler().close(self)
+
+    def readable(self, *args, **kwargs):
+        return True

Contents/Libraries/Shared/asio/file_opener.py

@@ -0,0 +1,21 @@
+class FileOpener(object):
+    def __init__(self, file_path, parameters=None):
+        self.file_path = file_path
+        self.parameters = parameters
+
+        self.file = None
+
+    def __enter__(self):
+        self.file = ASIO.get_handler().open(
+            self.file_path,
+            self.parameters.handlers.get(ASIO.get_handler())
+        )
+
+        return self.file
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self.file:
+            return
+
+        self.file.close()
+        self.file = None

Contents/Libraries/Shared/asio/interfaces/base.py

@@ -0,0 +1,41 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from asio.file import DEFAULT_BUFFER_SIZE
+
+
+class Interface(object):
+    @classmethod
+    def open(cls, file_path, parameters=None):
+        raise NotImplementedError()
+
+    @classmethod
+    def get_size(cls, fp):
+        raise NotImplementedError()
+
+    @classmethod
+    def get_path(cls, fp):
+        raise NotImplementedError()
+
+    @classmethod
+    def seek(cls, fp, pointer, distance):
+        raise NotImplementedError()
+
+    @classmethod
+    def read(cls, fp, n=DEFAULT_BUFFER_SIZE):
+        raise NotImplementedError()
+
+    @classmethod
+    def close(cls, fp):
+        raise NotImplementedError()

Contents/Libraries/Shared/asio/interfaces/posix.py

@@ -0,0 +1,123 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from asio.file import File, DEFAULT_BUFFER_SIZE
+from asio.interfaces.base import Interface
+
+import sys
+import os
+
+if os.name == 'posix':
+    import select
+
+    # fcntl is only required on darwin
+    if sys.platform == 'darwin':
+        import fcntl
+
+F_GETPATH = 50
+
+
+class PosixInterface(Interface):
+    @classmethod
+    def open(cls, file_path, parameters=None):
+        """
+        :type file_path: str
+        :rtype: asio.interfaces.posix.PosixFile
+        """
+        if not parameters:
+            parameters = {}
+
+        if not parameters.get('mode'):
+            parameters.pop('mode')
+
+        if not parameters.get('buffering'):
+            parameters.pop('buffering')
+
+        fd = os.open(file_path, os.O_RDONLY | os.O_NONBLOCK)
+
+        return PosixFile(fd)
+
+    @classmethod
+    def get_size(cls, fp):
+        """
+        :type fp: asio.interfaces.posix.PosixFile
+        :rtype: int
+        """
+        return os.fstat(fp.fd).st_size
+
+    @classmethod
+    def get_path(cls, fp):
+        """
+        :type fp: asio.interfaces.posix.PosixFile
+        :rtype: int
+        """
+
+        # readlink /dev/fd fails on darwin, so instead use fcntl F_GETPATH
+        if sys.platform == 'darwin':
+            return fcntl.fcntl(fp.fd, F_GETPATH, '\0' * 1024).rstrip('\0')
+
+        # Use /proc/self/fd if available
+        if os.path.lexists("/proc/self/fd/"):
+            return os.readlink("/proc/self/fd/%s" % fp.fd)
+
+        # Fallback to /dev/fd
+        if os.path.lexists("/dev/fd/"):
+            return os.readlink("/dev/fd/%s" % fp.fd)
+
+        raise NotImplementedError('Environment not supported (fdescfs not mounted?)')
+
+    @classmethod
+    def seek(cls, fp, offset, origin):
+        """
+        :type fp: asio.interfaces.posix.PosixFile
+        :type offset: int
+        :type origin: int
+        """
+        os.lseek(fp.fd, offset, origin)
+
+    @classmethod
+    def read(cls, fp, n=DEFAULT_BUFFER_SIZE):
+        """
+        :type fp: asio.interfaces.posix.PosixFile
+        :type n: int
+        :rtype: str
+        """
+        r, w, x = select.select([fp.fd], [], [], 5)
+
+        if r:
+            return os.read(fp.fd, n)
+
+        return None
+
+    @classmethod
+    def close(cls, fp):
+        """
+        :type fp: asio.interfaces.posix.PosixFile
+        """
+        os.close(fp.fd)
+
+
+class PosixFile(File):
+    platform_handler = PosixInterface
+
+    def __init__(self, fd, *args, **kwargs):
+        """
+        :type fd: asio.file.File
+        """
+        super(PosixFile, self).__init__(*args, **kwargs)
+
+        self.fd = fd
+
+    def __str__(self):
+        return "<asio_posix.PosixFile file: %s>" % self.fd

Contents/Libraries/Shared/asio/interfaces/windows/__init__.py

@@ -0,0 +1,201 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from asio.file import File, DEFAULT_BUFFER_SIZE
+from asio.interfaces.base import Interface
+
+import os
+
+
+NULL = 0
+
+if os.name == 'nt':
+    from asio.interfaces.windows.interop import WindowsInterop
+
+
+class WindowsInterface(Interface):
+    @classmethod
+    def open(cls, file_path, parameters=None):
+        """
+        :type file_path: str
+        :rtype: asio.interfaces.windows.WindowsFile
+        """
+        if not parameters:
+            parameters = {}
+
+        return WindowsFile(WindowsInterop.create_file(
+            file_path,
+            parameters.get('desired_access', WindowsInterface.GenericAccess.READ),
+            parameters.get('share_mode', WindowsInterface.ShareMode.ALL),
+            parameters.get('creation_disposition', WindowsInterface.CreationDisposition.OPEN_EXISTING),
+            parameters.get('flags_and_attributes', NULL)
+        ))
+
+    @classmethod
+    def get_size(cls, fp):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :rtype: int
+        """
+        return WindowsInterop.get_file_size(fp.handle)
+
+    @classmethod
+    def get_path(cls, fp):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :rtype: str
+        """
+
+        if not fp.file_map:
+            fp.file_map = WindowsInterop.create_file_mapping(fp.handle, WindowsInterface.Protection.READONLY)
+
+        if not fp.map_view:
+            fp.map_view = WindowsInterop.map_view_of_file(fp.file_map, WindowsInterface.FileMapAccess.READ, 1)
+
+        file_name = WindowsInterop.get_mapped_file_name(fp.map_view)
+
+        return file_name
+
+    @classmethod
+    def seek(cls, fp, offset, origin):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :type offset: int
+        :type origin: int
+        :rtype: int
+        """
+
+        return WindowsInterop.set_file_pointer(
+            fp.handle,
+            offset,
+            origin
+        )
+
+    @classmethod
+    def read(cls, fp, n=DEFAULT_BUFFER_SIZE):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :type n: int
+        :rtype: str
+        """
+        return WindowsInterop.read(fp.handle, n)
+
+    @classmethod
+    def read_into(cls, fp, b):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :type b: str
+        :rtype: int
+        """
+        return WindowsInterop.read_into(fp.handle, b)
+
+    @classmethod
+    def close(cls, fp):
+        """
+        :type fp: asio.interfaces.windows.WindowsFile
+        :rtype: bool
+        """
+        if fp.map_view:
+            WindowsInterop.unmap_view_of_file(fp.map_view)
+
+        if fp.file_map:
+            WindowsInterop.close_handle(fp.file_map)
+
+        return bool(WindowsInterop.close_handle(fp.handle))
+
+    class GenericAccess(object):
+        READ = 0x80000000
+        WRITE = 0x40000000
+        EXECUTE = 0x20000000
+        ALL = 0x10000000
+
+    class ShareMode(object):
+        READ = 0x00000001
+        WRITE = 0x00000002
+        DELETE = 0x00000004
+        ALL = READ | WRITE | DELETE
+
+    class CreationDisposition(object):
+        CREATE_NEW = 1
+        CREATE_ALWAYS = 2
+        OPEN_EXISTING = 3
+        OPEN_ALWAYS = 4
+        TRUNCATE_EXISTING = 5
+
+    class Attribute(object):
+        READONLY = 0x00000001
+        HIDDEN = 0x00000002
+        SYSTEM = 0x00000004
+        DIRECTORY = 0x00000010
+        ARCHIVE = 0x00000020
+        DEVICE = 0x00000040
+        NORMAL = 0x00000080
+        TEMPORARY = 0x00000100
+        SPARSE_FILE = 0x00000200
+        REPARSE_POINT = 0x00000400
+        COMPRESSED = 0x00000800
+        OFFLINE = 0x00001000
+        NOT_CONTENT_INDEXED = 0x00002000
+        ENCRYPTED = 0x00004000
+
+    class Flag(object):
+        WRITE_THROUGH = 0x80000000
+        OVERLAPPED = 0x40000000
+        NO_BUFFERING = 0x20000000
+        RANDOM_ACCESS = 0x10000000
+        SEQUENTIAL_SCAN = 0x08000000
+        DELETE_ON_CLOSE = 0x04000000
+        BACKUP_SEMANTICS = 0x02000000
+        POSIX_SEMANTICS = 0x01000000
+        OPEN_REPARSE_POINT = 0x00200000
+        OPEN_NO_RECALL = 0x00100000
+        FIRST_PIPE_INSTANCE = 0x00080000
+
+    class Protection(object):
+        NOACCESS = 0x01
+        READONLY = 0x02
+        READWRITE = 0x04
+        WRITECOPY = 0x08
+        EXECUTE = 0x10
+        EXECUTE_READ = 0x20,
+        EXECUTE_READWRITE = 0x40
+        EXECUTE_WRITECOPY = 0x80
+        GUARD = 0x100
+        NOCACHE = 0x200
+        WRITECOMBINE = 0x400
+
+    class FileMapAccess(object):
+        COPY = 0x0001
+        WRITE = 0x0002
+        READ = 0x0004
+        ALL_ACCESS = 0x001f
+        EXECUTE = 0x0020
+
+
+class WindowsFile(File):
+    platform_handler = WindowsInterface
+
+    def __init__(self, handle, *args, **kwargs):
+        super(WindowsFile, self).__init__(*args, **kwargs)
+
+        self.handle = handle
+
+        self.file_map = None
+        self.map_view = None
+
+    def readinto(self, b):
+        return self.get_handler().read_into(self, b)
+
+    def __str__(self):
+        return "<asio_windows.WindowsFile file: %s>" % self.handle

Contents/Libraries/Shared/asio/interfaces/windows/interop.py

@@ -0,0 +1,230 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ctypes.wintypes import *
+from ctypes import *
+import logging
+
+log = logging.getLogger(__name__)
+
+
+CreateFileW = windll.kernel32.CreateFileW
+CreateFileW.argtypes = (LPCWSTR, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE)
+CreateFileW.restype = HANDLE
+
+ReadFile = windll.kernel32.ReadFile
+ReadFile.argtypes = (HANDLE, c_void_p, DWORD, POINTER(DWORD), HANDLE)
+ReadFile.restype = BOOL
+
+
+NULL = 0
+MAX_PATH = 260
+DEFAULT_BUFFER_SIZE = 4096
+LPSECURITY_ATTRIBUTES = c_void_p
+
+
+class WindowsInterop(object):
+    ri_buffer = None
+
+    @classmethod
+    def create_file(cls, path, desired_access, share_mode, creation_disposition, flags_and_attributes):
+        h = CreateFileW(
+            path,
+            desired_access,
+            share_mode,
+            NULL,
+            creation_disposition,
+            flags_and_attributes,
+            NULL
+        )
+
+        error = GetLastError()
+        if error != 0:
+            raise Exception('[WindowsASIO.open] "%s"' % FormatError(error))
+
+        return h
+
+    @classmethod
+    def read(cls, handle, buf_size=DEFAULT_BUFFER_SIZE):
+        buf = create_string_buffer(buf_size)
+        bytes_read = c_ulong(0)
+
+        success = ReadFile(handle, buf, buf_size, byref(bytes_read), NULL)
+
+        error = GetLastError()
+        if error:
+            log.debug('read_file - error: (%s) "%s"', error, FormatError(error))
+
+        if not success and error:
+            raise Exception('[WindowsInterop.read_file] (%s) "%s"' % (error, FormatError(error)))
+
+        # Return if we have a valid buffer
+        if success and bytes_read.value:
+            return buf.value
+
+        return None
+
+    @classmethod
+    def read_into(cls, handle, b):
+        if cls.ri_buffer is None or len(cls.ri_buffer) < len(b):
+            cls.ri_buffer = create_string_buffer(len(b))
+
+        bytes_read = c_ulong(0)
+
+        success = ReadFile(handle, cls.ri_buffer, len(b), byref(bytes_read), NULL)
+        bytes_read = int(bytes_read.value)
+
+        b[:bytes_read] = cls.ri_buffer[:bytes_read]
+
+        error = GetLastError()
+        
+        if not success and error:
+            raise Exception('[WindowsInterop.read_file] (%s) "%s"' % (error, FormatError(error)))
+
+        # Return if we have a valid buffer
+        if success and bytes_read:
+            return bytes_read
+
+        return None
+
+    @classmethod
+    def set_file_pointer(cls, handle, distance, method):
+        pos_high = DWORD(NULL)
+
+        result = windll.kernel32.SetFilePointer(
+            handle,
+            c_ulong(distance),
+            byref(pos_high),
+            DWORD(method)
+        )
+
+        if result == -1:
+            raise Exception('[WindowsASIO.seek] INVALID_SET_FILE_POINTER: "%s"' % FormatError(GetLastError()))
+
+        return result
+
+    @classmethod
+    def get_file_size(cls, handle):
+        return windll.kernel32.GetFileSize(
+            handle,
+            DWORD(NULL)
+        )
+
+    @classmethod
+    def close_handle(cls, handle):
+        return windll.kernel32.CloseHandle(handle)
+
+    @classmethod
+    def create_file_mapping(cls, handle, protect, maximum_size_high=0, maximum_size_low=1):
+        return HANDLE(windll.kernel32.CreateFileMappingW(
+            handle,
+            LPSECURITY_ATTRIBUTES(NULL),
+            DWORD(protect),
+            DWORD(maximum_size_high),
+            DWORD(maximum_size_low),
+            LPCSTR(NULL)
+        ))
+
+    @classmethod
+    def map_view_of_file(cls, map_handle, desired_access, num_bytes, file_offset_high=0, file_offset_low=0):
+        return HANDLE(windll.kernel32.MapViewOfFile(
+            map_handle,
+            DWORD(desired_access),
+            DWORD(file_offset_high),
+            DWORD(file_offset_low),
+            num_bytes
+        ))
+
+    @classmethod
+    def unmap_view_of_file(cls, view_handle):
+        return windll.kernel32.UnmapViewOfFile(view_handle)
+
+    @classmethod
+    def get_mapped_file_name(cls, view_handle, translate_device_name=True):
+        buf = create_string_buffer(MAX_PATH + 1)
+
+        result = windll.psapi.GetMappedFileNameW(
+            cls.get_current_process(),
+            view_handle,
+            buf,
+            MAX_PATH
+        )
+
+        # Raise exception on error
+        error = GetLastError()
+        if result == 0:
+            raise Exception(FormatError(error))
+
+        # Retrieve a clean file name (skipping over NUL bytes)
+        file_name = cls.clean_buffer_value(buf)
+
+        # If we are not translating the device name return here
+        if not translate_device_name:
+            return file_name
+
+        drives = cls.get_logical_drive_strings()
+
+        # Find the drive matching the file_name device name
+        translated = False
+        for drive in drives:
+            device_name = cls.query_dos_device(drive)
+
+            if file_name.startswith(device_name):
+                file_name = drive + file_name[len(device_name):]
+                translated = True
+                break
+
+        if not translated:
+            raise Exception('Unable to translate device name')
+
+        return file_name
+
+    @classmethod
+    def get_logical_drive_strings(cls, buf_size=512):
+        buf = create_string_buffer(buf_size)
+
+        result = windll.kernel32.GetLogicalDriveStringsW(buf_size, buf)
+
+        error = GetLastError()
+        if result == 0:
+            raise Exception(FormatError(error))
+
+        drive_strings = cls.clean_buffer_value(buf)
+        return [dr for dr in drive_strings.split('\\') if dr != '']
+
+    @classmethod
+    def query_dos_device(cls, drive, buf_size=MAX_PATH):
+        buf = create_string_buffer(buf_size)
+
+        result = windll.kernel32.QueryDosDeviceA(
+            drive,
+            buf,
+            buf_size
+        )
+
+        return cls.clean_buffer_value(buf)
+
+    @classmethod
+    def get_current_process(cls):
+        return HANDLE(windll.kernel32.GetCurrentProcess())
+
+    @classmethod
+    def clean_buffer_value(cls, buf):
+        value = ""
+
+        for ch in buf.raw:
+            if ord(ch) != 0:
+                value += ch
+
+        return value

Contents/Libraries/Shared/asio/open_parameters.py

@@ -0,0 +1,47 @@
+from asio.interfaces.posix import PosixInterface
+from asio.interfaces.windows import WindowsInterface
+
+
+class OpenParameters(object):
+    def __init__(self):
+        self.handlers = {}
+
+        # Update handler_parameters with defaults
+        self.posix()
+        self.windows()
+
+    def posix(self, mode=None, buffering=None):
+        """
+        :type mode: str
+        :type buffering: int
+        """
+        self.handlers.update({PosixInterface: {
+            'mode': mode,
+            'buffering': buffering
+        }})
+
+    def windows(self, desired_access=WindowsInterface.GenericAccess.READ,
+                share_mode=WindowsInterface.ShareMode.ALL,
+                creation_disposition=WindowsInterface.CreationDisposition.OPEN_EXISTING,
+                flags_and_attributes=0):
+
+        """
+        :param desired_access: WindowsInterface.DesiredAccess
+        :type desired_access: int
+
+        :param share_mode: WindowsInterface.ShareMode
+        :type share_mode: int
+
+        :param creation_disposition: WindowsInterface.CreationDisposition
+        :type creation_disposition: int
+
+        :param flags_and_attributes: WindowsInterface.Attribute, WindowsInterface.Flag
+        :type flags_and_attributes: int
+        """
+
+        self.handlers.update({WindowsInterface: {
+            'desired_access': desired_access,
+            'share_mode': share_mode,
+            'creation_disposition': creation_disposition,
+            'flags_and_attributes': flags_and_attributes
+        }})

Contents/Libraries/Shared/babelfish/converters/opensubtitles.py

@@ -17,7 +17,7 @@ class OpenSubtitlesConverter(LanguageReverseConverter):
         self.to_opensubtitles = {('por', 'BR'): 'pob', ('gre', None): 'ell', ('srp', None): 'scc', ('srp', 'ME'): 'mne'}
         self.from_opensubtitles = CaseInsensitiveDict({'pob': ('por', 'BR'), 'pb': ('por', 'BR'), 'ell': ('ell', None),
                                                        'scc': ('srp', None), 'mne': ('srp', 'ME')})
-        self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(['pob', 'pb', 'scc', 'mne']))
+        self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(self.from_opensubtitles.keys()))
 
     def convert(self, alpha3, country=None, script=None):
         alpha3b = self.alpha3b_converter.convert(alpha3, country, script)

Contents/Libraries/Shared/babelfish/country.py

@@ -82,7 +82,10 @@ class Country(CountryMeta(str('CountryBase'), (object,), {})):
         self.alpha2 = state
 
     def __getattr__(self, name):
-        return country_converters[name].convert(self.alpha2)
+        try:
+            return country_converters[name].convert(self.alpha2)
+        except KeyError:
+            raise AttributeError(name)
 
     def __hash__(self):
         return hash(self.alpha2)

Contents/Libraries/Shared/babelfish/tests.py

@@ -212,7 +212,7 @@ class TestLanguage(TestCase, _Py26FixTestCase):
         self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
         self.assertRaises(LanguageReverseError, lambda: Language.fromopensubtitles('zzz'))
         self.assertRaises(LanguageConvertError, lambda: Language('aaa').opensubtitles)
-        self.assertEqual(len(language_converters['opensubtitles'].codes), 606)
+        self.assertEqual(len(language_converters['opensubtitles'].codes), 607)
 
         # test with all the LANGUAGES from the opensubtitles api
         # downloaded from: http://www.opensubtitles.org/addons/export_languages.php
@@ -228,6 +228,10 @@ class TestLanguage(TestCase, _Py26FixTestCase):
                 self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
         f.close()
 
+    def test_converter_opensubtitles_codes(self):
+        for code in language_converters['opensubtitles'].from_opensubtitles.keys():
+            self.assertIn(code, language_converters['opensubtitles'].codes)
+
     def test_fromietf_country_script(self):
         language = Language.fromietf('fra-FR-Latn')
         self.assertEqual(language.alpha3, 'fra')
@@ -283,6 +287,11 @@ class TestLanguage(TestCase, _Py26FixTestCase):
         self.assertTrue(hasattr(Language('fra'), 'alpha2'))
         self.assertFalse(hasattr(Language('bej'), 'alpha2'))
 
+    def test_country_hasattr(self):
+        self.assertTrue(hasattr(Country('US'), 'name'))
+        self.assertTrue(hasattr(Country('FR'), 'alpha2'))
+        self.assertFalse(hasattr(Country('BE'), 'none'))
+
     def test_country(self):
         self.assertEqual(Language('por', 'BR').country, Country('BR'))
         self.assertEqual(Language('eng', Country('US')).country, Country('US'))

Contents/Libraries/Shared/certifi/__init__.py

@@ -0,0 +1,3 @@
+from .core import where, old_where
+
+__version__ = "2017.04.17"

Contents/Libraries/Shared/certifi/__main__.py

@@ -0,0 +1,2 @@
+from certifi import where
+print(where())

Contents/Libraries/Shared/certifi/core.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+certifi.py
+~~~~~~~~~~
+
+This module returns the installation location of cacert.pem.
+"""
+import os
+import warnings
+
+
+class DeprecatedBundleWarning(DeprecationWarning):
+    """
+    The weak security bundle is being deprecated. Please bother your service
+    provider to get them to stop using cross-signed roots.
+    """
+
+
+def where():
+    f = os.path.split(__file__)[0]
+
+    return os.path.join(f, 'cacert.pem')
+
+
+def old_where():
+    warnings.warn(
+        "The weak security bundle is being deprecated.",
+        DeprecatedBundleWarning
+    )
+    f = os.path.split(__file__)[0]
+    return os.path.join(f, 'weak.pem')
+
+if __name__ == '__main__':
+    print(where())

Contents/Libraries/Shared/certifi/old_root.pem

@@ -0,0 +1,414 @@
+# Issuer: CN=Entrust.net Secure Server Certification Authority O=Entrust.net OU=www.entrust.net/CPS incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited
+# Subject: CN=Entrust.net Secure Server Certification Authority O=Entrust.net OU=www.entrust.net/CPS incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited
+# Label: "Entrust.net Secure Server CA"
+# Serial: 927650371
+# MD5 Fingerprint: df:f2:80:73:cc:f1:e6:61:73:fc:f5:42:e9:c5:7c:ee
+# SHA1 Fingerprint: 99:a6:9b:e6:1a:fe:88:6b:4d:2b:82:00:7c:b8:54:fc:31:7e:15:39
+# SHA256 Fingerprint: 62:f2:40:27:8c:56:4c:4d:d8:bf:7d:9d:4f:6f:36:6e:a8:94:d2:2f:5f:34:d9:89:a9:83:ac:ec:2f:ff:ed:50
+-----BEGIN CERTIFICATE-----
+MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC
+VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u
+ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc
+KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u
+ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1
+MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE
+ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j
+b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF
+bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg
+U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA
+A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/
+I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3
+wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC
+AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb
+oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5
+BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p
+dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk
+MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp
+b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu
+dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0
+MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi
+E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa
+MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI
+hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN
+95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd
+2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI=
+-----END CERTIFICATE-----
+
+# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 2 Policy Validation Authority
+# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 2 Policy Validation Authority
+# Label: "ValiCert Class 2 VA"
+# Serial: 1
+# MD5 Fingerprint: a9:23:75:9b:ba:49:36:6e:31:c2:db:f2:e7:66:ba:87
+# SHA1 Fingerprint: 31:7a:2a:d0:7f:2b:33:5e:f5:a1:c3:4e:4b:57:e8:b7:d8:f1:fc:a6
+# SHA256 Fingerprint: 58:d0:17:27:9c:d4:dc:63:ab:dd:b1:96:a6:c9:90:6c:30:c4:e0:87:83:ea:e8:c1:60:99:54:d6:93:55:59:6b
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0
+IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz
+BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y
+aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG
+9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy
+NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y
+azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs
+YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw
+Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl
+cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY
+dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9
+WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS
+v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v
+UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu
+IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC
+W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd
+-----END CERTIFICATE-----
+
+# Issuer: CN=NetLock Expressz (Class C) Tanusitvanykiado O=NetLock Halozatbiztonsagi Kft. OU=Tanusitvanykiadok
+# Subject: CN=NetLock Expressz (Class C) Tanusitvanykiado O=NetLock Halozatbiztonsagi Kft. OU=Tanusitvanykiadok
+# Label: "NetLock Express (Class C) Root"
+# Serial: 104
+# MD5 Fingerprint: 4f:eb:f1:f0:70:c2:80:63:5d:58:9f:da:12:3c:a9:c4
+# SHA1 Fingerprint: e3:92:51:2f:0a:cf:f5:05:df:f6:de:06:7f:75:37:e1:65:ea:57:4b
+# SHA256 Fingerprint: 0b:5e:ed:4e:84:64:03:cf:55:e0:65:84:84:40:ed:2a:82:75:8b:f5:b9:aa:1f:25:3d:46:13:cf:a0:80:ff:3f
+-----BEGIN CERTIFICATE-----
+MIIFTzCCBLigAwIBAgIBaDANBgkqhkiG9w0BAQQFADCBmzELMAkGA1UEBhMCSFUx
+ETAPBgNVBAcTCEJ1ZGFwZXN0MScwJQYDVQQKEx5OZXRMb2NrIEhhbG96YXRiaXp0
+b25zYWdpIEtmdC4xGjAYBgNVBAsTEVRhbnVzaXR2YW55a2lhZG9rMTQwMgYDVQQD
+EytOZXRMb2NrIEV4cHJlc3N6IChDbGFzcyBDKSBUYW51c2l0dmFueWtpYWRvMB4X
+DTk5MDIyNTE0MDgxMVoXDTE5MDIyMDE0MDgxMVowgZsxCzAJBgNVBAYTAkhVMREw
+DwYDVQQHEwhCdWRhcGVzdDEnMCUGA1UEChMeTmV0TG9jayBIYWxvemF0Yml6dG9u
+c2FnaSBLZnQuMRowGAYDVQQLExFUYW51c2l0dmFueWtpYWRvazE0MDIGA1UEAxMr
+TmV0TG9jayBFeHByZXNzeiAoQ2xhc3MgQykgVGFudXNpdHZhbnlraWFkbzCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA6+ywbGGKIyWvYCDj2Z/8kwvbXY2wobNA
+OoLO/XXgeDIDhlqGlZHtU/qdQPzm6N3ZW3oDvV3zOwzDUXmbrVWg6dADEK8KuhRC
+2VImESLH0iDMgqSaqf64gXadarfSNnU+sYYJ9m5tfk63euyucYT2BDMIJTLrdKwW
+RMbkQJMdf60CAwEAAaOCAp8wggKbMBIGA1UdEwEB/wQIMAYBAf8CAQQwDgYDVR0P
+AQH/BAQDAgAGMBEGCWCGSAGG+EIBAQQEAwIABzCCAmAGCWCGSAGG+EIBDQSCAlEW
+ggJNRklHWUVMRU0hIEV6ZW4gdGFudXNpdHZhbnkgYSBOZXRMb2NrIEtmdC4gQWx0
+YWxhbm9zIFN6b2xnYWx0YXRhc2kgRmVsdGV0ZWxlaWJlbiBsZWlydCBlbGphcmFz
+b2sgYWxhcGphbiBrZXN6dWx0LiBBIGhpdGVsZXNpdGVzIGZvbHlhbWF0YXQgYSBO
+ZXRMb2NrIEtmdC4gdGVybWVrZmVsZWxvc3NlZy1iaXp0b3NpdGFzYSB2ZWRpLiBB
+IGRpZ2l0YWxpcyBhbGFpcmFzIGVsZm9nYWRhc2FuYWsgZmVsdGV0ZWxlIGF6IGVs
+b2lydCBlbGxlbm9yemVzaSBlbGphcmFzIG1lZ3RldGVsZS4gQXogZWxqYXJhcyBs
+ZWlyYXNhIG1lZ3RhbGFsaGF0byBhIE5ldExvY2sgS2Z0LiBJbnRlcm5ldCBob25s
+YXBqYW4gYSBodHRwczovL3d3dy5uZXRsb2NrLm5ldC9kb2NzIGNpbWVuIHZhZ3kg
+a2VyaGV0byBheiBlbGxlbm9yemVzQG5ldGxvY2submV0IGUtbWFpbCBjaW1lbi4g
+SU1QT1JUQU5UISBUaGUgaXNzdWFuY2UgYW5kIHRoZSB1c2Ugb2YgdGhpcyBjZXJ0
+aWZpY2F0ZSBpcyBzdWJqZWN0IHRvIHRoZSBOZXRMb2NrIENQUyBhdmFpbGFibGUg
+YXQgaHR0cHM6Ly93d3cubmV0bG9jay5uZXQvZG9jcyBvciBieSBlLW1haWwgYXQg
+Y3BzQG5ldGxvY2submV0LjANBgkqhkiG9w0BAQQFAAOBgQAQrX/XDDKACtiG8XmY
+ta3UzbM2xJZIwVzNmtkFLp++UOv0JhQQLdRmF/iewSf98e3ke0ugbLWrmldwpu2g
+pO0u9f38vf5NNwgMvOOWgyL1SRt/Syu0VMGAfJlOHdCM7tCs5ZL6dVb+ZKATj7i4
+Fp1hBWeAyNDYpQcCNJgEjTME1A==
+-----END CERTIFICATE-----
+
+# Issuer: CN=NetLock Uzleti (Class B) Tanusitvanykiado O=NetLock Halozatbiztonsagi Kft. OU=Tanusitvanykiadok
+# Subject: CN=NetLock Uzleti (Class B) Tanusitvanykiado O=NetLock Halozatbiztonsagi Kft. OU=Tanusitvanykiadok
+# Label: "NetLock Business (Class B) Root"
+# Serial: 105
+# MD5 Fingerprint: 39:16:aa:b9:6a:41:e1:14:69:df:9e:6c:3b:72:dc:b6
+# SHA1 Fingerprint: 87:9f:4b:ee:05:df:98:58:3b:e3:60:d6:33:e7:0d:3f:fe:98:71:af
+# SHA256 Fingerprint: 39:df:7b:68:2b:7b:93:8f:84:71:54:81:cc:de:8d:60:d8:f2:2e:c5:98:87:7d:0a:aa:c1:2b:59:18:2b:03:12
+-----BEGIN CERTIFICATE-----
+MIIFSzCCBLSgAwIBAgIBaTANBgkqhkiG9w0BAQQFADCBmTELMAkGA1UEBhMCSFUx
+ETAPBgNVBAcTCEJ1ZGFwZXN0MScwJQYDVQQKEx5OZXRMb2NrIEhhbG96YXRiaXp0
+b25zYWdpIEtmdC4xGjAYBgNVBAsTEVRhbnVzaXR2YW55a2lhZG9rMTIwMAYDVQQD
+EylOZXRMb2NrIFV6bGV0aSAoQ2xhc3MgQikgVGFudXNpdHZhbnlraWFkbzAeFw05
+OTAyMjUxNDEwMjJaFw0xOTAyMjAxNDEwMjJaMIGZMQswCQYDVQQGEwJIVTERMA8G
+A1UEBxMIQnVkYXBlc3QxJzAlBgNVBAoTHk5ldExvY2sgSGFsb3phdGJpenRvbnNh
+Z2kgS2Z0LjEaMBgGA1UECxMRVGFudXNpdHZhbnlraWFkb2sxMjAwBgNVBAMTKU5l
+dExvY2sgVXpsZXRpIChDbGFzcyBCKSBUYW51c2l0dmFueWtpYWRvMIGfMA0GCSqG
+SIb3DQEBAQUAA4GNADCBiQKBgQCx6gTsIKAjwo84YM/HRrPVG/77uZmeBNwcf4xK
+gZjupNTKihe5In+DCnVMm8Bp2GQ5o+2So/1bXHQawEfKOml2mrriRBf8TKPV/riX
+iK+IA4kfpPIEPsgHC+b5sy96YhQJRhTKZPWLgLViqNhr1nGTLbO/CVRY7QbrqHvc
+Q7GhaQIDAQABo4ICnzCCApswEgYDVR0TAQH/BAgwBgEB/wIBBDAOBgNVHQ8BAf8E
+BAMCAAYwEQYJYIZIAYb4QgEBBAQDAgAHMIICYAYJYIZIAYb4QgENBIICURaCAk1G
+SUdZRUxFTSEgRXplbiB0YW51c2l0dmFueSBhIE5ldExvY2sgS2Z0LiBBbHRhbGFu
+b3MgU3pvbGdhbHRhdGFzaSBGZWx0ZXRlbGVpYmVuIGxlaXJ0IGVsamFyYXNvayBh
+bGFwamFuIGtlc3p1bHQuIEEgaGl0ZWxlc2l0ZXMgZm9seWFtYXRhdCBhIE5ldExv
+Y2sgS2Z0LiB0ZXJtZWtmZWxlbG9zc2VnLWJpenRvc2l0YXNhIHZlZGkuIEEgZGln
+aXRhbGlzIGFsYWlyYXMgZWxmb2dhZGFzYW5hayBmZWx0ZXRlbGUgYXogZWxvaXJ0
+IGVsbGVub3J6ZXNpIGVsamFyYXMgbWVndGV0ZWxlLiBBeiBlbGphcmFzIGxlaXJh
+c2EgbWVndGFsYWxoYXRvIGEgTmV0TG9jayBLZnQuIEludGVybmV0IGhvbmxhcGph
+biBhIGh0dHBzOi8vd3d3Lm5ldGxvY2submV0L2RvY3MgY2ltZW4gdmFneSBrZXJo
+ZXRvIGF6IGVsbGVub3J6ZXNAbmV0bG9jay5uZXQgZS1tYWlsIGNpbWVuLiBJTVBP
+UlRBTlQhIFRoZSBpc3N1YW5jZSBhbmQgdGhlIHVzZSBvZiB0aGlzIGNlcnRpZmlj
+YXRlIGlzIHN1YmplY3QgdG8gdGhlIE5ldExvY2sgQ1BTIGF2YWlsYWJsZSBhdCBo
+dHRwczovL3d3dy5uZXRsb2NrLm5ldC9kb2NzIG9yIGJ5IGUtbWFpbCBhdCBjcHNA
+bmV0bG9jay5uZXQuMA0GCSqGSIb3DQEBBAUAA4GBAATbrowXr/gOkDFOzT4JwG06
+sPgzTEdM43WIEJessDgVkcYplswhwG08pXTP2IKlOcNl40JwuyKQ433bNXbhoLXa
+n3BukxowOR0w2y7jfLKRstE3Kfq51hdcR0/jHTjrn9V7lagonhVK0dHQKwCXoOKS
+NitjrFgBazMpUIaD8QFI
+-----END CERTIFICATE-----
+
+# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 3 Policy Validation Authority
+# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 3 Policy Validation Authority
+# Label: "RSA Root Certificate 1"
+# Serial: 1
+# MD5 Fingerprint: a2:6f:53:b7:ee:40:db:4a:68:e7:fa:18:d9:10:4b:72
+# SHA1 Fingerprint: 69:bd:8c:f4:9c:d3:00:fb:59:2e:17:93:ca:55:6a:f3:ec:aa:35:fb
+# SHA256 Fingerprint: bc:23:f9:8a:31:3c:b9:2d:e3:bb:fc:3a:5a:9f:44:61:ac:39:49:4c:4a:e1:5a:9e:9d:f1:31:e9:9b:73:01:9a
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0
+IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz
+BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDMgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y
+aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG
+9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMjIzM1oXDTE5MDYy
+NjAwMjIzM1owgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y
+azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs
+YXNzIDMgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw
+Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl
+cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDjmFGWHOjVsQaBalfD
+cnWTq8+epvzzFlLWLU2fNUSoLgRNB0mKOCn1dzfnt6td3zZxFJmP3MKS8edgkpfs
+2Ejcv8ECIMYkpChMMFp2bbFc893enhBxoYjHW5tBbcqwuI4V7q0zK89HBFx1cQqY
+JJgpp0lZpd34t0NiYfPT4tBVPwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAFa7AliE
+Zwgs3x/be0kz9dNnnfS0ChCzycUs4pJqcXgn8nCDQtM+z6lU9PHYkhaM0QTLS6vJ
+n0WuPIqpsHEzXcjFV9+vqDWzf4mH6eglkrh/hXqu1rweN1gqZ8mRzyqBPu3GOd/A
+PhmcGcwTTYJBtYze4D1gCCAPRX5ron+jjBXu
+-----END CERTIFICATE-----
+
+# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 1 Policy Validation Authority
+# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 1 Policy Validation Authority
+# Label: "ValiCert Class 1 VA"
+# Serial: 1
+# MD5 Fingerprint: 65:58:ab:15:ad:57:6c:1e:a8:a7:b5:69:ac:bf:ff:eb
+# SHA1 Fingerprint: e5:df:74:3c:b6:01:c4:9b:98:43:dc:ab:8c:e8:6a:81:10:9f:e4:8e
+# SHA256 Fingerprint: f4:c1:49:55:1a:30:13:a3:5b:c7:bf:fe:17:a7:f3:44:9b:c1:ab:5b:5a:0a:e7:4b:06:c2:3b:90:00:4c:01:04
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0
+IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz
+BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDEgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y
+aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG
+9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNTIyMjM0OFoXDTE5MDYy
+NTIyMjM0OFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y
+azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs
+YXNzIDEgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw
+Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl
+cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDYWYJ6ibiWuqYvaG9Y
+LqdUHAZu9OqNSLwxlBfw8068srg1knaw0KWlAdcAAxIiGQj4/xEjm84H9b9pGib+
+TunRf50sQB1ZaG6m+FiwnRqP0z/x3BkGgagO4DrdyFNFCQbmD3DD+kCmDuJWBQ8Y
+TfwggtFzVXSNdnKgHZ0dwN0/cQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAFBoPUn0
+LBwGlN+VYH+Wexf+T3GtZMjdd9LvWVXoP+iOBSoh8gfStadS/pyxtuJbdxdA6nLW
+I8sogTLDAHkY7FkXicnGah5xyf23dKUlRWnFSKsZ4UWKJWsZ7uW7EvV/96aNUcPw
+nXS3qT6gpf+2SQMT2iLM7XGCK5nPOrf1LXLI
+-----END CERTIFICATE-----
+
+# Issuer: CN=Equifax Secure eBusiness CA-1 O=Equifax Secure Inc.
+# Subject: CN=Equifax Secure eBusiness CA-1 O=Equifax Secure Inc.
+# Label: "Equifax Secure eBusiness CA 1"
+# Serial: 4
+# MD5 Fingerprint: 64:9c:ef:2e:44:fc:c6:8f:52:07:d0:51:73:8f:cb:3d
+# SHA1 Fingerprint: da:40:18:8b:91:89:a3:ed:ee:ae:da:97:fe:2f:9d:f5:b7:d1:8a:41
+# SHA256 Fingerprint: cf:56:ff:46:a4:a1:86:10:9d:d9:65:84:b5:ee:b5:8a:51:0c:42:75:b0:e5:f9:4f:40:bb:ae:86:5e:19:f6:73
+-----BEGIN CERTIFICATE-----
+MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc
+MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT
+ZWN1cmUgZUJ1c2luZXNzIENBLTEwHhcNOTkwNjIxMDQwMDAwWhcNMjAwNjIxMDQw
+MDAwWjBTMQswCQYDVQQGEwJVUzEcMBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5j
+LjEmMCQGA1UEAxMdRXF1aWZheCBTZWN1cmUgZUJ1c2luZXNzIENBLTEwgZ8wDQYJ
+KoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vGbwXt3fek6lfWg0XTzQaDJj0ItlZ1MRo
+RvC0NcWFAyDGr0WlIVFFQesWWDYyb+JQYmT5/VGcqiTZ9J2DKocKIdMSODRsjQBu
+WqDZQu4aIZX5UkxVWsUPOE9G+m34LjXWHXzr4vCwdYDIqROsvojvOm6rXyo4YgKw
+Env+j6YDAgMBAAGjZjBkMBEGCWCGSAGG+EIBAQQEAwIABzAPBgNVHRMBAf8EBTAD
+AQH/MB8GA1UdIwQYMBaAFEp4MlIR21kWNl7fwRQ2QGpHfEyhMB0GA1UdDgQWBBRK
+eDJSEdtZFjZe38EUNkBqR3xMoTANBgkqhkiG9w0BAQQFAAOBgQB1W6ibAxHm6VZM
+zfmpTMANmvPMZWnmJXbMWbfWVMMdzZmsGd20hdXgPfxiIKeES1hl8eL5lSE/9dR+
+WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN
+/Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ==
+-----END CERTIFICATE-----
+
+# Issuer: CN=Equifax Secure Global eBusiness CA-1 O=Equifax Secure Inc.
+# Subject: CN=Equifax Secure Global eBusiness CA-1 O=Equifax Secure Inc.
+# Label: "Equifax Secure Global eBusiness CA"
+# Serial: 1
+# MD5 Fingerprint: 8f:5d:77:06:27:c4:98:3c:5b:93:78:e7:d7:7d:9b:cc
+# SHA1 Fingerprint: 7e:78:4a:10:1c:82:65:cc:2d:e1:f1:6d:47:b4:40:ca:d9:0a:19:45
+# SHA256 Fingerprint: 5f:0b:62:ea:b5:e3:53:ea:65:21:65:16:58:fb:b6:53:59:f4:43:28:0a:4a:fb:d1:04:d7:7d:10:f9:f0:4c:07
+-----BEGIN CERTIFICATE-----
+MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc
+MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT
+ZWN1cmUgR2xvYmFsIGVCdXNpbmVzcyBDQS0xMB4XDTk5MDYyMTA0MDAwMFoXDTIw
+MDYyMTA0MDAwMFowWjELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0VxdWlmYXggU2Vj
+dXJlIEluYy4xLTArBgNVBAMTJEVxdWlmYXggU2VjdXJlIEdsb2JhbCBlQnVzaW5l
+c3MgQ0EtMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAuucXkAJlsTRVPEnC
+UdXfp9E3j9HngXNBUmCbnaEXJnitx7HoJpQytd4zjTov2/KaelpzmKNc6fuKcxtc
+58O/gGzNqfTWK8D3+ZmqY6KxRwIP1ORROhI8bIpaVIRw28HFkM9yRcuoWcDNM50/
+o5brhTMhHD4ePmBudpxnhcXIw2ECAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgAH
+MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUvqigdHJQa0S3ySPY+6j/s1dr
+aGwwHQYDVR0OBBYEFL6ooHRyUGtEt8kj2Puo/7NXa2hsMA0GCSqGSIb3DQEBBAUA
+A4GBADDiAVGqx+pf2rnQZQ8w1j7aDRRJbpGTJxQx78T3LUX47Me/okENI7SS+RkA
+Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv
+8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV
+-----END CERTIFICATE-----
+
+# Issuer: CN=Thawte Premium Server CA O=Thawte Consulting cc OU=Certification Services Division
+# Subject: CN=Thawte Premium Server CA O=Thawte Consulting cc OU=Certification Services Division
+# Label: "Thawte Premium Server CA"
+# Serial: 1
+# MD5 Fingerprint: 06:9f:69:79:16:66:90:02:1b:8c:8c:a2:c3:07:6f:3a
+# SHA1 Fingerprint: 62:7f:8d:78:27:65:63:99:d2:7d:7f:90:44:c9:fe:b3:f3:3e:fa:9a
+# SHA256 Fingerprint: ab:70:36:36:5c:71:54:aa:29:c2:c2:9f:5d:41:91:16:3b:16:2a:22:25:01:13:57:d5:6d:07:ff:a7:bc:1f:72
+-----BEGIN CERTIFICATE-----
+MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy
+dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t
+MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB
+MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG
+A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp
+b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl
+cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv
+bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE
+VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ
+ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR
+uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG
+9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI
+hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM
+pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg==
+-----END CERTIFICATE-----
+
+# Issuer: CN=Thawte Server CA O=Thawte Consulting cc OU=Certification Services Division
+# Subject: CN=Thawte Server CA O=Thawte Consulting cc OU=Certification Services Division
+# Label: "Thawte Server CA"
+# Serial: 1
+# MD5 Fingerprint: c5:70:c4:a2:ed:53:78:0c:c8:10:53:81:64:cb:d0:1d
+# SHA1 Fingerprint: 23:e5:94:94:51:95:f2:41:48:03:b4:d5:64:d2:a3:a3:f5:d8:8b:8c
+# SHA256 Fingerprint: b4:41:0b:73:e2:e6:ea:ca:47:fb:c4:2f:8f:a4:01:8a:f4:38:1d:c5:4c:fa:a8:44:50:46:1e:ed:09:45:4d:e9
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEm
+MCQGCSqGSIb3DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wHhcNOTYwODAx
+MDAwMDAwWhcNMjAxMjMxMjM1OTU5WjCBxDELMAkGA1UEBhMCWkExFTATBgNVBAgT
+DFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYDVQQKExRUaGF3
+dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNl
+cyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEmMCQGCSqGSIb3
+DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD
+gY0AMIGJAoGBANOkUG7I/1Zr5s9dtuoMaHVHoqrC2oQl/Kj0R1HahbUgdJSGHg91
+yekIYfUGbTBuFRkC6VLAYttNmZ7iagxEOM3+vuNkCXDF/rFrKbYvScg71CcEJRCX
+L+eQbcAoQpnXTEPew/UhbVSfXcNY4cDk2VuwuNy0e982OsK1ZiIS1ocNAgMBAAGj
+EzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEEBQADgYEAB/pMaVz7lcxG
+7oWDTSEwjsrZqG9JGubaUeNgcGyEYRGhGshIPllDfU+VPaGLtwtimHp1it2ITk6e
+QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ
+qdq5snUb9kLy78fyGPmJvKP/iiMucEc=
+-----END CERTIFICATE-----
+
+# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority
+# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority
+# Label: "Verisign Class 3 Public Primary Certification Authority"
+# Serial: 149843929435818692848040365716851702463
+# MD5 Fingerprint: 10:fc:63:5d:f6:26:3e:0d:f3:25:be:5f:79:cd:67:67
+# SHA1 Fingerprint: 74:2c:31:92:e6:07:e4:24:eb:45:49:54:2b:e1:bb:c5:3e:61:74:e2
+# SHA256 Fingerprint: e7:68:56:34:ef:ac:f6:9a:ce:93:9a:6b:25:5b:7b:4f:ab:ef:42:93:5b:50:a2:65:ac:b5:cb:60:27:e4:4e:70
+-----BEGIN CERTIFICATE-----
+MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG
+A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz
+cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2
+MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV
+BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt
+YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN
+ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE
+BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is
+I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G
+CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do
+lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc
+AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k
+-----END CERTIFICATE-----
+
+# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority
+# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority
+# Label: "Verisign Class 3 Public Primary Certification Authority"
+# Serial: 80507572722862485515306429940691309246
+# MD5 Fingerprint: ef:5a:f1:33:ef:f1:cd:bb:51:02:ee:12:14:4b:96:c4
+# SHA1 Fingerprint: a1:db:63:93:91:6f:17:e4:18:55:09:40:04:15:c7:02:40:b0:ae:6b
+# SHA256 Fingerprint: a4:b6:b3:99:6f:c2:f3:06:b3:fd:86:81:bd:63:41:3d:8c:50:09:cc:4f:a3:29:c2:cc:f0:e2:fa:1b:14:03:05
+-----BEGIN CERTIFICATE-----
+MIICPDCCAaUCEDyRMcsf9tAbDpq40ES/Er4wDQYJKoZIhvcNAQEFBQAwXzELMAkG
+A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz
+cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2
+MDEyOTAwMDAwMFoXDTI4MDgwMjIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV
+BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt
+YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN
+ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE
+BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is
+I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G
+CSqGSIb3DQEBBQUAA4GBABByUqkFFBkyCEHwxWsKzH4PIRnN5GfcX6kb5sroc50i
+2JhucwNhkcV8sEVAbkSdjbCxlnRhLQ2pRdKkkirWmnWXbj9T/UWZYB2oK0z5XqcJ
+2HUw19JlYD1n1khVdWk/kfVIC0dpImmClr7JyDiGSnoscxlIaU5rfGW/D/xwzoiQ
+-----END CERTIFICATE-----
+
+# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority - G2/(c) 1998 VeriSign, Inc. - For authorized use only/VeriSign Trust Network
+# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority - G2/(c) 1998 VeriSign, Inc. - For authorized use only/VeriSign Trust Network
+# Label: "Verisign Class 3 Public Primary Certification Authority - G2"
+# Serial: 167285380242319648451154478808036881606
+# MD5 Fingerprint: a2:33:9b:4c:74:78:73:d4:6c:e7:c1:f3:8d:cb:5c:e9
+# SHA1 Fingerprint: 85:37:1c:a6:e5:50:14:3d:ce:28:03:47:1b:de:3a:09:e8:f8:77:0f
+# SHA256 Fingerprint: 83:ce:3c:12:29:68:8a:59:3d:48:5f:81:97:3c:0f:91:95:43:1e:da:37:cc:5e:36:43:0e:79:c7:a8:88:63:8b
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ
+BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh
+c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy
+MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp
+emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X
+DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw
+FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg
+UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo
+YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5
+MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB
+AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4
+pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0
+13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID
+AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk
+U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i
+F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY
+oJ2daZH9
+-----END CERTIFICATE-----
+
+# Issuer: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc.
+# Subject: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc.
+# Label: "GTE CyberTrust Global Root"
+# Serial: 421
+# MD5 Fingerprint: ca:3d:d3:68:f1:03:5c:d0:32:fa:b8:2b:59:e8:5a:db
+# SHA1 Fingerprint: 97:81:79:50:d8:1c:96:70:cc:34:d8:09:cf:79:44:31:36:7e:f4:74
+# SHA256 Fingerprint: a5:31:25:18:8d:21:10:aa:96:4b:02:c7:b7:c6:da:32:03:17:08:94:e5:fb:71:ff:fb:66:67:d5:e6:81:0a:36
+-----BEGIN CERTIFICATE-----
+MIICWjCCAcMCAgGlMA0GCSqGSIb3DQEBBAUAMHUxCzAJBgNVBAYTAlVTMRgwFgYD
+VQQKEw9HVEUgQ29ycG9yYXRpb24xJzAlBgNVBAsTHkdURSBDeWJlclRydXN0IFNv
+bHV0aW9ucywgSW5jLjEjMCEGA1UEAxMaR1RFIEN5YmVyVHJ1c3QgR2xvYmFsIFJv
+b3QwHhcNOTgwODEzMDAyOTAwWhcNMTgwODEzMjM1OTAwWjB1MQswCQYDVQQGEwJV
+UzEYMBYGA1UEChMPR1RFIENvcnBvcmF0aW9uMScwJQYDVQQLEx5HVEUgQ3liZXJU
+cnVzdCBTb2x1dGlvbnMsIEluYy4xIzAhBgNVBAMTGkdURSBDeWJlclRydXN0IEds
+b2JhbCBSb290MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCVD6C28FCc6HrH
+iM3dFw4usJTQGz0O9pTAipTHBsiQl8i4ZBp6fmw8U+E3KHNgf7KXUwefU/ltWJTS
+r41tiGeA5u2ylc9yMcqlHHK6XALnZELn+aks1joNrI1CqiQBOeacPwGFVw1Yh0X4
+04Wqk2kmhXBIgD8SFcd5tB8FLztimQIDAQABMA0GCSqGSIb3DQEBBAUAA4GBAG3r
+GwnpXtlR22ciYaQqPEh346B8pt5zohQDhT37qw4wxYMWM4ETCJ57NE7fQMh017l9
+3PR2VX2bY1QY6fDq81yx2YtCHrnAlU66+tXifPVoYb+O7AWXX1uw16OFNMQkpw0P
+lZPvy5TYnh+dXIVtx6quTx8itc2VrbqnzPmrC3p/
+-----END CERTIFICATE-----
+
+# Issuer: C=US, O=Equifax, OU=Equifax Secure Certificate Authority
+# Subject: C=US, O=Equifax, OU=Equifax Secure Certificate Authority
+# Label: "Equifax Secure Certificate Authority"
+# Serial: 903804111
+# MD5 Fingerprint: 67:cb:9d:c0:13:24:8a:82:9b:b2:17:1e:d1:1b:ec:d4
+# SHA1 Fingerprint: d2:32:09:ad:23:d3:14:23:21:74:e4:0d:7f:9d:62:13:97:86:63:3a
+# SHA256 Fingerprint: 08:29:7a:40:47:db:a2:36:80:c7:31:db:6e:31:76:53:ca:78:48:e1:be:bd:3a:0b:01:79:a7:07:f9:2c:f1:78
+-----BEGIN CERTIFICATE-----
+MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV
+UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy
+dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1
+MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VxdWlmYXgxLTArBgNVBAsTJEVx
+dWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPRfM6f
+BeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+A
+cJkVV5MW8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kC
+AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQ
+MA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlm
+aWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTgw
+ODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvSspXXR9gj
+IBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQF
+MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA
+A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y
+7qj/WsjTVbJmcVfewCHrPSqnI0kBBIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh
+1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4
+-----END CERTIFICATE-----

Contents/Libraries/Shared/click/__init__.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+    click
+    ~~~~~
+
+    Click is a simple Python module that wraps the stdlib's optparse to make
+    writing command line scripts fun.  Unlike other modules, it's based around
+    a simple API that does not come with too much magic and is composable.
+
+    In case optparse ever gets removed from the stdlib, it will be shipped by
+    this module.
+
+    :copyright: (c) 2014 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+
+# Core classes
+from .core import Context, BaseCommand, Command, MultiCommand, Group, \
+     CommandCollection, Parameter, Option, Argument
+
+# Globals
+from .globals import get_current_context
+
+# Decorators
+from .decorators import pass_context, pass_obj, make_pass_decorator, \
+     command, group, argument, option, confirmation_option, \
+     password_option, version_option, help_option
+
+# Types
+from .types import ParamType, File, Path, Choice, IntRange, Tuple, \
+     STRING, INT, FLOAT, BOOL, UUID, UNPROCESSED
+
+# Utilities
+from .utils import echo, get_binary_stream, get_text_stream, open_file, \
+     format_filename, get_app_dir, get_os_args
+
+# Terminal functions
+from .termui import prompt, confirm, get_terminal_size, echo_via_pager, \
+     progressbar, clear, style, unstyle, secho, edit, launch, getchar, \
+     pause
+
+# Exceptions
+from .exceptions import ClickException, UsageError, BadParameter, \
+     FileError, Abort, NoSuchOption, BadOptionUsage, BadArgumentUsage, \
+     MissingParameter
+
+# Formatting
+from .formatting import HelpFormatter, wrap_text
+
+# Parsing
+from .parser import OptionParser
+
+
+__all__ = [
+    # Core classes
+    'Context', 'BaseCommand', 'Command', 'MultiCommand', 'Group',
+    'CommandCollection', 'Parameter', 'Option', 'Argument',
+
+    # Globals
+    'get_current_context',
+
+    # Decorators
+    'pass_context', 'pass_obj', 'make_pass_decorator', 'command', 'group',
+    'argument', 'option', 'confirmation_option', 'password_option',
+    'version_option', 'help_option',
+
+    # Types
+    'ParamType', 'File', 'Path', 'Choice', 'IntRange', 'Tuple', 'STRING',
+    'INT', 'FLOAT', 'BOOL', 'UUID', 'UNPROCESSED',
+
+    # Utilities
+    'echo', 'get_binary_stream', 'get_text_stream', 'open_file',
+    'format_filename', 'get_app_dir', 'get_os_args',
+
+    # Terminal functions
+    'prompt', 'confirm', 'get_terminal_size', 'echo_via_pager',
+    'progressbar', 'clear', 'style', 'unstyle', 'secho', 'edit', 'launch',
+    'getchar', 'pause',
+
+    # Exceptions
+    'ClickException', 'UsageError', 'BadParameter', 'FileError',
+    'Abort', 'NoSuchOption', 'BadOptionUsage', 'BadArgumentUsage',
+    'MissingParameter',
+
+    # Formatting
+    'HelpFormatter', 'wrap_text',
+
+    # Parsing
+    'OptionParser',
+]
+
+
+# Controls if click should emit the warning about the use of unicode
+# literals.
+disable_unicode_literals_warning = False
+
+
+__version__ = '6.7'

Contents/Libraries/Shared/click/_bashcomplete.py

@@ -0,0 +1,83 @@
+import os
+import re
+from .utils import echo
+from .parser import split_arg_string
+from .core import MultiCommand, Option
+
+
+COMPLETION_SCRIPT = '''
+%(complete_func)s() {
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\
+                   COMP_CWORD=$COMP_CWORD \\
+                   %(autocomplete_var)s=complete $1 ) )
+    return 0
+}
+
+complete -F %(complete_func)s -o default %(script_names)s
+'''
+
+_invalid_ident_char_re = re.compile(r'[^a-zA-Z0-9_]')
+
+
+def get_completion_script(prog_name, complete_var):
+    cf_name = _invalid_ident_char_re.sub('', prog_name.replace('-', '_'))
+    return (COMPLETION_SCRIPT % {
+        'complete_func': '_%s_completion' % cf_name,
+        'script_names': prog_name,
+        'autocomplete_var': complete_var,
+    }).strip() + ';'
+
+
+def resolve_ctx(cli, prog_name, args):
+    ctx = cli.make_context(prog_name, args, resilient_parsing=True)
+    while ctx.protected_args + ctx.args and isinstance(ctx.command, MultiCommand):
+        a = ctx.protected_args + ctx.args
+        cmd = ctx.command.get_command(ctx, a[0])
+        if cmd is None:
+            return None
+        ctx = cmd.make_context(a[0], a[1:], parent=ctx, resilient_parsing=True)
+    return ctx
+
+
+def get_choices(cli, prog_name, args, incomplete):
+    ctx = resolve_ctx(cli, prog_name, args)
+    if ctx is None:
+        return
+
+    choices = []
+    if incomplete and not incomplete[:1].isalnum():
+        for param in ctx.command.params:
+            if not isinstance(param, Option):
+                continue
+            choices.extend(param.opts)
+            choices.extend(param.secondary_opts)
+    elif isinstance(ctx.command, MultiCommand):
+        choices.extend(ctx.command.list_commands(ctx))
+
+    for item in choices:
+        if item.startswith(incomplete):
+            yield item
+
+
+def do_complete(cli, prog_name):
+    cwords = split_arg_string(os.environ['COMP_WORDS'])
+    cword = int(os.environ['COMP_CWORD'])
+    args = cwords[1:cword]
+    try:
+        incomplete = cwords[cword]
+    except IndexError:
+        incomplete = ''
+
+    for item in get_choices(cli, prog_name, args, incomplete):
+        echo(item)
+
+    return True
+
+
+def bashcomplete(cli, prog_name, complete_var, complete_instr):
+    if complete_instr == 'source':
+        echo(get_completion_script(prog_name, complete_var))
+        return True
+    elif complete_instr == 'complete':
+        return do_complete(cli, prog_name)
+    return False

Contents/Libraries/Shared/click/_compat.py

@@ -0,0 +1,648 @@
+import re
+import io
+import os
+import sys
+import codecs
+from weakref import WeakKeyDictionary
+
+
+PY2 = sys.version_info[0] == 2
+WIN = sys.platform.startswith('win')
+DEFAULT_COLUMNS = 80
+
+
+_ansi_re = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')
+
+
+def get_filesystem_encoding():
+    return sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
+def _make_text_stream(stream, encoding, errors):
+    if encoding is None:
+        encoding = get_best_encoding(stream)
+    if errors is None:
+        errors = 'replace'
+    return _NonClosingTextIOWrapper(stream, encoding, errors,
+                                    line_buffering=True)
+
+
+def is_ascii_encoding(encoding):
+    """Checks if a given encoding is ascii."""
+    try:
+        return codecs.lookup(encoding).name == 'ascii'
+    except LookupError:
+        return False
+
+
+def get_best_encoding(stream):
+    """Returns the default stream encoding if not found."""
+    rv = getattr(stream, 'encoding', None) or sys.getdefaultencoding()
+    if is_ascii_encoding(rv):
+        return 'utf-8'
+    return rv
+
+
+class _NonClosingTextIOWrapper(io.TextIOWrapper):
+
+    def __init__(self, stream, encoding, errors, **extra):
+        self._stream = stream = _FixupStream(stream)
+        io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra)
+
+    # The io module is a place where the Python 3 text behavior
+    # was forced upon Python 2, so we need to unbreak
+    # it to look like Python 2.
+    if PY2:
+        def write(self, x):
+            if isinstance(x, str) or is_bytes(x):
+                try:
+                    self.flush()
+                except Exception:
+                    pass
+                return self.buffer.write(str(x))
+            return io.TextIOWrapper.write(self, x)
+
+        def writelines(self, lines):
+            for line in lines:
+                self.write(line)
+
+    def __del__(self):
+        try:
+            self.detach()
+        except Exception:
+            pass
+
+    def isatty(self):
+        # https://bitbucket.org/pypy/pypy/issue/1803
+        return self._stream.isatty()
+
+
+class _FixupStream(object):
+    """The new io interface needs more from streams than streams
+    traditionally implement.  As such, this fix-up code is necessary in
+    some circumstances.
+    """
+
+    def __init__(self, stream):
+        self._stream = stream
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+    def read1(self, size):
+        f = getattr(self._stream, 'read1', None)
+        if f is not None:
+            return f(size)
+        # We only dispatch to readline instead of read in Python 2 as we
+        # do not want cause problems with the different implementation
+        # of line buffering.
+        if PY2:
+            return self._stream.readline(size)
+        return self._stream.read(size)
+
+    def readable(self):
+        x = getattr(self._stream, 'readable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.read(0)
+        except Exception:
+            return False
+        return True
+
+    def writable(self):
+        x = getattr(self._stream, 'writable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.write('')
+        except Exception:
+            try:
+                self._stream.write(b'')
+            except Exception:
+                return False
+        return True
+
+    def seekable(self):
+        x = getattr(self._stream, 'seekable', None)
+        if x is not None:
+            return x()
+        try:
+            self._stream.seek(self._stream.tell())
+        except Exception:
+            return False
+        return True
+
+
+if PY2:
+    text_type = unicode
+    bytes = str
+    raw_input = raw_input
+    string_types = (str, unicode)
+    iteritems = lambda x: x.iteritems()
+    range_type = xrange
+
+    def is_bytes(x):
+        return isinstance(x, (buffer, bytearray))
+
+    _identifier_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
+
+    # For Windows, we need to force stdout/stdin/stderr to binary if it's
+    # fetched for that.  This obviously is not the most correct way to do
+    # it as it changes global state.  Unfortunately, there does not seem to
+    # be a clear better way to do it as just reopening the file in binary
+    # mode does not change anything.
+    #
+    # An option would be to do what Python 3 does and to open the file as
+    # binary only, patch it back to the system, and then use a wrapper
+    # stream that converts newlines.  It's not quite clear what's the
+    # correct option here.
+    #
+    # This code also lives in _winconsole for the fallback to the console
+    # emulation stream.
+    #
+    # There are also Windows environments where the `msvcrt` module is not
+    # available (which is why we use try-catch instead of the WIN variable
+    # here), such as the Google App Engine development server on Windows. In
+    # those cases there is just nothing we can do.
+    try:
+        import msvcrt
+    except ImportError:
+        set_binary_mode = lambda x: x
+    else:
+        def set_binary_mode(f):
+            try:
+                fileno = f.fileno()
+            except Exception:
+                pass
+            else:
+                msvcrt.setmode(fileno, os.O_BINARY)
+            return f
+
+    def isidentifier(x):
+        return _identifier_re.search(x) is not None
+
+    def get_binary_stdin():
+        return set_binary_mode(sys.stdin)
+
+    def get_binary_stdout():
+        return set_binary_mode(sys.stdout)
+
+    def get_binary_stderr():
+        return set_binary_mode(sys.stderr)
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdin, encoding, errors)
+
+    def get_text_stdout(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stdout, encoding, errors)
+
+    def get_text_stderr(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _make_text_stream(sys.stderr, encoding, errors)
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), 'replace')
+        return value
+else:
+    import io
+    text_type = str
+    raw_input = input
+    string_types = (str,)
+    range_type = range
+    isidentifier = lambda x: x.isidentifier()
+    iteritems = lambda x: iter(x.items())
+
+    def is_bytes(x):
+        return isinstance(x, (bytes, memoryview, bytearray))
+
+    def _is_binary_reader(stream, default=False):
+        try:
+            return isinstance(stream.read(0), bytes)
+        except Exception:
+            return default
+            # This happens in some cases where the stream was already
+            # closed.  In this case, we assume the default.
+
+    def _is_binary_writer(stream, default=False):
+        try:
+            stream.write(b'')
+        except Exception:
+            try:
+                stream.write('')
+                return False
+            except Exception:
+                pass
+            return default
+        return True
+
+    def _find_binary_reader(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detaching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_reader(stream, False):
+            return stream
+
+        buf = getattr(stream, 'buffer', None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_reader(buf, True):
+            return buf
+
+    def _find_binary_writer(stream):
+        # We need to figure out if the given stream is already binary.
+        # This can happen because the official docs recommend detatching
+        # the streams to get binary streams.  Some code might do this, so
+        # we need to deal with this case explicitly.
+        if _is_binary_writer(stream, False):
+            return stream
+
+        buf = getattr(stream, 'buffer', None)
+
+        # Same situation here; this time we assume that the buffer is
+        # actually binary in case it's closed.
+        if buf is not None and _is_binary_writer(buf, True):
+            return buf
+
+    def _stream_is_misconfigured(stream):
+        """A stream is misconfigured if its encoding is ASCII."""
+        # If the stream does not have an encoding set, we assume it's set
+        # to ASCII.  This appears to happen in certain unittest
+        # environments.  It's not quite clear what the correct behavior is
+        # but this at least will force Click to recover somehow.
+        return is_ascii_encoding(getattr(stream, 'encoding', None) or 'ascii')
+
+    def _is_compatible_text_stream(stream, encoding, errors):
+        stream_encoding = getattr(stream, 'encoding', None)
+        stream_errors = getattr(stream, 'errors', None)
+
+        # Perfect match.
+        if stream_encoding == encoding and stream_errors == errors:
+            return True
+
+        # Otherwise, it's only a compatible stream if we did not ask for
+        # an encoding.
+        if encoding is None:
+            return stream_encoding is not None
+
+        return False
+
+    def _force_correct_text_reader(text_reader, encoding, errors):
+        if _is_binary_reader(text_reader, False):
+            binary_reader = text_reader
+        else:
+            # If there is no target encoding set, we need to verify that the
+            # reader is not actually misconfigured.
+            if encoding is None and not _stream_is_misconfigured(text_reader):
+                return text_reader
+
+            if _is_compatible_text_stream(text_reader, encoding, errors):
+                return text_reader
+
+            # If the reader has no encoding, we try to find the underlying
+            # binary reader for it.  If that fails because the environment is
+            # misconfigured, we silently go with the same reader because this
+            # is too common to happen.  In that case, mojibake is better than
+            # exceptions.
+            binary_reader = _find_binary_reader(text_reader)
+            if binary_reader is None:
+                return text_reader
+
+        # At this point, we default the errors to replace instead of strict
+        # because nobody handles those errors anyways and at this point
+        # we're so fundamentally fucked that nothing can repair it.
+        if errors is None:
+            errors = 'replace'
+        return _make_text_stream(binary_reader, encoding, errors)
+
+    def _force_correct_text_writer(text_writer, encoding, errors):
+        if _is_binary_writer(text_writer, False):
+            binary_writer = text_writer
+        else:
+            # If there is no target encoding set, we need to verify that the
+            # writer is not actually misconfigured.
+            if encoding is None and not _stream_is_misconfigured(text_writer):
+                return text_writer
+
+            if _is_compatible_text_stream(text_writer, encoding, errors):
+                return text_writer
+
+            # If the writer has no encoding, we try to find the underlying
+            # binary writer for it.  If that fails because the environment is
+            # misconfigured, we silently go with the same writer because this
+            # is too common to happen.  In that case, mojibake is better than
+            # exceptions.
+            binary_writer = _find_binary_writer(text_writer)
+            if binary_writer is None:
+                return text_writer
+
+        # At this point, we default the errors to replace instead of strict
+        # because nobody handles those errors anyways and at this point
+        # we're so fundamentally fucked that nothing can repair it.
+        if errors is None:
+            errors = 'replace'
+        return _make_text_stream(binary_writer, encoding, errors)
+
+    def get_binary_stdin():
+        reader = _find_binary_reader(sys.stdin)
+        if reader is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stdin.')
+        return reader
+
+    def get_binary_stdout():
+        writer = _find_binary_writer(sys.stdout)
+        if writer is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stdout.')
+        return writer
+
+    def get_binary_stderr():
+        writer = _find_binary_writer(sys.stderr)
+        if writer is None:
+            raise RuntimeError('Was not able to determine binary '
+                               'stream for sys.stderr.')
+        return writer
+
+    def get_text_stdin(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_reader(sys.stdin, encoding, errors)
+
+    def get_text_stdout(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(sys.stdout, encoding, errors)
+
+    def get_text_stderr(encoding=None, errors=None):
+        rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+        if rv is not None:
+            return rv
+        return _force_correct_text_writer(sys.stderr, encoding, errors)
+
+    def filename_to_ui(value):
+        if isinstance(value, bytes):
+            value = value.decode(get_filesystem_encoding(), 'replace')
+        else:
+            value = value.encode('utf-8', 'surrogateescape') \
+                .decode('utf-8', 'replace')
+        return value
+
+
+def get_streerror(e, default=None):
+    if hasattr(e, 'strerror'):
+        msg = e.strerror
+    else:
+        if default is not None:
+            msg = default
+        else:
+            msg = str(e)
+    if isinstance(msg, bytes):
+        msg = msg.decode('utf-8', 'replace')
+    return msg
+
+
+def open_stream(filename, mode='r', encoding=None, errors='strict',
+                atomic=False):
+    # Standard streams first.  These are simple because they don't need
+    # special handling for the atomic flag.  It's entirely ignored.
+    if filename == '-':
+        if 'w' in mode:
+            if 'b' in mode:
+                return get_binary_stdout(), False
+            return get_text_stdout(encoding=encoding, errors=errors), False
+        if 'b' in mode:
+            return get_binary_stdin(), False
+        return get_text_stdin(encoding=encoding, errors=errors), False
+
+    # Non-atomic writes directly go out through the regular open functions.
+    if not atomic:
+        if encoding is None:
+            return open(filename, mode), True
+        return io.open(filename, mode, encoding=encoding, errors=errors), True
+
+    # Some usability stuff for atomic writes
+    if 'a' in mode:
+        raise ValueError(
+            'Appending to an existing file is not supported, because that '
+            'would involve an expensive `copy`-operation to a temporary '
+            'file. Open the file in normal `w`-mode and copy explicitly '
+            'if that\'s what you\'re after.'
+        )
+    if 'x' in mode:
+        raise ValueError('Use the `overwrite`-parameter instead.')
+    if 'w' not in mode:
+        raise ValueError('Atomic writes only make sense with `w`-mode.')
+
+    # Atomic writes are more complicated.  They work by opening a file
+    # as a proxy in the same folder and then using the fdopen
+    # functionality to wrap it in a Python file.  Then we wrap it in an
+    # atomic file that moves the file over on close.
+    import tempfile
+    fd, tmp_filename = tempfile.mkstemp(dir=os.path.dirname(filename),
+                                        prefix='.__atomic-write')
+
+    if encoding is not None:
+        f = io.open(fd, mode, encoding=encoding, errors=errors)
+    else:
+        f = os.fdopen(fd, mode)
+
+    return _AtomicFile(f, tmp_filename, filename), True
+
+
+# Used in a destructor call, needs extra protection from interpreter cleanup.
+if hasattr(os, 'replace'):
+    _replace = os.replace
+    _can_replace = True
+else:
+    _replace = os.rename
+    _can_replace = not WIN
+
+
+class _AtomicFile(object):
+
+    def __init__(self, f, tmp_filename, real_filename):
+        self._f = f
+        self._tmp_filename = tmp_filename
+        self._real_filename = real_filename
+        self.closed = False
+
+    @property
+    def name(self):
+        return self._real_filename
+
+    def close(self, delete=False):
+        if self.closed:
+            return
+        self._f.close()
+        if not _can_replace:
+            try:
+                os.remove(self._real_filename)
+            except OSError:
+                pass
+        _replace(self._tmp_filename, self._real_filename)
+        self.closed = True
+
+    def __getattr__(self, name):
+        return getattr(self._f, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close(delete=exc_type is not None)
+
+    def __repr__(self):
+        return repr(self._f)
+
+
+auto_wrap_for_ansi = None
+colorama = None
+get_winterm_size = None
+
+
+def strip_ansi(value):
+    return _ansi_re.sub('', value)
+
+
+def should_strip_ansi(stream=None, color=None):
+    if color is None:
+        if stream is None:
+            stream = sys.stdin
+        return not isatty(stream)
+    return not color
+
+
+# If we're on Windows, we provide transparent integration through
+# colorama.  This will make ANSI colors through the echo function
+# work automatically.
+if WIN:
+    # Windows has a smaller terminal
+    DEFAULT_COLUMNS = 79
+
+    from ._winconsole import _get_windows_console_stream
+
+    def _get_argv_encoding():
+        import locale
+        return locale.getpreferredencoding()
+
+    if PY2:
+        def raw_input(prompt=''):
+            sys.stderr.flush()
+            if prompt:
+                stdout = _default_text_stdout()
+                stdout.write(prompt)
+            stdin = _default_text_stdin()
+            return stdin.readline().rstrip('\r\n')
+
+    try:
+        import colorama
+    except ImportError:
+        pass
+    else:
+        _ansi_stream_wrappers = WeakKeyDictionary()
+
+        def auto_wrap_for_ansi(stream, color=None):
+            """This function wraps a stream so that calls through colorama
+            are issued to the win32 console API to recolor on demand.  It
+            also ensures to reset the colors if a write call is interrupted
+            to not destroy the console afterwards.
+            """
+            try:
+                cached = _ansi_stream_wrappers.get(stream)
+            except Exception:
+                cached = None
+            if cached is not None:
+                return cached
+            strip = should_strip_ansi(stream, color)
+            ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip)
+            rv = ansi_wrapper.stream
+            _write = rv.write
+
+            def _safe_write(s):
+                try:
+                    return _write(s)
+                except:
+                    ansi_wrapper.reset_all()
+                    raise
+
+            rv.write = _safe_write
+            try:
+                _ansi_stream_wrappers[stream] = rv
+            except Exception:
+                pass
+            return rv
+
+        def get_winterm_size():
+            win = colorama.win32.GetConsoleScreenBufferInfo(
+                colorama.win32.STDOUT).srWindow
+            return win.Right - win.Left, win.Bottom - win.Top
+else:
+    def _get_argv_encoding():
+        return getattr(sys.stdin, 'encoding', None) or get_filesystem_encoding()
+
+    _get_windows_console_stream = lambda *x: None
+
+
+def term_len(x):
+    return len(strip_ansi(x))
+
+
+def isatty(stream):
+    try:
+        return stream.isatty()
+    except Exception:
+        return False
+
+
+def _make_cached_stream_func(src_func, wrapper_func):
+    cache = WeakKeyDictionary()
+    def func():
+        stream = src_func()
+        try:
+            rv = cache.get(stream)
+        except Exception:
+            rv = None
+        if rv is not None:
+            return rv
+        rv = wrapper_func()
+        try:
+            cache[stream] = rv
+        except Exception:
+            pass
+        return rv
+    return func
+
+
+_default_text_stdin = _make_cached_stream_func(
+    lambda: sys.stdin, get_text_stdin)
+_default_text_stdout = _make_cached_stream_func(
+    lambda: sys.stdout, get_text_stdout)
+_default_text_stderr = _make_cached_stream_func(
+    lambda: sys.stderr, get_text_stderr)
+
+
+binary_streams = {
+    'stdin': get_binary_stdin,
+    'stdout': get_binary_stdout,
+    'stderr': get_binary_stderr,
+}
+
+text_streams = {
+    'stdin': get_text_stdin,
+    'stdout': get_text_stdout,
+    'stderr': get_text_stderr,
+}

Contents/Libraries/Shared/click/_termui_impl.py

@@ -0,0 +1,547 @@
+"""
+    click._termui_impl
+    ~~~~~~~~~~~~~~~~~~
+
+    This module contains implementations for the termui module.  To keep the
+    import time of Click down, some infrequently used functionality is placed
+    in this module and only imported as needed.
+
+    :copyright: (c) 2014 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import time
+import math
+from ._compat import _default_text_stdout, range_type, PY2, isatty, \
+     open_stream, strip_ansi, term_len, get_best_encoding, WIN
+from .utils import echo
+from .exceptions import ClickException
+
+
+if os.name == 'nt':
+    BEFORE_BAR = '\r'
+    AFTER_BAR = '\n'
+else:
+    BEFORE_BAR = '\r\033[?25l'
+    AFTER_BAR = '\033[?25h\n'
+
+
+def _length_hint(obj):
+    """Returns the length hint of an object."""
+    try:
+        return len(obj)
+    except (AttributeError, TypeError):
+        try:
+            get_hint = type(obj).__length_hint__
+        except AttributeError:
+            return None
+        try:
+            hint = get_hint(obj)
+        except TypeError:
+            return None
+        if hint is NotImplemented or \
+           not isinstance(hint, (int, long)) or \
+           hint < 0:
+            return None
+        return hint
+
+
+class ProgressBar(object):
+
+    def __init__(self, iterable, length=None, fill_char='#', empty_char=' ',
+                 bar_template='%(bar)s', info_sep='  ', show_eta=True,
+                 show_percent=None, show_pos=False, item_show_func=None,
+                 label=None, file=None, color=None, width=30):
+        self.fill_char = fill_char
+        self.empty_char = empty_char
+        self.bar_template = bar_template
+        self.info_sep = info_sep
+        self.show_eta = show_eta
+        self.show_percent = show_percent
+        self.show_pos = show_pos
+        self.item_show_func = item_show_func
+        self.label = label or ''
+        if file is None:
+            file = _default_text_stdout()
+        self.file = file
+        self.color = color
+        self.width = width
+        self.autowidth = width == 0
+
+        if length is None:
+            length = _length_hint(iterable)
+        if iterable is None:
+            if length is None:
+                raise TypeError('iterable or length is required')
+            iterable = range_type(length)
+        self.iter = iter(iterable)
+        self.length = length
+        self.length_known = length is not None
+        self.pos = 0
+        self.avg = []
+        self.start = self.last_eta = time.time()
+        self.eta_known = False
+        self.finished = False
+        self.max_width = None
+        self.entered = False
+        self.current_item = None
+        self.is_hidden = not isatty(self.file)
+        self._last_line = None
+
+    def __enter__(self):
+        self.entered = True
+        self.render_progress()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.render_finish()
+
+    def __iter__(self):
+        if not self.entered:
+            raise RuntimeError('You need to use progress bars in a with block.')
+        self.render_progress()
+        return self
+
+    def render_finish(self):
+        if self.is_hidden:
+            return
+        self.file.write(AFTER_BAR)
+        self.file.flush()
+
+    @property
+    def pct(self):
+        if self.finished:
+            return 1.0
+        return min(self.pos / (float(self.length) or 1), 1.0)
+
+    @property
+    def time_per_iteration(self):
+        if not self.avg:
+            return 0.0
+        return sum(self.avg) / float(len(self.avg))
+
+    @property
+    def eta(self):
+        if self.length_known and not self.finished:
+            return self.time_per_iteration * (self.length - self.pos)
+        return 0.0
+
+    def format_eta(self):
+        if self.eta_known:
+            t = self.eta + 1
+            seconds = t % 60
+            t /= 60
+            minutes = t % 60
+            t /= 60
+            hours = t % 24
+            t /= 24
+            if t > 0:
+                days = t
+                return '%dd %02d:%02d:%02d' % (days, hours, minutes, seconds)
+            else:
+                return '%02d:%02d:%02d' % (hours, minutes, seconds)
+        return ''
+
+    def format_pos(self):
+        pos = str(self.pos)
+        if self.length_known:
+            pos += '/%s' % self.length
+        return pos
+
+    def format_pct(self):
+        return ('% 4d%%' % int(self.pct * 100))[1:]
+
+    def format_progress_line(self):
+        show_percent = self.show_percent
+
+        info_bits = []
+        if self.length_known:
+            bar_length = int(self.pct * self.width)
+            bar = self.fill_char * bar_length
+            bar += self.empty_char * (self.width - bar_length)
+            if show_percent is None:
+                show_percent = not self.show_pos
+        else:
+            if self.finished:
+                bar = self.fill_char * self.width
+            else:
+                bar = list(self.empty_char * (self.width or 1))
+                if self.time_per_iteration != 0:
+                    bar[int((math.cos(self.pos * self.time_per_iteration)
+                        / 2.0 + 0.5) * self.width)] = self.fill_char
+                bar = ''.join(bar)
+
+        if self.show_pos:
+            info_bits.append(self.format_pos())
+        if show_percent:
+            info_bits.append(self.format_pct())
+        if self.show_eta and self.eta_known and not self.finished:
+            info_bits.append(self.format_eta())
+        if self.item_show_func is not None:
+            item_info = self.item_show_func(self.current_item)
+            if item_info is not None:
+                info_bits.append(item_info)
+
+        return (self.bar_template % {
+            'label': self.label,
+            'bar': bar,
+            'info': self.info_sep.join(info_bits)
+        }).rstrip()
+
+    def render_progress(self):
+        from .termui import get_terminal_size
+        nl = False
+
+        if self.is_hidden:
+            buf = [self.label]
+            nl = True
+        else:
+            buf = []
+            # Update width in case the terminal has been resized
+            if self.autowidth:
+                old_width = self.width
+                self.width = 0
+                clutter_length = term_len(self.format_progress_line())
+                new_width = max(0, get_terminal_size()[0] - clutter_length)
+                if new_width < old_width:
+                    buf.append(BEFORE_BAR)
+                    buf.append(' ' * self.max_width)
+                    self.max_width = new_width
+                self.width = new_width
+
+            clear_width = self.width
+            if self.max_width is not None:
+                clear_width = self.max_width
+
+            buf.append(BEFORE_BAR)
+            line = self.format_progress_line()
+            line_len = term_len(line)
+            if self.max_width is None or self.max_width < line_len:
+                self.max_width = line_len
+            buf.append(line)
+
+            buf.append(' ' * (clear_width - line_len))
+        line = ''.join(buf)
+
+        # Render the line only if it changed.
+        if line != self._last_line:
+            self._last_line = line
+            echo(line, file=self.file, color=self.color, nl=nl)
+            self.file.flush()
+
+    def make_step(self, n_steps):
+        self.pos += n_steps
+        if self.length_known and self.pos >= self.length:
+            self.finished = True
+
+        if (time.time() - self.last_eta) < 1.0:
+            return
+
+        self.last_eta = time.time()
+        self.avg = self.avg[-6:] + [-(self.start - time.time()) / (self.pos)]
+
+        self.eta_known = self.length_known
+
+    def update(self, n_steps):
+        self.make_step(n_steps)
+        self.render_progress()
+
+    def finish(self):
+        self.eta_known = 0
+        self.current_item = None
+        self.finished = True
+
+    def next(self):
+        if self.is_hidden:
+            return next(self.iter)
+        try:
+            rv = next(self.iter)
+            self.current_item = rv
+        except StopIteration:
+            self.finish()
+            self.render_progress()
+            raise StopIteration()
+        else:
+            self.update(1)
+            return rv
+
+    if not PY2:
+        __next__ = next
+        del next
+
+
+def pager(text, color=None):
+    """Decide what method to use for paging through text."""
+    stdout = _default_text_stdout()
+    if not isatty(sys.stdin) or not isatty(stdout):
+        return _nullpager(stdout, text, color)
+    pager_cmd = (os.environ.get('PAGER', None) or '').strip()
+    if pager_cmd:
+        if WIN:
+            return _tempfilepager(text, pager_cmd, color)
+        return _pipepager(text, pager_cmd, color)
+    if os.environ.get('TERM') in ('dumb', 'emacs'):
+        return _nullpager(stdout, text, color)
+    if WIN or sys.platform.startswith('os2'):
+        return _tempfilepager(text, 'more <', color)
+    if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
+        return _pipepager(text, 'less', color)
+
+    import tempfile
+    fd, filename = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
+            return _pipepager(text, 'more', color)
+        return _nullpager(stdout, text, color)
+    finally:
+        os.unlink(filename)
+
+
+def _pipepager(text, cmd, color):
+    """Page through text by feeding it to another program.  Invoking a
+    pager through this might support colors.
+    """
+    import subprocess
+    env = dict(os.environ)
+
+    # If we're piping to less we might support colors under the
+    # condition that
+    cmd_detail = cmd.rsplit('/', 1)[-1].split()
+    if color is None and cmd_detail[0] == 'less':
+        less_flags = os.environ.get('LESS', '') + ' '.join(cmd_detail[1:])
+        if not less_flags:
+            env['LESS'] = '-R'
+            color = True
+        elif 'r' in less_flags or 'R' in less_flags:
+            color = True
+
+    if not color:
+        text = strip_ansi(text)
+
+    c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+                         env=env)
+    encoding = get_best_encoding(c.stdin)
+    try:
+        c.stdin.write(text.encode(encoding, 'replace'))
+        c.stdin.close()
+    except (IOError, KeyboardInterrupt):
+        pass
+
+    # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
+    # search or other commands inside less).
+    #
+    # That means when the user hits ^C, the parent process (click) terminates,
+    # but less is still alive, paging the output and messing up the terminal.
+    #
+    # If the user wants to make the pager exit on ^C, they should set
+    # `LESS='-K'`. It's not our decision to make.
+    while True:
+        try:
+            c.wait()
+        except KeyboardInterrupt:
+            pass
+        else:
+            break
+
+
+def _tempfilepager(text, cmd, color):
+    """Page through text by invoking a program on a temporary file."""
+    import tempfile
+    filename = tempfile.mktemp()
+    if not color:
+        text = strip_ansi(text)
+    encoding = get_best_encoding(sys.stdout)
+    with open_stream(filename, 'wb')[0] as f:
+        f.write(text.encode(encoding))
+    try:
+        os.system(cmd + ' "' + filename + '"')
+    finally:
+        os.unlink(filename)
+
+
+def _nullpager(stream, text, color):
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    if not color:
+        text = strip_ansi(text)
+    stream.write(text)
+
+
+class Editor(object):
+
+    def __init__(self, editor=None, env=None, require_save=True,
+                 extension='.txt'):
+        self.editor = editor
+        self.env = env
+        self.require_save = require_save
+        self.extension = extension
+
+    def get_editor(self):
+        if self.editor is not None:
+            return self.editor
+        for key in 'VISUAL', 'EDITOR':
+            rv = os.environ.get(key)
+            if rv:
+                return rv
+        if WIN:
+            return 'notepad'
+        for editor in 'vim', 'nano':
+            if os.system('which %s >/dev/null 2>&1' % editor) == 0:
+                return editor
+        return 'vi'
+
+    def edit_file(self, filename):
+        import subprocess
+        editor = self.get_editor()
+        if self.env:
+            environ = os.environ.copy()
+            environ.update(self.env)
+        else:
+            environ = None
+        try:
+            c = subprocess.Popen('%s "%s"' % (editor, filename),
+                                 env=environ, shell=True)
+            exit_code = c.wait()
+            if exit_code != 0:
+                raise ClickException('%s: Editing failed!' % editor)
+        except OSError as e:
+            raise ClickException('%s: Editing failed: %s' % (editor, e))
+
+    def edit(self, text):
+        import tempfile
+
+        text = text or ''
+        if text and not text.endswith('\n'):
+            text += '\n'
+
+        fd, name = tempfile.mkstemp(prefix='editor-', suffix=self.extension)
+        try:
+            if WIN:
+                encoding = 'utf-8-sig'
+                text = text.replace('\n', '\r\n')
+            else:
+                encoding = 'utf-8'
+            text = text.encode(encoding)
+
+            f = os.fdopen(fd, 'wb')
+            f.write(text)
+            f.close()
+            timestamp = os.path.getmtime(name)
+
+            self.edit_file(name)
+
+            if self.require_save \
+               and os.path.getmtime(name) == timestamp:
+                return None
+
+            f = open(name, 'rb')
+            try:
+                rv = f.read()
+            finally:
+                f.close()
+            return rv.decode('utf-8-sig').replace('\r\n', '\n')
+        finally:
+            os.unlink(name)
+
+
+def open_url(url, wait=False, locate=False):
+    import subprocess
+
+    def _unquote_file(url):
+        try:
+            import urllib
+        except ImportError:
+            import urllib
+        if url.startswith('file://'):
+            url = urllib.unquote(url[7:])
+        return url
+
+    if sys.platform == 'darwin':
+        args = ['open']
+        if wait:
+            args.append('-W')
+        if locate:
+            args.append('-R')
+        args.append(_unquote_file(url))
+        null = open('/dev/null', 'w')
+        try:
+            return subprocess.Popen(args, stderr=null).wait()
+        finally:
+            null.close()
+    elif WIN:
+        if locate:
+            url = _unquote_file(url)
+            args = 'explorer /select,"%s"' % _unquote_file(
+                url.replace('"', ''))
+        else:
+            args = 'start %s "" "%s"' % (
+                wait and '/WAIT' or '', url.replace('"', ''))
+        return os.system(args)
+
+    try:
+        if locate:
+            url = os.path.dirname(_unquote_file(url)) or '.'
+        else:
+            url = _unquote_file(url)
+        c = subprocess.Popen(['xdg-open', url])
+        if wait:
+            return c.wait()
+        return 0
+    except OSError:
+        if url.startswith(('http://', 'https://')) and not locate and not wait:
+            import webbrowser
+            webbrowser.open(url)
+            return 0
+        return 1
+
+
+def _translate_ch_to_exc(ch):
+    if ch == '\x03':
+        raise KeyboardInterrupt()
+    if ch == '\x04':
+        raise EOFError()
+
+
+if WIN:
+    import msvcrt
+
+    def getchar(echo):
+        rv = msvcrt.getch()
+        if echo:
+            msvcrt.putchar(rv)
+        _translate_ch_to_exc(rv)
+        if PY2:
+            enc = getattr(sys.stdin, 'encoding', None)
+            if enc is not None:
+                rv = rv.decode(enc, 'replace')
+            else:
+                rv = rv.decode('cp1252', 'replace')
+        return rv
+else:
+    import tty
+    import termios
+
+    def getchar(echo):
+        if not isatty(sys.stdin):
+            f = open('/dev/tty')
+            fd = f.fileno()
+        else:
+            fd = sys.stdin.fileno()
+            f = None
+        try:
+            old_settings = termios.tcgetattr(fd)
+            try:
+                tty.setraw(fd)
+                ch = os.read(fd, 32)
+                if echo and isatty(sys.stdout):
+                    sys.stdout.write(ch)
+            finally:
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                sys.stdout.flush()
+                if f is not None:
+                    f.close()
+        except termios.error:
+            pass
+        _translate_ch_to_exc(ch)
+        return ch.decode(get_best_encoding(sys.stdin), 'replace')

Contents/Libraries/Shared/click/_textwrap.py

@@ -0,0 +1,38 @@
+import textwrap
+from contextlib import contextmanager
+
+
+class TextWrapper(textwrap.TextWrapper):
+
+    def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
+        space_left = max(width - cur_len, 1)
+
+        if self.break_long_words:
+            last = reversed_chunks[-1]
+            cut = last[:space_left]
+            res = last[space_left:]
+            cur_line.append(cut)
+            reversed_chunks[-1] = res
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+    @contextmanager
+    def extra_indent(self, indent):
+        old_initial_indent = self.initial_indent
+        old_subsequent_indent = self.subsequent_indent
+        self.initial_indent += indent
+        self.subsequent_indent += indent
+        try:
+            yield
+        finally:
+            self.initial_indent = old_initial_indent
+            self.subsequent_indent = old_subsequent_indent
+
+    def indent_only(self, text):
+        rv = []
+        for idx, line in enumerate(text.splitlines()):
+            indent = self.initial_indent
+            if idx > 0:
+                indent = self.subsequent_indent
+            rv.append(indent + line)
+        return '\n'.join(rv)

Contents/Libraries/Shared/click/_unicodefun.py

@@ -0,0 +1,118 @@
+import os
+import sys
+import codecs
+
+from ._compat import PY2
+
+
+# If someone wants to vendor click, we want to ensure the
+# correct package is discovered.  Ideally we could use a
+# relative import here but unfortunately Python does not
+# support that.
+click = sys.modules[__name__.rsplit('.', 1)[0]]
+
+
+def _find_unicode_literals_frame():
+    import __future__
+    frm = sys._getframe(1)
+    idx = 1
+    while frm is not None:
+        if frm.f_globals.get('__name__', '').startswith('click.'):
+            frm = frm.f_back
+            idx += 1
+        elif frm.f_code.co_flags & __future__.unicode_literals.compiler_flag:
+            return idx
+        else:
+            break
+    return 0
+
+
+def _check_for_unicode_literals():
+    if not __debug__:
+        return
+    if not PY2 or click.disable_unicode_literals_warning:
+        return
+    bad_frame = _find_unicode_literals_frame()
+    if bad_frame <= 0:
+        return
+    from warnings import warn
+    warn(Warning('Click detected the use of the unicode_literals '
+                 '__future__ import.  This is heavily discouraged '
+                 'because it can introduce subtle bugs in your '
+                 'code.  You should instead use explicit u"" literals '
+                 'for your unicode strings.  For more information see '
+                 'http://click.pocoo.org/python3/'),
+         stacklevel=bad_frame)
+
+
+def _verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    if PY2:
+        return
+    try:
+        import locale
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = 'ascii'
+    if fs_enc != 'ascii':
+        return
+
+    extra = ''
+    if os.name == 'posix':
+        import subprocess
+        rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE).communicate()[0]
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode('ascii', 'replace')
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith(('.utf-8', '.utf8')):
+                good_locales.add(locale)
+                if locale.lower() in ('c.utf8', 'c.utf-8'):
+                    has_c_utf8 = True
+
+        extra += '\n\n'
+        if not good_locales:
+            extra += (
+                'Additional information: on this system no suitable UTF-8\n'
+                'locales were discovered.  This most likely requires resolving\n'
+                'by reconfiguring the locale system.'
+            )
+        elif has_c_utf8:
+            extra += (
+                'This system supports the C.UTF-8 locale which is recommended.\n'
+                'You might be able to resolve your issue by exporting the\n'
+                'following environment variables:\n\n'
+                '    export LC_ALL=C.UTF-8\n'
+                '    export LANG=C.UTF-8'
+            )
+        else:
+            extra += (
+                'This system lists a couple of UTF-8 supporting locales that\n'
+                'you can pick from.  The following suitable locales where\n'
+                'discovered: %s'
+            ) % ', '.join(sorted(good_locales))
+
+        bad_locale = None
+        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
+            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                '\n\nClick discovered that you exported a UTF-8 locale\n'
+                'but the locale system could not pick up from it because\n'
+                'it does not exist.  The exported locale is "%s" but it\n'
+                'is not supported'
+            ) % bad_locale
+
+    raise RuntimeError('Click will abort further execution because Python 3 '
+                       'was configured to use ASCII as encoding for the '
+                       'environment.  Consult http://click.pocoo.org/python3/'
+                       'for mitigation steps.' + extra)

Contents/Libraries/Shared/click/_winconsole.py

@@ -0,0 +1,273 @@
+# -*- coding: utf-8 -*-
+# This module is based on the excellent work by Adam Bartoš who
+# provided a lot of what went into the implementation here in
+# the discussion to issue1602 in the Python bug tracker.
+#
+# There are some general differences in regards to how this works
+# compared to the original patches as we do not need to patch
+# the entire interpreter but just work in our little world of
+# echo and prmopt.
+
+import io
+import os
+import sys
+import zlib
+import time
+import ctypes
+import msvcrt
+from click._compat import _NonClosingTextIOWrapper, text_type, PY2
+from ctypes import byref, POINTER, c_int, c_char, c_char_p, \
+     c_void_p, py_object, c_ssize_t, c_ulong, windll, WINFUNCTYPE
+try:
+    from ctypes import pythonapi
+    PyObject_GetBuffer = pythonapi.PyObject_GetBuffer
+    PyBuffer_Release = pythonapi.PyBuffer_Release
+except ImportError:
+    pythonapi = None
+from ctypes.wintypes import LPWSTR, LPCWSTR
+
+
+c_ssize_p = POINTER(c_ssize_t)
+
+kernel32 = windll.kernel32
+GetStdHandle = kernel32.GetStdHandle
+ReadConsoleW = kernel32.ReadConsoleW
+WriteConsoleW = kernel32.WriteConsoleW
+GetLastError = kernel32.GetLastError
+GetCommandLineW = WINFUNCTYPE(LPWSTR)(
+    ('GetCommandLineW', windll.kernel32))
+CommandLineToArgvW = WINFUNCTYPE(
+    POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
+        ('CommandLineToArgvW', windll.shell32))
+
+
+STDIN_HANDLE = GetStdHandle(-10)
+STDOUT_HANDLE = GetStdHandle(-11)
+STDERR_HANDLE = GetStdHandle(-12)
+
+
+PyBUF_SIMPLE = 0
+PyBUF_WRITABLE = 1
+
+ERROR_SUCCESS = 0
+ERROR_NOT_ENOUGH_MEMORY = 8
+ERROR_OPERATION_ABORTED = 995
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+EOF = b'\x1a'
+MAX_BYTES_WRITTEN = 32767
+
+
+class Py_buffer(ctypes.Structure):
+    _fields_ = [
+        ('buf', c_void_p),
+        ('obj', py_object),
+        ('len', c_ssize_t),
+        ('itemsize', c_ssize_t),
+        ('readonly', c_int),
+        ('ndim', c_int),
+        ('format', c_char_p),
+        ('shape', c_ssize_p),
+        ('strides', c_ssize_p),
+        ('suboffsets', c_ssize_p),
+        ('internal', c_void_p)
+    ]
+
+    if PY2:
+        _fields_.insert(-1, ('smalltable', c_ssize_t * 2))
+
+
+# On PyPy we cannot get buffers so our ability to operate here is
+# serverly limited.
+if pythonapi is None:
+    get_buffer = None
+else:
+    def get_buffer(obj, writable=False):
+        buf = Py_buffer()
+        flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE
+        PyObject_GetBuffer(py_object(obj), byref(buf), flags)
+        try:
+            buffer_type = c_char * buf.len
+            return buffer_type.from_address(buf.buf)
+        finally:
+            PyBuffer_Release(byref(buf))
+
+
+class _WindowsConsoleRawIOBase(io.RawIOBase):
+
+    def __init__(self, handle):
+        self.handle = handle
+
+    def isatty(self):
+        io.RawIOBase.isatty(self)
+        return True
+
+
+class _WindowsConsoleReader(_WindowsConsoleRawIOBase):
+
+    def readable(self):
+        return True
+
+    def readinto(self, b):
+        bytes_to_be_read = len(b)
+        if not bytes_to_be_read:
+            return 0
+        elif bytes_to_be_read % 2:
+            raise ValueError('cannot read odd number of bytes from '
+                             'UTF-16-LE encoded console')
+
+        buffer = get_buffer(b, writable=True)
+        code_units_to_be_read = bytes_to_be_read // 2
+        code_units_read = c_ulong()
+
+        rv = ReadConsoleW(self.handle, buffer, code_units_to_be_read,
+                          byref(code_units_read), None)
+        if GetLastError() == ERROR_OPERATION_ABORTED:
+            # wait for KeyboardInterrupt
+            time.sleep(0.1)
+        if not rv:
+            raise OSError('Windows error: %s' % GetLastError())
+
+        if buffer[0] == EOF:
+            return 0
+        return 2 * code_units_read.value
+
+
+class _WindowsConsoleWriter(_WindowsConsoleRawIOBase):
+
+    def writable(self):
+        return True
+
+    @staticmethod
+    def _get_error_message(errno):
+        if errno == ERROR_SUCCESS:
+            return 'ERROR_SUCCESS'
+        elif errno == ERROR_NOT_ENOUGH_MEMORY:
+            return 'ERROR_NOT_ENOUGH_MEMORY'
+        return 'Windows error %s' % errno
+
+    def write(self, b):
+        bytes_to_be_written = len(b)
+        buf = get_buffer(b)
+        code_units_to_be_written = min(bytes_to_be_written,
+                                       MAX_BYTES_WRITTEN) // 2
+        code_units_written = c_ulong()
+
+        WriteConsoleW(self.handle, buf, code_units_to_be_written,
+                      byref(code_units_written), None)
+        bytes_written = 2 * code_units_written.value
+
+        if bytes_written == 0 and bytes_to_be_written > 0:
+            raise OSError(self._get_error_message(GetLastError()))
+        return bytes_written
+
+
+class ConsoleStream(object):
+
+    def __init__(self, text_stream, byte_stream):
+        self._text_stream = text_stream
+        self.buffer = byte_stream
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def write(self, x):
+        if isinstance(x, text_type):
+            return self._text_stream.write(x)
+        try:
+            self.flush()
+        except Exception:
+            pass
+        return self.buffer.write(x)
+
+    def writelines(self, lines):
+        for line in lines:
+            self.write(line)
+
+    def __getattr__(self, name):
+        return getattr(self._text_stream, name)
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def __repr__(self):
+        return '<ConsoleStream name=%r encoding=%r>' % (
+            self.name,
+            self.encoding,
+        )
+
+
+def _get_text_stdin(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stdout(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        _WindowsConsoleWriter(STDOUT_HANDLE),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+def _get_text_stderr(buffer_stream):
+    text_stream = _NonClosingTextIOWrapper(
+        _WindowsConsoleWriter(STDERR_HANDLE),
+        'utf-16-le', 'strict', line_buffering=True)
+    return ConsoleStream(text_stream, buffer_stream)
+
+
+if PY2:
+    def _hash_py_argv():
+        return zlib.crc32('\x00'.join(sys.argv[1:]))
+
+    _initial_argv_hash = _hash_py_argv()
+
+    def _get_windows_argv():
+        argc = c_int(0)
+        argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+        argv = [argv_unicode[i] for i in range(0, argc.value)]
+
+        if not hasattr(sys, 'frozen'):
+            argv = argv[1:]
+            while len(argv) > 0:
+                arg = argv[0]
+                if not arg.startswith('-') or arg == '-':
+                    break
+                argv = argv[1:]
+                if arg.startswith(('-c', '-m')):
+                    break
+
+        return argv[1:]
+
+
+_stream_factories = {
+    0: _get_text_stdin,
+    1: _get_text_stdout,
+    2: _get_text_stderr,
+}
+
+
+def _get_windows_console_stream(f, encoding, errors):
+    if get_buffer is not None and \
+       encoding in ('utf-16-le', None) \
+       and errors in ('strict', None) and \
+       hasattr(f, 'isatty') and f.isatty():
+        func = _stream_factories.get(f.fileno())
+        if func is not None:
+            if not PY2:
+                f = getattr(f, 'buffer')
+                if f is None:
+                    return None
+            else:
+                # If we are on Python 2 we need to set the stream that we
+                # deal with to binary mode as otherwise the exercise if a
+                # bit moot.  The same problems apply as for
+                # get_binary_stdin and friends from _compat.
+                msvcrt.setmode(f.fileno(), os.O_BINARY)
+            return func(f)

Contents/Libraries/Shared/click/decorators.py

@@ -0,0 +1,304 @@
+import sys
+import inspect
+
+from functools import update_wrapper
+
+from ._compat import iteritems
+from ._unicodefun import _check_for_unicode_literals
+from .utils import echo
+from .globals import get_current_context
+
+
+def pass_context(f):
+    """Marks a callback as wanting to receive the current context
+    object as first argument.
+    """
+    def new_func(*args, **kwargs):
+        return f(get_current_context(), *args, **kwargs)
+    return update_wrapper(new_func, f)
+
+
+def pass_obj(f):
+    """Similar to :func:`pass_context`, but only pass the object on the
+    context onwards (:attr:`Context.obj`).  This is useful if that object
+    represents the state of a nested system.
+    """
+    def new_func(*args, **kwargs):
+        return f(get_current_context().obj, *args, **kwargs)
+    return update_wrapper(new_func, f)
+
+
+def make_pass_decorator(object_type, ensure=False):
+    """Given an object type this creates a decorator that will work
+    similar to :func:`pass_obj` but instead of passing the object of the
+    current context, it will find the innermost context of type
+    :func:`object_type`.
+
+    This generates a decorator that works roughly like this::
+
+        from functools import update_wrapper
+
+        def decorator(f):
+            @pass_context
+            def new_func(ctx, *args, **kwargs):
+                obj = ctx.find_object(object_type)
+                return ctx.invoke(f, obj, *args, **kwargs)
+            return update_wrapper(new_func, f)
+        return decorator
+
+    :param object_type: the type of the object to pass.
+    :param ensure: if set to `True`, a new object will be created and
+                   remembered on the context if it's not there yet.
+    """
+    def decorator(f):
+        def new_func(*args, **kwargs):
+            ctx = get_current_context()
+            if ensure:
+                obj = ctx.ensure_object(object_type)
+            else:
+                obj = ctx.find_object(object_type)
+            if obj is None:
+                raise RuntimeError('Managed to invoke callback without a '
+                                   'context object of type %r existing'
+                                   % object_type.__name__)
+            return ctx.invoke(f, obj, *args[1:], **kwargs)
+        return update_wrapper(new_func, f)
+    return decorator
+
+
+def _make_command(f, name, attrs, cls):
+    if isinstance(f, Command):
+        raise TypeError('Attempted to convert a callback into a '
+                        'command twice.')
+    try:
+        params = f.__click_params__
+        params.reverse()
+        del f.__click_params__
+    except AttributeError:
+        params = []
+    help = attrs.get('help')
+    if help is None:
+        help = inspect.getdoc(f)
+        if isinstance(help, bytes):
+            help = help.decode('utf-8')
+    else:
+        help = inspect.cleandoc(help)
+    attrs['help'] = help
+    _check_for_unicode_literals()
+    return cls(name=name or f.__name__.lower(),
+               callback=f, params=params, **attrs)
+
+
+def command(name=None, cls=None, **attrs):
+    """Creates a new :class:`Command` and uses the decorated function as
+    callback.  This will also automatically attach all decorated
+    :func:`option`\s and :func:`argument`\s as parameters to the command.
+
+    The name of the command defaults to the name of the function.  If you
+    want to change that, you can pass the intended name as the first
+    argument.
+
+    All keyword arguments are forwarded to the underlying command class.
+
+    Once decorated the function turns into a :class:`Command` instance
+    that can be invoked as a command line utility or be attached to a
+    command :class:`Group`.
+
+    :param name: the name of the command.  This defaults to the function
+                 name.
+    :param cls: the command class to instantiate.  This defaults to
+                :class:`Command`.
+    """
+    if cls is None:
+        cls = Command
+    def decorator(f):
+        cmd = _make_command(f, name, attrs, cls)
+        cmd.__doc__ = f.__doc__
+        return cmd
+    return decorator
+
+
+def group(name=None, **attrs):
+    """Creates a new :class:`Group` with a function as callback.  This
+    works otherwise the same as :func:`command` just that the `cls`
+    parameter is set to :class:`Group`.
+    """
+    attrs.setdefault('cls', Group)
+    return command(name, **attrs)
+
+
+def _param_memo(f, param):
+    if isinstance(f, Command):
+        f.params.append(param)
+    else:
+        if not hasattr(f, '__click_params__'):
+            f.__click_params__ = []
+        f.__click_params__.append(param)
+
+
+def argument(*param_decls, **attrs):
+    """Attaches an argument to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Argument`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Argument` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the argument class to instantiate.  This defaults to
+                :class:`Argument`.
+    """
+    def decorator(f):
+        ArgumentClass = attrs.pop('cls', Argument)
+        _param_memo(f, ArgumentClass(param_decls, **attrs))
+        return f
+    return decorator
+
+
+def option(*param_decls, **attrs):
+    """Attaches an option to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Option`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Option` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    :param cls: the option class to instantiate.  This defaults to
+                :class:`Option`.
+    """
+    def decorator(f):
+        if 'help' in attrs:
+            attrs['help'] = inspect.cleandoc(attrs['help'])
+        OptionClass = attrs.pop('cls', Option)
+        _param_memo(f, OptionClass(param_decls, **attrs))
+        return f
+    return decorator
+
+
+def confirmation_option(*param_decls, **attrs):
+    """Shortcut for confirmation prompts that can be ignored by passing
+    ``--yes`` as parameter.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+
+        @click.command()
+        @click.option('--yes', is_flag=True, callback=callback,
+                      expose_value=False, prompt='Do you want to continue?')
+        def dropdb():
+            pass
+    """
+    def decorator(f):
+        def callback(ctx, param, value):
+            if not value:
+                ctx.abort()
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('callback', callback)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('prompt', 'Do you want to continue?')
+        attrs.setdefault('help', 'Confirm the action without prompting.')
+        return option(*(param_decls or ('--yes',)), **attrs)(f)
+    return decorator
+
+
+def password_option(*param_decls, **attrs):
+    """Shortcut for password prompts.
+
+    This is equivalent to decorating a function with :func:`option` with
+    the following parameters::
+
+        @click.command()
+        @click.option('--password', prompt=True, confirmation_prompt=True,
+                      hide_input=True)
+        def changeadmin(password):
+            pass
+    """
+    def decorator(f):
+        attrs.setdefault('prompt', True)
+        attrs.setdefault('confirmation_prompt', True)
+        attrs.setdefault('hide_input', True)
+        return option(*(param_decls or ('--password',)), **attrs)(f)
+    return decorator
+
+
+def version_option(version=None, *param_decls, **attrs):
+    """Adds a ``--version`` option which immediately ends the program
+    printing out the version number.  This is implemented as an eager
+    option that prints the version and exits the program in the callback.
+
+    :param version: the version number to show.  If not provided Click
+                    attempts an auto discovery via setuptools.
+    :param prog_name: the name of the program (defaults to autodetection)
+    :param message: custom message to show instead of the default
+                    (``'%(prog)s, version %(version)s'``)
+    :param others: everything else is forwarded to :func:`option`.
+    """
+    if version is None:
+        module = sys._getframe(1).f_globals.get('__name__')
+    def decorator(f):
+        prog_name = attrs.pop('prog_name', None)
+        message = attrs.pop('message', '%(prog)s, version %(version)s')
+
+        def callback(ctx, param, value):
+            if not value or ctx.resilient_parsing:
+                return
+            prog = prog_name
+            if prog is None:
+                prog = ctx.find_root().info_name
+            ver = version
+            if ver is None:
+                try:
+                    import pkg_resources
+                except ImportError:
+                    pass
+                else:
+                    for dist in pkg_resources.working_set:
+                        scripts = dist.get_entry_map().get('console_scripts') or {}
+                        for script_name, entry_point in iteritems(scripts):
+                            if entry_point.module_name == module:
+                                ver = dist.version
+                                break
+                if ver is None:
+                    raise RuntimeError('Could not determine version')
+            echo(message % {
+                'prog': prog,
+                'version': ver,
+            }, color=ctx.color)
+            ctx.exit()
+
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('is_eager', True)
+        attrs.setdefault('help', 'Show the version and exit.')
+        attrs['callback'] = callback
+        return option(*(param_decls or ('--version',)), **attrs)(f)
+    return decorator
+
+
+def help_option(*param_decls, **attrs):
+    """Adds a ``--help`` option which immediately ends the program
+    printing out the help page.  This is usually unnecessary to add as
+    this is added by default to all commands unless suppressed.
+
+    Like :func:`version_option`, this is implemented as eager option that
+    prints in the callback and exits.
+
+    All arguments are forwarded to :func:`option`.
+    """
+    def decorator(f):
+        def callback(ctx, param, value):
+            if value and not ctx.resilient_parsing:
+                echo(ctx.get_help(), color=ctx.color)
+                ctx.exit()
+        attrs.setdefault('is_flag', True)
+        attrs.setdefault('expose_value', False)
+        attrs.setdefault('help', 'Show this message and exit.')
+        attrs.setdefault('is_eager', True)
+        attrs['callback'] = callback
+        return option(*(param_decls or ('--help',)), **attrs)(f)
+    return decorator
+
+
+# Circular dependencies between core and decorators
+from .core import Command, Group, Argument, Option

Contents/Libraries/Shared/click/exceptions.py

@@ -0,0 +1,201 @@
+from ._compat import PY2, filename_to_ui, get_text_stderr
+from .utils import echo
+
+
+class ClickException(Exception):
+    """An exception that Click can handle and show to the user."""
+
+    #: The exit code for this exception
+    exit_code = 1
+
+    def __init__(self, message):
+        if PY2:
+            if message is not None:
+                message = message.encode('utf-8')
+        Exception.__init__(self, message)
+        self.message = message
+
+    def format_message(self):
+        return self.message
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        echo('Error: %s' % self.format_message(), file=file)
+
+
+class UsageError(ClickException):
+    """An internal exception that signals a usage error.  This typically
+    aborts any further handling.
+
+    :param message: the error message to display.
+    :param ctx: optionally the context that caused this error.  Click will
+                fill in the context automatically in some situations.
+    """
+    exit_code = 2
+
+    def __init__(self, message, ctx=None):
+        ClickException.__init__(self, message)
+        self.ctx = ctx
+
+    def show(self, file=None):
+        if file is None:
+            file = get_text_stderr()
+        color = None
+        if self.ctx is not None:
+            color = self.ctx.color
+            echo(self.ctx.get_usage() + '\n', file=file, color=color)
+        echo('Error: %s' % self.format_message(), file=file, color=color)
+
+
+class BadParameter(UsageError):
+    """An exception that formats out a standardized error message for a
+    bad parameter.  This is useful when thrown from a callback or type as
+    Click will attach contextual information to it (for instance, which
+    parameter it is).
+
+    .. versionadded:: 2.0
+
+    :param param: the parameter object that caused this error.  This can
+                  be left out, and Click will attach this info itself
+                  if possible.
+    :param param_hint: a string that shows up as parameter name.  This
+                       can be used as alternative to `param` in cases
+                       where custom validation should happen.  If it is
+                       a string it's used as such, if it's a list then
+                       each item is quoted and separated.
+    """
+
+    def __init__(self, message, ctx=None, param=None,
+                 param_hint=None):
+        UsageError.__init__(self, message, ctx)
+        self.param = param
+        self.param_hint = param_hint
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.opts or [self.param.human_readable_name]
+        else:
+            return 'Invalid value: %s' % self.message
+        if isinstance(param_hint, (tuple, list)):
+            param_hint = ' / '.join('"%s"' % x for x in param_hint)
+        return 'Invalid value for %s: %s' % (param_hint, self.message)
+
+
+class MissingParameter(BadParameter):
+    """Raised if click required an option or argument but it was not
+    provided when invoking the script.
+
+    .. versionadded:: 4.0
+
+    :param param_type: a string that indicates the type of the parameter.
+                       The default is to inherit the parameter type from
+                       the given `param`.  Valid values are ``'parameter'``,
+                       ``'option'`` or ``'argument'``.
+    """
+
+    def __init__(self, message=None, ctx=None, param=None,
+                 param_hint=None, param_type=None):
+        BadParameter.__init__(self, message, ctx, param, param_hint)
+        self.param_type = param_type
+
+    def format_message(self):
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.opts or [self.param.human_readable_name]
+        else:
+            param_hint = None
+        if isinstance(param_hint, (tuple, list)):
+            param_hint = ' / '.join('"%s"' % x for x in param_hint)
+
+        param_type = self.param_type
+        if param_type is None and self.param is not None:
+            param_type = self.param.param_type_name
+
+        msg = self.message
+        if self.param is not None:
+            msg_extra = self.param.type.get_missing_message(self.param)
+            if msg_extra:
+                if msg:
+                    msg += '.  ' + msg_extra
+                else:
+                    msg = msg_extra
+
+        return 'Missing %s%s%s%s' % (
+            param_type,
+            param_hint and ' %s' % param_hint or '',
+            msg and '.  ' or '.',
+            msg or '',
+        )
+
+
+class NoSuchOption(UsageError):
+    """Raised if click attempted to handle an option that does not
+    exist.
+
+    .. versionadded:: 4.0
+    """
+
+    def __init__(self, option_name, message=None, possibilities=None,
+                 ctx=None):
+        if message is None:
+            message = 'no such option: %s' % option_name
+        UsageError.__init__(self, message, ctx)
+        self.option_name = option_name
+        self.possibilities = possibilities
+
+    def format_message(self):
+        bits = [self.message]
+        if self.possibilities:
+            if len(self.possibilities) == 1:
+                bits.append('Did you mean %s?' % self.possibilities[0])
+            else:
+                possibilities = sorted(self.possibilities)
+                bits.append('(Possible options: %s)' % ', '.join(possibilities))
+        return '  '.join(bits)
+
+
+class BadOptionUsage(UsageError):
+    """Raised if an option is generally supplied but the use of the option
+    was incorrect.  This is for instance raised if the number of arguments
+    for an option is not correct.
+
+    .. versionadded:: 4.0
+    """
+
+    def __init__(self, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+
+
+class BadArgumentUsage(UsageError):
+    """Raised if an argument is generally supplied but the use of the argument
+    was incorrect.  This is for instance raised if the number of values
+    for an argument is not correct.
+
+    .. versionadded:: 6.0
+    """
+
+    def __init__(self, message, ctx=None):
+        UsageError.__init__(self, message, ctx)
+
+
+class FileError(ClickException):
+    """Raised if a file cannot be opened."""
+
+    def __init__(self, filename, hint=None):
+        ui_filename = filename_to_ui(filename)
+        if hint is None:
+            hint = 'unknown error'
+        ClickException.__init__(self, hint)
+        self.ui_filename = ui_filename
+        self.filename = filename
+
+    def format_message(self):
+        return 'Could not open file %s: %s' % (self.ui_filename, self.message)
+
+
+class Abort(RuntimeError):
+    """An internal signalling exception that signals Click to abort."""

Contents/Libraries/Shared/click/formatting.py

@@ -0,0 +1,256 @@
+from contextlib import contextmanager
+from .termui import get_terminal_size
+from .parser import split_opt
+from ._compat import term_len
+
+
+# Can force a width.  This is used by the test system
+FORCED_WIDTH = None
+
+
+def measure_table(rows):
+    widths = {}
+    for row in rows:
+        for idx, col in enumerate(row):
+            widths[idx] = max(widths.get(idx, 0), term_len(col))
+    return tuple(y for x, y in sorted(widths.items()))
+
+
+def iter_rows(rows, col_count):
+    for row in rows:
+        row = tuple(row)
+        yield row + ('',) * (col_count - len(row))
+
+
+def wrap_text(text, width=78, initial_indent='', subsequent_indent='',
+              preserve_paragraphs=False):
+    """A helper function that intelligently wraps text.  By default, it
+    assumes that it operates on a single paragraph of text but if the
+    `preserve_paragraphs` parameter is provided it will intelligently
+    handle paragraphs (defined by two empty lines).
+
+    If paragraphs are handled, a paragraph can be prefixed with an empty
+    line containing the ``\\b`` character (``\\x08``) to indicate that
+    no rewrapping should happen in that block.
+
+    :param text: the text that should be rewrapped.
+    :param width: the maximum width for the text.
+    :param initial_indent: the initial indent that should be placed on the
+                           first line as a string.
+    :param subsequent_indent: the indent string that should be placed on
+                              each consecutive line.
+    :param preserve_paragraphs: if this flag is set then the wrapping will
+                                intelligently handle paragraphs.
+    """
+    from ._textwrap import TextWrapper
+    text = text.expandtabs()
+    wrapper = TextWrapper(width, initial_indent=initial_indent,
+                          subsequent_indent=subsequent_indent,
+                          replace_whitespace=False)
+    if not preserve_paragraphs:
+        return wrapper.fill(text)
+
+    p = []
+    buf = []
+    indent = None
+
+    def _flush_par():
+        if not buf:
+            return
+        if buf[0].strip() == '\b':
+            p.append((indent or 0, True, '\n'.join(buf[1:])))
+        else:
+            p.append((indent or 0, False, ' '.join(buf)))
+        del buf[:]
+
+    for line in text.splitlines():
+        if not line:
+            _flush_par()
+            indent = None
+        else:
+            if indent is None:
+                orig_len = term_len(line)
+                line = line.lstrip()
+                indent = orig_len - term_len(line)
+            buf.append(line)
+    _flush_par()
+
+    rv = []
+    for indent, raw, text in p:
+        with wrapper.extra_indent(' ' * indent):
+            if raw:
+                rv.append(wrapper.indent_only(text))
+            else:
+                rv.append(wrapper.fill(text))
+
+    return '\n\n'.join(rv)
+
+
+class HelpFormatter(object):
+    """This class helps with formatting text-based help pages.  It's
+    usually just needed for very special internal cases, but it's also
+    exposed so that developers can write their own fancy outputs.
+
+    At present, it always writes into memory.
+
+    :param indent_increment: the additional increment for each level.
+    :param width: the width for the text.  This defaults to the terminal
+                  width clamped to a maximum of 78.
+    """
+
+    def __init__(self, indent_increment=2, width=None, max_width=None):
+        self.indent_increment = indent_increment
+        if max_width is None:
+            max_width = 80
+        if width is None:
+            width = FORCED_WIDTH
+            if width is None:
+                width = max(min(get_terminal_size()[0], max_width) - 2, 50)
+        self.width = width
+        self.current_indent = 0
+        self.buffer = []
+
+    def write(self, string):
+        """Writes a unicode string into the internal buffer."""
+        self.buffer.append(string)
+
+    def indent(self):
+        """Increases the indentation."""
+        self.current_indent += self.indent_increment
+
+    def dedent(self):
+        """Decreases the indentation."""
+        self.current_indent -= self.indent_increment
+
+    def write_usage(self, prog, args='', prefix='Usage: '):
+        """Writes a usage line into the buffer.
+
+        :param prog: the program name.
+        :param args: whitespace separated list of arguments.
+        :param prefix: the prefix for the first line.
+        """
+        usage_prefix = '%*s%s ' % (self.current_indent, prefix, prog)
+        text_width = self.width - self.current_indent
+
+        if text_width >= (term_len(usage_prefix) + 20):
+            # The arguments will fit to the right of the prefix.
+            indent = ' ' * term_len(usage_prefix)
+            self.write(wrap_text(args, text_width,
+                                 initial_indent=usage_prefix,
+                                 subsequent_indent=indent))
+        else:
+            # The prefix is too long, put the arguments on the next line.
+            self.write(usage_prefix)
+            self.write('\n')
+            indent = ' ' * (max(self.current_indent, term_len(prefix)) + 4)
+            self.write(wrap_text(args, text_width,
+                                 initial_indent=indent,
+                                 subsequent_indent=indent))
+
+        self.write('\n')
+
+    def write_heading(self, heading):
+        """Writes a heading into the buffer."""
+        self.write('%*s%s:\n' % (self.current_indent, '', heading))
+
+    def write_paragraph(self):
+        """Writes a paragraph into the buffer."""
+        if self.buffer:
+            self.write('\n')
+
+    def write_text(self, text):
+        """Writes re-indented text into the buffer.  This rewraps and
+        preserves paragraphs.
+        """
+        text_width = max(self.width - self.current_indent, 11)
+        indent = ' ' * self.current_indent
+        self.write(wrap_text(text, text_width,
+                             initial_indent=indent,
+                             subsequent_indent=indent,
+                             preserve_paragraphs=True))
+        self.write('\n')
+
+    def write_dl(self, rows, col_max=30, col_spacing=2):
+        """Writes a definition list into the buffer.  This is how options
+        and commands are usually formatted.
+
+        :param rows: a list of two item tuples for the terms and values.
+        :param col_max: the maximum width of the first column.
+        :param col_spacing: the number of spaces between the first and
+                            second column.
+        """
+        rows = list(rows)
+        widths = measure_table(rows)
+        if len(widths) != 2:
+            raise TypeError('Expected two columns for definition list')
+
+        first_col = min(widths[0], col_max) + col_spacing
+
+        for first, second in iter_rows(rows, len(widths)):
+            self.write('%*s%s' % (self.current_indent, '', first))
+            if not second:
+                self.write('\n')
+                continue
+            if term_len(first) <= first_col - col_spacing:
+                self.write(' ' * (first_col - term_len(first)))
+            else:
+                self.write('\n')
+                self.write(' ' * (first_col + self.current_indent))
+
+            text_width = max(self.width - first_col - 2, 10)
+            lines = iter(wrap_text(second, text_width).splitlines())
+            if lines:
+                self.write(next(lines) + '\n')
+                for line in lines:
+                    self.write('%*s%s\n' % (
+                        first_col + self.current_indent, '', line))
+            else:
+                self.write('\n')
+
+    @contextmanager
+    def section(self, name):
+        """Helpful context manager that writes a paragraph, a heading,
+        and the indents.
+
+        :param name: the section name that is written as heading.
+        """
+        self.write_paragraph()
+        self.write_heading(name)
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    @contextmanager
+    def indentation(self):
+        """A context manager that increases the indentation."""
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    def getvalue(self):
+        """Returns the buffer contents."""
+        return ''.join(self.buffer)
+
+
+def join_options(options):
+    """Given a list of option strings this joins them in the most appropriate
+    way and returns them in the form ``(formatted_string,
+    any_prefix_is_slash)`` where the second item in the tuple is a flag that
+    indicates if any of the option prefixes was a slash.
+    """
+    rv = []
+    any_prefix_is_slash = False
+    for opt in options:
+        prefix = split_opt(opt)[0]
+        if prefix == '/':
+            any_prefix_is_slash = True
+        rv.append((len(prefix), opt))
+
+    rv.sort(key=lambda x: x[0])
+
+    rv = ', '.join(x[1] for x in rv)
+    return rv, any_prefix_is_slash

Contents/Libraries/Shared/click/globals.py

@@ -0,0 +1,48 @@
+from threading import local
+
+
+_local = local()
+
+
+def get_current_context(silent=False):
+    """Returns the current click context.  This can be used as a way to
+    access the current context object from anywhere.  This is a more implicit
+    alternative to the :func:`pass_context` decorator.  This function is
+    primarily useful for helpers such as :func:`echo` which might be
+    interested in changing it's behavior based on the current context.
+
+    To push the current context, :meth:`Context.scope` can be used.
+
+    .. versionadded:: 5.0
+
+    :param silent: is set to `True` the return value is `None` if no context
+                   is available.  The default behavior is to raise a
+                   :exc:`RuntimeError`.
+    """
+    try:
+        return getattr(_local, 'stack')[-1]
+    except (AttributeError, IndexError):
+        if not silent:
+            raise RuntimeError('There is no active click context.')
+
+
+def push_context(ctx):
+    """Pushes a new context to the current stack."""
+    _local.__dict__.setdefault('stack', []).append(ctx)
+
+
+def pop_context():
+    """Removes the top level from the stack."""
+    _local.stack.pop()
+
+
+def resolve_color_default(color=None):
+    """"Internal helper to get the default value of the color flag.  If a
+    value is passed it's returned unchanged, otherwise it's looked up from
+    the current context.
+    """
+    if color is not None:
+        return color
+    ctx = get_current_context(silent=True)
+    if ctx is not None:
+        return ctx.color

Contents/Libraries/Shared/click/parser.py

@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+"""
+    click.parser
+    ~~~~~~~~~~~~
+
+    This module started out as largely a copy paste from the stdlib's
+    optparse module with the features removed that we do not need from
+    optparse because we implement them in Click on a higher level (for
+    instance type handling, help formatting and a lot more).
+
+    The plan is to remove more and more from here over time.
+
+    The reason this is a different module and not optparse from the stdlib
+    is that there are differences in 2.x and 3.x about the error messages
+    generated and optparse in the stdlib uses gettext for no good reason
+    and might cause us issues.
+"""
+import re
+from collections import deque
+from .exceptions import UsageError, NoSuchOption, BadOptionUsage, \
+     BadArgumentUsage
+
+
+def _unpack_args(args, nargs_spec):
+    """Given an iterable of arguments and an iterable of nargs specifications,
+    it returns a tuple with all the unpacked arguments at the first index
+    and all remaining arguments as the second.
+
+    The nargs specification is the number of arguments that should be consumed
+    or `-1` to indicate that this position should eat up all the remainders.
+
+    Missing items are filled with `None`.
+    """
+    args = deque(args)
+    nargs_spec = deque(nargs_spec)
+    rv = []
+    spos = None
+
+    def _fetch(c):
+        try:
+            if spos is None:
+                return c.popleft()
+            else:
+                return c.pop()
+        except IndexError:
+            return None
+
+    while nargs_spec:
+        nargs = _fetch(nargs_spec)
+        if nargs == 1:
+            rv.append(_fetch(args))
+        elif nargs > 1:
+            x = [_fetch(args) for _ in range(nargs)]
+            # If we're reversed, we're pulling in the arguments in reverse,
+            # so we need to turn them around.
+            if spos is not None:
+                x.reverse()
+            rv.append(tuple(x))
+        elif nargs < 0:
+            if spos is not None:
+                raise TypeError('Cannot have two nargs < 0')
+            spos = len(rv)
+            rv.append(None)
+
+    # spos is the position of the wildcard (star).  If it's not `None`,
+    # we fill it with the remainder.
+    if spos is not None:
+        rv[spos] = tuple(args)
+        args = []
+        rv[spos + 1:] = reversed(rv[spos + 1:])
+
+    return tuple(rv), list(args)
+
+
+def _error_opt_args(nargs, opt):
+    if nargs == 1:
+        raise BadOptionUsage('%s option requires an argument' % opt)
+    raise BadOptionUsage('%s option requires %d arguments' % (opt, nargs))
+
+
+def split_opt(opt):
+    first = opt[:1]
+    if first.isalnum():
+        return '', opt
+    if opt[1:2] == first:
+        return opt[:2], opt[2:]
+    return first, opt[1:]
+
+
+def normalize_opt(opt, ctx):
+    if ctx is None or ctx.token_normalize_func is None:
+        return opt
+    prefix, opt = split_opt(opt)
+    return prefix + ctx.token_normalize_func(opt)
+
+
+def split_arg_string(string):
+    """Given an argument string this attempts to split it into small parts."""
+    rv = []
+    for match in re.finditer(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+                             r'|"([^"\\]*(?:\\.[^"\\]*)*)"'
+                             r'|\S+)\s*', string, re.S):
+        arg = match.group().strip()
+        if arg[:1] == arg[-1:] and arg[:1] in '"\'':
+            arg = arg[1:-1].encode('ascii', 'backslashreplace') \
+                .decode('unicode-escape')
+        try:
+            arg = type(string)(arg)
+        except UnicodeError:
+            pass
+        rv.append(arg)
+    return rv
+
+
+class Option(object):
+
+    def __init__(self, opts, dest, action=None, nargs=1, const=None, obj=None):
+        self._short_opts = []
+        self._long_opts = []
+        self.prefixes = set()
+
+        for opt in opts:
+            prefix, value = split_opt(opt)
+            if not prefix:
+                raise ValueError('Invalid start character for option (%s)'
+                                 % opt)
+            self.prefixes.add(prefix[0])
+            if len(prefix) == 1 and len(value) == 1:
+                self._short_opts.append(opt)
+            else:
+                self._long_opts.append(opt)
+                self.prefixes.add(prefix)
+
+        if action is None:
+            action = 'store'
+
+        self.dest = dest
+        self.action = action
+        self.nargs = nargs
+        self.const = const
+        self.obj = obj
+
+    @property
+    def takes_value(self):
+        return self.action in ('store', 'append')
+
+    def process(self, value, state):
+        if self.action == 'store':
+            state.opts[self.dest] = value
+        elif self.action == 'store_const':
+            state.opts[self.dest] = self.const
+        elif self.action == 'append':
+            state.opts.setdefault(self.dest, []).append(value)
+        elif self.action == 'append_const':
+            state.opts.setdefault(self.dest, []).append(self.const)
+        elif self.action == 'count':
+            state.opts[self.dest] = state.opts.get(self.dest, 0) + 1
+        else:
+            raise ValueError('unknown action %r' % self.action)
+        state.order.append(self.obj)
+
+
+class Argument(object):
+
+    def __init__(self, dest, nargs=1, obj=None):
+        self.dest = dest
+        self.nargs = nargs
+        self.obj = obj
+
+    def process(self, value, state):
+        if self.nargs > 1:
+            holes = sum(1 for x in value if x is None)
+            if holes == len(value):
+                value = None
+            elif holes != 0:
+                raise BadArgumentUsage('argument %s takes %d values'
+                                       % (self.dest, self.nargs))
+        state.opts[self.dest] = value
+        state.order.append(self.obj)
+
+
+class ParsingState(object):
+
+    def __init__(self, rargs):
+        self.opts = {}
+        self.largs = []
+        self.rargs = rargs
+        self.order = []
+
+
+class OptionParser(object):
+    """The option parser is an internal class that is ultimately used to
+    parse options and arguments.  It's modelled after optparse and brings
+    a similar but vastly simplified API.  It should generally not be used
+    directly as the high level Click classes wrap it for you.
+
+    It's not nearly as extensible as optparse or argparse as it does not
+    implement features that are implemented on a higher level (such as
+    types or defaults).
+
+    :param ctx: optionally the :class:`~click.Context` where this parser
+                should go with.
+    """
+
+    def __init__(self, ctx=None):
+        #: The :class:`~click.Context` for this parser.  This might be
+        #: `None` for some advanced use cases.
+        self.ctx = ctx
+        #: This controls how the parser deals with interspersed arguments.
+        #: If this is set to `False`, the parser will stop on the first
+        #: non-option.  Click uses this to implement nested subcommands
+        #: safely.
+        self.allow_interspersed_args = True
+        #: This tells the parser how to deal with unknown options.  By
+        #: default it will error out (which is sensible), but there is a
+        #: second mode where it will ignore it and continue processing
+        #: after shifting all the unknown options into the resulting args.
+        self.ignore_unknown_options = False
+        if ctx is not None:
+            self.allow_interspersed_args = ctx.allow_interspersed_args
+            self.ignore_unknown_options = ctx.ignore_unknown_options
+        self._short_opt = {}
+        self._long_opt = {}
+        self._opt_prefixes = set(['-', '--'])
+        self._args = []
+
+    def add_option(self, opts, dest, action=None, nargs=1, const=None,
+                   obj=None):
+        """Adds a new option named `dest` to the parser.  The destination
+        is not inferred (unlike with optparse) and needs to be explicitly
+        provided.  Action can be any of ``store``, ``store_const``,
+        ``append``, ``appnd_const`` or ``count``.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        opts = [normalize_opt(opt, self.ctx) for opt in opts]
+        option = Option(opts, dest, action=action, nargs=nargs,
+                        const=const, obj=obj)
+        self._opt_prefixes.update(option.prefixes)
+        for opt in option._short_opts:
+            self._short_opt[opt] = option
+        for opt in option._long_opts:
+            self._long_opt[opt] = option
+
+    def add_argument(self, dest, nargs=1, obj=None):
+        """Adds a positional argument named `dest` to the parser.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        if obj is None:
+            obj = dest
+        self._args.append(Argument(dest=dest, nargs=nargs, obj=obj))
+
+    def parse_args(self, args):
+        """Parses positional arguments and returns ``(values, args, order)``
+        for the parsed options and arguments as well as the leftover
+        arguments if there are any.  The order is a list of objects as they
+        appear on the command line.  If arguments appear multiple times they
+        will be memorized multiple times as well.
+        """
+        state = ParsingState(args)
+        try:
+            self._process_args_for_options(state)
+            self._process_args_for_args(state)
+        except UsageError:
+            if self.ctx is None or not self.ctx.resilient_parsing:
+                raise
+        return state.opts, state.largs, state.order
+
+    def _process_args_for_args(self, state):
+        pargs, args = _unpack_args(state.largs + state.rargs,
+                                   [x.nargs for x in self._args])
+
+        for idx, arg in enumerate(self._args):
+            arg.process(pargs[idx], state)
+
+        state.largs = args
+        state.rargs = []
+
+    def _process_args_for_options(self, state):
+        while state.rargs:
+            arg = state.rargs.pop(0)
+            arglen = len(arg)
+            # Double dashes always handled explicitly regardless of what
+            # prefixes are valid.
+            if arg == '--':
+                return
+            elif arg[:1] in self._opt_prefixes and arglen > 1:
+                self._process_opts(arg, state)
+            elif self.allow_interspersed_args:
+                state.largs.append(arg)
+            else:
+                state.rargs.insert(0, arg)
+                return
+
+        # Say this is the original argument list:
+        # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)]
+        #                            ^
+        # (we are about to process arg(i)).
+        #
+        # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of
+        # [arg0, ..., arg(i-1)] (any options and their arguments will have
+        # been removed from largs).
+        #
+        # The while loop will usually consume 1 or more arguments per pass.
+        # If it consumes 1 (eg. arg is an option that takes no arguments),
+        # then after _process_arg() is done the situation is:
+        #
+        #   largs = subset of [arg0, ..., arg(i)]
+        #   rargs = [arg(i+1), ..., arg(N-1)]
+        #
+        # If allow_interspersed_args is false, largs will always be
+        # *empty* -- still a subset of [arg0, ..., arg(i-1)], but
+        # not a very interesting subset!
+
+    def _match_long_opt(self, opt, explicit_value, state):
+        if opt not in self._long_opt:
+            possibilities = [word for word in self._long_opt
+                             if word.startswith(opt)]
+            raise NoSuchOption(opt, possibilities=possibilities)
+
+        option = self._long_opt[opt]
+        if option.takes_value:
+            # At this point it's safe to modify rargs by injecting the
+            # explicit value, because no exception is raised in this
+            # branch.  This means that the inserted value will be fully
+            # consumed.
+            if explicit_value is not None:
+                state.rargs.insert(0, explicit_value)
+
+            nargs = option.nargs
+            if len(state.rargs) < nargs:
+                _error_opt_args(nargs, opt)
+            elif nargs == 1:
+                value = state.rargs.pop(0)
+            else:
+                value = tuple(state.rargs[:nargs])
+                del state.rargs[:nargs]
+
+        elif explicit_value is not None:
+            raise BadOptionUsage('%s option does not take a value' % opt)
+
+        else:
+            value = None
+
+        option.process(value, state)
+
+    def _match_short_opt(self, arg, state):
+        stop = False
+        i = 1
+        prefix = arg[0]
+        unknown_options = []
+
+        for ch in arg[1:]:
+            opt = normalize_opt(prefix + ch, self.ctx)
+            option = self._short_opt.get(opt)
+            i += 1
+
+            if not option:
+                if self.ignore_unknown_options:
+                    unknown_options.append(ch)
+                    continue
+                raise NoSuchOption(opt)
+            if option.takes_value:
+                # Any characters left in arg?  Pretend they're the
+                # next arg, and stop consuming characters of arg.
+                if i < len(arg):
+                    state.rargs.insert(0, arg[i:])
+                    stop = True
+
+                nargs = option.nargs
+                if len(state.rargs) < nargs:
+                    _error_opt_args(nargs, opt)
+                elif nargs == 1:
+                    value = state.rargs.pop(0)
+                else:
+                    value = tuple(state.rargs[:nargs])
+                    del state.rargs[:nargs]
+
+            else:
+                value = None
+
+            option.process(value, state)
+
+            if stop:
+                break
+
+        # If we got any unknown options we re-combinate the string of the
+        # remaining options and re-attach the prefix, then report that
+        # to the state as new larg.  This way there is basic combinatorics
+        # that can be achieved while still ignoring unknown arguments.
+        if self.ignore_unknown_options and unknown_options:
+            state.largs.append(prefix + ''.join(unknown_options))
+
+    def _process_opts(self, arg, state):
+        explicit_value = None
+        # Long option handling happens in two parts.  The first part is
+        # supporting explicitly attached values.  In any case, we will try
+        # to long match the option first.
+        if '=' in arg:
+            long_opt, explicit_value = arg.split('=', 1)
+        else:
+            long_opt = arg
+        norm_long_opt = normalize_opt(long_opt, self.ctx)
+
+        # At this point we will match the (assumed) long option through
+        # the long option matching code.  Note that this allows options
+        # like "-foo" to be matched as long options.
+        try:
+            self._match_long_opt(norm_long_opt, explicit_value, state)
+        except NoSuchOption:
+            # At this point the long option matching failed, and we need
+            # to try with short options.  However there is a special rule
+            # which says, that if we have a two character options prefix
+            # (applies to "--foo" for instance), we do not dispatch to the
+            # short option code and will instead raise the no option
+            # error.
+            if arg[:2] not in self._opt_prefixes:
+                return self._match_short_opt(arg, state)
+            if not self.ignore_unknown_options:
+                raise
+            state.largs.append(arg)

Contents/Libraries/Shared/click/termui.py

@@ -0,0 +1,539 @@
+import os
+import sys
+import struct
+
+from ._compat import raw_input, text_type, string_types, \
+     isatty, strip_ansi, get_winterm_size, DEFAULT_COLUMNS, WIN
+from .utils import echo
+from .exceptions import Abort, UsageError
+from .types import convert_type
+from .globals import resolve_color_default
+
+
+# The prompt functions to use.  The doc tools currently override these
+# functions to customize how they work.
+visible_prompt_func = raw_input
+
+_ansi_colors = ('black', 'red', 'green', 'yellow', 'blue', 'magenta',
+                'cyan', 'white', 'reset')
+_ansi_reset_all = '\033[0m'
+
+
+def hidden_prompt_func(prompt):
+    import getpass
+    return getpass.getpass(prompt)
+
+
+def _build_prompt(text, suffix, show_default=False, default=None):
+    prompt = text
+    if default is not None and show_default:
+        prompt = '%s [%s]' % (prompt, default)
+    return prompt + suffix
+
+
+def prompt(text, default=None, hide_input=False,
+           confirmation_prompt=False, type=None,
+           value_proc=None, prompt_suffix=': ',
+           show_default=True, err=False):
+    """Prompts a user for input.  This is a convenience function that can
+    be used to prompt a user for input later.
+
+    If the user aborts the input by sending a interrupt signal, this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 6.0
+       Added unicode support for cmd.exe on Windows.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the text to show for the prompt.
+    :param default: the default value to use if no input happens.  If this
+                    is not given it will prompt until it's aborted.
+    :param hide_input: if this is set to true then the input value will
+                       be hidden.
+    :param confirmation_prompt: asks for confirmation for the value.
+    :param type: the type to use to check the value against.
+    :param value_proc: if this parameter is provided it's a function that
+                       is invoked instead of the type conversion to
+                       convert a value.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    result = None
+
+    def prompt_func(text):
+        f = hide_input and hidden_prompt_func or visible_prompt_func
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(text, nl=False, err=err)
+            return f('')
+        except (KeyboardInterrupt, EOFError):
+            # getpass doesn't print a newline if the user aborts input with ^C.
+            # Allegedly this behavior is inherited from getpass(3).
+            # A doc bug has been filed at https://bugs.python.org/issue24711
+            if hide_input:
+                echo(None, err=err)
+            raise Abort()
+
+    if value_proc is None:
+        value_proc = convert_type(type, default)
+
+    prompt = _build_prompt(text, prompt_suffix, show_default, default)
+
+    while 1:
+        while 1:
+            value = prompt_func(prompt)
+            if value:
+                break
+            # If a default is set and used, then the confirmation
+            # prompt is always skipped because that's the only thing
+            # that really makes sense.
+            elif default is not None:
+                return default
+        try:
+            result = value_proc(value)
+        except UsageError as e:
+            echo('Error: %s' % e.message, err=err)
+            continue
+        if not confirmation_prompt:
+            return result
+        while 1:
+            value2 = prompt_func('Repeat for confirmation: ')
+            if value2:
+                break
+        if value == value2:
+            return result
+        echo('Error: the two entered values do not match', err=err)
+
+
+def confirm(text, default=False, abort=False, prompt_suffix=': ',
+            show_default=True, err=False):
+    """Prompts for confirmation (yes/no question).
+
+    If the user aborts the input by sending a interrupt signal this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param text: the question to ask.
+    :param default: the default for the prompt.
+    :param abort: if this is set to `True` a negative answer aborts the
+                  exception by raising :exc:`Abort`.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    prompt = _build_prompt(text, prompt_suffix, show_default,
+                           default and 'Y/n' or 'y/N')
+    while 1:
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(prompt, nl=False, err=err)
+            value = visible_prompt_func('').lower().strip()
+        except (KeyboardInterrupt, EOFError):
+            raise Abort()
+        if value in ('y', 'yes'):
+            rv = True
+        elif value in ('n', 'no'):
+            rv = False
+        elif value == '':
+            rv = default
+        else:
+            echo('Error: invalid input', err=err)
+            continue
+        break
+    if abort and not rv:
+        raise Abort()
+    return rv
+
+
+def get_terminal_size():
+    """Returns the current size of the terminal as tuple in the form
+    ``(width, height)`` in columns and rows.
+    """
+    # If shutil has get_terminal_size() (Python 3.3 and later) use that
+    if sys.version_info >= (3, 3):
+        import shutil
+        shutil_get_terminal_size = getattr(shutil, 'get_terminal_size', None)
+        if shutil_get_terminal_size:
+            sz = shutil_get_terminal_size()
+            return sz.columns, sz.lines
+
+    if get_winterm_size is not None:
+        return get_winterm_size()
+
+    def ioctl_gwinsz(fd):
+        try:
+            import fcntl
+            import termios
+            cr = struct.unpack(
+                'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
+        except Exception:
+            return
+        return cr
+
+    cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
+    if not cr:
+        try:
+            fd = os.open(os.ctermid(), os.O_RDONLY)
+            try:
+                cr = ioctl_gwinsz(fd)
+            finally:
+                os.close(fd)
+        except Exception:
+            pass
+    if not cr or not cr[0] or not cr[1]:
+        cr = (os.environ.get('LINES', 25),
+              os.environ.get('COLUMNS', DEFAULT_COLUMNS))
+    return int(cr[1]), int(cr[0])
+
+
+def echo_via_pager(text, color=None):
+    """This function takes a text and shows it via an environment specific
+    pager on stdout.
+
+    .. versionchanged:: 3.0
+       Added the `color` flag.
+
+    :param text: the text to page.
+    :param color: controls if the pager supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    color = resolve_color_default(color)
+    if not isinstance(text, string_types):
+        text = text_type(text)
+    from ._termui_impl import pager
+    return pager(text + '\n', color)
+
+
+def progressbar(iterable=None, length=None, label=None, show_eta=True,
+                show_percent=None, show_pos=False,
+                item_show_func=None, fill_char='#', empty_char='-',
+                bar_template='%(label)s  [%(bar)s]  %(info)s',
+                info_sep='  ', width=36, file=None, color=None):
+    """This function creates an iterable context manager that can be used
+    to iterate over something while showing a progress bar.  It will
+    either iterate over the `iterable` or `length` items (that are counted
+    up).  While iteration happens, this function will print a rendered
+    progress bar to the given `file` (defaults to stdout) and will attempt
+    to calculate remaining time and more.  By default, this progress bar
+    will not be rendered if the file is not a terminal.
+
+    The context manager creates the progress bar.  When the context
+    manager is entered the progress bar is already displayed.  With every
+    iteration over the progress bar, the iterable passed to the bar is
+    advanced and the bar is updated.  When the context manager exits,
+    a newline is printed and the progress bar is finalized on screen.
+
+    No printing must happen or the progress bar will be unintentionally
+    destroyed.
+
+    Example usage::
+
+        with progressbar(items) as bar:
+            for item in bar:
+                do_something_with(item)
+
+    Alternatively, if no iterable is specified, one can manually update the
+    progress bar through the `update()` method instead of directly
+    iterating over the progress bar.  The update method accepts the number
+    of steps to increment the bar with::
+
+        with progressbar(length=chunks.total_bytes) as bar:
+            for chunk in chunks:
+                process_chunk(chunk)
+                bar.update(chunks.bytes)
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `color` parameter.  Added a `update` method to the
+       progressbar object.
+
+    :param iterable: an iterable to iterate over.  If not provided the length
+                     is required.
+    :param length: the number of items to iterate over.  By default the
+                   progressbar will attempt to ask the iterator about its
+                   length, which might or might not work.  If an iterable is
+                   also provided this parameter can be used to override the
+                   length.  If an iterable is not provided the progress bar
+                   will iterate over a range of that length.
+    :param label: the label to show next to the progress bar.
+    :param show_eta: enables or disables the estimated time display.  This is
+                     automatically disabled if the length cannot be
+                     determined.
+    :param show_percent: enables or disables the percentage display.  The
+                         default is `True` if the iterable has a length or
+                         `False` if not.
+    :param show_pos: enables or disables the absolute position display.  The
+                     default is `False`.
+    :param item_show_func: a function called with the current item which
+                           can return a string to show the current item
+                           next to the progress bar.  Note that the current
+                           item can be `None`!
+    :param fill_char: the character to use to show the filled part of the
+                      progress bar.
+    :param empty_char: the character to use to show the non-filled part of
+                       the progress bar.
+    :param bar_template: the format string to use as template for the bar.
+                         The parameters in it are ``label`` for the label,
+                         ``bar`` for the progress bar and ``info`` for the
+                         info section.
+    :param info_sep: the separator between multiple info items (eta etc.)
+    :param width: the width of the progress bar in characters, 0 means full
+                  terminal width
+    :param file: the file to write to.  If this is not a terminal then
+                 only the label is printed.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.  This is only needed if ANSI
+                  codes are included anywhere in the progress bar output
+                  which is not the case by default.
+    """
+    from ._termui_impl import ProgressBar
+    color = resolve_color_default(color)
+    return ProgressBar(iterable=iterable, length=length, show_eta=show_eta,
+                       show_percent=show_percent, show_pos=show_pos,
+                       item_show_func=item_show_func, fill_char=fill_char,
+                       empty_char=empty_char, bar_template=bar_template,
+                       info_sep=info_sep, file=file, label=label,
+                       width=width, color=color)
+
+
+def clear():
+    """Clears the terminal screen.  This will have the effect of clearing
+    the whole visible space of the terminal and moving the cursor to the
+    top left.  This does not do anything if not connected to a terminal.
+
+    .. versionadded:: 2.0
+    """
+    if not isatty(sys.stdout):
+        return
+    # If we're on Windows and we don't have colorama available, then we
+    # clear the screen by shelling out.  Otherwise we can use an escape
+    # sequence.
+    if WIN:
+        os.system('cls')
+    else:
+        sys.stdout.write('\033[2J\033[1;1H')
+
+
+def style(text, fg=None, bg=None, bold=None, dim=None, underline=None,
+          blink=None, reverse=None, reset=True):
+    """Styles a text with ANSI styles and returns the new string.  By
+    default the styling is self contained which means that at the end
+    of the string a reset code is issued.  This can be prevented by
+    passing ``reset=False``.
+
+    Examples::
+
+        click.echo(click.style('Hello World!', fg='green'))
+        click.echo(click.style('ATTENTION!', blink=True))
+        click.echo(click.style('Some things', reverse=True, fg='cyan'))
+
+    Supported color names:
+
+    * ``black`` (might be a gray)
+    * ``red``
+    * ``green``
+    * ``yellow`` (might be an orange)
+    * ``blue``
+    * ``magenta``
+    * ``cyan``
+    * ``white`` (might be light gray)
+    * ``reset`` (reset the color code only)
+
+    .. versionadded:: 2.0
+
+    :param text: the string to style with ansi codes.
+    :param fg: if provided this will become the foreground color.
+    :param bg: if provided this will become the background color.
+    :param bold: if provided this will enable or disable bold mode.
+    :param dim: if provided this will enable or disable dim mode.  This is
+                badly supported.
+    :param underline: if provided this will enable or disable underline.
+    :param blink: if provided this will enable or disable blinking.
+    :param reverse: if provided this will enable or disable inverse
+                    rendering (foreground becomes background and the
+                    other way round).
+    :param reset: by default a reset-all code is added at the end of the
+                  string which means that styles do not carry over.  This
+                  can be disabled to compose styles.
+    """
+    bits = []
+    if fg:
+        try:
+            bits.append('\033[%dm' % (_ansi_colors.index(fg) + 30))
+        except ValueError:
+            raise TypeError('Unknown color %r' % fg)
+    if bg:
+        try:
+            bits.append('\033[%dm' % (_ansi_colors.index(bg) + 40))
+        except ValueError:
+            raise TypeError('Unknown color %r' % bg)
+    if bold is not None:
+        bits.append('\033[%dm' % (1 if bold else 22))
+    if dim is not None:
+        bits.append('\033[%dm' % (2 if dim else 22))
+    if underline is not None:
+        bits.append('\033[%dm' % (4 if underline else 24))
+    if blink is not None:
+        bits.append('\033[%dm' % (5 if blink else 25))
+    if reverse is not None:
+        bits.append('\033[%dm' % (7 if reverse else 27))
+    bits.append(text)
+    if reset:
+        bits.append(_ansi_reset_all)
+    return ''.join(bits)
+
+
+def unstyle(text):
+    """Removes ANSI styling information from a string.  Usually it's not
+    necessary to use this function as Click's echo function will
+    automatically remove styling if necessary.
+
+    .. versionadded:: 2.0
+
+    :param text: the text to remove style information from.
+    """
+    return strip_ansi(text)
+
+
+def secho(text, file=None, nl=True, err=False, color=None, **styles):
+    """This function combines :func:`echo` and :func:`style` into one
+    call.  As such the following two calls are the same::
+
+        click.secho('Hello World!', fg='green')
+        click.echo(click.style('Hello World!', fg='green'))
+
+    All keyword arguments are forwarded to the underlying functions
+    depending on which one they go with.
+
+    .. versionadded:: 2.0
+    """
+    return echo(style(text, **styles), file=file, nl=nl, err=err, color=color)
+
+
+def edit(text=None, editor=None, env=None, require_save=True,
+         extension='.txt', filename=None):
+    r"""Edits the given text in the defined editor.  If an editor is given
+    (should be the full path to the executable but the regular operating
+    system search path is used for finding the executable) it overrides
+    the detected editor.  Optionally, some environment variables can be
+    used.  If the editor is closed without changes, `None` is returned.  In
+    case a file is edited directly the return value is always `None` and
+    `require_save` and `extension` are ignored.
+
+    If the editor cannot be opened a :exc:`UsageError` is raised.
+
+    Note for Windows: to simplify cross-platform usage, the newlines are
+    automatically converted from POSIX to Windows and vice versa.  As such,
+    the message here will have ``\n`` as newline markers.
+
+    :param text: the text to edit.
+    :param editor: optionally the editor to use.  Defaults to automatic
+                   detection.
+    :param env: environment variables to forward to the editor.
+    :param require_save: if this is true, then not saving in the editor
+                         will make the return value become `None`.
+    :param extension: the extension to tell the editor about.  This defaults
+                      to `.txt` but changing this might change syntax
+                      highlighting.
+    :param filename: if provided it will edit this file instead of the
+                     provided text contents.  It will not use a temporary
+                     file as an indirection in that case.
+    """
+    from ._termui_impl import Editor
+    editor = Editor(editor=editor, env=env, require_save=require_save,
+                    extension=extension)
+    if filename is None:
+        return editor.edit(text)
+    editor.edit_file(filename)
+
+
+def launch(url, wait=False, locate=False):
+    """This function launches the given URL (or filename) in the default
+    viewer application for this file type.  If this is an executable, it
+    might launch the executable in a new session.  The return value is
+    the exit code of the launched application.  Usually, ``0`` indicates
+    success.
+
+    Examples::
+
+        click.launch('http://click.pocoo.org/')
+        click.launch('/my/downloaded/file', locate=True)
+
+    .. versionadded:: 2.0
+
+    :param url: URL or filename of the thing to launch.
+    :param wait: waits for the program to stop.
+    :param locate: if this is set to `True` then instead of launching the
+                   application associated with the URL it will attempt to
+                   launch a file manager with the file located.  This
+                   might have weird effects if the URL does not point to
+                   the filesystem.
+    """
+    from ._termui_impl import open_url
+    return open_url(url, wait=wait, locate=locate)
+
+
+# If this is provided, getchar() calls into this instead.  This is used
+# for unittesting purposes.
+_getchar = None
+
+
+def getchar(echo=False):
+    """Fetches a single character from the terminal and returns it.  This
+    will always return a unicode character and under certain rare
+    circumstances this might return more than one character.  The
+    situations which more than one character is returned is when for
+    whatever reason multiple characters end up in the terminal buffer or
+    standard input was not actually a terminal.
+
+    Note that this will always read from the terminal, even if something
+    is piped into the standard input.
+
+    .. versionadded:: 2.0
+
+    :param echo: if set to `True`, the character read will also show up on
+                 the terminal.  The default is to not show it.
+    """
+    f = _getchar
+    if f is None:
+        from ._termui_impl import getchar as f
+    return f(echo)
+
+
+def pause(info='Press any key to continue ...', err=False):
+    """This command stops execution and waits for the user to press any
+    key to continue.  This is similar to the Windows batch "pause"
+    command.  If the program is not run through a terminal, this command
+    will instead do nothing.
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param info: the info string to print before pausing.
+    :param err: if set to message goes to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    if not isatty(sys.stdin) or not isatty(sys.stdout):
+        return
+    try:
+        if info:
+            echo(info, nl=False, err=err)
+        try:
+            getchar()
+        except (KeyboardInterrupt, EOFError):
+            pass
+    finally:
+        if info:
+            echo(err=err)

Contents/Libraries/Shared/click/testing.py

@@ -0,0 +1,322 @@
+import os
+import sys
+import shutil
+import tempfile
+import contextlib
+
+from ._compat import iteritems, PY2
+
+
+# If someone wants to vendor click, we want to ensure the
+# correct package is discovered.  Ideally we could use a
+# relative import here but unfortunately Python does not
+# support that.
+clickpkg = sys.modules[__name__.rsplit('.', 1)[0]]
+
+
+if PY2:
+    from cStringIO import StringIO
+else:
+    import io
+    from ._compat import _find_binary_reader
+
+
+class EchoingStdin(object):
+
+    def __init__(self, input, output):
+        self._input = input
+        self._output = output
+
+    def __getattr__(self, x):
+        return getattr(self._input, x)
+
+    def _echo(self, rv):
+        self._output.write(rv)
+        return rv
+
+    def read(self, n=-1):
+        return self._echo(self._input.read(n))
+
+    def readline(self, n=-1):
+        return self._echo(self._input.readline(n))
+
+    def readlines(self):
+        return [self._echo(x) for x in self._input.readlines()]
+
+    def __iter__(self):
+        return iter(self._echo(x) for x in self._input)
+
+    def __repr__(self):
+        return repr(self._input)
+
+
+def make_input_stream(input, charset):
+    # Is already an input stream.
+    if hasattr(input, 'read'):
+        if PY2:
+            return input
+        rv = _find_binary_reader(input)
+        if rv is not None:
+            return rv
+        raise TypeError('Could not find binary reader for input stream.')
+
+    if input is None:
+        input = b''
+    elif not isinstance(input, bytes):
+        input = input.encode(charset)
+    if PY2:
+        return StringIO(input)
+    return io.BytesIO(input)
+
+
+class Result(object):
+    """Holds the captured result of an invoked CLI script."""
+
+    def __init__(self, runner, output_bytes, exit_code, exception,
+                 exc_info=None):
+        #: The runner that created the result
+        self.runner = runner
+        #: The output as bytes.
+        self.output_bytes = output_bytes
+        #: The exit code as integer.
+        self.exit_code = exit_code
+        #: The exception that happend if one did.
+        self.exception = exception
+        #: The traceback
+        self.exc_info = exc_info
+
+    @property
+    def output(self):
+        """The output as unicode string."""
+        return self.output_bytes.decode(self.runner.charset, 'replace') \
+            .replace('\r\n', '\n')
+
+    def __repr__(self):
+        return '<Result %s>' % (
+            self.exception and repr(self.exception) or 'okay',
+        )
+
+
+class CliRunner(object):
+    """The CLI runner provides functionality to invoke a Click command line
+    script for unittesting purposes in a isolated environment.  This only
+    works in single-threaded systems without any concurrency as it changes the
+    global interpreter state.
+
+    :param charset: the character set for the input and output data.  This is
+                    UTF-8 by default and should not be changed currently as
+                    the reporting to Click only works in Python 2 properly.
+    :param env: a dictionary with environment variables for overriding.
+    :param echo_stdin: if this is set to `True`, then reading from stdin writes
+                       to stdout.  This is useful for showing examples in
+                       some circumstances.  Note that regular prompts
+                       will automatically echo the input.
+    """
+
+    def __init__(self, charset=None, env=None, echo_stdin=False):
+        if charset is None:
+            charset = 'utf-8'
+        self.charset = charset
+        self.env = env or {}
+        self.echo_stdin = echo_stdin
+
+    def get_default_prog_name(self, cli):
+        """Given a command object it will return the default program name
+        for it.  The default is the `name` attribute or ``"root"`` if not
+        set.
+        """
+        return cli.name or 'root'
+
+    def make_env(self, overrides=None):
+        """Returns the environment overrides for invoking a script."""
+        rv = dict(self.env)
+        if overrides:
+            rv.update(overrides)
+        return rv
+
+    @contextlib.contextmanager
+    def isolation(self, input=None, env=None, color=False):
+        """A context manager that sets up the isolation for invoking of a
+        command line tool.  This sets up stdin with the given input data
+        and `os.environ` with the overrides from the given dictionary.
+        This also rebinds some internals in Click to be mocked (like the
+        prompt functionality).
+
+        This is automatically done in the :meth:`invoke` method.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param input: the input stream to put into sys.stdin.
+        :param env: the environment overrides as dictionary.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        input = make_input_stream(input, self.charset)
+
+        old_stdin = sys.stdin
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        old_forced_width = clickpkg.formatting.FORCED_WIDTH
+        clickpkg.formatting.FORCED_WIDTH = 80
+
+        env = self.make_env(env)
+
+        if PY2:
+            sys.stdout = sys.stderr = bytes_output = StringIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+        else:
+            bytes_output = io.BytesIO()
+            if self.echo_stdin:
+                input = EchoingStdin(input, bytes_output)
+            input = io.TextIOWrapper(input, encoding=self.charset)
+            sys.stdout = sys.stderr = io.TextIOWrapper(
+                bytes_output, encoding=self.charset)
+
+        sys.stdin = input
+
+        def visible_input(prompt=None):
+            sys.stdout.write(prompt or '')
+            val = input.readline().rstrip('\r\n')
+            sys.stdout.write(val + '\n')
+            sys.stdout.flush()
+            return val
+
+        def hidden_input(prompt=None):
+            sys.stdout.write((prompt or '') + '\n')
+            sys.stdout.flush()
+            return input.readline().rstrip('\r\n')
+
+        def _getchar(echo):
+            char = sys.stdin.read(1)
+            if echo:
+                sys.stdout.write(char)
+                sys.stdout.flush()
+            return char
+
+        default_color = color
+        def should_strip_ansi(stream=None, color=None):
+            if color is None:
+                return not default_color
+            return not color
+
+        old_visible_prompt_func = clickpkg.termui.visible_prompt_func
+        old_hidden_prompt_func = clickpkg.termui.hidden_prompt_func
+        old__getchar_func = clickpkg.termui._getchar
+        old_should_strip_ansi = clickpkg.utils.should_strip_ansi
+        clickpkg.termui.visible_prompt_func = visible_input
+        clickpkg.termui.hidden_prompt_func = hidden_input
+        clickpkg.termui._getchar = _getchar
+        clickpkg.utils.should_strip_ansi = should_strip_ansi
+
+        old_env = {}
+        try:
+            for key, value in iteritems(env):
+                old_env[key] = os.environ.get(key)
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            yield bytes_output
+        finally:
+            for key, value in iteritems(old_env):
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+            sys.stdin = old_stdin
+            clickpkg.termui.visible_prompt_func = old_visible_prompt_func
+            clickpkg.termui.hidden_prompt_func = old_hidden_prompt_func
+            clickpkg.termui._getchar = old__getchar_func
+            clickpkg.utils.should_strip_ansi = old_should_strip_ansi
+            clickpkg.formatting.FORCED_WIDTH = old_forced_width
+
+    def invoke(self, cli, args=None, input=None, env=None,
+               catch_exceptions=True, color=False, **extra):
+        """Invokes a command in an isolated environment.  The arguments are
+        forwarded directly to the command line script, the `extra` keyword
+        arguments are passed to the :meth:`~clickpkg.Command.main` function of
+        the command.
+
+        This returns a :class:`Result` object.
+
+        .. versionadded:: 3.0
+           The ``catch_exceptions`` parameter was added.
+
+        .. versionchanged:: 3.0
+           The result object now has an `exc_info` attribute with the
+           traceback if available.
+
+        .. versionadded:: 4.0
+           The ``color`` parameter was added.
+
+        :param cli: the command to invoke
+        :param args: the arguments to invoke
+        :param input: the input data for `sys.stdin`.
+        :param env: the environment overrides.
+        :param catch_exceptions: Whether to catch any other exceptions than
+                                 ``SystemExit``.
+        :param extra: the keyword arguments to pass to :meth:`main`.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+        """
+        exc_info = None
+        with self.isolation(input=input, env=env, color=color) as out:
+            exception = None
+            exit_code = 0
+
+            try:
+                cli.main(args=args or (),
+                         prog_name=self.get_default_prog_name(cli), **extra)
+            except SystemExit as e:
+                if e.code != 0:
+                    exception = e
+
+                exc_info = sys.exc_info()
+
+                exit_code = e.code
+                if not isinstance(exit_code, int):
+                    sys.stdout.write(str(exit_code))
+                    sys.stdout.write('\n')
+                    exit_code = 1
+            except Exception as e:
+                if not catch_exceptions:
+                    raise
+                exception = e
+                exit_code = -1
+                exc_info = sys.exc_info()
+            finally:
+                sys.stdout.flush()
+                output = out.getvalue()
+
+        return Result(runner=self,
+                      output_bytes=output,
+                      exit_code=exit_code,
+                      exception=exception,
+                      exc_info=exc_info)
+
+    @contextlib.contextmanager
+    def isolated_filesystem(self):
+        """A context manager that creates a temporary folder and changes
+        the current working directory to it for isolated filesystem tests.
+        """
+        cwd = os.getcwd()
+        t = tempfile.mkdtemp()
+        os.chdir(t)
+        try:
+            yield t
+        finally:
+            os.chdir(cwd)
+            try:
+                shutil.rmtree(t)
+            except (OSError, IOError):
+                pass

Contents/Libraries/Shared/click/types.py

@@ -0,0 +1,550 @@
+import os
+import stat
+
+from ._compat import open_stream, text_type, filename_to_ui, \
+    get_filesystem_encoding, get_streerror, _get_argv_encoding, PY2
+from .exceptions import BadParameter
+from .utils import safecall, LazyFile
+
+
+class ParamType(object):
+    """Helper for converting values through types.  The following is
+    necessary for a valid type:
+
+    *   it needs a name
+    *   it needs to pass through None unchanged
+    *   it needs to convert from a string
+    *   it needs to convert its result type through unchanged
+        (eg: needs to be idempotent)
+    *   it needs to be able to deal with param and context being `None`.
+        This can be the case when the object is used with prompt
+        inputs.
+    """
+    is_composite = False
+
+    #: the descriptive name of this type
+    name = None
+
+    #: if a list of this type is expected and the value is pulled from a
+    #: string environment variable, this is what splits it up.  `None`
+    #: means any whitespace.  For all parameters the general rule is that
+    #: whitespace splits them up.  The exception are paths and files which
+    #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on
+    #: Windows).
+    envvar_list_splitter = None
+
+    def __call__(self, value, param=None, ctx=None):
+        if value is not None:
+            return self.convert(value, param, ctx)
+
+    def get_metavar(self, param):
+        """Returns the metavar default for this param if it provides one."""
+
+    def get_missing_message(self, param):
+        """Optionally might return extra information about a missing
+        parameter.
+
+        .. versionadded:: 2.0
+        """
+
+    def convert(self, value, param, ctx):
+        """Converts the value.  This is not invoked for values that are
+        `None` (the missing value).
+        """
+        return value
+
+    def split_envvar_value(self, rv):
+        """Given a value from an environment variable this splits it up
+        into small chunks depending on the defined envvar list splitter.
+
+        If the splitter is set to `None`, which means that whitespace splits,
+        then leading and trailing whitespace is ignored.  Otherwise, leading
+        and trailing splitters usually lead to empty items being included.
+        """
+        return (rv or '').split(self.envvar_list_splitter)
+
+    def fail(self, message, param=None, ctx=None):
+        """Helper method to fail with an invalid value message."""
+        raise BadParameter(message, ctx=ctx, param=param)
+
+
+class CompositeParamType(ParamType):
+    is_composite = True
+
+    @property
+    def arity(self):
+        raise NotImplementedError()
+
+
+class FuncParamType(ParamType):
+
+    def __init__(self, func):
+        self.name = func.__name__
+        self.func = func
+
+    def convert(self, value, param, ctx):
+        try:
+            return self.func(value)
+        except ValueError:
+            try:
+                value = text_type(value)
+            except UnicodeError:
+                value = str(value).decode('utf-8', 'replace')
+            self.fail(value, param, ctx)
+
+
+class UnprocessedParamType(ParamType):
+    name = 'text'
+
+    def convert(self, value, param, ctx):
+        return value
+
+    def __repr__(self):
+        return 'UNPROCESSED'
+
+
+class StringParamType(ParamType):
+    name = 'text'
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bytes):
+            enc = _get_argv_encoding()
+            try:
+                value = value.decode(enc)
+            except UnicodeError:
+                fs_enc = get_filesystem_encoding()
+                if fs_enc != enc:
+                    try:
+                        value = value.decode(fs_enc)
+                    except UnicodeError:
+                        value = value.decode('utf-8', 'replace')
+            return value
+        return value
+
+    def __repr__(self):
+        return 'STRING'
+
+
+class Choice(ParamType):
+    """The choice type allows a value to be checked against a fixed set of
+    supported values.  All of these values have to be strings.
+
+    See :ref:`choice-opts` for an example.
+    """
+    name = 'choice'
+
+    def __init__(self, choices):
+        self.choices = choices
+
+    def get_metavar(self, param):
+        return '[%s]' % '|'.join(self.choices)
+
+    def get_missing_message(self, param):
+        return 'Choose from %s.' % ', '.join(self.choices)
+
+    def convert(self, value, param, ctx):
+        # Exact match
+        if value in self.choices:
+            return value
+
+        # Match through normalization
+        if ctx is not None and \
+           ctx.token_normalize_func is not None:
+            value = ctx.token_normalize_func(value)
+            for choice in self.choices:
+                if ctx.token_normalize_func(choice) == value:
+                    return choice
+
+        self.fail('invalid choice: %s. (choose from %s)' %
+                  (value, ', '.join(self.choices)), param, ctx)
+
+    def __repr__(self):
+        return 'Choice(%r)' % list(self.choices)
+
+
+class IntParamType(ParamType):
+    name = 'integer'
+
+    def convert(self, value, param, ctx):
+        try:
+            return int(value)
+        except (ValueError, UnicodeError):
+            self.fail('%s is not a valid integer' % value, param, ctx)
+
+    def __repr__(self):
+        return 'INT'
+
+
+class IntRange(IntParamType):
+    """A parameter that works similar to :data:`click.INT` but restricts
+    the value to fit into a range.  The default behavior is to fail if the
+    value falls outside the range, but it can also be silently clamped
+    between the two edges.
+
+    See :ref:`ranges` for an example.
+    """
+    name = 'integer range'
+
+    def __init__(self, min=None, max=None, clamp=False):
+        self.min = min
+        self.max = max
+        self.clamp = clamp
+
+    def convert(self, value, param, ctx):
+        rv = IntParamType.convert(self, value, param, ctx)
+        if self.clamp:
+            if self.min is not None and rv < self.min:
+                return self.min
+            if self.max is not None and rv > self.max:
+                return self.max
+        if self.min is not None and rv < self.min or \
+           self.max is not None and rv > self.max:
+            if self.min is None:
+                self.fail('%s is bigger than the maximum valid value '
+                          '%s.' % (rv, self.max), param, ctx)
+            elif self.max is None:
+                self.fail('%s is smaller than the minimum valid value '
+                          '%s.' % (rv, self.min), param, ctx)
+            else:
+                self.fail('%s is not in the valid range of %s to %s.'
+                          % (rv, self.min, self.max), param, ctx)
+        return rv
+
+    def __repr__(self):
+        return 'IntRange(%r, %r)' % (self.min, self.max)
+
+
+class BoolParamType(ParamType):
+    name = 'boolean'
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bool):
+            return bool(value)
+        value = value.lower()
+        if value in ('true', '1', 'yes', 'y'):
+            return True
+        elif value in ('false', '0', 'no', 'n'):
+            return False
+        self.fail('%s is not a valid boolean' % value, param, ctx)
+
+    def __repr__(self):
+        return 'BOOL'
+
+
+class FloatParamType(ParamType):
+    name = 'float'
+
+    def convert(self, value, param, ctx):
+        try:
+            return float(value)
+        except (UnicodeError, ValueError):
+            self.fail('%s is not a valid floating point value' %
+                      value, param, ctx)
+
+    def __repr__(self):
+        return 'FLOAT'
+
+
+class UUIDParameterType(ParamType):
+    name = 'uuid'
+
+    def convert(self, value, param, ctx):
+        import uuid
+        try:
+            if PY2 and isinstance(value, text_type):
+                value = value.encode('ascii')
+            return uuid.UUID(value)
+        except (UnicodeError, ValueError):
+            self.fail('%s is not a valid UUID value' % value, param, ctx)
+
+    def __repr__(self):
+        return 'UUID'
+
+
+class File(ParamType):
+    """Declares a parameter to be a file for reading or writing.  The file
+    is automatically closed once the context tears down (after the command
+    finished working).
+
+    Files can be opened for reading or writing.  The special value ``-``
+    indicates stdin or stdout depending on the mode.
+
+    By default, the file is opened for reading text data, but it can also be
+    opened in binary mode or for writing.  The encoding parameter can be used
+    to force a specific encoding.
+
+    The `lazy` flag controls if the file should be opened immediately or
+    upon first IO.  The default is to be non lazy for standard input and
+    output streams as well as files opened for reading, lazy otherwise.
+
+    Starting with Click 2.0, files can also be opened atomically in which
+    case all writes go into a separate file in the same folder and upon
+    completion the file will be moved over to the original location.  This
+    is useful if a file regularly read by other users is modified.
+
+    See :ref:`file-args` for more information.
+    """
+    name = 'filename'
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(self, mode='r', encoding=None, errors='strict', lazy=None,
+                 atomic=False):
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.lazy = lazy
+        self.atomic = atomic
+
+    def resolve_lazy_flag(self, value):
+        if self.lazy is not None:
+            return self.lazy
+        if value == '-':
+            return False
+        elif 'w' in self.mode:
+            return True
+        return False
+
+    def convert(self, value, param, ctx):
+        try:
+            if hasattr(value, 'read') or hasattr(value, 'write'):
+                return value
+
+            lazy = self.resolve_lazy_flag(value)
+
+            if lazy:
+                f = LazyFile(value, self.mode, self.encoding, self.errors,
+                             atomic=self.atomic)
+                if ctx is not None:
+                    ctx.call_on_close(f.close_intelligently)
+                return f
+
+            f, should_close = open_stream(value, self.mode,
+                                          self.encoding, self.errors,
+                                          atomic=self.atomic)
+            # If a context is provided, we automatically close the file
+            # at the end of the context execution (or flush out).  If a
+            # context does not exist, it's the caller's responsibility to
+            # properly close the file.  This for instance happens when the
+            # type is used with prompts.
+            if ctx is not None:
+                if should_close:
+                    ctx.call_on_close(safecall(f.close))
+                else:
+                    ctx.call_on_close(safecall(f.flush))
+            return f
+        except (IOError, OSError) as e:
+            self.fail('Could not open file: %s: %s' % (
+                filename_to_ui(value),
+                get_streerror(e),
+            ), param, ctx)
+
+
+class Path(ParamType):
+    """The path type is similar to the :class:`File` type but it performs
+    different checks.  First of all, instead of returning an open file
+    handle it returns just the filename.  Secondly, it can perform various
+    basic checks about what the file or directory should be.
+
+    .. versionchanged:: 6.0
+       `allow_dash` was added.
+
+    :param exists: if set to true, the file or directory needs to exist for
+                   this value to be valid.  If this is not required and a
+                   file does indeed not exist, then all further checks are
+                   silently skipped.
+    :param file_okay: controls if a file is a possible value.
+    :param dir_okay: controls if a directory is a possible value.
+    :param writable: if true, a writable check is performed.
+    :param readable: if true, a readable check is performed.
+    :param resolve_path: if this is true, then the path is fully resolved
+                         before the value is passed onwards.  This means
+                         that it's absolute and symlinks are resolved.
+    :param allow_dash: If this is set to `True`, a single dash to indicate
+                       standard streams is permitted.
+    :param type: optionally a string type that should be used to
+                 represent the path.  The default is `None` which
+                 means the return value will be either bytes or
+                 unicode depending on what makes most sense given the
+                 input data Click deals with.
+    """
+    envvar_list_splitter = os.path.pathsep
+
+    def __init__(self, exists=False, file_okay=True, dir_okay=True,
+                 writable=False, readable=True, resolve_path=False,
+                 allow_dash=False, path_type=None):
+        self.exists = exists
+        self.file_okay = file_okay
+        self.dir_okay = dir_okay
+        self.writable = writable
+        self.readable = readable
+        self.resolve_path = resolve_path
+        self.allow_dash = allow_dash
+        self.type = path_type
+
+        if self.file_okay and not self.dir_okay:
+            self.name = 'file'
+            self.path_type = 'File'
+        if self.dir_okay and not self.file_okay:
+            self.name = 'directory'
+            self.path_type = 'Directory'
+        else:
+            self.name = 'path'
+            self.path_type = 'Path'
+
+    def coerce_path_result(self, rv):
+        if self.type is not None and not isinstance(rv, self.type):
+            if self.type is text_type:
+                rv = rv.decode(get_filesystem_encoding())
+            else:
+                rv = rv.encode(get_filesystem_encoding())
+        return rv
+
+    def convert(self, value, param, ctx):
+        rv = value
+
+        is_dash = self.file_okay and self.allow_dash and rv in (b'-', '-')
+
+        if not is_dash:
+            if self.resolve_path:
+                rv = os.path.realpath(rv)
+
+            try:
+                st = os.stat(rv)
+            except OSError:
+                if not self.exists:
+                    return self.coerce_path_result(rv)
+                self.fail('%s "%s" does not exist.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+
+            if not self.file_okay and stat.S_ISREG(st.st_mode):
+                self.fail('%s "%s" is a file.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if not self.dir_okay and stat.S_ISDIR(st.st_mode):
+                self.fail('%s "%s" is a directory.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if self.writable and not os.access(value, os.W_OK):
+                self.fail('%s "%s" is not writable.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+            if self.readable and not os.access(value, os.R_OK):
+                self.fail('%s "%s" is not readable.' % (
+                    self.path_type,
+                    filename_to_ui(value)
+                ), param, ctx)
+
+        return self.coerce_path_result(rv)
+
+
+class Tuple(CompositeParamType):
+    """The default behavior of Click is to apply a type on a value directly.
+    This works well in most cases, except for when `nargs` is set to a fixed
+    count and different types should be used for different items.  In this
+    case the :class:`Tuple` type can be used.  This type can only be used
+    if `nargs` is set to a fixed number.
+
+    For more information see :ref:`tuple-type`.
+
+    This can be selected by using a Python tuple literal as a type.
+
+    :param types: a list of types that should be used for the tuple items.
+    """
+
+    def __init__(self, types):
+        self.types = [convert_type(ty) for ty in types]
+
+    @property
+    def name(self):
+        return "<" + " ".join(ty.name for ty in self.types) + ">"
+
+    @property
+    def arity(self):
+        return len(self.types)
+
+    def convert(self, value, param, ctx):
+        if len(value) != len(self.types):
+            raise TypeError('It would appear that nargs is set to conflict '
+                            'with the composite type arity.')
+        return tuple(ty(x, param, ctx) for ty, x in zip(self.types, value))
+
+
+def convert_type(ty, default=None):
+    """Converts a callable or python ty into the most appropriate param
+    ty.
+    """
+    guessed_type = False
+    if ty is None and default is not None:
+        if isinstance(default, tuple):
+            ty = tuple(map(type, default))
+        else:
+            ty = type(default)
+        guessed_type = True
+
+    if isinstance(ty, tuple):
+        return Tuple(ty)
+    if isinstance(ty, ParamType):
+        return ty
+    if ty is text_type or ty is str or ty is None:
+        return STRING
+    if ty is int:
+        return INT
+    # Booleans are only okay if not guessed.  This is done because for
+    # flags the default value is actually a bit of a lie in that it
+    # indicates which of the flags is the one we want.  See get_default()
+    # for more information.
+    if ty is bool and not guessed_type:
+        return BOOL
+    if ty is float:
+        return FLOAT
+    if guessed_type:
+        return STRING
+
+    # Catch a common mistake
+    if __debug__:
+        try:
+            if issubclass(ty, ParamType):
+                raise AssertionError('Attempted to use an uninstantiated '
+                                     'parameter type (%s).' % ty)
+        except TypeError:
+            pass
+    return FuncParamType(ty)
+
+
+#: A dummy parameter type that just does nothing.  From a user's
+#: perspective this appears to just be the same as `STRING` but internally
+#: no string conversion takes place.  This is necessary to achieve the
+#: same bytes/unicode behavior on Python 2/3 in situations where you want
+#: to not convert argument types.  This is usually useful when working
+#: with file paths as they can appear in bytes and unicode.
+#:
+#: For path related uses the :class:`Path` type is a better choice but
+#: there are situations where an unprocessed type is useful which is why
+#: it is is provided.
+#:
+#: .. versionadded:: 4.0
+UNPROCESSED = UnprocessedParamType()
+
+#: A unicode string parameter type which is the implicit default.  This
+#: can also be selected by using ``str`` as type.
+STRING = StringParamType()
+
+#: An integer parameter.  This can also be selected by using ``int`` as
+#: type.
+INT = IntParamType()
+
+#: A floating point value parameter.  This can also be selected by using
+#: ``float`` as type.
+FLOAT = FloatParamType()
+
+#: A boolean parameter.  This is the default for boolean flags.  This can
+#: also be selected by using ``bool`` as a type.
+BOOL = BoolParamType()
+
+#: A UUID parameter.
+UUID = UUIDParameterType()

Contents/Libraries/Shared/click/utils.py

@@ -0,0 +1,415 @@
+import os
+import sys
+
+from .globals import resolve_color_default
+
+from ._compat import text_type, open_stream, get_filesystem_encoding, \
+    get_streerror, string_types, PY2, binary_streams, text_streams, \
+    filename_to_ui, auto_wrap_for_ansi, strip_ansi, should_strip_ansi, \
+    _default_text_stdout, _default_text_stderr, is_bytes, WIN
+
+if not PY2:
+    from ._compat import _find_binary_writer
+elif WIN:
+    from ._winconsole import _get_windows_argv, \
+         _hash_py_argv, _initial_argv_hash
+
+
+echo_native_types = string_types + (bytes, bytearray)
+
+
+def _posixify(name):
+    return '-'.join(name.split()).lower()
+
+
+def safecall(func):
+    """Wraps a function so that it swallows exceptions."""
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            pass
+    return wrapper
+
+
+def make_str(value):
+    """Converts a value into a valid string."""
+    if isinstance(value, bytes):
+        try:
+            return value.decode(get_filesystem_encoding())
+        except UnicodeError:
+            return value.decode('utf-8', 'replace')
+    return text_type(value)
+
+
+def make_default_short_help(help, max_length=45):
+    words = help.split()
+    total_length = 0
+    result = []
+    done = False
+
+    for word in words:
+        if word[-1:] == '.':
+            done = True
+        new_length = result and 1 + len(word) or len(word)
+        if total_length + new_length > max_length:
+            result.append('...')
+            done = True
+        else:
+            if result:
+                result.append(' ')
+            result.append(word)
+        if done:
+            break
+        total_length += new_length
+
+    return ''.join(result)
+
+
+class LazyFile(object):
+    """A lazy file works like a regular file but it does not fully open
+    the file but it does perform some basic checks early to see if the
+    filename parameter does make sense.  This is useful for safely opening
+    files for writing.
+    """
+
+    def __init__(self, filename, mode='r', encoding=None, errors='strict',
+                 atomic=False):
+        self.name = filename
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.atomic = atomic
+
+        if filename == '-':
+            self._f, self.should_close = open_stream(filename, mode,
+                                                     encoding, errors)
+        else:
+            if 'r' in mode:
+                # Open and close the file in case we're opening it for
+                # reading so that we can catch at least some errors in
+                # some cases early.
+                open(filename, mode).close()
+            self._f = None
+            self.should_close = True
+
+    def __getattr__(self, name):
+        return getattr(self.open(), name)
+
+    def __repr__(self):
+        if self._f is not None:
+            return repr(self._f)
+        return '<unopened file %r %s>' % (self.name, self.mode)
+
+    def open(self):
+        """Opens the file if it's not yet open.  This call might fail with
+        a :exc:`FileError`.  Not handling this error will produce an error
+        that Click shows.
+        """
+        if self._f is not None:
+            return self._f
+        try:
+            rv, self.should_close = open_stream(self.name, self.mode,
+                                                self.encoding,
+                                                self.errors,
+                                                atomic=self.atomic)
+        except (IOError, OSError) as e:
+            from .exceptions import FileError
+            raise FileError(self.name, hint=get_streerror(e))
+        self._f = rv
+        return rv
+
+    def close(self):
+        """Closes the underlying file, no matter what."""
+        if self._f is not None:
+            self._f.close()
+
+    def close_intelligently(self):
+        """This function only closes the file if it was opened by the lazy
+        file wrapper.  For instance this will never close stdin.
+        """
+        if self.should_close:
+            self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close_intelligently()
+
+    def __iter__(self):
+        self.open()
+        return iter(self._f)
+
+
+class KeepOpenFile(object):
+
+    def __init__(self, file):
+        self._file = file
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        pass
+
+    def __repr__(self):
+        return repr(self._file)
+
+    def __iter__(self):
+        return iter(self._file)
+
+
+def echo(message=None, file=None, nl=True, err=False, color=None):
+    """Prints a message plus a newline to the given file or stdout.  On
+    first sight, this looks like the print function, but it has improved
+    support for handling Unicode and binary data that does not fail no
+    matter how badly configured the system is.
+
+    Primarily it means that you can print binary data as well as Unicode
+    data on both 2.x and 3.x to the given file in the most appropriate way
+    possible.  This is a very carefree function as in that it will try its
+    best to not fail.  As of Click 6.0 this includes support for unicode
+    output on the Windows console.
+
+    In addition to that, if `colorama`_ is installed, the echo function will
+    also support clever handling of ANSI codes.  Essentially it will then
+    do the following:
+
+    -   add transparent handling of ANSI color codes on Windows.
+    -   hide ANSI codes automatically if the destination file is not a
+        terminal.
+
+    .. _colorama: http://pypi.python.org/pypi/colorama
+
+    .. versionchanged:: 6.0
+       As of Click 6.0 the echo function will properly support unicode
+       output on the windows console.  Not that click does not modify
+       the interpreter in any way which means that `sys.stdout` or the
+       print statement or function will still not provide unicode support.
+
+    .. versionchanged:: 2.0
+       Starting with version 2.0 of Click, the echo function will work
+       with colorama if it's installed.
+
+    .. versionadded:: 3.0
+       The `err` parameter was added.
+
+    .. versionchanged:: 4.0
+       Added the `color` flag.
+
+    :param message: the message to print
+    :param file: the file to write to (defaults to ``stdout``)
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``.  This is faster and easier than calling
+                :func:`get_text_stderr` yourself.
+    :param nl: if set to `True` (the default) a newline is printed afterwards.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    if file is None:
+        if err:
+            file = _default_text_stderr()
+        else:
+            file = _default_text_stdout()
+
+    # Convert non bytes/text into the native string type.
+    if message is not None and not isinstance(message, echo_native_types):
+        message = text_type(message)
+
+    if nl:
+        message = message or u''
+        if isinstance(message, text_type):
+            message += u'\n'
+        else:
+            message += b'\n'
+
+    # If there is a message, and we're in Python 3, and the value looks
+    # like bytes, we manually need to find the binary stream and write the
+    # message in there.  This is done separately so that most stream
+    # types will work as you would expect.  Eg: you can write to StringIO
+    # for other cases.
+    if message and not PY2 and is_bytes(message):
+        binary_file = _find_binary_writer(file)
+        if binary_file is not None:
+            file.flush()
+            binary_file.write(message)
+            binary_file.flush()
+            return
+
+    # ANSI-style support.  If there is no message or we are dealing with
+    # bytes nothing is happening.  If we are connected to a file we want
+    # to strip colors.  If we are on windows we either wrap the stream
+    # to strip the color or we use the colorama support to translate the
+    # ansi codes to API calls.
+    if message and not is_bytes(message):
+        color = resolve_color_default(color)
+        if should_strip_ansi(file, color):
+            message = strip_ansi(message)
+        elif WIN:
+            if auto_wrap_for_ansi is not None:
+                file = auto_wrap_for_ansi(file)
+            elif not color:
+                message = strip_ansi(message)
+
+    if message:
+        file.write(message)
+    file.flush()
+
+
+def get_binary_stream(name):
+    """Returns a system stream for byte processing.  This essentially
+    returns the stream from the sys module with the given name but it
+    solves some compatibility issues between different Python versions.
+    Primarily this function is necessary for getting binary streams on
+    Python 3.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    """
+    opener = binary_streams.get(name)
+    if opener is None:
+        raise TypeError('Unknown standard stream %r' % name)
+    return opener()
+
+
+def get_text_stream(name, encoding=None, errors='strict'):
+    """Returns a system stream for text processing.  This usually returns
+    a wrapped stream around a binary stream returned from
+    :func:`get_binary_stream` but it also can take shortcuts on Python 3
+    for already correctly configured streams.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    :param encoding: overrides the detected default encoding.
+    :param errors: overrides the default error mode.
+    """
+    opener = text_streams.get(name)
+    if opener is None:
+        raise TypeError('Unknown standard stream %r' % name)
+    return opener(encoding, errors)
+
+
+def open_file(filename, mode='r', encoding=None, errors='strict',
+              lazy=False, atomic=False):
+    """This is similar to how the :class:`File` works but for manual
+    usage.  Files are opened non lazy by default.  This can open regular
+    files as well as stdin/stdout if ``'-'`` is passed.
+
+    If stdin/stdout is returned the stream is wrapped so that the context
+    manager will not close the stream accidentally.  This makes it possible
+    to always use the function like this without having to worry to
+    accidentally close a standard stream::
+
+        with open_file(filename) as f:
+            ...
+
+    .. versionadded:: 3.0
+
+    :param filename: the name of the file to open (or ``'-'`` for stdin/stdout).
+    :param mode: the mode in which to open the file.
+    :param encoding: the encoding to use.
+    :param errors: the error handling for this file.
+    :param lazy: can be flipped to true to open the file lazily.
+    :param atomic: in atomic mode writes go into a temporary file and it's
+                   moved on close.
+    """
+    if lazy:
+        return LazyFile(filename, mode, encoding, errors, atomic=atomic)
+    f, should_close = open_stream(filename, mode, encoding, errors,
+                                  atomic=atomic)
+    if not should_close:
+        f = KeepOpenFile(f)
+    return f
+
+
+def get_os_args():
+    """This returns the argument part of sys.argv in the most appropriate
+    form for processing.  What this means is that this return value is in
+    a format that works for Click to process but does not necessarily
+    correspond well to what's actually standard for the interpreter.
+
+    On most environments the return value is ``sys.argv[:1]`` unchanged.
+    However if you are on Windows and running Python 2 the return value
+    will actually be a list of unicode strings instead because the
+    default behavior on that platform otherwise will not be able to
+    carry all possible values that sys.argv can have.
+
+    .. versionadded:: 6.0
+    """
+    # We can only extract the unicode argv if sys.argv has not been
+    # changed since the startup of the application.
+    if PY2 and WIN and _initial_argv_hash == _hash_py_argv():
+        return _get_windows_argv()
+    return sys.argv[1:]
+
+
+def format_filename(filename, shorten=False):
+    """Formats a filename for user display.  The main purpose of this
+    function is to ensure that the filename can be displayed at all.  This
+    will decode the filename to unicode if necessary in a way that it will
+    not fail.  Optionally, it can shorten the filename to not include the
+    full path to the filename.
+
+    :param filename: formats a filename for UI display.  This will also convert
+                     the filename into unicode without failing.
+    :param shorten: this optionally shortens the filename to strip of the
+                    path that leads up to it.
+    """
+    if shorten:
+        filename = os.path.basename(filename)
+    return filename_to_ui(filename)
+
+
+def get_app_dir(app_name, roaming=True, force_posix=False):
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Win XP (roaming):
+      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo Bar``
+    Win XP (not roaming):
+      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
+    Win 7 (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Win 7 (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no affect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+    if WIN:
+        key = roaming and 'APPDATA' or 'LOCALAPPDATA'
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser('~')
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(os.path.expanduser('~/.' + _posixify(app_name)))
+    if sys.platform == 'darwin':
+        return os.path.join(os.path.expanduser(
+            '~/Library/Application Support'), app_name)
+    return os.path.join(
+        os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')),
+        _posixify(app_name))

Contents/Libraries/Shared/concurrent/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

Contents/Libraries/Shared/concurrent/futures/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2009 Brian Quinlan. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Execute computations asynchronously using threads or processes."""
+
+__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+
+from concurrent.futures._base import (FIRST_COMPLETED,
+                                      FIRST_EXCEPTION,
+                                      ALL_COMPLETED,
+                                      CancelledError,
+                                      TimeoutError,
+                                      Future,
+                                      Executor,
+                                      wait,
+                                      as_completed)
+from concurrent.futures.thread import ThreadPoolExecutor
+
+try:
+    from concurrent.futures.process import ProcessPoolExecutor
+except ImportError:
+    # some platforms don't have multiprocessing
+    pass

Contents/Libraries/Shared/concurrent/futures/_base.py

@@ -0,0 +1,607 @@
+# Copyright 2009 Brian Quinlan. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+import collections
+import logging
+import threading
+import itertools
+import time
+
+__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+
+FIRST_COMPLETED = 'FIRST_COMPLETED'
+FIRST_EXCEPTION = 'FIRST_EXCEPTION'
+ALL_COMPLETED = 'ALL_COMPLETED'
+_AS_COMPLETED = '_AS_COMPLETED'
+
+# Possible future states (for internal use by the futures package).
+PENDING = 'PENDING'
+RUNNING = 'RUNNING'
+# The future was cancelled by the user...
+CANCELLED = 'CANCELLED'
+# ...and _Waiter.add_cancelled() was called by a worker.
+CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
+FINISHED = 'FINISHED'
+
+_FUTURE_STATES = [
+    PENDING,
+    RUNNING,
+    CANCELLED,
+    CANCELLED_AND_NOTIFIED,
+    FINISHED
+]
+
+_STATE_TO_DESCRIPTION_MAP = {
+    PENDING: "pending",
+    RUNNING: "running",
+    CANCELLED: "cancelled",
+    CANCELLED_AND_NOTIFIED: "cancelled",
+    FINISHED: "finished"
+}
+
+# Logger for internal use by the futures package.
+LOGGER = logging.getLogger("concurrent.futures")
+
+class Error(Exception):
+    """Base class for all future-related exceptions."""
+    pass
+
+class CancelledError(Error):
+    """The Future was cancelled."""
+    pass
+
+class TimeoutError(Error):
+    """The operation exceeded the given deadline."""
+    pass
+
+class _Waiter(object):
+    """Provides the event that wait() and as_completed() block on."""
+    def __init__(self):
+        self.event = threading.Event()
+        self.finished_futures = []
+
+    def add_result(self, future):
+        self.finished_futures.append(future)
+
+    def add_exception(self, future):
+        self.finished_futures.append(future)
+
+    def add_cancelled(self, future):
+        self.finished_futures.append(future)
+
+class _AsCompletedWaiter(_Waiter):
+    """Used by as_completed()."""
+
+    def __init__(self):
+        super(_AsCompletedWaiter, self).__init__()
+        self.lock = threading.Lock()
+
+    def add_result(self, future):
+        with self.lock:
+            super(_AsCompletedWaiter, self).add_result(future)
+            self.event.set()
+
+    def add_exception(self, future):
+        with self.lock:
+            super(_AsCompletedWaiter, self).add_exception(future)
+            self.event.set()
+
+    def add_cancelled(self, future):
+        with self.lock:
+            super(_AsCompletedWaiter, self).add_cancelled(future)
+            self.event.set()
+
+class _FirstCompletedWaiter(_Waiter):
+    """Used by wait(return_when=FIRST_COMPLETED)."""
+
+    def add_result(self, future):
+        super(_FirstCompletedWaiter, self).add_result(future)
+        self.event.set()
+
+    def add_exception(self, future):
+        super(_FirstCompletedWaiter, self).add_exception(future)
+        self.event.set()
+
+    def add_cancelled(self, future):
+        super(_FirstCompletedWaiter, self).add_cancelled(future)
+        self.event.set()
+
+class _AllCompletedWaiter(_Waiter):
+    """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""
+
+    def __init__(self, num_pending_calls, stop_on_exception):
+        self.num_pending_calls = num_pending_calls
+        self.stop_on_exception = stop_on_exception
+        self.lock = threading.Lock()
+        super(_AllCompletedWaiter, self).__init__()
+
+    def _decrement_pending_calls(self):
+        with self.lock:
+            self.num_pending_calls -= 1
+            if not self.num_pending_calls:
+                self.event.set()
+
+    def add_result(self, future):
+        super(_AllCompletedWaiter, self).add_result(future)
+        self._decrement_pending_calls()
+
+    def add_exception(self, future):
+        super(_AllCompletedWaiter, self).add_exception(future)
+        if self.stop_on_exception:
+            self.event.set()
+        else:
+            self._decrement_pending_calls()
+
+    def add_cancelled(self, future):
+        super(_AllCompletedWaiter, self).add_cancelled(future)
+        self._decrement_pending_calls()
+
+class _AcquireFutures(object):
+    """A context manager that does an ordered acquire of Future conditions."""
+
+    def __init__(self, futures):
+        self.futures = sorted(futures, key=id)
+
+    def __enter__(self):
+        for future in self.futures:
+            future._condition.acquire()
+
+    def __exit__(self, *args):
+        for future in self.futures:
+            future._condition.release()
+
+def _create_and_install_waiters(fs, return_when):
+    if return_when == _AS_COMPLETED:
+        waiter = _AsCompletedWaiter()
+    elif return_when == FIRST_COMPLETED:
+        waiter = _FirstCompletedWaiter()
+    else:
+        pending_count = sum(
+                f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs)
+
+        if return_when == FIRST_EXCEPTION:
+            waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True)
+        elif return_when == ALL_COMPLETED:
+            waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False)
+        else:
+            raise ValueError("Invalid return condition: %r" % return_when)
+
+    for f in fs:
+        f._waiters.append(waiter)
+
+    return waiter
+
+def as_completed(fs, timeout=None):
+    """An iterator over the given futures that yields each as it completes.
+
+    Args:
+        fs: The sequence of Futures (possibly created by different Executors) to
+            iterate over.
+        timeout: The maximum number of seconds to wait. If None, then there
+            is no limit on the wait time.
+
+    Returns:
+        An iterator that yields the given Futures as they complete (finished or
+        cancelled). If any given Futures are duplicated, they will be returned
+        once.
+
+    Raises:
+        TimeoutError: If the entire result iterator could not be generated
+            before the given timeout.
+    """
+    if timeout is not None:
+        end_time = timeout + time.time()
+
+    fs = set(fs)
+    with _AcquireFutures(fs):
+        finished = set(
+                f for f in fs
+                if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
+        pending = fs - finished
+        waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
+
+    try:
+        for future in finished:
+            yield future
+
+        while pending:
+            if timeout is None:
+                wait_timeout = None
+            else:
+                wait_timeout = end_time - time.time()
+                if wait_timeout < 0:
+                    raise TimeoutError(
+                            '%d (of %d) futures unfinished' % (
+                            len(pending), len(fs)))
+
+            waiter.event.wait(wait_timeout)
+
+            with waiter.lock:
+                finished = waiter.finished_futures
+                waiter.finished_futures = []
+                waiter.event.clear()
+
+            for future in finished:
+                yield future
+                pending.remove(future)
+
+    finally:
+        for f in fs:
+            with f._condition:
+                f._waiters.remove(waiter)
+
+DoneAndNotDoneFutures = collections.namedtuple(
+        'DoneAndNotDoneFutures', 'done not_done')
+def wait(fs, timeout=None, return_when=ALL_COMPLETED):
+    """Wait for the futures in the given sequence to complete.
+
+    Args:
+        fs: The sequence of Futures (possibly created by different Executors) to
+            wait upon.
+        timeout: The maximum number of seconds to wait. If None, then there
+            is no limit on the wait time.
+        return_when: Indicates when this function should return. The options
+            are:
+
+            FIRST_COMPLETED - Return when any future finishes or is
+                              cancelled.
+            FIRST_EXCEPTION - Return when any future finishes by raising an
+                              exception. If no future raises an exception
+                              then it is equivalent to ALL_COMPLETED.
+            ALL_COMPLETED -   Return when all futures finish or are cancelled.
+
+    Returns:
+        A named 2-tuple of sets. The first set, named 'done', contains the
+        futures that completed (is finished or cancelled) before the wait
+        completed. The second set, named 'not_done', contains uncompleted
+        futures.
+    """
+    with _AcquireFutures(fs):
+        done = set(f for f in fs
+                   if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
+        not_done = set(fs) - done
+
+        if (return_when == FIRST_COMPLETED) and done:
+            return DoneAndNotDoneFutures(done, not_done)
+        elif (return_when == FIRST_EXCEPTION) and done:
+            if any(f for f in done
+                   if not f.cancelled() and f.exception() is not None):
+                return DoneAndNotDoneFutures(done, not_done)
+
+        if len(done) == len(fs):
+            return DoneAndNotDoneFutures(done, not_done)
+
+        waiter = _create_and_install_waiters(fs, return_when)
+
+    waiter.event.wait(timeout)
+    for f in fs:
+        with f._condition:
+            f._waiters.remove(waiter)
+
+    done.update(waiter.finished_futures)
+    return DoneAndNotDoneFutures(done, set(fs) - done)
+
+class Future(object):
+    """Represents the result of an asynchronous computation."""
+
+    def __init__(self):
+        """Initializes the future. Should not be called by clients."""
+        self._condition = threading.Condition()
+        self._state = PENDING
+        self._result = None
+        self._exception = None
+        self._traceback = None
+        self._waiters = []
+        self._done_callbacks = []
+
+    def _invoke_callbacks(self):
+        for callback in self._done_callbacks:
+            try:
+                callback(self)
+            except Exception:
+                LOGGER.exception('exception calling callback for %r', self)
+
+    def __repr__(self):
+        with self._condition:
+            if self._state == FINISHED:
+                if self._exception:
+                    return '<Future at %s state=%s raised %s>' % (
+                        hex(id(self)),
+                        _STATE_TO_DESCRIPTION_MAP[self._state],
+                        self._exception.__class__.__name__)
+                else:
+                    return '<Future at %s state=%s returned %s>' % (
+                        hex(id(self)),
+                        _STATE_TO_DESCRIPTION_MAP[self._state],
+                        self._result.__class__.__name__)
+            return '<Future at %s state=%s>' % (
+                    hex(id(self)),
+                   _STATE_TO_DESCRIPTION_MAP[self._state])
+
+    def cancel(self):
+        """Cancel the future if possible.
+
+        Returns True if the future was cancelled, False otherwise. A future
+        cannot be cancelled if it is running or has already completed.
+        """
+        with self._condition:
+            if self._state in [RUNNING, FINISHED]:
+                return False
+
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                return True
+
+            self._state = CANCELLED
+            self._condition.notify_all()
+
+        self._invoke_callbacks()
+        return True
+
+    def cancelled(self):
+        """Return True if the future has cancelled."""
+        with self._condition:
+            return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
+
+    def running(self):
+        """Return True if the future is currently executing."""
+        with self._condition:
+            return self._state == RUNNING
+
+    def done(self):
+        """Return True of the future was cancelled or finished executing."""
+        with self._condition:
+            return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
+
+    def __get_result(self):
+        if self._exception:
+            raise type(self._exception), self._exception, self._traceback
+        else:
+            return self._result
+
+    def add_done_callback(self, fn):
+        """Attaches a callable that will be called when the future finishes.
+
+        Args:
+            fn: A callable that will be called with this future as its only
+                argument when the future completes or is cancelled. The callable
+                will always be called by a thread in the same process in which
+                it was added. If the future has already completed or been
+                cancelled then the callable will be called immediately. These
+                callables are called in the order that they were added.
+        """
+        with self._condition:
+            if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
+                self._done_callbacks.append(fn)
+                return
+        fn(self)
+
+    def result(self, timeout=None):
+        """Return the result of the call that the future represents.
+
+        Args:
+            timeout: The number of seconds to wait for the result if the future
+                isn't done. If None, then there is no limit on the wait time.
+
+        Returns:
+            The result of the call that the future represents.
+
+        Raises:
+            CancelledError: If the future was cancelled.
+            TimeoutError: If the future didn't finish executing before the given
+                timeout.
+            Exception: If the call raised then that exception will be raised.
+        """
+        with self._condition:
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self.__get_result()
+
+            self._condition.wait(timeout)
+
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self.__get_result()
+            else:
+                raise TimeoutError()
+
+    def exception_info(self, timeout=None):
+        """Return a tuple of (exception, traceback) raised by the call that the
+        future represents.
+
+        Args:
+            timeout: The number of seconds to wait for the exception if the
+                future isn't done. If None, then there is no limit on the wait
+                time.
+
+        Returns:
+            The exception raised by the call that the future represents or None
+            if the call completed without raising.
+
+        Raises:
+            CancelledError: If the future was cancelled.
+            TimeoutError: If the future didn't finish executing before the given
+                timeout.
+        """
+        with self._condition:
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self._exception, self._traceback
+
+            self._condition.wait(timeout)
+
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self._exception, self._traceback
+            else:
+                raise TimeoutError()
+
+    def exception(self, timeout=None):
+        """Return the exception raised by the call that the future represents.
+
+        Args:
+            timeout: The number of seconds to wait for the exception if the
+                future isn't done. If None, then there is no limit on the wait
+                time.
+
+        Returns:
+            The exception raised by the call that the future represents or None
+            if the call completed without raising.
+
+        Raises:
+            CancelledError: If the future was cancelled.
+            TimeoutError: If the future didn't finish executing before the given
+                timeout.
+        """
+        return self.exception_info(timeout)[0]
+
+    # The following methods should only be used by Executors and in tests.
+    def set_running_or_notify_cancel(self):
+        """Mark the future as running or process any cancel notifications.
+
+        Should only be used by Executor implementations and unit tests.
+
+        If the future has been cancelled (cancel() was called and returned
+        True) then any threads waiting on the future completing (though calls
+        to as_completed() or wait()) are notified and False is returned.
+
+        If the future was not cancelled then it is put in the running state
+        (future calls to running() will return True) and True is returned.
+
+        This method should be called by Executor implementations before
+        executing the work associated with this future. If this method returns
+        False then the work should not be executed.
+
+        Returns:
+            False if the Future was cancelled, True otherwise.
+
+        Raises:
+            RuntimeError: if this method was already called or if set_result()
+                or set_exception() was called.
+        """
+        with self._condition:
+            if self._state == CANCELLED:
+                self._state = CANCELLED_AND_NOTIFIED
+                for waiter in self._waiters:
+                    waiter.add_cancelled(self)
+                # self._condition.notify_all() is not necessary because
+                # self.cancel() triggers a notification.
+                return False
+            elif self._state == PENDING:
+                self._state = RUNNING
+                return True
+            else:
+                LOGGER.critical('Future %s in unexpected state: %s',
+                                id(self),
+                                self._state)
+                raise RuntimeError('Future in unexpected state')
+
+    def set_result(self, result):
+        """Sets the return value of work associated with the future.
+
+        Should only be used by Executor implementations and unit tests.
+        """
+        with self._condition:
+            self._result = result
+            self._state = FINISHED
+            for waiter in self._waiters:
+                waiter.add_result(self)
+            self._condition.notify_all()
+        self._invoke_callbacks()
+
+    def set_exception_info(self, exception, traceback):
+        """Sets the result of the future as being the given exception
+        and traceback.
+
+        Should only be used by Executor implementations and unit tests.
+        """
+        with self._condition:
+            self._exception = exception
+            self._traceback = traceback
+            self._state = FINISHED
+            for waiter in self._waiters:
+                waiter.add_exception(self)
+            self._condition.notify_all()
+        self._invoke_callbacks()
+
+    def set_exception(self, exception):
+        """Sets the result of the future as being the given exception.
+
+        Should only be used by Executor implementations and unit tests.
+        """
+        self.set_exception_info(exception, None)
+
+class Executor(object):
+    """This is an abstract base class for concrete asynchronous executors."""
+
+    def submit(self, fn, *args, **kwargs):
+        """Submits a callable to be executed with the given arguments.
+
+        Schedules the callable to be executed as fn(*args, **kwargs) and returns
+        a Future instance representing the execution of the callable.
+
+        Returns:
+            A Future representing the given call.
+        """
+        raise NotImplementedError()
+
+    def map(self, fn, *iterables, **kwargs):
+        """Returns a iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+
+        Returns:
+            An iterator equivalent to: map(func, *iterables) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If fn(*args) raises for any values.
+        """
+        timeout = kwargs.get('timeout')
+        if timeout is not None:
+            end_time = timeout + time.time()
+
+        fs = [self.submit(fn, *args) for args in itertools.izip(*iterables)]
+
+        # Yield must be hidden in closure so that the futures are submitted
+        # before the first iterator value is required.
+        def result_iterator():
+            try:
+                for future in fs:
+                    if timeout is None:
+                        yield future.result()
+                    else:
+                        yield future.result(end_time - time.time())
+            finally:
+                for future in fs:
+                    future.cancel()
+        return result_iterator()
+
+    def shutdown(self, wait=True):
+        """Clean-up the resources associated with the Executor.
+
+        It is safe to call this method several times. Otherwise, no other
+        methods can be called after this one.
+
+        Args:
+            wait: If True then shutdown will not return until all running
+                futures have finished executing and the resources used by the
+                executor have been reclaimed.
+        """
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.shutdown(wait=True)
+        return False

Contents/Libraries/Shared/concurrent/futures/process.py

@@ -0,0 +1,359 @@
+# Copyright 2009 Brian Quinlan. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Implements ProcessPoolExecutor.
+
+The follow diagram and text describe the data-flow through the system:
+
+|======================= In-process =====================|== Out-of-process ==|
+
++----------+     +----------+       +--------+     +-----------+    +---------+
+|          |  => | Work Ids |    => |        |  => | Call Q    | => |         |
+|          |     +----------+       |        |     +-----------+    |         |
+|          |     | ...      |       |        |     | ...       |    |         |
+|          |     | 6        |       |        |     | 5, call() |    |         |
+|          |     | 7        |       |        |     | ...       |    |         |
+| Process  |     | ...      |       | Local  |     +-----------+    | Process |
+|  Pool    |     +----------+       | Worker |                      |  #1..n  |
+| Executor |                        | Thread |                      |         |
+|          |     +----------- +     |        |     +-----------+    |         |
+|          | <=> | Work Items | <=> |        | <=  | Result Q  | <= |         |
+|          |     +------------+     |        |     +-----------+    |         |
+|          |     | 6: call()  |     |        |     | ...       |    |         |
+|          |     |    future  |     |        |     | 4, result |    |         |
+|          |     | ...        |     |        |     | 3, except |    |         |
++----------+     +------------+     +--------+     +-----------+    +---------+
+
+Executor.submit() called:
+- creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict
+- adds the id of the _WorkItem to the "Work Ids" queue
+
+Local worker thread:
+- reads work ids from the "Work Ids" queue and looks up the corresponding
+  WorkItem from the "Work Items" dict: if the work item has been cancelled then
+  it is simply removed from the dict, otherwise it is repackaged as a
+  _CallItem and put in the "Call Q". New _CallItems are put in the "Call Q"
+  until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because
+  calls placed in the "Call Q" can no longer be cancelled with Future.cancel().
+- reads _ResultItems from "Result Q", updates the future stored in the
+  "Work Items" dict and deletes the dict entry
+
+Process #1..n:
+- reads _CallItems from "Call Q", executes the calls, and puts the resulting
+  _ResultItems in "Request Q"
+"""
+
+import atexit
+from concurrent.futures import _base
+import Queue as queue
+import multiprocessing
+import threading
+import weakref
+import sys
+
+__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+
+# Workers are created as daemon threads and processes. This is done to allow the
+# interpreter to exit when there are still idle processes in a
+# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
+# allowing workers to die with the interpreter has two undesirable properties:
+#   - The workers would still be running during interpretor shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads/processes finish.
+
+_threads_queues = weakref.WeakKeyDictionary()
+_shutdown = False
+
+def _python_exit():
+    global _shutdown
+    _shutdown = True
+    items = list(_threads_queues.items()) if _threads_queues else ()
+    for t, q in items:
+        q.put(None)
+    for t, q in items:
+        t.join(sys.maxint)
+
+# Controls how many more calls than processes will be queued in the call queue.
+# A smaller number will mean that processes spend more time idle waiting for
+# work while a larger number will make Future.cancel() succeed less frequently
+# (Futures in the call queue cannot be cancelled).
+EXTRA_QUEUED_CALLS = 1
+
+class _WorkItem(object):
+    def __init__(self, future, fn, args, kwargs):
+        self.future = future
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+class _ResultItem(object):
+    def __init__(self, work_id, exception=None, result=None):
+        self.work_id = work_id
+        self.exception = exception
+        self.result = result
+
+class _CallItem(object):
+    def __init__(self, work_id, fn, args, kwargs):
+        self.work_id = work_id
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+def _process_worker(call_queue, result_queue):
+    """Evaluates calls from call_queue and places the results in result_queue.
+
+    This worker is run in a separate process.
+
+    Args:
+        call_queue: A multiprocessing.Queue of _CallItems that will be read and
+            evaluated by the worker.
+        result_queue: A multiprocessing.Queue of _ResultItems that will written
+            to by the worker.
+        shutdown: A multiprocessing.Event that will be set as a signal to the
+            worker that it should exit when call_queue is empty.
+    """
+    while True:
+        call_item = call_queue.get(block=True)
+        if call_item is None:
+            # Wake up queue management thread
+            result_queue.put(None)
+            return
+        try:
+            r = call_item.fn(*call_item.args, **call_item.kwargs)
+        except BaseException:
+            e = sys.exc_info()[1]
+            result_queue.put(_ResultItem(call_item.work_id,
+                                         exception=e))
+        else:
+            result_queue.put(_ResultItem(call_item.work_id,
+                                         result=r))
+
+def _add_call_item_to_queue(pending_work_items,
+                            work_ids,
+                            call_queue):
+    """Fills call_queue with _WorkItems from pending_work_items.
+
+    This function never blocks.
+
+    Args:
+        pending_work_items: A dict mapping work ids to _WorkItems e.g.
+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
+        work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
+            are consumed and the corresponding _WorkItems from
+            pending_work_items are transformed into _CallItems and put in
+            call_queue.
+        call_queue: A multiprocessing.Queue that will be filled with _CallItems
+            derived from _WorkItems.
+    """
+    while True:
+        if call_queue.full():
+            return
+        try:
+            work_id = work_ids.get(block=False)
+        except queue.Empty:
+            return
+        else:
+            work_item = pending_work_items[work_id]
+
+            if work_item.future.set_running_or_notify_cancel():
+                call_queue.put(_CallItem(work_id,
+                                         work_item.fn,
+                                         work_item.args,
+                                         work_item.kwargs),
+                               block=True)
+            else:
+                del pending_work_items[work_id]
+                continue
+
+def _queue_management_worker(executor_reference,
+                             processes,
+                             pending_work_items,
+                             work_ids_queue,
+                             call_queue,
+                             result_queue):
+    """Manages the communication between this process and the worker processes.
+
+    This function is run in a local thread.
+
+    Args:
+        executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
+            this thread. Used to determine if the ProcessPoolExecutor has been
+            garbage collected and that this function can exit.
+        process: A list of the multiprocessing.Process instances used as
+            workers.
+        pending_work_items: A dict mapping work ids to _WorkItems e.g.
+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
+        work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
+        call_queue: A multiprocessing.Queue that will be filled with _CallItems
+            derived from _WorkItems for processing by the process workers.
+        result_queue: A multiprocessing.Queue of _ResultItems generated by the
+            process workers.
+    """
+    nb_shutdown_processes = [0]
+    def shutdown_one_process():
+        """Tell a worker to terminate, which will in turn wake us again"""
+        call_queue.put(None)
+        nb_shutdown_processes[0] += 1
+    while True:
+        _add_call_item_to_queue(pending_work_items,
+                                work_ids_queue,
+                                call_queue)
+
+        result_item = result_queue.get(block=True)
+        if result_item is not None:
+            work_item = pending_work_items[result_item.work_id]
+            del pending_work_items[result_item.work_id]
+
+            if result_item.exception:
+                work_item.future.set_exception(result_item.exception)
+            else:
+                work_item.future.set_result(result_item.result)
+            # Delete references to object. See issue16284
+            del work_item
+        # Check whether we should start shutting down.
+        executor = executor_reference()
+        # No more work items can be added if:
+        #   - The interpreter is shutting down OR
+        #   - The executor that owns this worker has been collected OR
+        #   - The executor that owns this worker has been shutdown.
+        if _shutdown or executor is None or executor._shutdown_thread:
+            # Since no new work items can be added, it is safe to shutdown
+            # this thread if there are no pending work items.
+            if not pending_work_items:
+                while nb_shutdown_processes[0] < len(processes):
+                    shutdown_one_process()
+                # If .join() is not called on the created processes then
+                # some multiprocessing.Queue methods may deadlock on Mac OS
+                # X.
+                for p in processes:
+                    p.join()
+                call_queue.close()
+                return
+        del executor
+
+_system_limits_checked = False
+_system_limited = None
+def _check_system_limits():
+    global _system_limits_checked, _system_limited
+    if _system_limits_checked:
+        if _system_limited:
+            raise NotImplementedError(_system_limited)
+    _system_limits_checked = True
+    try:
+        import os
+        nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
+    except (AttributeError, ValueError):
+        # sysconf not available or setting not available
+        return
+    if nsems_max == -1:
+        # indetermine limit, assume that limit is determined
+        # by available memory only
+        return
+    if nsems_max >= 256:
+        # minimum number of semaphores available
+        # according to POSIX
+        return
+    _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
+    raise NotImplementedError(_system_limited)
+
+class ProcessPoolExecutor(_base.Executor):
+    def __init__(self, max_workers=None):
+        """Initializes a new ProcessPoolExecutor instance.
+
+        Args:
+            max_workers: The maximum number of processes that can be used to
+                execute the given calls. If None or not given then as many
+                worker processes will be created as the machine has processors.
+        """
+        _check_system_limits()
+
+        if max_workers is None:
+            self._max_workers = multiprocessing.cpu_count()
+        else:
+            self._max_workers = max_workers
+
+        # Make the call queue slightly larger than the number of processes to
+        # prevent the worker processes from idling. But don't make it too big
+        # because futures in the call queue cannot be cancelled.
+        self._call_queue = multiprocessing.Queue(self._max_workers +
+                                                 EXTRA_QUEUED_CALLS)
+        self._result_queue = multiprocessing.Queue()
+        self._work_ids = queue.Queue()
+        self._queue_management_thread = None
+        self._processes = set()
+
+        # Shutdown is a two-step process.
+        self._shutdown_thread = False
+        self._shutdown_lock = threading.Lock()
+        self._queue_count = 0
+        self._pending_work_items = {}
+
+    def _start_queue_management_thread(self):
+        # When the executor gets lost, the weakref callback will wake up
+        # the queue management thread.
+        def weakref_cb(_, q=self._result_queue):
+            q.put(None)
+        if self._queue_management_thread is None:
+            self._queue_management_thread = threading.Thread(
+                    target=_queue_management_worker,
+                    args=(weakref.ref(self, weakref_cb),
+                          self._processes,
+                          self._pending_work_items,
+                          self._work_ids,
+                          self._call_queue,
+                          self._result_queue))
+            self._queue_management_thread.daemon = True
+            self._queue_management_thread.start()
+            _threads_queues[self._queue_management_thread] = self._result_queue
+
+    def _adjust_process_count(self):
+        for _ in range(len(self._processes), self._max_workers):
+            p = multiprocessing.Process(
+                    target=_process_worker,
+                    args=(self._call_queue,
+                          self._result_queue))
+            p.start()
+            self._processes.add(p)
+
+    def submit(self, fn, *args, **kwargs):
+        with self._shutdown_lock:
+            if self._shutdown_thread:
+                raise RuntimeError('cannot schedule new futures after shutdown')
+
+            f = _base.Future()
+            w = _WorkItem(f, fn, args, kwargs)
+
+            self._pending_work_items[self._queue_count] = w
+            self._work_ids.put(self._queue_count)
+            self._queue_count += 1
+            # Wake up queue management thread
+            self._result_queue.put(None)
+
+            self._start_queue_management_thread()
+            self._adjust_process_count()
+            return f
+    submit.__doc__ = _base.Executor.submit.__doc__
+
+    def shutdown(self, wait=True):
+        with self._shutdown_lock:
+            self._shutdown_thread = True
+        if self._queue_management_thread:
+            # Wake up queue management thread
+            self._result_queue.put(None)
+            if wait:
+                self._queue_management_thread.join(sys.maxint)
+        # To reduce the risk of openning too many files, remove references to
+        # objects that use file descriptors.
+        self._queue_management_thread = None
+        self._call_queue = None
+        self._result_queue = None
+        self._processes = None
+    shutdown.__doc__ = _base.Executor.shutdown.__doc__
+
+atexit.register(_python_exit)

Contents/Libraries/Shared/concurrent/futures/thread.py

@@ -0,0 +1,134 @@
+# Copyright 2009 Brian Quinlan. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Implements ThreadPoolExecutor."""
+
+import atexit
+from concurrent.futures import _base
+import Queue as queue
+import threading
+import weakref
+import sys
+
+__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+
+# Workers are created as daemon threads. This is done to allow the interpreter
+# to exit when there are still idle threads in a ThreadPoolExecutor's thread
+# pool (i.e. shutdown() was not called). However, allowing workers to die with
+# the interpreter has two undesirable properties:
+#   - The workers would still be running during interpretor shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads finish.
+
+_threads_queues = weakref.WeakKeyDictionary()
+_shutdown = False
+
+def _python_exit():
+    global _shutdown
+    _shutdown = True
+    items = list(_threads_queues.items()) if _threads_queues else ()
+    for t, q in items:
+        q.put(None)
+    for t, q in items:
+        t.join(sys.maxint)
+
+atexit.register(_python_exit)
+
+class _WorkItem(object):
+    def __init__(self, future, fn, args, kwargs):
+        self.future = future
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+    def run(self):
+        if not self.future.set_running_or_notify_cancel():
+            return
+
+        try:
+            result = self.fn(*self.args, **self.kwargs)
+        except BaseException:
+            e, tb = sys.exc_info()[1:]
+            self.future.set_exception_info(e, tb)
+        else:
+            self.future.set_result(result)
+
+def _worker(executor_reference, work_queue):
+    try:
+        while True:
+            work_item = work_queue.get(block=True)
+            if work_item is not None:
+                work_item.run()
+                # Delete references to object. See issue16284
+                del work_item
+                continue
+            executor = executor_reference()
+            # Exit if:
+            #   - The interpreter is shutting down OR
+            #   - The executor that owns the worker has been collected OR
+            #   - The executor that owns the worker has been shutdown.
+            if _shutdown or executor is None or executor._shutdown:
+                # Notice other workers
+                work_queue.put(None)
+                return
+            del executor
+    except BaseException:
+        _base.LOGGER.critical('Exception in worker', exc_info=True)
+
+class ThreadPoolExecutor(_base.Executor):
+    def __init__(self, max_workers):
+        """Initializes a new ThreadPoolExecutor instance.
+
+        Args:
+            max_workers: The maximum number of threads that can be used to
+                execute the given calls.
+        """
+        self._max_workers = max_workers
+        self._work_queue = queue.Queue()
+        self._threads = set()
+        self._shutdown = False
+        self._shutdown_lock = threading.Lock()
+
+    def submit(self, fn, *args, **kwargs):
+        with self._shutdown_lock:
+            if self._shutdown:
+                raise RuntimeError('cannot schedule new futures after shutdown')
+
+            f = _base.Future()
+            w = _WorkItem(f, fn, args, kwargs)
+
+            self._work_queue.put(w)
+            self._adjust_thread_count()
+            return f
+    submit.__doc__ = _base.Executor.submit.__doc__
+
+    def _adjust_thread_count(self):
+        # When the executor gets lost, the weakref callback will wake up
+        # the worker threads.
+        def weakref_cb(_, q=self._work_queue):
+            q.put(None)
+        # TODO(bquinlan): Should avoid creating new threads if there are more
+        # idle threads than items in the work queue.
+        if len(self._threads) < self._max_workers:
+            t = threading.Thread(target=_worker,
+                                 args=(weakref.ref(self, weakref_cb),
+                                       self._work_queue))
+            t.daemon = True
+            t.start()
+            self._threads.add(t)
+            _threads_queues[t] = self._work_queue
+
+    def shutdown(self, wait=True):
+        with self._shutdown_lock:
+            self._shutdown = True
+            self._work_queue.put(None)
+        if wait:
+            for t in self._threads:
+                t.join(sys.maxint)
+    shutdown.__doc__ = _base.Executor.shutdown.__doc__

Contents/Libraries/Shared/dateutil/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__version__ = "2.6.0"

Contents/Libraries/Shared/dateutil/_common.py

@@ -0,0 +1,33 @@
+"""
+Common code used in multiple modules.
+"""
+
+class weekday(object):
+    __slots__ = ["weekday", "n"]
+
+    def __init__(self, weekday, n=None):
+        self.weekday = weekday
+        self.n = n
+
+    def __call__(self, n):
+        if n == self.n:
+            return self
+        else:
+            return self.__class__(self.weekday, n)
+
+    def __eq__(self, other):
+        try:
+            if self.weekday != other.weekday or self.n != other.n:
+                return False
+        except AttributeError:
+            return False
+        return True
+
+    __hash__ = None
+
+    def __repr__(self):
+        s = ("MO", "TU", "WE", "TH", "FR", "SA", "SU")[self.weekday]
+        if not self.n:
+            return s
+        else:
+            return "%s(%+d)" % (s, self.n)

Contents/Libraries/Shared/dateutil/easter.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a generic easter computing method for any given year, using
+Western, Orthodox or Julian algorithms.
+"""
+
+import datetime
+
+__all__ = ["easter", "EASTER_JULIAN", "EASTER_ORTHODOX", "EASTER_WESTERN"]
+
+EASTER_JULIAN = 1
+EASTER_ORTHODOX = 2
+EASTER_WESTERN = 3
+
+
+def easter(year, method=EASTER_WESTERN):
+    """
+    This method was ported from the work done by GM Arts,
+    on top of the algorithm by Claus Tondering, which was
+    based in part on the algorithm of Ouding (1940), as
+    quoted in "Explanatory Supplement to the Astronomical
+    Almanac", P.  Kenneth Seidelmann, editor.
+
+    This algorithm implements three different easter
+    calculation methods:
+
+    1 - Original calculation in Julian calendar, valid in
+        dates after 326 AD
+    2 - Original method, with date converted to Gregorian
+        calendar, valid in years 1583 to 4099
+    3 - Revised method, in Gregorian calendar, valid in
+        years 1583 to 4099 as well
+
+    These methods are represented by the constants:
+
+    * ``EASTER_JULIAN   = 1``
+    * ``EASTER_ORTHODOX = 2``
+    * ``EASTER_WESTERN  = 3``
+
+    The default method is method 3.
+
+    More about the algorithm may be found at:
+
+    http://users.chariot.net.au/~gmarts/eastalg.htm
+
+    and
+
+    http://www.tondering.dk/claus/calendar.html
+
+    """
+
+    if not (1 <= method <= 3):
+        raise ValueError("invalid method")
+
+    # g - Golden year - 1
+    # c - Century
+    # h - (23 - Epact) mod 30
+    # i - Number of days from March 21 to Paschal Full Moon
+    # j - Weekday for PFM (0=Sunday, etc)
+    # p - Number of days from March 21 to Sunday on or before PFM
+    #     (-6 to 28 methods 1 & 3, to 56 for method 2)
+    # e - Extra days to add for method 2 (converting Julian
+    #     date to Gregorian date)
+
+    y = year
+    g = y % 19
+    e = 0
+    if method < 3:
+        # Old method
+        i = (19*g + 15) % 30
+        j = (y + y//4 + i) % 7
+        if method == 2:
+            # Extra dates to convert Julian to Gregorian date
+            e = 10
+            if y > 1600:
+                e = e + y//100 - 16 - (y//100 - 16)//4
+    else:
+        # New method
+        c = y//100
+        h = (c - c//4 - (8*c + 13)//25 + 19*g + 15) % 30
+        i = h - (h//28)*(1 - (h//28)*(29//(h + 1))*((21 - g)//11))
+        j = (y + y//4 + i + 2 - c + c//4) % 7
+
+    # p can be from -6 to 56 corresponding to dates 22 March to 23 May
+    # (later dates apply to method 2, although 23 May never actually occurs)
+    p = i - j + e
+    d = 1 + (p + 27 + (p + 6)//40) % 31
+    m = 3 + (p + 26)//30
+    return datetime.date(int(y), int(m), int(d))

Contents/Libraries/Shared/dateutil/relativedelta.py

@@ -0,0 +1,531 @@
+# -*- coding: utf-8 -*-
+import datetime
+import calendar
+
+import operator
+from math import copysign
+
+from six import integer_types
+from warnings import warn
+
+from ._common import weekday
+
+MO, TU, WE, TH, FR, SA, SU = weekdays = tuple([weekday(x) for x in range(7)])
+
+__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"]
+
+
+class relativedelta(object):
+    """
+    The relativedelta type is based on the specification of the excellent
+    work done by M.-A. Lemburg in his
+    `mx.DateTime <http://www.egenix.com/files/python/mxDateTime.html>`_ extension.
+    However, notice that this type does *NOT* implement the same algorithm as
+    his work. Do *NOT* expect it to behave like mx.DateTime's counterpart.
+
+    There are two different ways to build a relativedelta instance. The
+    first one is passing it two date/datetime classes::
+
+        relativedelta(datetime1, datetime2)
+
+    The second one is passing it any number of the following keyword arguments::
+
+        relativedelta(arg1=x,arg2=y,arg3=z...)
+
+        year, month, day, hour, minute, second, microsecond:
+            Absolute information (argument is singular); adding or subtracting a
+            relativedelta with absolute information does not perform an aritmetic
+            operation, but rather REPLACES the corresponding value in the
+            original datetime with the value(s) in relativedelta.
+
+        years, months, weeks, days, hours, minutes, seconds, microseconds:
+            Relative information, may be negative (argument is plural); adding
+            or subtracting a relativedelta with relative information performs
+            the corresponding aritmetic operation on the original datetime value
+            with the information in the relativedelta.
+
+        weekday:
+            One of the weekday instances (MO, TU, etc). These instances may
+            receive a parameter N, specifying the Nth weekday, which could
+            be positive or negative (like MO(+1) or MO(-2). Not specifying
+            it is the same as specifying +1. You can also use an integer,
+            where 0=MO.
+
+        leapdays:
+            Will add given days to the date found, if year is a leap
+            year, and the date found is post 28 of february.
+
+        yearday, nlyearday:
+            Set the yearday or the non-leap year day (jump leap days).
+            These are converted to day/month/leapdays information.
+
+    Here is the behavior of operations with relativedelta:
+
+    1. Calculate the absolute year, using the 'year' argument, or the
+       original datetime year, if the argument is not present.
+
+    2. Add the relative 'years' argument to the absolute year.
+
+    3. Do steps 1 and 2 for month/months.
+
+    4. Calculate the absolute day, using the 'day' argument, or the
+       original datetime day, if the argument is not present. Then,
+       subtract from the day until it fits in the year and month
+       found after their operations.
+
+    5. Add the relative 'days' argument to the absolute day. Notice
+       that the 'weeks' argument is multiplied by 7 and added to
+       'days'.
+
+    6. Do steps 1 and 2 for hour/hours, minute/minutes, second/seconds,
+       microsecond/microseconds.
+
+    7. If the 'weekday' argument is present, calculate the weekday,
+       with the given (wday, nth) tuple. wday is the index of the
+       weekday (0-6, 0=Mon), and nth is the number of weeks to add
+       forward or backward, depending on its signal. Notice that if
+       the calculated date is already Monday, for example, using
+       (0, 1) or (0, -1) won't change the day.
+    """
+
+    def __init__(self, dt1=None, dt2=None,
+                 years=0, months=0, days=0, leapdays=0, weeks=0,
+                 hours=0, minutes=0, seconds=0, microseconds=0,
+                 year=None, month=None, day=None, weekday=None,
+                 yearday=None, nlyearday=None,
+                 hour=None, minute=None, second=None, microsecond=None):
+
+        # Check for non-integer values in integer-only quantities
+        if any(x is not None and x != int(x) for x in (years, months)):
+            raise ValueError("Non-integer years and months are "
+                             "ambiguous and not currently supported.")
+
+        if dt1 and dt2:
+            # datetime is a subclass of date. So both must be date
+            if not (isinstance(dt1, datetime.date) and
+                    isinstance(dt2, datetime.date)):
+                raise TypeError("relativedelta only diffs datetime/date")
+
+            # We allow two dates, or two datetimes, so we coerce them to be
+            # of the same type
+            if (isinstance(dt1, datetime.datetime) !=
+                    isinstance(dt2, datetime.datetime)):
+                if not isinstance(dt1, datetime.datetime):
+                    dt1 = datetime.datetime.fromordinal(dt1.toordinal())
+                elif not isinstance(dt2, datetime.datetime):
+                    dt2 = datetime.datetime.fromordinal(dt2.toordinal())
+
+            self.years = 0
+            self.months = 0
+            self.days = 0
+            self.leapdays = 0
+            self.hours = 0
+            self.minutes = 0
+            self.seconds = 0
+            self.microseconds = 0
+            self.year = None
+            self.month = None
+            self.day = None
+            self.weekday = None
+            self.hour = None
+            self.minute = None
+            self.second = None
+            self.microsecond = None
+            self._has_time = 0
+
+            # Get year / month delta between the two
+            months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month)
+            self._set_months(months)
+
+            # Remove the year/month delta so the timedelta is just well-defined
+            # time units (seconds, days and microseconds)
+            dtm = self.__radd__(dt2)
+
+            # If we've overshot our target, make an adjustment
+            if dt1 < dt2:
+                compare = operator.gt
+                increment = 1
+            else:
+                compare = operator.lt
+                increment = -1
+
+            while compare(dt1, dtm):
+                months += increment
+                self._set_months(months)
+                dtm = self.__radd__(dt2)
+
+            # Get the timedelta between the "months-adjusted" date and dt1
+            delta = dt1 - dtm
+            self.seconds = delta.seconds + delta.days * 86400
+            self.microseconds = delta.microseconds
+        else:
+            # Relative information
+            self.years = years
+            self.months = months
+            self.days = days + weeks * 7
+            self.leapdays = leapdays
+            self.hours = hours
+            self.minutes = minutes
+            self.seconds = seconds
+            self.microseconds = microseconds
+
+            # Absolute information
+            self.year = year
+            self.month = month
+            self.day = day
+            self.hour = hour
+            self.minute = minute
+            self.second = second
+            self.microsecond = microsecond
+
+            if any(x is not None and int(x) != x
+                   for x in (year, month, day, hour,
+                             minute, second, microsecond)):
+                # For now we'll deprecate floats - later it'll be an error.
+                warn("Non-integer value passed as absolute information. " +
+                     "This is not a well-defined condition and will raise " +
+                     "errors in future versions.", DeprecationWarning)
+
+
+            if isinstance(weekday, integer_types):
+                self.weekday = weekdays[weekday]
+            else:
+                self.weekday = weekday
+
+            yday = 0
+            if nlyearday:
+                yday = nlyearday
+            elif yearday:
+                yday = yearday
+                if yearday > 59:
+                    self.leapdays = -1
+            if yday:
+                ydayidx = [31, 59, 90, 120, 151, 181, 212,
+                           243, 273, 304, 334, 366]
+                for idx, ydays in enumerate(ydayidx):
+                    if yday <= ydays:
+                        self.month = idx+1
+                        if idx == 0:
+                            self.day = yday
+                        else:
+                            self.day = yday-ydayidx[idx-1]
+                        break
+                else:
+                    raise ValueError("invalid year day (%d)" % yday)
+
+        self._fix()
+
+    def _fix(self):
+        if abs(self.microseconds) > 999999:
+            s = _sign(self.microseconds)
+            div, mod = divmod(self.microseconds * s, 1000000)
+            self.microseconds = mod * s
+            self.seconds += div * s
+        if abs(self.seconds) > 59:
+            s = _sign(self.seconds)
+            div, mod = divmod(self.seconds * s, 60)
+            self.seconds = mod * s
+            self.minutes += div * s
+        if abs(self.minutes) > 59:
+            s = _sign(self.minutes)
+            div, mod = divmod(self.minutes * s, 60)
+            self.minutes = mod * s
+            self.hours += div * s
+        if abs(self.hours) > 23:
+            s = _sign(self.hours)
+            div, mod = divmod(self.hours * s, 24)
+            self.hours = mod * s
+            self.days += div * s
+        if abs(self.months) > 11:
+            s = _sign(self.months)
+            div, mod = divmod(self.months * s, 12)
+            self.months = mod * s
+            self.years += div * s
+        if (self.hours or self.minutes or self.seconds or self.microseconds
+                or self.hour is not None or self.minute is not None or
+                self.second is not None or self.microsecond is not None):
+            self._has_time = 1
+        else:
+            self._has_time = 0
+
+    @property
+    def weeks(self):
+        return self.days // 7
+    @weeks.setter
+    def weeks(self, value):
+        self.days = self.days - (self.weeks * 7) + value * 7
+
+    def _set_months(self, months):
+        self.months = months
+        if abs(self.months) > 11:
+            s = _sign(self.months)
+            div, mod = divmod(self.months * s, 12)
+            self.months = mod * s
+            self.years = div * s
+        else:
+            self.years = 0
+
+    def normalized(self):
+        """
+        Return a version of this object represented entirely using integer
+        values for the relative attributes.
+
+        >>> relativedelta(days=1.5, hours=2).normalized()
+        relativedelta(days=1, hours=14)
+
+        :return:
+            Returns a :class:`dateutil.relativedelta.relativedelta` object.
+        """
+        # Cascade remainders down (rounding each to roughly nearest microsecond)
+        days = int(self.days)
+
+        hours_f = round(self.hours + 24 * (self.days - days), 11)
+        hours = int(hours_f)
+
+        minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
+        minutes = int(minutes_f)
+
+        seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
+        seconds = int(seconds_f)
+
+        microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))
+
+        # Constructor carries overflow back up with call to _fix()
+        return self.__class__(years=self.years, months=self.months,
+                              days=days, hours=hours, minutes=minutes,
+                              seconds=seconds, microseconds=microseconds,
+                              leapdays=self.leapdays, year=self.year,
+                              month=self.month, day=self.day,
+                              weekday=self.weekday, hour=self.hour,
+                              minute=self.minute, second=self.second,
+                              microsecond=self.microsecond)
+
+    def __add__(self, other):
+        if isinstance(other, relativedelta):
+            return self.__class__(years=other.years + self.years,
+                                 months=other.months + self.months,
+                                 days=other.days + self.days,
+                                 hours=other.hours + self.hours,
+                                 minutes=other.minutes + self.minutes,
+                                 seconds=other.seconds + self.seconds,
+                                 microseconds=(other.microseconds +
+                                               self.microseconds),
+                                 leapdays=other.leapdays or self.leapdays,
+                                 year=other.year or self.year,
+                                 month=other.month or self.month,
+                                 day=other.day or self.day,
+                                 weekday=other.weekday or self.weekday,
+                                 hour=other.hour or self.hour,
+                                 minute=other.minute or self.minute,
+                                 second=other.second or self.second,
+                                 microsecond=(other.microsecond or
+                                              self.microsecond))
+        if isinstance(other, datetime.timedelta):
+            return self.__class__(years=self.years,
+                                  months=self.months,
+                                  days=self.days + other.days,
+                                  hours=self.hours,
+                                  minutes=self.minutes,
+                                  seconds=self.seconds + other.seconds,
+                                  microseconds=self.microseconds + other.microseconds,
+                                  leapdays=self.leapdays,
+                                  year=self.year,
+                                  month=self.month,
+                                  day=self.day,
+                                  weekday=self.weekday,
+                                  hour=self.hour,
+                                  minute=self.minute,
+                                  second=self.second,
+                                  microsecond=self.microsecond)
+        if not isinstance(other, datetime.date):
+            return NotImplemented
+        elif self._has_time and not isinstance(other, datetime.datetime):
+            other = datetime.datetime.fromordinal(other.toordinal())
+        year = (self.year or other.year)+self.years
+        month = self.month or other.month
+        if self.months:
+            assert 1 <= abs(self.months) <= 12
+            month += self.months
+            if month > 12:
+                year += 1
+                month -= 12
+            elif month < 1:
+                year -= 1
+                month += 12
+        day = min(calendar.monthrange(year, month)[1],
+                  self.day or other.day)
+        repl = {"year": year, "month": month, "day": day}
+        for attr in ["hour", "minute", "second", "microsecond"]:
+            value = getattr(self, attr)
+            if value is not None:
+                repl[attr] = value
+        days = self.days
+        if self.leapdays and month > 2 and calendar.isleap(year):
+            days += self.leapdays
+        ret = (other.replace(**repl)
+               + datetime.timedelta(days=days,
+                                    hours=self.hours,
+                                    minutes=self.minutes,
+                                    seconds=self.seconds,
+                                    microseconds=self.microseconds))
+        if self.weekday:
+            weekday, nth = self.weekday.weekday, self.weekday.n or 1
+            jumpdays = (abs(nth) - 1) * 7
+            if nth > 0:
+                jumpdays += (7 - ret.weekday() + weekday) % 7
+            else:
+                jumpdays += (ret.weekday() - weekday) % 7
+                jumpdays *= -1
+            ret += datetime.timedelta(days=jumpdays)
+        return ret
+
+    def __radd__(self, other):
+        return self.__add__(other)
+
+    def __rsub__(self, other):
+        return self.__neg__().__radd__(other)
+
+    def __sub__(self, other):
+        if not isinstance(other, relativedelta):
+            return NotImplemented   # In case the other object defines __rsub__
+        return self.__class__(years=self.years - other.years,
+                             months=self.months - other.months,
+                             days=self.days - other.days,
+                             hours=self.hours - other.hours,
+                             minutes=self.minutes - other.minutes,
+                             seconds=self.seconds - other.seconds,
+                             microseconds=self.microseconds - other.microseconds,
+                             leapdays=self.leapdays or other.leapdays,
+                             year=self.year or other.year,
+                             month=self.month or other.month,
+                             day=self.day or other.day,
+                             weekday=self.weekday or other.weekday,
+                             hour=self.hour or other.hour,
+                             minute=self.minute or other.minute,
+                             second=self.second or other.second,
+                             microsecond=self.microsecond or other.microsecond)
+
+    def __neg__(self):
+        return self.__class__(years=-self.years,
+                             months=-self.months,
+                             days=-self.days,
+                             hours=-self.hours,
+                             minutes=-self.minutes,
+                             seconds=-self.seconds,
+                             microseconds=-self.microseconds,
+                             leapdays=self.leapdays,
+                             year=self.year,
+                             month=self.month,
+                             day=self.day,
+                             weekday=self.weekday,
+                             hour=self.hour,
+                             minute=self.minute,
+                             second=self.second,
+                             microsecond=self.microsecond)
+
+    def __bool__(self):
+        return not (not self.years and
+                    not self.months and
+                    not self.days and
+                    not self.hours and
+                    not self.minutes and
+                    not self.seconds and
+                    not self.microseconds and
+                    not self.leapdays and
+                    self.year is None and
+                    self.month is None and
+                    self.day is None and
+                    self.weekday is None and
+                    self.hour is None and
+                    self.minute is None and
+                    self.second is None and
+                    self.microsecond is None)
+    # Compatibility with Python 2.x
+    __nonzero__ = __bool__
+
+    def __mul__(self, other):
+        try:
+            f = float(other)
+        except TypeError:
+            return NotImplemented
+
+        return self.__class__(years=int(self.years * f),
+                             months=int(self.months * f),
+                             days=int(self.days * f),
+                             hours=int(self.hours * f),
+                             minutes=int(self.minutes * f),
+                             seconds=int(self.seconds * f),
+                             microseconds=int(self.microseconds * f),
+                             leapdays=self.leapdays,
+                             year=self.year,
+                             month=self.month,
+                             day=self.day,
+                             weekday=self.weekday,
+                             hour=self.hour,
+                             minute=self.minute,
+                             second=self.second,
+                             microsecond=self.microsecond)
+
+    __rmul__ = __mul__
+
+    def __eq__(self, other):
+        if not isinstance(other, relativedelta):
+            return NotImplemented
+        if self.weekday or other.weekday:
+            if not self.weekday or not other.weekday:
+                return False
+            if self.weekday.weekday != other.weekday.weekday:
+                return False
+            n1, n2 = self.weekday.n, other.weekday.n
+            if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)):
+                return False
+        return (self.years == other.years and
+                self.months == other.months and
+                self.days == other.days and
+                self.hours == other.hours and
+                self.minutes == other.minutes and
+                self.seconds == other.seconds and
+                self.microseconds == other.microseconds and
+                self.leapdays == other.leapdays and
+                self.year == other.year and
+                self.month == other.month and
+                self.day == other.day and
+                self.hour == other.hour and
+                self.minute == other.minute and
+                self.second == other.second and
+                self.microsecond == other.microsecond)
+
+    __hash__ = None
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __div__(self, other):
+        try:
+            reciprocal = 1 / float(other)
+        except TypeError:
+            return NotImplemented
+
+        return self.__mul__(reciprocal)
+
+    __truediv__ = __div__
+
+    def __repr__(self):
+        l = []
+        for attr in ["years", "months", "days", "leapdays",
+                     "hours", "minutes", "seconds", "microseconds"]:
+            value = getattr(self, attr)
+            if value:
+                l.append("{attr}={value:+g}".format(attr=attr, value=value))
+        for attr in ["year", "month", "day", "weekday",
+                     "hour", "minute", "second", "microsecond"]:
+            value = getattr(self, attr)
+            if value is not None:
+                l.append("{attr}={value}".format(attr=attr, value=repr(value)))
+        return "{classname}({attrs})".format(classname=self.__class__.__name__,
+                                             attrs=", ".join(l))
+
+def _sign(x):
+    return int(copysign(1, x))
+
+# vim:ts=4:sw=4:et

Contents/Libraries/Shared/dateutil/test/_common.py

@@ -0,0 +1,288 @@
+from __future__ import unicode_literals
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+import os
+import datetime
+import time
+import subprocess
+import warnings
+import tempfile
+import pickle
+
+
+class WarningTestMixin(object):
+    # Based on https://stackoverflow.com/a/12935176/467366
+    class _AssertWarnsContext(warnings.catch_warnings):
+        def __init__(self, expected_warnings, parent, **kwargs):
+            super(WarningTestMixin._AssertWarnsContext, self).__init__(**kwargs)
+
+            self.parent = parent
+            try:
+                self.expected_warnings = list(expected_warnings)
+            except TypeError:
+                self.expected_warnings = [expected_warnings]
+
+            self._warning_log = []
+
+        def __enter__(self, *args, **kwargs):
+            rv = super(WarningTestMixin._AssertWarnsContext, self).__enter__(*args, **kwargs)
+
+            if self._showwarning is not self._module.showwarning:
+                super_showwarning = self._module.showwarning
+            else:
+                super_showwarning = None
+
+            def showwarning(*args, **kwargs):
+                if super_showwarning is not None:
+                    super_showwarning(*args, **kwargs)
+
+                self._warning_log.append(warnings.WarningMessage(*args, **kwargs))
+
+            self._module.showwarning = showwarning
+            return rv
+
+        def __exit__(self, *args, **kwargs):
+            super(WarningTestMixin._AssertWarnsContext, self).__exit__(self, *args, **kwargs)
+
+            self.parent.assertTrue(any(issubclass(item.category, warning)
+                                       for warning in self.expected_warnings
+                                       for item in self._warning_log))
+
+    def assertWarns(self, warning, callable=None, *args, **kwargs):
+        warnings.simplefilter('always')
+        context = self.__class__._AssertWarnsContext(warning, self)
+        if callable is None:
+            return context
+        else:
+            with context:
+                callable(*args, **kwargs)
+
+
+class PicklableMixin(object):
+    def _get_nobj_bytes(self, obj, dump_kwargs, load_kwargs):
+        """
+        Pickle and unpickle an object using ``pickle.dumps`` / ``pickle.loads``
+        """
+        pkl = pickle.dumps(obj, **dump_kwargs)
+        return pickle.loads(pkl, **load_kwargs)
+
+    def _get_nobj_file(self, obj, dump_kwargs, load_kwargs):
+        """
+        Pickle and unpickle an object using ``pickle.dump`` / ``pickle.load`` on
+        a temporary file.
+        """
+        with tempfile.TemporaryFile('w+b') as pkl:
+            pickle.dump(obj, pkl, **dump_kwargs)
+            pkl.seek(0)         # Reset the file to the beginning to read it
+            nobj = pickle.load(pkl, **load_kwargs)
+
+        return nobj
+
+    def assertPicklable(self, obj, asfile=False,
+                        dump_kwargs=None, load_kwargs=None):
+        """
+        Assert that an object can be pickled and unpickled. This assertion
+        assumes that the desired behavior is that the unpickled object compares
+        equal to the original object, but is not the same object.
+        """
+        get_nobj = self._get_nobj_file if asfile else self._get_nobj_bytes
+        dump_kwargs = dump_kwargs or {}
+        load_kwargs = load_kwargs or {}
+
+        nobj = get_nobj(obj, dump_kwargs, load_kwargs)
+        self.assertIsNot(obj, nobj)
+        self.assertEqual(obj, nobj)
+
+
+class TZContextBase(object):
+    """
+    Base class for a context manager which allows changing of time zones.
+
+    Subclasses may define a guard variable to either block or or allow time
+    zone changes by redefining ``_guard_var_name`` and ``_guard_allows_change``.
+    The default is that the guard variable must be affirmatively set.
+
+    Subclasses must define ``get_current_tz`` and ``set_current_tz``.
+    """
+    _guard_var_name = "DATEUTIL_MAY_CHANGE_TZ"
+    _guard_allows_change = True
+
+    def __init__(self, tzval):
+        self.tzval = tzval
+        self._old_tz = None
+
+    @classmethod
+    def tz_change_allowed(cls):
+        """
+        Class method used to query whether or not this class allows time zone
+        changes.
+        """
+        guard = bool(os.environ.get(cls._guard_var_name, False))
+
+        # _guard_allows_change gives the "default" behavior - if True, the
+        # guard is overcoming a block. If false, the guard is causing a block.
+        # Whether tz_change is allowed is therefore the XNOR of the two.
+        return guard == cls._guard_allows_change
+
+    @classmethod
+    def tz_change_disallowed_message(cls):
+        """ Generate instructions on how to allow tz changes """
+        msg = ('Changing time zone not allowed. Set {envar} to {gval} '
+               'if you would like to allow this behavior')
+
+        return msg.format(envar=cls._guard_var_name,
+                          gval=cls._guard_allows_change)
+
+    def __enter__(self):
+        if not self.tz_change_allowed():
+            raise ValueError(self.tz_change_disallowed_message())
+
+        self._old_tz = self.get_current_tz()
+        self.set_current_tz(self.tzval)
+
+    def __exit__(self, type, value, traceback):
+        if self._old_tz is not None:
+            self.set_current_tz(self._old_tz)
+
+        self._old_tz = None
+
+    def get_current_tz(self):
+        raise NotImplementedError
+
+    def set_current_tz(self):
+        raise NotImplementedError
+
+
+class TZEnvContext(TZContextBase):
+    """
+    Context manager that temporarily sets the `TZ` variable (for use on
+    *nix-like systems). Because the effect is local to the shell anyway, this
+    will apply *unless* a guard is set.
+
+    If you do not want the TZ environment variable set, you may set the
+    ``DATEUTIL_MAY_NOT_CHANGE_TZ_VAR`` variable to a truthy value.
+    """
+    _guard_var_name = "DATEUTIL_MAY_NOT_CHANGE_TZ_VAR"
+    _guard_allows_change = False
+
+    def get_current_tz(self):
+        return os.environ.get('TZ', UnsetTz)
+
+    def set_current_tz(self, tzval):
+        if tzval is UnsetTz and 'TZ' in os.environ:
+            del os.environ['TZ']
+        else:
+            os.environ['TZ'] = tzval
+
+        time.tzset()
+
+
+class TZWinContext(TZContextBase):
+    """
+    Context manager for changing local time zone on Windows.
+
+    Because the effect of this is system-wide and global, it may have
+    unintended side effect. Set the ``DATEUTIL_MAY_CHANGE_TZ`` environment
+    variable to a truthy value before using this context manager.
+    """
+    def get_current_tz(self):
+        p = subprocess.Popen(['tzutil', '/g'], stdout=subprocess.PIPE)
+
+        ctzname, err = p.communicate()
+        ctzname = ctzname.decode()     # Popen returns 
+
+        if p.returncode:
+            raise OSError('Failed to get current time zone: ' + err)
+
+        return ctzname
+
+    def set_current_tz(self, tzname):
+        p = subprocess.Popen('tzutil /s "' + tzname + '"')
+
+        out, err = p.communicate()
+
+        if p.returncode:
+            raise OSError('Failed to set current time zone: ' +
+                          (err or 'Unknown error.'))
+
+
+###
+# Compatibility functions
+
+def _total_seconds(td):
+    # Python 2.6 doesn't have a total_seconds() method on timedelta objects
+    return ((td.seconds + td.days * 86400) * 1000000 +
+            td.microseconds) // 1000000
+
+total_seconds = getattr(datetime.timedelta, 'total_seconds', _total_seconds)
+
+
+###
+# Utility classes
+class NotAValueClass(object):
+    """
+    A class analogous to NaN that has operations defined for any type.
+    """
+    def _op(self, other):
+        return self             # Operation with NotAValue returns NotAValue
+
+    def _cmp(self, other):
+        return False
+
+    __add__ = __radd__ = _op
+    __sub__ = __rsub__ = _op
+    __mul__ = __rmul__ = _op
+    __div__ = __rdiv__ = _op
+    __truediv__ = __rtruediv__ = _op
+    __floordiv__ = __rfloordiv__ = _op
+
+    __lt__ = __rlt__ = _op
+    __gt__ = __rgt__ = _op
+    __eq__ = __req__ = _op
+    __le__ = __rle__ = _op
+    __ge__ = __rge__ = _op
+
+NotAValue = NotAValueClass()
+
+
+class ComparesEqualClass(object):
+    """
+    A class that is always equal to whatever you compare it to.
+    """
+
+    def __eq__(self, other):
+        return True
+
+    def __ne__(self, other):
+        return False
+
+    def __le__(self, other):
+        return True
+
+    def __ge__(self, other):
+        return True
+
+    def __lt__(self, other):
+        return False
+
+    def __gt__(self, other):
+        return False
+
+    __req__ = __eq__
+    __rne__ = __ne__
+    __rle__ = __le__
+    __rge__ = __ge__
+    __rlt__ = __lt__
+    __rgt__ = __gt__
+
+ComparesEqual = ComparesEqualClass()
+
+class UnsetTzClass(object):
+    """ Sentinel class for unset time zone variable """
+    pass
+
+UnsetTz = UnsetTzClass()
+

Contents/Libraries/Shared/dateutil/test/test_easter.py

@@ -0,0 +1,99 @@
+from dateutil.easter import easter
+from dateutil.easter import EASTER_WESTERN, EASTER_ORTHODOX, EASTER_JULIAN
+
+from datetime import date
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+# List of easters between 1990 and 2050
+western_easter_dates = [
+    date(1990, 4, 15), date(1991, 3, 31), date(1992, 4, 19), date(1993, 4, 11),
+    date(1994, 4,  3), date(1995, 4, 16), date(1996, 4,  7), date(1997, 3, 30),
+    date(1998, 4, 12), date(1999, 4,  4),
+
+    date(2000, 4, 23), date(2001, 4, 15), date(2002, 3, 31), date(2003, 4, 20),
+    date(2004, 4, 11), date(2005, 3, 27), date(2006, 4, 16), date(2007, 4,  8),
+    date(2008, 3, 23), date(2009, 4, 12),
+
+    date(2010, 4,  4), date(2011, 4, 24), date(2012, 4,  8), date(2013, 3, 31),
+    date(2014, 4, 20), date(2015, 4,  5), date(2016, 3, 27), date(2017, 4, 16),
+    date(2018, 4,  1), date(2019, 4, 21),
+
+    date(2020, 4, 12), date(2021, 4,  4), date(2022, 4, 17), date(2023, 4,  9),
+    date(2024, 3, 31), date(2025, 4, 20), date(2026, 4,  5), date(2027, 3, 28),
+    date(2028, 4, 16), date(2029, 4,  1),
+
+    date(2030, 4, 21), date(2031, 4, 13), date(2032, 3, 28), date(2033, 4, 17),
+    date(2034, 4,  9), date(2035, 3, 25), date(2036, 4, 13), date(2037, 4,  5),
+    date(2038, 4, 25), date(2039, 4, 10),
+
+    date(2040, 4,  1), date(2041, 4, 21), date(2042, 4,  6), date(2043, 3, 29),
+    date(2044, 4, 17), date(2045, 4,  9), date(2046, 3, 25), date(2047, 4, 14),
+    date(2048, 4,  5), date(2049, 4, 18), date(2050, 4, 10)
+    ]
+
+orthodox_easter_dates = [
+    date(1990, 4, 15), date(1991, 4,  7), date(1992, 4, 26), date(1993, 4, 18),
+    date(1994, 5,  1), date(1995, 4, 23), date(1996, 4, 14), date(1997, 4, 27),
+    date(1998, 4, 19), date(1999, 4, 11),
+
+    date(2000, 4, 30), date(2001, 4, 15), date(2002, 5,  5), date(2003, 4, 27),
+    date(2004, 4, 11), date(2005, 5,  1), date(2006, 4, 23), date(2007, 4,  8),
+    date(2008, 4, 27), date(2009, 4, 19),
+
+    date(2010, 4,  4), date(2011, 4, 24), date(2012, 4, 15), date(2013, 5,  5),
+    date(2014, 4, 20), date(2015, 4, 12), date(2016, 5,  1), date(2017, 4, 16),
+    date(2018, 4,  8), date(2019, 4, 28),
+
+    date(2020, 4, 19), date(2021, 5,  2), date(2022, 4, 24), date(2023, 4, 16),
+    date(2024, 5,  5), date(2025, 4, 20), date(2026, 4, 12), date(2027, 5,  2),
+    date(2028, 4, 16), date(2029, 4,  8),
+
+    date(2030, 4, 28), date(2031, 4, 13), date(2032, 5,  2), date(2033, 4, 24),
+    date(2034, 4,  9), date(2035, 4, 29), date(2036, 4, 20), date(2037, 4,  5),
+    date(2038, 4, 25), date(2039, 4, 17),
+
+    date(2040, 5,  6), date(2041, 4, 21), date(2042, 4, 13), date(2043, 5,  3),
+    date(2044, 4, 24), date(2045, 4,  9), date(2046, 4, 29), date(2047, 4, 21),
+    date(2048, 4,  5), date(2049, 4, 25), date(2050, 4, 17)
+]
+
+# A random smattering of Julian dates.
+# Pulled values from http://www.kevinlaughery.com/east4099.html
+julian_easter_dates = [
+    date( 326, 4,  3), date( 375, 4,  5), date( 492, 4,  5), date( 552, 3, 31),
+    date( 562, 4,  9), date( 569, 4, 21), date( 597, 4, 14), date( 621, 4, 19),
+    date( 636, 3, 31), date( 655, 3, 29), date( 700, 4, 11), date( 725, 4,  8),
+    date( 750, 3, 29), date( 782, 4,  7), date( 835, 4, 18), date( 849, 4, 14),
+    date( 867, 3, 30), date( 890, 4, 12), date( 922, 4, 21), date( 934, 4,  6),
+    date(1049, 3, 26), date(1058, 4, 19), date(1113, 4,  6), date(1119, 3, 30),
+    date(1242, 4, 20), date(1255, 3, 28), date(1257, 4,  8), date(1258, 3, 24),
+    date(1261, 4, 24), date(1278, 4, 17), date(1333, 4,  4), date(1351, 4, 17),
+    date(1371, 4,  6), date(1391, 3, 26), date(1402, 3, 26), date(1412, 4,  3),
+    date(1439, 4,  5), date(1445, 3, 28), date(1531, 4,  9), date(1555, 4, 14)
+]
+
+
+class EasterTest(unittest.TestCase):
+    def testEasterWestern(self):
+        for easter_date in western_easter_dates:
+            self.assertEqual(easter_date,
+                             easter(easter_date.year, EASTER_WESTERN))
+
+    def testEasterOrthodox(self):
+        for easter_date in orthodox_easter_dates:
+            self.assertEqual(easter_date,
+                             easter(easter_date.year, EASTER_ORTHODOX))
+
+    def testEasterJulian(self):
+        for easter_date in julian_easter_dates:
+            self.assertEqual(easter_date,
+                             easter(easter_date.year, EASTER_JULIAN))
+
+    def testEasterBadMethod(self):
+        # Invalid methods raise ValueError
+        with self.assertRaises(ValueError):
+            easter(1975, 4)

Contents/Libraries/Shared/dateutil/test/test_imports.py

@@ -0,0 +1,149 @@
+import sys
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+
+class ImportEasterTest(unittest.TestCase):
+    """ Test that dateutil.easter-related imports work properly """
+
+    def testEasterDirect(self):
+        import dateutil.easter
+
+    def testEasterFrom(self):
+        from dateutil import easter
+
+    def testEasterStar(self):
+        from dateutil.easter import easter
+
+
+class ImportParserTest(unittest.TestCase):
+    """ Test that dateutil.parser-related imports work properly """
+    def testParserDirect(self):
+        import dateutil.parser
+
+    def testParserFrom(self):
+        from dateutil import parser
+
+    def testParserAll(self):
+        # All interface
+        from dateutil.parser import parse
+        from dateutil.parser import parserinfo
+
+        # Other public classes
+        from dateutil.parser import parser
+
+        for var in (parse, parserinfo, parser):
+            self.assertIsNot(var, None)
+
+
+class ImportRelativeDeltaTest(unittest.TestCase):
+    """ Test that dateutil.relativedelta-related imports work properly """
+    def testRelativeDeltaDirect(self):
+        import dateutil.relativedelta
+
+    def testRelativeDeltaFrom(self):
+        from dateutil import relativedelta
+
+    def testRelativeDeltaAll(self):
+        from dateutil.relativedelta import relativedelta
+        from dateutil.relativedelta import MO, TU, WE, TH, FR, SA, SU
+
+        for var in (relativedelta, MO, TU, WE, TH, FR, SA, SU):
+            self.assertIsNot(var, None)
+
+        # In the public interface but not in all
+        from dateutil.relativedelta import weekday
+        self.assertIsNot(weekday, None)
+
+
+class ImportRRuleTest(unittest.TestCase):
+    """ Test that dateutil.rrule related imports work properly """
+    def testRRuleDirect(self):
+        import dateutil.rrule
+
+    def testRRuleFrom(self):
+        from dateutil import rrule
+
+    def testRRuleAll(self):
+        from dateutil.rrule import rrule
+        from dateutil.rrule import rruleset
+        from dateutil.rrule import rrulestr
+        from dateutil.rrule import YEARLY, MONTHLY, WEEKLY, DAILY
+        from dateutil.rrule import HOURLY, MINUTELY, SECONDLY
+        from dateutil.rrule import MO, TU, WE, TH, FR, SA, SU
+
+        rr_all = (rrule, rruleset, rrulestr,
+                  YEARLY, MONTHLY, WEEKLY, DAILY,
+                  HOURLY, MINUTELY, SECONDLY,
+                  MO, TU, WE, TH, FR, SA, SU)
+
+        for var in rr_all:
+            self.assertIsNot(var, None)
+
+        # In the public interface but not in all
+        from dateutil.rrule import weekday
+        self.assertIsNot(weekday, None)
+
+
+class ImportTZTest(unittest.TestCase):
+    """ Test that dateutil.tz related imports work properly """
+    def testTzDirect(self):
+        import dateutil.tz
+
+    def testTzFrom(self):
+        from dateutil import tz
+
+    def testTzAll(self):
+        from dateutil.tz import tzutc
+        from dateutil.tz import tzoffset
+        from dateutil.tz import tzlocal
+        from dateutil.tz import tzfile
+        from dateutil.tz import tzrange
+        from dateutil.tz import tzstr
+        from dateutil.tz import tzical
+        from dateutil.tz import gettz
+        from dateutil.tz import tzwin
+        from dateutil.tz import tzwinlocal
+
+        tz_all = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange",
+                  "tzstr", "tzical", "gettz"]
+
+        tz_all += ["tzwin", "tzwinlocal"] if sys.platform.startswith("win") else []
+        lvars = locals()
+
+        for var in tz_all:
+            self.assertIsNot(lvars[var], None)
+
+
+@unittest.skipUnless(sys.platform.startswith('win'), "Requires Windows")
+class ImportTZWinTest(unittest.TestCase):
+    """ Test that dateutil.tzwin related imports work properly """
+    def testTzwinDirect(self):
+        import dateutil.tzwin
+
+    def testTzwinFrom(self):
+        from dateutil import tzwin
+
+    def testTzwinStar(self):
+        tzwin_all = ["tzwin", "tzwinlocal"]
+
+
+class ImportZoneInfoTest(unittest.TestCase):
+    def testZoneinfoDirect(self):
+        import dateutil.zoneinfo
+
+    def testZoneinfoFrom(self):
+        from dateutil import zoneinfo
+
+    def testZoneinfoStar(self):
+        from dateutil.zoneinfo import gettz
+        from dateutil.zoneinfo import gettz_db_metadata
+        from dateutil.zoneinfo import rebuild
+
+        zi_all = (gettz, gettz_db_metadata, rebuild)
+
+        for var in zi_all:
+            self.assertIsNot(var, None)

Contents/Libraries/Shared/dateutil/test/test_parser.py

@@ -0,0 +1,861 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from ._common import unittest
+
+from datetime import datetime, timedelta, date
+
+from dateutil.tz import tzoffset
+from dateutil.parser import *
+
+import six
+from six import assertRaisesRegex, PY3
+from six.moves import StringIO
+
+class ParserTest(unittest.TestCase):
+
+    def setUp(self):
+        self.tzinfos = {"BRST": -10800}
+        self.brsttz = tzoffset("BRST", -10800)
+        self.default = datetime(2003, 9, 25)
+
+        # Parser should be able to handle bytestring and unicode
+        base_str = '2014-05-01 08:00:00'
+        try:
+            # Python 2.x
+            self.uni_str = unicode(base_str)
+            self.str_str = str(base_str)
+        except NameError:
+            self.uni_str = str(base_str)
+            self.str_str = bytes(base_str.encode())
+
+    def testEmptyString(self):
+        with self.assertRaises(ValueError):
+            parse('')
+
+    def testNone(self):
+        with self.assertRaises(TypeError):
+            parse(None)
+
+    def testInvalidType(self):
+        with self.assertRaises(TypeError):
+            parse(13)
+
+    def testDuckTyping(self):
+        # We want to support arbitrary classes that implement the stream
+        # interface.
+
+        class StringPassThrough(object):
+            def __init__(self, stream):
+                self.stream = stream
+
+            def read(self, *args, **kwargs):
+                return self.stream.read(*args, **kwargs)
+
+
+        dstr = StringPassThrough(StringIO('2014 January 19'))
+
+        self.assertEqual(parse(dstr), datetime(2014, 1, 19))
+
+    def testParseStream(self):
+        dstr = StringIO('2014 January 19')
+
+        self.assertEqual(parse(dstr), datetime(2014, 1, 19))
+
+    def testParseStr(self):
+        self.assertEqual(parse(self.str_str),
+                         parse(self.uni_str))
+
+    def testParserParseStr(self):
+        from dateutil.parser import parser
+
+        self.assertEqual(parser().parse(self.str_str),
+                         parser().parse(self.uni_str))
+
+    def testParseUnicodeWords(self):
+
+        class rus_parserinfo(parserinfo):
+            MONTHS = [("янв", "Январь"),
+                      ("фев", "Февраль"),
+                      ("мар", "Март"),
+                      ("апр", "Апрель"),
+                      ("май", "Май"),
+                      ("июн", "Июнь"),
+                      ("июл", "Июль"),
+                      ("авг", "Август"),
+                      ("сен", "Сентябрь"),
+                      ("окт", "Октябрь"),
+                      ("ноя", "Ноябрь"),
+                      ("дек", "Декабрь")]
+
+        self.assertEqual(parse('10 Сентябрь 2015 10:20',
+                               parserinfo=rus_parserinfo()),
+                         datetime(2015, 9, 10, 10, 20))
+
+    def testParseWithNulls(self):
+        # This relies on the from __future__ import unicode_literals, because
+        # explicitly specifying a unicode literal is a syntax error in Py 3.2
+        # May want to switch to u'...' if we ever drop Python 3.2 support.
+        pstring = '\x00\x00August 29, 1924'
+
+        self.assertEqual(parse(pstring),
+                         datetime(1924, 8, 29))
+
+    def testDateCommandFormat(self):
+        self.assertEqual(parse("Thu Sep 25 10:36:28 BRST 2003",
+                               tzinfos=self.tzinfos),
+                         datetime(2003, 9, 25, 10, 36, 28,
+                                  tzinfo=self.brsttz))
+
+    def testDateCommandFormatUnicode(self):
+        self.assertEqual(parse("Thu Sep 25 10:36:28 BRST 2003",
+                               tzinfos=self.tzinfos),
+                         datetime(2003, 9, 25, 10, 36, 28,
+                                  tzinfo=self.brsttz))
+
+
+    def testDateCommandFormatReversed(self):
+        self.assertEqual(parse("2003 10:36:28 BRST 25 Sep Thu",
+                               tzinfos=self.tzinfos),
+                         datetime(2003, 9, 25, 10, 36, 28,
+                                  tzinfo=self.brsttz))
+
+    def testDateCommandFormatWithLong(self):
+        if not PY3:
+            self.assertEqual(parse("Thu Sep 25 10:36:28 BRST 2003",
+                                   tzinfos={"BRST": long(-10800)}),
+                             datetime(2003, 9, 25, 10, 36, 28,
+                                      tzinfo=self.brsttz))
+    def testDateCommandFormatIgnoreTz(self):
+        self.assertEqual(parse("Thu Sep 25 10:36:28 BRST 2003",
+                               ignoretz=True),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip1(self):
+        self.assertEqual(parse("Thu Sep 25 10:36:28 2003"),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip2(self):
+        self.assertEqual(parse("Thu Sep 25 10:36:28", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip3(self):
+        self.assertEqual(parse("Thu Sep 10:36:28", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip4(self):
+        self.assertEqual(parse("Thu 10:36:28", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip5(self):
+        self.assertEqual(parse("Sep 10:36:28", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip6(self):
+        self.assertEqual(parse("10:36:28", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testDateCommandFormatStrip7(self):
+        self.assertEqual(parse("10:36", default=self.default),
+                         datetime(2003, 9, 25, 10, 36))
+
+    def testDateCommandFormatStrip8(self):
+        self.assertEqual(parse("Thu Sep 25 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateCommandFormatStrip9(self):
+        self.assertEqual(parse("Sep 25 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateCommandFormatStrip10(self):
+        self.assertEqual(parse("Sep 2003", default=self.default),
+                         datetime(2003, 9, 25))
+
+    def testDateCommandFormatStrip11(self):
+        self.assertEqual(parse("Sep", default=self.default),
+                         datetime(2003, 9, 25))
+
+    def testDateCommandFormatStrip12(self):
+        self.assertEqual(parse("2003", default=self.default),
+                         datetime(2003, 9, 25))
+
+    def testDateRCommandFormat(self):
+        self.assertEqual(parse("Thu, 25 Sep 2003 10:49:41 -0300"),
+                         datetime(2003, 9, 25, 10, 49, 41,
+                                  tzinfo=self.brsttz))
+
+    def testISOFormat(self):
+        self.assertEqual(parse("2003-09-25T10:49:41.5-03:00"),
+                         datetime(2003, 9, 25, 10, 49, 41, 500000,
+                                  tzinfo=self.brsttz))
+
+    def testISOFormatStrip1(self):
+        self.assertEqual(parse("2003-09-25T10:49:41-03:00"),
+                         datetime(2003, 9, 25, 10, 49, 41,
+                                  tzinfo=self.brsttz))
+
+    def testISOFormatStrip2(self):
+        self.assertEqual(parse("2003-09-25T10:49:41"),
+                         datetime(2003, 9, 25, 10, 49, 41))
+
+    def testISOFormatStrip3(self):
+        self.assertEqual(parse("2003-09-25T10:49"),
+                         datetime(2003, 9, 25, 10, 49))
+
+    def testISOFormatStrip4(self):
+        self.assertEqual(parse("2003-09-25T10"),
+                         datetime(2003, 9, 25, 10))
+
+    def testISOFormatStrip5(self):
+        self.assertEqual(parse("2003-09-25"),
+                         datetime(2003, 9, 25))
+
+    def testISOStrippedFormat(self):
+        self.assertEqual(parse("20030925T104941.5-0300"),
+                         datetime(2003, 9, 25, 10, 49, 41, 500000,
+                                  tzinfo=self.brsttz))
+
+    def testISOStrippedFormatStrip1(self):
+        self.assertEqual(parse("20030925T104941-0300"),
+                         datetime(2003, 9, 25, 10, 49, 41,
+                                  tzinfo=self.brsttz))
+
+    def testISOStrippedFormatStrip2(self):
+        self.assertEqual(parse("20030925T104941"),
+                         datetime(2003, 9, 25, 10, 49, 41))
+
+    def testISOStrippedFormatStrip3(self):
+        self.assertEqual(parse("20030925T1049"),
+                         datetime(2003, 9, 25, 10, 49, 0))
+
+    def testISOStrippedFormatStrip4(self):
+        self.assertEqual(parse("20030925T10"),
+                         datetime(2003, 9, 25, 10))
+
+    def testISOStrippedFormatStrip5(self):
+        self.assertEqual(parse("20030925"),
+                         datetime(2003, 9, 25))
+
+    def testPythonLoggerFormat(self):
+        self.assertEqual(parse("2003-09-25 10:49:41,502"),
+                         datetime(2003, 9, 25, 10, 49, 41, 502000))
+
+    def testNoSeparator1(self):
+        self.assertEqual(parse("199709020908"),
+                         datetime(1997, 9, 2, 9, 8))
+
+    def testNoSeparator2(self):
+        self.assertEqual(parse("19970902090807"),
+                         datetime(1997, 9, 2, 9, 8, 7))
+
+    def testDateWithDash1(self):
+        self.assertEqual(parse("2003-09-25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash2(self):
+        self.assertEqual(parse("2003-Sep-25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash3(self):
+        self.assertEqual(parse("25-Sep-2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash4(self):
+        self.assertEqual(parse("25-Sep-2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash5(self):
+        self.assertEqual(parse("Sep-25-2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash6(self):
+        self.assertEqual(parse("09-25-2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash7(self):
+        self.assertEqual(parse("25-09-2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDash8(self):
+        self.assertEqual(parse("10-09-2003", dayfirst=True),
+                         datetime(2003, 9, 10))
+
+    def testDateWithDash9(self):
+        self.assertEqual(parse("10-09-2003"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithDash10(self):
+        self.assertEqual(parse("10-09-03"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithDash11(self):
+        self.assertEqual(parse("10-09-03", yearfirst=True),
+                         datetime(2010, 9, 3))
+
+    def testDateWithDot1(self):
+        self.assertEqual(parse("2003.09.25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot2(self):
+        self.assertEqual(parse("2003.Sep.25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot3(self):
+        self.assertEqual(parse("25.Sep.2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot4(self):
+        self.assertEqual(parse("25.Sep.2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot5(self):
+        self.assertEqual(parse("Sep.25.2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot6(self):
+        self.assertEqual(parse("09.25.2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot7(self):
+        self.assertEqual(parse("25.09.2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithDot8(self):
+        self.assertEqual(parse("10.09.2003", dayfirst=True),
+                         datetime(2003, 9, 10))
+
+    def testDateWithDot9(self):
+        self.assertEqual(parse("10.09.2003"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithDot10(self):
+        self.assertEqual(parse("10.09.03"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithDot11(self):
+        self.assertEqual(parse("10.09.03", yearfirst=True),
+                         datetime(2010, 9, 3))
+
+    def testDateWithSlash1(self):
+        self.assertEqual(parse("2003/09/25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash2(self):
+        self.assertEqual(parse("2003/Sep/25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash3(self):
+        self.assertEqual(parse("25/Sep/2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash4(self):
+        self.assertEqual(parse("25/Sep/2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash5(self):
+        self.assertEqual(parse("Sep/25/2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash6(self):
+        self.assertEqual(parse("09/25/2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash7(self):
+        self.assertEqual(parse("25/09/2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSlash8(self):
+        self.assertEqual(parse("10/09/2003", dayfirst=True),
+                         datetime(2003, 9, 10))
+
+    def testDateWithSlash9(self):
+        self.assertEqual(parse("10/09/2003"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithSlash10(self):
+        self.assertEqual(parse("10/09/03"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithSlash11(self):
+        self.assertEqual(parse("10/09/03", yearfirst=True),
+                         datetime(2010, 9, 3))
+
+    def testDateWithSpace1(self):
+        self.assertEqual(parse("2003 09 25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace2(self):
+        self.assertEqual(parse("2003 Sep 25"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace3(self):
+        self.assertEqual(parse("25 Sep 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace4(self):
+        self.assertEqual(parse("25 Sep 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace5(self):
+        self.assertEqual(parse("Sep 25 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace6(self):
+        self.assertEqual(parse("09 25 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace7(self):
+        self.assertEqual(parse("25 09 2003"),
+                         datetime(2003, 9, 25))
+
+    def testDateWithSpace8(self):
+        self.assertEqual(parse("10 09 2003", dayfirst=True),
+                         datetime(2003, 9, 10))
+
+    def testDateWithSpace9(self):
+        self.assertEqual(parse("10 09 2003"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithSpace10(self):
+        self.assertEqual(parse("10 09 03"),
+                         datetime(2003, 10, 9))
+
+    def testDateWithSpace11(self):
+        self.assertEqual(parse("10 09 03", yearfirst=True),
+                         datetime(2010, 9, 3))
+
+    def testDateWithSpace12(self):
+        self.assertEqual(parse("25 09 03"),
+                         datetime(2003, 9, 25))
+
+    def testStrangelyOrderedDate1(self):
+        self.assertEqual(parse("03 25 Sep"),
+                         datetime(2003, 9, 25))
+
+    def testStrangelyOrderedDate2(self):
+        self.assertEqual(parse("2003 25 Sep"),
+                         datetime(2003, 9, 25))
+
+    def testStrangelyOrderedDate3(self):
+        self.assertEqual(parse("25 03 Sep"),
+                         datetime(2025, 9, 3))
+
+    def testHourWithLetters(self):
+        self.assertEqual(parse("10h36m28.5s", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28, 500000))
+
+    def testHourWithLettersStrip1(self):
+        self.assertEqual(parse("10h36m28s", default=self.default),
+                         datetime(2003, 9, 25, 10, 36, 28))
+
+    def testHourWithLettersStrip2(self):
+        self.assertEqual(parse("10h36m", default=self.default),
+                         datetime(2003, 9, 25, 10, 36))
+
+    def testHourWithLettersStrip3(self):
+        self.assertEqual(parse("10h", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourWithLettersStrip4(self):
+        self.assertEqual(parse("10 h 36", default=self.default),
+                         datetime(2003, 9, 25, 10, 36))
+
+    def testAMPMNoHour(self):
+        with self.assertRaises(ValueError):
+            parse("AM")
+
+        with self.assertRaises(ValueError):
+            parse("Jan 20, 2015 PM")
+
+    def testHourAmPm1(self):
+        self.assertEqual(parse("10h am", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm2(self):
+        self.assertEqual(parse("10h pm", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testHourAmPm3(self):
+        self.assertEqual(parse("10am", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm4(self):
+        self.assertEqual(parse("10pm", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testHourAmPm5(self):
+        self.assertEqual(parse("10:00 am", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm6(self):
+        self.assertEqual(parse("10:00 pm", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testHourAmPm7(self):
+        self.assertEqual(parse("10:00am", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm8(self):
+        self.assertEqual(parse("10:00pm", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testHourAmPm9(self):
+        self.assertEqual(parse("10:00a.m", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm10(self):
+        self.assertEqual(parse("10:00p.m", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testHourAmPm11(self):
+        self.assertEqual(parse("10:00a.m.", default=self.default),
+                         datetime(2003, 9, 25, 10))
+
+    def testHourAmPm12(self):
+        self.assertEqual(parse("10:00p.m.", default=self.default),
+                         datetime(2003, 9, 25, 22))
+
+    def testAMPMRange(self):
+        with self.assertRaises(ValueError):
+            parse("13:44 AM")
+
+        with self.assertRaises(ValueError):
+            parse("January 25, 1921 23:13 PM")
+
+    def testPertain(self):
+        self.assertEqual(parse("Sep 03", default=self.default),
+                         datetime(2003, 9, 3))
+        self.assertEqual(parse("Sep of 03", default=self.default),
+                         datetime(2003, 9, 25))
+
+    def testWeekdayAlone(self):
+        self.assertEqual(parse("Wed", default=self.default),
+                         datetime(2003, 10, 1))
+
+    def testLongWeekday(self):
+        self.assertEqual(parse("Wednesday", default=self.default),
+                         datetime(2003, 10, 1))
+
+    def testLongMonth(self):
+        self.assertEqual(parse("October", default=self.default),
+                         datetime(2003, 10, 25))
+
+    def testZeroYear(self):
+        self.assertEqual(parse("31-Dec-00", default=self.default),
+                         datetime(2000, 12, 31))
+
+    def testFuzzy(self):
+        s = "Today is 25 of September of 2003, exactly " \
+            "at 10:49:41 with timezone -03:00."
+        self.assertEqual(parse(s, fuzzy=True),
+                         datetime(2003, 9, 25, 10, 49, 41,
+                                  tzinfo=self.brsttz))
+
+    def testFuzzyWithTokens(self):
+        s = "Today is 25 of September of 2003, exactly " \
+            "at 10:49:41 with timezone -03:00."
+        self.assertEqual(parse(s, fuzzy_with_tokens=True),
+                         (datetime(2003, 9, 25, 10, 49, 41,
+                                   tzinfo=self.brsttz),
+                         ('Today is ', 'of ', ', exactly at ',
+                          ' with timezone ', '.')))
+
+    def testFuzzyAMPMProblem(self):
+        # Sometimes fuzzy parsing results in AM/PM flag being set without
+        # hours - if it's fuzzy it should ignore that.
+        s1 = "I have a meeting on March 1, 1974."
+        s2 = "On June 8th, 2020, I am going to be the first man on Mars"
+
+        # Also don't want any erroneous AM or PMs changing the parsed time
+        s3 = "Meet me at the AM/PM on Sunset at 3:00 AM on December 3rd, 2003"
+        s4 = "Meet me at 3:00AM on December 3rd, 2003 at the AM/PM on Sunset"
+
+        self.assertEqual(parse(s1, fuzzy=True), datetime(1974, 3, 1))
+        self.assertEqual(parse(s2, fuzzy=True), datetime(2020, 6, 8))
+        self.assertEqual(parse(s3, fuzzy=True), datetime(2003, 12, 3, 3))
+        self.assertEqual(parse(s4, fuzzy=True), datetime(2003, 12, 3, 3))
+
+    def testFuzzyIgnoreAMPM(self):
+        s1 = "Jan 29, 1945 14:45 AM I going to see you there?"
+
+        self.assertEqual(parse(s1, fuzzy=True), datetime(1945, 1, 29, 14, 45))
+
+    def testExtraSpace(self):
+        self.assertEqual(parse("  July   4 ,  1976   12:01:02   am  "),
+                         datetime(1976, 7, 4, 0, 1, 2))
+
+    def testRandomFormat1(self):
+        self.assertEqual(parse("Wed, July 10, '96"),
+                         datetime(1996, 7, 10, 0, 0))
+
+    def testRandomFormat2(self):
+        self.assertEqual(parse("1996.07.10 AD at 15:08:56 PDT",
+                               ignoretz=True),
+                         datetime(1996, 7, 10, 15, 8, 56))
+
+    def testRandomFormat3(self):
+        self.assertEqual(parse("1996.July.10 AD 12:08 PM"),
+                         datetime(1996, 7, 10, 12, 8))
+
+    def testRandomFormat4(self):
+        self.assertEqual(parse("Tuesday, April 12, 1952 AD 3:30:42pm PST",
+                               ignoretz=True),
+                         datetime(1952, 4, 12, 15, 30, 42))
+
+    def testRandomFormat5(self):
+        self.assertEqual(parse("November 5, 1994, 8:15:30 am EST",
+                               ignoretz=True),
+                         datetime(1994, 11, 5, 8, 15, 30))
+
+    def testRandomFormat6(self):
+        self.assertEqual(parse("1994-11-05T08:15:30-05:00",
+                               ignoretz=True),
+                         datetime(1994, 11, 5, 8, 15, 30))
+
+    def testRandomFormat7(self):
+        self.assertEqual(parse("1994-11-05T08:15:30Z",
+                               ignoretz=True),
+                         datetime(1994, 11, 5, 8, 15, 30))
+
+    def testRandomFormat8(self):
+        self.assertEqual(parse("July 4, 1976"), datetime(1976, 7, 4))
+
+    def testRandomFormat9(self):
+        self.assertEqual(parse("7 4 1976"), datetime(1976, 7, 4))
+
+    def testRandomFormat10(self):
+        self.assertEqual(parse("4 jul 1976"), datetime(1976, 7, 4))
+
+    def testRandomFormat11(self):
+        self.assertEqual(parse("7-4-76"), datetime(1976, 7, 4))
+
+    def testRandomFormat12(self):
+        self.assertEqual(parse("19760704"), datetime(1976, 7, 4))
+
+    def testRandomFormat13(self):
+        self.assertEqual(parse("0:01:02", default=self.default),
+                         datetime(2003, 9, 25, 0, 1, 2))
+
+    def testRandomFormat14(self):
+        self.assertEqual(parse("12h 01m02s am", default=self.default),
+                         datetime(2003, 9, 25, 0, 1, 2))
+
+    def testRandomFormat15(self):
+        self.assertEqual(parse("0:01:02 on July 4, 1976"),
+                         datetime(1976, 7, 4, 0, 1, 2))
+
+    def testRandomFormat16(self):
+        self.assertEqual(parse("0:01:02 on July 4, 1976"),
+                         datetime(1976, 7, 4, 0, 1, 2))
+
+    def testRandomFormat17(self):
+        self.assertEqual(parse("1976-07-04T00:01:02Z", ignoretz=True),
+                         datetime(1976, 7, 4, 0, 1, 2))
+
+    def testRandomFormat18(self):
+        self.assertEqual(parse("July 4, 1976 12:01:02 am"),
+                         datetime(1976, 7, 4, 0, 1, 2))
+
+    def testRandomFormat19(self):
+        self.assertEqual(parse("Mon Jan  2 04:24:27 1995"),
+                         datetime(1995, 1, 2, 4, 24, 27))
+
+    def testRandomFormat20(self):
+        self.assertEqual(parse("Tue Apr 4 00:22:12 PDT 1995", ignoretz=True),
+                         datetime(1995, 4, 4, 0, 22, 12))
+
+    def testRandomFormat21(self):
+        self.assertEqual(parse("04.04.95 00:22"),
+                         datetime(1995, 4, 4, 0, 22))
+
+    def testRandomFormat22(self):
+        self.assertEqual(parse("Jan 1 1999 11:23:34.578"),
+                         datetime(1999, 1, 1, 11, 23, 34, 578000))
+
+    def testRandomFormat23(self):
+        self.assertEqual(parse("950404 122212"),
+                         datetime(1995, 4, 4, 12, 22, 12))
+
+    def testRandomFormat24(self):
+        self.assertEqual(parse("0:00 PM, PST", default=self.default,
+                               ignoretz=True),
+                         datetime(2003, 9, 25, 12, 0))
+
+    def testRandomFormat25(self):
+        self.assertEqual(parse("12:08 PM", default=self.default),
+                         datetime(2003, 9, 25, 12, 8))
+
+    def testRandomFormat26(self):
+        self.assertEqual(parse("5:50 A.M. on June 13, 1990"),
+                         datetime(1990, 6, 13, 5, 50))
+
+    def testRandomFormat27(self):
+        self.assertEqual(parse("3rd of May 2001"), datetime(2001, 5, 3))
+
+    def testRandomFormat28(self):
+        self.assertEqual(parse("5th of March 2001"), datetime(2001, 3, 5))
+
+    def testRandomFormat29(self):
+        self.assertEqual(parse("1st of May 2003"), datetime(2003, 5, 1))
+
+    def testRandomFormat30(self):
+        self.assertEqual(parse("01h02m03", default=self.default),
+                         datetime(2003, 9, 25, 1, 2, 3))
+
+    def testRandomFormat31(self):
+        self.assertEqual(parse("01h02", default=self.default),
+                         datetime(2003, 9, 25, 1, 2))
+
+    def testRandomFormat32(self):
+        self.assertEqual(parse("01h02s", default=self.default),
+                         datetime(2003, 9, 25, 1, 0, 2))
+
+    def testRandomFormat33(self):
+        self.assertEqual(parse("01m02", default=self.default),
+                         datetime(2003, 9, 25, 0, 1, 2))
+
+    def testRandomFormat34(self):
+        self.assertEqual(parse("01m02h", default=self.default),
+                         datetime(2003, 9, 25, 2, 1))
+
+    def testRandomFormat35(self):
+        self.assertEqual(parse("2004 10 Apr 11h30m", default=self.default),
+                         datetime(2004, 4, 10, 11, 30))
+
+    def test_99_ad(self):
+        self.assertEqual(parse('0099-01-01T00:00:00'),
+                         datetime(99, 1, 1, 0, 0))
+
+    def test_31_ad(self):
+        self.assertEqual(parse('0031-01-01T00:00:00'),
+                         datetime(31, 1, 1, 0, 0))
+
+    def testInvalidDay(self):
+        with self.assertRaises(ValueError):
+            parse("Feb 30, 2007")
+
+    def testUnspecifiedDayFallback(self):
+        # Test that for an unspecified day, the fallback behavior is correct.
+        self.assertEqual(parse("April 2009", default=datetime(2010, 1, 31)),
+                         datetime(2009, 4, 30))
+
+    def testUnspecifiedDayFallbackFebNoLeapYear(self):        
+        self.assertEqual(parse("Feb 2007", default=datetime(2010, 1, 31)),
+                         datetime(2007, 2, 28))
+
+    def testUnspecifiedDayFallbackFebLeapYear(self):        
+        self.assertEqual(parse("Feb 2008", default=datetime(2010, 1, 31)),
+                         datetime(2008, 2, 29))
+
+    def testErrorType01(self):
+        self.assertRaises(ValueError,
+                          parse, 'shouldfail')
+
+    def testCorrectErrorOnFuzzyWithTokens(self):
+        assertRaisesRegex(self, ValueError, 'Unknown string format',
+                          parse, '04/04/32/423', fuzzy_with_tokens=True)
+        assertRaisesRegex(self, ValueError, 'Unknown string format',
+                          parse, '04/04/04 +32423', fuzzy_with_tokens=True)
+        assertRaisesRegex(self, ValueError, 'Unknown string format',
+                          parse, '04/04/0d4', fuzzy_with_tokens=True)
+
+    def testIncreasingCTime(self):
+        # This test will check 200 different years, every month, every day,
+        # every hour, every minute, every second, and every weekday, using
+        # a delta of more or less 1 year, 1 month, 1 day, 1 minute and
+        # 1 second.
+        delta = timedelta(days=365+31+1, seconds=1+60+60*60)
+        dt = datetime(1900, 1, 1, 0, 0, 0, 0)
+        for i in range(200):
+            self.assertEqual(parse(dt.ctime()), dt)
+            dt += delta
+
+    def testIncreasingISOFormat(self):
+        delta = timedelta(days=365+31+1, seconds=1+60+60*60)
+        dt = datetime(1900, 1, 1, 0, 0, 0, 0)
+        for i in range(200):
+            self.assertEqual(parse(dt.isoformat()), dt)
+            dt += delta
+
+    def testMicrosecondsPrecisionError(self):
+        # Skip found out that sad precision problem. :-(
+        dt1 = parse("00:11:25.01")
+        dt2 = parse("00:12:10.01")
+        self.assertEqual(dt1.microsecond, 10000)
+        self.assertEqual(dt2.microsecond, 10000)
+
+    def testMicrosecondPrecisionErrorReturns(self):
+        # One more precision issue, discovered by Eric Brown.  This should
+        # be the last one, as we're no longer using floating points.
+        for ms in [100001, 100000, 99999, 99998,
+                    10001,  10000,  9999,  9998,
+                     1001,   1000,   999,   998,
+                      101,    100,    99,    98]:
+            dt = datetime(2008, 2, 27, 21, 26, 1, ms)
+            self.assertEqual(parse(dt.isoformat()), dt)
+
+    def testHighPrecisionSeconds(self):
+        self.assertEqual(parse("20080227T21:26:01.123456789"),
+                          datetime(2008, 2, 27, 21, 26, 1, 123456))
+
+    def testCustomParserInfo(self):
+        # Custom parser info wasn't working, as Michael Elsdörfer discovered.
+        from dateutil.parser import parserinfo, parser
+
+        class myparserinfo(parserinfo):
+            MONTHS = parserinfo.MONTHS[:]
+            MONTHS[0] = ("Foo", "Foo")
+        myparser = parser(myparserinfo())
+        dt = myparser.parse("01/Foo/2007")
+        self.assertEqual(dt, datetime(2007, 1, 1))
+
+    def testNoYearFirstNoDayFirst(self):
+        dtstr = '090107'
+
+        # Should be MMDDYY
+        self.assertEqual(parse(dtstr),
+                         datetime(2007, 9, 1))
+
+        self.assertEqual(parse(dtstr, yearfirst=False, dayfirst=False),
+                         datetime(2007, 9, 1))
+
+    def testYearFirst(self):
+        dtstr = '090107'
+
+        # Should be MMDDYY
+        self.assertEqual(parse(dtstr, yearfirst=True),
+                         datetime(2009, 1, 7))
+
+        self.assertEqual(parse(dtstr, yearfirst=True, dayfirst=False),
+                         datetime(2009, 1, 7))
+
+    def testDayFirst(self):
+        dtstr = '090107'
+
+        # Should be DDMMYY
+        self.assertEqual(parse(dtstr, dayfirst=True),
+                         datetime(2007, 1, 9))
+
+        self.assertEqual(parse(dtstr, yearfirst=False, dayfirst=True),
+                         datetime(2007, 1, 9))
+
+    def testDayFirstYearFirst(self):
+        dtstr = '090107'
+        # Should be YYDDMM
+        self.assertEqual(parse(dtstr, yearfirst=True, dayfirst=True),
+                         datetime(2009, 7, 1))
+
+    def testUnambiguousYearFirst(self):
+        dtstr = '2015 09 25'
+        self.assertEqual(parse(dtstr, yearfirst=True),
+                         datetime(2015, 9, 25))
+
+    def testUnambiguousDayFirst(self):
+        dtstr = '2015 09 25'
+        self.assertEqual(parse(dtstr, dayfirst=True), 
+                         datetime(2015, 9, 25))
+
+    def testUnambiguousDayFirstYearFirst(self):
+        dtstr = '2015 09 25'
+        self.assertEqual(parse(dtstr, dayfirst=True, yearfirst=True),
+                         datetime(2015, 9, 25))
+

Contents/Libraries/Shared/dateutil/test/test_relativedelta.py

@@ -0,0 +1,571 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from ._common import unittest, WarningTestMixin, NotAValue
+
+import calendar
+from datetime import datetime, date, timedelta
+
+from dateutil.relativedelta import *
+
+
+class RelativeDeltaTest(WarningTestMixin, unittest.TestCase):
+    now = datetime(2003, 9, 17, 20, 54, 47, 282310)
+    today = date(2003, 9, 17)
+
+    def testInheritance(self):
+        # Ensure that relativedelta is inheritance-friendly.
+        class rdChildClass(relativedelta):
+            pass
+
+        ccRD = rdChildClass(years=1, months=1, days=1, leapdays=1, weeks=1,
+                            hours=1, minutes=1, seconds=1, microseconds=1)
+
+        rd = relativedelta(years=1, months=1, days=1, leapdays=1, weeks=1,
+                           hours=1, minutes=1, seconds=1, microseconds=1)
+
+        self.assertEqual(type(ccRD + rd), type(ccRD),
+                         msg='Addition does not inherit type.')
+
+        self.assertEqual(type(ccRD - rd), type(ccRD),
+                         msg='Subtraction does not inherit type.')
+
+        self.assertEqual(type(-ccRD), type(ccRD),
+                         msg='Negation does not inherit type.')
+
+        self.assertEqual(type(ccRD * 5.0), type(ccRD),
+                         msg='Multiplication does not inherit type.')
+
+        self.assertEqual(type(ccRD / 5.0), type(ccRD),
+                         msg='Division does not inherit type.')
+
+    def testMonthEndMonthBeginning(self):
+        self.assertEqual(relativedelta(datetime(2003, 1, 31, 23, 59, 59),
+                                       datetime(2003, 3, 1, 0, 0, 0)),
+                         relativedelta(months=-1, seconds=-1))
+
+        self.assertEqual(relativedelta(datetime(2003, 3, 1, 0, 0, 0),
+                                       datetime(2003, 1, 31, 23, 59, 59)),
+                         relativedelta(months=1, seconds=1))
+
+    def testMonthEndMonthBeginningLeapYear(self):
+        self.assertEqual(relativedelta(datetime(2012, 1, 31, 23, 59, 59),
+                                       datetime(2012, 3, 1, 0, 0, 0)),
+                         relativedelta(months=-1, seconds=-1))
+
+        self.assertEqual(relativedelta(datetime(2003, 3, 1, 0, 0, 0),
+                                       datetime(2003, 1, 31, 23, 59, 59)),
+                         relativedelta(months=1, seconds=1))
+
+    def testNextMonth(self):
+        self.assertEqual(self.now+relativedelta(months=+1),
+                         datetime(2003, 10, 17, 20, 54, 47, 282310))
+
+    def testNextMonthPlusOneWeek(self):
+        self.assertEqual(self.now+relativedelta(months=+1, weeks=+1),
+                         datetime(2003, 10, 24, 20, 54, 47, 282310))
+
+    def testNextMonthPlusOneWeek10am(self):
+        self.assertEqual(self.today +
+                         relativedelta(months=+1, weeks=+1, hour=10),
+                         datetime(2003, 10, 24, 10, 0))
+
+    def testNextMonthPlusOneWeek10amDiff(self):
+        self.assertEqual(relativedelta(datetime(2003, 10, 24, 10, 0),
+                                       self.today),
+                         relativedelta(months=+1, days=+7, hours=+10))
+
+    def testOneMonthBeforeOneYear(self):
+        self.assertEqual(self.now+relativedelta(years=+1, months=-1),
+                         datetime(2004, 8, 17, 20, 54, 47, 282310))
+
+    def testMonthsOfDiffNumOfDays(self):
+        self.assertEqual(date(2003, 1, 27)+relativedelta(months=+1),
+                         date(2003, 2, 27))
+        self.assertEqual(date(2003, 1, 31)+relativedelta(months=+1),
+                         date(2003, 2, 28))
+        self.assertEqual(date(2003, 1, 31)+relativedelta(months=+2),
+                         date(2003, 3, 31))
+
+    def testMonthsOfDiffNumOfDaysWithYears(self):
+        self.assertEqual(date(2000, 2, 28)+relativedelta(years=+1),
+                         date(2001, 2, 28))
+        self.assertEqual(date(2000, 2, 29)+relativedelta(years=+1),
+                         date(2001, 2, 28))
+
+        self.assertEqual(date(1999, 2, 28)+relativedelta(years=+1),
+                         date(2000, 2, 28))
+        self.assertEqual(date(1999, 3, 1)+relativedelta(years=+1),
+                         date(2000, 3, 1))
+        self.assertEqual(date(1999, 3, 1)+relativedelta(years=+1),
+                         date(2000, 3, 1))
+
+        self.assertEqual(date(2001, 2, 28)+relativedelta(years=-1),
+                         date(2000, 2, 28))
+        self.assertEqual(date(2001, 3, 1)+relativedelta(years=-1),
+                         date(2000, 3, 1))
+
+    def testNextFriday(self):
+        self.assertEqual(self.today+relativedelta(weekday=FR),
+                         date(2003, 9, 19))
+
+    def testNextFridayInt(self):
+        self.assertEqual(self.today+relativedelta(weekday=calendar.FRIDAY),
+                         date(2003, 9, 19))
+
+    def testLastFridayInThisMonth(self):
+        self.assertEqual(self.today+relativedelta(day=31, weekday=FR(-1)),
+                         date(2003, 9, 26))
+
+    def testNextWednesdayIsToday(self):
+        self.assertEqual(self.today+relativedelta(weekday=WE),
+                         date(2003, 9, 17))
+
+    def testNextWenesdayNotToday(self):
+        self.assertEqual(self.today+relativedelta(days=+1, weekday=WE),
+                         date(2003, 9, 24))
+
+    def test15thISOYearWeek(self):
+        self.assertEqual(date(2003, 1, 1) +
+                         relativedelta(day=4, weeks=+14, weekday=MO(-1)),
+                         date(2003, 4, 7))
+
+    def testMillenniumAge(self):
+        self.assertEqual(relativedelta(self.now, date(2001, 1, 1)),
+                         relativedelta(years=+2, months=+8, days=+16,
+                                       hours=+20, minutes=+54, seconds=+47,
+                                       microseconds=+282310))
+
+    def testJohnAge(self):
+        self.assertEqual(relativedelta(self.now,
+                                       datetime(1978, 4, 5, 12, 0)),
+                         relativedelta(years=+25, months=+5, days=+12,
+                                       hours=+8, minutes=+54, seconds=+47,
+                                       microseconds=+282310))
+
+    def testJohnAgeWithDate(self):
+        self.assertEqual(relativedelta(self.today,
+                                       datetime(1978, 4, 5, 12, 0)),
+                         relativedelta(years=+25, months=+5, days=+11,
+                                       hours=+12))
+
+    def testYearDay(self):
+        self.assertEqual(date(2003, 1, 1)+relativedelta(yearday=260),
+                         date(2003, 9, 17))
+        self.assertEqual(date(2002, 1, 1)+relativedelta(yearday=260),
+                         date(2002, 9, 17))
+        self.assertEqual(date(2000, 1, 1)+relativedelta(yearday=260),
+                         date(2000, 9, 16))
+        self.assertEqual(self.today+relativedelta(yearday=261),
+                         date(2003, 9, 18))
+
+    def testYearDayBug(self):
+        # Tests a problem reported by Adam Ryan.
+        self.assertEqual(date(2010, 1, 1)+relativedelta(yearday=15),
+                         date(2010, 1, 15))
+
+    def testNonLeapYearDay(self):
+        self.assertEqual(date(2003, 1, 1)+relativedelta(nlyearday=260),
+                         date(2003, 9, 17))
+        self.assertEqual(date(2002, 1, 1)+relativedelta(nlyearday=260),
+                         date(2002, 9, 17))
+        self.assertEqual(date(2000, 1, 1)+relativedelta(nlyearday=260),
+                         date(2000, 9, 17))
+        self.assertEqual(self.today+relativedelta(yearday=261),
+                         date(2003, 9, 18))
+
+    def testAddition(self):
+        self.assertEqual(relativedelta(days=10) +
+                         relativedelta(years=1, months=2, days=3, hours=4,
+                                       minutes=5, microseconds=6),
+                         relativedelta(years=1, months=2, days=13, hours=4,
+                                       minutes=5, microseconds=6))
+
+    def testAdditionToDatetime(self):
+        self.assertEqual(datetime(2000, 1, 1) + relativedelta(days=1),
+                         datetime(2000, 1, 2))
+
+    def testRightAdditionToDatetime(self):
+        self.assertEqual(relativedelta(days=1) + datetime(2000, 1, 1),
+                         datetime(2000, 1, 2))
+
+    def testAdditionInvalidType(self):
+        with self.assertRaises(TypeError):
+            relativedelta(days=3) + 9
+
+    def testAdditionUnsupportedType(self):
+        # For unsupported types that define their own comparators, etc.
+        self.assertIs(relativedelta(days=1) + NotAValue, NotAValue)
+
+    def testSubtraction(self):
+        self.assertEqual(relativedelta(days=10) -
+                         relativedelta(years=1, months=2, days=3, hours=4,
+                                       minutes=5, microseconds=6),
+                         relativedelta(years=-1, months=-2, days=7, hours=-4,
+                                       minutes=-5, microseconds=-6))
+
+    def testRightSubtractionFromDatetime(self):
+        self.assertEqual(datetime(2000, 1, 2) - relativedelta(days=1),
+                         datetime(2000, 1, 1))
+
+    def testSubractionWithDatetime(self):
+        self.assertRaises(TypeError, lambda x, y: x - y,
+                          (relativedelta(days=1), datetime(2000, 1, 1)))
+
+    def testSubtractionInvalidType(self):
+        with self.assertRaises(TypeError):
+            relativedelta(hours=12) - 14
+
+    def testSubtractionUnsupportedType(self):
+        self.assertIs(relativedelta(days=1) + NotAValue, NotAValue)
+
+    def testMultiplication(self):
+        self.assertEqual(datetime(2000, 1, 1) + relativedelta(days=1) * 28,
+                         datetime(2000, 1, 29))
+        self.assertEqual(datetime(2000, 1, 1) + 28 * relativedelta(days=1),
+                         datetime(2000, 1, 29))
+
+    def testMultiplicationUnsupportedType(self):
+        self.assertIs(relativedelta(days=1) * NotAValue, NotAValue)
+
+    def testDivision(self):
+        self.assertEqual(datetime(2000, 1, 1) + relativedelta(days=28) / 28,
+                         datetime(2000, 1, 2))
+
+    def testDivisionUnsupportedType(self):
+        self.assertIs(relativedelta(days=1) / NotAValue, NotAValue)
+
+    def testBoolean(self):
+        self.assertFalse(relativedelta(days=0))
+        self.assertTrue(relativedelta(days=1))
+
+    def testComparison(self):
+        d1 = relativedelta(years=1, months=1, days=1, leapdays=0, hours=1,
+                           minutes=1, seconds=1, microseconds=1)
+        d2 = relativedelta(years=1, months=1, days=1, leapdays=0, hours=1,
+                           minutes=1, seconds=1, microseconds=1)
+        d3 = relativedelta(years=1, months=1, days=1, leapdays=0, hours=1,
+                           minutes=1, seconds=1, microseconds=2)
+
+        self.assertEqual(d1, d2)
+        self.assertNotEqual(d1, d3)
+
+    def testInequalityTypeMismatch(self):
+        # Different type
+        self.assertFalse(relativedelta(year=1) == 19)
+
+    def testInequalityUnsupportedType(self):
+        self.assertIs(relativedelta(hours=3) == NotAValue, NotAValue)
+
+    def testInequalityWeekdays(self):
+        # Different weekdays
+        no_wday = relativedelta(year=1997, month=4)
+        wday_mo_1 = relativedelta(year=1997, month=4, weekday=MO(+1))
+        wday_mo_2 = relativedelta(year=1997, month=4, weekday=MO(+2))
+        wday_tu = relativedelta(year=1997, month=4, weekday=TU)
+
+        self.assertTrue(wday_mo_1 == wday_mo_1)
+
+        self.assertFalse(no_wday == wday_mo_1)
+        self.assertFalse(wday_mo_1 == no_wday)
+
+        self.assertFalse(wday_mo_1 == wday_mo_2)
+        self.assertFalse(wday_mo_2 == wday_mo_1)
+
+        self.assertFalse(wday_mo_1 == wday_tu)
+        self.assertFalse(wday_tu == wday_mo_1)
+
+    def testMonthOverflow(self):
+        self.assertEqual(relativedelta(months=273),
+                         relativedelta(years=22, months=9))
+
+    def testWeeks(self):
+        # Test that the weeks property is working properly.
+        rd = relativedelta(years=4, months=2, weeks=8, days=6)
+        self.assertEqual((rd.weeks, rd.days), (8, 8 * 7 + 6))
+
+        rd.weeks = 3
+        self.assertEqual((rd.weeks, rd.days), (3, 3 * 7 + 6))
+
+    def testRelativeDeltaRepr(self):
+        self.assertEqual(repr(relativedelta(years=1, months=-1, days=15)),
+                         'relativedelta(years=+1, months=-1, days=+15)')
+
+        self.assertEqual(repr(relativedelta(months=14, seconds=-25)),
+                         'relativedelta(years=+1, months=+2, seconds=-25)')
+
+        self.assertEqual(repr(relativedelta(month=3, hour=3, weekday=SU(3))),
+                         'relativedelta(month=3, weekday=SU(+3), hour=3)')
+
+    def testRelativeDeltaFractionalYear(self):
+        with self.assertRaises(ValueError):
+            relativedelta(years=1.5)
+
+    def testRelativeDeltaFractionalMonth(self):
+        with self.assertRaises(ValueError):
+            relativedelta(months=1.5)
+
+    def testRelativeDeltaFractionalAbsolutes(self):
+        # Fractional absolute values will soon be unsupported,
+        # check for the deprecation warning.
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(year=2.86)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(month=1.29)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(day=0.44)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(hour=23.98)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(minute=45.21)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(second=13.2)
+
+        with self.assertWarns(DeprecationWarning):
+            relativedelta(microsecond=157221.93)
+
+    def testRelativeDeltaFractionalRepr(self):
+        rd = relativedelta(years=3, months=-2, days=1.25)
+
+        self.assertEqual(repr(rd),
+                         'relativedelta(years=+3, months=-2, days=+1.25)')
+
+        rd = relativedelta(hours=0.5, seconds=9.22)
+        self.assertEqual(repr(rd),
+                         'relativedelta(hours=+0.5, seconds=+9.22)')
+
+    def testRelativeDeltaFractionalWeeks(self):
+        # Equivalent to days=8, hours=18
+        rd = relativedelta(weeks=1.25)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd,
+                         datetime(2009, 9, 11, 18))
+
+    def testRelativeDeltaFractionalDays(self):
+        rd1 = relativedelta(days=1.48)
+
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd1,
+                         datetime(2009, 9, 4, 11, 31, 12))
+
+        rd2 = relativedelta(days=1.5)
+        self.assertEqual(d1 + rd2,
+                         datetime(2009, 9, 4, 12, 0, 0))
+
+    def testRelativeDeltaFractionalHours(self):
+        rd = relativedelta(days=1, hours=12.5)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd,
+                         datetime(2009, 9, 4, 12, 30, 0))
+
+    def testRelativeDeltaFractionalMinutes(self):
+        rd = relativedelta(hours=1, minutes=30.5)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd,
+                         datetime(2009, 9, 3, 1, 30, 30))
+
+    def testRelativeDeltaFractionalSeconds(self):
+        rd = relativedelta(hours=5, minutes=30, seconds=30.5)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd,
+                         datetime(2009, 9, 3, 5, 30, 30, 500000))
+
+    def testRelativeDeltaFractionalPositiveOverflow(self):
+        # Equivalent to (days=1, hours=14)
+        rd1 = relativedelta(days=1.5, hours=2)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd1,
+                         datetime(2009, 9, 4, 14, 0, 0))
+
+        # Equivalent to (days=1, hours=14, minutes=45)
+        rd2 = relativedelta(days=1.5, hours=2.5, minutes=15)
+        d1 = datetime(2009, 9, 3, 0, 0)
+        self.assertEqual(d1 + rd2,
+                         datetime(2009, 9, 4, 14, 45))
+
+        # Carry back up - equivalent to (days=2, hours=2, minutes=0, seconds=1)
+        rd3 = relativedelta(days=1.5, hours=13, minutes=59.5, seconds=31)
+        self.assertEqual(d1 + rd3,
+                         datetime(2009, 9, 5, 2, 0, 1))
+
+    def testRelativeDeltaFractionalNegativeDays(self):
+        # Equivalent to (days=-1, hours=-1)
+        rd1 = relativedelta(days=-1.5, hours=11)
+        d1 = datetime(2009, 9, 3, 12, 0)
+        self.assertEqual(d1 + rd1,
+                         datetime(2009, 9, 2, 11, 0, 0))
+
+        # Equivalent to (days=-1, hours=-9)
+        rd2 = relativedelta(days=-1.25, hours=-3)
+        self.assertEqual(d1 + rd2,
+            datetime(2009, 9, 2, 3))
+
+    def testRelativeDeltaNormalizeFractionalDays(self):
+        # Equivalent to (days=2, hours=18)
+        rd1 = relativedelta(days=2.75)
+
+        self.assertEqual(rd1.normalized(), relativedelta(days=2, hours=18))
+
+        # Equvalent to (days=1, hours=11, minutes=31, seconds=12)
+        rd2 = relativedelta(days=1.48)
+
+        self.assertEqual(rd2.normalized(),
+            relativedelta(days=1, hours=11, minutes=31, seconds=12))
+
+    def testRelativeDeltaNormalizeFractionalDays(self):
+        # Equivalent to (hours=1, minutes=30)
+        rd1 = relativedelta(hours=1.5)
+
+        self.assertEqual(rd1.normalized(), relativedelta(hours=1, minutes=30))
+
+        # Equivalent to (hours=3, minutes=17, seconds=5, microseconds=100)
+        rd2 = relativedelta(hours=3.28472225)
+
+        self.assertEqual(rd2.normalized(),
+            relativedelta(hours=3, minutes=17, seconds=5, microseconds=100))
+
+    def testRelativeDeltaNormalizeFractionalMinutes(self):
+        # Equivalent to (minutes=15, seconds=36)
+        rd1 = relativedelta(minutes=15.6)
+
+        self.assertEqual(rd1.normalized(),
+            relativedelta(minutes=15, seconds=36))
+
+        # Equivalent to (minutes=25, seconds=20, microseconds=25000)
+        rd2 = relativedelta(minutes=25.33375)
+
+        self.assertEqual(rd2.normalized(),
+            relativedelta(minutes=25, seconds=20, microseconds=25000))
+
+    def testRelativeDeltaNormalizeFractionalSeconds(self):
+        # Equivalent to (seconds=45, microseconds=25000)
+        rd1 = relativedelta(seconds=45.025)
+        self.assertEqual(rd1.normalized(),
+            relativedelta(seconds=45, microseconds=25000))
+
+    def testRelativeDeltaFractionalPositiveOverflow(self):
+        # Equivalent to (days=1, hours=14)
+        rd1 = relativedelta(days=1.5, hours=2)
+        self.assertEqual(rd1.normalized(),
+            relativedelta(days=1, hours=14))
+
+        # Equivalent to (days=1, hours=14, minutes=45)
+        rd2 = relativedelta(days=1.5, hours=2.5, minutes=15)
+        self.assertEqual(rd2.normalized(),
+            relativedelta(days=1, hours=14, minutes=45))
+
+        # Carry back up - equivalent to:
+        # (days=2, hours=2, minutes=0, seconds=2, microseconds=3)
+        rd3 = relativedelta(days=1.5, hours=13, minutes=59.50045,
+                            seconds=31.473, microseconds=500003)
+        self.assertEqual(rd3.normalized(),
+            relativedelta(days=2, hours=2, minutes=0,
+                          seconds=2, microseconds=3))
+
+    def testRelativeDeltaFractionalNegativeOverflow(self):
+        # Equivalent to (days=-1)
+        rd1 = relativedelta(days=-0.5, hours=-12)
+        self.assertEqual(rd1.normalized(),
+            relativedelta(days=-1))
+
+        # Equivalent to (days=-1)
+        rd2 = relativedelta(days=-1.5, hours=12)
+        self.assertEqual(rd2.normalized(),
+            relativedelta(days=-1))
+
+        # Equivalent to (days=-1, hours=-14, minutes=-45)
+        rd3 = relativedelta(days=-1.5, hours=-2.5, minutes=-15)
+        self.assertEqual(rd3.normalized(),
+            relativedelta(days=-1, hours=-14, minutes=-45))
+
+        # Equivalent to (days=-1, hours=-14, minutes=+15)
+        rd4 = relativedelta(days=-1.5, hours=-2.5, minutes=45)
+        self.assertEqual(rd4.normalized(),
+            relativedelta(days=-1, hours=-14, minutes=+15))
+
+        # Carry back up - equivalent to:
+        # (days=-2, hours=-2, minutes=0, seconds=-2, microseconds=-3)
+        rd3 = relativedelta(days=-1.5, hours=-13, minutes=-59.50045,
+                            seconds=-31.473, microseconds=-500003)
+        self.assertEqual(rd3.normalized(),
+            relativedelta(days=-2, hours=-2, minutes=0,
+                          seconds=-2, microseconds=-3))
+
+    def testInvalidYearDay(self):
+        with self.assertRaises(ValueError):
+            relativedelta(yearday=367)
+
+    def testAddTimedeltaToUnpopulatedRelativedelta(self):
+        td = timedelta(
+            days=1,
+            seconds=1,
+            microseconds=1,
+            milliseconds=1,
+            minutes=1,
+            hours=1,
+            weeks=1
+        )
+
+        expected = relativedelta(
+            weeks=1,
+            days=1,
+            hours=1,
+            minutes=1,
+            seconds=1,
+            microseconds=1001
+        )
+
+        self.assertEqual(expected, relativedelta() + td)
+
+    def testAddTimedeltaToPopulatedRelativeDelta(self):
+        td = timedelta(
+            days=1,
+            seconds=1,
+            microseconds=1,
+            milliseconds=1,
+            minutes=1,
+            hours=1,
+            weeks=1
+        )
+
+        rd = relativedelta(
+            year=1,
+            month=1,
+            day=1,
+            hour=1,
+            minute=1,
+            second=1,
+            microsecond=1,
+            years=1,
+            months=1,
+            days=1,
+            weeks=1,
+            hours=1,
+            minutes=1,
+            seconds=1,
+            microseconds=1
+        )
+
+        expected = relativedelta(
+            year=1,
+            month=1,
+            day=1,
+            hour=1,
+            minute=1,
+            second=1,
+            microsecond=1,
+            years=1,
+            months=1,
+            weeks=2,
+            days=2,
+            hours=2,
+            minutes=2,
+            seconds=2,
+            microseconds=1002,
+        )
+
+        self.assertEqual(expected, rd + td)

Contents/Libraries/Shared/dateutil/tz/__init__.py

@@ -0,0 +1,4 @@
+from .tz import *
+
+__all__ = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange",
+           "tzstr", "tzical", "tzwin", "tzwinlocal", "gettz"]

Contents/Libraries/Shared/dateutil/tz/_common.py

@@ -0,0 +1,380 @@
+from six import PY3
+from six.moves import _thread
+
+from datetime import datetime, timedelta, tzinfo
+import copy
+
+ZERO = timedelta(0)
+
+__all__ = ['tzname_in_python2', 'enfold']
+
+
+def tzname_in_python2(namefunc):
+    """Change unicode output into bytestrings in Python 2
+
+    tzname() API changed in Python 3. It used to return bytes, but was changed
+    to unicode strings
+    """
+    def adjust_encoding(*args, **kwargs):
+        name = namefunc(*args, **kwargs)
+        if name is not None and not PY3:
+            name = name.encode()
+
+        return name
+
+    return adjust_encoding
+
+
+# The following is adapted from Alexander Belopolsky's tz library
+# https://github.com/abalkin/tz
+if hasattr(datetime, 'fold'):
+    # This is the pre-python 3.6 fold situation
+    def enfold(dt, fold=1):
+        """
+        Provides a unified interface for assigning the ``fold`` attribute to
+        datetimes both before and after the implementation of PEP-495.
+
+        :param fold:
+            The value for the ``fold`` attribute in the returned datetime. This
+            should be either 0 or 1.
+
+        :return:
+            Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+            ``fold`` for all versions of Python. In versions prior to
+            Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+            subclass of :py:class:`datetime.datetime` with the ``fold``
+            attribute added, if ``fold`` is 1.
+
+        ..versionadded:: 2.6.0
+        """
+        return dt.replace(fold=fold)
+
+else:
+    class _DatetimeWithFold(datetime):
+        """
+        This is a class designed to provide a PEP 495-compliant interface for
+        Python versions before 3.6. It is used only for dates in a fold, so
+        the ``fold`` attribute is fixed at ``1``.
+
+        ..versionadded:: 2.6.0
+        """
+        __slots__ = ()
+
+        @property
+        def fold(self):
+            return 1
+
+    def enfold(dt, fold=1):
+        """
+        Provides a unified interface for assigning the ``fold`` attribute to
+        datetimes both before and after the implementation of PEP-495.
+
+        :param fold:
+            The value for the ``fold`` attribute in the returned datetime. This
+            should be either 0 or 1.
+
+        :return:
+            Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+            ``fold`` for all versions of Python. In versions prior to
+            Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+            subclass of :py:class:`datetime.datetime` with the ``fold``
+            attribute added, if ``fold`` is 1.
+
+        ..versionadded:: 2.6.0
+        """
+        if getattr(dt, 'fold', 0) == fold:
+            return dt
+
+        args = dt.timetuple()[:6]
+        args += (dt.microsecond, dt.tzinfo)
+
+        if fold:
+            return _DatetimeWithFold(*args)
+        else:
+            return datetime(*args)
+
+
+class _tzinfo(tzinfo):
+    """
+    Base class for all ``dateutil`` ``tzinfo`` objects.
+    """
+
+    def is_ambiguous(self, dt):
+        """
+        Whether or not the "wall time" of a given datetime is ambiguous in this
+        zone.
+
+        :param dt:
+            A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+        :return:
+            Returns ``True`` if ambiguous, ``False`` otherwise.
+
+        ..versionadded:: 2.6.0
+        """
+
+        dt = dt.replace(tzinfo=self)
+
+        wall_0 = enfold(dt, fold=0)
+        wall_1 = enfold(dt, fold=1)
+
+        same_offset = wall_0.utcoffset() == wall_1.utcoffset()
+        same_dt = wall_0.replace(tzinfo=None) == wall_1.replace(tzinfo=None)
+        
+        return same_dt and not same_offset
+
+    def _fold_status(self, dt_utc, dt_wall):
+        """
+        Determine the fold status of a "wall" datetime, given a representation
+        of the same datetime as a (naive) UTC datetime. This is calculated based
+        on the assumption that ``dt.utcoffset() - dt.dst()`` is constant for all
+        datetimes, and that this offset is the actual number of hours separating
+        ``dt_utc`` and ``dt_wall``.
+
+        :param dt_utc:
+            Representation of the datetime as UTC
+
+        :param dt_wall:
+            Representation of the datetime as "wall time". This parameter must
+            either have a `fold` attribute or have a fold-naive
+            :class:`datetime.tzinfo` attached, otherwise the calculation may
+            fail.
+        """
+        if self.is_ambiguous(dt_wall):
+            delta_wall = dt_wall - dt_utc
+            _fold = int(delta_wall == (dt_utc.utcoffset() - dt_utc.dst()))
+        else:
+            _fold = 0
+
+        return _fold
+
+    def _fold(self, dt):
+        return getattr(dt, 'fold', 0)
+
+    def _fromutc(self, dt):
+        """
+        Given a timezone-aware datetime in a given timezone, calculates a
+        timezone-aware datetime in a new timezone.
+
+        Since this is the one time that we *know* we have an unambiguous
+        datetime object, we take this opportunity to determine whether the
+        datetime is ambiguous and in a "fold" state (e.g. if it's the first
+        occurence, chronologically, of the ambiguous datetime).
+
+        :param dt:
+            A timezone-aware :class:`datetime.dateime` object.
+        """
+
+        # Re-implement the algorithm from Python's datetime.py
+        if not isinstance(dt, datetime):
+            raise TypeError("fromutc() requires a datetime argument")
+        if dt.tzinfo is not self:
+            raise ValueError("dt.tzinfo is not self")
+
+        dtoff = dt.utcoffset()
+        if dtoff is None:
+            raise ValueError("fromutc() requires a non-None utcoffset() "
+                             "result")
+
+        # The original datetime.py code assumes that `dst()` defaults to
+        # zero during ambiguous times. PEP 495 inverts this presumption, so
+        # for pre-PEP 495 versions of python, we need to tweak the algorithm.
+        dtdst = dt.dst()
+        if dtdst is None:
+            raise ValueError("fromutc() requires a non-None dst() result")
+        delta = dtoff - dtdst
+        if delta:
+            dt += delta
+            # Set fold=1 so we can default to being in the fold for
+            # ambiguous dates.
+            dtdst = enfold(dt, fold=1).dst()
+            if dtdst is None:
+                raise ValueError("fromutc(): dt.dst gave inconsistent "
+                                 "results; cannot convert")
+        return dt + dtdst
+
+    def fromutc(self, dt):
+        """
+        Given a timezone-aware datetime in a given timezone, calculates a
+        timezone-aware datetime in a new timezone.
+
+        Since this is the one time that we *know* we have an unambiguous
+        datetime object, we take this opportunity to determine whether the
+        datetime is ambiguous and in a "fold" state (e.g. if it's the first
+        occurance, chronologically, of the ambiguous datetime).
+
+        :param dt:
+            A timezone-aware :class:`datetime.dateime` object.
+        """
+        dt_wall = self._fromutc(dt)
+
+        # Calculate the fold status given the two datetimes.
+        _fold = self._fold_status(dt, dt_wall)
+
+        # Set the default fold value for ambiguous dates
+        return enfold(dt_wall, fold=_fold)
+
+
+class tzrangebase(_tzinfo):
+    """
+    This is an abstract base class for time zones represented by an annual
+    transition into and out of DST. Child classes should implement the following
+    methods:
+
+        * ``__init__(self, *args, **kwargs)``
+        * ``transitions(self, year)`` - this is expected to return a tuple of
+          datetimes representing the DST on and off transitions in standard
+          time.
+
+    A fully initialized ``tzrangebase`` subclass should also provide the
+    following attributes:
+        * ``hasdst``: Boolean whether or not the zone uses DST.
+        * ``_dst_offset`` / ``_std_offset``: :class:`datetime.timedelta` objects
+          representing the respective UTC offsets.
+        * ``_dst_abbr`` / ``_std_abbr``: Strings representing the timezone short
+          abbreviations in DST and STD, respectively.
+        * ``_hasdst``: Whether or not the zone has DST.
+
+    ..versionadded:: 2.6.0
+    """
+    def __init__(self):
+        raise NotImplementedError('tzrangebase is an abstract base class')
+
+    def utcoffset(self, dt):
+        isdst = self._isdst(dt)
+
+        if isdst is None:
+            return None
+        elif isdst:
+            return self._dst_offset
+        else:
+            return self._std_offset
+
+    def dst(self, dt):
+        isdst = self._isdst(dt)
+
+        if isdst is None:
+            return None
+        elif isdst:
+            return self._dst_base_offset
+        else:
+            return ZERO
+
+    @tzname_in_python2
+    def tzname(self, dt):
+        if self._isdst(dt):
+            return self._dst_abbr
+        else:
+            return self._std_abbr
+
+    def fromutc(self, dt):
+        """ Given a datetime in UTC, return local time """
+        if not isinstance(dt, datetime):
+            raise TypeError("fromutc() requires a datetime argument")
+
+        if dt.tzinfo is not self:
+            raise ValueError("dt.tzinfo is not self")
+
+        # Get transitions - if there are none, fixed offset
+        transitions = self.transitions(dt.year)
+        if transitions is None:
+            return dt + self.utcoffset(dt)
+
+        # Get the transition times in UTC
+        dston, dstoff = transitions
+
+        dston -= self._std_offset
+        dstoff -= self._std_offset
+
+        utc_transitions = (dston, dstoff)
+        dt_utc = dt.replace(tzinfo=None)
+
+
+        isdst = self._naive_isdst(dt_utc, utc_transitions)
+
+        if isdst:
+            dt_wall = dt + self._dst_offset
+        else:
+            dt_wall = dt + self._std_offset
+
+        _fold = int(not isdst and self.is_ambiguous(dt_wall))
+
+        return enfold(dt_wall, fold=_fold)
+
+    def is_ambiguous(self, dt):
+        """
+        Whether or not the "wall time" of a given datetime is ambiguous in this
+        zone.
+
+        :param dt:
+            A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+        :return:
+            Returns ``True`` if ambiguous, ``False`` otherwise.
+
+        .. versionadded:: 2.6.0
+        """
+        if not self.hasdst:
+            return False
+
+        start, end = self.transitions(dt.year)
+
+        dt = dt.replace(tzinfo=None)
+        return (end <= dt < end + self._dst_base_offset)
+
+    def _isdst(self, dt):
+        if not self.hasdst:
+            return False
+        elif dt is None:
+            return None
+
+        transitions = self.transitions(dt.year)
+
+        if transitions is None:
+            return False
+
+        dt = dt.replace(tzinfo=None)
+
+        isdst = self._naive_isdst(dt, transitions)
+
+        # Handle ambiguous dates
+        if not isdst and self.is_ambiguous(dt):
+            return not self._fold(dt)
+        else:
+            return isdst
+
+    def _naive_isdst(self, dt, transitions):
+        dston, dstoff = transitions
+
+        dt = dt.replace(tzinfo=None)
+
+        if dston < dstoff:
+            isdst = dston <= dt < dstoff
+        else:
+            isdst = not dstoff <= dt < dston
+
+        return isdst
+
+    @property
+    def _dst_base_offset(self):
+        return self._dst_offset - self._std_offset
+    
+    __hash__ = None
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "%s(...)" % self.__class__.__name__
+
+    __reduce__ = object.__reduce__
+
+
+def _total_seconds(td):
+    # Python 2.6 doesn't have a total_seconds() method on timedelta objects
+    return ((td.seconds + td.days * 86400) * 1000000 +
+            td.microseconds) // 1000000
+
+_total_seconds = getattr(timedelta, 'total_seconds', _total_seconds)

Contents/Libraries/Shared/dateutil/tz/win.py

@@ -0,0 +1,332 @@
+# This code was originally contributed by Jeffrey Harris.
+import datetime
+import struct
+
+from six.moves import winreg
+from six import text_type
+
+try:
+    import ctypes
+    from ctypes import wintypes
+except ValueError:
+    # ValueError is raised on non-Windows systems for some horrible reason.
+    raise ImportError("Running tzwin on non-Windows system")
+
+from ._common import tzname_in_python2, _tzinfo
+from ._common import tzrangebase
+
+__all__ = ["tzwin", "tzwinlocal", "tzres"]
+
+ONEWEEK = datetime.timedelta(7)
+
+TZKEYNAMENT = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones"
+TZKEYNAME9X = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones"
+TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation"
+
+
+def _settzkeyname():
+    handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE)
+    try:
+        winreg.OpenKey(handle, TZKEYNAMENT).Close()
+        TZKEYNAME = TZKEYNAMENT
+    except WindowsError:
+        TZKEYNAME = TZKEYNAME9X
+    handle.Close()
+    return TZKEYNAME
+
+TZKEYNAME = _settzkeyname()
+
+
+class tzres(object):
+    """
+    Class for accessing `tzres.dll`, which contains timezone name related
+    resources.
+
+    .. versionadded:: 2.5.0
+    """
+    p_wchar = ctypes.POINTER(wintypes.WCHAR)        # Pointer to a wide char
+
+    def __init__(self, tzres_loc='tzres.dll'):
+        # Load the user32 DLL so we can load strings from tzres
+        user32 = ctypes.WinDLL('user32')
+        
+        # Specify the LoadStringW function
+        user32.LoadStringW.argtypes = (wintypes.HINSTANCE,
+                                       wintypes.UINT,
+                                       wintypes.LPWSTR,
+                                       ctypes.c_int)
+
+        self.LoadStringW = user32.LoadStringW
+        self._tzres = ctypes.WinDLL(tzres_loc)
+        self.tzres_loc = tzres_loc
+
+    def load_name(self, offset):
+        """
+        Load a timezone name from a DLL offset (integer).
+        
+        >>> from dateutil.tzwin import tzres
+        >>> tzr = tzres()
+        >>> print(tzr.load_name(112))
+        'Eastern Standard Time'
+
+        :param offset:
+            A positive integer value referring to a string from the tzres dll.
+
+        ..note:
+            Offsets found in the registry are generally of the form
+            `@tzres.dll,-114`. The offset in this case if 114, not -114.
+
+        """
+        resource = self.p_wchar()
+        lpBuffer = ctypes.cast(ctypes.byref(resource), wintypes.LPWSTR)
+        nchar = self.LoadStringW(self._tzres._handle, offset, lpBuffer, 0)
+        return resource[:nchar]
+
+    def name_from_string(self, tzname_str):
+        """
+        Parse strings as returned from the Windows registry into the time zone
+        name as defined in the registry.
+
+        >>> from dateutil.tzwin import tzres
+        >>> tzr = tzres()
+        >>> print(tzr.name_from_string('@tzres.dll,-251'))
+        'Dateline Daylight Time'
+        >>> print(tzr.name_from_string('Eastern Standard Time'))
+        'Eastern Standard Time'
+
+        :param tzname_str:
+            A timezone name string as returned from a Windows registry key.
+
+        :return:
+            Returns the localized timezone string from tzres.dll if the string
+            is of the form `@tzres.dll,-offset`, else returns the input string.
+        """
+        if not tzname_str.startswith('@'):
+            return tzname_str
+
+        name_splt = tzname_str.split(',-')
+        try:
+            offset = int(name_splt[1])
+        except:
+            raise ValueError("Malformed timezone string.")
+
+        return self.load_name(offset)
+
+
+class tzwinbase(tzrangebase):
+    """tzinfo class based on win32's timezones available in the registry."""
+    def __init__(self):
+        raise NotImplementedError('tzwinbase is an abstract base class')
+
+    def __eq__(self, other):
+        # Compare on all relevant dimensions, including name.
+        if not isinstance(other, tzwinbase):
+            return NotImplemented
+
+        return  (self._std_offset == other._std_offset and
+                 self._dst_offset == other._dst_offset and
+                 self._stddayofweek == other._stddayofweek and
+                 self._dstdayofweek == other._dstdayofweek and
+                 self._stdweeknumber == other._stdweeknumber and
+                 self._dstweeknumber == other._dstweeknumber and
+                 self._stdhour == other._stdhour and
+                 self._dsthour == other._dsthour and
+                 self._stdminute == other._stdminute and
+                 self._dstminute == other._dstminute and
+                 self._std_abbr == other._std_abbr and
+                 self._dst_abbr == other._dst_abbr)
+
+    @staticmethod
+    def list():
+        """Return a list of all time zones known to the system."""
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            with winreg.OpenKey(handle, TZKEYNAME) as tzkey:
+                result = [winreg.EnumKey(tzkey, i)
+                          for i in range(winreg.QueryInfoKey(tzkey)[0])]
+        return result
+
+    def display(self):
+        return self._display
+
+    def transitions(self, year):
+        """
+        For a given year, get the DST on and off transition times, expressed
+        always on the standard time side. For zones with no transitions, this
+        function returns ``None``.
+
+        :param year:
+            The year whose transitions you would like to query.
+
+        :return:
+            Returns a :class:`tuple` of :class:`datetime.datetime` objects,
+            ``(dston, dstoff)`` for zones with an annual DST transition, or
+            ``None`` for fixed offset zones.
+        """
+
+        if not self.hasdst:
+            return None
+
+        dston = picknthweekday(year, self._dstmonth, self._dstdayofweek,
+                               self._dsthour, self._dstminute,
+                               self._dstweeknumber)
+
+        dstoff = picknthweekday(year, self._stdmonth, self._stddayofweek,
+                                self._stdhour, self._stdminute,
+                                self._stdweeknumber)
+
+        # Ambiguous dates default to the STD side
+        dstoff -= self._dst_base_offset
+
+        return dston, dstoff
+
+    def _get_hasdst(self):
+        return self._dstmonth != 0
+
+    @property
+    def _dst_base_offset(self):
+        return self._dst_base_offset_
+
+
+class tzwin(tzwinbase):
+
+    def __init__(self, name):
+        self._name = name
+
+        # multiple contexts only possible in 2.7 and 3.1, we still support 2.6
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            tzkeyname = text_type("{kn}\{name}").format(kn=TZKEYNAME, name=name)
+            with winreg.OpenKey(handle, tzkeyname) as tzkey:
+                keydict = valuestodict(tzkey)
+
+        self._std_abbr = keydict["Std"]
+        self._dst_abbr = keydict["Dlt"]
+
+        self._display = keydict["Display"]
+
+        # See http://ww_winreg.jsiinc.com/SUBA/tip0300/rh0398.htm
+        tup = struct.unpack("=3l16h", keydict["TZI"])
+        stdoffset = -tup[0]-tup[1]          # Bias + StandardBias * -1
+        dstoffset = stdoffset-tup[2]        # + DaylightBias * -1
+        self._std_offset = datetime.timedelta(minutes=stdoffset)
+        self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+        # for the meaning see the win32 TIME_ZONE_INFORMATION structure docs
+        # http://msdn.microsoft.com/en-us/library/windows/desktop/ms725481(v=vs.85).aspx
+        (self._stdmonth,
+         self._stddayofweek,   # Sunday = 0
+         self._stdweeknumber,  # Last = 5
+         self._stdhour,
+         self._stdminute) = tup[4:9]
+
+        (self._dstmonth,
+         self._dstdayofweek,   # Sunday = 0
+         self._dstweeknumber,  # Last = 5
+         self._dsthour,
+         self._dstminute) = tup[12:17]
+
+        self._dst_base_offset_ = self._dst_offset - self._std_offset
+        self.hasdst = self._get_hasdst()
+
+    def __repr__(self):
+        return "tzwin(%s)" % repr(self._name)
+
+    def __reduce__(self):
+        return (self.__class__, (self._name,))
+
+
+class tzwinlocal(tzwinbase):
+    def __init__(self):
+        with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+            with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey:
+                keydict = valuestodict(tzlocalkey)
+
+            self._std_abbr = keydict["StandardName"]
+            self._dst_abbr = keydict["DaylightName"]
+
+            try:
+                tzkeyname = text_type('{kn}\{sn}').format(kn=TZKEYNAME,
+                                                          sn=self._std_abbr)
+                with winreg.OpenKey(handle, tzkeyname) as tzkey:
+                    _keydict = valuestodict(tzkey)
+                    self._display = _keydict["Display"]
+            except OSError:
+                self._display = None
+
+        stdoffset = -keydict["Bias"]-keydict["StandardBias"]
+        dstoffset = stdoffset-keydict["DaylightBias"]
+
+        self._std_offset = datetime.timedelta(minutes=stdoffset)
+        self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+        # For reasons unclear, in this particular key, the day of week has been
+        # moved to the END of the SYSTEMTIME structure.
+        tup = struct.unpack("=8h", keydict["StandardStart"])
+
+        (self._stdmonth,
+         self._stdweeknumber,  # Last = 5
+         self._stdhour,
+         self._stdminute) = tup[1:5]
+        
+        self._stddayofweek = tup[7]
+
+        tup = struct.unpack("=8h", keydict["DaylightStart"])
+
+        (self._dstmonth,
+         self._dstweeknumber,  # Last = 5
+         self._dsthour,
+         self._dstminute) = tup[1:5]
+
+        self._dstdayofweek = tup[7]
+
+        self._dst_base_offset_ = self._dst_offset - self._std_offset
+        self.hasdst = self._get_hasdst()
+
+    def __repr__(self):
+        return "tzwinlocal()"
+
+    def __str__(self):
+        # str will return the standard name, not the daylight name.
+        return "tzwinlocal(%s)" % repr(self._std_abbr)
+
+    def __reduce__(self):
+        return (self.__class__, ())
+
+
+def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
+    """ dayofweek == 0 means Sunday, whichweek 5 means last instance """
+    first = datetime.datetime(year, month, 1, hour, minute)
+
+    # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
+    # Because 7 % 7 = 0
+    weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
+    wd = weekdayone + ((whichweek - 1) * ONEWEEK)
+    if (wd.month != month):
+        wd -= ONEWEEK
+
+    return wd
+
+
+def valuestodict(key):
+    """Convert a registry key's values to a dictionary."""
+    dout = {}
+    size = winreg.QueryInfoKey(key)[1]
+    tz_res = None
+
+    for i in range(size):
+        key_name, value, dtype = winreg.EnumValue(key, i)
+        if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
+            # If it's a DWORD (32-bit integer), it's stored as unsigned - convert
+            # that to a proper signed integer
+            if value & (1 << 31):
+                value = value - (1 << 32)
+        elif dtype == winreg.REG_SZ:
+            # If it's a reference to the tzres DLL, load the actual string
+            if value.startswith('@tzres'):
+                tz_res = tz_res or tzres()
+                value = tz_res.name_from_string(value)
+
+            value = value.rstrip('\x00')    # Remove trailing nulls
+
+        dout[key_name] = value
+
+    return dout

Contents/Libraries/Shared/dateutil/tzwin.py

@@ -0,0 +1,2 @@
+# tzwin has moved to dateutil.tz.win
+from .tz.win import *

Contents/Libraries/Shared/dateutil/zoneinfo/__init__.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+import logging
+import os
+import warnings
+import tempfile
+import shutil
+import json
+
+from tarfile import TarFile
+from pkgutil import get_data
+from io import BytesIO
+from contextlib import closing
+
+from dateutil.tz import tzfile
+
+__all__ = ["get_zonefile_instance", "gettz", "gettz_db_metadata", "rebuild"]
+
+ZONEFILENAME = "dateutil-zoneinfo.tar.gz"
+METADATA_FN = 'METADATA'
+
+# python2.6 compatability. Note that TarFile.__exit__ != TarFile.close, but
+# it's close enough for python2.6
+tar_open = TarFile.open
+if not hasattr(TarFile, '__exit__'):
+    def tar_open(*args, **kwargs):
+        return closing(TarFile.open(*args, **kwargs))
+
+
+class tzfile(tzfile):
+    def __reduce__(self):
+        return (gettz, (self._filename,))
+
+
+def getzoneinfofile_stream():
+    try:
+        return BytesIO(get_data(__name__, ZONEFILENAME))
+    except IOError as e:  # TODO  switch to FileNotFoundError?
+        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
+        return None
+
+
+class ZoneInfoFile(object):
+    def __init__(self, zonefile_stream=None):
+        if zonefile_stream is not None:
+            with tar_open(fileobj=zonefile_stream, mode='r') as tf:
+                # dict comprehension does not work on python2.6
+                # TODO: get back to the nicer syntax when we ditch python2.6
+                # self.zones = {zf.name: tzfile(tf.extractfile(zf),
+                #               filename = zf.name)
+                #              for zf in tf.getmembers() if zf.isfile()}
+                self.zones = dict((zf.name, tzfile(tf.extractfile(zf),
+                                                   filename=zf.name))
+                                  for zf in tf.getmembers()
+                                  if zf.isfile() and zf.name != METADATA_FN)
+                # deal with links: They'll point to their parent object. Less
+                # waste of memory
+                # links = {zl.name: self.zones[zl.linkname]
+                #        for zl in tf.getmembers() if zl.islnk() or zl.issym()}
+                links = dict((zl.name, self.zones[zl.linkname])
+                             for zl in tf.getmembers() if
+                             zl.islnk() or zl.issym())
+                self.zones.update(links)
+                try:
+                    metadata_json = tf.extractfile(tf.getmember(METADATA_FN))
+                    metadata_str = metadata_json.read().decode('UTF-8')
+                    self.metadata = json.loads(metadata_str)
+                except KeyError:
+                    # no metadata in tar file
+                    self.metadata = None
+        else:
+            self.zones = dict()
+            self.metadata = None
+
+    def get(self, name, default=None):
+        """
+        Wrapper for :func:`ZoneInfoFile.zones.get`. This is a convenience method
+        for retrieving zones from the zone dictionary.
+        
+        :param name:
+            The name of the zone to retrieve. (Generally IANA zone names)
+
+        :param default:
+            The value to return in the event of a missing key.
+
+        .. versionadded:: 2.6.0
+
+        """
+        return self.zones.get(name, default)
+
+
+# The current API has gettz as a module function, although in fact it taps into
+# a stateful class. So as a workaround for now, without changing the API, we
+# will create a new "global" class instance the first time a user requests a
+# timezone. Ugly, but adheres to the api.
+#
+# TODO: Remove after deprecation period.
+_CLASS_ZONE_INSTANCE = list()
+
+def get_zonefile_instance(new_instance=False):
+    """
+    This is a convenience function which provides a :class:`ZoneInfoFile`
+    instance using the data provided by the ``dateutil`` package. By default, it
+    caches a single instance of the ZoneInfoFile object and returns that.
+
+    :param new_instance:
+        If ``True``, a new instance of :class:`ZoneInfoFile` is instantiated and
+        used as the cached instance for the next call. Otherwise, new instances
+        are created only as necessary.
+
+    :return:
+        Returns a :class:`ZoneInfoFile` object.
+
+    .. versionadded:: 2.6
+    """
+    if new_instance:
+        zif = None
+    else:
+        zif = getattr(get_zonefile_instance, '_cached_instance', None)
+
+    if zif is None:
+        zif = ZoneInfoFile(getzoneinfofile_stream())
+
+        get_zonefile_instance._cached_instance = zif
+
+    return zif
+
+def gettz(name):
+    """
+    This retrieves a time zone from the local zoneinfo tarball that is packaged
+    with dateutil.
+
+    :param name:
+        An IANA-style time zone name, as found in the zoneinfo file.
+
+    :return:
+        Returns a :class:`dateutil.tz.tzfile` time zone object.
+
+    .. warning::
+        It is generally inadvisable to use this function, and it is only
+        provided for API compatibility with earlier versions. This is *not*
+        equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
+        time zone based on the inputs, favoring system zoneinfo. This is ONLY
+        for accessing the dateutil-specific zoneinfo (which may be out of
+        date compared to the system zoneinfo).
+
+    .. deprecated:: 2.6
+        If you need to use a specific zoneinfofile over the system zoneinfo,
+        instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
+        :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.
+
+        Use :func:`get_zonefile_instance` to retrieve an instance of the
+        dateutil-provided zoneinfo.
+    """
+    warnings.warn("zoneinfo.gettz() will be removed in future versions, "
+                  "to use the dateutil-provided zoneinfo files, instantiate a "
+                  "ZoneInfoFile object and use ZoneInfoFile.zones.get() "
+                  "instead. See the documentation for details.",
+                  DeprecationWarning)
+
+    if len(_CLASS_ZONE_INSTANCE) == 0:
+        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+    return _CLASS_ZONE_INSTANCE[0].zones.get(name)
+
+
+def gettz_db_metadata():
+    """ Get the zonefile metadata
+
+    See `zonefile_metadata`_
+
+    :returns:
+        A dictionary with the database metadata
+
+    .. deprecated:: 2.6
+        See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
+        query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
+    """
+    warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
+                  "versions, to use the dateutil-provided zoneinfo files, "
+                  "ZoneInfoFile object and query the 'metadata' attribute "
+                  "instead. See the documentation for details.",
+                  DeprecationWarning)
+
+    if len(_CLASS_ZONE_INSTANCE) == 0:
+        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+    return _CLASS_ZONE_INSTANCE[0].metadata
+
+

Contents/Libraries/Shared/dateutil/zoneinfo/rebuild.py

@@ -0,0 +1,51 @@
+import logging
+import os
+import tempfile
+import shutil
+import json
+from subprocess import check_call
+
+from dateutil.zoneinfo import tar_open, METADATA_FN, ZONEFILENAME
+
+
+def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
+    """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
+
+    filename is the timezone tarball from ftp.iana.org/tz.
+
+    """
+    tmpdir = tempfile.mkdtemp()
+    zonedir = os.path.join(tmpdir, "zoneinfo")
+    moduledir = os.path.dirname(__file__)
+    try:
+        with tar_open(filename) as tf:
+            for name in zonegroups:
+                tf.extract(name, tmpdir)
+            filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
+            try:
+                check_call(["zic", "-d", zonedir] + filepaths)
+            except OSError as e:
+                _print_on_nosuchfile(e)
+                raise
+        # write metadata file
+        with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
+            json.dump(metadata, f, indent=4, sort_keys=True)
+        target = os.path.join(moduledir, ZONEFILENAME)
+        with tar_open(target, "w:%s" % format) as tf:
+            for entry in os.listdir(zonedir):
+                entrypath = os.path.join(zonedir, entry)
+                tf.add(entrypath, entry)
+    finally:
+        shutil.rmtree(tmpdir)
+
+def _print_on_nosuchfile(e):
+    """Print helpful troubleshooting message
+
+    e is an exception raised by subprocess.check_call()
+
+    """
+    if e.errno == 2:
+        logging.error(
+            "Could not find zic. Perhaps you need to install "
+            "libc-bin or some other package that provides it, "
+            "or it's not in your PATH?")

Contents/Libraries/Shared/dogpile/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2011-2014 Mike Bayer
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-3. The name of the author or contributors may not be used to endorse or
-   promote products derived from this software without specific prior
-   written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.

Contents/Libraries/Shared/dogpile/README.rst

@@ -1,91 +0,0 @@
-dogpile.cache
-=============
-
-A caching API built around the concept of a "dogpile lock", which allows
-continued access to an expiring data value while a single thread generates a
-new value.
-
-dogpile.cache builds on the `dogpile.core <http://pypi.python.org/pypi/dogpile.core>`_
-locking system, which implements the idea of "allow one creator to write while
-others read" in the abstract.   Overall, dogpile.cache is intended as a
-replacement to the `Beaker <http://beaker.groovie.org>`_ caching system, the internals
-of which are written by the same author.   All the ideas of Beaker which "work"
-are re-implemented in dogpile.cache in a more efficient and succinct manner,
-and all the cruft (Beaker's internals were first written in 2005) relegated
-to the trash heap.
-
-Features
---------
-
-* A succinct API which encourages up-front configuration of pre-defined
-  "regions", each one defining a set of caching characteristics including
-  storage backend, configuration options, and default expiration time.
-* A standard get/set/delete API as well as a function decorator API is
-  provided.
-* The mechanics of key generation are fully customizable.   The function
-  decorator API features a pluggable "key generator" to customize how
-  cache keys are made to correspond to function calls, and an optional
-  "key mangler" feature provides for pluggable mangling of keys
-  (such as encoding, SHA-1 hashing) as desired for each region.
-* The dogpile lock, first developed as the core engine behind the Beaker
-  caching system, here vastly simplified, improved, and better tested.
-  Some key performance
-  issues that were intrinsic to Beaker's architecture, particularly that
-  values would frequently be "double-fetched" from the cache, have been fixed.
-* Backends implement their own version of a "distributed" lock, where the
-  "distribution" matches the backend's storage system.  For example, the
-  memcached backends allow all clients to coordinate creation of values
-  using memcached itself.   The dbm file backend uses a lockfile
-  alongside the dbm file.  New backends, such as a Redis-based backend,
-  can provide their own locking mechanism appropriate to the storage
-  engine.
-* Writing new backends or hacking on the existing backends is intended to be
-  routine - all that's needed are basic get/set/delete methods. A distributed
-  lock tailored towards the backend is an optional addition, else dogpile uses
-  a regular thread mutex. New backends can be registered with dogpile.cache
-  directly or made available via setuptools entry points.
-* Included backends feature three memcached backends (python-memcached, pylibmc,
-  bmemcached), a Redis backend, a backend based on Python's
-  anydbm, and a plain dictionary backend.
-* Space for third party plugins, including the first which provides the
-  dogpile.cache engine to Mako templates.
-* Python 3 compatible in place - no 2to3 required.
-
-Synopsis
---------
-
-dogpile.cache features a single public usage object known as the ``CacheRegion``.
-This object then refers to a particular ``CacheBackend``.   Typical usage
-generates a region using ``make_region()``, which can then be used at the
-module level to decorate functions, or used directly in code with a traditional
-get/set interface.   Configuration of the backend is applied to the region
-using ``configure()`` or ``configure_from_config()``, allowing deferred
-config-file based configuration to occur after modules have been imported::
-
-    from dogpile.cache import make_region
-
-    region = make_region().configure(
-        'dogpile.cache.pylibmc',
-        expiration_time = 3600,
-        arguments = {
-            'url':["127.0.0.1"],
-            'binary':True,
-            'behaviors':{"tcp_nodelay": True,"ketama":True}
-        }
-    )
-
-    @region.cache_on_arguments()
-    def load_user_info(user_id):
-        return some_database.lookup_user_by_id(user_id)
-
-
-Documentation
--------------
-
-See dogpile.cache's full documentation at
-`dogpile.cache documentation <http://dogpilecache.readthedocs.org>`_.
-
-
-
-
-

Contents/Libraries/Shared/dogpile/__init__.py

@@ -1,6 +1,4 @@
-# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
-try:
-    __import__('pkg_resources').declare_namespace(__name__)
-except ImportError:
-    from pkgutil import extend_path
-    __path__ = extend_path(__path__, __name__)
+__version__ = '0.6.2'
+
+from .lock import Lock  # noqa
+from .lock import NeedRegenerationException  # noqa

Contents/Libraries/Shared/dogpile/cache/__init__.py

@@ -1,3 +1,4 @@
-__version__ = '0.5.4'
+from .region import CacheRegion, register_backend, make_region  # noqa
 
-from .region import CacheRegion, register_backend, make_region
+# backwards compat
+from .. import __version__  # noqa

Contents/Libraries/Shared/dogpile/cache/api.py

@@ -1,5 +1,5 @@
 import operator
-from .compat import py3k
+from ..util.compat import py3k
 
 
 class NoValue(object):
@@ -14,16 +14,17 @@ class NoValue(object):
         return self
 
     if py3k:
-        def __bool__(self): #pragma NO COVERAGE
+        def __bool__(self):  # pragma NO COVERAGE
             return False
     else:
-        def __nonzero__(self): #pragma NO COVERAGE
+        def __nonzero__(self):  # pragma NO COVERAGE
             return False
 
 NO_VALUE = NoValue()
 """Value returned from ``get()`` that describes
 a  key not present."""
 
+
 class CachedValue(tuple):
     """Represent a value stored in the cache.
 
@@ -47,6 +48,7 @@ class CachedValue(tuple):
     def __reduce__(self):
         return CachedValue, (self.payload, self.metadata)
 
+
 class CacheBackend(object):
     """Base class for backend implementations."""
 
@@ -58,7 +60,7 @@ class CacheBackend(object):
 
     """
 
-    def __init__(self, arguments): #pragma NO COVERAGE
+    def __init__(self, arguments):  # pragma NO COVERAGE
         """Construct a new :class:`.CacheBackend`.
 
         Subclasses should override this to
@@ -74,12 +76,15 @@ class CacheBackend(object):
     def from_config_dict(cls, config_dict, prefix):
         prefix_len = len(prefix)
         return cls(
-                dict(
-                    (key[prefix_len:], config_dict[key])
-                    for key in config_dict
-                    if key.startswith(prefix)
-                )
+            dict(
+                (key[prefix_len:], config_dict[key])
+                for key in config_dict
+                if key.startswith(prefix)
             )
+        )
+
+    def has_lock_timeout(self):
+        return False
 
     def get_mutex(self, key):
         """Return an optional mutexing object for the given key.
@@ -114,7 +119,7 @@ class CacheBackend(object):
         """
         return None
 
-    def get(self, key): #pragma NO COVERAGE
+    def get(self, key):  # pragma NO COVERAGE
         """Retrieve a value from the cache.
 
         The returned value should be an instance of
@@ -124,7 +129,7 @@ class CacheBackend(object):
         """
         raise NotImplementedError()
 
-    def get_multi(self, keys): #pragma NO COVERAGE
+    def get_multi(self, keys):  # pragma NO COVERAGE
         """Retrieve multiple values from the cache.
 
         The returned value should be a list, corresponding
@@ -135,7 +140,7 @@ class CacheBackend(object):
         """
         raise NotImplementedError()
 
-    def set(self, key, value): #pragma NO COVERAGE
+    def set(self, key, value):  # pragma NO COVERAGE
         """Set a value in the cache.
 
         The key will be whatever was passed
@@ -147,21 +152,30 @@ class CacheBackend(object):
         """
         raise NotImplementedError()
 
-    def set_multi(self, mapping): #pragma NO COVERAGE
+    def set_multi(self, mapping):  # pragma NO COVERAGE
         """Set multiple values in the cache.
 
-        The key will be whatever was passed
+        ``mapping`` is a dict in which
+        the key will be whatever was passed
         to the registry, processed by the
         "key mangling" function, if any.
         The value will always be an instance
         of :class:`.CachedValue`.
 
+        When implementing a new :class:`.CacheBackend` or cutomizing via
+        :class:`.ProxyBackend`, be aware that when this method is invoked by
+        :meth:`.Region.get_or_create_multi`, the ``mapping`` values are the
+        same ones returned to the upstream caller. If the subclass alters the
+        values in any way, it must not do so 'in-place' on the ``mapping`` dict
+        -- that will have the undesirable effect of modifying the returned
+        values as well.
+
         .. versionadded:: 0.5.0
 
         """
         raise NotImplementedError()
 
-    def delete(self, key): #pragma NO COVERAGE
+    def delete(self, key):  # pragma NO COVERAGE
         """Delete a value from the cache.
 
         The key will be whatever was passed
@@ -175,7 +189,7 @@ class CacheBackend(object):
         """
         raise NotImplementedError()
 
-    def delete_multi(self, keys): #pragma NO COVERAGE
+    def delete_multi(self, keys):  # pragma NO COVERAGE
         """Delete multiple values from the cache.
 
         The key will be whatever was passed

Contents/Libraries/Shared/dogpile/cache/backends/__init__.py

@@ -1,10 +1,22 @@
 from dogpile.cache.region import register_backend
 
-register_backend("dogpile.cache.null", "dogpile.cache.backends.null", "NullBackend")
-register_backend("dogpile.cache.dbm", "dogpile.cache.backends.file", "DBMBackend")
-register_backend("dogpile.cache.pylibmc", "dogpile.cache.backends.memcached", "PylibmcBackend")
-register_backend("dogpile.cache.bmemcached", "dogpile.cache.backends.memcached", "BMemcachedBackend")
-register_backend("dogpile.cache.memcached", "dogpile.cache.backends.memcached", "MemcachedBackend")
-register_backend("dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend")
-register_backend("dogpile.cache.memory_pickle", "dogpile.cache.backends.memory", "MemoryPickleBackend")
-register_backend("dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend")
+register_backend(
+    "dogpile.cache.null", "dogpile.cache.backends.null", "NullBackend")
+register_backend(
+    "dogpile.cache.dbm", "dogpile.cache.backends.file", "DBMBackend")
+register_backend(
+    "dogpile.cache.pylibmc", "dogpile.cache.backends.memcached",
+    "PylibmcBackend")
+register_backend(
+    "dogpile.cache.bmemcached", "dogpile.cache.backends.memcached",
+    "BMemcachedBackend")
+register_backend(
+    "dogpile.cache.memcached", "dogpile.cache.backends.memcached",
+    "MemcachedBackend")
+register_backend(
+    "dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend")
+register_backend(
+    "dogpile.cache.memory_pickle", "dogpile.cache.backends.memory",
+    "MemoryPickleBackend")
+register_backend(
+    "dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend")

Contents/Libraries/Shared/dogpile/cache/backends/file.py

@@ -7,14 +7,15 @@ Provides backends that deal with local filesystem access.
 """
 
 from __future__ import with_statement
-from dogpile.cache.api import CacheBackend, NO_VALUE
+from ..api import CacheBackend, NO_VALUE
 from contextlib import contextmanager
-from dogpile.cache import compat
-from dogpile.cache import util
+from ...util import compat
+from ... import util
 import os
 
 __all__ = 'DBMBackend', 'FileLock', 'AbstractFileLock'
 
+
 class DBMBackend(CacheBackend):
     """A file-backend using a dbm file to store keys.
 
@@ -135,19 +136,19 @@ class DBMBackend(CacheBackend):
     """
     def __init__(self, arguments):
         self.filename = os.path.abspath(
-                            os.path.normpath(arguments['filename'])
-                        )
+            os.path.normpath(arguments['filename'])
+        )
         dir_, filename = os.path.split(self.filename)
 
         self.lock_factory = arguments.get("lock_factory", FileLock)
         self._rw_lock = self._init_lock(
-                                arguments.get('rw_lockfile'),
-                                ".rw.lock", dir_, filename)
+            arguments.get('rw_lockfile'),
+            ".rw.lock", dir_, filename)
         self._dogpile_lock = self._init_lock(
-                                arguments.get('dogpile_lockfile'),
-                                ".dogpile.lock",
-                                dir_, filename,
-                                util.KeyReentrantMutex.factory)
+            arguments.get('dogpile_lockfile'),
+            ".dogpile.lock",
+            dir_, filename,
+            util.KeyReentrantMutex.factory)
 
         # TODO: make this configurable
         if compat.py3k:
@@ -162,9 +163,9 @@ class DBMBackend(CacheBackend):
             lock = self.lock_factory(os.path.join(basedir, basefile + suffix))
         elif argument is not False:
             lock = self.lock_factory(
-                        os.path.abspath(
-                            os.path.normpath(argument)
-                        ))
+                os.path.abspath(
+                    os.path.normpath(argument)
+                ))
         else:
             return None
         if wrapper:
@@ -209,8 +210,9 @@ class DBMBackend(CacheBackend):
     @contextmanager
     def _dbm_file(self, write):
         with self._use_rw_lock(write):
-            dbm = self.dbmmodule.open(self.filename,
-                                "w" if write else "r")
+            dbm = self.dbmmodule.open(
+                self.filename,
+                "w" if write else "r")
             yield dbm
             dbm.close()
 
@@ -233,12 +235,14 @@ class DBMBackend(CacheBackend):
 
     def set(self, key, value):
         with self._dbm_file(True) as dbm:
-            dbm[key] = compat.pickle.dumps(value)
+            dbm[key] = compat.pickle.dumps(value,
+                                           compat.pickle.HIGHEST_PROTOCOL)
 
     def set_multi(self, mapping):
         with self._dbm_file(True) as dbm:
-            for key,value in mapping.items():
-                dbm[key] = compat.pickle.dumps(value)
+            for key, value in mapping.items():
+                dbm[key] = compat.pickle.dumps(value,
+                                               compat.pickle.HIGHEST_PROTOCOL)
 
     def delete(self, key):
         with self._dbm_file(True) as dbm:
@@ -255,6 +259,7 @@ class DBMBackend(CacheBackend):
                 except KeyError:
                     pass
 
+
 class AbstractFileLock(object):
     """Coordinate read/write access to a file.
 
@@ -275,10 +280,10 @@ class AbstractFileLock(object):
     file.
 
     Note that multithreaded environments must provide a thread-safe
-    version of this lock.  The recommended approach for file-descriptor-based
-    locks is to use a Python ``threading.local()`` so that a unique file descriptor
-    is held per thread.  See the source code of :class:`.FileLock` for an
-    implementation example.
+    version of this lock.  The recommended approach for file-
+    descriptor-based locks is to use a Python ``threading.local()`` so
+    that a unique file descriptor is held per thread.  See the source
+    code of :class:`.FileLock` for an implementation example.
 
 
     """
@@ -377,6 +382,7 @@ class AbstractFileLock(object):
         """
         raise NotImplementedError()
 
+
 class FileLock(AbstractFileLock):
     """Use lockfiles to coordinate read/write access to a file.
 

Contents/Libraries/Shared/dogpile/cache/backends/memcached.py

@@ -6,15 +6,16 @@ Provides backends for talking to `memcached <http://memcached.org>`_.
 
 """
 
-from dogpile.cache.api import CacheBackend, NO_VALUE
-from dogpile.cache import compat
-from dogpile.cache import util
+from ..api import CacheBackend, NO_VALUE
+from ...util import compat
+from ... import util
 import random
 import time
 
 __all__ = 'GenericMemcachedBackend', 'MemcachedBackend',\
           'PylibmcBackend', 'BMemcachedBackend', 'MemcachedLock'
 
+
 class MemcachedLock(object):
     """Simple distributed lock using memcached.
 
@@ -23,20 +24,21 @@ class MemcachedLock(object):
 
     """
 
-    def __init__(self, client_fn, key):
+    def __init__(self, client_fn, key, timeout=0):
         self.client_fn = client_fn
         self.key = "_lock" + key
+        self.timeout = timeout
 
     def acquire(self, wait=True):
         client = self.client_fn()
         i = 0
         while True:
-            if client.add(self.key, 1):
+            if client.add(self.key, 1, self.timeout):
                 return True
             elif not wait:
                 return False
             else:
-                sleep_time = (((i+1)*random.random()) + 2**i) / 2.5
+                sleep_time = (((i + 1) * random.random()) + 2 ** i) / 2.5
                 time.sleep(sleep_time)
             if i < 15:
                 i += 1
@@ -45,6 +47,7 @@ class MemcachedLock(object):
         client = self.client_fn()
         client.delete(self.key)
 
+
 class GenericMemcachedBackend(CacheBackend):
     """Base class for memcached backends.
 
@@ -60,6 +63,12 @@ class GenericMemcachedBackend(CacheBackend):
      processes will be talking to the same memcached instance.
      When left at False, dogpile will coordinate on a regular
      threading mutex.
+    :param lock_timeout: integer, number of seconds after acquiring a lock that
+     memcached should expire it.  This argument is only valid when
+     ``distributed_lock`` is ``True``.
+
+     .. versionadded:: 0.5.7
+
     :param memcached_expire_time: integer, when present will
      be passed as the ``time`` parameter to ``pylibmc.Client.set``.
      This is used to set the memcached expiry time for a value.
@@ -104,9 +113,13 @@ class GenericMemcachedBackend(CacheBackend):
         # automatically.
         self.url = util.to_list(arguments['url'])
         self.distributed_lock = arguments.get('distributed_lock', False)
+        self.lock_timeout = arguments.get('lock_timeout', 0)
         self.memcached_expire_time = arguments.get(
             'memcached_expire_time', 0)
 
+    def has_lock_timeout(self):
+        return self.lock_timeout != 0
+
     def _imports(self):
         """client library imports go here."""
         raise NotImplementedError()
@@ -118,6 +131,7 @@ class GenericMemcachedBackend(CacheBackend):
     @util.memoized_property
     def _clients(self):
         backend = self
+
         class ClientPool(compat.threading.local):
             def __init__(self):
                 self.memcached = backend._create_client()
@@ -138,7 +152,8 @@ class GenericMemcachedBackend(CacheBackend):
 
     def get_mutex(self, key):
         if self.distributed_lock:
-            return MemcachedLock(lambda: self.client, key)
+            return MemcachedLock(lambda: self.client, key,
+                                 timeout=self.lock_timeout)
         else:
             return None
 
@@ -157,13 +172,15 @@ class GenericMemcachedBackend(CacheBackend):
         ]
 
     def set(self, key, value):
-        self.client.set(key,
+        self.client.set(
+            key,
             value,
             **self.set_arguments
         )
 
     def set_multi(self, mapping):
-        self.client.set_multi(mapping,
+        self.client.set_multi(
+            mapping,
             **self.set_arguments
         )
 
@@ -173,6 +190,7 @@ class GenericMemcachedBackend(CacheBackend):
     def delete_multi(self, keys):
         self.client.delete_multi(keys)
 
+
 class MemcacheArgs(object):
     """Mixin which provides support for the 'time' argument to set(),
     'min_compress_len' to other methods.
@@ -183,13 +201,15 @@ class MemcacheArgs(object):
 
         self.set_arguments = {}
         if "memcached_expire_time" in arguments:
-            self.set_arguments["time"] =\
-            arguments["memcached_expire_time"]
+            self.set_arguments["time"] = arguments["memcached_expire_time"]
         if "min_compress_len" in arguments:
-            self.set_arguments["min_compress_len"] =\
-            arguments["min_compress_len"]
+            self.set_arguments["min_compress_len"] = \
+                arguments["min_compress_len"]
         super(MemcacheArgs, self).__init__(arguments)
 
+pylibmc = None
+
+
 class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
     """A backend for the
     `pylibmc <http://sendapatch.se/projects/pylibmc/index.html>`_
@@ -229,19 +249,24 @@ class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
         self.behaviors = arguments.get('behaviors', {})
         super(PylibmcBackend, self).__init__(arguments)
 
-
     def _imports(self):
         global pylibmc
-        import pylibmc
+        import pylibmc  # noqa
 
     def _create_client(self):
-        return pylibmc.Client(self.url,
+        return pylibmc.Client(
+            self.url,
             binary=self.binary,
             behaviors=self.behaviors
         )
 
+memcache = None
+
+
 class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
-    """A backend using the standard `Python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
+    """A backend using the standard
+    `Python-memcached <http://www.tummy.com/Community/software/\
+    python-memcached/>`_
     library.
 
     Example::
@@ -259,14 +284,19 @@ class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
     """
     def _imports(self):
         global memcache
-        import memcache
+        import memcache  # noqa
 
     def _create_client(self):
         return memcache.Client(self.url)
 
+
+bmemcached = None
+
+
 class BMemcachedBackend(GenericMemcachedBackend):
     """A backend for the
-    `python-binary-memcached <https://github.com/jaysonsantos/python-binary-memcached>`_
+    `python-binary-memcached <https://github.com/jaysonsantos/\
+    python-binary-memcached>`_
     memcached client.
 
     This is a pure Python memcached client which
@@ -312,16 +342,18 @@ class BMemcachedBackend(GenericMemcachedBackend):
 
             """
 
-            def add(self, key, value):
+            def add(self, key, value, timeout=0):
                 try:
-                    return super(RepairBMemcachedAPI, self).add(key, value)
+                    return super(RepairBMemcachedAPI, self).add(
+                        key, value, timeout)
                 except ValueError:
                     return False
 
         self.Client = RepairBMemcachedAPI
 
     def _create_client(self):
-        return self.Client(self.url,
+        return self.Client(
+            self.url,
             username=self.username,
             password=self.password
         )