Compare commits

...

83 Commits

Author SHA1 Message Date
panni fc2208f9e5 bump version 2017-05-05 19:32:12 +02:00
panni 1a4eb366bb add helping indicator to FPS mod; add 30fps 2017-05-05 19:31:43 +02:00
panni b89c64a2c2 add modification management menu 2017-05-05 19:19:34 +02:00
panni 68e8f6e753 don't remove HI by default 2017-05-05 19:11:43 +02:00
panni f15cc4cb3c add offset shifter submod 2017-05-05 19:10:32 +02:00
panni 903273e3ef add advanced submods; add global (non-line) submods; test implementation of ChangeFPS mod 2017-05-05 15:39:18 +02:00
panni 1c9b744d31 move subtitle modification menu to separate file 2017-05-05 14:58:19 +02:00
panni 7c0fb29886 fix init_cache whoopsie 2017-05-05 14:58:06 +02:00
panni 2505a7510c enzyme: incorporate 0.4.2 fixes 2017-05-05 14:44:59 +02:00
panni 0a66db40a2 fix findbetter 2017-05-05 14:30:49 +02:00
panni 6c68893979 add mod.long_description; add remove_last action to subtitle modification menu 2017-05-04 20:10:35 +02:00
panni c512eab0b6 testcommit 2017-05-04 20:00:12 +02:00
panni 3cedd4bd0f try getting plex token from environment by default 2017-05-04 19:33:05 +02:00
panni 0759c5e4c6 add environment debug 2017-05-04 19:31:07 +02:00
panni ad6cf4be79 move config debug to better position; verify readability of log files 2017-05-04 19:15:38 +02:00
panni 23c3899fb2 add fixme 2017-05-04 14:30:25 +02:00
panni 1a6515a660 add platform and os to config debug 2017-05-04 14:20:29 +02:00
panni 58815a7650 use external ip fallback when logs were requested from plex.tv 2017-05-04 14:16:10 +02:00
panni c15ec9fefc disable get_logs when universal plex token is None 2017-05-04 13:49:02 +02:00
panni 0e18d59680 2.0.0.12 2017-05-03 23:12:42 +02:00
panni 2d88efa5b4 add doc 2017-05-03 23:12:26 +02:00
panni b3da7572f3 add PartialWordsAlways to OCR_fixes 2017-05-03 23:11:02 +02:00
panni 099ec4e85d remove debug print; add doc 2017-05-03 23:04:25 +02:00
panni ff88a15c61 reset initialized mods after load 2017-05-03 22:59:47 +02:00
panni 839791b0fa add OCR fixes as default; fix little whoopsie in SubtitleModifications.modify 2017-05-03 22:52:33 +02:00
panni 159a533731 add precompiled patterns to data dict; add more parsed data; add OCR fixes finally 2017-05-03 22:44:54 +02:00
panni fb5835baa4 separate ocr fix data further into line, word, partial 2017-05-03 15:19:52 +02:00
panni a3f05cd597 separat partial and full replace data 2017-05-03 15:16:22 +02:00
panni f3af1672f6 use memory cache on windows for now; add config debug logging 2017-05-03 13:33:29 +02:00
panni c984c9849b only add better subtitle if its score is higher than the minimum configured 2017-05-02 21:37:40 +02:00
panni e28d264125 language conversion test 2017-05-02 19:22:57 +02:00
panni 7166ab9502 use default mods in tasks as well 2017-05-02 18:47:58 +02:00
panni ab242c2ecb add current find/replace data 2017-05-02 18:43:45 +02:00
panni 6f829dd4c7 move xmls to xml/; add make_data and test_data script; 2017-05-02 18:43:35 +02:00
panni 3e0602cdf0 add OCRFixReplaceList dictionaries of SubtitleEdit; commit 4f43a84c354d53251614fe6fa4c1b9df92839f57; add second test srt 2017-05-02 18:03:17 +02:00
panni 67cdebfb67 make subtitle modifications a subpackage of subzero 2017-05-02 18:01:46 +02:00
panni 0f87973742 modify test.srt to accomodate for specials chars in text-before-colon; handle special chars in HI_before_colon better 2017-05-02 17:42:39 +02:00
panni 92317f7730 add task run info logging 2017-05-01 05:37:38 +02:00
panni ce936c2553 add task debug 2017-05-01 05:37:09 +02:00
panni b995f16c34 2.0.0.10 DEV 2017-05-01 05:18:33 +02:00
panni 49c7adcc40 correctly use current_sub; correctly use add_mod with mods=mods 2017-05-01 05:18:02 +02:00
panni 88eee6fe48 lower opensubtitles timeout from 10 to 4 seconds 2017-05-01 04:52:05 +02:00
panni cbe425d150 add default hearing impaired removal; add optional mods to save_subtitles 2017-05-01 04:51:44 +02:00
panni 1c7d6b7bf8 debug stuff 2017-05-01 04:01:45 +02:00
panni 8323608558 add HI starting dash 2017-05-01 03:55:41 +02:00
panni 3f8a5ec125 add info to virtual subtitle instance 2017-05-01 03:50:45 +02:00
panni 464b1695a9 add subtitle modification processor debug option; fix remove_HI; 2017-05-01 03:50:30 +02:00
panni d85602612b subtitle modifications live! 2017-05-01 03:15:00 +02:00
panni 59440d251b rename support.background to support.scheduler; add subtitle modification to PatchedSubtitle 2017-05-01 02:39:44 +02:00
panni d774f09427 add doc; get_modified_content: always return unicode 2017-05-01 02:02:33 +02:00
panni 45be650db9 add exclusive mods; add processor/mod debug info; add to_string encoding parameter; add StoredSubtitle.add_mod; 2017-05-01 01:45:42 +02:00
panni d54847803f add subtitle modification menu; add get_current_sub; 2017-05-01 01:44:57 +02:00
panni ce3b66eda7 add mod registry 2017-05-01 00:19:21 +02:00
panni 5b6bcc7d12 add log warning for dogpile cache region setup; bump to 2.0.0.9 2017-04-30 23:55:31 +02:00
pannal 24d4c2ae2c memory backend fallback 2017-04-30 22:36:29 +02:00
pannal 98e451d57d move imports 2017-04-30 06:02:37 +02:00
pannal 8c491c45be re-merge GetLogsLink 2017-04-30 05:59:30 +02:00
pannal 6f271c5638 Merge branch '2.0_menu_maintenance' into develop-2.0
# Conflicts:
#	Contents/Code/interface/menu.py
2017-04-30 05:58:25 +02:00
pannal f9c083ebc6 split interface.menu up; first coarse try 2017-04-30 05:57:35 +02:00
pannal e79360915d remove redundancy 2017-04-30 04:56:35 +02:00
pannal 2fbd8fdc08 urlparse object has hostname, not host 2017-04-30 04:54:55 +02:00
pannal 5a9d5ec9a1 separate menu items 2017-04-30 04:53:11 +02:00
pannal 9ace798ee5 support older PMSs; use Referer for request origin determination 2017-04-30 04:37:16 +02:00
pannal 63e0dc0cb0 add debug if request origin can't be determined 2017-04-30 03:49:35 +02:00
pannal 974aae3ec6 precompile regex patterns 2017-04-30 00:37:26 +02:00
pannal 3268975849 hopefully fix #271 2017-04-29 23:59:07 +02:00
pannal b6adb4cff5 add SubtitleModifications.to_string; add mods to stored subtitle 2017-04-29 23:03:14 +02:00
pannal 78191bb750 doc; all caps line replaced with at least 3 chars; accept filename or subtitle content; optional known fps argument 2017-04-29 22:51:18 +02:00
pannal 2ab66671e5 add subtitle modification basics; add test script and test subtitle 2017-04-29 06:14:33 +02:00
pannal fdcfc630b3 fixed logging 2017-04-29 01:15:20 +02:00
pannal 3a717a8876 more logging 2017-04-29 00:47:32 +02:00
pannal 2dfb381b96 DEV 8 2017-04-29 00:44:52 +02:00
pannal d8a7e3331b improve logging for multiprocessing 2017-04-29 00:44:24 +02:00
pannal bedb097955 reflect DEV number in last version digit for now 2017-04-27 19:40:43 +02:00
pannal e6cebe41dc delete old libraries 2017-04-27 19:40:23 +02:00
pannal 5aa123d42b add automatic subtitle storage maintenance task 2017-04-27 19:08:15 +02:00
pannal 9adb7d18c0 bump DEV 2017-04-27 17:13:57 +02:00
pannal 73da57a4f7 add dogpile cache invalidation 2017-04-27 17:13:42 +02:00
pannal 261d3c5532 add data paths to config; add proper dbm cache for subliminal 2017-04-27 17:00:03 +02:00
pannal 91e55502f6 correct elif 2017-04-27 15:50:51 +02:00
pannal 26846a02b5 add hash_verifiable flag to subtitle and provider base classes 2017-04-27 15:46:42 +02:00
pannal f2ed289c70 add new Provider base class; streamlined base class patching 2017-04-27 15:42:18 +02:00
pannal 8b9109396a fix duplicate subtitles issue on synology/qnap #215 2017-04-27 14:00:09 +02:00
238 changed files with 26200 additions and 35770 deletions
+4 -3
View File
@@ -1,5 +1,7 @@
# coding=utf-8
import sys
import datetime
import os
from subliminal_patch import compute_score
from subzero.sandbox import restore_builtins
@@ -40,8 +42,7 @@ def Start():
HTTP.CacheTime = 0
HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
# configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
subliminal.region.configure('dogpile.cache.memory')
config.init_cache()
# clear expired intents
intent = get_intent()
@@ -216,7 +217,7 @@ class SubZeroAgent(object):
whack_missing_parts(scanned_video_part_map)
if downloaded_subtitles:
save_subtitles(scanned_video_part_map, downloaded_subtitles)
save_subtitles(scanned_video_part_map, downloaded_subtitles, mods=config.default_mods)
track_usage("Subtitle", "refreshed", "download", 1)
for video, video_subtitles in downloaded_subtitles.items():
+17 -1
View File
@@ -2,6 +2,22 @@ import sys
import menu
sys.modules["interface.menu"] = menu
sys.modules["menu"] = menu
import menu_helpers
sys.modules["interface.menu_helpers"] = menu_helpers
sys.modules["interface.menu_helpers"] = menu_helpers
import advanced
sys.modules["interface.advanced"] = advanced
import main
sys.modules["interface.main"] = main
import refresh_item
sys.modules["interface.refresh_item"] = refresh_item
import item_details
sys.modules["interface.item_details"] = item_details
import sub_mod
sys.modules["interface.modification"] = sub_mod
+241
View File
@@ -0,0 +1,241 @@
# coding=utf-8
import datetime
import StringIO
import glob
import os
import urlparse
from zipfile import ZipFile, ZIP_DEFLATED
from subzero.lib.io import FileIO
from subzero.constants import PREFIX, PLUGIN_IDENTIFIER
from menu_helpers import SubFolderObjectContainer, debounce, set_refresh_menu_state, ZipObject
from main import fatality
from support.helpers import timestamp, pad_title
from support.config import config
from support.lib import Plex
from support.storage import reset_storage, log_storage
from support.scheduler import scheduler
@route(PREFIX + '/advanced')
def AdvancedMenu(randomize=None, header=None, message=None):
oc = SubFolderObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True,
no_history=True,
replace_parent=False, title2="Advanced")
if config.lock_advanced_menu and not config.pin_correct:
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), success_go_to="advanced"),
title=pad_title("Enter PIN"),
summary="The owner has restricted the access to this menu. Please enter the correct pin",
))
return oc
oc.add(DirectoryObject(
key=Callback(TriggerRestart, randomize=timestamp()),
title=pad_title("Restart the plugin"),
))
oc.add(DirectoryObject(
key=Callback(GetLogsLink),
title="Get my logs (copy the appearing link and open it in your browser, please)",
summary="Copy the appearing link and open it in your browser, please",
))
oc.add(DirectoryObject(
key=Callback(TriggerBetterSubtitles, randomize=timestamp()),
title=pad_title("Trigger find better subtitles"),
))
oc.add(DirectoryObject(
key=Callback(TriggerStorageMaintenance, randomize=timestamp()),
title=pad_title("Trigger subtitle storage maintenance"),
))
oc.add(DirectoryObject(
key=Callback(LogStorage, key="tasks", randomize=timestamp()),
title=pad_title("Log the plugin's scheduled tasks state storage"),
))
oc.add(DirectoryObject(
key=Callback(LogStorage, key="ignore", randomize=timestamp()),
title=pad_title("Log the plugin's internal ignorelist storage"),
))
oc.add(DirectoryObject(
key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
title=pad_title("Reset the plugin's scheduled tasks state storage"),
))
oc.add(DirectoryObject(
key=Callback(ResetStorage, key="ignore", randomize=timestamp()),
title=pad_title("Reset the plugin's internal ignorelist storage"),
))
oc.add(DirectoryObject(
key=Callback(InvalidateCache, randomize=timestamp()),
title=pad_title("Invalidate Sub-Zero metadata caches (subliminal)"),
))
return oc
def DispatchRestart():
Thread.CreateTimer(1.0, Restart)
@route(PREFIX + '/advanced/restart/trigger')
@debounce
def TriggerRestart(randomize=None):
set_refresh_menu_state("Restarting the plugin")
DispatchRestart()
return fatality(header="Restart triggered, please wait about 5 seconds", force_title=" ", only_refresh=True,
replace_parent=True,
no_history=True, randomize=timestamp())
@route(PREFIX + '/advanced/restart/execute')
def Restart():
Plex[":/plugins"].restart(PLUGIN_IDENTIFIER)
@route(PREFIX + '/storage/reset', sure=bool)
def ResetStorage(key, randomize=None, sure=False):
if not sure:
oc = SubFolderObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
oc.add(DirectoryObject(
key=Callback(ResetStorage, key=key, sure=True, randomize=timestamp()),
title=pad_title("Are you really sure?"),
))
return oc
reset_storage(key)
if key == "tasks":
# reinitialize the scheduler
scheduler.init_storage()
scheduler.setup_tasks()
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='Information Storage (%s) reset' % key
)
@route(PREFIX + '/storage/log')
def LogStorage(key, randomize=None):
log_storage(key)
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='Information Storage (%s) logged' % key
)
@route(PREFIX + '/triggerbetter')
def TriggerBetterSubtitles(randomize=None):
scheduler.dispatch_task("FindBetterSubtitles")
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='FindBetterSubtitles triggered'
)
@route(PREFIX + '/triggermaintenance')
def TriggerStorageMaintenance(randomize=None):
scheduler.dispatch_task("SubtitleStorageMaintenance")
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='SubtitleStorageMaintenance triggered'
)
@route(PREFIX + '/get_logs_link')
def GetLogsLink():
if not config.plex_token:
oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
header="Sorry, feature unavailable",
message="Universal Plex token not available")
return oc
# try getting the link base via the request in context, first, otherwise use the public ip
req_headers = Core.sandbox.context.request.headers
get_external_ip = True
link_base = ""
if "Origin" in req_headers:
link_base = req_headers["Origin"]
Log.Debug("Using origin-based link_base")
get_external_ip = False
elif "Referer" in req_headers:
parsed = urlparse.urlparse(req_headers["Referer"])
link_base = "%s://%s:%s" % (parsed.scheme, parsed.hostname, parsed.port)
Log.Debug("Using referer-based link_base")
get_external_ip = False
if get_external_ip or "plex.tv" in link_base:
ip = Core.networking.http_request("http://www.plexapp.com/ip.php", cacheTime=7200).content.strip()
link_base = "https://%s:32400" % ip
Log.Debug("Using ip-based fallback link_base")
logs_link = "%s%s?X-Plex-Token=%s" % (link_base, PREFIX + '/logs', config.plex_token)
oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
header="Copy this link and open this in your browser, please",
message=logs_link)
return oc
@route(PREFIX + '/logs')
def DownloadLogs():
buff = StringIO.StringIO()
zip_archive = ZipFile(buff, mode='w', compression=ZIP_DEFLATED)
logs = sorted(glob.glob(config.plugin_log_path + '*')) + [config.server_log_path]
for path in logs:
data = StringIO.StringIO()
data.write(FileIO.read(path))
zip_archive.writestr(os.path.basename(path), data.getvalue())
zip_archive.close()
return ZipObject(buff.getvalue())
@route(PREFIX + '/invalidatecache')
def InvalidateCache(randomize=None):
from subliminal.cache import region
region.invalidate()
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='Cache invalidated'
)
@route(PREFIX + '/pin')
def PinMenu(pin="", randomize=None, success_go_to="channel"):
oc = ObjectContainer(title2="Enter PIN number %s" % (len(pin) + 1), no_cache=True, no_history=True,
skip_pin_lock=True)
if pin == config.pin:
Dict["pin_correct_time"] = datetime.datetime.now()
config.locked = False
if success_go_to == "channel":
return fatality(force_title="PIN correct", header="PIN correct", no_history=True)
elif success_go_to == "advanced":
return AdvancedMenu(randomize=timestamp())
for i in range(10):
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), pin=pin + str(i), success_go_to=success_go_to),
title=pad_title(str(i)),
))
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), success_go_to=success_go_to),
title=pad_title("Reset"),
))
return oc
@route(PREFIX + '/pin_lock')
def ClearPin(randomize=None):
Dict["pin_correct_time"] = None
config.locked = True
return fatality(force_title="Menu locked", header=" ", no_history=True)
+236
View File
@@ -0,0 +1,236 @@
# coding=utf-8
import os
from sub_mod import SubtitleModificationsMenu
from menu_helpers import debounce, SubFolderObjectContainer, default_thumb, add_ignore_options, get_item_task_data, \
set_refresh_menu_state
from refresh_item import RefreshItem
from subzero.constants import PREFIX
from support.config import config
from support.helpers import timestamp, cast_bool, df, get_language
from support.items import get_item_kind_from_rating_key, get_item, get_current_sub
from support.lib import Plex
from support.plex_media import get_plex_metadata, scan_videos
from support.scheduler import scheduler
from support.storage import get_subtitle_storage
@route(PREFIX + '/item/{rating_key}/actions')
@debounce
def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None):
"""
displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
:param rating_key:
:param title:
:param base_title:
:param item_title:
:param randomize:
:return:
"""
from interface.main import IgnoreMenu
title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title)
item = get_item(rating_key)
current_kind = get_item_kind_from_rating_key(rating_key)
timeout = 30
oc = SubFolderObjectContainer(title2=title, replace_parent=True)
oc.add(DirectoryObject(
key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp(),
timeout=timeout * 1000),
title=u"Refresh: %s" % item_title,
summary="Refreshes the %s, possibly searching for missing and picking up new subtitles on disk" % current_kind,
thumb=item.thumb or default_thumb
))
oc.add(DirectoryObject(
key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(),
timeout=timeout * 1000),
title=u"Auto-search: %s" % item_title,
summary="Issues a forced refresh, ignoring known subtitles and searching for new ones",
thumb=item.thumb or default_thumb
))
# get stored subtitle info for item id
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
# get the plex item
plex_item = list(Plex["library"].metadata(rating_key))[0]
# get current media info for that item
media = plex_item.media
# look for subtitles for all available media parts and all of their languages
for part in media.parts:
filename = os.path.basename(part.file)
part_id = str(part.id)
# iterate through all configured languages
for lang in config.lang_list:
lang_a2 = lang.alpha2
# ietf lang?
if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
lang_a2 = lang_a2.split("-")[0]
# get corresponding stored subtitle data for that media part (physical media item), for language
current_sub = stored_subs.get_any(part_id, lang_a2)
current_sub_id = None
current_sub_provider_name = None
summary = u"No current subtitle in storage"
current_score = None
if current_sub:
current_sub_id = current_sub.id
current_sub_provider_name = current_sub.provider_name
current_score = current_sub.score
summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
(current_sub.provider_name, df(current_sub.date_added), current_sub.mode_verbose, lang,
current_sub.score, current_sub.storage_type)
oc.add(DirectoryObject(
key=Callback(SubtitleOptionsMenu, rating_key=rating_key, part_id=part_id, title=title,
item_title=item_title, language=lang, language_name=lang.name, current_id=current_sub_id,
item_type=plex_item.type, filename=filename, current_data=summary,
randomize=timestamp(), current_provider=current_sub_provider_name,
current_score=current_score),
title=u"Actions for %s subtitle" % lang.name,
summary=summary
))
add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
return oc
@route(PREFIX + '/item/current_sub/{rating_key}/{part_id}', force=bool)
@debounce
def SubtitleOptionsMenu(**kwargs):
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
kwargs.pop("randomize")
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"],
title=kwargs["title"], randomize=timestamp()),
title=u"Back to: %s" % kwargs["title"],
summary=kwargs["current_data"],
thumb=default_thumb
))
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, randomize=timestamp(), **kwargs),
title=u"List %s subtitles" % kwargs["language_name"],
summary=kwargs["current_data"]
))
if current_sub:
oc.add(DirectoryObject(
key=Callback(SubtitleModificationsMenu, randomize=timestamp(), **kwargs),
title=u"Modify %s subtitle" % kwargs["language_name"],
summary=u"Currently applied mods: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
))
return oc
@route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool)
@debounce
def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None,
item_type="episode", language=None, language_name=None, force=False, current_id=None,
current_data=None,
current_provider=None, current_score=None, randomize=None):
assert rating_key, part_id
running = scheduler.is_task_running("AvailableSubsForItem")
search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
if (search_results is None or force) and not running:
scheduler.dispatch_task("AvailableSubsForItem", rating_key=rating_key, item_type=item_type, part_id=part_id,
language=language)
running = True
oc = SubFolderObjectContainer(title2=unicode(title), replace_parent=True)
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()),
title=u"Back to: %s" % title,
summary=current_data,
thumb=default_thumb
))
metadata = get_plex_metadata(rating_key, part_id, item_type)
scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
if not scanned_parts:
Log.Error("Couldn't list available subtitles for %s", rating_key)
return oc
video, plex_part = scanned_parts.items()[0]
video_display_data = [video.format] if video.format else []
if video.release_group:
video_display_data.append(u"by %s" % video.release_group)
video_display_data = " ".join(video_display_data)
current_display = (u"Current: %s (%s) " % (current_provider, current_score) if current_provider else "")
if not running:
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, language=language,
filename=filename, part_id=part_id, title=title, current_id=current_id, force=True,
current_provider=current_provider, current_score=current_score,
current_data=current_data, item_type=item_type, randomize=timestamp()),
title=u"Search for %s subs (%s)" % (get_language(language).name, video_display_data),
summary=u"%sFilename: %s" % (current_display, filename),
thumb=default_thumb
))
else:
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
language=language, filename=filename, current_data=current_data,
part_id=part_id, title=title, current_id=current_id, item_type=item_type,
current_provider=current_provider, current_score=current_score,
randomize=timestamp()),
title=u"Searching for %s subs (%s), refresh here ..." % (get_language(language).name, video_display_data),
summary=u"%sFilename: %s" % (current_display, filename),
thumb=default_thumb
))
if not search_results:
return oc
for subtitle in search_results:
oc.add(DirectoryObject(
key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title,
subtitle_id=str(subtitle.id), language=language),
title=u"%s: %s, score: %s" % ("Available" if current_id != subtitle.id else "Current",
subtitle.provider_name, subtitle.score),
summary=u"Release: %s, Matches: %s" % (subtitle.release_info, ", ".join(subtitle.matches)),
thumb=default_thumb
))
return oc
@route(PREFIX + '/download_subtitle/{rating_key}')
@debounce
def TriggerDownloadSubtitle(rating_key=None, subtitle_id=None, item_title=None, language=None, randomize=None):
from interface.main import fatality
set_refresh_menu_state("Downloading subtitle for %s" % item_title or rating_key)
search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
download_subtitle = None
for subtitle in search_results:
if str(subtitle.id) == subtitle_id:
download_subtitle = subtitle
break
if not download_subtitle:
Log.Error(u"Something went horribly wrong")
else:
scheduler.dispatch_task("DownloadSubtitleForItem", rating_key=rating_key, subtitle=download_subtitle)
return fatality(randomize=timestamp(), header=" ", replace_parent=True)
+387
View File
@@ -0,0 +1,387 @@
# coding=utf-8
from subzero.constants import PREFIX, TITLE, ART
from support.config import config
from support.helpers import pad_title, timestamp, df
from support.scheduler import scheduler
from support.ignore import ignore_list
from support.items import get_item_thumb, get_on_deck_items, get_all_items, get_items_info
from menu_helpers import main_icon, debounce, SubFolderObjectContainer, default_thumb, dig_tree, add_ignore_options
from item_details import ItemDetailsMenu
@handler(PREFIX, TITLE if not config.is_development else TITLE + " DEV", art=ART, thumb=main_icon)
@route(PREFIX)
def fatality(randomize=None, force_title=None, header=None, message=None, only_refresh=False, no_history=False,
replace_parent=False):
"""
subzero main menu
"""
from interface.advanced import PinMenu, ClearPin, AdvancedMenu
from interface.menu import RefreshMissing, IgnoreListMenu, HistoryMenu
title = config.full_version # force_title if force_title is not None else config.full_version
oc = ObjectContainer(title1=title, title2=title, header=unicode(header) if header else title, message=message,
no_history=no_history,
replace_parent=replace_parent, no_cache=True)
# always re-check permissions
config.refresh_permissions_status()
# always re-check enabled sections
config.refresh_enabled_sections()
if config.lock_menu and not config.pin_correct:
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp()),
title=pad_title("Enter PIN"),
summary="The owner has restricted the access to this menu. Please enter the correct pin",
))
return oc
if not config.permissions_ok and config.missing_permissions:
for title, path in config.missing_permissions:
oc.add(DirectoryObject(
key=Callback(fatality, randomize=timestamp()),
title=pad_title("Insufficient permissions"),
summary="Insufficient permissions on library %s, folder: %s" % (title, path),
))
return oc
if not config.enabled_sections:
oc.add(DirectoryObject(
key=Callback(fatality, randomize=timestamp()),
title=pad_title("I'm not enabled!"),
summary="Please enable me for some of your libraries in your server settings; currently I do nothing",
))
return oc
if not only_refresh:
if Dict["current_refresh_state"]:
oc.add(DirectoryObject(
key=Callback(fatality, force_title=" ", randomize=timestamp()),
title=pad_title("Working ... refresh here"),
summary="Current state: %s; Last state: %s" % (
(Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
(Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
)
))
oc.add(DirectoryObject(
key=Callback(OnDeckMenu),
title="On Deck items",
summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/"
"subtitles.",
thumb=R("icon-ondeck.jpg")
))
oc.add(DirectoryObject(
key=Callback(RecentlyAddedMenu),
title="Recently Added items",
summary="Shows the recently added items per section.",
thumb=R("icon-recent.jpg")
))
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()),
title="Items with missing subtitles",
summary="Shows the items honoring the configured 'Item age to be considered recent'-setting (%s)"
" and allowing you to individually (force-) refresh their metadata/subtitles. " %
Prefs["scheduler.item_is_recent_age"],
thumb=R("icon-missing.jpg")
))
oc.add(DirectoryObject(
key=Callback(SectionsMenu),
title="Browse all items",
summary="Go through your whole library and manage your ignore list. You can also "
"(force-) refresh the metadata/subtitles of individual items.",
thumb=R("icon-browse.jpg")
))
task_name = "SearchAllRecentlyAddedMissing"
task = scheduler.task(task_name)
if task.ready_for_display:
task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
else:
task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (
df(scheduler.last_run(task_name)) or "never",
df(scheduler.next_run(task_name)) or "never",
str(task.last_run_time).split(".")[0])
oc.add(DirectoryObject(
key=Callback(RefreshMissing, randomize=timestamp()),
title="Search for missing subtitles (in recently-added items, max-age: %s)" % Prefs[
"scheduler.item_is_recent_age"],
summary="Automatically run periodically by the scheduler, if configured. %s" % task_state,
thumb=R("icon-search.jpg")
))
oc.add(DirectoryObject(
key=Callback(IgnoreListMenu),
title="Display ignore list (%d)" % len(ignore_list),
summary="Show the current ignore list (mainly used for the automatic tasks)",
thumb=R("icon-ignore.jpg")
))
oc.add(DirectoryObject(
key=Callback(HistoryMenu),
title="History",
summary="Show the last %i downloaded subtitles" % int(Prefs["history_size"]),
thumb=R("icon-history.jpg")
))
oc.add(DirectoryObject(
key=Callback(fatality, force_title=" ", randomize=timestamp()),
title=pad_title("Refresh"),
summary="Current state: %s; Last state: %s" % (
(Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
(Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
),
thumb=R("icon-refresh.jpg")
))
# add re-lock after pin unlock
if config.pin:
oc.add(DirectoryObject(
key=Callback(ClearPin, randomize=timestamp()),
title=pad_title("Re-lock menu(s)"),
summary="Enabled the PIN again for menu(s)"
))
if not only_refresh:
oc.add(DirectoryObject(
key=Callback(AdvancedMenu),
title=pad_title("Advanced functions"),
summary="Use at your own risk",
thumb=R("icon-advanced.jpg")
))
return oc
@route(PREFIX + '/on_deck')
def OnDeckMenu(message=None):
"""
displays the items on deck
:param message:
:return:
"""
return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=get_on_deck_items)
@route(PREFIX + '/recently_added')
def RecentlyAddedMenu(message=None):
"""
displays the items recently added per section
:param message:
:return:
"""
return SectionsMenu(base_title="Recently added", section_items_key="recently_added", ignore_options=False)
@route(PREFIX + '/recent', force=bool)
@debounce
def RecentMissingSubtitlesMenu(force=False, randomize=None):
title = "Items with missing subtitles"
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
running = scheduler.is_task_running("MissingSubtitles")
task_data = scheduler.get_task_data("MissingSubtitles")
missing_items = task_data["missing_subtitles"] if task_data else None
if ((missing_items is None) or force) and not running:
scheduler.dispatch_task("MissingSubtitles")
running = True
if not running:
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, force=True, randomize=timestamp()),
title=u"Get items with missing subtitles",
thumb=default_thumb
))
else:
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, force=False, randomize=timestamp()),
title=u"Updating, refresh here ...",
thumb=default_thumb
))
if missing_items is not None:
for added_at, item_id, item_title, item, missing_languages in missing_items:
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, title=title + " > " + item_title, item_title=item_title,
rating_key=item_id),
title=item_title,
summary="Missing: %s" % ", ".join(l.name for l in missing_languages),
thumb=get_item_thumb(item) or default_thumb
))
scheduler.clear_task_data("MissingSubtitles")
return oc
def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *args, **kwargs):
"""
displays an item list of dynamic kinds of items
:param title:
:param itemGetter:
:param itemGetterKwArgs:
:param base_title:
:param args:
:param kwargs:
:return:
"""
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
items = itemGetter(*args, **kwargs)
for kind, title, item_id, deeper, item in items:
oc.add(DirectoryObject(
title=title,
key=Callback(ItemDetailsMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id),
thumb=get_item_thumb(item) or default_thumb
))
return oc
def determine_section_display(kind, item, pass_kwargs=None):
"""
returns the menu function for a section based on the size of it (amount of items)
:param kind:
:param item:
:return:
"""
if pass_kwargs and pass_kwargs.get("section_items_key", "all") != "all":
return SectionMenu
if item.size > 80:
return SectionFirstLetterMenu
return SectionMenu
@route(PREFIX + '/ignore/set/{kind}/{rating_key}/{todo}/sure={sure}', kind=str, rating_key=str, todo=str, sure=bool)
def IgnoreMenu(kind, rating_key, title=None, sure=False, todo="not_set"):
"""
displays the ignore options for a menu
:param kind:
:param rating_key:
:param title:
:param sure:
:param todo:
:return:
"""
is_ignored = rating_key in ignore_list[kind]
if not sure:
oc = SubFolderObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
"Add" if not is_ignored else "Remove", ignore_list.verbose(kind), title,
"to" if not is_ignored else "from"), title2="Are you sure?")
oc.add(DirectoryObject(
key=Callback(IgnoreMenu, kind=kind, rating_key=rating_key, title=title, sure=True,
todo="add" if not is_ignored else "remove"),
title=pad_title("Are you sure?"),
))
return oc
rel = ignore_list[kind]
dont_change = False
if todo == "remove":
if not is_ignored:
dont_change = True
else:
rel.remove(rating_key)
Log.Info("Removed %s (%s) from the ignore list", title, rating_key)
ignore_list.remove_title(kind, rating_key)
ignore_list.save()
state = "removed from"
elif todo == "add":
if is_ignored:
dont_change = True
else:
rel.append(rating_key)
Log.Info("Added %s (%s) to the ignore list", title, rating_key)
ignore_list.add_title(kind, rating_key, title)
ignore_list.save()
state = "added to"
else:
dont_change = True
if dont_change:
return fatality(force_title=" ", header="Didn't change the ignore list", no_history=True)
return fatality(force_title=" ", header="%s %s the ignore list" % (title, state), no_history=True)
@route(PREFIX + '/sections')
def SectionsMenu(base_title="Sections", section_items_key="all", ignore_options=True):
"""
displays the menu for all sections
:return:
"""
items = get_all_items("sections")
return dig_tree(SubFolderObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
menu_determination_callback=determine_section_display, pass_kwargs={"base_title": base_title,
"section_items_key": section_items_key,
"ignore_options": ignore_options},
fill_args={"title": "section_title"})
@route(PREFIX + '/section', ignore_options=bool)
def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
section_items_key="all"):
"""
displays the contents of a section
:param section_items_key:
:param rating_key:
:param title:
:param base_title:
:param section_title:
:param ignore_options:
:return:
"""
from menu import MetadataMenu
items = get_all_items(key=section_items_key, value=rating_key, base="library/sections")
kind, deeper = get_items_info(items)
title = unicode(title)
section_title = title
title = base_title + " > " + title
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
if ignore_options:
add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
return dig_tree(oc, items, MetadataMenu,
pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": "section",
"previous_rating_key": rating_key})
@route(PREFIX + '/section/firstLetter', deeper=bool)
def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
section_items_key="all"):
"""
displays the contents of a section indexed by its first char (A-Z, 0-9...)
:param ignore_options: ignored
:param section_items_key: ignored
:param rating_key:
:param title:
:param base_title:
:param section_title:
:return:
"""
from menu import FirstLetterMetadataMenu
items = get_all_items(key="first_character", value=rating_key, base="library/sections")
kind, deeper = get_items_info(items)
title = unicode(title)
oc = SubFolderObjectContainer(title2=section_title, no_cache=True, no_history=True)
title = base_title + " > " + title
add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
oc.add(DirectoryObject(
key=Callback(SectionMenu, title="All", base_title=title, rating_key=rating_key, ignore_options=False),
title="All"
)
)
return dig_tree(oc, items, FirstLetterMetadataMenu, force_rating_key=rating_key, fill_args={"key": "key"},
pass_kwargs={"base_title": title, "display_items": deeper, "previous_rating_key": rating_key})
+47 -763
View File
@@ -1,27 +1,22 @@
# coding=utf-8
import logging
import datetime
import logger
import os
import StringIO
import glob
from zipfile import ZipFile, ZIP_DEFLATED
import logger
from item_details import ItemDetailsMenu
from refresh_item import RefreshItem
from menu_helpers import add_ignore_options, dig_tree, set_refresh_menu_state, \
should_display_ignore, enable_channel_wrapper, default_thumb, debounce, ObjectContainer, SubFolderObjectContainer, \
ZipObject
from subzero.constants import TITLE, ART, ICON, PREFIX, PLUGIN_IDENTIFIER, DEPENDENCY_MODULE_NAMES
from support.background import scheduler
should_display_ignore, enable_channel_wrapper, default_thumb, debounce, ObjectContainer, SubFolderObjectContainer
from main import fatality, IgnoreMenu
from advanced import DispatchRestart
from subzero.constants import ART, PREFIX, DEPENDENCY_MODULE_NAMES
from support.scheduler import scheduler
from support.config import config
from support.helpers import pad_title, timestamp, get_language, df, cast_bool
from support.helpers import timestamp, df
from support.ignore import ignore_list
from support.items import get_item, get_on_deck_items, refresh_item, get_all_items, get_items_info, \
get_item_thumb, get_item_kind_from_rating_key
from support.lib import Plex
from subzero.lib.io import FileIO
from support.plex_media import get_plex_metadata, scan_videos
from support.storage import reset_storage, log_storage, get_subtitle_storage
from support.items import get_all_items, get_items_info, \
get_item_kind_from_rating_key
# init GUI
ObjectContainer.art = R(ART)
@@ -35,412 +30,6 @@ route = enable_channel_wrapper(route)
# noinspection PyUnboundLocalVariable
handler = enable_channel_wrapper(handler)
main_icon = ICON if not config.is_development else "icon-dev.jpg"
@handler(PREFIX, TITLE if not config.is_development else TITLE + " DEV", art=ART, thumb=main_icon)
@route(PREFIX)
def fatality(randomize=None, force_title=None, header=None, message=None, only_refresh=False, no_history=False,
replace_parent=False):
"""
subzero main menu
"""
title = config.full_version # force_title if force_title is not None else config.full_version
oc = ObjectContainer(title1=title, title2=title, header=unicode(header) if header else title, message=message,
no_history=no_history,
replace_parent=replace_parent, no_cache=True)
# always re-check permissions
config.refresh_permissions_status()
# always re-check enabled sections
config.refresh_enabled_sections()
if config.lock_menu and not config.pin_correct:
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp()),
title=pad_title("Enter PIN"),
summary="The owner has restricted the access to this menu. Please enter the correct pin",
))
return oc
if not config.permissions_ok and config.missing_permissions:
for title, path in config.missing_permissions:
oc.add(DirectoryObject(
key=Callback(fatality, randomize=timestamp()),
title=pad_title("Insufficient permissions"),
summary="Insufficient permissions on library %s, folder: %s" % (title, path),
))
return oc
if not config.enabled_sections:
oc.add(DirectoryObject(
key=Callback(fatality, randomize=timestamp()),
title=pad_title("I'm not enabled!"),
summary="Please enable me for some of your libraries in your server settings; currently I do nothing",
))
return oc
if not only_refresh:
if Dict["current_refresh_state"]:
oc.add(DirectoryObject(
key=Callback(fatality, force_title=" ", randomize=timestamp()),
title=pad_title("Working ... refresh here"),
summary="Current state: %s; Last state: %s" % (
(Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
(Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
)
))
oc.add(DirectoryObject(
key=Callback(OnDeckMenu),
title="On Deck items",
summary="Shows the current on deck items and allows you to individually (force-) refresh their metadata/"
"subtitles.",
thumb=R("icon-ondeck.jpg")
))
oc.add(DirectoryObject(
key=Callback(RecentlyAddedMenu),
title="Recently Added items",
summary="Shows the recently added items per section.",
thumb=R("icon-recent.jpg")
))
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, randomize=timestamp()),
title="Items with missing subtitles",
summary="Shows the items honoring the configured 'Item age to be considered recent'-setting (%s)"
" and allowing you to individually (force-) refresh their metadata/subtitles. " %
Prefs["scheduler.item_is_recent_age"],
thumb=R("icon-missing.jpg")
))
oc.add(DirectoryObject(
key=Callback(SectionsMenu),
title="Browse all items",
summary="Go through your whole library and manage your ignore list. You can also "
"(force-) refresh the metadata/subtitles of individual items.",
thumb=R("icon-browse.jpg")
))
task_name = "SearchAllRecentlyAddedMissing"
task = scheduler.task(task_name)
if task.ready_for_display:
task_state = "Running: %s/%s (%s%%)" % (len(task.items_done), len(task.items_searching), task.percentage)
else:
task_state = "Last scheduler run: %s; Next scheduled run: %s; Last runtime: %s" % (
df(scheduler.last_run(task_name)) or "never",
df(scheduler.next_run(task_name)) or "never",
str(task.last_run_time).split(".")[0])
oc.add(DirectoryObject(
key=Callback(RefreshMissing, randomize=timestamp()),
title="Search for missing subtitles (in recently-added items, max-age: %s)" % Prefs[
"scheduler.item_is_recent_age"],
summary="Automatically run periodically by the scheduler, if configured. %s" % task_state,
thumb=R("icon-search.jpg")
))
oc.add(DirectoryObject(
key=Callback(IgnoreListMenu),
title="Display ignore list (%d)" % len(ignore_list),
summary="Show the current ignore list (mainly used for the automatic tasks)",
thumb=R("icon-ignore.jpg")
))
oc.add(DirectoryObject(
key=Callback(HistoryMenu),
title="History",
summary="Show the last %i downloaded subtitles" % int(Prefs["history_size"]),
thumb=R("icon-history.jpg")
))
oc.add(DirectoryObject(
key=Callback(fatality, force_title=" ", randomize=timestamp()),
title=pad_title("Refresh"),
summary="Current state: %s; Last state: %s" % (
(Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
(Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
),
thumb=R("icon-refresh.jpg")
))
# add re-lock after pin unlock
if config.pin:
oc.add(DirectoryObject(
key=Callback(ClearPin, randomize=timestamp()),
title=pad_title("Re-lock menu(s)"),
summary="Enabled the PIN again for menu(s)"
))
if not only_refresh:
oc.add(DirectoryObject(
key=Callback(AdvancedMenu),
title=pad_title("Advanced functions"),
summary="Use at your own risk",
thumb=R("icon-advanced.jpg")
))
return oc
@route(PREFIX + '/pin')
def PinMenu(pin="", randomize=None, success_go_to="channel"):
oc = ObjectContainer(title2="Enter PIN number %s" % (len(pin) + 1), no_cache=True, no_history=True,
skip_pin_lock=True)
if pin == config.pin:
Dict["pin_correct_time"] = datetime.datetime.now()
config.locked = False
if success_go_to == "channel":
return fatality(force_title="PIN correct", header="PIN correct", no_history=True)
elif success_go_to == "advanced":
return AdvancedMenu(randomize=timestamp())
for i in range(10):
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), pin=pin + str(i), success_go_to=success_go_to),
title=pad_title(str(i)),
))
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), success_go_to=success_go_to),
title=pad_title("Reset"),
))
return oc
@route(PREFIX + '/pin_lock')
def ClearPin(randomize=None):
Dict["pin_correct_time"] = None
config.locked = True
return fatality(force_title="Menu locked", header=" ", no_history=True)
@route(PREFIX + '/on_deck')
def OnDeckMenu(message=None):
"""
displays the items on deck
:param message:
:return:
"""
return mergedItemsMenu(title="Items On Deck", base_title="Items On Deck", itemGetter=get_on_deck_items)
@route(PREFIX + '/recently_added')
def RecentlyAddedMenu(message=None):
"""
displays the items recently added per section
:param message:
:return:
"""
return SectionsMenu(base_title="Recently added", section_items_key="recently_added", ignore_options=False)
@route(PREFIX + '/recent', force=bool)
@debounce
def RecentMissingSubtitlesMenu(force=False, randomize=None):
title = "Items with missing subtitles"
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
running = scheduler.is_task_running("MissingSubtitles")
task_data = scheduler.get_task_data("MissingSubtitles")
missing_items = task_data["missing_subtitles"] if task_data else None
if ((missing_items is None) or force) and not running:
scheduler.dispatch_task("MissingSubtitles")
running = True
if not running:
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, force=True, randomize=timestamp()),
title=u"Get items with missing subtitles",
thumb=default_thumb
))
else:
oc.add(DirectoryObject(
key=Callback(RecentMissingSubtitlesMenu, force=False, randomize=timestamp()),
title=u"Updating, refresh here ...",
thumb=default_thumb
))
if missing_items is not None:
for added_at, item_id, item_title, item, missing_languages in missing_items:
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, title=title + " > " + item_title, item_title=item_title,
rating_key=item_id),
title=item_title,
summary="Missing: %s" % ", ".join(l.name for l in missing_languages),
thumb=get_item_thumb(item) or default_thumb
))
scheduler.clear_task_data("MissingSubtitles")
return oc
def mergedItemsMenu(title, itemGetter, itemGetterKwArgs=None, base_title=None, *args, **kwargs):
"""
displays an item list of dynamic kinds of items
:param title:
:param itemGetter:
:param itemGetterKwArgs:
:param base_title:
:param args:
:param kwargs:
:return:
"""
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
items = itemGetter(*args, **kwargs)
for kind, title, item_id, deeper, item in items:
oc.add(DirectoryObject(
title=title,
key=Callback(ItemDetailsMenu, title=base_title + " > " + title, item_title=title, rating_key=item_id),
thumb=get_item_thumb(item) or default_thumb
))
return oc
def determine_section_display(kind, item, pass_kwargs=None):
"""
returns the menu function for a section based on the size of it (amount of items)
:param kind:
:param item:
:return:
"""
if pass_kwargs and pass_kwargs.get("section_items_key", "all") != "all":
return SectionMenu
if item.size > 80:
return SectionFirstLetterMenu
return SectionMenu
@route(PREFIX + '/ignore/set/{kind}/{rating_key}/{todo}/sure={sure}', kind=str, rating_key=str, todo=str, sure=bool)
def IgnoreMenu(kind, rating_key, title=None, sure=False, todo="not_set"):
"""
displays the ignore options for a menu
:param kind:
:param rating_key:
:param title:
:param sure:
:param todo:
:return:
"""
is_ignored = rating_key in ignore_list[kind]
if not sure:
oc = SubFolderObjectContainer(no_history=True, replace_parent=True, title1="%s %s %s %s the ignore list" % (
"Add" if not is_ignored else "Remove", ignore_list.verbose(kind), title,
"to" if not is_ignored else "from"), title2="Are you sure?")
oc.add(DirectoryObject(
key=Callback(IgnoreMenu, kind=kind, rating_key=rating_key, title=title, sure=True,
todo="add" if not is_ignored else "remove"),
title=pad_title("Are you sure?"),
))
return oc
rel = ignore_list[kind]
dont_change = False
if todo == "remove":
if not is_ignored:
dont_change = True
else:
rel.remove(rating_key)
Log.Info("Removed %s (%s) from the ignore list", title, rating_key)
ignore_list.remove_title(kind, rating_key)
ignore_list.save()
state = "removed from"
elif todo == "add":
if is_ignored:
dont_change = True
else:
rel.append(rating_key)
Log.Info("Added %s (%s) to the ignore list", title, rating_key)
ignore_list.add_title(kind, rating_key, title)
ignore_list.save()
state = "added to"
else:
dont_change = True
if dont_change:
return fatality(force_title=" ", header="Didn't change the ignore list", no_history=True)
return fatality(force_title=" ", header="%s %s the ignore list" % (title, state), no_history=True)
@route(PREFIX + '/sections')
def SectionsMenu(base_title="Sections", section_items_key="all", ignore_options=True):
"""
displays the menu for all sections
:return:
"""
items = get_all_items("sections")
return dig_tree(SubFolderObjectContainer(title2="Sections", no_cache=True, no_history=True), items, None,
menu_determination_callback=determine_section_display, pass_kwargs={"base_title": base_title,
"section_items_key": section_items_key,
"ignore_options": ignore_options},
fill_args={"title": "section_title"})
@route(PREFIX + '/section', ignore_options=bool)
def SectionMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
section_items_key="all"):
"""
displays the contents of a section
:param section_items_key:
:param rating_key:
:param title:
:param base_title:
:param section_title:
:param ignore_options:
:return:
"""
items = get_all_items(key=section_items_key, value=rating_key, base="library/sections")
kind, deeper = get_items_info(items)
title = unicode(title)
section_title = title
title = base_title + " > " + title
oc = SubFolderObjectContainer(title2=title, no_cache=True, no_history=True)
if ignore_options:
add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
return dig_tree(oc, items, MetadataMenu,
pass_kwargs={"base_title": title, "display_items": deeper, "previous_item_type": "section",
"previous_rating_key": rating_key})
@route(PREFIX + '/section/firstLetter', deeper=bool)
def SectionFirstLetterMenu(rating_key, title=None, base_title=None, section_title=None, ignore_options=True,
section_items_key="all"):
"""
displays the contents of a section indexed by its first char (A-Z, 0-9...)
:param ignore_options: ignored
:param section_items_key: ignored
:param rating_key:
:param title:
:param base_title:
:param section_title:
:return:
"""
items = get_all_items(key="first_character", value=rating_key, base="library/sections")
kind, deeper = get_items_info(items)
title = unicode(title)
oc = SubFolderObjectContainer(title2=section_title, no_cache=True, no_history=True)
title = base_title + " > " + title
add_ignore_options(oc, "sections", title=section_title, rating_key=rating_key, callback_menu=IgnoreMenu)
oc.add(DirectoryObject(
key=Callback(SectionMenu, title="All", base_title=title, rating_key=rating_key, ignore_options=False),
title="All"
)
)
return dig_tree(oc, items, FirstLetterMetadataMenu, force_rating_key=rating_key, fill_args={"key": "key"},
pass_kwargs={"base_title": title, "display_items": deeper, "previous_rating_key": rating_key})
@route(PREFIX + '/section/firstLetter/key', deeper=bool)
def FirstLetterMetadataMenu(rating_key, key, title=None, base_title=None, display_items=False, previous_item_type=None,
@@ -549,210 +138,6 @@ def HistoryMenu():
return oc
@route(PREFIX + '/item/{rating_key}/actions')
@debounce
def ItemDetailsMenu(rating_key, title=None, base_title=None, item_title=None, randomize=None):
"""
displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
:param rating_key:
:param title:
:param base_title:
:param item_title:
:param randomize:
:return:
"""
title = unicode(base_title) + " > " + unicode(title) if base_title else unicode(title)
item = get_item(rating_key)
current_kind = get_item_kind_from_rating_key(rating_key)
timeout = 30
oc = SubFolderObjectContainer(title2=title, replace_parent=True)
oc.add(DirectoryObject(
key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, randomize=timestamp(),
timeout=timeout * 1000),
title=u"Refresh: %s" % item_title,
summary="Refreshes the %s, possibly searching for missing and picking up new subtitles on disk" % current_kind,
thumb=item.thumb or default_thumb
))
oc.add(DirectoryObject(
key=Callback(RefreshItem, rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(),
timeout=timeout * 1000),
title=u"Auto-search: %s" % item_title,
summary="Issues a forced refresh, ignoring known subtitles and searching for new ones",
thumb=item.thumb or default_thumb
))
# get stored subtitle info for item id
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
# get the plex item
plex_item = list(Plex["library"].metadata(rating_key))[0]
# get current media info for that item
media = plex_item.media
# look for subtitles for all available media parts and all of their languages
for part in media.parts:
filename = os.path.basename(part.file)
part_id = str(part.id)
# iterate through all configured languages
for lang in config.lang_list:
lang_a2 = lang.alpha2
# ietf lang?
if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
lang_a2 = lang_a2.split("-")[0]
# get corresponding stored subtitle data for that media part (physical media item), for language
current_sub = stored_subs.get_any(part_id, lang_a2)
current_sub_id = None
current_sub_provider_name = None
summary = u"No current subtitle in storage"
current_score = None
if current_sub:
current_sub_id = current_sub.id
current_sub_provider_name = current_sub.provider_name
current_score = current_sub.score
summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
(current_sub.provider_name, df(current_sub.date_added), current_sub.mode_verbose, lang,
current_sub.score, current_sub.storage_type)
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, part_id=part_id, title=title,
item_title=item_title, language=lang, current_id=current_sub_id,
item_type=plex_item.type, filename=filename, current_data=summary,
randomize=timestamp(), current_provider=current_sub_provider_name,
current_score=current_score),
title=u"List %s subtitles" % lang.name,
summary=summary
))
add_ignore_options(oc, "videos", title=item_title, rating_key=rating_key, callback_menu=IgnoreMenu)
return oc
def get_item_task_data(task_name, rating_key, language):
task_data = scheduler.get_task_data(task_name)
search_results = task_data.get(rating_key, {}) if task_data else {}
return search_results.get(language)
@route(PREFIX + '/item/search/{rating_key}/{part_id}', force=bool)
@debounce
def ListAvailableSubsForItemMenu(rating_key=None, part_id=None, title=None, item_title=None, filename=None,
item_type="episode", language=None, force=False, current_id=None, current_data=None,
current_provider=None, current_score=None, randomize=None):
assert rating_key, part_id
running = scheduler.is_task_running("AvailableSubsForItem")
search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
if (search_results is None or force) and not running:
scheduler.dispatch_task("AvailableSubsForItem", rating_key=rating_key, item_type=item_type, part_id=part_id,
language=language)
running = True
oc = SubFolderObjectContainer(title2=unicode(title), replace_parent=True)
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, rating_key=rating_key, item_title=item_title, title=title, randomize=timestamp()),
title=u"Back to: %s" % title,
summary=current_data,
thumb=default_thumb
))
metadata = get_plex_metadata(rating_key, part_id, item_type)
scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
if not scanned_parts:
Log.Error("Couldn't list available subtitles for %s", rating_key)
return oc
video, plex_part = scanned_parts.items()[0]
video_display_data = [video.format] if video.format else []
if video.release_group:
video_display_data.append(u"by %s" % video.release_group)
video_display_data = " ".join(video_display_data)
current_display = (u"Current: %s (%s) " % (current_provider, current_score) if current_provider else "")
if not running:
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title, language=language,
filename=filename, part_id=part_id, title=title, current_id=current_id, force=True,
current_provider=current_provider, current_score=current_score,
current_data=current_data, item_type=item_type, randomize=timestamp()),
title=u"Search for %s subs (%s)" % (get_language(language).name, video_display_data),
summary=u"%sFilename: %s" % (current_display, filename),
thumb=default_thumb
))
else:
oc.add(DirectoryObject(
key=Callback(ListAvailableSubsForItemMenu, rating_key=rating_key, item_title=item_title,
language=language, filename=filename, current_data=current_data,
part_id=part_id, title=title, current_id=current_id, item_type=item_type,
current_provider=current_provider, current_score=current_score,
randomize=timestamp()),
title=u"Searching for %s subs (%s), refresh here ..." % (get_language(language).name, video_display_data),
summary=u"%sFilename: %s" % (current_display, filename),
thumb=default_thumb
))
if not search_results:
return oc
for subtitle in search_results:
oc.add(DirectoryObject(
key=Callback(TriggerDownloadSubtitle, rating_key=rating_key, randomize=timestamp(), item_title=item_title,
subtitle_id=str(subtitle.id), language=language),
title=u"%s: %s, score: %s" % ("Available" if current_id != subtitle.id else "Current",
subtitle.provider_name, subtitle.score),
summary=u"Release: %s, Matches: %s" % (subtitle.release_info, ", ".join(subtitle.matches)),
thumb=default_thumb
))
return oc
@route(PREFIX + '/download_subtitle/{rating_key}')
@debounce
def TriggerDownloadSubtitle(rating_key=None, subtitle_id=None, item_title=None, language=None, randomize=None):
set_refresh_menu_state("Downloading subtitle for %s" % item_title or rating_key)
search_results = get_item_task_data("AvailableSubsForItem", rating_key, language)
download_subtitle = None
for subtitle in search_results:
if str(subtitle.id) == subtitle_id:
download_subtitle = subtitle
break
if not download_subtitle:
Log.Error(u"Something went horribly wrong")
else:
scheduler.dispatch_task("DownloadSubtitleForItem", rating_key=rating_key, subtitle=download_subtitle)
return fatality(randomize=timestamp(), header=" ", replace_parent=True)
@route(PREFIX + '/item/{rating_key}')
@debounce
def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None,
previous_rating_key=None, timeout=8000, randomize=None, trigger=True):
assert rating_key
header = " "
if trigger:
set_refresh_menu_state(u"Triggering %sRefresh for %s" % ("Force-" if force else "", item_title))
Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind,
parent_rating_key=previous_rating_key, timeout=int(timeout))
header = u"%s of item %s triggered" % ("Refresh" if not force else "Forced-refresh", rating_key)
return fatality(randomize=timestamp(), header=header, replace_parent=True)
@route(PREFIX + '/missing/refresh')
@debounce
def RefreshMissing(randomize=None):
@@ -761,56 +146,9 @@ def RefreshMissing(randomize=None):
return fatality(header=header, replace_parent=True)
@route(PREFIX + '/advanced')
def AdvancedMenu(randomize=None, header=None, message=None):
oc = SubFolderObjectContainer(header=header or "Internal stuff, pay attention!", message=message, no_cache=True,
no_history=True,
replace_parent=False, title2="Advanced")
if config.lock_advanced_menu and not config.pin_correct:
oc.add(DirectoryObject(
key=Callback(PinMenu, randomize=timestamp(), success_go_to="advanced"),
title=pad_title("Enter PIN"),
summary="The owner has restricted the access to this menu. Please enter the correct pin",
))
return oc
oc.add(DirectoryObject(
key=Callback(TriggerRestart, randomize=timestamp()),
title=pad_title("Restart the plugin"),
))
oc.add(DirectoryObject(
key=Callback(GetLogsLink),
title="Get my logs (copy the appearing link and open it in your browser, please)",
summary="Copy the appearing link and open it in your browser, please",
))
oc.add(DirectoryObject(
key=Callback(TriggerBetterSubtitles, randomize=timestamp()),
title=pad_title("Trigger find better subtitles"),
))
oc.add(DirectoryObject(
key=Callback(LogStorage, key="tasks", randomize=timestamp()),
title=pad_title("Log the plugin's scheduled tasks state storage"),
))
oc.add(DirectoryObject(
key=Callback(LogStorage, key="ignore", randomize=timestamp()),
title=pad_title("Log the plugin's internal ignorelist storage"),
))
oc.add(DirectoryObject(
key=Callback(ResetStorage, key="tasks", randomize=timestamp()),
title=pad_title("Reset the plugin's scheduled tasks state storage"),
))
oc.add(DirectoryObject(
key=Callback(ResetStorage, key="ignore", randomize=timestamp()),
title=pad_title("Reset the plugin's internal ignorelist storage"),
))
return oc
@route(PREFIX + '/ValidatePrefs', enforce_route=True)
def ValidatePrefs():
Core.log.setLevel(logging.DEBUG)
Log.Debug("Validate Prefs called.")
# cache the channel state
update_dict = False
@@ -845,98 +183,44 @@ def ValidatePrefs():
Core.log.removeHandler(logger.console_handler)
Log.Debug("Stop logging to console")
Log.Debug("Validate Prefs called.")
# SZ config debug
Log.Debug("--- SZ Config-Debug ---")
for attr in [
"app_support_path", "data_path", "data_items_path", "enable_agent",
"enable_channel", "permissions_ok", "missing_permissions", "fs_encoding"]:
Log.Debug("config.%s: %s", attr, getattr(config, attr))
for attr in ["plugin_log_path", "server_log_path"]:
value = getattr(config, attr)
access = os.access(value, os.R_OK)
if Core.runtime.os == "Windows":
try:
f = open(value, "r")
f.read(1)
f.close()
except:
access = False
Log.Debug("config.%s: %s (accessible: %s)", attr, value, access)
# fixme: check existance of and os access of logs
Log.Debug("Platform: %s", Core.runtime.platform)
Log.Debug("OS: %s", Core.runtime.os)
Log.Debug("----- Environment -----")
for key, value in os.environ.iteritems():
if key.startswith("PLEX"):
if "TOKEN" in key:
outval = "xxxxxxxxxxxxxxxxxxx"
else:
outval = value
Log.Debug("%s: %s", key, outval)
Log.Debug("-----------------------")
Log.Debug("Setting log-level to %s", Prefs["log_level"])
logger.register_logging_handler(DEPENDENCY_MODULE_NAMES, level=Prefs["log_level"])
Core.log.setLevel(logging.getLevelName(Prefs["log_level"]))
return
def DispatchRestart():
Thread.CreateTimer(1.0, Restart)
@route(PREFIX + '/advanced/restart/trigger')
@debounce
def TriggerRestart(randomize=None):
set_refresh_menu_state("Restarting the plugin")
DispatchRestart()
return fatality(header="Restart triggered, please wait about 5 seconds", force_title=" ", only_refresh=True,
replace_parent=True,
no_history=True, randomize=timestamp())
@route(PREFIX + '/advanced/restart/execute')
def Restart():
Plex[":/plugins"].restart(PLUGIN_IDENTIFIER)
@route(PREFIX + '/storage/reset', sure=bool)
def ResetStorage(key, randomize=None, sure=False):
if not sure:
oc = SubFolderObjectContainer(no_history=True, title1="Reset subtitle storage", title2="Are you sure?")
oc.add(DirectoryObject(
key=Callback(ResetStorage, key=key, sure=True, randomize=timestamp()),
title=pad_title("Are you really sure?"),
))
return oc
reset_storage(key)
if key == "tasks":
# reinitialize the scheduler
scheduler.init_storage()
scheduler.setup_tasks()
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='Information Storage (%s) reset' % key
)
@route(PREFIX + '/storage/log')
def LogStorage(key, randomize=None):
log_storage(key)
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='Information Storage (%s) logged' % key
)
@route(PREFIX + '/triggerbetter')
def TriggerBetterSubtitles(randomize=None):
scheduler.dispatch_task("FindBetterSubtitles")
return AdvancedMenu(
randomize=timestamp(),
header='Success',
message='FindBetterSubtitles triggered'
)
@route(PREFIX + '/get_logs_link')
def GetLogsLink():
ip = Core.networking.http_request("http://www.plexapp.com/ip.php", cacheTime=7200).content.strip()
logs_link = "http://%s:32400%s?X-Plex-Token=%s" % (ip, PREFIX + '/logs', config.universal_plex_token)
oc = ObjectContainer(title2="Download Logs", no_cache=True, no_history=True,
header="Copy this link and open this in your browser, please",
message=logs_link)
return oc
@route(PREFIX + '/logs')
def DownloadLogs():
buff = StringIO.StringIO()
zip_archive = ZipFile(buff, mode='w', compression=ZIP_DEFLATED)
logs = sorted(glob.glob(config.plugin_log_path + '*')) + [config.server_log_path]
for path in logs:
data = StringIO.StringIO()
data.write(FileIO.read(path))
zip_archive.writestr(os.path.basename(path), data.getvalue())
zip_archive.close()
return ZipObject(buff.getvalue())
+9 -1
View File
@@ -7,9 +7,11 @@ from support.helpers import get_video_display_title
from support.ignore import ignore_list
from support.lib import get_intent
from support.config import config
from subzero.constants import ICON_SUB
from subzero.constants import ICON_SUB, ICON
from support.scheduler import scheduler
default_thumb = R(ICON_SUB)
main_icon = ICON if not config.is_development else "icon-dev.jpg"
def should_display_ignore(items, previous=None):
@@ -104,6 +106,12 @@ def set_refresh_menu_state(state_or_media, media_type="movies"):
Dict["current_refresh_state"] = u"%sRefreshing %s" % ("Force-" if force_refresh else "", unicode(title))
def get_item_task_data(task_name, rating_key, language):
task_data = scheduler.get_task_data(task_name)
search_results = task_data.get(rating_key, {}) if task_data else {}
return search_results.get(language)
def enable_channel_wrapper(func):
"""
returns the original wrapper :func: (route or handler) if applicable, else the plain to-be-wrapped function
+22
View File
@@ -0,0 +1,22 @@
# coding=utf-8
from subzero.constants import PREFIX
from menu_helpers import debounce, set_refresh_menu_state
from support.items import refresh_item
from support.helpers import timestamp
@route(PREFIX + '/item/{rating_key}')
@debounce
def RefreshItem(rating_key=None, came_from="/recent", item_title=None, force=False, refresh_kind=None,
previous_rating_key=None, timeout=8000, randomize=None, trigger=True):
assert rating_key
from interface.main import fatality
header = " "
if trigger:
set_refresh_menu_state(u"Triggering %sRefresh for %s" % ("Force-" if force else "", item_title))
Thread.Create(refresh_item, rating_key=rating_key, force=force, refresh_kind=refresh_kind,
parent_rating_key=previous_rating_key, timeout=int(timeout))
header = u"%s of item %s triggered" % ("Refresh" if not force else "Forced-refresh", rating_key)
return fatality(randomize=timestamp(), header=header, replace_parent=True)
+231
View File
@@ -0,0 +1,231 @@
# coding=utf-8
import traceback
import types
from babelfish import Language
from menu_helpers import debounce, SubFolderObjectContainer
from subliminal_patch import PatchedSubtitle as Subtitle
from subzero.modification import registry as mod_registry, SubtitleModifications
from subzero.constants import PREFIX
from support.plex_media import get_plex_metadata, scan_videos
from support.storage import save_subtitles
from support.helpers import timestamp, pad_title
from support.items import get_current_sub
@route(PREFIX + '/item/sub_mods/{rating_key}/{part_id}', force=bool)
@debounce
def SubtitleModificationsMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
kwargs.pop("randomize")
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
for identifier, mod in mod_registry.mods.iteritems():
if mod.advanced:
continue
if mod.exclusive and identifier in current_sub.mods:
continue
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=identifier, mode="add", randomize=timestamp(), **kwargs),
title=pad_title(mod.description), summary=mod.long_description or ""
))
fps_mod = SubtitleModifications.get_mod_class("change_FPS")
oc.add(DirectoryObject(
key=Callback(SubtitleFPSModMenu, randomize=timestamp(), **kwargs),
title=pad_title(fps_mod.description), summary=fps_mod.long_description or ""
))
shift_mod = SubtitleModifications.get_mod_class("shift_offset")
oc.add(DirectoryObject(
key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs),
title=pad_title(shift_mod.description), summary=shift_mod.long_description or ""
))
if current_sub.mods:
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=None, mode="remove_last", randomize=timestamp(), **kwargs),
title=pad_title("Remove last applied mod (%s)" % current_sub.mods[-1]),
summary=u"Currently applied mods: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
))
oc.add(DirectoryObject(
key=Callback(SubtitleListMods, randomize=timestamp(), **kwargs),
title=pad_title("Manage applied mods"),
summary=u"Currently applied mods: %s" % (", ".join(current_sub.mods))
))
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=None, mode="clear", randomize=timestamp(), **kwargs),
title=pad_title("Restore original version"),
summary=u"Currently applied mods: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
))
return oc
@route(PREFIX + '/item/sub_mod_fps/{rating_key}/{part_id}', force=bool)
@debounce
def SubtitleFPSModMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
item_type = kwargs["item_type"]
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
metadata = get_plex_metadata(rating_key, part_id, item_type)
scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
video, plex_part = scanned_parts.items()[0]
target_fps = plex_part.fps
kwargs.pop("randomize")
for fps in ["23.976", "24.000", "25.000", "29.970", "30.000", "50.000", "59.940", "60.000"]:
if float(fps) == float(target_fps):
continue
if float(fps) > float(target_fps):
indicator = "subs constantly getting faster"
else:
indicator = "subs constantly getting slower"
mod_ident = SubtitleModifications.get_mod_signature("change_FPS", **{"from": fps, "to": target_fps})
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs),
title="%s fps -> %s fps (%s)" % (fps, target_fps, indicator)
))
return oc
POSSIBLE_UNITS = (("ms", "milliseconds"), ("s", "seconds"), ("m", "minutes"), ("h", "hours"))
POSSIBLE_UNITS_D = dict(POSSIBLE_UNITS)
@route(PREFIX + '/item/sub_mod_shift_unit/{rating_key}/{part_id}', force=bool)
@debounce
def SubtitleShiftModUnitMenu(**kwargs):
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
kwargs.pop("randomize")
for unit, title in POSSIBLE_UNITS:
oc.add(DirectoryObject(
key=Callback(SubtitleShiftModMenu, unit=unit, randomize=timestamp(), **kwargs),
title="Adjust by %s" % title
))
return oc
@route(PREFIX + '/item/sub_mod_shift/{rating_key}/{part_id}/{unit}', force=bool)
@debounce
def SubtitleShiftModMenu(unit=None, **kwargs):
if unit not in POSSIBLE_UNITS_D:
raise NotImplementedError
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
kwargs.pop("randomize")
rng = []
if unit == "h":
rng = range(-10, 11)
elif unit in ("m", "s"):
rng = range(-59, 60)
elif unit == "ms":
rng = range(-900, 1000, 100)
for i in rng:
if i == 0:
continue
mod_ident = SubtitleModifications.get_mod_signature("shift_offset", **{unit: i})
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=mod_ident, mode="add", randomize=timestamp(), **kwargs),
title="%s %s" % (("%s" if i < 0 else "+%s") % i, unit)
))
return oc
@route(PREFIX + '/item/sub_set_mods/{rating_key}/{part_id}/{mods}/{mode}', force=bool)
@debounce
def SubtitleSetMods(mods=None, mode=None, **kwargs):
if not isinstance(mods, types.ListType) and mods:
mods = [mods]
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
lang_a2 = kwargs["language"]
item_type = kwargs["item_type"]
language = Language.fromietf(lang_a2)
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
if mode == "add":
for mod in mods:
identifier, args = SubtitleModifications.parse_identifier(mod)
if identifier not in mod_registry.mods_available:
raise NotImplementedError("Mod unknown or not registered")
current_sub.add_mod(mod)
elif mode == "clear":
current_sub.add_mod(None)
elif mode == "remove":
for mod in mods:
current_sub.mods.remove(mod)
elif mode == "remove_last":
if current_sub.mods:
current_sub.mods.pop()
else:
raise NotImplementedError("Wrong mode given")
storage.save(stored_subs)
metadata = get_plex_metadata(rating_key, part_id, item_type)
scanned_parts = scan_videos([metadata], kind="series" if item_type == "episode" else "movie", ignore_all=True)
video, plex_part = scanned_parts.items()[0]
subtitle = Subtitle(language, mods=current_sub.mods)
subtitle.content = current_sub.content
subtitle.plex_media_fps = plex_part.fps
subtitle.page_link = "modify subtitles with: %s" % (", ".join(current_sub.mods) if current_sub.mods else "none")
subtitle.language = language
try:
save_subtitles(scanned_parts, {video: [subtitle]}, mode="m", bare_save=True)
Log.Debug("Modified %s subtitle for: %s:%s with: %s", language.name, rating_key, part_id,
", ".join(current_sub.mods) if current_sub.mods else "none")
except:
Log.Error("Something went wrong when modifying subtitle: %s", traceback.format_exc())
kwargs.pop("randomize")
return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
@route(PREFIX + '/item/sub_list_mods/{rating_key}/{part_id}', force=bool)
@debounce
def SubtitleListMods(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
kwargs.pop("randomize")
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
for identifier in current_sub.mods:
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=identifier, mode="remove", randomize=timestamp(), **kwargs),
title="Remove: %s" % identifier
))
return oc
+2 -2
View File
@@ -32,9 +32,9 @@ import missing_subtitles
sys.modules["support.missing_subtitles"] = missing_subtitles
import background
import scheduler
sys.modules["support.background"] = background
sys.modules["support.scheduler"] = scheduler
import tasks
+2 -2
View File
@@ -11,9 +11,9 @@ class PlexActivityManager(object):
def start(self):
activity_sources_enabled = None
if config.universal_plex_token:
if config.plex_token:
from plex import Plex
Plex.configuration.defaults.authentication(config.universal_plex_token)
Plex.configuration.defaults.authentication(config.plex_token)
activity_sources_enabled = ["websocket"]
Activity.on('websocket.playing', self.on_playing)
+39
View File
@@ -9,6 +9,7 @@ import datetime
import subliminal
import subliminal_patch
from babelfish import Language
from subliminal.cli import MutexLock
from subzero.lib.io import FileIO, get_viable_encoding
from subzero.constants import PLUGIN_NAME, PLUGIN_IDENTIFIER, MOVIE, SHOW
from lib import Plex
@@ -42,7 +43,10 @@ class Config(object):
plugin_log_path = None
server_log_path = None
app_support_path = None
data_path = None
data_items_path = None
universal_plex_token = None
plex_token = None
is_development = False
enable_channel = True
@@ -65,12 +69,15 @@ class Config(object):
notify_executable = None
sections = None
enabled_sections = None
remove_hi = False
fix_ocr = False
enforce_encoding = False
chmod = None
forced_only = False
exotic_ext = False
treat_und_as_first = False
ext_match_strictness = False
default_mods = None
use_activities = False
activity_mode = None
@@ -84,7 +91,10 @@ class Config(object):
self.full_version = u"%s %s" % (PLUGIN_NAME, self.version)
self.set_log_paths()
self.app_support_path = Core.app_support_path
self.data_path = getattr(Data, "_core").storage.data_path
self.data_items_path = os.path.join(self.data_path, "DataItems")
self.universal_plex_token = self.get_universal_plex_token()
self.plex_token = os.environ.get("PLEXTOKEN", self.universal_plex_token)
self.set_plugin_mode()
self.set_plugin_lock()
@@ -102,14 +112,32 @@ class Config(object):
self.enabled_sections = self.check_enabled_sections()
self.permissions_ok = self.check_permissions()
self.notify_executable = self.check_notify_executable()
self.remove_hi = cast_bool(Prefs['subtitles.remove_hi'])
self.fix_ocr = cast_bool(Prefs['subtitles.fix_ocr'])
self.enforce_encoding = cast_bool(Prefs['subtitles.enforce_encoding'])
self.chmod = self.check_chmod()
self.forced_only = cast_bool(Prefs["subtitles.only_foreign"])
self.exotic_ext = cast_bool(Prefs["subtitles.scan.exotic_ext"])
self.treat_und_as_first = cast_bool(Prefs["subtitles.language.treat_und_as_first"])
self.ext_match_strictness = self.determine_ext_sub_strictness()
self.default_mods = self.get_default_mods()
self.initialized = True
def init_cache(self):
use_fallback_cache = True
if Core.runtime.os != "Windows":
try:
subliminal.region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30),
arguments={'filename': os.path.join(config.data_items_path, 'subzero.dbm'),
'lock_factory': MutexLock})
use_fallback_cache = False
except:
pass
if use_fallback_cache:
Log.Warn("Not using file based cache!")
subliminal.region.configure('dogpile.cache.memory')
def set_log_paths(self):
# find log handler
for handler in Core.log.handlers:
@@ -136,6 +164,8 @@ class Config(object):
else:
Log("Did NOT find Preferences file - please check logfile and hierarchy. Aborting!")
# fixme: windows
def set_plugin_mode(self):
if Prefs["plugin_mode"] == "only agent":
self.enable_channel = False
@@ -400,6 +430,15 @@ class Config(object):
return "loose"
return "strict"
def get_default_mods(self):
mods = []
if self.remove_hi:
mods.append("remove_HI")
if self.fix_ocr:
mods.append("OCR_fixes")
return mods
def set_activity_modes(self):
val = Prefs["activity.on_playback"]
if val == "never":
+2 -2
View File
@@ -110,9 +110,9 @@ def str_pad(s, length, align='left', pad_char=' ', trim=False):
raise ValueError("Unknown align type, expected either 'left' or 'right'")
def pad_title(value):
def pad_title(value, width=49):
"""Pad a title to 30 characters to force the 'details' view."""
return str_pad(value, 49, pad_char=' ')
return str_pad(value, width, pad_char=' ')
def get_plex_item_display_title(item, kind, parent=None, parent_title=None, section_title=None,
+10
View File
@@ -283,3 +283,13 @@ def refresh_item(rating_key, force=False, timeout=8000, refresh_kind=None, paren
for key in refresh:
Log.Info("%s item %s", "Refreshing" if not force else "Forced-refreshing", key)
Plex["library/metadata"].refresh(key)
def get_current_sub(rating_key, part_id, language):
from support.storage import get_subtitle_storage
item = get_item(rating_key)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
current_sub = stored_subs.get_any(part_id, language)
return current_sub, stored_subs, subtitle_storage
+1 -1
View File
@@ -56,7 +56,7 @@ def find_subtitles(part):
global_folders.append(global_subtitle_folder)
# normalize all paths
paths = [os.path.normpath(os.path.realpath(helpers.unicodize(path))) for path in paths]
paths = [os.path.normpath(helpers.unicodize(path)) for path in paths]
# We start by building a dictionary of files to their absolute paths. We also need to know
# the number of media files that are actually present, in case the found local media asset
+1 -1
View File
@@ -72,7 +72,7 @@ class DefaultScheduler(object):
try:
task_frequency = Prefs["scheduler.tasks.%s.frequency" % task.name]
except KeyError:
task_frequency = None
task_frequency = getattr(task, "frequency", None)
self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)}
+28 -5
View File
@@ -6,16 +6,17 @@ import pprint
import copy
import subliminal
from items import get_item
from subzero.subtitle_storage import StoredSubtitlesManager
from subtitlehelpers import force_utf8
from config import config
from helpers import notify_executable, get_title_for_video_metadata, cast_bool, force_unicode
from plex_media import PMSMediaProxy
from support.items import get_item
get_subtitle_storage = lambda: StoredSubtitlesManager(Data, get_item)
def get_subtitle_storage():
return StoredSubtitlesManager(Data, get_item)
def whack_missing_parts(scanned_video_part_map, existing_parts=None):
@@ -156,9 +157,29 @@ def save_subtitles_to_metadata(videos, subtitles):
return True
def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a", bare_save=False, mods=None):
"""
:param scanned_video_part_map:
:param downloaded_subtitles:
:param mode:
:param bare_save: don't trigger anything; don't store information
:param mods: enabled mods
:return:
"""
meta_fallback = False
save_successful = False
if mods:
for video, video_subtitles in downloaded_subtitles.items():
if not video_subtitles:
continue
for subtitle in video_subtitles:
Log.Info("Applying mods: %s to %s", mods, subtitle)
subtitle.mods = mods
subtitle.plex_media_fps = video.fps
storage = "metadata"
if Prefs['subtitles.save.filesystem']:
storage = "filesystem"
@@ -180,7 +201,9 @@ def save_subtitles(scanned_video_part_map, downloaded_subtitles, mode="a"):
Log.Debug("Using metadata as subtitle storage")
save_successful = save_subtitles_to_metadata(scanned_video_part_map, downloaded_subtitles)
if save_successful and config.notify_executable:
if not bare_save and save_successful and config.notify_executable:
notify_executable(config.notify_executable, scanned_video_part_map, downloaded_subtitles, storage)
store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)
if not bare_save:
store_subtitle_info(scanned_video_part_map, downloaded_subtitles, storage, mode=mode)
+32 -5
View File
@@ -11,7 +11,7 @@ from subliminal import list_subtitles as list_all_subtitles
from babelfish import Language
from missing_subtitles import items_get_all_missing_subs, refresh_item
from background import scheduler
from scheduler import scheduler
from storage import save_subtitles, whack_missing_parts, get_subtitle_storage
from support.config import config
from support.items import get_recent_items, is_ignored, get_item
@@ -80,6 +80,7 @@ class Task(object):
return
def run(self):
Log.Info(u"Task: running: %s", self.name)
self.time_start = datetime.datetime.now()
def post_run(self, data_holder):
@@ -88,6 +89,7 @@ class Task(object):
if self.time_start:
self.last_run_time = self.last_run - self.time_start
self.time_start = None
Log.Info(u"Task: ran: %s", self.name)
class SearchAllRecentlyAddedMissing(Task):
@@ -248,7 +250,7 @@ class DownloadSubtitleMixin(object):
if subtitle.content:
try:
whack_missing_parts(scanned_parts)
save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode)
save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode, mods=config.default_mods)
Log.Debug("Manually downloaded subtitle for: %s", rating_key)
download_successful = True
refresh_item(rating_key)
@@ -362,13 +364,21 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
return
now = datetime.datetime.now()
min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
subtitle_storage = get_subtitle_storage()
recent_subs = subtitle_storage.load_recent_files(age_days=max_search_days)
for fn, stored_subs in recent_subs.iteritems():
video_id = stored_subs.video_id
cutoff = self.series_cutoff if stored_subs.item_type == "episode" else self.movies_cutoff
if stored_subs.item_type == "episode":
cutoff = self.series_cutoff
min_score = min_score_series
else:
cutoff = self.movies_cutoff
min_score = min_score_movies
# don't search for better subtitles until at least 30 minutes have passed
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
@@ -420,7 +430,7 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
better_downloaded = False
better_tried_download = 0
for sub in subs:
if sub.score > current_score:
if sub.score > current_score and sub.score > min_score:
Log.Debug("Better subtitle found for %s, downloading", video_id)
better_tried_download += 1
ret = self.download_subtitle(sub, video_id, mode="b")
@@ -446,7 +456,23 @@ class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
if better_found:
Log.Debug("Task: %s, done. Better subtitles found for %s items", self.name, better_found)
self.running = False
class SubtitleStorageMaintenance(Task):
periodic = True
frequency = "every 7 days"
def run(self):
super(SubtitleStorageMaintenance, self).run()
self.running = True
Log.Info("Running subtitle storage maintenance")
storage = get_subtitle_storage()
deleted_items = storage.delete_missing_files()
if deleted_items:
Log.Info("Subtitle information for %d non-existant videos have been cleaned up" % len(deleted_items))
Log.Debug("Videos: %s" % deleted_items)
else:
Log.Info("Nothing to do")
scheduler.register(SearchAllRecentlyAddedMissing)
@@ -454,3 +480,4 @@ scheduler.register(AvailableSubsForItem)
scheduler.register(DownloadSubtitleForItem)
scheduler.register(MissingSubtitles)
scheduler.register(FindBetterSubtitles)
scheduler.register(SubtitleStorageMaintenance)
+12
View File
@@ -395,6 +395,18 @@
],
"default": "don't prefer"
},
{
"id": "subtitles.remove_hi",
"label": "Remove Hearing Impaired tags from downloaded subtitles",
"type": "bool",
"default": "false"
},
{
"id": "subtitles.fix_ocr",
"label": "Fix common OCR errors in downloaded subtitles",
"type": "bool",
"default": "true"
},
{
"id": "subtitles.enforce_encoding",
"label": "Normalize subtitle encoding to UTF-8",
+2 -2
View File
@@ -13,7 +13,7 @@
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleVersion</key>
<string>2.0.0.0</string>
<string>2.0.0.13</string>
<key>PlexFrameworkVersion</key>
<string>2</string>
<key>PlexPluginClass</key>
@@ -32,7 +32,7 @@
&lt;h1&gt;Sub-Zero for Plex&lt;/h1&gt;&lt;i&gt;Subtitles done right&lt;/i&gt;
Version 2.0.0.0 DEV #6
Version 2.0.0.13 DEV
Originally based on @bramwalet's awesome &lt;a href=&quot;https://github.com/bramwalet/Subliminal.bundle&quot;&gt;Subliminal.bundle&lt;/a&gt;
@@ -1,25 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
__title__ = 'babelfish'
__version__ = '0.5.5-dev'
__author__ = 'Antoine Bertin'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015 the BabelFish authors'
import sys
if sys.version_info[0] >= 3:
basestr = str
else:
basestr = basestring
from .converters import (LanguageConverter, LanguageReverseConverter, LanguageEquivalenceConverter, CountryConverter,
CountryReverseConverter)
from .country import country_converters, COUNTRIES, COUNTRY_MATRIX, Country
from .exceptions import Error, LanguageConvertError, LanguageReverseError, CountryConvertError, CountryReverseError
from .language import language_converters, LANGUAGES, LANGUAGE_MATRIX, Language
from .script import SCRIPTS, SCRIPT_MATRIX, Script
@@ -1,287 +0,0 @@
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
import collections
from pkg_resources import iter_entry_points, EntryPoint
from ..exceptions import LanguageConvertError, LanguageReverseError
# from https://github.com/kennethreitz/requests/blob/master/requests/structures.py
class CaseInsensitiveDict(collections.MutableMapping):
"""A case-insensitive ``dict``-like object.
Implements all methods and operations of
``collections.MutableMapping`` as well as dict's ``copy``. Also
provides ``lower_items``.
All keys are expected to be strings. The structure remembers the
case of the last key to be set, and ``iter(instance)``,
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
will contain case-sensitive keys. However, querying and contains
testing is case insensitive:
cid = CaseInsensitiveDict()
cid['English'] = 'eng'
cid['ENGLISH'] == 'eng' # True
list(cid) == ['English'] # True
If the constructor, ``.update``, or equality comparison
operations are given keys that have equal ``.lower()``s, the
behavior is undefined.
"""
def __init__(self, data=None, **kwargs):
self._store = dict()
if data is None:
data = {}
self.update(data, **kwargs)
def __setitem__(self, key, value):
# Use the lowercased key for lookups, but store the actual
# key alongside the value.
self._store[key.lower()] = (key, value)
def __getitem__(self, key):
return self._store[key.lower()][1]
def __delitem__(self, key):
del self._store[key.lower()]
def __iter__(self):
return (casedkey for casedkey, mappedvalue in self._store.values())
def __len__(self):
return len(self._store)
def lower_items(self):
"""Like iteritems(), but with all lowercase keys."""
return (
(lowerkey, keyval[1])
for (lowerkey, keyval)
in self._store.items()
)
def __eq__(self, other):
if isinstance(other, collections.Mapping):
other = CaseInsensitiveDict(other)
else:
return NotImplemented
# Compare insensitively
return dict(self.lower_items()) == dict(other.lower_items())
# Copy is required
def copy(self):
return CaseInsensitiveDict(self._store.values())
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, dict(self.items()))
class LanguageConverter(object):
"""A :class:`LanguageConverter` supports converting an alpha3 language code with an
alpha2 country code and a script code into a custom code
.. attribute:: codes
Set of possible custom codes
"""
def convert(self, alpha3, country=None, script=None):
"""Convert an alpha3 language code with an alpha2 country code and a script code
into a custom code
:param string alpha3: ISO-639-3 language code
:param country: ISO-3166 country code, if any
:type country: string or None
:param script: ISO-15924 script code, if any
:type script: string or None
:return: the corresponding custom code
:rtype: string
:raise: :class:`~babelfish.exceptions.LanguageConvertError`
"""
raise NotImplementedError
class LanguageReverseConverter(LanguageConverter):
"""A :class:`LanguageConverter` able to reverse a custom code into a alpha3
ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code
"""
def reverse(self, code):
"""Reverse a custom code into alpha3, country and script code
:param string code: custom code to reverse
:return: the corresponding alpha3 ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code
:rtype: tuple
:raise: :class:`~babelfish.exceptions.LanguageReverseError`
"""
raise NotImplementedError
class LanguageEquivalenceConverter(LanguageReverseConverter):
"""A :class:`LanguageEquivalenceConverter` is a utility class that allows you to easily define a
:class:`LanguageReverseConverter` by only specifying the dict from alpha3 to their corresponding symbols.
You must specify the dict of equivalence as a class variable named SYMBOLS.
If you also set the class variable CASE_SENSITIVE to ``True`` then the reverse conversion function will be
case-sensitive (it is case-insensitive by default).
Example::
class MyCodeConverter(babelfish.LanguageEquivalenceConverter):
CASE_SENSITIVE = True
SYMBOLS = {'fra': 'mycode1', 'eng': 'mycode2'}
"""
CASE_SENSITIVE = False
def __init__(self):
self.codes = set()
self.to_symbol = {}
if self.CASE_SENSITIVE:
self.from_symbol = {}
else:
self.from_symbol = CaseInsensitiveDict()
for alpha3, symbol in self.SYMBOLS.items():
self.to_symbol[alpha3] = symbol
self.from_symbol[symbol] = (alpha3, None, None)
self.codes.add(symbol)
def convert(self, alpha3, country=None, script=None):
try:
return self.to_symbol[alpha3]
except KeyError:
raise LanguageConvertError(alpha3, country, script)
def reverse(self, code):
try:
return self.from_symbol[code]
except KeyError:
raise LanguageReverseError(code)
class CountryConverter(object):
"""A :class:`CountryConverter` supports converting an alpha2 country code
into a custom code
.. attribute:: codes
Set of possible custom codes
"""
def convert(self, alpha2):
"""Convert an alpha2 country code into a custom code
:param string alpha2: ISO-3166-1 language code
:return: the corresponding custom code
:rtype: string
:raise: :class:`~babelfish.exceptions.CountryConvertError`
"""
raise NotImplementedError
class CountryReverseConverter(CountryConverter):
"""A :class:`CountryConverter` able to reverse a custom code into a alpha2
ISO-3166-1 country code
"""
def reverse(self, code):
"""Reverse a custom code into alpha2 code
:param string code: custom code to reverse
:return: the corresponding alpha2 ISO-3166-1 country code
:rtype: string
:raise: :class:`~babelfish.exceptions.CountryReverseError`
"""
raise NotImplementedError
class ConverterManager(object):
"""Manager for babelfish converters behaving like a dict with lazy loading
Loading is done in this order:
* Entry point converters
* Registered converters
* Internal converters
.. attribute:: entry_point
The entry point where to look for converters
.. attribute:: internal_converters
Internal converters with entry point syntax
"""
entry_point = ''
internal_converters = []
def __init__(self):
#: Registered converters with entry point syntax
self.registered_converters = []
#: Loaded converters
self.converters = {}
def __getitem__(self, name):
"""Get a converter, lazy loading it if necessary"""
if name in self.converters:
return self.converters[name]
for ep in iter_entry_points(self.entry_point):
if ep.name == name:
self.converters[ep.name] = ep.load()()
return self.converters[ep.name]
for ep in (EntryPoint.parse(c) for c in self.registered_converters + self.internal_converters):
if ep.name == name:
# `require` argument of ep.load() is deprecated in newer versions of setuptools
if hasattr(ep, 'resolve'):
plugin = ep.resolve()
elif hasattr(ep, '_load'):
plugin = ep._load()
else:
plugin = ep.load(require=False)
self.converters[ep.name] = plugin()
return self.converters[ep.name]
raise KeyError(name)
def __setitem__(self, name, converter):
"""Load a converter"""
self.converters[name] = converter
def __delitem__(self, name):
"""Unload a converter"""
del self.converters[name]
def __iter__(self):
"""Iterator over loaded converters"""
return iter(self.converters)
def register(self, entry_point):
"""Register a converter
:param string entry_point: converter to register (entry point syntax)
:raise: ValueError if already registered
"""
if entry_point in self.registered_converters:
raise ValueError('Already registered')
self.registered_converters.insert(0, entry_point)
def unregister(self, entry_point):
"""Unregister a converter
:param string entry_point: converter to unregister (entry point syntax)
"""
self.registered_converters.remove(entry_point)
def __contains__(self, name):
return name in self.converters
@@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageEquivalenceConverter
from ..language import LANGUAGE_MATRIX
class Alpha2Converter(LanguageEquivalenceConverter):
CASE_SENSITIVE = True
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
if iso_language.alpha2:
SYMBOLS[iso_language.alpha3] = iso_language.alpha2
@@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageEquivalenceConverter
from ..language import LANGUAGE_MATRIX
class Alpha3BConverter(LanguageEquivalenceConverter):
CASE_SENSITIVE = True
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
if iso_language.alpha3b:
SYMBOLS[iso_language.alpha3] = iso_language.alpha3b
@@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageEquivalenceConverter
from ..language import LANGUAGE_MATRIX
class Alpha3TConverter(LanguageEquivalenceConverter):
CASE_SENSITIVE = True
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
if iso_language.alpha3t:
SYMBOLS[iso_language.alpha3] = iso_language.alpha3t
@@ -1,31 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import CountryReverseConverter, CaseInsensitiveDict
from ..country import COUNTRY_MATRIX
from ..exceptions import CountryConvertError, CountryReverseError
class CountryNameConverter(CountryReverseConverter):
def __init__(self):
self.codes = set()
self.to_name = {}
self.from_name = CaseInsensitiveDict()
for country in COUNTRY_MATRIX:
self.codes.add(country.name)
self.to_name[country.alpha2] = country.name
self.from_name[country.name] = country.alpha2
def convert(self, alpha2):
if alpha2 not in self.to_name:
raise CountryConvertError(alpha2)
return self.to_name[alpha2]
def reverse(self, name):
if name not in self.from_name:
raise CountryReverseError(name)
return self.from_name[name]
@@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageEquivalenceConverter
from ..language import LANGUAGE_MATRIX
class NameConverter(LanguageEquivalenceConverter):
CASE_SENSITIVE = False
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
if iso_language.name:
SYMBOLS[iso_language.alpha3] = iso_language.name
@@ -1,36 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageReverseConverter, CaseInsensitiveDict
from ..exceptions import LanguageReverseError
from ..language import language_converters
class OpenSubtitlesConverter(LanguageReverseConverter):
def __init__(self):
self.alpha3b_converter = language_converters['alpha3b']
self.alpha2_converter = language_converters['alpha2']
self.to_opensubtitles = {('por', 'BR'): 'pob', ('gre', None): 'ell', ('srp', None): 'scc', ('srp', 'ME'): 'mne'}
self.from_opensubtitles = CaseInsensitiveDict({'pob': ('por', 'BR'), 'pb': ('por', 'BR'), 'ell': ('ell', None),
'scc': ('srp', None), 'mne': ('srp', 'ME')})
self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(['pob', 'pb', 'scc', 'mne']))
def convert(self, alpha3, country=None, script=None):
alpha3b = self.alpha3b_converter.convert(alpha3, country, script)
if (alpha3b, country) in self.to_opensubtitles:
return self.to_opensubtitles[(alpha3b, country)]
return alpha3b
def reverse(self, opensubtitles):
if opensubtitles in self.from_opensubtitles:
return self.from_opensubtitles[opensubtitles]
for conv in [self.alpha3b_converter, self.alpha2_converter]:
try:
return conv.reverse(opensubtitles)
except LanguageReverseError:
pass
raise LanguageReverseError(opensubtitles)
@@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageConverter
from ..exceptions import LanguageConvertError
from ..language import LANGUAGE_MATRIX
class ScopeConverter(LanguageConverter):
FULLNAME = {'I': 'individual', 'M': 'macrolanguage', 'S': 'special'}
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
SYMBOLS[iso_language.alpha3] = iso_language.scope
codes = set(SYMBOLS.values())
def convert(self, alpha3, country=None, script=None):
if self.SYMBOLS[alpha3] in self.FULLNAME:
return self.FULLNAME[self.SYMBOLS[alpha3]]
raise LanguageConvertError(alpha3, country, script)
@@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from . import LanguageConverter
from ..exceptions import LanguageConvertError
from ..language import LANGUAGE_MATRIX
class LanguageTypeConverter(LanguageConverter):
FULLNAME = {'A': 'ancient', 'C': 'constructed', 'E': 'extinct', 'H': 'historical', 'L': 'living', 'S': 'special'}
SYMBOLS = {}
for iso_language in LANGUAGE_MATRIX:
SYMBOLS[iso_language.alpha3] = iso_language.type
codes = set(SYMBOLS.values())
def convert(self, alpha3, country=None, script=None):
if self.SYMBOLS[alpha3] in self.FULLNAME:
return self.FULLNAME[self.SYMBOLS[alpha3]]
raise LanguageConvertError(alpha3, country, script)
@@ -1,104 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from collections import namedtuple
from functools import partial
from pkg_resources import resource_stream # @UnresolvedImport
from .converters import ConverterManager
from . import basestr
COUNTRIES = {}
COUNTRY_MATRIX = []
#: The namedtuple used in the :data:`COUNTRY_MATRIX`
IsoCountry = namedtuple('IsoCountry', ['name', 'alpha2'])
f = resource_stream('babelfish', 'data/iso-3166-1.txt')
f.readline()
for l in f:
iso_country = IsoCountry(*l.decode('utf-8').strip().split(';'))
COUNTRIES[iso_country.alpha2] = iso_country.name
COUNTRY_MATRIX.append(iso_country)
f.close()
class CountryConverterManager(ConverterManager):
""":class:`~babelfish.converters.ConverterManager` for country converters"""
entry_point = 'babelfish.country_converters'
internal_converters = ['name = babelfish.converters.countryname:CountryNameConverter']
country_converters = CountryConverterManager()
class CountryMeta(type):
"""The :class:`Country` metaclass
Dynamically redirect :meth:`Country.frommycode` to :meth:`Country.fromcode` with the ``mycode`` `converter`
"""
def __getattr__(cls, name):
if name.startswith('from'):
return partial(cls.fromcode, converter=name[4:])
return type.__getattribute__(cls, name)
class Country(CountryMeta(str('CountryBase'), (object,), {})):
"""A country on Earth
A country is represented by a 2-letter code from the ISO-3166 standard
:param string country: 2-letter ISO-3166 country code
"""
def __init__(self, country):
if country not in COUNTRIES:
raise ValueError('%r is not a valid country' % country)
#: ISO-3166 2-letter country code
self.alpha2 = country
@classmethod
def fromcode(cls, code, converter):
"""Create a :class:`Country` by its `code` using `converter` to
:meth:`~babelfish.converters.CountryReverseConverter.reverse` it
:param string code: the code to reverse
:param string converter: name of the :class:`~babelfish.converters.CountryReverseConverter` to use
:return: the corresponding :class:`Country` instance
:rtype: :class:`Country`
"""
return cls(country_converters[converter].reverse(code))
def __getstate__(self):
return self.alpha2
def __setstate__(self, state):
self.alpha2 = state
def __getattr__(self, name):
return country_converters[name].convert(self.alpha2)
def __hash__(self):
return hash(self.alpha2)
def __eq__(self, other):
if isinstance(other, basestr):
return str(self) == other
if not isinstance(other, Country):
return False
return self.alpha2 == other.alpha2
def __ne__(self, other):
return not self == other
def __repr__(self):
return '<Country [%s]>' % self
def __str__(self):
return self.alpha2
@@ -1,45 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
import os.path
import tempfile
import zipfile
import requests
DATA_DIR = os.path.dirname(__file__)
# iso-3166-1.txt
print('Downloading ISO-3166-1 standard (ISO country codes)...')
with open(os.path.join(DATA_DIR, 'iso-3166-1.txt'), 'w') as f:
r = requests.get('http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt.htm')
f.write(r.content.strip())
# iso-639-3.tab
print('Downloading ISO-639-3 standard (ISO language codes)...')
with tempfile.TemporaryFile() as f:
r = requests.get('http://www-01.sil.org/iso639-3/iso-639-3_Code_Tables_20130531.zip')
f.write(r.content)
with zipfile.ZipFile(f) as z:
z.extract('iso-639-3.tab', DATA_DIR)
# iso-15924
print('Downloading ISO-15924 standard (ISO script codes)...')
with tempfile.TemporaryFile() as f:
r = requests.get('http://www.unicode.org/iso15924/iso15924.txt.zip')
f.write(r.content)
with zipfile.ZipFile(f) as z:
z.extract('iso15924-utf8-20131012.txt', DATA_DIR)
# opensubtitles supported languages
print('Downloading OpenSubtitles supported languages...')
with open(os.path.join(DATA_DIR, 'opensubtitles_languages.txt'), 'w') as f:
r = requests.get('http://www.opensubtitles.org/addons/export_languages.php')
f.write(r.content)
print('Done!')
@@ -1,250 +0,0 @@
Country Name;ISO 3166-1-alpha-2 code
AFGHANISTAN;AF
ÅLAND ISLANDS;AX
ALBANIA;AL
ALGERIA;DZ
AMERICAN SAMOA;AS
ANDORRA;AD
ANGOLA;AO
ANGUILLA;AI
ANTARCTICA;AQ
ANTIGUA AND BARBUDA;AG
ARGENTINA;AR
ARMENIA;AM
ARUBA;AW
AUSTRALIA;AU
AUSTRIA;AT
AZERBAIJAN;AZ
BAHAMAS;BS
BAHRAIN;BH
BANGLADESH;BD
BARBADOS;BB
BELARUS;BY
BELGIUM;BE
BELIZE;BZ
BENIN;BJ
BERMUDA;BM
BHUTAN;BT
BOLIVIA, PLURINATIONAL STATE OF;BO
BONAIRE, SINT EUSTATIUS AND SABA;BQ
BOSNIA AND HERZEGOVINA;BA
BOTSWANA;BW
BOUVET ISLAND;BV
BRAZIL;BR
BRITISH INDIAN OCEAN TERRITORY;IO
BRUNEI DARUSSALAM;BN
BULGARIA;BG
BURKINA FASO;BF
BURUNDI;BI
CAMBODIA;KH
CAMEROON;CM
CANADA;CA
CAPE VERDE;CV
CAYMAN ISLANDS;KY
CENTRAL AFRICAN REPUBLIC;CF
CHAD;TD
CHILE;CL
CHINA;CN
CHRISTMAS ISLAND;CX
COCOS (KEELING) ISLANDS;CC
COLOMBIA;CO
COMOROS;KM
CONGO;CG
CONGO, THE DEMOCRATIC REPUBLIC OF THE;CD
COOK ISLANDS;CK
COSTA RICA;CR
CÔTE D'IVOIRE;CI
CROATIA;HR
CUBA;CU
CURAÇAO;CW
CYPRUS;CY
CZECH REPUBLIC;CZ
DENMARK;DK
DJIBOUTI;DJ
DOMINICA;DM
DOMINICAN REPUBLIC;DO
ECUADOR;EC
EGYPT;EG
EL SALVADOR;SV
EQUATORIAL GUINEA;GQ
ERITREA;ER
ESTONIA;EE
ETHIOPIA;ET
FALKLAND ISLANDS (MALVINAS);FK
FAROE ISLANDS;FO
FIJI;FJ
FINLAND;FI
FRANCE;FR
FRENCH GUIANA;GF
FRENCH POLYNESIA;PF
FRENCH SOUTHERN TERRITORIES;TF
GABON;GA
GAMBIA;GM
GEORGIA;GE
GERMANY;DE
GHANA;GH
GIBRALTAR;GI
GREECE;GR
GREENLAND;GL
GRENADA;GD
GUADELOUPE;GP
GUAM;GU
GUATEMALA;GT
GUERNSEY;GG
GUINEA;GN
GUINEA-BISSAU;GW
GUYANA;GY
HAITI;HT
HEARD ISLAND AND MCDONALD ISLANDS;HM
HOLY SEE (VATICAN CITY STATE);VA
HONDURAS;HN
HONG KONG;HK
HUNGARY;HU
ICELAND;IS
INDIA;IN
INDONESIA;ID
IRAN, ISLAMIC REPUBLIC OF;IR
IRAQ;IQ
IRELAND;IE
ISLE OF MAN;IM
ISRAEL;IL
ITALY;IT
JAMAICA;JM
JAPAN;JP
JERSEY;JE
JORDAN;JO
KAZAKHSTAN;KZ
KENYA;KE
KIRIBATI;KI
KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF;KP
KOREA, REPUBLIC OF;KR
KUWAIT;KW
KYRGYZSTAN;KG
LAO PEOPLE'S DEMOCRATIC REPUBLIC;LA
LATVIA;LV
LEBANON;LB
LESOTHO;LS
LIBERIA;LR
LIBYA;LY
LIECHTENSTEIN;LI
LITHUANIA;LT
LUXEMBOURG;LU
MACAO;MO
MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF;MK
MADAGASCAR;MG
MALAWI;MW
MALAYSIA;MY
MALDIVES;MV
MALI;ML
MALTA;MT
MARSHALL ISLANDS;MH
MARTINIQUE;MQ
MAURITANIA;MR
MAURITIUS;MU
MAYOTTE;YT
MEXICO;MX
MICRONESIA, FEDERATED STATES OF;FM
MOLDOVA, REPUBLIC OF;MD
MONACO;MC
MONGOLIA;MN
MONTENEGRO;ME
MONTSERRAT;MS
MOROCCO;MA
MOZAMBIQUE;MZ
MYANMAR;MM
NAMIBIA;NA
NAURU;NR
NEPAL;NP
NETHERLANDS;NL
NEW CALEDONIA;NC
NEW ZEALAND;NZ
NICARAGUA;NI
NIGER;NE
NIGERIA;NG
NIUE;NU
NORFOLK ISLAND;NF
NORTHERN MARIANA ISLANDS;MP
NORWAY;NO
OMAN;OM
PAKISTAN;PK
PALAU;PW
PALESTINE, STATE OF;PS
PANAMA;PA
PAPUA NEW GUINEA;PG
PARAGUAY;PY
PERU;PE
PHILIPPINES;PH
PITCAIRN;PN
POLAND;PL
PORTUGAL;PT
PUERTO RICO;PR
QATAR;QA
RÉUNION;RE
ROMANIA;RO
RUSSIAN FEDERATION;RU
RWANDA;RW
SAINT BARTHÉLEMY;BL
SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA;SH
SAINT KITTS AND NEVIS;KN
SAINT LUCIA;LC
SAINT MARTIN (FRENCH PART);MF
SAINT PIERRE AND MIQUELON;PM
SAINT VINCENT AND THE GRENADINES;VC
SAMOA;WS
SAN MARINO;SM
SAO TOME AND PRINCIPE;ST
SAUDI ARABIA;SA
SENEGAL;SN
SERBIA;RS
SEYCHELLES;SC
SIERRA LEONE;SL
SINGAPORE;SG
SINT MAARTEN (DUTCH PART);SX
SLOVAKIA;SK
SLOVENIA;SI
SOLOMON ISLANDS;SB
SOMALIA;SO
SOUTH AFRICA;ZA
SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS;GS
SOUTH SUDAN;SS
SPAIN;ES
SRI LANKA;LK
SUDAN;SD
SURINAME;SR
SVALBARD AND JAN MAYEN;SJ
SWAZILAND;SZ
SWEDEN;SE
SWITZERLAND;CH
SYRIAN ARAB REPUBLIC;SY
TAIWAN, PROVINCE OF CHINA;TW
TAJIKISTAN;TJ
TANZANIA, UNITED REPUBLIC OF;TZ
THAILAND;TH
TIMOR-LESTE;TL
TOGO;TG
TOKELAU;TK
TONGA;TO
TRINIDAD AND TOBAGO;TT
TUNISIA;TN
TURKEY;TR
TURKMENISTAN;TM
TURKS AND CAICOS ISLANDS;TC
TUVALU;TV
UGANDA;UG
UKRAINE;UA
UNITED ARAB EMIRATES;AE
UNITED KINGDOM;GB
UNITED STATES;US
UNITED STATES MINOR OUTLYING ISLANDS;UM
URUGUAY;UY
UZBEKISTAN;UZ
VANUATU;VU
VENEZUELA, BOLIVARIAN REPUBLIC OF;VE
VIET NAM;VN
VIRGIN ISLANDS, BRITISH;VG
VIRGIN ISLANDS, U.S.;VI
WALLIS AND FUTUNA;WF
WESTERN SAHARA;EH
YEMEN;YE
ZAMBIA;ZM
ZIMBABWE;ZW
File diff suppressed because it is too large Load Diff
@@ -1,176 +0,0 @@
#
# ISO 15924 - Codes for the representation of names of scripts
# Codes pour la représentation des noms d’écritures
# Format:
# Code;N°;English Name;Nom français;PVA;Date
#
Afak;439;Afaka;afaka;;2010-12-21
Aghb;239;Caucasian Albanian;aghbanien;;2012-10-16
Ahom;338;Ahom, Tai Ahom;âhom;;2012-11-01
Arab;160;Arabic;arabe;Arabic;2004-05-01
Armi;124;Imperial Aramaic;araméen impérial;Imperial_Aramaic;2009-06-01
Armn;230;Armenian;arménien;Armenian;2004-05-01
Avst;134;Avestan;avestique;Avestan;2009-06-01
Bali;360;Balinese;balinais;Balinese;2006-10-10
Bamu;435;Bamum;bamoum;Bamum;2009-06-01
Bass;259;Bassa Vah;bassa;;2010-03-26
Batk;365;Batak;batik;Batak;2010-07-23
Beng;325;Bengali;bengalî;Bengali;2004-05-01
Blis;550;Blissymbols;symboles Bliss;;2004-05-01
Bopo;285;Bopomofo;bopomofo;Bopomofo;2004-05-01
Brah;300;Brahmi;brahma;Brahmi;2010-07-23
Brai;570;Braille;braille;Braille;2004-05-01
Bugi;367;Buginese;bouguis;Buginese;2006-06-21
Buhd;372;Buhid;bouhide;Buhid;2004-05-01
Cakm;349;Chakma;chakma;Chakma;2012-02-06
Cans;440;Unified Canadian Aboriginal Syllabics;syllabaire autochtone canadien unifié;Canadian_Aboriginal;2004-05-29
Cari;201;Carian;carien;Carian;2007-07-02
Cham;358;Cham;cham (čam, tcham);Cham;2009-11-11
Cher;445;Cherokee;tchérokî;Cherokee;2004-05-01
Cirt;291;Cirth;cirth;;2004-05-01
Copt;204;Coptic;copte;Coptic;2006-06-21
Cprt;403;Cypriot;syllabaire chypriote;Cypriot;2004-05-01
Cyrl;220;Cyrillic;cyrillique;Cyrillic;2004-05-01
Cyrs;221;Cyrillic (Old Church Slavonic variant);cyrillique (variante slavonne);;2004-05-01
Deva;315;Devanagari (Nagari);dévanâgarî;Devanagari;2004-05-01
Dsrt;250;Deseret (Mormon);déseret (mormon);Deseret;2004-05-01
Dupl;755;Duployan shorthand, Duployan stenography;sténographie Duployé;;2010-07-18
Egyd;070;Egyptian demotic;démotique égyptien;;2004-05-01
Egyh;060;Egyptian hieratic;hiératique égyptien;;2004-05-01
Egyp;050;Egyptian hieroglyphs;hiéroglyphes égyptiens;Egyptian_Hieroglyphs;2009-06-01
Elba;226;Elbasan;elbasan;;2010-07-18
Ethi;430;Ethiopic (Geʻez);éthiopien (geʻez, guèze);Ethiopic;2004-10-25
Geor;240;Georgian (Mkhedruli);géorgien (mkhédrouli);Georgian;2004-05-29
Geok;241;Khutsuri (Asomtavruli and Nuskhuri);khoutsouri (assomtavrouli et nouskhouri);Georgian;2012-10-16
Glag;225;Glagolitic;glagolitique;Glagolitic;2006-06-21
Goth;206;Gothic;gotique;Gothic;2004-05-01
Gran;343;Grantha;grantha;;2009-11-11
Grek;200;Greek;grec;Greek;2004-05-01
Gujr;320;Gujarati;goudjarâtî (gujrâtî);Gujarati;2004-05-01
Guru;310;Gurmukhi;gourmoukhî;Gurmukhi;2004-05-01
Hang;286;Hangul (Hangŭl, Hangeul);hangûl (hangŭl, hangeul);Hangul;2004-05-29
Hani;500;Han (Hanzi, Kanji, Hanja);idéogrammes han (sinogrammes);Han;2009-02-23
Hano;371;Hanunoo (Hanunóo);hanounóo;Hanunoo;2004-05-29
Hans;501;Han (Simplified variant);idéogrammes han (variante simplifiée);;2004-05-29
Hant;502;Han (Traditional variant);idéogrammes han (variante traditionnelle);;2004-05-29
Hatr;127;Hatran;hatrénien;;2012-11-01
Hebr;125;Hebrew;hébreu;Hebrew;2004-05-01
Hira;410;Hiragana;hiragana;Hiragana;2004-05-01
Hluw;080;Anatolian Hieroglyphs (Luwian Hieroglyphs, Hittite Hieroglyphs);hiéroglyphes anatoliens (hiéroglyphes louvites, hiéroglyphes hittites);;2011-12-09
Hmng;450;Pahawh Hmong;pahawh hmong;;2004-05-01
Hrkt;412;Japanese syllabaries (alias for Hiragana + Katakana);syllabaires japonais (alias pour hiragana + katakana);Katakana_Or_Hiragana;2011-06-21
Hung;176;Old Hungarian (Hungarian Runic);runes hongroises (ancien hongrois);;2012-10-16
Inds;610;Indus (Harappan);indus;;2004-05-01
Ital;210;Old Italic (Etruscan, Oscan, etc.);ancien italique (étrusque, osque, etc.);Old_Italic;2004-05-29
Java;361;Javanese;javanais;Javanese;2009-06-01
Jpan;413;Japanese (alias for Han + Hiragana + Katakana);japonais (alias pour han + hiragana + katakana);;2006-06-21
Jurc;510;Jurchen;jurchen;;2010-12-21
Kali;357;Kayah Li;kayah li;Kayah_Li;2007-07-02
Kana;411;Katakana;katakana;Katakana;2004-05-01
Khar;305;Kharoshthi;kharochthî;Kharoshthi;2006-06-21
Khmr;355;Khmer;khmer;Khmer;2004-05-29
Khoj;322;Khojki;khojkî;;2011-06-21
Knda;345;Kannada;kannara (canara);Kannada;2004-05-29
Kore;287;Korean (alias for Hangul + Han);coréen (alias pour hangûl + han);;2007-06-13
Kpel;436;Kpelle;kpèllé;;2010-03-26
Kthi;317;Kaithi;kaithî;Kaithi;2009-06-01
Lana;351;Tai Tham (Lanna);taï tham (lanna);Tai_Tham;2009-06-01
Laoo;356;Lao;laotien;Lao;2004-05-01
Latf;217;Latin (Fraktur variant);latin (variante brisée);;2004-05-01
Latg;216;Latin (Gaelic variant);latin (variante gaélique);;2004-05-01
Latn;215;Latin;latin;Latin;2004-05-01
Lepc;335;Lepcha (Róng);lepcha (róng);Lepcha;2007-07-02
Limb;336;Limbu;limbou;Limbu;2004-05-29
Lina;400;Linear A;linéaire A;;2004-05-01
Linb;401;Linear B;linéaire B;Linear_B;2004-05-29
Lisu;399;Lisu (Fraser);lisu (Fraser);Lisu;2009-06-01
Loma;437;Loma;loma;;2010-03-26
Lyci;202;Lycian;lycien;Lycian;2007-07-02
Lydi;116;Lydian;lydien;Lydian;2007-07-02
Mahj;314;Mahajani;mahâjanî;;2012-10-16
Mand;140;Mandaic, Mandaean;mandéen;Mandaic;2010-07-23
Mani;139;Manichaean;manichéen;;2007-07-15
Maya;090;Mayan hieroglyphs;hiéroglyphes mayas;;2004-05-01
Mend;438;Mende Kikakui;mendé kikakui;;2013-10-12
Merc;101;Meroitic Cursive;cursif méroïtique;Meroitic_Cursive;2012-02-06
Mero;100;Meroitic Hieroglyphs;hiéroglyphes méroïtiques;Meroitic_Hieroglyphs;2012-02-06
Mlym;347;Malayalam;malayâlam;Malayalam;2004-05-01
Modi;323;Modi, Moḍī;modî;;2013-10-12
Moon;218;Moon (Moon code, Moon script, Moon type);écriture Moon;;2006-12-11
Mong;145;Mongolian;mongol;Mongolian;2004-05-01
Mroo;199;Mro, Mru;mro;;2010-12-21
Mtei;337;Meitei Mayek (Meithei, Meetei);meitei mayek;Meetei_Mayek;2009-06-01
Mult;323; Multani;multanî;;2012-11-01
Mymr;350;Myanmar (Burmese);birman;Myanmar;2004-05-01
Narb;106;Old North Arabian (Ancient North Arabian);nord-arabique;;2010-03-26
Nbat;159;Nabataean;nabatéen;;2010-03-26
Nkgb;420;Nakhi Geba ('Na-'Khi ²Ggŏ-¹baw, Naxi Geba);nakhi géba;;2009-02-23
Nkoo;165;NKo;nko;Nko;2006-10-10
Nshu;499;Nüshu;nüshu;;2010-12-21
Ogam;212;Ogham;ogam;Ogham;2004-05-01
Olck;261;Ol Chiki (Ol Cemet, Ol, Santali);ol tchiki;Ol_Chiki;2007-07-02
Orkh;175;Old Turkic, Orkhon Runic;orkhon;Old_Turkic;2009-06-01
Orya;327;Oriya;oriyâ;Oriya;2004-05-01
Osma;260;Osmanya;osmanais;Osmanya;2004-05-01
Palm;126;Palmyrene;palmyrénien;;2010-03-26
Pauc;263;Pau Cin Hau;paou chin haou;;2013-10-12
Perm;227;Old Permic;ancien permien;;2004-05-01
Phag;331;Phags-pa;phags pa;Phags_Pa;2006-10-10
Phli;131;Inscriptional Pahlavi;pehlevi des inscriptions;Inscriptional_Pahlavi;2009-06-01
Phlp;132;Psalter Pahlavi;pehlevi des psautiers;;2007-11-26
Phlv;133;Book Pahlavi;pehlevi des livres;;2007-07-15
Phnx;115;Phoenician;phénicien;Phoenician;2006-10-10
Plrd;282;Miao (Pollard);miao (Pollard);Miao;2012-02-06
Prti;130;Inscriptional Parthian;parthe des inscriptions;Inscriptional_Parthian;2009-06-01
Qaaa;900;Reserved for private use (start);réservé à lusage privé (début);;2004-05-29
Qabx;949;Reserved for private use (end);réservé à lusage privé (fin);;2004-05-29
Rjng;363;Rejang (Redjang, Kaganga);redjang (kaganga);Rejang;2009-02-23
Roro;620;Rongorongo;rongorongo;;2004-05-01
Runr;211;Runic;runique;Runic;2004-05-01
Samr;123;Samaritan;samaritain;Samaritan;2009-06-01
Sara;292;Sarati;sarati;;2004-05-29
Sarb;105;Old South Arabian;sud-arabique, himyarite;Old_South_Arabian;2009-06-01
Saur;344;Saurashtra;saurachtra;Saurashtra;2007-07-02
Sgnw;095;SignWriting;SignÉcriture, SignWriting;;2006-10-10
Shaw;281;Shavian (Shaw);shavien (Shaw);Shavian;2004-05-01
Shrd;319;Sharada, Śāradā;charada, shard;Sharada;2012-02-06
Sidd;302;Siddham, Siddhaṃ, Siddhamātṛkā;siddham;;2013-10-12
Sind;318;Khudawadi, Sindhi;khoudawadî, sindhî;;2010-12-21
Sinh;348;Sinhala;singhalais;Sinhala;2004-05-01
Sora;398;Sora Sompeng;sora sompeng;Sora_Sompeng;2012-02-06
Sund;362;Sundanese;sundanais;Sundanese;2007-07-02
Sylo;316;Syloti Nagri;sylotî nâgrî;Syloti_Nagri;2006-06-21
Syrc;135;Syriac;syriaque;Syriac;2004-05-01
Syre;138;Syriac (Estrangelo variant);syriaque (variante estranghélo);;2004-05-01
Syrj;137;Syriac (Western variant);syriaque (variante occidentale);;2004-05-01
Syrn;136;Syriac (Eastern variant);syriaque (variante orientale);;2004-05-01
Tagb;373;Tagbanwa;tagbanoua;Tagbanwa;2004-05-01
Takr;321;Takri, Ṭākrī, Ṭāṅkrī;tâkrî;Takri;2012-02-06
Tale;353;Tai Le;taï-le;Tai_Le;2004-10-25
Talu;354;New Tai Lue;nouveau taï-lue;New_Tai_Lue;2006-06-21
Taml;346;Tamil;tamoul;Tamil;2004-05-01
Tang;520;Tangut;tangoute;;2010-12-21
Tavt;359;Tai Viet;taï viêt;Tai_Viet;2009-06-01
Telu;340;Telugu;télougou;Telugu;2004-05-01
Teng;290;Tengwar;tengwar;;2004-05-01
Tfng;120;Tifinagh (Berber);tifinagh (berbère);Tifinagh;2006-06-21
Tglg;370;Tagalog (Baybayin, Alibata);tagal (baybayin, alibata);Tagalog;2009-02-23
Thaa;170;Thaana;thâna;Thaana;2004-05-01
Thai;352;Thai;thaï;Thai;2004-05-01
Tibt;330;Tibetan;tibétain;Tibetan;2004-05-01
Tirh;326;Tirhuta;tirhouta;;2011-12-09
Ugar;040;Ugaritic;ougaritique;Ugaritic;2004-05-01
Vaii;470;Vai;vaï;Vai;2007-07-02
Visp;280;Visible Speech;parole visible;;2004-05-01
Wara;262;Warang Citi (Varang Kshiti);warang citi;;2009-11-11
Wole;480;Woleai;woléaï;;2010-12-21
Xpeo;030;Old Persian;cunéiforme persépolitain;Old_Persian;2006-06-21
Xsux;020;Cuneiform, Sumero-Akkadian;cunéiforme suméro-akkadien;Cuneiform;2006-10-10
Yiii;460;Yi;yi;Yi;2004-05-01
Zinh;994;Code for inherited script;codet pour écriture héritée;Inherited;2009-02-23
Zmth;995;Mathematical notation;notation mathématique;;2007-11-26
Zsym;996;Symbols;symboles;;2007-11-26
Zxxx;997;Code for unwritten documents;codet pour les documents non écrits;;2011-06-21
Zyyy;998;Code for undetermined script;codet pour écriture indéterminée;Common;2004-05-29
Zzzz;999;Code for uncoded script;codet pour écriture non codée;Unknown;2006-10-10
@@ -1,474 +0,0 @@
IdSubLanguage ISO639 LanguageName UploadEnabled WebEnabled
aar aa Afar, afar 0 0
abk ab Abkhazian 0 0
ace Achinese 0 0
ach Acoli 0 0
ada Adangme 0 0
ady adyghé 0 0
afa Afro-Asiatic (Other) 0 0
afh Afrihili 0 0
afr af Afrikaans 1 0
ain Ainu 0 0
aka ak Akan 0 0
akk Akkadian 0 0
alb sq Albanian 1 1
ale Aleut 0 0
alg Algonquian languages 0 0
alt Southern Altai 0 0
amh am Amharic 0 0
ang English, Old (ca.450-1100) 0 0
apa Apache languages 0 0
ara ar Arabic 1 1
arc Aramaic 0 0
arg an Aragonese 0 0
arm hy Armenian 1 0
arn Araucanian 0 0
arp Arapaho 0 0
art Artificial (Other) 0 0
arw Arawak 0 0
asm as Assamese 0 0
ast Asturian, Bable 0 0
ath Athapascan languages 0 0
aus Australian languages 0 0
ava av Avaric 0 0
ave ae Avestan 0 0
awa Awadhi 0 0
aym ay Aymara 0 0
aze az Azerbaijani 0 0
bad Banda 0 0
bai Bamileke languages 0 0
bak ba Bashkir 0 0
bal Baluchi 0 0
bam bm Bambara 0 0
ban Balinese 0 0
baq eu Basque 1 1
bas Basa 0 0
bat Baltic (Other) 0 0
bej Beja 0 0
bel be Belarusian 0 0
bem Bemba 0 0
ben bn Bengali 1 0
ber Berber (Other) 0 0
bho Bhojpuri 0 0
bih bh Bihari 0 0
bik Bikol 0 0
bin Bini 0 0
bis bi Bislama 0 0
bla Siksika 0 0
bnt Bantu (Other) 0 0
bos bs Bosnian 1 0
bra Braj 0 0
bre br Breton 1 0
btk Batak (Indonesia) 0 0
bua Buriat 0 0
bug Buginese 0 0
bul bg Bulgarian 1 1
bur my Burmese 1 0
byn Blin 0 0
cad Caddo 0 0
cai Central American Indian (Other) 0 0
car Carib 0 0
cat ca Catalan 1 1
cau Caucasian (Other) 0 0
ceb Cebuano 0 0
cel Celtic (Other) 0 0
cha ch Chamorro 0 0
chb Chibcha 0 0
che ce Chechen 0 0
chg Chagatai 0 0
chi zh Chinese 1 1
chk Chuukese 0 0
chm Mari 0 0
chn Chinook jargon 0 0
cho Choctaw 0 0
chp Chipewyan 0 0
chr Cherokee 0 0
chu cu Church Slavic 0 0
chv cv Chuvash 0 0
chy Cheyenne 0 0
cmc Chamic languages 0 0
cop Coptic 0 0
cor kw Cornish 0 0
cos co Corsican 0 0
cpe Creoles and pidgins, English based (Other) 0 0
cpf Creoles and pidgins, French-based (Other) 0 0
cpp Creoles and pidgins, Portuguese-based (Other) 0 0
cre cr Cree 0 0
crh Crimean Tatar 0 0
crp Creoles and pidgins (Other) 0 0
csb Kashubian 0 0
cus Cushitic (Other)' couchitiques, autres langues 0 0
cze cs Czech 1 1
dak Dakota 0 0
dan da Danish 1 1
dar Dargwa 0 0
day Dayak 0 0
del Delaware 0 0
den Slave (Athapascan) 0 0
dgr Dogrib 0 0
din Dinka 0 0
div dv Divehi 0 0
doi Dogri 0 0
dra Dravidian (Other) 0 0
dua Duala 0 0
dum Dutch, Middle (ca.1050-1350) 0 0
dut nl Dutch 1 1
dyu Dyula 0 0
dzo dz Dzongkha 0 0
efi Efik 0 0
egy Egyptian (Ancient) 0 0
eka Ekajuk 0 0
elx Elamite 0 0
eng en English 1 1
enm English, Middle (1100-1500) 0 0
epo eo Esperanto 1 0
est et Estonian 1 1
ewe ee Ewe 0 0
ewo Ewondo 0 0
fan Fang 0 0
fao fo Faroese 0 0
fat Fanti 0 0
fij fj Fijian 0 0
fil Filipino 0 0
fin fi Finnish 1 1
fiu Finno-Ugrian (Other) 0 0
fon Fon 0 0
fre fr French 1 1
frm French, Middle (ca.1400-1600) 0 0
fro French, Old (842-ca.1400) 0 0
fry fy Frisian 0 0
ful ff Fulah 0 0
fur Friulian 0 0
gaa Ga 0 0
gay Gayo 0 0
gba Gbaya 0 0
gem Germanic (Other) 0 0
geo ka Georgian 1 1
ger de German 1 1
gez Geez 0 0
gil Gilbertese 0 0
gla gd Gaelic 0 0
gle ga Irish 0 0
glg gl Galician 1 1
glv gv Manx 0 0
gmh German, Middle High (ca.1050-1500) 0 0
goh German, Old High (ca.750-1050) 0 0
gon Gondi 0 0
gor Gorontalo 0 0
got Gothic 0 0
grb Grebo 0 0
grc Greek, Ancient (to 1453) 0 0
ell el Greek 1 1
grn gn Guarani 0 0
guj gu Gujarati 0 0
gwi Gwich´in 0 0
hai Haida 0 0
hat ht Haitian 0 0
hau ha Hausa 0 0
haw Hawaiian 0 0
heb he Hebrew 1 1
her hz Herero 0 0
hil Hiligaynon 0 0
him Himachali 0 0
hin hi Hindi 1 1
hit Hittite 0 0
hmn Hmong 0 0
hmo ho Hiri Motu 0 0
hrv hr Croatian 1 1
hun hu Hungarian 1 1
hup Hupa 0 0
iba Iban 0 0
ibo ig Igbo 0 0
ice is Icelandic 1 1
ido io Ido 0 0
iii ii Sichuan Yi 0 0
ijo Ijo 0 0
iku iu Inuktitut 0 0
ile ie Interlingue 0 0
ilo Iloko 0 0
ina ia Interlingua (International Auxiliary Language Asso 0 0
inc Indic (Other) 0 0
ind id Indonesian 1 1
ine Indo-European (Other) 0 0
inh Ingush 0 0
ipk ik Inupiaq 0 0
ira Iranian (Other) 0 0
iro Iroquoian languages 0 0
ita it Italian 1 1
jav jv Javanese 0 0
jpn ja Japanese 1 1
jpr Judeo-Persian 0 0
jrb Judeo-Arabic 0 0
kaa Kara-Kalpak 0 0
kab Kabyle 0 0
kac Kachin 0 0
kal kl Kalaallisut 0 0
kam Kamba 0 0
kan kn Kannada 0 0
kar Karen 0 0
kas ks Kashmiri 0 0
kau kr Kanuri 0 0
kaw Kawi 0 0
kaz kk Kazakh 1 0
kbd Kabardian 0 0
kha Khasi 0 0
khi Khoisan (Other) 0 0
khm km Khmer 1 1
kho Khotanese 0 0
kik ki Kikuyu 0 0
kin rw Kinyarwanda 0 0
kir ky Kirghiz 0 0
kmb Kimbundu 0 0
kok Konkani 0 0
kom kv Komi 0 0
kon kg Kongo 0 0
kor ko Korean 1 1
kos Kosraean 0 0
kpe Kpelle 0 0
krc Karachay-Balkar 0 0
kro Kru 0 0
kru Kurukh 0 0
kua kj Kuanyama 0 0
kum Kumyk 0 0
kur ku Kurdish 0 0
kut Kutenai 0 0
lad Ladino 0 0
lah Lahnda 0 0
lam Lamba 0 0
lao lo Lao 0 0
lat la Latin 0 0
lav lv Latvian 1 0
lez Lezghian 0 0
lim li Limburgan 0 0
lin ln Lingala 0 0
lit lt Lithuanian 1 0
lol Mongo 0 0
loz Lozi 0 0
ltz lb Luxembourgish 1 0
lua Luba-Lulua 0 0
lub lu Luba-Katanga 0 0
lug lg Ganda 0 0
lui Luiseno 0 0
lun Lunda 0 0
luo Luo (Kenya and Tanzania) 0 0
lus lushai 0 0
mac mk Macedonian 1 1
mad Madurese 0 0
mag Magahi 0 0
mah mh Marshallese 0 0
mai Maithili 0 0
mak Makasar 0 0
mal ml Malayalam 1 0
man Mandingo 0 0
mao mi Maori 0 0
map Austronesian (Other) 0 0
mar mr Marathi 0 0
mas Masai 0 0
may ms Malay 1 1
mdf Moksha 0 0
mdr Mandar 0 0
men Mende 0 0
mga Irish, Middle (900-1200) 0 0
mic Mi'kmaq 0 0
min Minangkabau 0 0
mis Miscellaneous languages 0 0
mkh Mon-Khmer (Other) 0 0
mlg mg Malagasy 0 0
mlt mt Maltese 0 0
mnc Manchu 0 0
mni Manipuri 0 0
mno Manobo languages 0 0
moh Mohawk 0 0
mol mo Moldavian 0 0
mon mn Mongolian 1 0
mos Mossi 0 0
mwl Mirandese 0 0
mul Multiple languages 0 0
mun Munda languages 0 0
mus Creek 0 0
mwr Marwari 0 0
myn Mayan languages 0 0
myv Erzya 0 0
nah Nahuatl 0 0
nai North American Indian 0 0
nap Neapolitan 0 0
nau na Nauru 0 0
nav nv Navajo 0 0
nbl nr Ndebele, South 0 0
nde nd Ndebele, North 0 0
ndo ng Ndonga 0 0
nds Low German 0 0
nep ne Nepali 0 0
new Nepal Bhasa 0 0
nia Nias 0 0
nic Niger-Kordofanian (Other) 0 0
niu Niuean 0 0
nno nn Norwegian Nynorsk 0 0
nob nb Norwegian Bokmal 0 0
nog Nogai 0 0
non Norse, Old 0 0
nor no Norwegian 1 1
nso Northern Sotho 0 0
nub Nubian languages 0 0
nwc Classical Newari 0 0
nya ny Chichewa 0 0
nym Nyamwezi 0 0
nyn Nyankole 0 0
nyo Nyoro 0 0
nzi Nzima 0 0
oci oc Occitan 1 1
oji oj Ojibwa 0 0
ori or Oriya 0 0
orm om Oromo 0 0
osa Osage 0 0
oss os Ossetian 0 0
ota Turkish, Ottoman (1500-1928) 0 0
oto Otomian languages 0 0
paa Papuan (Other) 0 0
pag Pangasinan 0 0
pal Pahlavi 0 0
pam Pampanga 0 0
pan pa Panjabi 0 0
pap Papiamento 0 0
pau Palauan 0 0
peo Persian, Old (ca.600-400 B.C.) 0 0
per fa Persian 1 1
phi Philippine (Other) 0 0
phn Phoenician 0 0
pli pi Pali 0 0
pol pl Polish 1 1
pon Pohnpeian 0 0
por pt Portuguese 1 1
pra Prakrit languages 0 0
pro Provençal, Old (to 1500) 0 0
pus ps Pushto 0 0
que qu Quechua 0 0
raj Rajasthani 0 0
rap Rapanui 0 0
rar Rarotongan 0 0
roa Romance (Other) 0 0
roh rm Raeto-Romance 0 0
rom Romany 0 0
run rn Rundi 0 0
rup Aromanian 0 0
rus ru Russian 1 1
sad Sandawe 0 0
sag sg Sango 0 0
sah Yakut 0 0
sai South American Indian (Other) 0 0
sal Salishan languages 0 0
sam Samaritan Aramaic 0 0
san sa Sanskrit 0 0
sas Sasak 0 0
sat Santali 0 0
scc sr Serbian 1 1
scn Sicilian 0 0
sco Scots 0 0
sel Selkup 0 0
sem Semitic (Other) 0 0
sga Irish, Old (to 900) 0 0
sgn Sign Languages 0 0
shn Shan 0 0
sid Sidamo 0 0
sin si Sinhalese 1 1
sio Siouan languages 0 0
sit Sino-Tibetan (Other) 0 0
sla Slavic (Other) 0 0
slo sk Slovak 1 1
slv sl Slovenian 1 1
sma Southern Sami 0 0
sme se Northern Sami 0 0
smi Sami languages (Other) 0 0
smj Lule Sami 0 0
smn Inari Sami 0 0
smo sm Samoan 0 0
sms Skolt Sami 0 0
sna sn Shona 0 0
snd sd Sindhi 0 0
snk Soninke 0 0
sog Sogdian 0 0
som so Somali 0 0
son Songhai 0 0
sot st Sotho, Southern 0 0
spa es Spanish 1 1
srd sc Sardinian 0 0
srr Serer 0 0
ssa Nilo-Saharan (Other) 0 0
ssw ss Swati 0 0
suk Sukuma 0 0
sun su Sundanese 0 0
sus Susu 0 0
sux Sumerian 0 0
swa sw Swahili 1 0
swe sv Swedish 1 1
syr Syriac 1 0
tah ty Tahitian 0 0
tai Tai (Other) 0 0
tam ta Tamil 1 0
tat tt Tatar 0 0
tel te Telugu 1 0
tem Timne 0 0
ter Tereno 0 0
tet Tetum 0 0
tgk tg Tajik 0 0
tgl tl Tagalog 1 1
tha th Thai 1 1
tib bo Tibetan 0 0
tig Tigre 0 0
tir ti Tigrinya 0 0
tiv Tiv 0 0
tkl Tokelau 0 0
tlh Klingon 0 0
tli Tlingit 0 0
tmh Tamashek 0 0
tog Tonga (Nyasa) 0 0
ton to Tonga (Tonga Islands) 0 0
tpi Tok Pisin 0 0
tsi Tsimshian 0 0
tsn tn Tswana 0 0
tso ts Tsonga 0 0
tuk tk Turkmen 0 0
tum Tumbuka 0 0
tup Tupi languages 0 0
tur tr Turkish 1 1
tut Altaic (Other) 0 0
tvl Tuvalu 0 0
twi tw Twi 0 0
tyv Tuvinian 0 0
udm Udmurt 0 0
uga Ugaritic 0 0
uig ug Uighur 0 0
ukr uk Ukrainian 1 1
umb Umbundu 0 0
und Undetermined 0 0
urd ur Urdu 1 0
uzb uz Uzbek 0 0
vai Vai 0 0
ven ve Venda 0 0
vie vi Vietnamese 1 1
vol vo Volapük 0 0
vot Votic 0 0
wak Wakashan languages 0 0
wal Walamo 0 0
war Waray 0 0
was Washo 0 0
wel cy Welsh 0 0
wen Sorbian languages 0 0
wln wa Walloon 0 0
wol wo Wolof 0 0
xal Kalmyk 0 0
xho xh Xhosa 0 0
yao Yao 0 0
yap Yapese 0 0
yid yi Yiddish 0 0
yor yo Yoruba 0 0
ypk Yupik languages 0 0
zap Zapotec 0 0
zen Zenaga 0 0
zha za Zhuang 0 0
znd Zande 0 0
zul zu Zulu 0 0
zun Zuni 0 0
rum ro Romanian 1 1
pob pb Brazilian 1 1
mne Montenegrin 1 0
@@ -1,85 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
class Error(Exception):
"""Base class for all exceptions in babelfish"""
pass
class LanguageError(Error, AttributeError):
"""Base class for all language exceptions in babelfish"""
pass
class LanguageConvertError(LanguageError):
"""Exception raised by converters when :meth:`~babelfish.converters.LanguageConverter.convert` fails
:param string alpha3: alpha3 code that failed conversion
:param country: country code that failed conversion, if any
:type country: string or None
:param script: script code that failed conversion, if any
:type script: string or None
"""
def __init__(self, alpha3, country=None, script=None):
self.alpha3 = alpha3
self.country = country
self.script = script
def __str__(self):
s = self.alpha3
if self.country is not None:
s += '-' + self.country
if self.script is not None:
s += '-' + self.script
return s
class LanguageReverseError(LanguageError):
"""Exception raised by converters when :meth:`~babelfish.converters.LanguageReverseConverter.reverse` fails
:param string code: code that failed reverse conversion
"""
def __init__(self, code):
self.code = code
def __str__(self):
return repr(self.code)
class CountryError(Error, AttributeError):
"""Base class for all country exceptions in babelfish"""
pass
class CountryConvertError(CountryError):
"""Exception raised by converters when :meth:`~babelfish.converters.CountryConverter.convert` fails
:param string alpha2: alpha2 code that failed conversion
"""
def __init__(self, alpha2):
self.alpha2 = alpha2
def __str__(self):
return self.alpha2
class CountryReverseError(CountryError):
"""Exception raised by converters when :meth:`~babelfish.converters.CountryReverseConverter.reverse` fails
:param string code: code that failed reverse conversion
"""
def __init__(self, code):
self.code = code
def __str__(self):
return repr(self.code)
@@ -1,185 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from collections import namedtuple
from functools import partial
from pkg_resources import resource_stream # @UnresolvedImport
from .converters import ConverterManager
from .country import Country
from .exceptions import LanguageConvertError
from .script import Script
from . import basestr
LANGUAGES = set()
LANGUAGE_MATRIX = []
#: The namedtuple used in the :data:`LANGUAGE_MATRIX`
IsoLanguage = namedtuple('IsoLanguage', ['alpha3', 'alpha3b', 'alpha3t', 'alpha2', 'scope', 'type', 'name', 'comment'])
f = resource_stream('babelfish', 'data/iso-639-3.tab')
f.readline()
for l in f:
iso_language = IsoLanguage(*l.decode('utf-8').split('\t'))
LANGUAGES.add(iso_language.alpha3)
LANGUAGE_MATRIX.append(iso_language)
f.close()
class LanguageConverterManager(ConverterManager):
""":class:`~babelfish.converters.ConverterManager` for language converters"""
entry_point = 'babelfish.language_converters'
internal_converters = ['alpha2 = babelfish.converters.alpha2:Alpha2Converter',
'alpha3b = babelfish.converters.alpha3b:Alpha3BConverter',
'alpha3t = babelfish.converters.alpha3t:Alpha3TConverter',
'name = babelfish.converters.name:NameConverter',
'scope = babelfish.converters.scope:ScopeConverter',
'type = babelfish.converters.type:LanguageTypeConverter',
'opensubtitles = babelfish.converters.opensubtitles:OpenSubtitlesConverter']
language_converters = LanguageConverterManager()
class LanguageMeta(type):
"""The :class:`Language` metaclass
Dynamically redirect :meth:`Language.frommycode` to :meth:`Language.fromcode` with the ``mycode`` `converter`
"""
def __getattr__(cls, name):
if name.startswith('from'):
return partial(cls.fromcode, converter=name[4:])
return type.__getattribute__(cls, name)
class Language(LanguageMeta(str('LanguageBase'), (object,), {})):
"""A human language
A human language is composed of a language part following the ISO-639
standard and can be country-specific when a :class:`~babelfish.country.Country`
is specified.
The :class:`Language` is extensible with custom converters (see :ref:`custom_converters`)
:param string language: the language as a 3-letter ISO-639-3 code
:param country: the country (if any) as a 2-letter ISO-3166 code or :class:`~babelfish.country.Country` instance
:type country: string or :class:`~babelfish.country.Country` or None
:param script: the script (if any) as a 4-letter ISO-15924 code or :class:`~babelfish.script.Script` instance
:type script: string or :class:`~babelfish.script.Script` or None
:param unknown: the unknown language as a three-letters ISO-639-3 code to use as fallback
:type unknown: string or None
:raise: ValueError if the language could not be recognized and `unknown` is ``None``
"""
def __init__(self, language, country=None, script=None, unknown=None):
if unknown is not None and language not in LANGUAGES:
language = unknown
if language not in LANGUAGES:
raise ValueError('%r is not a valid language' % language)
self.alpha3 = language
self.country = None
if isinstance(country, Country):
self.country = country
elif country is None:
self.country = None
else:
self.country = Country(country)
self.script = None
if isinstance(script, Script):
self.script = script
elif script is None:
self.script = None
else:
self.script = Script(script)
@classmethod
def fromcode(cls, code, converter):
"""Create a :class:`Language` by its `code` using `converter` to
:meth:`~babelfish.converters.LanguageReverseConverter.reverse` it
:param string code: the code to reverse
:param string converter: name of the :class:`~babelfish.converters.LanguageReverseConverter` to use
:return: the corresponding :class:`Language` instance
:rtype: :class:`Language`
"""
return cls(*language_converters[converter].reverse(code))
@classmethod
def fromietf(cls, ietf):
"""Create a :class:`Language` by from an IETF language code
:param string ietf: the ietf code
:return: the corresponding :class:`Language` instance
:rtype: :class:`Language`
"""
subtags = ietf.split('-')
language_subtag = subtags.pop(0).lower()
if len(language_subtag) == 2:
language = cls.fromalpha2(language_subtag)
else:
language = cls(language_subtag)
while subtags:
subtag = subtags.pop(0)
if len(subtag) == 2:
language.country = Country(subtag.upper())
else:
language.script = Script(subtag.capitalize())
if language.script is not None:
if subtags:
raise ValueError('Wrong IETF format. Unmatched subtags: %r' % subtags)
break
return language
def __getstate__(self):
return self.alpha3, self.country, self.script
def __setstate__(self, state):
self.alpha3, self.country, self.script = state
def __getattr__(self, name):
alpha3 = self.alpha3
country = self.country.alpha2 if self.country is not None else None
script = self.script.code if self.script is not None else None
try:
return language_converters[name].convert(alpha3, country, script)
except KeyError:
raise AttributeError(name)
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
if isinstance(other, basestr):
return str(self) == other
if not isinstance(other, Language):
return False
return (self.alpha3 == other.alpha3 and
self.country == other.country and
self.script == other.script)
def __ne__(self, other):
return not self == other
def __bool__(self):
return self.alpha3 != 'und'
__nonzero__ = __bool__
def __repr__(self):
return '<Language [%s]>' % self
def __str__(self):
try:
s = self.alpha2
except LanguageConvertError:
s = self.alpha3
if self.country is not None:
s += '-' + str(self.country)
if self.script is not None:
s += '-' + str(self.script)
return s
@@ -1,76 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from collections import namedtuple
from pkg_resources import resource_stream # @UnresolvedImport
from . import basestr
#: Script code to script name mapping
SCRIPTS = {}
#: List of countries in the ISO-15924 as namedtuple of code, number, name, french_name, pva and date
SCRIPT_MATRIX = []
#: The namedtuple used in the :data:`SCRIPT_MATRIX`
IsoScript = namedtuple('IsoScript', ['code', 'number', 'name', 'french_name', 'pva', 'date'])
f = resource_stream('babelfish', 'data/iso15924-utf8-20131012.txt')
f.readline()
for l in f:
l = l.decode('utf-8').strip()
if not l or l.startswith('#'):
continue
script = IsoScript._make(l.split(';'))
SCRIPT_MATRIX.append(script)
SCRIPTS[script.code] = script.name
f.close()
class Script(object):
"""A human writing system
A script is represented by a 4-letter code from the ISO-15924 standard
:param string script: 4-letter ISO-15924 script code
"""
def __init__(self, script):
if script not in SCRIPTS:
raise ValueError('%r is not a valid script' % script)
#: ISO-15924 4-letter script code
self.code = script
@property
def name(self):
"""English name of the script"""
return SCRIPTS[self.code]
def __getstate__(self):
return self.code
def __setstate__(self, state):
self.code = state
def __hash__(self):
return hash(self.code)
def __eq__(self, other):
if isinstance(other, basestr):
return self.code == other
if not isinstance(other, Script):
return False
return self.code == other.code
def __ne__(self, other):
return not self == other
def __repr__(self):
return '<Script [%s]>' % self
def __str__(self):
return self.code
@@ -1,368 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
import re
import sys
import pickle
from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
from pkg_resources import resource_stream # @UnresolvedImport
from babelfish import (LANGUAGES, Language, Country, Script, language_converters, country_converters,
LanguageReverseConverter, LanguageConvertError, LanguageReverseError, CountryReverseError)
if sys.version_info[:2] <= (2, 6):
_MAX_LENGTH = 80
def safe_repr(obj, short=False):
try:
result = repr(obj)
except Exception:
result = object.__repr__(obj)
if not short or len(result) < _MAX_LENGTH:
return result
return result[:_MAX_LENGTH] + ' [truncated]...'
class _AssertRaisesContext(object):
"""A context manager used to implement TestCase.assertRaises* methods."""
def __init__(self, expected, test_case, expected_regexp=None):
self.expected = expected
self.failureException = test_case.failureException
self.expected_regexp = expected_regexp
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
if exc_type is None:
try:
exc_name = self.expected.__name__
except AttributeError:
exc_name = str(self.expected)
raise self.failureException(
"{0} not raised".format(exc_name))
if not issubclass(exc_type, self.expected):
# let unexpected exceptions pass through
return False
self.exception = exc_value # store for later retrieval
if self.expected_regexp is None:
return True
expected_regexp = self.expected_regexp
if isinstance(expected_regexp, basestring):
expected_regexp = re.compile(expected_regexp)
if not expected_regexp.search(str(exc_value)):
raise self.failureException('"%s" does not match "%s"' %
(expected_regexp.pattern, str(exc_value)))
return True
class _Py26FixTestCase(object):
def assertIsNone(self, obj, msg=None):
"""Same as self.assertTrue(obj is None), with a nicer default message."""
if obj is not None:
standardMsg = '%s is not None' % (safe_repr(obj),)
self.fail(self._formatMessage(msg, standardMsg))
def assertIsNotNone(self, obj, msg=None):
"""Included for symmetry with assertIsNone."""
if obj is None:
standardMsg = 'unexpectedly None'
self.fail(self._formatMessage(msg, standardMsg))
def assertIn(self, member, container, msg=None):
"""Just like self.assertTrue(a in b), but with a nicer default message."""
if member not in container:
standardMsg = '%s not found in %s' % (safe_repr(member),
safe_repr(container))
self.fail(self._formatMessage(msg, standardMsg))
def assertNotIn(self, member, container, msg=None):
"""Just like self.assertTrue(a not in b), but with a nicer default message."""
if member in container:
standardMsg = '%s unexpectedly found in %s' % (safe_repr(member),
safe_repr(container))
self.fail(self._formatMessage(msg, standardMsg))
def assertIs(self, expr1, expr2, msg=None):
"""Just like self.assertTrue(a is b), but with a nicer default message."""
if expr1 is not expr2:
standardMsg = '%s is not %s' % (safe_repr(expr1),
safe_repr(expr2))
self.fail(self._formatMessage(msg, standardMsg))
def assertIsNot(self, expr1, expr2, msg=None):
"""Just like self.assertTrue(a is not b), but with a nicer default message."""
if expr1 is expr2:
standardMsg = 'unexpectedly identical: %s' % (safe_repr(expr1),)
self.fail(self._formatMessage(msg, standardMsg))
else:
class _Py26FixTestCase(object):
pass
class TestScript(TestCase, _Py26FixTestCase):
def test_wrong_script(self):
self.assertRaises(ValueError, lambda: Script('Azer'))
def test_eq(self):
self.assertEqual(Script('Latn'), Script('Latn'))
def test_ne(self):
self.assertNotEqual(Script('Cyrl'), Script('Latn'))
def test_hash(self):
self.assertEqual(hash(Script('Hira')), hash('Hira'))
def test_pickle(self):
self.assertEqual(pickle.loads(pickle.dumps(Script('Latn'))), Script('Latn'))
class TestCountry(TestCase, _Py26FixTestCase):
def test_wrong_country(self):
self.assertRaises(ValueError, lambda: Country('ZZ'))
def test_eq(self):
self.assertEqual(Country('US'), Country('US'))
def test_ne(self):
self.assertNotEqual(Country('GB'), Country('US'))
self.assertIsNotNone(Country('US'))
def test_hash(self):
self.assertEqual(hash(Country('US')), hash('US'))
def test_pickle(self):
for country in [Country('GB'), Country('US')]:
self.assertEqual(pickle.loads(pickle.dumps(country)), country)
def test_converter_name(self):
self.assertEqual(Country('US').name, 'UNITED STATES')
self.assertEqual(Country.fromname('UNITED STATES'), Country('US'))
self.assertEqual(Country.fromcode('UNITED STATES', 'name'), Country('US'))
self.assertRaises(CountryReverseError, lambda: Country.fromname('ZZZZZ'))
self.assertEqual(len(country_converters['name'].codes), 249)
class TestLanguage(TestCase, _Py26FixTestCase):
def test_languages(self):
self.assertEqual(len(LANGUAGES), 7874)
def test_wrong_language(self):
self.assertRaises(ValueError, lambda: Language('zzz'))
def test_unknown_language(self):
self.assertEqual(Language('zzzz', unknown='und'), Language('und'))
def test_converter_alpha2(self):
self.assertEqual(Language('eng').alpha2, 'en')
self.assertEqual(Language.fromalpha2('en'), Language('eng'))
self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha2('zz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha2)
self.assertEqual(len(language_converters['alpha2'].codes), 184)
def test_converter_alpha3b(self):
self.assertEqual(Language('fra').alpha3b, 'fre')
self.assertEqual(Language.fromalpha3b('fre'), Language('fra'))
self.assertEqual(Language.fromcode('fre', 'alpha3b'), Language('fra'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3b('zzz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3b)
self.assertEqual(len(language_converters['alpha3b'].codes), 418)
def test_converter_alpha3t(self):
self.assertEqual(Language('fra').alpha3t, 'fra')
self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t)
self.assertEqual(len(language_converters['alpha3t'].codes), 418)
def test_converter_name(self):
self.assertEqual(Language('eng').name, 'English')
self.assertEqual(Language.fromname('English'), Language('eng'))
self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
self.assertRaises(LanguageReverseError, lambda: Language.fromname('Zzzzzzzzz'))
self.assertEqual(len(language_converters['name'].codes), 7874)
def test_converter_scope(self):
self.assertEqual(language_converters['scope'].codes, set(['I', 'S', 'M']))
self.assertEqual(Language('eng').scope, 'individual')
self.assertEqual(Language('und').scope, 'special')
def test_converter_type(self):
self.assertEqual(language_converters['type'].codes, set(['A', 'C', 'E', 'H', 'L', 'S']))
self.assertEqual(Language('eng').type, 'living')
self.assertEqual(Language('und').type, 'special')
def test_converter_opensubtitles(self):
self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
# Montenegrin is not recognized as an ISO language (yet?) but for now it is
# unofficially accepted as Serbian from Montenegro
self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
self.assertRaises(LanguageReverseError, lambda: Language.fromopensubtitles('zzz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').opensubtitles)
self.assertEqual(len(language_converters['opensubtitles'].codes), 606)
# test with all the LANGUAGES from the opensubtitles api
# downloaded from: http://www.opensubtitles.org/addons/export_languages.php
f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
f.readline()
for l in f:
idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
if not int(upload_enabled) and not int(web_enabled):
# do not test LANGUAGES that are too esoteric / not widely available
continue
self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
if alpha2:
self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
f.close()
def test_fromietf_country_script(self):
language = Language.fromietf('fra-FR-Latn')
self.assertEqual(language.alpha3, 'fra')
self.assertEqual(language.country, Country('FR'))
self.assertEqual(language.script, Script('Latn'))
def test_fromietf_country_no_script(self):
language = Language.fromietf('fra-FR')
self.assertEqual(language.alpha3, 'fra')
self.assertEqual(language.country, Country('FR'))
self.assertIsNone(language.script)
def test_fromietf_no_country_no_script(self):
language = Language.fromietf('fra-FR')
self.assertEqual(language.alpha3, 'fra')
self.assertEqual(language.country, Country('FR'))
self.assertIsNone(language.script)
def test_fromietf_no_country_script(self):
language = Language.fromietf('fra-Latn')
self.assertEqual(language.alpha3, 'fra')
self.assertIsNone(language.country)
self.assertEqual(language.script, Script('Latn'))
def test_fromietf_alpha2_language(self):
language = Language.fromietf('fr-Latn')
self.assertEqual(language.alpha3, 'fra')
self.assertIsNone(language.country)
self.assertEqual(language.script, Script('Latn'))
def test_fromietf_wrong_language(self):
self.assertRaises(ValueError, lambda: Language.fromietf('xyz-FR'))
def test_fromietf_wrong_country(self):
self.assertRaises(ValueError, lambda: Language.fromietf('fra-YZ'))
def test_fromietf_wrong_script(self):
self.assertRaises(ValueError, lambda: Language.fromietf('fra-FR-Wxyz'))
def test_eq(self):
self.assertEqual(Language('eng'), Language('eng'))
def test_ne(self):
self.assertNotEqual(Language('fra'), Language('eng'))
self.assertIsNotNone(Language('fra'))
def test_nonzero(self):
self.assertFalse(bool(Language('und')))
self.assertTrue(bool(Language('eng')))
def test_language_hasattr(self):
self.assertTrue(hasattr(Language('fra'), 'alpha3'))
self.assertTrue(hasattr(Language('fra'), 'alpha2'))
self.assertFalse(hasattr(Language('bej'), 'alpha2'))
def test_country(self):
self.assertEqual(Language('por', 'BR').country, Country('BR'))
self.assertEqual(Language('eng', Country('US')).country, Country('US'))
def test_eq_with_country(self):
self.assertEqual(Language('eng', 'US'), Language('eng', Country('US')))
def test_ne_with_country(self):
self.assertNotEqual(Language('eng', 'US'), Language('eng', Country('GB')))
def test_script(self):
self.assertEqual(Language('srp', script='Latn').script, Script('Latn'))
self.assertEqual(Language('srp', script=Script('Cyrl')).script, Script('Cyrl'))
def test_eq_with_script(self):
self.assertEqual(Language('srp', script='Latn'), Language('srp', script=Script('Latn')))
def test_ne_with_script(self):
self.assertNotEqual(Language('srp', script='Latn'), Language('srp', script=Script('Cyrl')))
def test_eq_with_country_and_script(self):
self.assertEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Latn')))
def test_ne_with_country_and_script(self):
self.assertNotEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Cyrl')))
def test_hash(self):
self.assertEqual(hash(Language('fra')), hash('fr'))
self.assertEqual(hash(Language('ace')), hash('ace'))
self.assertEqual(hash(Language('por', 'BR')), hash('pt-BR'))
self.assertEqual(hash(Language('srp', script='Cyrl')), hash('sr-Cyrl'))
self.assertEqual(hash(Language('eng', 'US', 'Latn')), hash('en-US-Latn'))
def test_pickle(self):
for lang in [Language('fra'),
Language('eng', 'US'),
Language('srp', script='Latn'),
Language('eng', 'US', 'Latn')]:
self.assertEqual(pickle.loads(pickle.dumps(lang)), lang)
def test_str(self):
self.assertEqual(Language.fromietf(str(Language('eng', 'US', 'Latn'))), Language('eng', 'US', 'Latn'))
self.assertEqual(Language.fromietf(str(Language('fra', 'FR'))), Language('fra', 'FR'))
self.assertEqual(Language.fromietf(str(Language('bel'))), Language('bel'))
def test_register_converter(self):
class TestConverter(LanguageReverseConverter):
def __init__(self):
self.to_test = {'fra': 'test1', 'eng': 'test2'}
self.from_test = {'test1': 'fra', 'test2': 'eng'}
def convert(self, alpha3, country=None, script=None):
if alpha3 not in self.to_test:
raise LanguageConvertError(alpha3, country, script)
return self.to_test[alpha3]
def reverse(self, test):
if test not in self.from_test:
raise LanguageReverseError(test)
return (self.from_test[test], None)
language = Language('fra')
self.assertFalse(hasattr(language, 'test'))
language_converters['test'] = TestConverter()
self.assertTrue(hasattr(language, 'test'))
self.assertIn('test', language_converters)
self.assertEqual(Language('fra').test, 'test1')
self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
del language_converters['test']
self.assertNotIn('test', language_converters)
self.assertRaises(KeyError, lambda: Language.fromtest('test1'))
self.assertRaises(AttributeError, lambda: Language('fra').test)
def suite():
suite = TestSuite()
suite.addTest(TestLoader().loadTestsFromTestCase(TestScript))
suite.addTest(TestLoader().loadTestsFromTestCase(TestCountry))
suite.addTest(TestLoader().loadTestsFromTestCase(TestLanguage))
return suite
if __name__ == '__main__':
TextTestRunner().run(suite())
@@ -1,27 +0,0 @@
Copyright (c) 2011-2014 Mike Bayer
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author or contributors may not be used to endorse or
promote products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
@@ -1,91 +0,0 @@
dogpile.cache
=============
A caching API built around the concept of a "dogpile lock", which allows
continued access to an expiring data value while a single thread generates a
new value.
dogpile.cache builds on the `dogpile.core <http://pypi.python.org/pypi/dogpile.core>`_
locking system, which implements the idea of "allow one creator to write while
others read" in the abstract. Overall, dogpile.cache is intended as a
replacement to the `Beaker <http://beaker.groovie.org>`_ caching system, the internals
of which are written by the same author. All the ideas of Beaker which "work"
are re-implemented in dogpile.cache in a more efficient and succinct manner,
and all the cruft (Beaker's internals were first written in 2005) relegated
to the trash heap.
Features
--------
* A succinct API which encourages up-front configuration of pre-defined
"regions", each one defining a set of caching characteristics including
storage backend, configuration options, and default expiration time.
* A standard get/set/delete API as well as a function decorator API is
provided.
* The mechanics of key generation are fully customizable. The function
decorator API features a pluggable "key generator" to customize how
cache keys are made to correspond to function calls, and an optional
"key mangler" feature provides for pluggable mangling of keys
(such as encoding, SHA-1 hashing) as desired for each region.
* The dogpile lock, first developed as the core engine behind the Beaker
caching system, here vastly simplified, improved, and better tested.
Some key performance
issues that were intrinsic to Beaker's architecture, particularly that
values would frequently be "double-fetched" from the cache, have been fixed.
* Backends implement their own version of a "distributed" lock, where the
"distribution" matches the backend's storage system. For example, the
memcached backends allow all clients to coordinate creation of values
using memcached itself. The dbm file backend uses a lockfile
alongside the dbm file. New backends, such as a Redis-based backend,
can provide their own locking mechanism appropriate to the storage
engine.
* Writing new backends or hacking on the existing backends is intended to be
routine - all that's needed are basic get/set/delete methods. A distributed
lock tailored towards the backend is an optional addition, else dogpile uses
a regular thread mutex. New backends can be registered with dogpile.cache
directly or made available via setuptools entry points.
* Included backends feature three memcached backends (python-memcached, pylibmc,
bmemcached), a Redis backend, a backend based on Python's
anydbm, and a plain dictionary backend.
* Space for third party plugins, including the first which provides the
dogpile.cache engine to Mako templates.
* Python 3 compatible in place - no 2to3 required.
Synopsis
--------
dogpile.cache features a single public usage object known as the ``CacheRegion``.
This object then refers to a particular ``CacheBackend``. Typical usage
generates a region using ``make_region()``, which can then be used at the
module level to decorate functions, or used directly in code with a traditional
get/set interface. Configuration of the backend is applied to the region
using ``configure()`` or ``configure_from_config()``, allowing deferred
config-file based configuration to occur after modules have been imported::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.pylibmc',
expiration_time = 3600,
arguments = {
'url':["127.0.0.1"],
'binary':True,
'behaviors':{"tcp_nodelay": True,"ketama":True}
}
)
@region.cache_on_arguments()
def load_user_info(user_id):
return some_database.lookup_user_by_id(user_id)
Documentation
-------------
See dogpile.cache's full documentation at
`dogpile.cache documentation <http://dogpilecache.readthedocs.org>`_.
@@ -1,6 +0,0 @@
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
@@ -1,3 +0,0 @@
__version__ = '0.5.4'
from .region import CacheRegion, register_backend, make_region
-193
View File
@@ -1,193 +0,0 @@
import operator
from .compat import py3k
class NoValue(object):
"""Describe a missing cache value.
The :attr:`.NO_VALUE` module global
should be used.
"""
@property
def payload(self):
return self
if py3k:
def __bool__(self): #pragma NO COVERAGE
return False
else:
def __nonzero__(self): #pragma NO COVERAGE
return False
NO_VALUE = NoValue()
"""Value returned from ``get()`` that describes
a key not present."""
class CachedValue(tuple):
"""Represent a value stored in the cache.
:class:`.CachedValue` is a two-tuple of
``(payload, metadata)``, where ``metadata``
is dogpile.cache's tracking information (
currently the creation time). The metadata
and tuple structure is pickleable, if
the backend requires serialization.
"""
payload = property(operator.itemgetter(0))
"""Named accessor for the payload."""
metadata = property(operator.itemgetter(1))
"""Named accessor for the dogpile.cache metadata dictionary."""
def __new__(cls, payload, metadata):
return tuple.__new__(cls, (payload, metadata))
def __reduce__(self):
return CachedValue, (self.payload, self.metadata)
class CacheBackend(object):
"""Base class for backend implementations."""
key_mangler = None
"""Key mangling function.
May be None, or otherwise declared
as an ordinary instance method.
"""
def __init__(self, arguments): #pragma NO COVERAGE
"""Construct a new :class:`.CacheBackend`.
Subclasses should override this to
handle the given arguments.
:param arguments: The ``arguments`` parameter
passed to :func:`.make_registry`.
"""
raise NotImplementedError()
@classmethod
def from_config_dict(cls, config_dict, prefix):
prefix_len = len(prefix)
return cls(
dict(
(key[prefix_len:], config_dict[key])
for key in config_dict
if key.startswith(prefix)
)
)
def get_mutex(self, key):
"""Return an optional mutexing object for the given key.
This object need only provide an ``acquire()``
and ``release()`` method.
May return ``None``, in which case the dogpile
lock will use a regular ``threading.Lock``
object to mutex concurrent threads for
value creation. The default implementation
returns ``None``.
Different backends may want to provide various
kinds of "mutex" objects, such as those which
link to lock files, distributed mutexes,
memcached semaphores, etc. Whatever
kind of system is best suited for the scope
and behavior of the caching backend.
A mutex that takes the key into account will
allow multiple regenerate operations across
keys to proceed simultaneously, while a mutex
that does not will serialize regenerate operations
to just one at a time across all keys in the region.
The latter approach, or a variant that involves
a modulus of the given key's hash value,
can be used as a means of throttling the total
number of value recreation operations that may
proceed at one time.
"""
return None
def get(self, key): #pragma NO COVERAGE
"""Retrieve a value from the cache.
The returned value should be an instance of
:class:`.CachedValue`, or ``NO_VALUE`` if
not present.
"""
raise NotImplementedError()
def get_multi(self, keys): #pragma NO COVERAGE
"""Retrieve multiple values from the cache.
The returned value should be a list, corresponding
to the list of keys given.
.. versionadded:: 0.5.0
"""
raise NotImplementedError()
def set(self, key, value): #pragma NO COVERAGE
"""Set a value in the cache.
The key will be whatever was passed
to the registry, processed by the
"key mangling" function, if any.
The value will always be an instance
of :class:`.CachedValue`.
"""
raise NotImplementedError()
def set_multi(self, mapping): #pragma NO COVERAGE
"""Set multiple values in the cache.
The key will be whatever was passed
to the registry, processed by the
"key mangling" function, if any.
The value will always be an instance
of :class:`.CachedValue`.
.. versionadded:: 0.5.0
"""
raise NotImplementedError()
def delete(self, key): #pragma NO COVERAGE
"""Delete a value from the cache.
The key will be whatever was passed
to the registry, processed by the
"key mangling" function, if any.
The behavior here should be idempotent,
that is, can be called any number of times
regardless of whether or not the
key exists.
"""
raise NotImplementedError()
def delete_multi(self, keys): #pragma NO COVERAGE
"""Delete multiple values from the cache.
The key will be whatever was passed
to the registry, processed by the
"key mangling" function, if any.
The behavior here should be idempotent,
that is, can be called any number of times
regardless of whether or not the
key exists.
.. versionadded:: 0.5.0
"""
raise NotImplementedError()
@@ -1,10 +0,0 @@
from dogpile.cache.region import register_backend
register_backend("dogpile.cache.null", "dogpile.cache.backends.null", "NullBackend")
register_backend("dogpile.cache.dbm", "dogpile.cache.backends.file", "DBMBackend")
register_backend("dogpile.cache.pylibmc", "dogpile.cache.backends.memcached", "PylibmcBackend")
register_backend("dogpile.cache.bmemcached", "dogpile.cache.backends.memcached", "BMemcachedBackend")
register_backend("dogpile.cache.memcached", "dogpile.cache.backends.memcached", "MemcachedBackend")
register_backend("dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend")
register_backend("dogpile.cache.memory_pickle", "dogpile.cache.backends.memory", "MemoryPickleBackend")
register_backend("dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend")
@@ -1,441 +0,0 @@
"""
File Backends
------------------
Provides backends that deal with local filesystem access.
"""
from __future__ import with_statement
from dogpile.cache.api import CacheBackend, NO_VALUE
from contextlib import contextmanager
from dogpile.cache import compat
from dogpile.cache import util
import os
__all__ = 'DBMBackend', 'FileLock', 'AbstractFileLock'
class DBMBackend(CacheBackend):
"""A file-backend using a dbm file to store keys.
Basic usage::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.dbm',
expiration_time = 3600,
arguments = {
"filename":"/path/to/cachefile.dbm"
}
)
DBM access is provided using the Python ``anydbm`` module,
which selects a platform-specific dbm module to use.
This may be made to be more configurable in a future
release.
Note that different dbm modules have different behaviors.
Some dbm implementations handle their own locking, while
others don't. The :class:`.DBMBackend` uses a read/write
lockfile by default, which is compatible even with those
DBM implementations for which this is unnecessary,
though the behavior can be disabled.
The DBM backend by default makes use of two lockfiles.
One is in order to protect the DBM file itself from
concurrent writes, the other is to coordinate
value creation (i.e. the dogpile lock). By default,
these lockfiles use the ``flock()`` system call
for locking; this is **only available on Unix
platforms**. An alternative lock implementation, such as one
which is based on threads or uses a third-party system
such as `portalocker <https://pypi.python.org/pypi/portalocker>`_,
can be dropped in using the ``lock_factory`` argument
in conjunction with the :class:`.AbstractFileLock` base class.
Currently, the dogpile lock is against the entire
DBM file, not per key. This means there can
only be one "creator" job running at a time
per dbm file.
A future improvement might be to have the dogpile lock
using a filename that's based on a modulus of the key.
Locking on a filename that uniquely corresponds to the
key is problematic, since it's not generally safe to
delete lockfiles as the application runs, implying an
unlimited number of key-based files would need to be
created and never deleted.
Parameters to the ``arguments`` dictionary are
below.
:param filename: path of the filename in which to
create the DBM file. Note that some dbm backends
will change this name to have additional suffixes.
:param rw_lockfile: the name of the file to use for
read/write locking. If omitted, a default name
is used by appending the suffix ".rw.lock" to the
DBM filename. If False, then no lock is used.
:param dogpile_lockfile: the name of the file to use
for value creation, i.e. the dogpile lock. If
omitted, a default name is used by appending the
suffix ".dogpile.lock" to the DBM filename. If
False, then dogpile.cache uses the default dogpile
lock, a plain thread-based mutex.
:param lock_factory: a function or class which provides
for a read/write lock. Defaults to :class:`.FileLock`.
Custom implementations need to implement context-manager
based ``read()`` and ``write()`` functions - the
:class:`.AbstractFileLock` class is provided as a base class
which provides these methods based on individual read/write lock
functions. E.g. to replace the lock with the dogpile.core
:class:`.ReadWriteMutex`::
from dogpile.core.readwrite_lock import ReadWriteMutex
from dogpile.cache.backends.file import AbstractFileLock
class MutexLock(AbstractFileLock):
def __init__(self, filename):
self.mutex = ReadWriteMutex()
def acquire_read_lock(self, wait):
ret = self.mutex.acquire_read_lock(wait)
return wait or ret
def acquire_write_lock(self, wait):
ret = self.mutex.acquire_write_lock(wait)
return wait or ret
def release_read_lock(self):
return self.mutex.release_read_lock()
def release_write_lock(self):
return self.mutex.release_write_lock()
from dogpile.cache import make_region
region = make_region().configure(
"dogpile.cache.dbm",
expiration_time=300,
arguments={
"filename": "file.dbm",
"lock_factory": MutexLock
}
)
While the included :class:`.FileLock` uses ``os.flock()``, a
windows-compatible implementation can be built using a library
such as `portalocker <https://pypi.python.org/pypi/portalocker>`_.
.. versionadded:: 0.5.2
"""
def __init__(self, arguments):
self.filename = os.path.abspath(
os.path.normpath(arguments['filename'])
)
dir_, filename = os.path.split(self.filename)
self.lock_factory = arguments.get("lock_factory", FileLock)
self._rw_lock = self._init_lock(
arguments.get('rw_lockfile'),
".rw.lock", dir_, filename)
self._dogpile_lock = self._init_lock(
arguments.get('dogpile_lockfile'),
".dogpile.lock",
dir_, filename,
util.KeyReentrantMutex.factory)
# TODO: make this configurable
if compat.py3k:
import dbm
else:
import anydbm as dbm
self.dbmmodule = dbm
self._init_dbm_file()
def _init_lock(self, argument, suffix, basedir, basefile, wrapper=None):
if argument is None:
lock = self.lock_factory(os.path.join(basedir, basefile + suffix))
elif argument is not False:
lock = self.lock_factory(
os.path.abspath(
os.path.normpath(argument)
))
else:
return None
if wrapper:
lock = wrapper(lock)
return lock
def _init_dbm_file(self):
exists = os.access(self.filename, os.F_OK)
if not exists:
for ext in ('db', 'dat', 'pag', 'dir'):
if os.access(self.filename + os.extsep + ext, os.F_OK):
exists = True
break
if not exists:
fh = self.dbmmodule.open(self.filename, 'c')
fh.close()
def get_mutex(self, key):
# using one dogpile for the whole file. Other ways
# to do this might be using a set of files keyed to a
# hash/modulus of the key. the issue is it's never
# really safe to delete a lockfile as this can
# break other processes trying to get at the file
# at the same time - so handling unlimited keys
# can't imply unlimited filenames
if self._dogpile_lock:
return self._dogpile_lock(key)
else:
return None
@contextmanager
def _use_rw_lock(self, write):
if self._rw_lock is None:
yield
elif write:
with self._rw_lock.write():
yield
else:
with self._rw_lock.read():
yield
@contextmanager
def _dbm_file(self, write):
with self._use_rw_lock(write):
dbm = self.dbmmodule.open(self.filename,
"w" if write else "r")
yield dbm
dbm.close()
def get(self, key):
with self._dbm_file(False) as dbm:
if hasattr(dbm, 'get'):
value = dbm.get(key, NO_VALUE)
else:
# gdbm objects lack a .get method
try:
value = dbm[key]
except KeyError:
value = NO_VALUE
if value is not NO_VALUE:
value = compat.pickle.loads(value)
return value
def get_multi(self, keys):
return [self.get(key) for key in keys]
def set(self, key, value):
with self._dbm_file(True) as dbm:
dbm[key] = compat.pickle.dumps(value)
def set_multi(self, mapping):
with self._dbm_file(True) as dbm:
for key,value in mapping.items():
dbm[key] = compat.pickle.dumps(value)
def delete(self, key):
with self._dbm_file(True) as dbm:
try:
del dbm[key]
except KeyError:
pass
def delete_multi(self, keys):
with self._dbm_file(True) as dbm:
for key in keys:
try:
del dbm[key]
except KeyError:
pass
class AbstractFileLock(object):
"""Coordinate read/write access to a file.
typically is a file-based lock but doesn't necessarily have to be.
The default implementation here is :class:`.FileLock`.
Implementations should provide the following methods::
* __init__()
* acquire_read_lock()
* acquire_write_lock()
* release_read_lock()
* release_write_lock()
The ``__init__()`` method accepts a single argument "filename", which
may be used as the "lock file", for those implementations that use a lock
file.
Note that multithreaded environments must provide a thread-safe
version of this lock. The recommended approach for file-descriptor-based
locks is to use a Python ``threading.local()`` so that a unique file descriptor
is held per thread. See the source code of :class:`.FileLock` for an
implementation example.
"""
def __init__(self, filename):
"""Constructor, is given the filename of a potential lockfile.
The usage of this filename is optional and no file is
created by default.
Raises ``NotImplementedError`` by default, must be
implemented by subclasses.
"""
raise NotImplementedError()
def acquire(self, wait=True):
"""Acquire the "write" lock.
This is a direct call to :meth:`.AbstractFileLock.acquire_write_lock`.
"""
return self.acquire_write_lock(wait)
def release(self):
"""Release the "write" lock.
This is a direct call to :meth:`.AbstractFileLock.release_write_lock`.
"""
self.release_write_lock()
@contextmanager
def read(self):
"""Provide a context manager for the "read" lock.
This method makes use of :meth:`.AbstractFileLock.acquire_read_lock`
and :meth:`.AbstractFileLock.release_read_lock`
"""
self.acquire_read_lock(True)
try:
yield
finally:
self.release_read_lock()
@contextmanager
def write(self):
"""Provide a context manager for the "write" lock.
This method makes use of :meth:`.AbstractFileLock.acquire_write_lock`
and :meth:`.AbstractFileLock.release_write_lock`
"""
self.acquire_write_lock(True)
try:
yield
finally:
self.release_write_lock()
@property
def is_open(self):
"""optional method."""
raise NotImplementedError()
def acquire_read_lock(self, wait):
"""Acquire a 'reader' lock.
Raises ``NotImplementedError`` by default, must be
implemented by subclasses.
"""
raise NotImplementedError()
def acquire_write_lock(self, wait):
"""Acquire a 'write' lock.
Raises ``NotImplementedError`` by default, must be
implemented by subclasses.
"""
raise NotImplementedError()
def release_read_lock(self):
"""Release a 'reader' lock.
Raises ``NotImplementedError`` by default, must be
implemented by subclasses.
"""
raise NotImplementedError()
def release_write_lock(self):
"""Release a 'writer' lock.
Raises ``NotImplementedError`` by default, must be
implemented by subclasses.
"""
raise NotImplementedError()
class FileLock(AbstractFileLock):
"""Use lockfiles to coordinate read/write access to a file.
Only works on Unix systems, using
`fcntl.flock() <http://docs.python.org/library/fcntl.html>`_.
"""
def __init__(self, filename):
self._filedescriptor = compat.threading.local()
self.filename = filename
@util.memoized_property
def _module(self):
import fcntl
return fcntl
@property
def is_open(self):
return hasattr(self._filedescriptor, 'fileno')
def acquire_read_lock(self, wait):
return self._acquire(wait, os.O_RDONLY, self._module.LOCK_SH)
def acquire_write_lock(self, wait):
return self._acquire(wait, os.O_WRONLY, self._module.LOCK_EX)
def release_read_lock(self):
self._release()
def release_write_lock(self):
self._release()
def _acquire(self, wait, wrflag, lockflag):
wrflag |= os.O_CREAT
fileno = os.open(self.filename, wrflag)
try:
if not wait:
lockflag |= self._module.LOCK_NB
self._module.flock(fileno, lockflag)
except IOError:
os.close(fileno)
if not wait:
# this is typically
# "[Errno 35] Resource temporarily unavailable",
# because of LOCK_NB
return False
else:
raise
else:
self._filedescriptor.fileno = fileno
return True
def _release(self):
try:
fileno = self._filedescriptor.fileno
except AttributeError:
return
else:
self._module.flock(fileno, self._module.LOCK_UN)
os.close(fileno)
del self._filedescriptor.fileno
@@ -1,332 +0,0 @@
"""
Memcached Backends
------------------
Provides backends for talking to `memcached <http://memcached.org>`_.
"""
from dogpile.cache.api import CacheBackend, NO_VALUE
from dogpile.cache import compat
from dogpile.cache import util
import random
import time
__all__ = 'GenericMemcachedBackend', 'MemcachedBackend',\
'PylibmcBackend', 'BMemcachedBackend', 'MemcachedLock'
class MemcachedLock(object):
"""Simple distributed lock using memcached.
This is an adaptation of the lock featured at
http://amix.dk/blog/post/19386
"""
def __init__(self, client_fn, key):
self.client_fn = client_fn
self.key = "_lock" + key
def acquire(self, wait=True):
client = self.client_fn()
i = 0
while True:
if client.add(self.key, 1):
return True
elif not wait:
return False
else:
sleep_time = (((i+1)*random.random()) + 2**i) / 2.5
time.sleep(sleep_time)
if i < 15:
i += 1
def release(self):
client = self.client_fn()
client.delete(self.key)
class GenericMemcachedBackend(CacheBackend):
"""Base class for memcached backends.
This base class accepts a number of paramters
common to all backends.
:param url: the string URL to connect to. Can be a single
string or a list of strings. This is the only argument
that's required.
:param distributed_lock: boolean, when True, will use a
memcached-lock as the dogpile lock (see :class:`.MemcachedLock`).
Use this when multiple
processes will be talking to the same memcached instance.
When left at False, dogpile will coordinate on a regular
threading mutex.
:param memcached_expire_time: integer, when present will
be passed as the ``time`` parameter to ``pylibmc.Client.set``.
This is used to set the memcached expiry time for a value.
.. note::
This parameter is **different** from Dogpile's own
``expiration_time``, which is the number of seconds after
which Dogpile will consider the value to be expired.
When Dogpile considers a value to be expired,
it **continues to use the value** until generation
of a new value is complete, when using
:meth:`.CacheRegion.get_or_create`.
Therefore, if you are setting ``memcached_expire_time``, you'll
want to make sure it is greater than ``expiration_time``
by at least enough seconds for new values to be generated,
else the value won't be available during a regeneration,
forcing all threads to wait for a regeneration each time
a value expires.
The :class:`.GenericMemachedBackend` uses a ``threading.local()``
object to store individual client objects per thread,
as most modern memcached clients do not appear to be inherently
threadsafe.
In particular, ``threading.local()`` has the advantage over pylibmc's
built-in thread pool in that it automatically discards objects
associated with a particular thread when that thread ends.
"""
set_arguments = {}
"""Additional arguments which will be passed
to the :meth:`set` method."""
def __init__(self, arguments):
self._imports()
# using a plain threading.local here. threading.local
# automatically deletes the __dict__ when a thread ends,
# so the idea is that this is superior to pylibmc's
# own ThreadMappedPool which doesn't handle this
# automatically.
self.url = util.to_list(arguments['url'])
self.distributed_lock = arguments.get('distributed_lock', False)
self.memcached_expire_time = arguments.get(
'memcached_expire_time', 0)
def _imports(self):
"""client library imports go here."""
raise NotImplementedError()
def _create_client(self):
"""Creation of a Client instance goes here."""
raise NotImplementedError()
@util.memoized_property
def _clients(self):
backend = self
class ClientPool(compat.threading.local):
def __init__(self):
self.memcached = backend._create_client()
return ClientPool()
@property
def client(self):
"""Return the memcached client.
This uses a threading.local by
default as it appears most modern
memcached libs aren't inherently
threadsafe.
"""
return self._clients.memcached
def get_mutex(self, key):
if self.distributed_lock:
return MemcachedLock(lambda: self.client, key)
else:
return None
def get(self, key):
value = self.client.get(key)
if value is None:
return NO_VALUE
else:
return value
def get_multi(self, keys):
values = self.client.get_multi(keys)
return [
NO_VALUE if key not in values
else values[key] for key in keys
]
def set(self, key, value):
self.client.set(key,
value,
**self.set_arguments
)
def set_multi(self, mapping):
self.client.set_multi(mapping,
**self.set_arguments
)
def delete(self, key):
self.client.delete(key)
def delete_multi(self, keys):
self.client.delete_multi(keys)
class MemcacheArgs(object):
"""Mixin which provides support for the 'time' argument to set(),
'min_compress_len' to other methods.
"""
def __init__(self, arguments):
self.min_compress_len = arguments.get('min_compress_len', 0)
self.set_arguments = {}
if "memcached_expire_time" in arguments:
self.set_arguments["time"] =\
arguments["memcached_expire_time"]
if "min_compress_len" in arguments:
self.set_arguments["min_compress_len"] =\
arguments["min_compress_len"]
super(MemcacheArgs, self).__init__(arguments)
class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
"""A backend for the
`pylibmc <http://sendapatch.se/projects/pylibmc/index.html>`_
memcached client.
A configuration illustrating several of the optional
arguments described in the pylibmc documentation::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.pylibmc',
expiration_time = 3600,
arguments = {
'url':["127.0.0.1"],
'binary':True,
'behaviors':{"tcp_nodelay": True,"ketama":True}
}
)
Arguments accepted here include those of
:class:`.GenericMemcachedBackend`, as well as
those below.
:param binary: sets the ``binary`` flag understood by
``pylibmc.Client``.
:param behaviors: a dictionary which will be passed to
``pylibmc.Client`` as the ``behaviors`` parameter.
:param min_compress_len: Integer, will be passed as the
``min_compress_len`` parameter to the ``pylibmc.Client.set``
method.
"""
def __init__(self, arguments):
self.binary = arguments.get('binary', False)
self.behaviors = arguments.get('behaviors', {})
super(PylibmcBackend, self).__init__(arguments)
def _imports(self):
global pylibmc
import pylibmc
def _create_client(self):
return pylibmc.Client(self.url,
binary=self.binary,
behaviors=self.behaviors
)
class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
"""A backend using the standard `Python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
library.
Example::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.memcached',
expiration_time = 3600,
arguments = {
'url':"127.0.0.1:11211"
}
)
"""
def _imports(self):
global memcache
import memcache
def _create_client(self):
return memcache.Client(self.url)
class BMemcachedBackend(GenericMemcachedBackend):
"""A backend for the
`python-binary-memcached <https://github.com/jaysonsantos/python-binary-memcached>`_
memcached client.
This is a pure Python memcached client which
includes the ability to authenticate with a memcached
server using SASL.
A typical configuration using username/password::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.bmemcached',
expiration_time = 3600,
arguments = {
'url':["127.0.0.1"],
'username':'scott',
'password':'tiger'
}
)
Arguments which can be passed to the ``arguments``
dictionary include:
:param username: optional username, will be used for
SASL authentication.
:param password: optional password, will be used for
SASL authentication.
"""
def __init__(self, arguments):
self.username = arguments.get('username', None)
self.password = arguments.get('password', None)
super(BMemcachedBackend, self).__init__(arguments)
def _imports(self):
global bmemcached
import bmemcached
class RepairBMemcachedAPI(bmemcached.Client):
"""Repairs BMemcached's non-standard method
signatures, which was fixed in BMemcached
ef206ed4473fec3b639e.
"""
def add(self, key, value):
try:
return super(RepairBMemcachedAPI, self).add(key, value)
except ValueError:
return False
self.Client = RepairBMemcachedAPI
def _create_client(self):
return self.Client(self.url,
username=self.username,
password=self.password
)
def delete_multi(self, keys):
"""python-binary-memcached api does not implements delete_multi"""
for key in keys:
self.delete(key)
@@ -1,122 +0,0 @@
"""
Memory Backends
---------------
Provides simple dictionary-based backends.
The two backends are :class:`.MemoryBackend` and :class:`.MemoryPickleBackend`;
the latter applies a serialization step to cached values while the former
places the value as given into the dictionary.
"""
from dogpile.cache.api import CacheBackend, NO_VALUE
from dogpile.cache.compat import pickle
class MemoryBackend(CacheBackend):
"""A backend that uses a plain dictionary.
There is no size management, and values which
are placed into the dictionary will remain
until explicitly removed. Note that
Dogpile's expiration of items is based on
timestamps and does not remove them from
the cache.
E.g.::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.memory'
)
To use a Python dictionary of your choosing,
it can be passed in with the ``cache_dict``
argument::
my_dictionary = {}
region = make_region().configure(
'dogpile.cache.memory',
arguments={
"cache_dict":my_dictionary
}
)
"""
pickle_values = False
def __init__(self, arguments):
self._cache = arguments.pop("cache_dict", {})
def get(self, key):
value = self._cache.get(key, NO_VALUE)
if value is not NO_VALUE and self.pickle_values:
value = pickle.loads(value)
return value
def get_multi(self, keys):
ret = [self._cache.get(key, NO_VALUE)
for key in keys]
if self.pickle_values:
ret = [
pickle.loads(value)
if value is not NO_VALUE else value
for value in ret
]
return ret
def set(self, key, value):
if self.pickle_values:
value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = value
def set_multi(self, mapping):
pickle_values = self.pickle_values
for key, value in mapping.items():
if pickle_values:
value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = value
def delete(self, key):
self._cache.pop(key, None)
def delete_multi(self, keys):
for key in keys:
self._cache.pop(key, None)
class MemoryPickleBackend(MemoryBackend):
"""A backend that uses a plain dictionary, but serializes objects on
:meth:`.MemoryBackend.set` and deserializes :meth:`.MemoryBackend.get`.
E.g.::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.memory_pickle'
)
The usage of pickle to serialize cached values allows an object
as placed in the cache to be a copy of the original given object, so
that any subsequent changes to the given object aren't reflected
in the cached value, thus making the backend behave the same way
as other backends which make use of serialization.
The serialization is performed via pickle, and incurs the same
performance hit in doing so as that of other backends; in this way
the :class:`.MemoryPickleBackend` performance is somewhere in between
that of the pure :class:`.MemoryBackend` and the remote server oriented
backends such as that of Memcached or Redis.
Pickle behavior here is the same as that of the Redis backend, using
either ``cPickle`` or ``pickle`` and specifying ``HIGHEST_PROTOCOL``
upon serialize.
.. versionadded:: 0.5.3
"""
pickle_values = True
@@ -1,62 +0,0 @@
"""
Null Backend
-------------
The Null backend does not do any caching at all. It can be
used to test behavior without caching, or as a means of disabling
caching for a region that is otherwise used normally.
.. versionadded:: 0.5.4
"""
from dogpile.cache.api import CacheBackend, NO_VALUE
__all__ = ['NullBackend']
class NullLock(object):
def acquire(self):
pass
def release(self):
pass
class NullBackend(CacheBackend):
"""A "null" backend that effectively disables all cache operations.
Basic usage::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.null'
)
"""
def __init__(self, arguments):
pass
def get_mutex(self, key):
return NullLock()
def get(self, key):
return NO_VALUE
def get_multi(self, keys):
return [NO_VALUE for k in keys]
def set(self, key, value):
pass
def set_multi(self, mapping):
pass
def delete(self, key):
pass
def delete_multi(self, keys):
pass
@@ -1,181 +0,0 @@
"""
Redis Backends
------------------
Provides backends for talking to `Redis <http://redis.io>`_.
"""
from __future__ import absolute_import
from dogpile.cache.api import CacheBackend, NO_VALUE
from dogpile.cache.compat import pickle, u
redis = None
__all__ = 'RedisBackend',
class RedisBackend(CacheBackend):
"""A `Redis <http://redis.io/>`_ backend, using the
`redis-py <http://pypi.python.org/pypi/redis/>`_ backend.
Example configuration::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.redis',
arguments = {
'host': 'localhost',
'port': 6379,
'db': 0,
'redis_expiration_time': 60*60*2, # 2 hours
'distributed_lock':True
}
)
Arguments accepted in the arguments dictionary:
:param url: string. If provided, will override separate host/port/db
params. The format is that accepted by ``StrictRedis.from_url()``.
.. versionadded:: 0.4.1
:param host: string, default is ``localhost``.
:param password: string, default is no password.
.. versionadded:: 0.4.1
:param port: integer, default is ``6379``.
:param db: integer, default is ``0``.
:param redis_expiration_time: integer, number of seconds after setting
a value that Redis should expire it. This should be larger than dogpile's
cache expiration. By default no expiration is set.
:param distributed_lock: boolean, when True, will use a
redis-lock as the dogpile lock.
Use this when multiple
processes will be talking to the same redis instance.
When left at False, dogpile will coordinate on a regular
threading mutex.
:param lock_timeout: integer, number of seconds after acquiring a lock that
Redis should expire it. This argument is only valid when
``distributed_lock`` is ``True``.
.. versionadded:: 0.5.0
:param socket_timeout: float, seconds for socket timeout.
Default is None (no timeout).
.. versionadded:: 0.5.4
:param lock_sleep: integer, number of seconds to sleep when failed to
acquire a lock. This argument is only valid when
``distributed_lock`` is ``True``.
.. versionadded:: 0.5.0
:param connection_pool: ``redis.ConnectionPool`` object. If provided,
this object supersedes other connection arguments passed to the
``redis.StrictRedis`` instance, including url and/or host as well as
socket_timeout, and will be passed to ``redis.StrictRedis`` as the
source of connectivity.
.. versionadded:: 0.5.4
"""
def __init__(self, arguments):
self._imports()
self.url = arguments.pop('url', None)
self.host = arguments.pop('host', 'localhost')
self.password = arguments.pop('password', None)
self.port = arguments.pop('port', 6379)
self.db = arguments.pop('db', 0)
self.distributed_lock = arguments.get('distributed_lock', False)
self.socket_timeout = arguments.pop('socket_timeout', None)
self.lock_timeout = arguments.get('lock_timeout', None)
self.lock_sleep = arguments.get('lock_sleep', 0.1)
self.redis_expiration_time = arguments.pop('redis_expiration_time', 0)
self.connection_pool = arguments.get('connection_pool', None)
self.client = self._create_client()
def _imports(self):
# defer imports until backend is used
global redis
import redis
def _create_client(self):
if self.connection_pool is not None:
# the connection pool already has all other connection
# options present within, so here we disregard socket_timeout
# and others.
return redis.StrictRedis(connection_pool=self.connection_pool)
args = {}
if self.socket_timeout:
args['socket_timeout'] = self.socket_timeout
if self.url is not None:
args.update(url=self.url)
return redis.StrictRedis.from_url(**args)
else:
args.update(
host=self.host, password=self.password,
port=self.port, db=self.db
)
return redis.StrictRedis(**args)
def get_mutex(self, key):
if self.distributed_lock:
return self.client.lock(u('_lock{0}').format(key),
self.lock_timeout, self.lock_sleep)
else:
return None
def get(self, key):
value = self.client.get(key)
if value is None:
return NO_VALUE
return pickle.loads(value)
def get_multi(self, keys):
values = self.client.mget(keys)
return [pickle.loads(v) if v is not None else NO_VALUE
for v in values]
def set(self, key, value):
if self.redis_expiration_time:
self.client.setex(key, self.redis_expiration_time,
pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
else:
self.client.set(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
def set_multi(self, mapping):
mapping = dict(
(k, pickle.dumps(v, pickle.HIGHEST_PROTOCOL))
for k, v in mapping.items()
)
if not self.redis_expiration_time:
self.client.mset(mapping)
else:
pipe = self.client.pipeline()
for key, value in mapping.items():
pipe.setex(key, self.redis_expiration_time, value)
pipe.execute()
def delete(self, key):
self.client.delete(key)
def delete_multi(self, keys):
self.client.delete(*keys)
-68
View File
@@ -1,68 +0,0 @@
import sys
py2k = sys.version_info < (3, 0)
py3k = sys.version_info >= (3, 0)
py32 = sys.version_info >= (3, 2)
py27 = sys.version_info >= (2, 7)
jython = sys.platform.startswith('java')
win32 = sys.platform.startswith('win')
try:
import threading
except ImportError:
import dummy_threading as threading
if py3k: # pragma: no cover
string_types = str,
text_type = str
string_type = str
if py32:
callable = callable
else:
def callable(fn):
return hasattr(fn, '__call__')
def u(s):
return s
def ue(s):
return s
import configparser
import io
import _thread as thread
else:
string_types = basestring,
text_type = unicode
string_type = str
def u(s):
return unicode(s, "utf-8")
def ue(s):
return unicode(s, "unicode_escape")
import ConfigParser as configparser
import StringIO as io
callable = callable
import thread
if py3k or jython:
import pickle
else:
import cPickle as pickle
def timedelta_total_seconds(td):
if py27:
return td.total_seconds()
else:
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6
@@ -1,17 +0,0 @@
"""Exception classes for dogpile.cache."""
class DogpileCacheException(Exception):
"""Base Exception for dogpile.cache exceptions to inherit from."""
class RegionAlreadyConfigured(DogpileCacheException):
"""CacheRegion instance is already configured."""
class RegionNotConfigured(DogpileCacheException):
"""CacheRegion instance has not been configured."""
class ValidationError(DogpileCacheException):
"""Error validating a value or option."""
@@ -1,87 +0,0 @@
"""
Mako Integration
----------------
dogpile.cache includes a `Mako <http://www.makotemplates.org>`_ plugin that replaces `Beaker <http://beaker.groovie.org>`_
as the cache backend.
Setup a Mako template lookup using the "dogpile.cache" cache implementation
and a region dictionary::
from dogpile.cache import make_region
from mako.lookup import TemplateLookup
my_regions = {
"local":make_region().configure(
"dogpile.cache.dbm",
expiration_time=360,
arguments={"filename":"file.dbm"}
),
"memcached":make_region().configure(
"dogpile.cache.pylibmc",
expiration_time=3600,
arguments={"url":["127.0.0.1"]}
)
}
mako_lookup = TemplateLookup(
directories=["/myapp/templates"],
cache_impl="dogpile.cache",
cache_args={
'regions':my_regions
}
)
To use the above configuration in a template, use the ``cached=True`` argument on any
Mako tag which accepts it, in conjunction with the name of the desired region
as the ``cache_region`` argument::
<%def name="mysection()" cached="True" cache_region="memcached">
some content that's cached
</%def>
"""
from mako.cache import CacheImpl
class MakoPlugin(CacheImpl):
"""A Mako ``CacheImpl`` which talks to dogpile.cache."""
def __init__(self, cache):
super(MakoPlugin, self).__init__(cache)
try:
self.regions = self.cache.template.cache_args['regions']
except KeyError:
raise KeyError(
"'cache_regions' argument is required on the "
"Mako Lookup or Template object for usage "
"with the dogpile.cache plugin.")
def _get_region(self, **kw):
try:
region = kw['region']
except KeyError:
raise KeyError(
"'cache_region' argument must be specified with 'cache=True'"
"within templates for usage with the dogpile.cache plugin.")
try:
return self.regions[region]
except KeyError:
raise KeyError("No such region '%s'" % region)
def get_and_replace(self, key, creation_function, **kw):
expiration_time = kw.pop("timeout", None)
return self._get_region(**kw).get_or_create(key, creation_function,
expiration_time=expiration_time)
def get_or_create(self, key, creation_function, **kw):
return self.get_and_replace(key, creation_function, **kw)
def put(self, key, value, **kw):
self._get_region(**kw).put(key, value)
def get(self, key, **kw):
expiration_time = kw.pop("timeout", None)
return self._get_region(**kw).get(key, expiration_time=expiration_time)
def invalidate(self, key, **kw):
self._get_region(**kw).delete(key)
-93
View File
@@ -1,93 +0,0 @@
"""
Proxy Backends
------------------
Provides a utility and a decorator class that allow for modifying the behavior
of different backends without altering the class itself or having to extend the
base backend.
.. versionadded:: 0.5.0 Added support for the :class:`.ProxyBackend` class.
"""
from .api import CacheBackend
class ProxyBackend(CacheBackend):
"""A decorator class for altering the functionality of backends.
Basic usage::
from dogpile.cache import make_region
from dogpile.cache.proxy import ProxyBackend
class MyFirstProxy(ProxyBackend):
def get(self, key):
# ... custom code goes here ...
return self.proxied.get(key)
def set(self, key, value):
# ... custom code goes here ...
self.proxied.set(key)
class MySecondProxy(ProxyBackend):
def get(self, key):
# ... custom code goes here ...
return self.proxied.get(key)
region = make_region().configure(
'dogpile.cache.dbm',
expiration_time = 3600,
arguments = {
"filename":"/path/to/cachefile.dbm"
},
wrap = [ MyFirstProxy, MySecondProxy ]
)
Classes that extend :class:`.ProxyBackend` can be stacked
together. The ``.proxied`` property will always
point to either the concrete backend instance or
the next proxy in the chain that a method can be
delegated towards.
.. versionadded:: 0.5.0
"""
def __init__(self, *args, **kwargs):
self.proxied = None
def wrap(self, backend):
''' Take a backend as an argument and setup the self.proxied property.
Return an object that be used as a backend by a :class:`.CacheRegion`
object.
'''
assert(isinstance(backend, CacheBackend) or isinstance(backend, ProxyBackend))
self.proxied = backend
return self
#
# Delegate any functions that are not already overridden to
# the proxies backend
#
def get(self, key):
return self.proxied.get(key)
def set(self, key, value):
self.proxied.set(key, value)
def delete(self, key):
self.proxied.delete(key)
def get_multi(self, keys):
return self.proxied.get_multi(keys)
def set_multi(self, keys):
self.proxied.set_multi(keys)
def delete_multi(self, keys):
self.proxied.delete_multi(keys)
def get_mutex(self, key):
return self.proxied.get_mutex(key)
File diff suppressed because it is too large Load Diff
-189
View File
@@ -1,189 +0,0 @@
from hashlib import sha1
import inspect
import re
import collections
from . import compat
def coerce_string_conf(d):
result = {}
for k, v in d.items():
if not isinstance(v, compat.string_types):
result[k] = v
continue
v = v.strip()
if re.match(r'^[-+]?\d+$', v):
result[k] = int(v)
elif re.match(r'^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?$', v):
result[k] = float(v)
elif v.lower() in ('false', 'true'):
result[k] = v.lower() == 'true'
elif v == 'None':
result[k] = None
else:
result[k] = v
return result
class PluginLoader(object):
def __init__(self, group):
self.group = group
self.impls = {}
def load(self, name):
if name in self.impls:
return self.impls[name]()
else: # pragma NO COVERAGE
import pkg_resources
for impl in pkg_resources.iter_entry_points(
self.group,
name):
self.impls[name] = impl.load
return impl.load()
else:
raise Exception(
"Can't load plugin %s %s" %
(self.group, name))
def register(self, name, modulepath, objname):
def load():
mod = __import__(modulepath)
for token in modulepath.split(".")[1:]:
mod = getattr(mod, token)
return getattr(mod, objname)
self.impls[name] = load
def function_key_generator(namespace, fn, to_str=compat.string_type):
"""Return a function that generates a string
key, based on a given function as well as
arguments to the returned function itself.
This is used by :meth:`.CacheRegion.cache_on_arguments`
to generate a cache key from a decorated function.
It can be replaced using the ``function_key_generator``
argument passed to :func:`.make_region`.
"""
if namespace is None:
namespace = '%s:%s' % (fn.__module__, fn.__name__)
else:
namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
args = inspect.getargspec(fn)
has_self = args[0] and args[0][0] in ('self', 'cls')
def generate_key(*args, **kw):
if kw:
raise ValueError(
"dogpile.cache's default key creation "
"function does not accept keyword arguments.")
if has_self:
args = args[1:]
return namespace + "|" + " ".join(map(to_str, args))
return generate_key
def function_multi_key_generator(namespace, fn, to_str=compat.string_type):
if namespace is None:
namespace = '%s:%s' % (fn.__module__, fn.__name__)
else:
namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
args = inspect.getargspec(fn)
has_self = args[0] and args[0][0] in ('self', 'cls')
def generate_keys(*args, **kw):
if kw:
raise ValueError(
"dogpile.cache's default key creation "
"function does not accept keyword arguments.")
if has_self:
args = args[1:]
return [namespace + "|" + key for key in map(to_str, args)]
return generate_keys
def sha1_mangle_key(key):
"""a SHA1 key mangler."""
return sha1(key).hexdigest()
def length_conditional_mangler(length, mangler):
"""a key mangler that mangles if the length of the key is
past a certain threshold.
"""
def mangle(key):
if len(key) >= length:
return mangler(key)
else:
return key
return mangle
class memoized_property(object):
"""A read-only @property that is only evaluated once."""
def __init__(self, fget, doc=None):
self.fget = fget
self.__doc__ = doc or fget.__doc__
self.__name__ = fget.__name__
def __get__(self, obj, cls):
if obj is None:
return self
obj.__dict__[self.__name__] = result = self.fget(obj)
return result
def to_list(x, default=None):
"""Coerce to a list."""
if x is None:
return default
if not isinstance(x, (list, tuple)):
return [x]
else:
return x
class KeyReentrantMutex(object):
def __init__(self, key, mutex, keys):
self.key = key
self.mutex = mutex
self.keys = keys
@classmethod
def factory(cls, mutex):
# this collection holds zero or one
# thread idents as the key; a set of
# keynames held as the value.
keystore = collections.defaultdict(set)
def fac(key):
return KeyReentrantMutex(key, mutex, keystore)
return fac
def acquire(self, wait=True):
current_thread = compat.threading.current_thread().ident
keys = self.keys.get(current_thread)
if keys is not None and \
self.key not in keys:
# current lockholder, new key. add it in
keys.add(self.key)
return True
elif self.mutex.acquire(wait=wait):
# after acquire, create new set and add our key
self.keys[current_thread].add(self.key)
return True
else:
return False
def release(self):
current_thread = compat.threading.current_thread().ident
keys = self.keys.get(current_thread)
assert keys is not None, "this thread didn't do the acquire"
assert self.key in keys, "No acquire held for key '%s'" % self.key
keys.remove(self.key)
if not keys:
# when list of keys empty, remove
# the thread ident and unlock.
del self.keys[current_thread]
self.mutex.release()
@@ -1,11 +0,0 @@
from .dogpile import NeedRegenerationException, Lock
from .nameregistry import NameRegistry
from .readwrite_lock import ReadWriteMutex
from .legacy import Dogpile, SyncReaderDogpile
__all__ = [
'Dogpile', 'SyncReaderDogpile', 'NeedRegenerationException',
'NameRegistry', 'ReadWriteMutex', 'Lock']
__version__ = '0.4.1'
@@ -1,162 +0,0 @@
import time
import logging
log = logging.getLogger(__name__)
class NeedRegenerationException(Exception):
"""An exception that when raised in the 'with' block,
forces the 'has_value' flag to False and incurs a
regeneration of the value.
"""
NOT_REGENERATED = object()
class Lock(object):
"""Dogpile lock class.
Provides an interface around an arbitrary mutex
that allows one thread/process to be elected as
the creator of a new value, while other threads/processes
continue to return the previous version
of that value.
.. versionadded:: 0.4.0
The :class:`.Lock` class was added as a single-use object
representing the dogpile API without dependence on
any shared state between multiple instances.
:param mutex: A mutex object that provides ``acquire()``
and ``release()`` methods.
:param creator: Callable which returns a tuple of the form
(new_value, creation_time). "new_value" should be a newly
generated value representing completed state. "creation_time"
should be a floating point time value which is relative
to Python's ``time.time()`` call, representing the time
at which the value was created. This time value should
be associated with the created value.
:param value_and_created_fn: Callable which returns
a tuple of the form (existing_value, creation_time). This
basically should return what the last local call to the ``creator()``
callable has returned, i.e. the value and the creation time,
which would be assumed here to be from a cache. If the
value is not available, the :class:`.NeedRegenerationException`
exception should be thrown.
:param expiretime: Expiration time in seconds. Set to
``None`` for never expires. This timestamp is compared
to the creation_time result and ``time.time()`` to determine if
the value returned by value_and_created_fn is "expired".
:param async_creator: A callable. If specified, this callable will be
passed the mutex as an argument and is responsible for releasing the mutex
after it finishes some asynchronous value creation. The intent is for
this to be used to defer invocation of the creator callable until some
later time.
.. versionadded:: 0.4.1 added the async_creator argument.
"""
def __init__(self,
mutex,
creator,
value_and_created_fn,
expiretime,
async_creator=None,
):
self.mutex = mutex
self.creator = creator
self.value_and_created_fn = value_and_created_fn
self.expiretime = expiretime
self.async_creator = async_creator
def _is_expired(self, createdtime):
"""Return true if the expiration time is reached, or no
value is available."""
return not self._has_value(createdtime) or \
(
self.expiretime is not None and
time.time() - createdtime > self.expiretime
)
def _has_value(self, createdtime):
"""Return true if the creation function has proceeded
at least once."""
return createdtime > 0
def _enter(self):
value_fn = self.value_and_created_fn
try:
value = value_fn()
value, createdtime = value
except NeedRegenerationException:
log.debug("NeedRegenerationException")
value = NOT_REGENERATED
createdtime = -1
generated = self._enter_create(createdtime)
if generated is not NOT_REGENERATED:
generated, createdtime = generated
return generated
elif value is NOT_REGENERATED:
try:
value, createdtime = value_fn()
return value
except NeedRegenerationException:
raise Exception("Generation function should "
"have just been called by a concurrent "
"thread.")
else:
return value
def _enter_create(self, createdtime):
if not self._is_expired(createdtime):
return NOT_REGENERATED
async = False
if self._has_value(createdtime):
if not self.mutex.acquire(False):
log.debug("creation function in progress "
"elsewhere, returning")
return NOT_REGENERATED
else:
log.debug("no value, waiting for create lock")
self.mutex.acquire()
try:
log.debug("value creation lock %r acquired" % self.mutex)
# see if someone created the value already
try:
value, createdtime = self.value_and_created_fn()
except NeedRegenerationException:
pass
else:
if not self._is_expired(createdtime):
log.debug("value already present")
return value, createdtime
elif self.async_creator:
log.debug("Passing creation lock to async runner")
self.async_creator(self.mutex)
async = True
return value, createdtime
log.debug("Calling creation function")
created = self.creator()
return created
finally:
if not async:
self.mutex.release()
log.debug("Released creation lock")
def __enter__(self):
return self._enter()
def __exit__(self, type, value, traceback):
pass
@@ -1,154 +0,0 @@
from __future__ import with_statement
from .util import threading
from .readwrite_lock import ReadWriteMutex
from .dogpile import Lock
import time
import contextlib
class Dogpile(object):
"""Dogpile lock class.
.. deprecated:: 0.4.0
The :class:`.Lock` object specifies the full
API of the :class:`.Dogpile` object in a single way,
rather than providing multiple modes of usage which
don't necessarily work in the majority of cases.
:class:`.Dogpile` is now a wrapper around the :class:`.Lock` object
which provides dogpile.core's original usage pattern.
This usage pattern began as something simple, but was
not of general use in real-world caching environments without
several extra complicating factors; the :class:`.Lock`
object presents the "real-world" API more succinctly,
and also fixes a cross-process concurrency issue.
:param expiretime: Expiration time in seconds. Set to
``None`` for never expires.
:param init: if True, set the 'createdtime' to the
current time.
:param lock: a mutex object that provides
``acquire()`` and ``release()`` methods.
"""
def __init__(self, expiretime, init=False, lock=None):
"""Construct a new :class:`.Dogpile`.
"""
if lock:
self.dogpilelock = lock
else:
self.dogpilelock = threading.Lock()
self.expiretime = expiretime
if init:
self.createdtime = time.time()
createdtime = -1
"""The last known 'creation time' of the value,
stored as an epoch (i.e. from ``time.time()``).
If the value here is -1, it is assumed the value
should recreate immediately.
"""
def acquire(self, creator,
value_fn=None,
value_and_created_fn=None):
"""Acquire the lock, returning a context manager.
:param creator: Creation function, used if this thread
is chosen to create a new value.
:param value_fn: Optional function that returns
the value from some datasource. Will be returned
if regeneration is not needed.
:param value_and_created_fn: Like value_fn, but returns a tuple
of (value, createdtime). The returned createdtime
will replace the "createdtime" value on this dogpile
lock. This option removes the need for the dogpile lock
itself to remain persistent across usages; another
dogpile can come along later and pick up where the
previous one left off.
"""
if value_and_created_fn is None:
if value_fn is None:
def value_and_created_fn():
return None, self.createdtime
else:
def value_and_created_fn():
return value_fn(), self.createdtime
def creator_wrapper():
value = creator()
self.createdtime = time.time()
return value, self.createdtime
else:
def creator_wrapper():
value = creator()
self.createdtime = time.time()
return value
return Lock(
self.dogpilelock,
creator_wrapper,
value_and_created_fn,
self.expiretime
)
@property
def is_expired(self):
"""Return true if the expiration time is reached, or no
value is available."""
return not self.has_value or \
(
self.expiretime is not None and
time.time() - self.createdtime > self.expiretime
)
@property
def has_value(self):
"""Return true if the creation function has proceeded
at least once."""
return self.createdtime > 0
class SyncReaderDogpile(Dogpile):
"""Provide a read-write lock function on top of the :class:`.Dogpile`
class.
.. deprecated:: 0.4.0
The :class:`.ReadWriteMutex` object can be used directly.
"""
def __init__(self, *args, **kw):
super(SyncReaderDogpile, self).__init__(*args, **kw)
self.readwritelock = ReadWriteMutex()
@contextlib.contextmanager
def acquire_write_lock(self):
"""Return the "write" lock context manager.
This will provide a section that is mutexed against
all readers/writers for the dogpile-maintained value.
"""
self.readwritelock.acquire_write_lock()
try:
yield
finally:
self.readwritelock.release_write_lock()
@contextlib.contextmanager
def acquire(self, *arg, **kw):
with super(SyncReaderDogpile, self).acquire(*arg, **kw) as value:
self.readwritelock.acquire_read_lock()
try:
yield value
finally:
self.readwritelock.release_read_lock()
@@ -1,83 +0,0 @@
from .util import threading
import weakref
class NameRegistry(object):
"""Generates and return an object, keeping it as a
singleton for a certain identifier for as long as its
strongly referenced.
e.g.::
class MyFoo(object):
"some important object."
def __init__(self, identifier):
self.identifier = identifier
registry = NameRegistry(MyFoo)
# thread 1:
my_foo = registry.get("foo1")
# thread 2
my_foo = registry.get("foo1")
Above, ``my_foo`` in both thread #1 and #2 will
be *the same object*. The constructor for
``MyFoo`` will be called once, passing the
identifier ``foo1`` as the argument.
When thread 1 and thread 2 both complete or
otherwise delete references to ``my_foo``, the
object is *removed* from the :class:`.NameRegistry` as
a result of Python garbage collection.
:param creator: A function that will create a new
value, given the identifier passed to the :meth:`.NameRegistry.get`
method.
"""
_locks = weakref.WeakValueDictionary()
_mutex = threading.RLock()
def __init__(self, creator):
"""Create a new :class:`.NameRegistry`.
"""
self._values = weakref.WeakValueDictionary()
self._mutex = threading.RLock()
self.creator = creator
def get(self, identifier, *args, **kw):
"""Get and possibly create the value.
:param identifier: Hash key for the value.
If the creation function is called, this identifier
will also be passed to the creation function.
:param \*args, \**kw: Additional arguments which will
also be passed to the creation function if it is
called.
"""
try:
if identifier in self._values:
return self._values[identifier]
else:
return self._sync_get(identifier, *args, **kw)
except KeyError:
return self._sync_get(identifier, *args, **kw)
def _sync_get(self, identifier, *args, **kw):
self._mutex.acquire()
try:
try:
if identifier in self._values:
return self._values[identifier]
else:
self._values[identifier] = value = self.creator(identifier, *args, **kw)
return value
except KeyError:
self._values[identifier] = value = self.creator(identifier, *args, **kw)
return value
finally:
self._mutex.release()
@@ -1,130 +0,0 @@
from .util import threading
import logging
log = logging.getLogger(__name__)
class LockError(Exception):
pass
class ReadWriteMutex(object):
"""A mutex which allows multiple readers, single writer.
:class:`.ReadWriteMutex` uses a Python ``threading.Condition``
to provide this functionality across threads within a process.
The Beaker package also contained a file-lock based version
of this concept, so that readers/writers could be synchronized
across processes with a common filesystem. A future Dogpile
release may include this additional class at some point.
"""
def __init__(self):
# counts how many asynchronous methods are executing
self.async = 0
# pointer to thread that is the current sync operation
self.current_sync_operation = None
# condition object to lock on
self.condition = threading.Condition(threading.Lock())
def acquire_read_lock(self, wait = True):
"""Acquire the 'read' lock."""
self.condition.acquire()
try:
# see if a synchronous operation is waiting to start
# or is already running, in which case we wait (or just
# give up and return)
if wait:
while self.current_sync_operation is not None:
self.condition.wait()
else:
if self.current_sync_operation is not None:
return False
self.async += 1
log.debug("%s acquired read lock", self)
finally:
self.condition.release()
if not wait:
return True
def release_read_lock(self):
"""Release the 'read' lock."""
self.condition.acquire()
try:
self.async -= 1
# check if we are the last asynchronous reader thread
# out the door.
if self.async == 0:
# yes. so if a sync operation is waiting, notifyAll to wake
# it up
if self.current_sync_operation is not None:
self.condition.notifyAll()
elif self.async < 0:
raise LockError("Synchronizer error - too many "
"release_read_locks called")
log.debug("%s released read lock", self)
finally:
self.condition.release()
def acquire_write_lock(self, wait = True):
"""Acquire the 'write' lock."""
self.condition.acquire()
try:
# here, we are not a synchronous reader, and after returning,
# assuming waiting or immediate availability, we will be.
if wait:
# if another sync is working, wait
while self.current_sync_operation is not None:
self.condition.wait()
else:
# if another sync is working,
# we dont want to wait, so forget it
if self.current_sync_operation is not None:
return False
# establish ourselves as the current sync
# this indicates to other read/write operations
# that they should wait until this is None again
self.current_sync_operation = threading.currentThread()
# now wait again for asyncs to finish
if self.async > 0:
if wait:
# wait
self.condition.wait()
else:
# we dont want to wait, so forget it
self.current_sync_operation = None
return False
log.debug("%s acquired write lock", self)
finally:
self.condition.release()
if not wait:
return True
def release_write_lock(self):
"""Release the 'write' lock."""
self.condition.acquire()
try:
if self.current_sync_operation is not threading.currentThread():
raise LockError("Synchronizer error - current thread doesn't "
"have the write lock")
# reset the current sync operation so
# another can get it
self.current_sync_operation = None
# tell everyone to get ready
self.condition.notifyAll()
log.debug("%s released write lock", self)
finally:
# everyone go !!
self.condition.release()
@@ -1,8 +0,0 @@
import sys
py3k = sys.version_info >= (3, 0)
try:
import threading
except ImportError:
import dummy_threading as threading
+2 -1
View File
@@ -369,7 +369,8 @@ class Chapter(object):
if chapterdisplays:
string = chapterdisplays[0].get('ChapString')
language = chapterdisplays[0].get('ChapLanguage')
return cls(start, hidden, enabled, end, string, language)
return cls(start, hidden, enabled, end, string, language)
return cls(start, hidden, enabled, end)
def __repr__(self):
return '<%s [%s, enabled=%s]>' % (self.__class__.__name__, self.start, self.enabled)
@@ -1,165 +0,0 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
@@ -1,227 +0,0 @@
GuessIt
=======
.. image:: http://img.shields.io/pypi/v/guessit.svg
:target: https://pypi.python.org/pypi/guessit
:alt: Latest Version
.. image:: http://img.shields.io/badge/license-LGPLv3-blue.svg
:target: https://pypi.python.org/pypi/guessit
:alt: License
.. image:: http://img.shields.io/travis/wackou/guessit.svg?branch=master
:target: http://travis-ci.org/wackou/guessit
:alt: Build Status
.. image:: http://img.shields.io/coveralls/wackou/guessit.svg?branch=master
:target: https://coveralls.io/r/wackou/guessit
:alt: Coveralls
`HuBoard <https://huboard.com/wackou/guessit>`_
GuessIt is a python library that extracts as much information as
possible from a video file.
It has a very powerful filename matcher that allows to guess a lot of
metadata from a video using its filename only. This matcher works with
both movies and tv shows episodes.
For example, GuessIt can do the following::
$ guessit "Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.avi"
For: Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.avi
GuessIt found: {
[1.00] "mimetype": "video/x-msvideo",
[0.80] "episodeNumber": 3,
[0.80] "videoCodec": "XviD",
[1.00] "container": "avi",
[1.00] "format": "HDTV",
[0.70] "series": "Treme",
[0.50] "title": "Right Place, Wrong Time",
[0.80] "releaseGroup": "NoTV",
[0.80] "season": 1,
[1.00] "type": "episode"
}
Install
-------
Installing GuessIt is simple with `pip <http://www.pip-installer.org/>`_::
$ pip install guessit
or, with `easy_install <http://pypi.python.org/pypi/setuptools>`_::
$ easy_install guessit
But, you really `shouldn't do that <http://stackoverflow.com/questions/3220404/why-use-pip-over-easy-install>`_.
You can now launch a demo::
$ guessit -d
and guess your own filename::
$ guessit "Breaking.Bad.S05E08.720p.MP4.BDRip.[KoTuWa].mkv"
For: Breaking.Bad.S05E08.720p.MP4.BDRip.[KoTuWa].mkv
GuessIt found: {
[1.00] "mimetype": "video/x-matroska",
[1.00] "episodeNumber": 8,
[0.30] "container": "mkv",
[1.00] "format": "BluRay",
[0.70] "series": "Breaking Bad",
[1.00] "releaseGroup": "KoTuWa",
[1.00] "screenSize": "720p",
[1.00] "season": 5,
[1.00] "type": "episode"
}
Filename matcher
----------------
The filename matcher is based on pattern matching and is able to recognize many properties from the filename,
like ``title``, ``year``, ``series``, ``episodeNumber``, ``seasonNumber``,
``videoCodec``, ``screenSize``, ``language``. Guessed values are cleaned up and given in a readable format
which may not match exactly the raw filename.
The full list of available properties can be seen in the
`main documentation <http://guessit.readthedocs.org/en/latest/user/properties.html>`_.
Other features
--------------
GuessIt also allows you to compute a whole lot of hashes from a file,
namely all the ones you can find in the hashlib python module (md5,
sha1, ...), but also the Media Player Classic hash that is used (amongst
others) by OpenSubtitles and SMPlayer, as well as the ed2k hash.
If you have the 'guess-language' python package installed, GuessIt can also
analyze a subtitle file's contents and detect which language it is written in.
If you have the 'enzyme' python package installed, GuessIt can also detect the
properties from the actual video file metadata.
Usage
-----
guessit can be use from command line::
$ guessit
usage: guessit [-h] [-t TYPE] [-n] [-c] [-X DISABLED_TRANSFORMERS] [-v]
[-P SHOW_PROPERTY] [-u] [-a] [-y] [-f INPUT_FILE] [-d] [-p]
[-V] [-s] [--version] [-b] [-i INFO] [-S EXPECTED_SERIES]
[-T EXPECTED_TITLE] [-Y] [-D] [-L ALLOWED_LANGUAGES] [-E]
[-C ALLOWED_COUNTRIES] [-G EXPECTED_GROUP]
[filename [filename ...]]
positional arguments:
filename Filename or release name to guess
optional arguments:
-h, --help show this help message and exit
Naming:
-t TYPE, --type TYPE The suggested file type: movie, episode. If undefined,
type will be guessed.
-n, --name-only Parse files as name only. Disable folder parsing,
extension parsing, and file content analysis.
-c, --split-camel Split camel case part of filename.
-X DISABLED_TRANSFORMERS, --disabled-transformer DISABLED_TRANSFORMERS
Transformer to disable (can be used multiple time)
-S EXPECTED_SERIES, --expected-series EXPECTED_SERIES
Expected series to parse (can be used multiple times)
-T EXPECTED_TITLE, --expected-title EXPECTED_TITLE
Expected title (can be used multiple times)
-Y, --date-year-first
If short date is found, consider the first digits as
the year.
-D, --date-day-first If short date is found, consider the second digits as
the day.
-L ALLOWED_LANGUAGES, --allowed-languages ALLOWED_LANGUAGES
Allowed language (can be used multiple times)
-E, --episode-prefer-number
Guess "serie.213.avi" as the episodeNumber 213.
Without this option, it will be guessed as season 2,
episodeNumber 13
-C ALLOWED_COUNTRIES, --allowed-country ALLOWED_COUNTRIES
Allowed country (can be used multiple times)
-G EXPECTED_GROUP, --expected-group EXPECTED_GROUP
Expected release group (can be used multiple times)
Output:
-v, --verbose Display debug output
-P SHOW_PROPERTY, --show-property SHOW_PROPERTY
Display the value of a single property (title, series,
videoCodec, year, type ...)
-u, --unidentified Display the unidentified parts.
-a, --advanced Display advanced information for filename guesses, as
json output
-y, --yaml Display information for filename guesses as yaml
output (like unit-test)
-f INPUT_FILE, --input-file INPUT_FILE
Read filenames from an input file.
-d, --demo Run a few builtin tests instead of analyzing a file
Information:
-p, --properties Display properties that can be guessed.
-V, --values Display property values that can be guessed.
-s, --transformers Display transformers that can be used.
--version Display the guessit version.
guessit.io:
-b, --bug Submit a wrong detection to the guessit.io service
Other features:
-i INFO, --info INFO The desired information type: filename, video,
hash_mpc or a hash from python's hashlib module, such
as hash_md5, hash_sha1, ...; or a list of any of them,
comma-separated
It can also be used as a python module::
>>> from guessit import guess_file_info
>>> guess_file_info('Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.avi')
{u'mimetype': 'video/x-msvideo', u'episodeNumber': 3, u'videoCodec': u'XviD', u'container': u'avi', u'format': u'HDTV', u'series': u'Treme', u'title': u'Right Place, Wrong Time', u'releaseGroup': u'NoTV', u'season': 1, u'type': u'episode'}
Support
-------
The project website for GuessIt is hosted at `ReadTheDocs <http://guessit.readthedocs.org/>`_.
There you will also find the User guide and Developer documentation.
This project is hosted on GitHub: `<https://github.com/wackou/guessit>`_
Please report issues and/or feature requests via the `bug tracker <https://github.com/wackou/guessit/issues>`_.
You can also report issues using the command-line tool::
$ guessit --bug "filename.that.fails.avi"
Contribute
----------
GuessIt is under active development, and contributions are more than welcome!
#. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug.
There is a Contributor Friendly tag for issues that should be ideal for people who are not very
familiar with the codebase yet.
#. Fork `the repository`_ on Github to start making your changes to the **master**
branch (or branch off of it).
#. Write a test which shows that the bug was fixed or that the feature works as expected.
#. Send a pull request and bug the maintainer until it gets merged and published. :)
.. _the repository: https://github.com/wackou/guessit
License
-------
GuessIt is licensed under the `LGPLv3 license <http://www.gnu.org/licenses/lgpl.html>`_.
@@ -1,370 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from .__version__ import __version__
__all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info',
'default_options']
# Do python3 detection before importing any other module, to be sure that
# it will then always be available
# with code from http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
import sys
if sys.version_info[0] >= 3: # pragma: no cover
PY2, PY3 = False, True
unicode_text_type = str
native_text_type = str
base_text_type = str
def u(x):
return str(x)
def s(x):
return x
class UnicodeMixin(object):
__str__ = lambda x: x.__unicode__()
import binascii
def to_hex(x):
return binascii.hexlify(x).decode('utf-8')
else: # pragma: no cover
PY2, PY3 = True, False
__all__ = [str(s) for s in __all__] # fix imports for python2
unicode_text_type = unicode
native_text_type = str
base_text_type = basestring
def u(x):
if isinstance(x, str):
return x.decode('utf-8')
if isinstance(x, list):
return [u(s) for s in x]
return unicode(x)
def s(x):
if isinstance(x, unicode):
return x.encode('utf-8')
if isinstance(x, list):
return [s(y) for y in x]
if isinstance(x, tuple):
return tuple(s(y) for y in x)
if isinstance(x, dict):
return dict((s(key), s(value)) for key, value in x.items())
return x
class UnicodeMixin(object):
__str__ = lambda x: unicode(x).encode('utf-8')
def to_hex(x):
return x.encode('hex')
range = xrange
from guessit.guess import Guess, smart_merge
from guessit.language import Language
from guessit.matcher import IterativeMatcher
from guessit.textutils import clean_default, is_camel, from_camel
from copy import deepcopy
import babelfish
import os.path
import logging
from guessit.options import get_opts
import shlex
# Needed for guessit.plugins.transformers.reload() to be called.
from guessit.plugins import transformers
log = logging.getLogger(__name__)
class NullHandler(logging.Handler):
def emit(self, record):
pass
# let's be a nicely behaving library
h = NullHandler()
log.addHandler(h)
def _guess_filename(filename, options=None, **kwargs):
mtree = _build_filename_mtree(filename, options=options, **kwargs)
if options.get('split_camel'):
_add_camel_properties(mtree, options=options)
return mtree.matched()
def _build_filename_mtree(filename, options=None, **kwargs):
mtree = IterativeMatcher(filename, options=options, **kwargs)
second_pass_options = mtree.second_pass_options
if second_pass_options:
log.debug('Running 2nd pass with options: %s' % second_pass_options)
merged_options = dict(options)
merged_options.update(second_pass_options)
mtree = IterativeMatcher(filename, options=merged_options, **kwargs)
return mtree
def _add_camel_properties(mtree, options=None, **kwargs):
prop = 'title' if mtree.matched().get('type') != 'episode' else 'series'
value = mtree.matched().get(prop)
_guess_camel_string(mtree, value, options=options, skip_title=False, **kwargs)
for leaf in mtree.match_tree.unidentified_leaves():
value = leaf.value
_guess_camel_string(mtree, value, options=options, skip_title=True, **kwargs)
def _guess_camel_string(mtree, string, options=None, skip_title=False, **kwargs):
if string and is_camel(string):
log.debug('"%s" is camel cased. Try to detect more properties.' % (string,))
uncameled_value = from_camel(string)
merged_options = dict(options)
if 'type' in mtree.match_tree.info:
current_type = mtree.match_tree.info.get('type')
if current_type and current_type != 'unknown':
merged_options['type'] = current_type
camel_tree = _build_filename_mtree(uncameled_value, options=merged_options, name_only=True, skip_title=skip_title, **kwargs)
if len(camel_tree.matched()) > 0:
mtree.matched().update(camel_tree.matched())
return True
return False
def guess_video_metadata(filename):
"""Gets the video metadata properties out of a given file. The file needs to
exist on the filesystem to be able to be analyzed. An empty guess is
returned otherwise.
You need to have the Enzyme python package installed for this to work."""
result = Guess()
def found(prop, value):
result[prop] = value
log.debug('Found with enzyme %s: %s' % (prop, value))
# first get the size of the file, in bytes
try:
size = os.stat(filename).st_size
found('fileSize', size)
except Exception as e:
log.error('Cannot get video file size: %s' % e)
# file probably does not exist, we might as well return now
return result
# then get additional metadata from the file using enzyme, if available
try:
import enzyme
with open(filename) as f:
mkv = enzyme.MKV(f)
found('duration', mkv.info.duration.total_seconds())
if mkv.video_tracks:
video_track = mkv.video_tracks[0]
# resolution
if video_track.height in (480, 720, 1080):
if video_track.interlaced:
found('screenSize', '%di' % video_track.height)
else:
found('screenSize', '%dp' % video_track.height)
else:
# TODO: do we want this?
#found('screenSize', '%dx%d' % (video_track.width, video_track.height))
pass
# video codec
if video_track.codec_id == 'V_MPEG4/ISO/AVC':
found('videoCodec', 'h264')
elif video_track.codec_id == 'V_MPEG4/ISO/SP':
found('videoCodec', 'DivX')
elif video_track.codec_id == 'V_MPEG4/ISO/ASP':
found('videoCodec', 'XviD')
else:
log.warning('MKV has no video track')
if mkv.audio_tracks:
audio_track = mkv.audio_tracks[0]
# audio codec
if audio_track.codec_id == 'A_AC3':
found('audioCodec', 'AC3')
elif audio_track.codec_id == 'A_DTS':
found('audioCodec', 'DTS')
elif audio_track.codec_id == 'A_AAC':
found('audioCodec', 'AAC')
else:
log.warning('MKV has no audio track')
if mkv.subtitle_tracks:
embedded_subtitle_languages = set()
for st in mkv.subtitle_tracks:
try:
if st.language:
lang = babelfish.Language.fromalpha3b(st.language)
elif st.name:
lang = babelfish.Language.fromname(st.name)
else:
lang = babelfish.Language('und')
except babelfish.Error:
lang = babelfish.Language('und')
embedded_subtitle_languages.add(lang)
found('subtitleLanguage', embedded_subtitle_languages)
else:
log.debug('MKV has no subtitle track')
return result
except ImportError:
log.error('Cannot get video file metadata, missing dependency: enzyme')
log.error('Please install it from PyPI, by doing eg: pip install enzyme')
return result
except IOError as e:
log.error('Could not open file: %s' % filename)
log.error('Make sure it exists and is available for reading on the filesystem')
log.error('Error: %s' % e)
return result
except enzyme.Error as e:
log.error('Cannot guess video file metadata')
log.error('enzyme.Error while reading file: %s' % filename)
log.error('Error: %s' % e)
return result
default_options = {}
def guess_file_info(filename, info=None, options=None, **kwargs):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
>>> testfile = os.path.join(os.path.dirname(__file__), 'test/dummy.srt')
>>> g = guess_file_info(testfile, info = ['hash_md5', 'hash_sha1'])
>>> g['hash_md5'], g['hash_sha1']
('64de6b5893cac24456c46a935ef9c359', 'a703fc0fa4518080505809bf562c6fc6f7b3c98c')
"""
info = info or 'filename'
options = options or {}
if isinstance(options, base_text_type):
args = shlex.split(options)
options = vars(get_opts().parse_args(args))
if default_options:
if isinstance(default_options, base_text_type):
default_args = shlex.split(default_options)
merged_options = vars(get_opts().parse_args(default_args))
else:
merged_options = deepcopy(default_options)
merged_options.update(options)
options = merged_options
result = []
hashers = []
# Force unicode as soon as possible
filename = u(filename)
if isinstance(info, base_text_type):
info = [info]
for infotype in info:
if infotype == 'filename':
result.append(_guess_filename(filename, options, **kwargs))
elif infotype == 'hash_mpc':
from guessit.hash_mpc import hash_file
try:
result.append(Guess({infotype: hash_file(filename)},
confidence=1.0))
except Exception as e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
from guessit.hash_ed2k import hash_file
try:
result.append(Guess({infotype: hash_file(filename)},
confidence=1.0))
except Exception as e:
log.warning('Could not compute ed2k hash because: %s' % e)
elif infotype.startswith('hash_'):
import hashlib
hashname = infotype[5:]
try:
hasher = getattr(hashlib, hashname)()
hashers.append((infotype, hasher))
except AttributeError:
log.warning('Could not compute %s hash because it is not available from python\'s hashlib module' % hashname)
elif infotype == 'video':
g = guess_video_metadata(filename)
if g:
result.append(g)
else:
log.warning('Invalid infotype: %s' % infotype)
# do all the hashes now, but on a single pass
if hashers:
try:
blocksize = 8192
hasherobjs = dict(hashers).values()
with open(filename, 'rb') as f:
chunk = f.read(blocksize)
while chunk:
for hasher in hasherobjs:
hasher.update(chunk)
chunk = f.read(blocksize)
for infotype, hasher in hashers:
result.append(Guess({infotype: hasher.hexdigest()},
confidence=1.0))
except Exception as e:
log.warning('Could not compute hash because: %s' % e)
result = smart_merge(result)
return result
def guess_video_info(filename, info=None, options=None, **kwargs):
return guess_file_info(filename, info=info, options=options, type='video', **kwargs)
def guess_movie_info(filename, info=None, options=None, **kwargs):
return guess_file_info(filename, info=info, options=options, type='movie', **kwargs)
def guess_episode_info(filename, info=None, options=None, **kwargs):
return guess_file_info(filename, info=info, options=options, type='episode', **kwargs)
@@ -1,285 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from collections import defaultdict
import logging
import os
from guessit import PY2, u, guess_file_info
from guessit.options import get_opts
from guessit.__version__ import __version__
def guess_file(filename, info='filename', options=None, **kwargs):
options = options or {}
filename = u(filename)
if not options.get('yaml') and not options.get('show_property'):
print('For:', filename)
guess = guess_file_info(filename, info, options, **kwargs)
if not options.get('unidentified'):
try:
del guess['unidentified']
except KeyError:
pass
if options.get('show_property'):
print(guess.get(options.get('show_property'), ''))
return
if options.get('yaml'):
import yaml
for k, v in guess.items():
if isinstance(v, list) and len(v) == 1:
guess[k] = v[0]
ystr = yaml.safe_dump({filename: dict(guess)}, default_flow_style=False, allow_unicode=True)
i = 0
for yline in ystr.splitlines():
if i == 0:
print("? " + yline[:-1])
elif i == 1:
print(":" + yline[1:])
else:
print(yline)
i += 1
return
print('GuessIt found:', guess.nice_string(options.get('advanced')))
def _supported_properties():
all_properties = defaultdict(list)
transformers_properties = []
from guessit.plugins import transformers
for transformer in transformers.all_transformers():
supported_properties = transformer.supported_properties()
transformers_properties.append((transformer, supported_properties))
if isinstance(supported_properties, dict):
for property_name, possible_values in supported_properties.items():
all_properties[property_name].extend(possible_values)
else:
for property_name in supported_properties:
all_properties[property_name] # just make sure it exists
return all_properties, transformers_properties
def display_transformers():
print('GuessIt transformers:')
_, transformers_properties = _supported_properties()
for transformer, _ in transformers_properties:
print('[@] %s (%s)' % (transformer.name, transformer.priority))
def display_properties(options):
values = options.values
transformers = options.transformers
name_only = options.name_only
print('GuessIt properties:')
all_properties, transformers_properties = _supported_properties()
if name_only:
# the 'container' property does not apply when using the --name-only
# option
del all_properties['container']
if transformers:
for transformer, properties_list in transformers_properties:
print('[@] %s (%s)' % (transformer.name, transformer.priority))
for property_name in properties_list:
property_values = all_properties.get(property_name)
print(' [+] %s' % (property_name,))
if property_values and values:
_display_property_values(property_name, indent=4)
else:
properties_list = sorted(all_properties.keys())
for property_name in properties_list:
property_values = all_properties.get(property_name)
print(' [+] %s' % (property_name,))
if property_values and values:
_display_property_values(property_name, indent=4)
def _display_property_values(property_name, indent=2):
all_properties, _ = _supported_properties()
property_values = all_properties.get(property_name)
for property_value in property_values:
print(indent * ' ' + '[!] %s' % (property_value,))
def run_demo(episodes=True, movies=True, options=None):
# NOTE: tests should not be added here but rather in the tests/ folder
# this is just intended as a quick example
if episodes:
testeps = ['Series/Californication/Season 2/Californication.2x05.Vaginatown.HDTV.XviD-0TV.[tvu.org.ru].avi',
'Series/dexter/Dexter.5x02.Hello,.Bandit.ENG.-.sub.FR.HDTV.XviD-AlFleNi-TeaM.[tvu.org.ru].avi',
'Series/Treme/Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.[tvu.org.ru].avi',
'Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi',
'Series/Duckman/Duckman - S1E13 Joking The Chicken (unedited).avi',
'Series/Simpsons/The_simpsons_s13e18_-_i_am_furious_yellow.mpg',
'Series/Simpsons/Saison 12 Français/Simpsons,.The.12x08.A.Bas.Le.Sergent.Skinner.FR.[tvu.org.ru].avi',
'Series/Dr._Slump_-_002_DVB-Rip_Catalan_by_kelf.avi',
'Series/Kaamelott/Kaamelott - Livre V - Second Volet - HD 704x396 Xvid 2 pass - Son 5.1 - TntRip by Slurm.avi']
for f in testeps:
print('-' * 80)
guess_file(f, options=options, type='episode')
if movies:
testmovies = ['Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv',
'Movies/El Dia de la Bestia (1995)/El.dia.de.la.bestia.DVDrip.Spanish.DivX.by.Artik[SEDG].avi',
'Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director\'s.Cut).CD1.DVDRip.XviD.AC3-WAF.avi',
'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv',
'Movies/Sin City (BluRay) (2005)/Sin.City.2005.BDRip.720p.x264.AC3-SEPTiC.mkv',
'Movies/Borat (2006)/Borat.(2006).R5.PROPER.REPACK.DVDRip.XviD-PUKKA.avi',
'[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv',
'Battle Royale (2000)/Battle.Royale.(Batoru.Rowaiaru).(2000).(Special.Edition).CD1of2.DVDRiP.XviD-[ZeaL].avi',
'Movies/Brazil (1985)/Brazil_Criterion_Edition_(1985).CD2.English.srt',
'Movies/Persepolis (2007)/[XCT] Persepolis [H264+Aac-128(Fr-Eng)+ST(Fr-Eng)+Ind].mkv',
'Movies/Toy Story (1995)/Toy Story [HDTV 720p English-Spanish].mkv',
'Movies/Pirates of the Caribbean: The Curse of the Black Pearl (2003)/Pirates.Of.The.Carribean.DC.2003.iNT.DVDRip.XviD.AC3-NDRT.CD1.avi',
'Movies/Office Space (1999)/Office.Space.[Dual-DVDRip].[Spanish-English].[XviD-AC3-AC3].[by.Oswald].avi',
'Movies/The NeverEnding Story (1984)/The.NeverEnding.Story.1.1984.DVDRip.AC3.Xvid-Monteque.avi',
'Movies/Juno (2007)/Juno KLAXXON.avi',
'Movies/Chat noir, chat blanc (1998)/Chat noir, Chat blanc - Emir Kusturica (VO - VF - sub FR - Chapters).mkv',
'Movies/Wild Zero (2000)/Wild.Zero.DVDivX-EPiC.srt',
'Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720x432].avi',
'testsmewt_bugs/movies/Baraka_Edition_Collector.avi'
]
for f in testmovies:
print('-' * 80)
guess_file(f, options=options, type='movie')
def submit_bug(filename, options):
import requests # only import when needed
from requests.exceptions import RequestException
try:
opts = dict((k, v) for k, v in options.__dict__.items()
if v and k != 'submit_bug')
r = requests.post('http://guessit.io/bugs', {'filename': filename,
'version': __version__,
'options': str(opts)})
if r.status_code == 200:
print('Successfully submitted file: %s' % r.text)
else:
print('Could not submit bug at the moment, please try again later: %s %s' % (r.status_code, r.reason))
except RequestException as e:
print('Could not submit bug at the moment, please try again later: %s' % e)
def main(args=None, setup_logging=True):
if setup_logging:
from guessit import slogging
slogging.setup_logging()
if PY2: # pragma: no cover
import codecs
import locale
import sys
# see http://bugs.python.org/issue2128
if os.name == 'nt':
for i, a in enumerate(sys.argv):
sys.argv[i] = a.decode(locale.getpreferredencoding())
# see https://github.com/wackou/guessit/issues/43
# and http://stackoverflow.com/questions/4545661/unicodedecodeerror-when-redirecting-to-file
# Wrap sys.stdout into a StreamWriter to allow writing unicode.
sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
# Needed for guessit.plugins.transformers.reload() to be called.
from guessit.plugins import transformers
if args:
options = get_opts().parse_args(args)
else: # pragma: no cover
options = get_opts().parse_args()
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
help_required = True
if options.properties or options.values:
display_properties(options)
help_required = False
elif options.transformers:
display_transformers()
help_required = False
if options.demo:
run_demo(episodes=True, movies=True, options=vars(options))
help_required = False
if options.version:
print('+-------------------------------------------------------+')
print('+ GuessIt ' + __version__ + (28-len(__version__)) * ' ' + '+')
print('+-------------------------------------------------------+')
print('| Please report any bug or feature request at |')
print('| https://github.com/wackou/guessit/issues. |')
print('+-------------------------------------------------------+')
help_required = False
if options.yaml:
try:
import yaml, babelfish
def default_representer(dumper, data):
return dumper.represent_str(str(data))
yaml.SafeDumper.add_representer(babelfish.Language, default_representer)
yaml.SafeDumper.add_representer(babelfish.Country, default_representer)
except ImportError: # pragma: no cover
print('PyYAML not found. Using default output.')
filenames = []
if options.filename:
filenames.extend(options.filename)
if options.input_file:
input_file = open(options.input_file, 'r')
try:
filenames.extend([line.strip() for line in input_file.readlines()])
finally:
input_file.close()
filenames = filter(lambda f: f, filenames)
if filenames:
if options.submit_bug:
for filename in filenames:
help_required = False
submit_bug(filename, options)
else:
for filename in filenames:
help_required = False
guess_file(filename,
info=options.info.split(','),
options=vars(options))
if help_required: # pragma: no cover
get_opts().print_help()
if __name__ == '__main__':
main()
@@ -1,20 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.11.1.dev0'
@@ -1,802 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import types
from .patterns import compile_pattern, sep
from . import base_text_type
from .guess import Guess
def _get_span(prop, match):
"""Retrieves span for a match"""
if not prop.global_span and match.re.groups:
start = None
end = None
for i in range(1, match.re.groups + 1):
span = match.span(i)
if start is None or span[0] < start:
start = span[0]
if end is None or span[1] > end:
end = span[1]
return start, end
else:
return match.span()
def _trim_span(span, value, blanks = sep):
start, end = span
for i in range(0, len(value)):
if value[i] in blanks:
start += 1
else:
break
for i in reversed(range(0, len(value))):
if value[i] in blanks:
end -= 1
else:
break
if end <= start:
return -1, -1
return start, end
def _get_groups(compiled_re):
"""
Retrieves groups from re
:return: list of group names
"""
if compiled_re.groups:
indexgroup = {}
for k, i in compiled_re.groupindex.items():
indexgroup[i] = k
ret = []
for i in range(1, compiled_re.groups + 1):
ret.append(indexgroup.get(i, i))
return ret
else:
return [None]
class NoValidator(object):
@staticmethod
def validate(prop, string, node, match, entry_start, entry_end):
return True
class LeftValidator(object):
"""Make sure our match is starting by separator, or by another entry"""
@staticmethod
def validate(prop, string, node, match, entry_start, entry_end):
span = _get_span(prop, match)
span = _trim_span(span, string[span[0]:span[1]])
start, end = span
sep_start = start <= 0 or string[start - 1] in sep
start_by_other = start in entry_end
if not sep_start and not start_by_other:
return False
return True
class RightValidator(object):
"""Make sure our match is ended by separator, or by another entry"""
@staticmethod
def validate(prop, string, node, match, entry_start, entry_end):
span = _get_span(prop, match)
span = _trim_span(span, string[span[0]:span[1]])
start, end = span
sep_end = end >= len(string) or string[end] in sep
end_by_other = end in entry_start
if not sep_end and not end_by_other:
return False
return True
class ChainedValidator(object):
def __init__(self, *validators):
self._validators = validators
def validate(self, prop, string, node, match, entry_start, entry_end):
for validator in self._validators:
if not validator.validate(prop, string, node, match, entry_start, entry_end):
return False
return True
class SameKeyValidator(object):
def __init__(self, validator_function):
self.validator_function = validator_function
def validate(self, prop, string, node, match, entry_start, entry_end):
path_nodes = [path_node for path_node in node.ancestors if path_node.category == 'path']
if path_nodes:
path_node = path_nodes[0]
else:
path_node = node.root
for key in prop.keys:
for same_value_leaf in path_node.leaves_containing(key):
ret = self.validator_function(same_value_leaf, key, prop, string, node, match, entry_start, entry_end)
if ret is not None:
return ret
return True
class OnlyOneValidator(SameKeyValidator):
"""
Check that there's only one occurence of key for current directory
"""
def __init__(self):
super(OnlyOneValidator, self).__init__(lambda same_value_leaf, key, prop, string, node, match, entry_start, entry_end: False)
class DefaultValidator(object):
"""Make sure our match is surrounded by separators, or by another entry"""
def validate(self, prop, string, node, match, entry_start, entry_end):
span = _get_span(prop, match)
span = _trim_span(span, string[span[0]:span[1]])
return DefaultValidator.validate_string(string, span, entry_start, entry_end)
@staticmethod
def validate_string(string, span, entry_start=None, entry_end=None):
start, end = span
sep_start = start <= 0 or string[start - 1] in sep
sep_end = end >= len(string) or string[end] in sep
start_by_other = start in entry_end if entry_end else False
end_by_other = end in entry_start if entry_start else False
if (sep_start or start_by_other) and (sep_end or end_by_other):
return True
return False
class FunctionValidator(object):
def __init__(self, function):
self.function = function
def validate(self, prop, string, node, match, entry_start, entry_end):
return self.function(prop, string, node, match, entry_start, entry_end)
class FormatterValidator(object):
def __init__(self, group_name=None, formatted_validator=None):
self.group_name = group_name
self.formatted_validator = formatted_validator
def validate(self, prop, string, node, match, entry_start, entry_end):
if self.group_name:
formatted = prop.format(match.group(self.group_name), self.group_name)
else:
formatted = prop.format(match.group())
if self.formatted_validator:
return self.formatted_validator(formatted)
else:
return formatted
def _get_positions(prop, string, node, match, entry_start, entry_end):
span = match.span()
start = span[0]
end = span[1]
at_start = True
at_end = True
while start > 0:
start -= 1
if string[start] not in sep:
at_start = False
break
while end < len(string) - 1:
end += 1
if string[end] not in sep:
at_end = False
break
return at_start, at_end
class WeakValidator(DefaultValidator):
"""Make sure our match is surrounded by separators and is the first or last element in the string"""
def validate(self, prop, string, node, match, entry_start, entry_end):
if super(WeakValidator, self).validate(prop, string, node, match, entry_start, entry_end):
at_start, at_end = _get_positions(prop, string, node, match, entry_start, entry_end)
return at_start or at_end
return False
class NeighborValidator(DefaultValidator):
"""Make sure the node is next another one"""
def validate(self, prop, string, node, match, entry_start, entry_end):
at_start, at_end = _get_positions(prop, string, node, match, entry_start, entry_end)
if at_start:
previous_leaf = node.root.previous_leaf(node)
if previous_leaf is not None:
return True
if at_end:
next_leaf = node.root.next_leaf(node)
if next_leaf is not None:
return True
return False
class FullMatchValidator(DefaultValidator):
"""Make sure the node match fully"""
def validate(self, prop, string, node, match, entry_start, entry_end):
at_start, at_end = _get_positions(prop, string, node, match, entry_start, entry_end)
return at_start and at_end
class LeavesValidator(DefaultValidator):
def __init__(self, lambdas=None, previous_lambdas=None, next_lambdas=None, both_side=False, default_=True):
self.previous_lambdas = previous_lambdas if previous_lambdas is not None else []
self.next_lambdas = next_lambdas if next_lambdas is not None else []
if lambdas:
self.previous_lambdas.extend(lambdas)
self.next_lambdas.extend(lambdas)
self.both_side = both_side
self.default_ = default_
"""Make sure our match is surrounded by separators and validates defined lambdas"""
def validate(self, prop, string, node, match, entry_start, entry_end):
if self.default_:
super_ret = super(LeavesValidator, self).validate(prop, string, node, match, entry_start, entry_end)
else:
super_ret = True
if not super_ret:
return False
previous_ = self._validate_previous(prop, string, node, match, entry_start, entry_end)
next_ = self._validate_next(prop, string, node, match, entry_start, entry_end)
if previous_ is None and next_ is None:
return super_ret
if self.both_side:
return previous_ and next_
else:
return previous_ or next_
def _validate_previous(self, prop, string, node, match, entry_start, entry_end):
if self.previous_lambdas:
for leaf in node.root.previous_leaves(node):
for lambda_ in self.previous_lambdas:
ret = self._check_rule(lambda_, leaf)
if ret is not None:
return ret
return False
def _validate_next(self, prop, string, node, match, entry_start, entry_end):
if self.next_lambdas:
for leaf in node.root.next_leaves(node):
for lambda_ in self.next_lambdas:
ret = self._check_rule(lambda_, leaf)
if ret is not None:
return ret
return False
@staticmethod
def _check_rule(lambda_, previous_leaf):
return lambda_(previous_leaf)
class _Property:
"""Represents a property configuration."""
def __init__(self, keys=None, pattern=None, canonical_form=None, canonical_from_pattern=True, confidence=1.0, enhance=True, global_span=False, validator=DefaultValidator(), formatter=None, disabler=None, confidence_lambda=None, remove_duplicates=False):
"""
:param keys: Keys of the property (format, screenSize, ...)
:type keys: string
:param canonical_form: Unique value of the property (DVD, 720p, ...)
:type canonical_form: string
:param pattern: Regexp pattern
:type pattern: string
:param confidence: confidence
:type confidence: float
:param enhance: enhance the pattern
:type enhance: boolean
:param global_span: if True, the whole match span will used to create the Guess.
Else, the span from the capturing groups will be used.
:type global_span: boolean
:param validator: Validator to use
:type validator: :class:`DefaultValidator`
:param formatter: Formater to use
:type formatter: function
:param remove_duplicates: Keep only the last match if multiple values are found
:type remove_duplicates: bool
"""
if isinstance(keys, list):
self.keys = keys
elif isinstance(keys, base_text_type):
self.keys = [keys]
else:
self.keys = []
self.canonical_form = canonical_form
if pattern is not None:
self.pattern = pattern
else:
self.pattern = canonical_form
if self.canonical_form is None and canonical_from_pattern:
self.canonical_form = self.pattern
self.compiled = compile_pattern(self.pattern, enhance=enhance)
for group_name in _get_groups(self.compiled):
if isinstance(group_name, base_text_type) and not group_name in self.keys:
self.keys.append(group_name)
if not self.keys:
raise ValueError("No property key is defined")
self.confidence = confidence
self.confidence_lambda = confidence_lambda
self.global_span = global_span
self.validator = validator
self.formatter = formatter
self.disabler = disabler
self.remove_duplicates = remove_duplicates
def disabled(self, options):
if self.disabler:
return self.disabler(options)
return False
def format(self, value, group_name=None):
"""Retrieves the final value from re group match value"""
formatter = None
if isinstance(self.formatter, dict):
formatter = self.formatter.get(group_name)
if formatter is None and group_name is not None:
formatter = self.formatter.get(None)
else:
formatter = self.formatter
if isinstance(formatter, types.FunctionType):
return formatter(value)
elif formatter is not None:
return formatter.format(value)
return value
def __repr__(self):
return "%s: %s" % (self.keys, self.canonical_form if self.canonical_form else self.pattern)
class PropertiesContainer(object):
def __init__(self, **kwargs):
self._properties = []
self.default_property_kwargs = kwargs
def unregister_property(self, name, *canonical_forms):
"""Unregister a property canonical forms
If canonical_forms are specified, only those values will be unregistered
:param name: Property name to unregister
:type name: string
:param canonical_forms: Values to unregister
:type canonical_forms: varargs of string
"""
_properties = [prop for prop in self._properties if prop.name == name and (not canonical_forms or prop.canonical_form in canonical_forms)]
def register_property(self, name, *patterns, **property_params):
"""Register property with defined canonical form and patterns.
:param name: name of the property (format, screenSize, ...)
:type name: string
:param patterns: regular expression patterns to register for the property canonical_form
:type patterns: varargs of string
"""
properties = []
for pattern in patterns:
params = dict(self.default_property_kwargs)
params.update(property_params)
if isinstance(pattern, dict):
params.update(pattern)
prop = _Property(name, **params)
else:
prop = _Property(name, pattern, **params)
self._properties.append(prop)
properties.append(prop)
return properties
def register_canonical_properties(self, name, *canonical_forms, **property_params):
"""Register properties from their canonical forms.
:param name: name of the property (releaseGroup, ...)
:type name: string
:param canonical_forms: values of the property ('ESiR', 'WAF', 'SEPTiC', ...)
:type canonical_forms: varargs of strings
"""
properties = []
for canonical_form in canonical_forms:
params = dict(property_params)
params['canonical_form'] = canonical_form
properties.extend(self.register_property(name, canonical_form, **property_params))
return properties
def unregister_all_properties(self):
"""Unregister all defined properties"""
self._properties.clear()
def find_properties(self, string, node, options, name=None, validate=True, re_match=False, sort=True, multiple=False):
"""Find all distinct properties for given string
If no capturing group is defined in the property, value will be grabbed from the entire match.
If one ore more unnamed capturing group is defined in the property, first capturing group will be used.
If named capturing group are defined in the property, they will be returned as property key.
If validate, found properties will be validated by their defined validator
If re_match, re.match will be used instead of re.search.
if sort, found properties will be sorted from longer match to shorter match.
If multiple is False and multiple values are found for the same property, the more confident one will be returned.
If multiple is False and multiple values are found for the same property and the same confidence, the longer will be returned.
:param string: input string
:type string: string
:param node: current node of the matching tree
:type node: :class:`guessit.matchtree.MatchTree`
:param name: name of property to find
:type name: string
:param re_match: use re.match instead of re.search
:type re_match: bool
:param multiple: Allows multiple property values to be returned
:type multiple: bool
:return: found properties
:rtype: list of tuples (:class:`_Property`, match, list of tuples (property_name, tuple(value_start, value_end)))
:see: `_Property`
:see: `register_property`
:see: `register_canonical_properties`
"""
entry_start = {}
entry_end = {}
entries = []
duplicate_matches = {}
ret = []
if not string.strip():
return ret
# search all properties
for prop in self.get_properties(name):
if not prop.disabled(options):
valid_match = None
if re_match:
match = prop.compiled.match(string)
if match:
entries.append((prop, match))
else:
matches = list(prop.compiled.finditer(string))
if prop.remove_duplicates:
duplicate_matches[prop] = matches
for match in matches:
entries.append((prop, match))
for prop, match in entries:
# compute confidence
if prop.confidence_lambda:
computed_confidence = prop.confidence_lambda(match)
if computed_confidence is not None:
prop.confidence = computed_confidence
entries.sort(key=lambda entry: -entry[0].confidence)
# sort entries, from most confident to less confident
if validate:
# compute entries start and ends
for prop, match in entries:
start, end = _get_span(prop, match)
if start not in entry_start:
entry_start[start] = [prop]
else:
entry_start[start].append(prop)
if end not in entry_end:
entry_end[end] = [prop]
else:
entry_end[end].append(prop)
# remove invalid values
while True:
invalid_entries = []
for entry in entries:
prop, match = entry
if not prop.validator.validate(prop, string, node, match, entry_start, entry_end):
invalid_entries.append(entry)
if not invalid_entries:
break
for entry in invalid_entries:
prop, match = entry
entries.remove(entry)
prop_duplicate_matches = duplicate_matches.get(prop)
if prop_duplicate_matches:
prop_duplicate_matches.remove(match)
invalid_span = _get_span(prop, match)
start = invalid_span[0]
end = invalid_span[1]
entry_start[start].remove(prop)
if not entry_start.get(start):
del entry_start[start]
entry_end[end].remove(prop)
if not entry_end.get(end):
del entry_end[end]
for prop, prop_duplicate_matches in duplicate_matches.items():
# Keeping the last valid match only.
# Needed for the.100.109.hdtv-lol.mp4
for duplicate_match in prop_duplicate_matches[:-1]:
entries.remove((prop, duplicate_match))
if multiple:
ret = entries
else:
# keep only best match if multiple values where found
entries_dict = {}
for entry in entries:
for key in prop.keys:
if key not in entries_dict:
entries_dict[key] = []
entries_dict[key].append(entry)
for key_entries in entries_dict.values():
if multiple:
for entry in key_entries:
ret.append(entry)
else:
best_ret = {}
best_prop, best_match = None, None
if len(key_entries) == 1:
best_prop, best_match = key_entries[0]
else:
for prop, match in key_entries:
start, end = _get_span(prop, match)
if not best_prop or \
best_prop.confidence < prop.confidence or \
best_prop.confidence == prop.confidence and \
best_match.span()[1] - best_match.span()[0] < match.span()[1] - match.span()[0]:
best_prop, best_match = prop, match
best_ret[best_prop] = best_match
for prop, match in best_ret.items():
ret.append((prop, match))
if sort:
def _sorting(x):
_, x_match = x
x_start, x_end = x_match.span()
return x_start - x_end
ret.sort(key=_sorting)
return ret
def as_guess(self, found_properties, input=None, filter_=None, sep_replacement=None, multiple=False, *args, **kwargs):
if filter_ is None:
filter_ = lambda property, *args, **kwargs: True
guesses = [] if multiple else None
for prop, match in found_properties:
first_key = None
for key in prop.keys:
# First property key will be used as base for effective name
if isinstance(key, base_text_type):
if first_key is None:
first_key = key
break
property_name = first_key if first_key else None
span = _get_span(prop, match)
guess = Guess(confidence=prop.confidence, input=input, span=span, prop=property_name)
groups = _get_groups(match.re)
for group_name in groups:
name = group_name if isinstance(group_name, base_text_type) else property_name if property_name not in groups else None
if name:
value = self._effective_prop_value(prop, group_name, input, match.span(group_name) if group_name else match.span(), sep_replacement)
if not value is None:
is_string = isinstance(value, base_text_type)
if not is_string or is_string and value: # Keep non empty strings and other defined objects
if isinstance(value, dict):
for k, v in value.items():
if k is None:
k = name
guess[k] = v
else:
if name in guess:
if not isinstance(guess[name], list):
guess[name] = [guess[name]]
guess[name].append(value)
else:
guess[name] = value
if group_name:
guess.metadata(prop).span = match.span(group_name)
if filter_(guess):
if multiple:
guesses.append(guess)
else:
return guess
return guesses
@staticmethod
def _effective_prop_value(prop, group_name, input=None, span=None, sep_replacement=None):
if prop.canonical_form:
return prop.canonical_form
if input is None:
return None
value = input
if span is not None:
value = value[span[0]:span[1]]
value = input[span[0]:span[1]] if input else None
if sep_replacement:
for sep_char in sep:
value = value.replace(sep_char, sep_replacement)
if value:
value = prop.format(value, group_name)
return value
def get_properties(self, name=None, canonical_form=None):
"""Retrieve properties
:return: Properties
:rtype: generator
"""
for prop in self._properties:
if (name is None or name in prop.keys) and (canonical_form is None or prop.canonical_form == canonical_form):
yield prop
def get_supported_properties(self):
supported_properties = {}
for prop in self.get_properties():
for k in prop.keys:
values = supported_properties.get(k)
if not values:
values = set()
supported_properties[k] = values
if prop.canonical_form:
values.add(prop.canonical_form)
return supported_properties
class QualitiesContainer():
def __init__(self):
self._qualities = {}
def register_quality(self, name, canonical_form, rating):
"""Register a quality rating.
:param name: Name of the property
:type name: string
:param canonical_form: Value of the property
:type canonical_form: string
:param rating: Estimated quality rating for the property
:type rating: int
"""
property_qualities = self._qualities.get(name)
if property_qualities is None:
property_qualities = {}
self._qualities[name] = property_qualities
property_qualities[canonical_form] = rating
def unregister_quality(self, name, *canonical_forms):
"""Unregister quality ratings for given property name.
If canonical_forms are specified, only those values will be unregistered
:param name: Name of the property
:type name: string
:param canonical_forms: Value of the property
:type canonical_forms: string
"""
if not canonical_forms:
if name in self._qualities:
del self._qualities[name]
else:
property_qualities = self._qualities.get(name)
if property_qualities is not None:
for property_canonical_form in canonical_forms:
if property_canonical_form in property_qualities:
del property_qualities[property_canonical_form]
if not property_qualities:
del self._qualities[name]
def clear_qualities(self,):
"""Unregister all defined quality ratings.
"""
self._qualities.clear()
def rate_quality(self, guess, *props):
"""Rate the quality of guess.
:param guess: Guess to rate
:type guess: :class:`guessit.guess.Guess`
:param props: Properties to include in the rating. if empty, rating will be performed for all guess properties.
:type props: varargs of string
:return: Quality of the guess. The higher, the better.
:rtype: int
"""
rate = 0
if not props:
props = guess.keys()
for prop in props:
prop_value = guess.get(prop)
prop_qualities = self._qualities.get(prop)
if prop_value is not None and prop_qualities is not None:
rate += prop_qualities.get(prop_value, 0)
return rate
def best_quality_properties(self, props, *guesses):
"""Retrieve the best quality guess, based on given properties
:param props: Properties to include in the rating
:type props: list of strings
:param guesses: Guesses to rate
:type guesses: :class:`guessit.guess.Guess`
:return: Best quality guess from all passed guesses
:rtype: :class:`guessit.guess.Guess`
"""
best_guess = None
best_rate = None
for guess in guesses:
rate = self.rate_quality(guess, *props)
if best_rate is None or best_rate < rate:
best_rate = rate
best_guess = guess
return best_guess
def best_quality(self, *guesses):
"""Retrieve the best quality guess.
:param guesses: Guesses to rate
:type guesses: :class:`guessit.guess.Guess`
:return: Best quality guess from all passed guesses
:rtype: :class:`guessit.guess.Guess`
"""
best_guess = None
best_rate = None
for guess in guesses:
rate = self.rate_quality(guess)
if best_rate is None or best_rate < rate:
best_rate = rate
best_guess = guess
return best_guess
@@ -1,128 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import datetime
import re
from dateutil import parser
_dsep = r'[-/ \.]'
_dsep_bis = r'[-/ \.x]'
date_regexps = [
re.compile('%s(\d{8})%s' % (_dsep, _dsep), re.IGNORECASE),
re.compile('%s(\d{6})%s' % (_dsep, _dsep), re.IGNORECASE),
re.compile('[^\d](\d{2})%s(\d{1,2})%s(\d{1,2})[^\d]' % (_dsep, _dsep), re.IGNORECASE),
re.compile('[^\d](\d{1,2})%s(\d{1,2})%s(\d{2})[^\d]' % (_dsep, _dsep), re.IGNORECASE),
re.compile('[^\d](\d{4})%s(\d{1,2})%s(\d{1,2})[^\d]' % (_dsep_bis, _dsep), re.IGNORECASE),
re.compile('[^\d](\d{1,2})%s(\d{1,2})%s(\d{4})[^\d]' % (_dsep, _dsep_bis), re.IGNORECASE),
re.compile('[^\d](\d{1,2}(?:st|nd|rd|th)?%s(?:[a-z]{3,10})%s\d{4})[^\d]' % (_dsep, _dsep), re.IGNORECASE)]
def valid_year(year, today=None):
"""Check if number is a valid year"""
if not today:
today = datetime.date.today()
return 1920 < year < today.year + 5
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
Note this only looks for valid production years, that is between 1920
and now + 5 years, so for instance 2000 would be returned as a valid
year but 1492 would not.
>>> search_year(' in the year 2000... ')
(2000, (13, 17))
>>> search_year(' they arrived in 1492. ')
(None, None)
"""
match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string)
if match:
year = int(match.group(1))
if valid_year(year):
return year, match.span(1)
return None, None
def search_date(string, year_first=None, day_first=True):
"""Looks for date patterns, and if found return the date and group span.
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
Year can be defined on two digit only. It will return the nearest possible
date from today.
>>> search_date(' This happened on 2002-04-22. ')
(datetime.date(2002, 4, 22), (18, 28))
>>> search_date(' And this on 17-06-1998. ')
(datetime.date(1998, 6, 17), (13, 23))
>>> search_date(' no date in here ')
(None, None)
"""
start, end = None, None
match = None
for date_re in date_regexps:
s = date_re.search(string)
if s and (match is None or s.end() - s.start() > len(match)):
start, end = s.start(), s.end()
if date_re.groups:
match = '-'.join(s.groups())
else:
match = s.group()
if match is None:
return None, None
today = datetime.date.today()
# If day_first/year_first is undefined, parse is made using both possible values.
yearfirst_opts = [False, True]
if year_first is not None:
yearfirst_opts = [year_first]
dayfirst_opts = [True, False]
if day_first is not None:
dayfirst_opts = [day_first]
kwargs_list = ({'dayfirst': d, 'yearfirst': y} for d in dayfirst_opts for y in yearfirst_opts)
for kwargs in kwargs_list:
try:
date = parser.parse(match, **kwargs)
except (ValueError, TypeError) as e: #see https://bugs.launchpad.net/dateutil/+bug/1247643
date = None
pass
# check date plausibility
if date and valid_year(date.year, today=today):
return date.date(), (start+1, end-1) #compensate for sentinels
return None, None
@@ -1,88 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import os.path
import zipfile
import io
from guessit import s, u
def split_path(path):
r"""Splits the given path into the list of folders and the filename (or the
last folder if you gave it a folder path.
If the given path was an absolute path, the first element will always be:
- the '/' root folder on Unix systems
- the drive letter on Windows systems (eg: r'C:\')
- the mount point '\\' on Windows systems (eg: r'\\host\share')
>>> s(split_path('/usr/bin/smewt'))
['/', 'usr', 'bin', 'smewt']
>>> s(split_path('relative_path/to/my_folder/'))
['relative_path', 'to', 'my_folder']
"""
result = []
while True:
head, tail = os.path.split(path)
if not head and not tail:
return result
if not tail and head == path:
# Make sure we won't have an infinite loop.
result = [head] + result
return result
# we just split a directory ending with '/', so tail is empty
if not tail:
path = head
continue
# otherwise, add the last path fragment and keep splitting
result = [tail] + result
path = head
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
>>> s(file_in_same_dir('~/smewt/smewt.db', 'smewt.settings')) == os.path.normpath('~/smewt/smewt.settings')
True
"""
return os.path.join(*(split_path(ref_file)[:-1] + [desired_file]))
def load_file_in_same_dir(ref_file, filename):
"""Load a given file. Works even when the file is contained inside a zip."""
path = split_path(ref_file)[:-1] + [filename]
for i, p in enumerate(path):
if p.endswith('.zip'):
zfilename = os.path.join(*path[:i + 1])
zfile = zipfile.ZipFile(zfilename)
return u(zfile.read('/'.join(path[i + 1:])))
return u(io.open(os.path.join(*path), encoding='utf-8').read())
@@ -1,521 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import json
import datetime
import logging
from guessit import UnicodeMixin, s, u, base_text_type
from babelfish import Language, Country
from guessit.textutils import common_words
log = logging.getLogger(__name__)
class GuessMetadata(object):
"""GuessMetadata contains confidence, an input string, span and related property.
If defined on a property of Guess object, it overrides the object defined as global.
:param parent: The parent metadata, used for undefined properties in self object
:type parent: :class: `GuessMedata`
:param confidence: The confidence (from 0.0 to 1.0)
:type confidence: number
:param input: The input string
:type input: string
:param span: The input string
:type span: tuple (int, int)
:param prop: The found property definition
:type prop: :class `guessit.containers._Property`
"""
def __init__(self, parent=None, confidence=None, input=None, span=None, prop=None, *args, **kwargs):
self.parent = parent
if confidence is None and self.parent is None:
self._confidence = 1.0
else:
self._confidence = confidence
self._input = input
self._span = span
self._prop = prop
@property
def confidence(self):
"""The confidence
:rtype: int
:return: confidence value
"""
return self._confidence if self._confidence is not None else self.parent.confidence if self.parent else None
@confidence.setter
def confidence(self, confidence):
self._confidence = confidence
@property
def input(self):
"""The input
:rtype: string
:return: String used to find this guess value
"""
return self._input if self._input is not None else self.parent.input if self.parent else None
@input.setter
def input(self, input):
"""The input
:rtype: string
"""
self._input = input
@property
def span(self):
"""The span
:rtype: tuple (int, int)
:return: span of input string used to find this guess value
"""
return self._span if self._span is not None else self.parent.span if self.parent else None
@span.setter
def span(self, span):
"""The span
:rtype: tuple (int, int)
:return: span of input string used to find this guess value
"""
self._span = span
@property
def prop(self):
"""The property
:rtype: :class:`_Property`
:return: The property
"""
return self._prop if self._prop is not None else self.parent.prop if self.parent else None
@property
def raw(self):
"""Return the raw information (original match from the string,
not the cleaned version) associated with the given property name."""
if self.input and self.span:
return self.input[self.span[0]:self.span[1]]
return None
def __repr__(self, *args, **kwargs):
return object.__repr__(self, *args, **kwargs)
def _split_kwargs(**kwargs):
metadata_args = {}
for prop in dir(GuessMetadata):
try:
metadata_args[prop] = kwargs.pop(prop)
except KeyError:
pass
return metadata_args, kwargs
class Guess(UnicodeMixin, dict):
"""A Guess is a dictionary which has an associated confidence for each of
its values.
As it is a subclass of dict, you can use it everywhere you expect a
simple dict."""
def __init__(self, *args, **kwargs):
metadata_kwargs, kwargs = _split_kwargs(**kwargs)
self._global_metadata = GuessMetadata(**metadata_kwargs)
dict.__init__(self, *args, **kwargs)
self._metadata = {}
for prop in self:
self._metadata[prop] = GuessMetadata(parent=self._global_metadata)
def rename(self, old_name, new_name):
if old_name in self._metadata:
metadata = self._metadata[old_name]
del self._metadata[old_name]
self._metadata[new_name] = metadata
if old_name in self:
value = self[old_name]
del self[old_name]
self[new_name] = value
return True
return False
def to_dict(self, advanced=False):
"""Return the guess as a dict containing only base types, ie:
where dates, languages, countries, etc. are converted to strings.
if advanced is True, return the data as a json string containing
also the raw information of the properties."""
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (UnicodeMixin, base_text_type)):
data[prop] = u(value)
elif isinstance(value, (Language, Country)):
data[prop] = value.guessit
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
if advanced:
metadata = self.metadata(prop)
prop_data = {'value': data[prop]}
if metadata.raw:
prop_data['raw'] = metadata.raw
if metadata.confidence:
prop_data['confidence'] = metadata.confidence
data[prop] = prop_data
return data
def nice_string(self, advanced=False):
"""Return a string with the property names and their values,
that also displays the associated confidence to each property.
FIXME: doc with param"""
if advanced:
data = self.to_dict(advanced)
return json.dumps(data, indent=4, ensure_ascii=False)
else:
data = self.to_dict()
parts = json.dumps(data, indent=4, ensure_ascii=False).split('\n')
for i, p in enumerate(parts):
if p[:5] != ' "':
continue
prop = p.split('"')[1]
parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:]
return '\n'.join(parts)
def __unicode__(self):
return u(self.to_dict())
def metadata(self, prop=None):
"""Return the metadata associated with the given property name
If no property name is given, get the global_metadata
"""
if prop is None:
return self._global_metadata
if prop not in self._metadata:
self._metadata[prop] = GuessMetadata(parent=self._global_metadata)
return self._metadata[prop]
def confidence(self, prop=None):
return self.metadata(prop).confidence
def set_confidence(self, prop, confidence):
self.metadata(prop).confidence = confidence
def raw(self, prop):
return self.metadata(prop).raw
def set(self, prop_name, value, *args, **kwargs):
if value is None:
try:
del self[prop_name]
except KeyError:
pass
try:
del self._metadata[prop_name]
except KeyError:
pass
else:
self[prop_name] = value
if 'metadata' in kwargs.keys():
self._metadata[prop_name] = kwargs['metadata']
else:
self._metadata[prop_name] = GuessMetadata(parent=self._global_metadata, *args, **kwargs)
def update(self, other, confidence=None):
dict.update(self, other)
if isinstance(other, Guess):
for prop in other:
try:
self._metadata[prop] = other._metadata[prop]
except KeyError:
pass
if confidence is not None:
for prop in other:
self.set_confidence(prop, confidence)
def update_highest_confidence(self, other):
"""Update this guess with the values from the given one. In case
there is property present in both, only the one with the highest one
is kept."""
if not isinstance(other, Guess):
raise ValueError('Can only call this function on Guess instances')
for prop in other:
if prop in self and self.metadata(prop).confidence >= other.metadata(prop).confidence:
continue
self[prop] = other[prop]
self._metadata[prop] = other.metadata(prop)
def choose_int(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are integers."""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if v1 == v2:
return v1, 1 - (1 - c1) * (1 - c2)
else:
if c1 >= c2:
return v1, c1 - c2 / 2
else:
return v2, c2 - c1 / 2
def choose_string(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are strings.
If the 2 strings are similar or have common words longer than 3 letters,
the one with highest confidence is returned with an increased confidence.
If the 2 strings are dissimilar, the one with the higher confidence is returned, with
a weaker confidence.
Note that here, 'similar' means that 2 strings are either equal, or that they
differ very little, such as one string being the other one with the 'the' word
prepended to it.
>>> s(choose_string(('Hello', 0.75), ('World', 0.5)))
('Hello', 0.5)
>>> s(choose_string(('Hello', 0.5), ('hello', 0.5)))
('Hello', 0.75)
>>> s(choose_string(('Hello', 0.4), ('Hello World', 0.4)))
('Hello', 0.64)
>>> s(choose_string(('simpsons', 0.5), ('The Simpsons', 0.5)))
('The Simpsons', 0.75)
"""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if not v1:
return g2
elif not v2:
return g1
v1, v2 = v1.strip(), v2.strip()
v1l, v2l = v1.lower(), v2.lower()
combined_prob = 1 - (1 - c1) * (1 - c2)
if v1l == v2l:
return v1, combined_prob
# check for common patterns
elif v1l == 'the ' + v2l:
return v1, combined_prob
elif v2l == 'the ' + v1l:
return v2, combined_prob
# If the 2 strings have common words longer than 3 letters,
# return the one with highest confidence.
commons = common_words(v1l, v2l)
for common_word in commons:
if len(common_word) > 3:
if c1 >= c2:
return v1, combined_prob
else:
return v2, combined_prob
# in case of conflict, return the one with highest confidence
else:
if c1 >= c2:
return v1, c1 - c2 / 2
else:
return v2, c2 - c1 / 2
def _merge_similar_guesses_nocheck(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar.
This function assumes there are at least 2 valid guesses."""
similar = [guess for guess in guesses if prop in guess]
g1, g2 = similar[0], similar[1]
# merge only this prop of s2 into s1, updating the confidence for the
# considered property
v1, v2 = g1[prop], g2[prop]
c1, c2 = g1.confidence(prop), g2.confidence(prop)
new_value, new_confidence = choose((v1, c1), (v2, c2))
if new_confidence >= c1:
msg = "Updating matching property '%s' with confidence %.2f"
else:
msg = "Updating non-matching property '%s' with confidence %.2f"
log.debug(msg % (prop, new_confidence))
g1.set(prop, new_value, confidence=new_confidence)
g2.pop(prop)
# remove g2 if there are no properties left
if not g2.keys():
guesses.remove(g2)
def merge_similar_guesses(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar."""
similar = [guess for guess in guesses if prop in guess]
if len(similar) < 2:
# nothing to merge
return
if len(similar) == 2:
_merge_similar_guesses_nocheck(guesses, prop, choose)
if len(similar) > 2:
log.debug('complex merge, trying our best...')
before = len(guesses)
_merge_similar_guesses_nocheck(guesses, prop, choose)
after = len(guesses)
if after < before:
# recurse only when the previous call actually did something,
# otherwise we end up in an infinite loop
merge_similar_guesses(guesses, prop, choose)
def merge_all(guesses, append=None):
"""Merge all the guesses in a single result, remove very unlikely values,
and return it.
You can specify a list of properties that should be appended into a list
instead of being merged.
>>> s(merge_all([ Guess({'season': 2}, confidence=0.6),
... Guess({'episodeNumber': 13}, confidence=0.8) ])
... ) == {'season': 2, 'episodeNumber': 13}
True
>>> s(merge_all([ Guess({'episodeNumber': 27}, confidence=0.02),
... Guess({'season': 1}, confidence=0.2) ])
... ) == {'season': 1}
True
>>> s(merge_all([ Guess({'other': 'PROPER'}, confidence=0.8),
... Guess({'releaseGroup': '2HD'}, confidence=0.8) ],
... append=['other'])
... ) == {'releaseGroup': '2HD', 'other': ['PROPER']}
True
"""
result = Guess()
if not guesses:
return result
if append is None:
append = []
for g in guesses:
# first append our appendable properties
for prop in append:
if prop in g:
if isinstance(g[prop], (list, set)):
new_values = result.get(prop, []) + list(g[prop])
else:
new_values = result.get(prop, []) + [g[prop]]
result.set(prop, new_values,
# TODO: what to do with confidence here? maybe an
# arithmetic mean...
confidence=g.metadata(prop).confidence,
input=g.metadata(prop).input,
span=g.metadata(prop).span,
prop=g.metadata(prop).prop)
del g[prop]
# then merge the remaining ones
dups = set(result) & set(g)
if dups:
log.debug('duplicate properties %s in merged result...' % [(result[p], g[p]) for p in dups])
result.update_highest_confidence(g)
# delete very unlikely values
for p in list(result.keys()):
if result.confidence(p) < 0.05:
del result[p]
# make sure our appendable properties contain unique values
for prop in append:
try:
value = result[prop]
if isinstance(value, list):
result[prop] = list(set(value))
else:
result[prop] = [value]
except KeyError:
pass
return result
def smart_merge(guesses):
"""First tries to merge well-known similar properties, and then merges
the rest with a merge_all call.
Should be the function to call in most cases, unless one wants to have more
control.
Warning: this function is destructive, ie: it will merge the list in-place.
"""
# 1- try to merge similar information together and give it a higher
# confidence
for int_part in ('year', 'season', 'episodeNumber'):
merge_similar_guesses(guesses, int_part, choose_int)
for string_part in ('title', 'series', 'container', 'format',
'releaseGroup', 'website', 'audioCodec',
'videoCodec', 'screenSize', 'episodeFormat',
'audioChannels', 'idNumber', 'container'):
merge_similar_guesses(guesses, string_part, choose_string)
# 2- merge the rest, potentially discarding information not properly
# merged before
result = merge_all(guesses,
append=['language', 'subtitleLanguage', 'other',
'episodeDetails', 'unidentified'])
return result
@@ -1,69 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import hashlib
import os.path
from functools import reduce
from guessit import s, to_hex
def hash_file(filename):
"""Returns the ed2k hash of a given file.
>>> testfile = os.path.join(os.path.dirname(__file__), 'test/dummy.srt')
>>> s(hash_file(testfile))
'ed2k://|file|dummy.srt|59|41F58B913AB3973F593BEBA8B8DF6510|/'
"""
return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename),
os.path.getsize(filename),
hash_filehash(filename).upper())
def hash_filehash(filename):
"""Returns the ed2k hash of a given file.
This function is taken from:
http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
"""
md4 = hashlib.new('md4').copy
def gen(f):
while True:
x = f.read(9728000)
if x:
yield x
else:
return
def md4_hash(data):
m = md4()
m.update(data)
return m
with open(filename, 'rb') as f:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return to_hex(hashes[0])
else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd
@@ -1,58 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import struct
import os
def hash_file(filename):
"""This function is taken from:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
and is licensed under the GPL."""
longlongformat = b'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(filename, "rb")
filesize = os.path.getsize(filename)
hash_value = filesize
if filesize < 65536 * 2:
raise Exception("SizeError: size is %d, should be > 132K..." % filesize)
for x in range(int(65536 / bytesize)):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
hash_value &= 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
f.seek(max(0, filesize - 65536), 0)
for x in range(int(65536 / bytesize)):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
hash_value &= 0xFFFFFFFFFFFFFFFF
f.close()
return "%016x" % hash_value
@@ -1,319 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import re
import logging
from guessit import u
from guessit.textutils import find_words
from babelfish import Language, Country
import babelfish
from guessit.guess import Guess
__all__ = ['Language', 'UNDETERMINED',
'search_language', 'guess_language']
log = logging.getLogger(__name__)
UNDETERMINED = babelfish.Language('und')
SYN = {('und', None): ['unknown', 'inconnu', 'unk', 'un'],
('ell', None): ['gr', 'greek'],
('spa', None): ['esp', 'español'],
('fra', None): ['français', 'vf', 'vff', 'vfi'],
('swe', None): ['se'],
('por', 'BR'): ['po', 'pb', 'pob', 'br', 'brazilian'],
('cat', None): ['català'],
('ces', None): ['cz'],
('ukr', None): ['ua'],
('zho', None): ['cn'],
('jpn', None): ['jp'],
('hrv', None): ['scr'],
('mul', None): ['multi', 'dl'], # http://scenelingo.wordpress.com/2009/03/24/what-does-dl-mean/
}
class GuessitConverter(babelfish.LanguageReverseConverter):
_with_country_regexp = re.compile('(.*)\((.*)\)')
_with_country_regexp2 = re.compile('(.*)-(.*)')
def __init__(self):
self.guessit_exceptions = {}
for (alpha3, country), synlist in SYN.items():
for syn in synlist:
self.guessit_exceptions[syn.lower()] = (alpha3, country, None)
@property
def codes(self):
return (babelfish.language_converters['alpha3b'].codes |
babelfish.language_converters['alpha2'].codes |
babelfish.language_converters['name'].codes |
babelfish.language_converters['opensubtitles'].codes |
babelfish.country_converters['name'].codes |
frozenset(self.guessit_exceptions.keys()))
@staticmethod
def convert(alpha3, country=None, script=None):
return str(babelfish.Language(alpha3, country, script))
def reverse(self, name):
with_country = (GuessitConverter._with_country_regexp.match(name) or
GuessitConverter._with_country_regexp2.match(name))
name = u(name.lower())
if with_country:
lang = Language.fromguessit(with_country.group(1).strip())
lang.country = babelfish.Country.fromguessit(with_country.group(2).strip())
return lang.alpha3, lang.country.alpha2 if lang.country else None, lang.script or None
# exceptions come first, as they need to override a potential match
# with any of the other guessers
try:
return self.guessit_exceptions[name]
except KeyError:
pass
for conv in [babelfish.Language,
babelfish.Language.fromalpha3b,
babelfish.Language.fromalpha2,
babelfish.Language.fromname,
babelfish.Language.fromopensubtitles]:
try:
c = conv(name)
return c.alpha3, c.country, c.script
except (ValueError, babelfish.LanguageReverseError):
pass
raise babelfish.LanguageReverseError(name)
babelfish.language_converters['guessit'] = GuessitConverter()
COUNTRIES_SYN = {'ES': ['españa'],
'GB': ['UK'],
'BR': ['brazilian', 'bra'],
# FIXME: this one is a bit of a stretch, not sure how to do
# it properly, though...
'MX': ['Latinoamérica', 'latin america']
}
class GuessitCountryConverter(babelfish.CountryReverseConverter):
def __init__(self):
self.guessit_exceptions = {}
for alpha2, synlist in COUNTRIES_SYN.items():
for syn in synlist:
self.guessit_exceptions[syn.lower()] = alpha2
@property
def codes(self):
return (babelfish.country_converters['name'].codes |
frozenset(babelfish.COUNTRIES.values()) |
frozenset(self.guessit_exceptions.keys()))
@staticmethod
def convert(alpha2):
if alpha2 == 'GB':
return 'UK'
return str(Country(alpha2))
def reverse(self, name):
# exceptions come first, as they need to override a potential match
# with any of the other guessers
try:
return self.guessit_exceptions[name.lower()]
except KeyError:
pass
try:
return babelfish.Country(name.upper()).alpha2
except ValueError:
pass
for conv in [babelfish.Country.fromname]:
try:
return conv(name).alpha2
except babelfish.CountryReverseError:
pass
raise babelfish.CountryReverseError(name)
babelfish.country_converters['guessit'] = GuessitCountryConverter()
# list of common words which could be interpreted as languages, but which
# are far too common to be able to say they represent a language in the
# middle of a string (where they most likely carry their commmon meaning)
LNG_COMMON_WORDS = frozenset([
# english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'one', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', 'rum', 'pi', 'bb',
'bt', 'tv', 'aw', 'by', 'md', 'mp', 'cd', 'lt', 'gt', 'in', 'ad', 'ice',
'ay', 'at', 'star', 'so',
# french words
'bas', 'de', 'le', 'son', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se', 'je', 'tu', 'me',
'ne', 'ma', 'va', 'au',
# japanese words,
'wa', 'ga', 'ao',
# spanish words
'la', 'el', 'del', 'por', 'mar', 'al',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
'vi', 'ben', 'da', 'lt', 'ch', 'sr', 'ps', 'cx',
# new from babelfish
'mkv', 'avi', 'dmd', 'the', 'dis', 'cut', 'stv', 'des', 'dia', 'and',
'cab', 'sub', 'mia', 'rim', 'las', 'une', 'par', 'srt', 'ano', 'toy',
'job', 'gag', 'reel', 'www', 'for', 'ayu', 'csi', 'ren', 'moi', 'sur',
'fer', 'fun', 'two', 'big', 'psy', 'air',
# movie title
'brazil',
# release groups
'bs', # Bosnian
'kz',
# countries
'gt', 'lt', 'im',
# part/pt
'pt'
])
LNG_COMMON_WORDS_STRICT = frozenset(['brazil'])
subtitle_prefixes = ['sub', 'subs', 'st', 'vost', 'subforced', 'fansub', 'hardsub']
subtitle_suffixes = ['subforced', 'fansub', 'hardsub', 'sub', 'subs']
lang_prefixes = ['true']
all_lang_prefixes_suffixes = subtitle_prefixes + subtitle_suffixes + lang_prefixes
def find_possible_languages(string, allowed_languages=None):
"""Find possible languages in the string
:return: list of tuple (property, Language, lang_word, word)
"""
common_words = None
if allowed_languages:
common_words = LNG_COMMON_WORDS_STRICT
else:
common_words = LNG_COMMON_WORDS
words = find_words(string)
valid_words = []
for word in words:
lang_word = word.lower()
key = 'language'
for prefix in subtitle_prefixes:
if lang_word.startswith(prefix):
lang_word = lang_word[len(prefix):]
key = 'subtitleLanguage'
for suffix in subtitle_suffixes:
if lang_word.endswith(suffix):
lang_word = lang_word[:len(suffix)]
key = 'subtitleLanguage'
for prefix in lang_prefixes:
if lang_word.startswith(prefix):
lang_word = lang_word[len(prefix):]
if lang_word not in common_words and word.lower() not in common_words:
try:
lang = Language.fromguessit(lang_word)
if allowed_languages:
if lang.name.lower() in allowed_languages or lang.alpha2.lower() in allowed_languages or lang.alpha3.lower() in allowed_languages:
valid_words.append((key, lang, lang_word, word))
# Keep language with alpha2 equivalent. Others are probably
# uncommon languages.
elif lang == 'mul' or hasattr(lang, 'alpha2'):
valid_words.append((key, lang, lang_word, word))
except babelfish.Error:
pass
return valid_words
def search_language(string, allowed_languages=None):
"""Looks for language patterns, and if found return the language object,
its group span and an associated confidence.
you can specify a list of allowed languages using the lang_filter argument,
as in lang_filter = [ 'fr', 'eng', 'spanish' ]
>>> search_language('movie [en].avi')['language']
<Language [en]>
>>> search_language('the zen fat cat and the gay mad men got a new fan', allowed_languages = ['en', 'fr', 'es'])
"""
if allowed_languages:
allowed_languages = set(Language.fromguessit(lang) for lang in allowed_languages)
confidence = 1.0 # for all of them
for prop, language, lang, word in find_possible_languages(string, allowed_languages):
pos = string.find(word)
end = pos + len(word)
# only allow those languages that have a 2-letter code, those that
# don't are too esoteric and probably false matches
# if language.lang not in lng3_to_lng2:
# continue
# confidence depends on alpha2, alpha3, english name, ...
if len(lang) == 2:
confidence = 0.8
elif len(lang) == 3:
confidence = 0.9
elif prop == 'subtitleLanguage':
confidence = 0.6 # Subtitle prefix found with language
else:
# Note: we could either be really confident that we found a
# language or assume that full language names are too
# common words and lower their confidence accordingly
confidence = 0.3 # going with the low-confidence route here
return Guess({prop: language}, confidence=confidence, input=string, span=(pos, end))
return None
def guess_language(text): # pragma: no cover
"""Guess the language in which a body of text is written.
This uses the external guess-language python module, and will fail and return
Language(Undetermined) if it is not installed.
"""
try:
from guess_language import guessLanguage
return Language.fromguessit(guessLanguage(text))
except ImportError:
log.error('Cannot detect the language of the given text body, missing dependency: guess-language')
log.error('Please install it from PyPI, by doing eg: pip install guess-language')
return UNDETERMINED
@@ -1,314 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, \
unicode_literals
import logging
import inspect
from guessit import PY3, u
from guessit.transfo import TransformerException
from guessit.matchtree import MatchTree
from guessit.textutils import normalize_unicode, clean_default
from guessit.guess import Guess
log = logging.getLogger(__name__)
class IterativeMatcher(object):
"""An iterative matcher tries to match different patterns that appear
in the filename.
The ``filetype`` argument indicates which type of file you want to match.
If it is undefined, the matcher will try to see whether it can guess
that the file corresponds to an episode, or otherwise will assume it is
a movie.
The recognized ``filetype`` values are:
``['subtitle', 'info', 'movie', 'moviesubtitle', 'movieinfo', 'episode',
'episodesubtitle', 'episodeinfo']``
``options`` is a dict of options values to be passed to the transformations used
by the matcher.
The IterativeMatcher works mainly in 2 steps:
First, it splits the filename into a match_tree, which is a tree of groups
which have a semantic meaning, such as episode number, movie title,
etc...
The match_tree created looks like the following::
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, ``x264`` (in the middle) is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter ``v`` in the 4th line.
(for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
containing all the found properties, and does some (basic) conflict
resolution when they arise.
"""
def __init__(self, filename, options=None, **kwargs):
options = dict(options or {})
for k, v in kwargs.items():
if k not in options or not options[k]:
options[k] = v # options dict has priority over keyword arguments
self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
if options and options.get('clean_function'):
clean_function = options.get('clean_function')
if not hasattr(clean_function, '__call__'):
module, function = clean_function.rsplit('.')
if not module:
module = 'guessit.textutils'
clean_function = getattr(__import__(module), function)
if not clean_function:
log.error('Can\'t find clean function %s. Default will be used.' % options.get('clean_function'))
clean_function = clean_default
else:
clean_function = clean_default
self.match_tree = MatchTree(filename, clean_function=clean_function)
self.options = options
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_function(filename).strip() == '':
return
from guessit.plugins import transformers
try:
mtree = self.match_tree
if 'type' in self.options:
mtree.guess.set('type', self.options['type'], confidence=0.0)
# Process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or transformer.name not in disabled:
self._process(transformer, False)
# Post-process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or transformer.name not in disabled:
self._process(transformer, True)
log.debug('Found match tree:\n%s' % u(mtree))
except TransformerException as e:
log.debug('An error has occurred in Transformer %s: %s' % (e.transformer, e))
def _process(self, transformer, post=False):
if not hasattr(transformer, 'should_process') or transformer.should_process(self.match_tree, self.options):
if post:
transformer.post_process(self.match_tree, self.options)
else:
transformer.process(self.match_tree, self.options)
self._transfo_calls.append(transformer)
@property
def second_pass_options(self):
second_pass_options = {}
for transformer in self._transfo_calls:
if hasattr(transformer, 'second_pass_options'):
transformer_second_pass_options = transformer.second_pass_options(self.match_tree, self.options)
if transformer_second_pass_options:
second_pass_options.update(transformer_second_pass_options)
return second_pass_options
@staticmethod
def _validate_options(options):
valid_filetypes = ('subtitle', 'info', 'video',
'movie', 'moviesubtitle', 'movieinfo',
'episode', 'episodesubtitle', 'episodeinfo')
type_ = options.get('type')
if type_ and type_ not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % (valid_filetypes,))
def matched(self):
return self.match_tree.matched()
def build_guess(node, name, value=None, confidence=1.0):
guess = Guess({name: node.clean_value if value is None else value}, confidence=confidence)
guess.metadata().input = node.value if value is None else value
if value is None:
left_offset = 0
right_offset = 0
clean_value = node.clean_value
if clean_value:
for i in range(0, len(node.value)):
if clean_value[0] == node.value[i]:
break
left_offset += 1
for i in reversed(range(0, len(node.value))):
if clean_value[-1] == node.value[i]:
break
right_offset += 1
guess.metadata().span = (node.span[0] - node.offset + left_offset, node.span[1] - node.offset - right_offset)
return guess
def found_property(node, name, value=None, confidence=1.0, update_guess=True, logger=None):
# automatically retrieve the log object from the caller frame
if not logger:
caller_frame = inspect.stack()[1][0]
logger = caller_frame.f_locals['self'].log
guess = build_guess(node, name, value, confidence)
return found_guess(node, guess, update_guess=update_guess, logger=logger)
def found_guess(node, guess, update_guess=True, logger=None):
if node.guess:
if update_guess:
node.guess.update_highest_confidence(guess)
else:
child = node.add_child(guess.metadata().span)
child.guess = guess
else:
node.guess = guess
log_found_guess(guess, logger)
return node.guess
def log_found_guess(guess, logger=None):
for k, v in guess.items():
(logger or log).debug('Property found: %s=%s (%s) (confidence=%.2f)' %
(k, v, guess.raw(k), guess.confidence(k)))
class GuessFinder(object):
def __init__(self, guess_func, confidence=None, logger=None, options=None):
self.guess_func = guess_func
self.confidence = confidence
self.logger = logger or log
self.options = options or {}
def process_nodes(self, nodes):
for node in nodes:
self.process_node(node)
def process_node(self, node, iterative=True, partial_span=None, skip_nodes=True):
if skip_nodes and not isinstance(skip_nodes, list):
skip_nodes = self.options.get('skip_nodes')
elif not isinstance(skip_nodes, list):
skip_nodes = []
if partial_span:
value = node.value[partial_span[0]:partial_span[1]]
else:
value = node.value
string = ' %s ' % value # add sentinels
matcher_result = self.guess_func(string, node, self.options)
if not matcher_result:
return
if not isinstance(matcher_result, Guess):
result, span = matcher_result
else:
result, span = matcher_result, matcher_result.metadata().span
#log.error('span2 %s' % (span,))
if not result:
return
if span[1] == len(string):
# somehow, the sentinel got included in the span. Remove it
span = (span[0], span[1] - 1)
# readjust span to compensate for sentinels
span = (span[0] - 1, span[1] - 1)
# readjust span to compensate for partial_span
if partial_span:
span = (span[0] + partial_span[0], span[1] + partial_span[0])
if skip_nodes:
skip_nodes = [skip_node for skip_node in self.options.get('skip_nodes') if skip_node.parent.span[0] == node.span[0] or skip_node.parent.span[1] == node.span[1]]
# if we guessed a node that we need to skip, recurse down the tree and ignore that node
indices = set()
skip_nodes_spans = []
next_skip_nodes = []
for skip_node in skip_nodes:
skip_for_next = False
skip_nodes_spans.append(skip_node.span)
if node.offset <= skip_node.span[0] <= node.span[1]:
indices.add(skip_node.span[0] - node.offset)
skip_for_next = True
if node.offset <= skip_node.span[1] <= node.span[1]:
indices.add(skip_node.span[1] - node.offset)
skip_for_next = True
if not skip_for_next:
next_skip_nodes.append(skip_node)
if indices:
partition_spans = [s for s in node.get_partition_spans(indices) if s not in skip_nodes_spans]
for partition_span in partition_spans:
relative_span = (partition_span[0] - node.offset, partition_span[1] - node.offset)
self.process_node(node, partial_span=relative_span, skip_nodes=next_skip_nodes)
return
# restore sentinels compensation
if isinstance(result, Guess):
guess = result
else:
no_sentinel_string =string[1:-1]
guess = Guess(result, confidence=self.confidence, input=no_sentinel_string, span=span)
if not iterative:
found_guess(node, guess, logger=self.logger)
else:
absolute_span = (span[0] + node.offset, span[1] + node.offset)
node.partition(span)
found_child = None
for child in node.children:
if child.span == absolute_span:
# if we have a match on one of our children, mark it as such...
found_guess(child, guess, logger=self.logger)
found_child = child
break
# ...and only then recurse on the other children
for child in node.children:
if child is not found_child:
self.process_node(child)
@@ -1,491 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import copy
import logging
import guessit # @UnusedImport needed for doctests
from guessit import UnicodeMixin, base_text_type
from guessit.textutils import clean_default, str_fill
from guessit.patterns import group_delimiters
from guessit.guess import smart_merge, Guess
log = logging.getLogger(__name__)
class BaseMatchTree(UnicodeMixin):
"""A BaseMatchTree is a tree covering the filename, where each
node represents a substring in the filename and can have a ``Guess``
associated with it that contains the information that has been guessed
in this node. Nodes can be further split into subnodes until a proper
split has been found.
Each node has the following attributes:
- string = the original string of which this node represents a region
- span = a pair of (begin, end) indices delimiting the substring
- parent = parent node
- children = list of children nodes
- guess = Guess()
BaseMatchTrees are displayed in the following way:
>>> path = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv'
>>> print(guessit.IterativeMatcher(path).match_tree)
000000 1111111111111111 2222222222222222222222222222222222222222222 333
000000 0000000000111111 0000000000111111222222222222222222222222222 000
011112 011112000011111222222222222222222 000
011112222222222222
0000011112222
01112 0111
Movies/__________(____)/Dark.City.(____).DC._____.____.___.____-___.___
tttttttttt yyyy yyyy fffff ssss aaa vvvv rrr ccc
Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv
The last line contains the filename, which you can use a reference.
The previous line contains the type of property that has been found.
The line before that contains the filename, where all the found groups
have been blanked. Basically, what is left on this line are the leftover
groups which could not be identified.
The lines before that indicate the indices of the groups in the tree.
For instance, the part of the filename 'BDRip' is the leaf with index
``(2, 2, 1)`` (read from top to bottom), and its meaning is 'format'
(as shown by the ``f``'s on the last-but-one line).
"""
def __init__(self, string='', span=None, parent=None, clean_function=None, category=None):
self.string = string
self.span = span or (0, len(string))
self.parent = parent
self.children = []
self.guess = Guess()
self._clean_value = None
self._clean_function = clean_function or clean_default
self.category = category
@property
def value(self):
"""Return the substring that this node matches."""
return self.string[self.span[0]:self.span[1]]
@property
def clean_value(self):
"""Return a cleaned value of the matched substring, with better
presentation formatting (punctuation marks removed, duplicate
spaces, ...)"""
if self._clean_value is None:
self._clean_value = self.clean_string(self.value)
return self._clean_value
def clean_string(self, string):
return self._clean_function(string)
@property
def offset(self):
return self.span[0]
@property
def info(self):
"""Return a dict containing all the info guessed by this node,
subnodes included."""
result = dict(self.guess)
for c in self.children:
result.update(c.info)
return result
@property
def raw(self):
result = {}
for guess in self.guesses:
for k in guess.keys():
result[k] = guess.raw(k)
return result
@property
def guesses(self):
"""
List all guesses, including children ones.
:return: list of guesses objects
"""
result = []
if self.guess:
result.append(self.guess)
for c in self.children:
result.extend(c.guesses)
return result
@property
def root(self):
"""Return the root node of the tree."""
if not self.parent:
return self
return self.parent.root
@property
def ancestors(self):
"""
Retrieve all ancestors, from this node to root node.
:return: a list of MatchTree objects
"""
ret = [self]
if not self.parent:
return ret
parent_ancestors = self.parent.ancestors
ret.extend(parent_ancestors)
return ret
@property
def depth(self):
"""Return the depth of this node."""
if self.is_leaf():
return 0
return 1 + max(c.depth for c in self.children)
def is_leaf(self):
"""Return whether this node is a leaf or not."""
return self.children == []
def add_child(self, span, category=None):
"""Add a new child node to this node with the given span.
:param span: span of the new MatchTree
:param category: category of the new MatchTree
:return: A new MatchTree instance having self as a parent
"""
child = MatchTree(self.string, span=span, parent=self, clean_function=self._clean_function, category=category)
self.children.append(child)
return child
def get_partition_spans(self, indices):
"""Return the list of absolute spans for the regions of the original
string defined by splitting this node at the given indices (relative
to this node)
:param indices: indices of the partition spans
:return: a list of tuple of the spans
"""
indices = sorted(indices)
if indices[-1] > len(self.value):
log.error('Filename: {}'.format(self.string))
log.error('Invalid call to get_partitions_spans, indices are too high: {}, len({}) == {:d}'
.format(indices, self.value, len(self.value)))
if indices[0] != 0:
indices.insert(0, 0)
if indices[-1] != len(self.value):
indices.append(len(self.value))
spans = []
for start, end in zip(indices[:-1], indices[1:]):
spans.append((self.offset + start,
self.offset + end))
return spans
def partition(self, indices, category=None):
"""Partition this node by splitting it at the given indices,
relative to this node.
:param indices: indices of the partition spans
:param category: category of the new MatchTree
:return: a list of created MatchTree instances
"""
created = []
for partition_span in self.get_partition_spans(indices):
created.append(self.add_child(span=partition_span, category=category))
return created
def split_on_components(self, components, category=None):
offset = 0
created = []
for c in components:
start = self.value.find(c, offset)
end = start + len(c)
created.append(self.add_child(span=(self.offset + start,
self.offset + end), category=category))
offset = end
return created
def nodes_at_depth(self, depth):
"""Return all the nodes at a given depth in the tree"""
if depth == 0:
yield self
for child in self.children:
for node in child.nodes_at_depth(depth - 1):
yield node
@property
def node_idx(self):
"""Return this node's index in the tree, as a tuple.
If this node is the root of the tree, then return ()."""
if self.parent is None:
return ()
return self.parent.node_idx + (self.node_last_idx,)
@property
def node_last_idx(self):
if self.parent is None:
return None
return self.parent.children.index(self)
def node_at(self, idx):
"""Return the node at the given index in the subtree rooted at
this node."""
if not idx:
return self
try:
return self.children[idx[0]].node_at(idx[1:])
except IndexError:
raise ValueError('Non-existent node index: %s' % (idx,))
def nodes(self):
"""Return a generator of all nodes and subnodes in this tree."""
yield self
for child in self.children:
for node in child.nodes():
yield node
def leaves(self):
"""Return a generator over all the nodes that are leaves."""
if self.is_leaf():
yield self
else:
for child in self.children:
for leaf in child.leaves():
yield leaf
def group_node(self):
return self._other_group_node(0)
def previous_group_node(self):
return self._other_group_node(-1)
def next_group_node(self):
return self._other_group_node(+1)
def _other_group_node(self, offset):
if len(self.node_idx) > 1:
group_idx = self.node_idx[:2]
if group_idx[1] + offset >= 0:
other_group_idx = (group_idx[0], group_idx[1] + offset)
try:
other_group_node = self.root.node_at(other_group_idx)
return other_group_node
except ValueError:
pass
return None
def previous_leaf(self, leaf):
"""Return previous leaf for this node"""
return self._other_leaf(leaf, -1)
def next_leaf(self, leaf):
"""Return next leaf for this node"""
return self._other_leaf(leaf, +1)
def _other_leaf(self, leaf, offset):
leaves = list(self.leaves())
index = leaves.index(leaf) + offset
if 0 < index < len(leaves):
return leaves[index]
return None
def previous_leaves(self, leaf):
"""Return previous leaves for this node"""
leaves = list(self.leaves())
index = leaves.index(leaf)
if 0 < index < len(leaves):
previous_leaves = leaves[:index]
previous_leaves.reverse()
return previous_leaves
return []
def next_leaves(self, leaf):
"""Return next leaves for this node"""
leaves = list(self.leaves())
index = leaves.index(leaf)
if 0 < index < len(leaves):
return leaves[index + 1:len(leaves)]
return []
def to_string(self):
"""Return a readable string representation of this tree.
The result is a multi-line string, where the lines are:
- line 1 -> N-2: each line contains the nodes at the given depth in the tree
- line N-2: original string where all the found groups have been blanked
- line N-1: type of property that has been found
- line N: the original string, which you can use a reference.
"""
empty_line = ' ' * len(self.string)
def to_hex(x):
if isinstance(x, int):
return str(x) if x < 10 else chr(55 + x)
return x
def meaning(result):
mmap = {'episodeNumber': 'E',
'season': 'S',
'extension': 'e',
'format': 'f',
'language': 'l',
'country': 'C',
'videoCodec': 'v',
'videoProfile': 'v',
'audioCodec': 'a',
'audioProfile': 'a',
'audioChannels': 'a',
'website': 'w',
'container': 'c',
'series': 'T',
'title': 't',
'date': 'd',
'year': 'y',
'releaseGroup': 'r',
'screenSize': 's',
'other': 'o'
}
if result is None:
return ' '
for prop, l in mmap.items():
if prop in result:
return l
return 'x'
lines = [empty_line] * (self.depth + 2) # +2: remaining, meaning
lines[-2] = self.string
for node in self.nodes():
if node == self:
continue
idx = node.node_idx
depth = len(idx) - 1
if idx:
lines[depth] = str_fill(lines[depth], node.span,
to_hex(idx[-1]))
if node.guess:
lines[-2] = str_fill(lines[-2], node.span, '_')
lines[-1] = str_fill(lines[-1], node.span, meaning(node.guess))
lines.append(self.string)
return '\n'.join(l.rstrip() for l in lines)
def __unicode__(self):
return self.to_string()
def __repr__(self):
return '<MatchTree: root=%s>' % self.value
class MatchTree(BaseMatchTree):
"""The MatchTree contains a few "utility" methods which are not necessary
for the BaseMatchTree, but add a lot of convenience for writing
higher-level rules.
"""
def unidentified_leaves(self,
valid=lambda leaf: len(leaf.clean_value) > 0):
"""Return a generator of leaves that are not empty."""
for leaf in self.leaves():
if not leaf.guess and valid(leaf):
yield leaf
def leaves_containing(self, property_name):
"""Return a generator of leaves that guessed the given property."""
if isinstance(property_name, base_text_type):
property_name = [property_name]
for leaf in self.leaves():
for prop in property_name:
if prop in leaf.guess:
yield leaf
break
def first_leaf_containing(self, property_name):
"""Return the first leaf containing the given property."""
try:
return next(self.leaves_containing(property_name))
except StopIteration:
return None
def previous_unidentified_leaves(self, node):
"""Return a generator of non-empty leaves that are before the given
node (in the string)."""
node_idx = node.node_idx
for leaf in self.unidentified_leaves():
if leaf.node_idx < node_idx:
yield leaf
def previous_leaves_containing(self, node, property_name):
"""Return a generator of leaves containing the given property that are
before the given node (in the string)."""
node_idx = node.node_idx
for leaf in self.leaves_containing(property_name):
if leaf.node_idx < node_idx:
yield leaf
def is_explicit(self):
"""Return whether the group was explicitly enclosed by
parentheses/square brackets/etc."""
return (self.value[0] + self.value[-1]) in group_delimiters
def matched(self):
"""Return a single guess that contains all the info found in the
nodes of this tree, trying to merge properties as good as possible.
"""
if not getattr(self, '_matched_result', None):
# we need to make a copy here, as the merge functions work in place and
# calling them on the match tree would modify it
parts = [copy.copy(node.guess) for node in self.nodes() if node.guess]
result = smart_merge(parts)
log.debug('Final result: ' + result.nice_string())
self._matched_result = result
for leaf in self.unidentified_leaves():
if 'unidentified' not in self._matched_result:
self._matched_result['unidentified'] = []
self._matched_result['unidentified'].append(leaf.clean_value)
return self._matched_result
@@ -1,69 +0,0 @@
from argparse import ArgumentParser
def build_opts(transformers=None):
opts = ArgumentParser()
opts.add_argument(dest='filename', help='Filename or release name to guess', nargs='*')
naming_opts = opts.add_argument_group("Naming")
naming_opts.add_argument('-t', '--type', dest='type', default=None,
help='The suggested file type: movie, episode. If undefined, type will be guessed.')
naming_opts.add_argument('-n', '--name-only', dest='name_only', action='store_true', default=False,
help='Parse files as name only. Disable folder parsing, extension parsing, and file content analysis.')
naming_opts.add_argument('-c', '--split-camel', dest='split_camel', action='store_true', default=False,
help='Split camel case part of filename.')
naming_opts.add_argument('-X', '--disabled-transformer', action='append', dest='disabled_transformers',
help='Transformer to disable (can be used multiple time)')
output_opts = opts.add_argument_group("Output")
output_opts.add_argument('-v', '--verbose', action='store_true', dest='verbose', default=False,
help='Display debug output')
output_opts.add_argument('-P', '--show-property', dest='show_property', default=None,
help='Display the value of a single property (title, series, videoCodec, year, type ...)'),
output_opts.add_argument('-u', '--unidentified', dest='unidentified', action='store_true', default=False,
help='Display the unidentified parts.'),
output_opts.add_argument('-a', '--advanced', dest='advanced', action='store_true', default=False,
help='Display advanced information for filename guesses, as json output')
output_opts.add_argument('-y', '--yaml', dest='yaml', action='store_true', default=False,
help='Display information for filename guesses as yaml output (like unit-test)')
output_opts.add_argument('-f', '--input-file', dest='input_file', default=False,
help='Read filenames from an input file.')
output_opts.add_argument('-d', '--demo', action='store_true', dest='demo', default=False,
help='Run a few builtin tests instead of analyzing a file')
information_opts = opts.add_argument_group("Information")
information_opts.add_argument('-p', '--properties', dest='properties', action='store_true', default=False,
help='Display properties that can be guessed.')
information_opts.add_argument('-V', '--values', dest='values', action='store_true', default=False,
help='Display property values that can be guessed.')
information_opts.add_argument('-s', '--transformers', dest='transformers', action='store_true', default=False,
help='Display transformers that can be used.')
information_opts.add_argument('--version', dest='version', action='store_true', default=False,
help='Display the guessit version.')
webservice_opts = opts.add_argument_group("guessit.io")
webservice_opts.add_argument('-b', '--bug', action='store_true', dest='submit_bug', default=False,
help='Submit a wrong detection to the guessit.io service')
other_opts = opts.add_argument_group("Other features")
other_opts.add_argument('-i', '--info', dest='info', default='filename',
help='The desired information type: filename, video, hash_mpc or a hash from python\'s '
'hashlib module, such as hash_md5, hash_sha1, ...; or a list of any of '
'them, comma-separated')
if transformers:
for transformer in transformers:
transformer.register_arguments(opts, naming_opts, output_opts, information_opts, webservice_opts, other_opts)
return opts, naming_opts, output_opts, information_opts, webservice_opts, other_opts
_opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts = None, None, None, None, None, None
def reload(transformers=None):
global _opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts
_opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts = build_opts(transformers)
def get_opts():
return _opts
@@ -1,75 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import re
group_delimiters = ['()', '[]', '{}']
# separator character regexp
sep = r'[][,)(}:{+ /~/\._-]' # regexp art, hehe :D
_dash = '-'
_psep = '[\W_]?'
def build_or_pattern(patterns, escape=False):
"""Build a or pattern string from a list of possible patterns
"""
or_pattern = []
for pattern in patterns:
if not or_pattern:
or_pattern.append('(?:')
else:
or_pattern.append('|')
or_pattern.append('(?:%s)' % re.escape(pattern) if escape else pattern)
or_pattern.append(')')
return ''.join(or_pattern)
def compile_pattern(pattern, enhance=True):
"""Compile and enhance a pattern
:param pattern: Pattern to compile (regexp).
:type pattern: string
:param pattern: Enhance pattern before compiling.
:type pattern: string
:return: The compiled pattern
:rtype: regular expression object
"""
return re.compile(enhance_pattern(pattern) if enhance else pattern, re.IGNORECASE)
def enhance_pattern(pattern):
"""Enhance pattern to match more equivalent values.
'-' are replaced by '[\W_]?', which matches more types of separators (or none)
:param pattern: Pattern to enhance (regexp).
:type pattern: string
:return: The enhanced pattern
:rtype: string
"""
return pattern.replace(_dash, _psep)
@@ -1,32 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
subtitle_exts = ['srt', 'idx', 'sub', 'ssa', 'ass']
info_exts = ['nfo']
video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2',
'mka', 'mkv', 'mov', 'mp4', 'mp4a', 'mpeg', 'mpg', 'ogg', 'ogm',
'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv',
'iso', 'vob']
@@ -1,80 +0,0 @@
import re
from guessit.patterns import sep, build_or_pattern
from guessit.patterns.numeral import parse_numeral
range_separators = ['-', 'to', 'a']
discrete_separators = ['&', 'and', 'et']
excluded_separators = ['.'] # Dot cannot serve as a discrete_separator
discrete_sep = sep
for range_separator in range_separators:
discrete_sep = discrete_sep.replace(range_separator, '')
for excluded_separator in excluded_separators:
discrete_sep = discrete_sep.replace(excluded_separator, '')
discrete_separators.append(discrete_sep)
all_separators = list(range_separators)
all_separators.extend(discrete_separators)
range_separators_re = re.compile(build_or_pattern(range_separators), re.IGNORECASE)
discrete_separators_re = re.compile(build_or_pattern(discrete_separators), re.IGNORECASE)
all_separators_re = re.compile(build_or_pattern(all_separators), re.IGNORECASE)
def list_parser(value, property_list_name, discrete_separators_re=discrete_separators_re, range_separators_re=range_separators_re, allow_discrete=False, fill_gaps=False):
discrete_elements = filter(lambda x: x != '', discrete_separators_re.split(value))
discrete_elements = [x.strip() for x in discrete_elements]
proper_discrete_elements = []
i = 0
while i < len(discrete_elements):
if i < len(discrete_elements) - 2 and range_separators_re.match(discrete_elements[i+1]):
proper_discrete_elements.append(discrete_elements[i] + discrete_elements[i+1] + discrete_elements[i+2])
i += 3
else:
match = range_separators_re.search(discrete_elements[i])
if match and match.start() == 0:
proper_discrete_elements[i - 1] += discrete_elements[i]
elif match and match.end() == len(discrete_elements[i]):
proper_discrete_elements.append(discrete_elements[i] + discrete_elements[i + 1])
else:
proper_discrete_elements.append(discrete_elements[i])
i += 1
discrete_elements = proper_discrete_elements
ret = []
for discrete_element in discrete_elements:
range_values = filter(lambda x: x != '', range_separators_re.split(discrete_element))
range_values = [x.strip() for x in range_values]
if len(range_values) > 1:
for x in range(0, len(range_values) - 1):
start_range_ep = parse_numeral(range_values[x])
end_range_ep = parse_numeral(range_values[x+1])
for range_ep in range(start_range_ep, end_range_ep + 1):
if range_ep not in ret:
ret.append(range_ep)
else:
discrete_value = parse_numeral(discrete_element)
if discrete_value not in ret:
ret.append(discrete_value)
if len(ret) > 1:
if not allow_discrete:
valid_ret = list()
# replace discrete elements by ranges
valid_ret.append(ret[0])
for i in range(0, len(ret) - 1):
previous = valid_ret[len(valid_ret) - 1]
if ret[i+1] < previous:
pass
else:
valid_ret.append(ret[i+1])
ret = valid_ret
if fill_gaps:
ret = list(range(min(ret), max(ret) + 1))
if len(ret) > 1:
return {None: ret[0], property_list_name: ret}
if len(ret) > 0:
return ret[0]
return None
@@ -1,150 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import re
digital_numeral = '\d{1,4}'
roman_numeral = "(?=[MCDLXVI]+)M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})"
english_word_numeral_list = [
'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty'
]
french_word_numeral_list = [
'zéro', 'un', 'deux', 'trois', 'quatre', 'cinq', 'six', 'sept', 'huit', 'neuf', 'dix',
'onze', 'douze', 'treize', 'quatorze', 'quinze', 'seize', 'dix-sept', 'dix-huit', 'dix-neuf', 'vingt'
]
french_alt_word_numeral_list = [
'zero', 'une', 'deux', 'trois', 'quatre', 'cinq', 'six', 'sept', 'huit', 'neuf', 'dix',
'onze', 'douze', 'treize', 'quatorze', 'quinze', 'seize', 'dixsept', 'dixhuit', 'dixneuf', 'vingt'
]
def __build_word_numeral(*args, **kwargs):
re_ = None
for word_list in args:
for word in word_list:
if not re_:
re_ = '(?:(?=\w+)'
else:
re_ += '|'
re_ += word
re_ += ')'
return re_
word_numeral = __build_word_numeral(english_word_numeral_list, french_word_numeral_list, french_alt_word_numeral_list)
numeral = '(?:' + digital_numeral + '|' + roman_numeral + '|' + word_numeral + ')'
__romanNumeralMap = (
('M', 1000),
('CM', 900),
('D', 500),
('CD', 400),
('C', 100),
('XC', 90),
('L', 50),
('XL', 40),
('X', 10),
('IX', 9),
('V', 5),
('IV', 4),
('I', 1)
)
__romanNumeralPattern = re.compile('^' + roman_numeral + '$')
def __parse_roman(value):
"""convert Roman numeral to integer"""
if not __romanNumeralPattern.search(value):
raise ValueError('Invalid Roman numeral: %s' % value)
result = 0
index = 0
for num, integer in __romanNumeralMap:
while value[index:index + len(num)] == num:
result += integer
index += len(num)
return result
def __parse_word(value):
"""Convert Word numeral to integer"""
for word_list in [english_word_numeral_list, french_word_numeral_list, french_alt_word_numeral_list]:
try:
return word_list.index(value.lower())
except ValueError:
pass
raise ValueError
_clean_re = re.compile('[^\d]*(\d+)[^\d]*')
def parse_numeral(value, int_enabled=True, roman_enabled=True, word_enabled=True, clean=True):
"""Parse a numeric value into integer.
input can be an integer as a string, a roman numeral or a word
:param value: Value to parse. Can be an integer, roman numeral or word.
:type value: string
:return: Numeric value, or None if value can't be parsed
:rtype: int
"""
if int_enabled:
try:
if clean:
match = _clean_re.match(value)
if match:
clean_value = match.group(1)
return int(clean_value)
return int(value)
except ValueError:
pass
if roman_enabled:
try:
if clean:
for word in value.split():
try:
return __parse_roman(word.upper())
except ValueError:
pass
return __parse_roman(value)
except ValueError:
pass
if word_enabled:
try:
if clean:
for word in value.split():
try:
return __parse_word(word)
except ValueError:
pass
return __parse_word(value)
except ValueError:
pass
raise ValueError('Invalid numeral: ' + value)
@@ -1,21 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
@@ -1,222 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from logging import getLogger
from pkg_resources import EntryPoint
from guessit.options import reload as reload_options
from stevedore import ExtensionManager
from stevedore.extension import Extension
log = getLogger(__name__)
class Transformer(object): # pragma: no cover
def __init__(self, priority=0):
self.priority = priority
self.log = getLogger(self.name)
@property
def name(self):
return self.__class__.__name__
def supported_properties(self):
return {}
def second_pass_options(self, mtree, options=None):
return None
def should_process(self, mtree, options=None):
return True
def process(self, mtree, options=None):
pass
def post_process(self, mtree, options=None):
pass
def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
pass
def rate_quality(self, guess, *props):
return 0
class CustomTransformerExtensionManager(ExtensionManager):
def __init__(self, namespace='guessit.transformer', invoke_on_load=True,
invoke_args=(), invoke_kwds={}, propagate_map_exceptions=True, on_load_failure_callback=None,
verify_requirements=False):
super(CustomTransformerExtensionManager, self).__init__(namespace=namespace,
invoke_on_load=invoke_on_load,
invoke_args=invoke_args,
invoke_kwds=invoke_kwds,
propagate_map_exceptions=propagate_map_exceptions,
on_load_failure_callback=on_load_failure_callback,
verify_requirements=verify_requirements)
@staticmethod
def order_extensions(extensions):
"""Order the loaded transformers
It should follow those rules
- website before language (eg: tvu.org.ru vs russian)
- language before episodes_rexps
- properties before language (eg: he-aac vs hebrew)
- release_group before properties (eg: XviD-?? vs xvid)
"""
extensions.sort(key=lambda ext: -ext.obj.priority)
return extensions
@staticmethod
def _load_one_plugin(ep, invoke_on_load, invoke_args, invoke_kwds, verify_requirements=True):
if not ep.dist:
# `require` argument of ep.load() is deprecated in newer versions of setuptools
if hasattr(ep, 'resolve'):
plugin = ep.resolve()
elif hasattr(ep, '_load'):
plugin = ep._load()
else:
plugin = ep.load(require=False)
else:
plugin = ep.load()
if invoke_on_load:
obj = plugin(*invoke_args, **invoke_kwds)
else:
obj = None
return Extension(ep.name, ep, plugin, obj)
def _load_plugins(self, invoke_on_load, invoke_args, invoke_kwds, verify_requirements):
return self.order_extensions(super(CustomTransformerExtensionManager, self)._load_plugins(invoke_on_load, invoke_args, invoke_kwds, verify_requirements))
def objects(self):
return self.map(self._get_obj)
@staticmethod
def _get_obj(ext):
return ext.obj
def object(self, name):
try:
return self[name].obj
except KeyError:
return None
def register_module(self, name=None, module_name=None, attrs=(), entry_point=None):
if entry_point:
ep = EntryPoint.parse(entry_point)
else:
ep = EntryPoint(name, module_name, attrs)
loaded = self._load_one_plugin(ep, invoke_on_load=True, invoke_args=(), invoke_kwds={})
if loaded:
self.extensions.append(loaded)
self.extensions = self.order_extensions(self.extensions)
self._extensions_by_name = None
class DefaultTransformerExtensionManager(CustomTransformerExtensionManager):
@property
def _internal_entry_points(self):
return ['split_path_components = guessit.transfo.split_path_components:SplitPathComponents',
'guess_filetype = guessit.transfo.guess_filetype:GuessFiletype',
'split_explicit_groups = guessit.transfo.split_explicit_groups:SplitExplicitGroups',
'guess_date = guessit.transfo.guess_date:GuessDate',
'guess_website = guessit.transfo.guess_website:GuessWebsite',
'guess_release_group = guessit.transfo.guess_release_group:GuessReleaseGroup',
'guess_properties = guessit.transfo.guess_properties:GuessProperties',
'guess_language = guessit.transfo.guess_language:GuessLanguage',
'guess_video_rexps = guessit.transfo.guess_video_rexps:GuessVideoRexps',
'guess_episodes_rexps = guessit.transfo.guess_episodes_rexps:GuessEpisodesRexps',
'guess_weak_episodes_rexps = guessit.transfo.guess_weak_episodes_rexps:GuessWeakEpisodesRexps',
'guess_bonus_features = guessit.transfo.guess_bonus_features:GuessBonusFeatures',
'guess_year = guessit.transfo.guess_year:GuessYear',
'guess_country = guessit.transfo.guess_country:GuessCountry',
'guess_idnumber = guessit.transfo.guess_idnumber:GuessIdnumber',
'split_on_dash = guessit.transfo.split_on_dash:SplitOnDash',
'guess_episode_info_from_position = guessit.transfo.guess_episode_info_from_position:GuessEpisodeInfoFromPosition',
'guess_movie_title_from_position = guessit.transfo.guess_movie_title_from_position:GuessMovieTitleFromPosition',
'guess_episode_details = guessit.transfo.guess_episode_details:GuessEpisodeDetails',
'expected_series = guessit.transfo.expected_series:ExpectedSeries',
'expected_title = guessit.transfo.expected_title:ExpectedTitle',]
def _find_entry_points(self, namespace):
entry_points = {}
# Internal entry points
if namespace == self.namespace:
for internal_entry_point_str in self._internal_entry_points:
internal_entry_point = EntryPoint.parse(internal_entry_point_str)
entry_points[internal_entry_point.name] = internal_entry_point
# Package entry points
setuptools_entrypoints = super(DefaultTransformerExtensionManager, self)._find_entry_points(namespace)
for setuptools_entrypoint in setuptools_entrypoints:
entry_points[setuptools_entrypoint.name] = setuptools_entrypoint
return list(entry_points.values())
_extensions = None
def all_transformers():
return _extensions.objects()
def get_transformer(name):
return _extensions.object(name)
def add_transformer(name, module_name, class_name):
"""
Add a transformer
:param name: the name of the transformer. ie: 'guess_regexp_id'
:param name: the module name. ie: 'flexget.utils.parsers.transformers.guess_regexp_id'
:param class_name: the class name. ie: 'GuessRegexpId'
"""
_extensions.register_module(name, module_name, (class_name,))
def add_transformer(entry_point):
"""
Add a transformer
:param entry_point: entry point spec format. ie: 'guess_regexp_id = flexget.utils.parsers.transformers.guess_regexp_id:GuessRegexpId'
"""
_extensions.register_module(entry_point = entry_point)
def reload(custom=False):
"""
Reload extension manager with default or custom one.
:param custom: if True, custom manager will be used, else default one.
Default manager will load default extensions from guessit and setuptools packaging extensions
Custom manager will not load default extensions from guessit, using only setuptools packaging extensions.
:type custom: boolean
"""
global _extensions
if custom:
_extensions = CustomTransformerExtensionManager()
else:
_extensions = DefaultTransformerExtensionManager()
reload_options(all_transformers())
reload()
@@ -1,65 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from guessit.plugins.transformers import all_transformers
def best_quality_properties(props, *guesses):
"""Retrieve the best quality guess, based on given properties
:param props: Properties to include in the rating
:type props: list of strings
:param guesses: Guesses to rate
:type guesses: :class:`guessit.guess.Guess`
:return: Best quality guess from all passed guesses
:rtype: :class:`guessit.guess.Guess`
"""
best_guess = None
best_rate = None
for guess in guesses:
for transformer in all_transformers():
rate = transformer.rate_quality(guess, *props)
if best_rate is None or best_rate < rate:
best_rate = rate
best_guess = guess
return best_guess
def best_quality(*guesses):
"""Retrieve the best quality guess.
:param guesses: Guesses to rate
:type guesses: :class:`guessit.guess.Guess`
:return: Best quality guess from all passed guesses
:rtype: :class:`guessit.guess.Guess`
"""
best_guess = None
best_rate = None
for guess in guesses:
for transformer in all_transformers():
rate = transformer.rate_quality(guess)
if best_rate is None or best_rate < rate:
best_rate = rate
best_guess = guess
return best_guess
@@ -1,116 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from functools import wraps
import logging
import sys
import os
log = logging.getLogger(__name__)
GREEN_FONT = "\x1B[0;32m"
YELLOW_FONT = "\x1B[0;33m"
BLUE_FONT = "\x1B[0;34m"
RED_FONT = "\x1B[0;31m"
RESET_FONT = "\x1B[0m"
def setup_logging(colored=True, with_time=False, with_thread=False, filename=None, with_lineno=False): # pragma: no cover
"""Set up a nice colored logger as the main application logger."""
class SimpleFormatter(logging.Formatter):
def __init__(self, with_time, with_thread):
self.fmt = (('%(asctime)s ' if with_time else '') +
'%(levelname)-8s ' +
'[%(name)s:%(funcName)s' +
(':%(lineno)s' if with_lineno else '') + ']' +
('[%(threadName)s]' if with_thread else '') +
' -- %(message)s')
logging.Formatter.__init__(self, self.fmt)
class ColoredFormatter(logging.Formatter):
def __init__(self, with_time, with_thread):
self.fmt = (('%(asctime)s ' if with_time else '') +
'-CC-%(levelname)-8s ' +
BLUE_FONT + '[%(name)s:%(funcName)s' +
(':%(lineno)s' if with_lineno else '') + ']' +
RESET_FONT + ('[%(threadName)s]' if with_thread else '') +
' -- %(message)s')
logging.Formatter.__init__(self, self.fmt)
def format(self, record):
modpath = record.name.split('.')
record.mname = modpath[0]
record.mmodule = '.'.join(modpath[1:])
result = logging.Formatter.format(self, record)
if record.levelno == logging.DEBUG:
color = BLUE_FONT
elif record.levelno == logging.INFO:
color = GREEN_FONT
elif record.levelno == logging.WARNING:
color = YELLOW_FONT
else:
color = RED_FONT
result = result.replace('-CC-', color)
return result
if filename is not None:
# make sure we can write to our log file
logdir = os.path.dirname(filename)
if not os.path.exists(logdir):
os.makedirs(logdir)
ch = logging.FileHandler(filename, mode='w')
ch.setFormatter(SimpleFormatter(with_time, with_thread))
else:
ch = logging.StreamHandler()
if colored and sys.platform != 'win32':
ch.setFormatter(ColoredFormatter(with_time, with_thread))
else:
ch.setFormatter(SimpleFormatter(with_time, with_thread))
logging.getLogger().addHandler(ch)
def trace_func_call(f):
@wraps(f)
def wrapper(*args, **kwargs):
is_method = (f.__name__ != f.__qualname__) # method is still not bound, we need to get around it
if is_method:
no_self_args = args[1:]
else:
no_self_args = args
args_str = ', '.join(repr(arg) for arg in no_self_args)
kwargs_str = ', '.join('{}={}'.format(k, v) for k, v in kwargs.items())
if not args_str:
args_str = kwargs_str
elif not kwargs_str:
args_str = args_str
else:
args_str = '{}, {}'.format(args_str, kwargs_str)
log.debug('Calling {}({})'.format(f.__name__, args_str))
return f(*args, **kwargs)
return wrapper
Binary file not shown.
@@ -1,28 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
import logging
from guessit.slogging import setup_logging
setup_logging()
logging.disable(logging.INFO)
@@ -1,553 +0,0 @@
? Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv
: type: movie
title: Fear and Loathing in Las Vegas
year: 1998
screenSize: 720p
format: HD-DVD
audioCodec: DTS
videoCodec: h264
releaseGroup: ESiR
? Leopard.dmg
: type: unknown
extension: dmg
? Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi
: type: episode
series: Duckman
season: 1
episodeNumber: 1
title: I, Duckman
date: 2002-11-07
? Series/Neverwhere/Neverwhere.05.Down.Street.[tvu.org.ru].avi
: type: episode
series: Neverwhere
episodeNumber: 5
title: Down Street
website: tvu.org.ru
? Neverwhere.05.Down.Street.[tvu.org.ru].avi
: type: episode
series: Neverwhere
episodeNumber: 5
title: Down Street
website: tvu.org.ru
? Series/Breaking Bad/Minisodes/Breaking.Bad.(Minisodes).01.Good.Cop.Bad.Cop.WEBRip.XviD.avi
: type: episode
series: Breaking Bad
episodeFormat: Minisode
episodeNumber: 1
title: Good Cop Bad Cop
format: WEBRip
videoCodec: XviD
? Series/Kaamelott/Kaamelott - Livre V - Ep 23 - Le Forfait.avi
: type: episode
series: Kaamelott
episodeNumber: 23
title: Le Forfait
? Movies/The Doors (1991)/09.03.08.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
: type: movie
title: The Doors
year: 1991
date: 2008-03-09
format: BluRay
screenSize: 720p
audioCodec: AC3
videoCodec: h264
releaseGroup: HiS@SiLUHD
language: english
website: sharethefiles.com
? Movies/M.A.S.H. (1970)/MASH.(1970).[Divx.5.02][Dual-Subtitulos][DVDRip].ogm
: type: movie
title: M.A.S.H.
year: 1970
videoCodec: DivX
format: DVD
? the.mentalist.501.hdtv-lol.mp4
: type: episode
series: The Mentalist
season: 5
episodeNumber: 1
format: HDTV
releaseGroup: LOL
? the.simpsons.2401.hdtv-lol.mp4
: type: episode
series: The Simpsons
season: 24
episodeNumber: 1
format: HDTV
releaseGroup: LOL
? Homeland.S02E01.HDTV.x264-EVOLVE.mp4
: type: episode
series: Homeland
season: 2
episodeNumber: 1
format: HDTV
videoCodec: h264
releaseGroup: EVOLVE
? /media/Band_of_Brothers-e01-Currahee.mkv
: type: episode
series: Band of Brothers
episodeNumber: 1
title: Currahee
? /media/Band_of_Brothers-x02-We_Stand_Alone_Together.mkv
: type: episode
series: Band of Brothers
bonusNumber: 2
bonusTitle: We Stand Alone Together
? /movies/James_Bond-f21-Casino_Royale-x02-Stunts.mkv
: type: movie
title: Casino Royale
filmSeries: James Bond
filmNumber: 21
bonusNumber: 2
bonusTitle: Stunts
? /TV Shows/new.girl.117.hdtv-lol.mp4
: type: episode
series: New Girl
season: 1
episodeNumber: 17
format: HDTV
releaseGroup: LOL
? The.Office.(US).1x03.Health.Care.HDTV.XviD-LOL.avi
: type: episode
series: The Office (US)
country: US
season: 1
episodeNumber: 3
title: Health Care
format: HDTV
videoCodec: XviD
releaseGroup: LOL
? The_Insider-(1999)-x02-60_Minutes_Interview-1996.mp4
: type: movie
title: The Insider
year: 1999
bonusNumber: 2
bonusTitle: 60 Minutes Interview-1996
? OSS_117--Cairo,_Nest_of_Spies.mkv
: type: movie
title: OSS 117--Cairo, Nest of Spies
? Rush.._Beyond_The_Lighted_Stage-x09-Between_Sun_and_Moon-2002_Hartford.mkv
: type: movie
title: Rush Beyond The Lighted Stage
bonusNumber: 9
bonusTitle: Between Sun and Moon-2002 Hartford
? House.Hunters.International.S56E06.720p.hdtv.x264.mp4
: type: episode
series: House Hunters International
season: 56
episodeNumber: 6
screenSize: 720p
format: HDTV
videoCodec: h264
? White.House.Down.2013.1080p.BluRay.DTS-HD.MA.5.1.x264-PublicHD.mkv
: type: movie
title: White House Down
year: 2013
screenSize: 1080p
format: BluRay
audioCodec: DTS
audioProfile: HDMA
videoCodec: h264
releaseGroup: PublicHD
audioChannels: "5.1"
? White.House.Down.2013.1080p.BluRay.DTSHD.MA.5.1.x264-PublicHD.mkv
: type: movie
title: White House Down
year: 2013
screenSize: 1080p
format: BluRay
audioCodec: DTS
audioProfile: HDMA
videoCodec: h264
releaseGroup: PublicHD
audioChannels: "5.1"
? Hostages.S01E01.Pilot.for.Air.720p.WEB-DL.DD5.1.H.264-NTb.nfo
: type: episodeinfo
series: Hostages
title: Pilot for Air
season: 1
episodeNumber: 1
screenSize: 720p
format: WEB-DL
audioChannels: "5.1"
videoCodec: h264
audioCodec: DolbyDigital
releaseGroup: NTb
? Despicable.Me.2.2013.1080p.BluRay.x264-VeDeTT.nfo
: type: movieinfo
title: Despicable Me 2
year: 2013
screenSize: 1080p
format: BluRay
videoCodec: h264
releaseGroup: VeDeTT
? Le Cinquieme Commando 1971 SUBFORCED FRENCH DVDRiP XViD AC3 Bandix.mkv
: type: movie
audioCodec: AC3
format: DVD
releaseGroup: Bandix
subtitleLanguage: French
title: Le Cinquieme Commando
videoCodec: XviD
year: 1971
? Le Seigneur des Anneaux - La Communauté de l'Anneau - Version Longue - BDRip.mkv
: type: movie
format: BluRay
title: Le Seigneur des Anneaux
? La petite bande (Michel Deville - 1983) VF PAL MP4 x264 AAC.mkv
: type: movie
audioCodec: AAC
language: French
title: La petite bande
videoCodec: h264
year: 1983
other: PAL
? Retour de Flammes (Gregor Schnitzler 2003) FULL DVD.iso
: type: movie
format: DVD
title: Retour de Flammes
type: movie
year: 2003
? A.Common.Title.Special.2014.avi
: type: movie
year: 2014
title: A Common Title Special
? A.Common.Title.2014.Special.avi
: type: episode
year: 2014
series: A Common Title
title: Special
episodeDetails: Special
? A.Common.Title.2014.Special.Edition.avi
: type: movie
year: 2014
title: A Common Title
edition: Special Edition
? Downton.Abbey.2013.Christmas.Special.HDTV.x264-FoV.mp4
: type: episode
year: 2013
series: Downton Abbey
title: Christmas Special
videoCodec: h264
releaseGroup: FoV
format: HDTV
episodeDetails: Special
? Doctor_Who_2013_Christmas_Special.The_Time_of_The_Doctor.HD
: options: -n
type: episode
series: Doctor Who
other: HD
episodeDetails: Special
title: Christmas Special The Time of The Doctor
year: 2013
? Doctor Who 2005 50th Anniversary Special The Day of the Doctor 3.avi
: type: episode
series: Doctor Who
episodeDetails: Special
title: 50th Anniversary Special The Day of the Doctor 3
year: 2005
? Robot Chicken S06-Born Again Virgin Christmas Special HDTV x264.avi
: type: episode
series: Robot Chicken
format: HDTV
season: 6
title: Born Again Virgin Christmas Special
videoCodec: h264
episodeDetails: Special
? Wicked.Tuna.S03E00.Head.To.Tail.Special.HDTV.x264-YesTV
: options: -n
type: episode
series: Wicked Tuna
title: Head To Tail Special
releaseGroup: YesTV
season: 3
episodeNumber: 0
videoCodec: h264
format: HDTV
episodeDetails: Special
? The.Voice.UK.S03E12.HDTV.x264-C4TV
: options: -n
episodeNumber: 12
videoCodec: h264
format: HDTV
series: The Voice (UK)
releaseGroup: C4TV
season: 3
country: United Kingdom
type: episode
? /tmp/star.trek.9/star.trek.9.mkv
: type: movie
title: star trek 9
? star.trek.9.mkv
: type: movie
title: star trek 9
? FlexGet.S01E02.TheName.HDTV.xvid
: options: -n
episodeNumber: 2
format: HDTV
season: 1
series: FlexGet
title: TheName
type: episode
videoCodec: XviD
? FlexGet.S01E02.TheName.HDTV.xvid
: options: -n
episodeNumber: 2
format: HDTV
season: 1
series: FlexGet
title: TheName
type: episode
videoCodec: XviD
? some.series.S03E14.Title.Here.720p
: options: -n
episodeNumber: 14
screenSize: 720p
season: 3
series: some series
title: Title Here
type: episode
? '[the.group] Some.Series.S03E15.Title.Two.720p'
: options: -n
episodeNumber: 15
releaseGroup: the.group
screenSize: 720p
season: 3
series: Some Series
title: Title Two
type: episode
? 'HD 720p: Some series.S03E16.Title.Three'
: options: -n
episodeNumber: 16
other: HD
screenSize: 720p
season: 3
series: Some series
title: Title Three
type: episode
? Something.Season.2.1of4.Ep.Title.HDTV.torrent
: episodeCount: 4
episodeNumber: 1
format: HDTV
season: 2
series: Something
title: Title
type: episode
extension: torrent
? Show-A (US) - Episode Title S02E09 hdtv
: options: -n
country: US
episodeNumber: 9
format: HDTV
season: 2
series: Show-A (US)
type: episode
? Jack's.Show.S03E01.blah.1080p
: options: -n
episodeNumber: 1
screenSize: 1080p
season: 3
series: Jack's Show
title: blah
type: episode
? FlexGet.epic
: options: -n
title: FlexGet epic
type: movie
? FlexGet.Apt.1
: options: -n
title: FlexGet Apt 1
type: movie
? FlexGet.aptitude
: options: -n
title: FlexGet aptitude
type: movie
? FlexGet.Step1
: options: -n
title: FlexGet Step1
type: movie
? Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720 * 432].avi
: format: DVD
screenSize: 720x432
title: El Bosque Animado
videoCodec: XviD
year: 1987
type: movie
? Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720x432].avi
: format: DVD
screenSize: 720x432
title: El Bosque Animado
videoCodec: XviD
year: 1987
type: movie
? 2009.shoot.fruit.chan.multi.dvd9.pal
: options: -n
format: DVD
language: mul
other: PAL
title: shoot fruit chan
type: movie
year: 2009
? 2009.shoot.fruit.chan.multi.dvd5.pal
: options: -n
format: DVD
language: mul
other: PAL
title: shoot fruit chan
type: movie
year: 2009
? The.Flash.2014.S01E01.PREAIR.WEBRip.XviD-EVO.avi
: episodeNumber: 1
format: WEBRip
other: Preair
releaseGroup: EVO
season: 1
series: The Flash
type: episode
videoCodec: XviD
year: 2014
? Ice.Lake.Rebels.S01E06.Ice.Lake.Games.720p.HDTV.x264-DHD
: options: -n
episodeNumber: 6
format: HDTV
releaseGroup: DHD
screenSize: 720p
season: 1
series: Ice Lake Rebels
title: Ice Lake Games
type: episode
videoCodec: h264
? The League - S06E10 - Epi Sexy.mkv
: episodeNumber: 10
season: 6
series: The League
title: Epi Sexy
type: episode
? Stay (2005) [1080p]/Stay.2005.1080p.BluRay.x264.YIFY.mp4
: format: BluRay
releaseGroup: YIFY
screenSize: 1080p
title: Stay
type: movie
videoCodec: h264
year: 2005
? /media/live/A/Anger.Management.S02E82.720p.HDTV.X264-DIMENSION.mkv
: format: HDTV
releaseGroup: DIMENSION
screenSize: 720p
series: Anger Management
type: episode
season: 2
episodeNumber: 82
videoCodec: h264
? "[Figmentos] Monster 34 - At the End of Darkness [781219F1].mkv"
: type: episode
releaseGroup: Figmentos
series: Monster
episodeNumber: 34
title: At the End of Darkness
crc32: 781219F1
? Game.of.Thrones.S05E07.720p.HDTV-KILLERS.mkv
: type: episode
episodeNumber: 7
format: HDTV
releaseGroup: KILLERS
screenSize: 720p
season: 5
series: Game of Thrones
? Game.of.Thrones.S05E07.HDTV.720p-KILLERS.mkv
: type: episode
episodeNumber: 7
format: HDTV
releaseGroup: KILLERS
screenSize: 720p
season: 5
series: Game of Thrones
? Parks and Recreation - [04x12] - Ad Campaign.avi
: type: episode
series: Parks and Recreation
season: 4
episodeNumber: 12
title: Ad Campaign
? Star Trek Into Darkness (2013)/star.trek.into.darkness.2013.720p.web-dl.h264-publichd.mkv
: type: movie
title: Star Trek Into Darkness
year: 2013
screenSize: 720p
format: WEB-DL
videoCodec: h264
releaseGroup: PublicHD
? /var/medias/series/The Originals/Season 02/The.Originals.S02E15.720p.HDTV.X264-DIMENSION.mkv
: type: episode
series: The Originals
season: 2
episodeNumber: 15
screenSize: 720p
format: HDTV
videoCodec: h264
releaseGroup: DIMENSION
@@ -1 +0,0 @@
Just a dummy srt file (used for unittests: do not remove!)
File diff suppressed because it is too large Load Diff
@@ -1,184 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from collections import defaultdict
from unittest import TestCase, TestLoader
import logging
import os
import sys
from os.path import *
import babelfish
import yaml
def currentPath():
"""Returns the path in which the calling file is located."""
return dirname(join(os.getcwd(), sys._getframe(1).f_globals['__file__']))
def addImportPath(path):
"""Function that adds the specified path to the import path. The path can be
absolute or relative to the calling file."""
importPath = abspath(join(currentPath(), path))
sys.path = [importPath] + sys.path
log = logging.getLogger(__name__)
from guessit.options import get_opts
from guessit import base_text_type
from guessit import *
from guessit.matcher import *
from guessit.fileutils import *
import guessit
def allTests(testClass):
return TestLoader().loadTestsFromTestCase(testClass)
class TestGuessit(TestCase):
def checkMinimumFieldsCorrect(self, filename, filetype=None, remove_type=True,
exclude_files=None):
groundTruth = yaml.load(load_file_in_same_dir(__file__, filename))
def guess_func(string, options=None):
return guess_file_info(string, options=options, type=filetype)
return self.checkFields(groundTruth, guess_func, remove_type, exclude_files)
def checkFields(self, groundTruth, guess_func, remove_type=True,
exclude_files=None):
total = 0
exclude_files = exclude_files or []
fails = defaultdict(list)
additionals = defaultdict(list)
for filename, required_fields in groundTruth.items():
filename = u(filename)
if filename in exclude_files:
continue
log.debug('\n' + '-' * 120)
log.info('Guessing information for file: %s' % filename)
options = required_fields.pop('options') if 'options' in required_fields else None
try:
found = guess_func(filename, options)
except Exception as e:
fails[filename].append("An exception has occured in %s: %s" % (filename, e))
log.exception("An exception has occured in %s: %s" % (filename, e))
continue
total += 1
# no need for these in the unittests
if remove_type:
try:
del found['type']
except:
pass
for prop in ('container', 'mimetype', 'unidentified'):
if prop in found:
del found[prop]
# props which are list of just 1 elem should be opened for easier writing of the tests
for prop in ('language', 'subtitleLanguage', 'other', 'episodeDetails', 'unidentified'):
value = found.get(prop, None)
if isinstance(value, list) and len(value) == 1:
found[prop] = value[0]
# look for missing properties
for prop, value in required_fields.items():
if prop not in found:
log.debug("Prop '%s' not found in: %s" % (prop, filename))
fails[filename].append("'%s' not found in: %s" % (prop, filename))
continue
# if both properties are strings, do a case-insensitive comparison
if (isinstance(value, base_text_type) and
isinstance(found[prop], base_text_type)):
if value.lower() != found[prop].lower():
log.debug("Wrong prop value [str] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(value, list) and isinstance(found[prop], list):
if found[prop] and isinstance(found[prop][0], babelfish.Language):
# list of languages
s1 = set(Language.fromguessit(s) for s in value)
s2 = set(found[prop])
else:
# by default we assume list of strings and do a case-insensitive
# comparison on their elements
s1 = set(u(s).lower() for s in value)
s2 = set(u(s).lower() for s in found[prop])
if s1 != s2:
log.debug("Wrong prop value [list] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Language):
try:
if babelfish.Language.fromguessit(value) != found[prop]:
raise ValueError
except:
log.debug("Wrong prop value [Language] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Country):
try:
if babelfish.Country.fromguessit(value) != found[prop]:
raise ValueError
except:
log.debug("Wrong prop value [Country] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
# otherwise, just compare their values directly
else:
if found[prop] != value:
log.debug("Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
fails[filename].append("'%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
# look for additional properties
for prop, value in found.items():
if prop not in required_fields:
log.debug("Found additional info for prop = '%s': '%s'" % (prop, u(value)))
additionals[filename].append("'%s': '%s'" % (prop, u(value)))
correct = total - len(fails)
log.info('SUMMARY: Guessed correctly %d out of %d filenames' % (correct, total))
for failed_entry, failed_properties in fails.items():
log.error('---- ' + failed_entry + ' ----')
for failed_property in failed_properties:
log.error("FAILED: " + failed_property)
for additional_entry, additional_properties in additionals.items():
log.warning('---- ' + additional_entry + ' ----')
for additional_property in additional_properties:
log.warning("ADDITIONAL: " + additional_property)
assert correct == total, 'Correct: %d < Total: %d' % (correct, total)
@@ -1,779 +0,0 @@
? Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv
: title: Fear and Loathing in Las Vegas
year: 1998
screenSize: 720p
format: HD-DVD
audioCodec: DTS
videoCodec: h264
releaseGroup: ESiR
? Movies/El Dia de la Bestia (1995)/El.dia.de.la.bestia.DVDrip.Spanish.DivX.by.Artik[SEDG].avi
: title: El Dia de la Bestia
year: 1995
format: DVD
language: spanish
videoCodec: DivX
releaseGroup: Artik[SEDG]
? Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv
: title: Dark City
year: 1998
format: BluRay
screenSize: 720p
audioCodec: DTS
videoCodec: h264
releaseGroup: CHD
? Movies/Sin City (BluRay) (2005)/Sin.City.2005.BDRip.720p.x264.AC3-SEPTiC.mkv
: title: Sin City
year: 2005
format: BluRay
screenSize: 720p
videoCodec: h264
audioCodec: AC3
releaseGroup: SEPTiC
? Movies/Borat (2006)/Borat.(2006).R5.PROPER.REPACK.DVDRip.XviD-PUKKA.avi
: title: Borat
year: 2006
other: PROPER
properCount: 2
format: DVD
other: [ R5, Proper ]
videoCodec: XviD
releaseGroup: PUKKA
? "[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv"
: title: Le Prestige
format: DVD
videoCodec: h264
videoProfile: HP
audioCodec: AAC
audioProfile: HE
language: [ french, english ]
subtitleLanguage: [ french, english ]
releaseGroup: XCT
? Battle Royale (2000)/Battle.Royale.(Batoru.Rowaiaru).(2000).(Special.Edition).CD1of2.DVDRiP.XviD-[ZeaL].avi
: title: Battle Royale
year: 2000
edition: special edition
cdNumber: 1
cdNumberTotal: 2
format: DVD
videoCodec: XviD
releaseGroup: ZeaL
? Movies/Brazil (1985)/Brazil_Criterion_Edition_(1985).CD2.avi
: title: Brazil
edition: Criterion Edition
year: 1985
cdNumber: 2
? Movies/Persepolis (2007)/[XCT] Persepolis [H264+Aac-128(Fr-Eng)+ST(Fr-Eng)+Ind].mkv
: title: Persepolis
year: 2007
videoCodec: h264
audioCodec: AAC
language: [ French, English ]
subtitleLanguage: [ French, English ]
releaseGroup: XCT
? Movies/Toy Story (1995)/Toy Story [HDTV 720p English-Spanish].mkv
: title: Toy Story
year: 1995
format: HDTV
screenSize: 720p
language: [ english, spanish ]
? Movies/Office Space (1999)/Office.Space.[Dual-DVDRip].[Spanish-English].[XviD-AC3-AC3].[by.Oswald].avi
: title: Office Space
year: 1999
format: DVD
language: [ english, spanish ]
videoCodec: XviD
audioCodec: AC3
? Movies/Wild Zero (2000)/Wild.Zero.DVDivX-EPiC.avi
: title: Wild Zero
year: 2000
videoCodec: DivX
releaseGroup: EPiC
? movies/Baraka_Edition_Collector.avi
: title: Baraka
edition: collector edition
? Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director's.Cut).CD1.DVDRip.XviD.AC3-WAF.avi
: title: Blade Runner
year: 1982
edition: Director's Cut
cdNumber: 1
format: DVD
videoCodec: XviD
audioCodec: AC3
releaseGroup: WAF
? movies/American.The.Bill.Hicks.Story.2009.DVDRip.XviD-EPiSODE.[UsaBit.com]/UsaBit.com_esd-americanbh.avi
: title: American The Bill Hicks Story
year: 2009
format: DVD
videoCodec: XviD
releaseGroup: EPiSODE
website: UsaBit.com
? movies/Charlie.And.Boots.DVDRip.XviD-TheWretched/wthd-cab.avi
: title: Charlie And Boots
format: DVD
videoCodec: XviD
releaseGroup: TheWretched
? movies/Steig Larsson Millenium Trilogy (2009) BRrip 720 AAC x264/(1)The Girl With The Dragon Tattoo (2009) BRrip 720 AAC x264.mkv
: title: The Girl With The Dragon Tattoo
filmSeries: Steig Larsson Millenium Trilogy
filmNumber: 1
year: 2009
format: BluRay
audioCodec: AAC
videoCodec: h264
screenSize: 720p
? movies/Greenberg.REPACK.LiMiTED.DVDRip.XviD-ARROW/arw-repack-greenberg.dvdrip.xvid.avi
: title: Greenberg
format: DVD
videoCodec: XviD
releaseGroup: ARROW
other: ['Proper', 'Limited']
properCount: 2
? Movies/Fr - Paris 2054, Renaissance (2005) - De Christian Volckman - (Film Divx Science Fiction Fantastique Thriller Policier N&B).avi
: title: Paris 2054, Renaissance
year: 2005
language: french
videoCodec: DivX
? Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
: title: Avida
year: 2006
language: french
format: DVD
videoCodec: XviD
releaseGroup: PROD
? Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
: title: Alice in Wonderland
format: DVD
videoCodec: XviD
releaseGroup: DiAMOND
? Movies/Ne.Le.Dis.A.Personne.Fr 2 cd/personnea_mp.avi
: title: Ne Le Dis A Personne
language: french
cdNumberTotal: 2
? Movies/Bunker Palace Hôtel (Enki Bilal) (1989)/Enki Bilal - Bunker Palace Hotel (Fr Vhs Rip).avi
: title: Bunker Palace Hôtel
year: 1989
language: french
format: VHS
? Movies/21 (2008)/21.(2008).DVDRip.x264.AC3-FtS.[sharethefiles.com].mkv
: title: "21"
year: 2008
format: DVD
videoCodec: h264
audioCodec: AC3
releaseGroup: FtS
website: sharethefiles.com
? Movies/9 (2009)/9.2009.Blu-ray.DTS.720p.x264.HDBRiSe.[sharethefiles.com].mkv
: title: "9"
year: 2009
format: BluRay
audioCodec: DTS
screenSize: 720p
videoCodec: h264
releaseGroup: HDBRiSe
website: sharethefiles.com
? Movies/Mamma.Mia.2008.DVDRip.AC3.XviD-CrazyTeam/Mamma.Mia.2008.DVDRip.AC3.XviD-CrazyTeam.avi
: title: Mamma Mia
year: 2008
format: DVD
audioCodec: AC3
videoCodec: XviD
releaseGroup: CrazyTeam
? Movies/M.A.S.H. (1970)/MASH.(1970).[Divx.5.02][Dual-Subtitulos][DVDRip].ogm
: title: M.A.S.H.
year: 1970
videoCodec: DivX
format: DVD
? Movies/The Doors (1991)/09.03.08.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
: title: The Doors
year: 1991
date: 2008-03-09
format: BluRay
screenSize: 720p
audioCodec: AC3
videoCodec: h264
releaseGroup: HiS@SiLUHD
language: english
website: sharethefiles.com
? Movies/The Doors (1991)/08.03.09.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
: options: --date-year-first
title: The Doors
year: 1991
date: 2008-03-09
format: BluRay
screenSize: 720p
audioCodec: AC3
videoCodec: h264
releaseGroup: HiS@SiLUHD
language: english
website: sharethefiles.com
? Movies/Ratatouille/video_ts-ratatouille.srt
: title: Ratatouille
format: DVD
? Movies/001 __ A classer/Fantomas se déchaine - Louis de Funès.avi
: title: Fantomas se déchaine
? Movies/Comme une Image (2004)/Comme.Une.Image.FRENCH.DVDRiP.XViD-NTK.par-www.divx-overnet.com.avi
: title: Comme une Image
year: 2004
language: french
format: DVD
videoCodec: XviD
releaseGroup: NTK
website: www.divx-overnet.com
? Movies/Fantastic Mr Fox/Fantastic.Mr.Fox.2009.DVDRip.{x264+LC-AAC.5.1}{Fr-Eng}{Sub.Fr-Eng}-™.[sharethefiles.com].mkv
: title: Fantastic Mr Fox
year: 2009
format: DVD
videoCodec: h264
audioCodec: AAC
audioProfile: LC
audioChannels: "5.1"
language: [ french, english ]
subtitleLanguage: [ french, english ]
website: sharethefiles.com
? Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi
: title: Somewhere
year: 2010
format: DVD
videoCodec: XviD
releaseGroup: iLG
? Movies/Moon_(2009).mkv
: title: Moon
year: 2009
? Movies/Moon_(2009)-x01.mkv
: title: Moon
year: 2009
bonusNumber: 1
? Movies/Moon_(2009)-x02-Making_Of.mkv
: title: Moon
year: 2009
bonusNumber: 2
bonusTitle: Making Of
? movies/James_Bond-f17-Goldeneye.mkv
: title: Goldeneye
filmSeries: James Bond
filmNumber: 17
? /movies/James_Bond-f21-Casino_Royale.mkv
: title: Casino Royale
filmSeries: James Bond
filmNumber: 21
? /movies/James_Bond-f21-Casino_Royale-x01-Becoming_Bond.mkv
: title: Casino Royale
filmSeries: James Bond
filmNumber: 21
bonusNumber: 1
bonusTitle: Becoming Bond
? /movies/James_Bond-f21-Casino_Royale-x02-Stunts.mkv
: title: Casino Royale
filmSeries: James Bond
filmNumber: 21
bonusNumber: 2
bonusTitle: Stunts
? OSS_117--Cairo,_Nest_of_Spies.mkv
: title: OSS 117--Cairo, Nest of Spies
? The Godfather Part III.mkv
: title: The Godfather
part: 3
? Foobar Part VI.mkv
: title: Foobar
part: 6
? The_Insider-(1999)-x02-60_Minutes_Interview-1996.mp4
: title: The Insider
year: 1999
bonusNumber: 2
bonusTitle: 60 Minutes Interview-1996
? Rush.._Beyond_The_Lighted_Stage-x09-Between_Sun_and_Moon-2002_Hartford.mkv
: title: Rush Beyond The Lighted Stage
bonusNumber: 9
bonusTitle: Between Sun and Moon-2002 Hartford
? /public/uTorrent/Downloads Finished/Movies/Indiana.Jones.and.the.Temple.of.Doom.1984.HDTV.720p.x264.AC3.5.1-REDµX/Indiana.Jones.and.the.Temple.of.Doom.1984.HDTV.720p.x264.AC3.5.1-REDµX.mkv
: title: Indiana Jones and the Temple of Doom
year: 1984
format: HDTV
screenSize: 720p
videoCodec: h264
audioCodec: AC3
audioChannels: "5.1"
releaseGroup: REDµX
? The.Directors.Notebook.2006.Blu-Ray.x264.DXVA.720p.AC3-de[42].mkv
: title: The Directors Notebook
year: 2006
format: BluRay
videoCodec: h264
videoApi: DXVA
screenSize: 720p
audioCodec: AC3
releaseGroup: de[42]
? Movies/Cosmopolis.2012.LiMiTED.720p.BluRay.x264-AN0NYM0US[bb]/ano-cosmo.720p.mkv
: title: Cosmopolis
year: 2012
screenSize: 720p
videoCodec: h264
releaseGroup: AN0NYM0US[bb]
format: BluRay
other: LIMITED
? movies/La Science des Rêves (2006)/La.Science.Des.Reves.FRENCH.DVDRip.XviD-MP-AceBot.avi
: title: La Science des Rêves
year: 2006
format: DVD
videoCodec: XviD
videoProfile: MP
releaseGroup: AceBot
language: French
? The_Italian_Job.mkv
: title: The Italian Job
? The.Rum.Diary.2011.1080p.BluRay.DTS.x264.D-Z0N3.mkv
: title: The Rum Diary
year: 2011
screenSize: 1080p
format: BluRay
videoCodec: h264
audioCodec: DTS
releaseGroup: D-Z0N3
? Life.Of.Pi.2012.1080p.BluRay.DTS.x264.D-Z0N3.mkv
: title: Life Of Pi
year: 2012
screenSize: 1080p
format: BluRay
videoCodec: h264
audioCodec: DTS
releaseGroup: D-Z0N3
? The.Kings.Speech.2010.1080p.BluRay.DTS.x264.D Z0N3.mkv
: title: The Kings Speech
year: 2010
screenSize: 1080p
format: BluRay
audioCodec: DTS
videoCodec: h264
releaseGroup: D Z0N3
? Street.Kings.2008.BluRay.1080p.DTS.x264.dxva EuReKA.mkv
: title: Street Kings
year: 2008
format: BluRay
screenSize: 1080p
audioCodec: DTS
videoCodec: h264
videoApi: DXVA
releaseGroup: EuReKa
? 2001.A.Space.Odyssey.1968.HDDVD.1080p.DTS.x264.dxva EuReKA.mkv
: title: 2001 A Space Odyssey
year: 1968
format: HD-DVD
screenSize: 1080p
audioCodec: DTS
videoCodec: h264
videoApi: DXVA
releaseGroup: EuReKa
? 2012.2009.720p.BluRay.x264.DTS WiKi.mkv
: title: "2012"
year: 2009
screenSize: 720p
format: BluRay
videoCodec: h264
audioCodec: DTS
releaseGroup: WiKi
? /share/Download/movie/Dead Man Down (2013) BRRiP XViD DD5_1 Custom NLSubs =-_lt Q_o_Q gt-=_/XD607ebb-BRc59935-5155473f-1c5f49/XD607ebb-BRc59935-5155473f-1c5f49.avi
: title: Dead Man Down
year: 2013
format: BluRay
videoCodec: XviD
audioChannels: "5.1"
audioCodec: DolbyDigital
idNumber: XD607ebb-BRc59935-5155473f-1c5f49
? Pacific.Rim.3D.2013.COMPLETE.BLURAY-PCH.avi
: title: Pacific Rim
year: 2013
format: BluRay
other:
- complete
- 3D
releaseGroup: PCH
? Immersion.French.2011.STV.READNFO.QC.FRENCH.ENGLISH.NTSC.DVDR.nfo
: title: Immersion French
year: 2011
language:
- French
- English
format: DVD
other: NTSC
? Immersion.French.2011.STV.READNFO.QC.FRENCH.NTSC.DVDR.nfo
: title: Immersion French
year: 2011
language: French
format: DVD
other: NTSC
? Immersion.French.2011.STV.READNFO.QC.NTSC.DVDR.nfo
: title: Immersion French
year: 2011
format: DVD
other: NTSC
? French.Immersion.2011.STV.READNFO.QC.ENGLISH.NTSC.DVDR.nfo
: title: French Immersion
year: 2011
language: ENGLISH
format: DVD
other: NTSC
? Howl's_Moving_Castle_(2004)_[720p,HDTV,x264,DTS]-FlexGet.avi
: videoCodec: h264
format: HDTV
title: Howl's Moving Castle
screenSize: 720p
year: 2004
audioCodec: DTS
releaseGroup: FlexGet
? Pirates de langkasuka.2008.FRENCH.1920X1080.h264.AVC.AsiaRa.mkv
: screenSize: 1080p
year: 2008
language: French
videoCodec: h264
title: Pirates de langkasuka
releaseGroup: AsiaRa
? Masala (2013) Telugu Movie HD DVDScr XviD - Exclusive.avi
: year: 2013
videoCodec: XviD
title: Masala
format: HD-DVD
other: screener
language: Telugu
releaseGroup: Exclusive
? Django Unchained 2012 DVDSCR X264 AAC-P2P.nfo
: year: 2012
other: screener
videoCodec: h264
title: Django Unchained
audioCodec: AAC
format: DVD
releaseGroup: P2P
? Ejecutiva.En.Apuros(2009).BLURAY.SCR.Xvid.Spanish.LanzamientosD.nfo
: year: 2009
other: screener
format: BluRay
videoCodec: XviD
language: Spanish
title: Ejecutiva En Apuros
? Die.Schluempfe.2.German.DL.1080p.BluRay.x264-EXQUiSiTE.mkv
: title: Die Schluempfe 2
format: BluRay
language:
- Multiple languages
- German
videoCodec: h264
releaseGroup: EXQUiSiTE
screenSize: 1080p
? Rocky 1976 French SubForced BRRip x264 AC3-FUNKY.mkv
: title: Rocky
year: 1976
subtitleLanguage: French
format: BluRay
videoCodec: h264
audioCodec: AC3
releaseGroup: FUNKY
? REDLINE (BD 1080p H264 10bit FLAC) [3xR].mkv
: title: REDLINE
format: BluRay
videoCodec: h264
videoProfile: 10bit
audioCodec: Flac
screenSize: 1080p
? The.Lizzie.McGuire.Movie.(2003).HR.DVDRiP.avi
: title: The Lizzie McGuire Movie
year: 2003
format: DVD
other: HR
? Hua.Mulan.BRRIP.MP4.x264.720p-HR.avi
: title: Hua Mulan
videoCodec: h264
format: BluRay
screenSize: 720p
other: HR
? Dr.Seuss.The.Lorax.2012.DVDRip.LiNE.XviD.AC3.HQ.Hive-CM8.mp4
: videoCodec: XviD
title: Dr Seuss The Lorax
format: DVD
other: LiNE
year: 2012
audioCodec: AC3
audioProfile: HQ
releaseGroup: Hive-CM8
? "Star Wars: Episode IV - A New Hope (2004) Special Edition.MKV"
: title: Star Wars Episode IV
year: 2004
edition: Special Edition
? Dr.LiNE.The.Lorax.2012.DVDRip.LiNE.XviD.AC3.HQ.Hive-CM8.mp4
: videoCodec: XviD
title: Dr LiNE The Lorax
format: DVD
other: LiNE
year: 2012
audioCodec: AC3
audioProfile: HQ
releaseGroup: Hive-CM8
? Perfect Child-2007-TRUEFRENCH-TVRip.Xvid-h@mster.avi
: releaseGroup: h@mster
title: Perfect Child
videoCodec: XviD
language: French
format: TV
year: 2007
? entre.ciel.et.terre.(1994).dvdrip.h264.aac-psypeon.avi
: audioCodec: AAC
format: DVD
releaseGroup: psypeon
title: entre ciel et terre
videoCodec: h264
year: 1994
? Yves.Saint.Laurent.2013.FRENCH.DVDSCR.MD.XviD-ViVARiUM.avi
: format: DVD
language: French
other:
- MD
- Screener
releaseGroup: ViVARiUM
title: Yves Saint Laurent
videoCodec: XviD
year: 2013
? Echec et Mort - Hard to Kill - Steven Seagal Multi 1080p BluRay x264 CCATS.avi
: format: BluRay
language: Multiple languages
releaseGroup: CCATS
screenSize: 1080p
title: Echec et Mort
videoCodec: h264
? Paparazzi - Timsit/Lindon (MKV 1080p tvripHD)
: options: -n
title: Paparazzi
screenSize: 1080p
format: HDTV
? some.movie.720p.bluray.x264-mind
: options: -n
title: some movie
screenSize: 720p
videoCodec: h264
releaseGroup: mind
format: BluRay
? Dr LiNE The Lorax 720p h264 BluRay
: options: -n
title: Dr LiNE The Lorax
screenSize: 720p
videoCodec: h264
format: BluRay
? BeatdownFrenchDVDRip.mkv
: options: -c
title: Beatdown
language: French
format: DVD
? YvesSaintLaurent2013FrenchDVDScrXvid.avi
: options: -c
format: DVD
language: French
other: Screener
title: Yves saint laurent
videoCodec: XviD
year: 2013
? Elle.s.en.va.720p.mkv
: screenSize: 720p
title: Elle s en va
? FooBar.7.PDTV-FlexGet
: options: -n
format: DVB
releaseGroup: FlexGet
title: FooBar 7
? h265 - HEVC Riddick Unrated Director Cut French 1080p DTS.mkv
: audioCodec: DTS
edition: Director's cut
language: fr
screenSize: 1080p
title: Riddick Unrated
videoCodec: h265
? "[h265 - HEVC] Riddick Unrated Director Cut French [1080p DTS].mkv"
: audioCodec: DTS
edition: Director's cut
language: fr
screenSize: 1080p
title: Riddick Unrated
videoCodec: h265
? Barbecue-2014-French-mHD-1080p
: options: -n
language: fr
other: mHD
screenSize: 1080p
title: Barbecue
year: 2014
? Underworld Quadrilogie VO+VFF+VFQ 1080p HDlight.x264~Tonyk~Monde Infernal
: options: -n
language:
- fr
- vo
other: HDLight
screenSize: 1080p
title: Underworld Quadrilogie
videoCodec: h264
? A Bout Portant (The Killers).PAL.Multi.DVD-R-KZ
: options: -n
format: DVD
language: mul
releaseGroup: KZ
title: A Bout Portant
? "Mise à Sac (Alain Cavalier, 1967) [Vhs.Rip.Vff]"
: options: -n
format: VHS
language: fr
title: "Mise à Sac"
year: 1967
? A Bout Portant (The Killers).PAL.Multi.DVD-R-KZ
: options: -n
format: DVD
other: PAL
language: mul
releaseGroup: KZ
title: A Bout Portant
? Youth.In.Revolt.(Be.Bad).2009.MULTI.1080p.LAME3*92-MEDIOZZ
: options: -n
audioCodec: MP3
language: mul
releaseGroup: MEDIOZZ
screenSize: 1080p
title: Youth In Revolt
year: 2009
? La Defense Lincoln (The Lincoln Lawyer) 2011 [DVDRIP][Vostfr]
: options: -n
format: DVD
subtitleLanguage: fr
title: La Defense Lincoln
year: 2011
? '[h265 - HEVC] Fight Club French 1080p DTS.'
: options: -n
audioCodec: DTS
language: fr
screenSize: 1080p
title: Fight Club
videoCodec: h265
? Love Gourou (Mike Myers) - FR
: options: -n
language: fr
title: Love Gourou
? '[h265 - hevc] transformers 2 1080p french ac3 6ch.'
: options: -n
audioChannels: '5.1'
audioCodec: AC3
language: fr
screenSize: 1080p
title: transformers 2
videoCodec: h265
? 1.Angry.Man.1957.mkv
: title: 1 Angry Man
year: 1957
? 12.Angry.Men.1957.mkv
: title: 12 Angry Men
year: 1957
? 123.Angry.Men.1957.mkv
: title: 123 Angry Men
year: 1957
? "Looney Tunes 1444x866 Porky's Last Stand.mkv"
: screenSize: 1444x866
title: Looney Tunes
@@ -1,473 +0,0 @@
IdSubLanguage ISO639 LanguageName UploadEnabled WebEnabled
aar aa Afar, afar 0 0
abk ab Abkhazian 0 0
ace Achinese 0 0
ach Acoli 0 0
ada Adangme 0 0
ady adyghé 0 0
afa Afro-Asiatic (Other) 0 0
afh Afrihili 0 0
afr af Afrikaans 0 0
ain Ainu 0 0
aka ak Akan 0 0
akk Akkadian 0 0
alb sq Albanian 1 1
ale Aleut 0 0
alg Algonquian languages 0 0
alt Southern Altai 0 0
amh am Amharic 0 0
ang English, Old (ca.450-1100) 0 0
apa Apache languages 0 0
ara ar Arabic 1 1
arc Aramaic 0 0
arg an Aragonese 0 0
arm hy Armenian 1 0
arn Araucanian 0 0
arp Arapaho 0 0
art Artificial (Other) 0 0
arw Arawak 0 0
asm as Assamese 0 0
ast Asturian, Bable 0 0
ath Athapascan languages 0 0
aus Australian languages 0 0
ava av Avaric 0 0
ave ae Avestan 0 0
awa Awadhi 0 0
aym ay Aymara 0 0
aze az Azerbaijani 0 0
bad Banda 0 0
bai Bamileke languages 0 0
bak ba Bashkir 0 0
bal Baluchi 0 0
bam bm Bambara 0 0
ban Balinese 0 0
baq eu Basque 1 1
bas Basa 0 0
bat Baltic (Other) 0 0
bej Beja 0 0
bel be Belarusian 0 0
bem Bemba 0 0
ben bn Bengali 1 0
ber Berber (Other) 0 0
bho Bhojpuri 0 0
bih bh Bihari 0 0
bik Bikol 0 0
bin Bini 0 0
bis bi Bislama 0 0
bla Siksika 0 0
bnt Bantu (Other) 0 0
bos bs Bosnian 1 0
bra Braj 0 0
bre br Breton 1 0
btk Batak (Indonesia) 0 0
bua Buriat 0 0
bug Buginese 0 0
bul bg Bulgarian 1 1
bur my Burmese 0 0
byn Blin 0 0
cad Caddo 0 0
cai Central American Indian (Other) 0 0
car Carib 0 0
cat ca Catalan 1 1
cau Caucasian (Other) 0 0
ceb Cebuano 0 0
cel Celtic (Other) 0 0
cha ch Chamorro 0 0
chb Chibcha 0 0
che ce Chechen 0 0
chg Chagatai 0 0
chi zh Chinese 1 1
chk Chuukese 0 0
chm Mari 0 0
chn Chinook jargon 0 0
cho Choctaw 0 0
chp Chipewyan 0 0
chr Cherokee 0 0
chu cu Church Slavic 0 0
chv cv Chuvash 0 0
chy Cheyenne 0 0
cmc Chamic languages 0 0
cop Coptic 0 0
cor kw Cornish 0 0
cos co Corsican 0 0
cpe Creoles and pidgins, English based (Other) 0 0
cpf Creoles and pidgins, French-based (Other) 0 0
cpp Creoles and pidgins, Portuguese-based (Other) 0 0
cre cr Cree 0 0
crh Crimean Tatar 0 0
crp Creoles and pidgins (Other) 0 0
csb Kashubian 0 0
cus Cushitic (Other)' couchitiques, autres langues 0 0
cze cs Czech 1 1
dak Dakota 0 0
dan da Danish 1 1
dar Dargwa 0 0
day Dayak 0 0
del Delaware 0 0
den Slave (Athapascan) 0 0
dgr Dogrib 0 0
din Dinka 0 0
div dv Divehi 0 0
doi Dogri 0 0
dra Dravidian (Other) 0 0
dua Duala 0 0
dum Dutch, Middle (ca.1050-1350) 0 0
dut nl Dutch 1 1
dyu Dyula 0 0
dzo dz Dzongkha 0 0
efi Efik 0 0
egy Egyptian (Ancient) 0 0
eka Ekajuk 0 0
elx Elamite 0 0
eng en English 1 1
enm English, Middle (1100-1500) 0 0
epo eo Esperanto 1 0
est et Estonian 1 1
ewe ee Ewe 0 0
ewo Ewondo 0 0
fan Fang 0 0
fao fo Faroese 0 0
fat Fanti 0 0
fij fj Fijian 0 0
fil Filipino 0 0
fin fi Finnish 1 1
fiu Finno-Ugrian (Other) 0 0
fon Fon 0 0
fre fr French 1 1
frm French, Middle (ca.1400-1600) 0 0
fro French, Old (842-ca.1400) 0 0
fry fy Frisian 0 0
ful ff Fulah 0 0
fur Friulian 0 0
gaa Ga 0 0
gay Gayo 0 0
gba Gbaya 0 0
gem Germanic (Other) 0 0
geo ka Georgian 1 1
ger de German 1 1
gez Geez 0 0
gil Gilbertese 0 0
gla gd Gaelic 0 0
gle ga Irish 0 0
glg gl Galician 1 1
glv gv Manx 0 0
gmh German, Middle High (ca.1050-1500) 0 0
goh German, Old High (ca.750-1050) 0 0
gon Gondi 0 0
gor Gorontalo 0 0
got Gothic 0 0
grb Grebo 0 0
grc Greek, Ancient (to 1453) 0 0
ell el Greek 1 1
grn gn Guarani 0 0
guj gu Gujarati 0 0
gwi Gwich´in 0 0
hai Haida 0 0
hat ht Haitian 0 0
hau ha Hausa 0 0
haw Hawaiian 0 0
heb he Hebrew 1 1
her hz Herero 0 0
hil Hiligaynon 0 0
him Himachali 0 0
hin hi Hindi 1 1
hit Hittite 0 0
hmn Hmong 0 0
hmo ho Hiri Motu 0 0
hrv hr Croatian 1 1
hun hu Hungarian 1 1
hup Hupa 0 0
iba Iban 0 0
ibo ig Igbo 0 0
ice is Icelandic 1 1
ido io Ido 0 0
iii ii Sichuan Yi 0 0
ijo Ijo 0 0
iku iu Inuktitut 0 0
ile ie Interlingue 0 0
ilo Iloko 0 0
ina ia Interlingua (International Auxiliary Language Asso 0 0
inc Indic (Other) 0 0
ind id Indonesian 1 1
ine Indo-European (Other) 0 0
inh Ingush 0 0
ipk ik Inupiaq 0 0
ira Iranian (Other) 0 0
iro Iroquoian languages 0 0
ita it Italian 1 1
jav jv Javanese 0 0
jpn ja Japanese 1 1
jpr Judeo-Persian 0 0
jrb Judeo-Arabic 0 0
kaa Kara-Kalpak 0 0
kab Kabyle 0 0
kac Kachin 0 0
kal kl Kalaallisut 0 0
kam Kamba 0 0
kan kn Kannada 0 0
kar Karen 0 0
kas ks Kashmiri 0 0
kau kr Kanuri 0 0
kaw Kawi 0 0
kaz kk Kazakh 1 0
kbd Kabardian 0 0
kha Khasi 0 0
khi Khoisan (Other) 0 0
khm km Khmer 1 1
kho Khotanese 0 0
kik ki Kikuyu 0 0
kin rw Kinyarwanda 0 0
kir ky Kirghiz 0 0
kmb Kimbundu 0 0
kok Konkani 0 0
kom kv Komi 0 0
kon kg Kongo 0 0
kor ko Korean 1 1
kos Kosraean 0 0
kpe Kpelle 0 0
krc Karachay-Balkar 0 0
kro Kru 0 0
kru Kurukh 0 0
kua kj Kuanyama 0 0
kum Kumyk 0 0
kur ku Kurdish 0 0
kut Kutenai 0 0
lad Ladino 0 0
lah Lahnda 0 0
lam Lamba 0 0
lao lo Lao 0 0
lat la Latin 0 0
lav lv Latvian 1 0
lez Lezghian 0 0
lim li Limburgan 0 0
lin ln Lingala 0 0
lit lt Lithuanian 1 0
lol Mongo 0 0
loz Lozi 0 0
ltz lb Luxembourgish 1 0
lua Luba-Lulua 0 0
lub lu Luba-Katanga 0 0
lug lg Ganda 0 0
lui Luiseno 0 0
lun Lunda 0 0
luo Luo (Kenya and Tanzania) 0 0
lus lushai 0 0
mac mk Macedonian 1 1
mad Madurese 0 0
mag Magahi 0 0
mah mh Marshallese 0 0
mai Maithili 0 0
mak Makasar 0 0
mal ml Malayalam 0 0
man Mandingo 0 0
mao mi Maori 0 0
map Austronesian (Other) 0 0
mar mr Marathi 0 0
mas Masai 0 0
may ms Malay 1 1
mdf Moksha 0 0
mdr Mandar 0 0
men Mende 0 0
mga Irish, Middle (900-1200) 0 0
mic Mi'kmaq 0 0
min Minangkabau 0 0
mis Miscellaneous languages 0 0
mkh Mon-Khmer (Other) 0 0
mlg mg Malagasy 0 0
mlt mt Maltese 0 0
mnc Manchu 0 0
mni Manipuri 0 0
mno Manobo languages 0 0
moh Mohawk 0 0
mol mo Moldavian 0 0
mon mn Mongolian 1 0
mos Mossi 0 0
mwl Mirandese 0 0
mul Multiple languages 0 0
mun Munda languages 0 0
mus Creek 0 0
mwr Marwari 0 0
myn Mayan languages 0 0
myv Erzya 0 0
nah Nahuatl 0 0
nai North American Indian 0 0
nap Neapolitan 0 0
nau na Nauru 0 0
nav nv Navajo 0 0
nbl nr Ndebele, South 0 0
nde nd Ndebele, North 0 0
ndo ng Ndonga 0 0
nds Low German 0 0
nep ne Nepali 0 0
new Nepal Bhasa 0 0
nia Nias 0 0
nic Niger-Kordofanian (Other) 0 0
niu Niuean 0 0
nno nn Norwegian Nynorsk 0 0
nob nb Norwegian Bokmal 0 0
nog Nogai 0 0
non Norse, Old 0 0
nor no Norwegian 1 1
nso Northern Sotho 0 0
nub Nubian languages 0 0
nwc Classical Newari 0 0
nya ny Chichewa 0 0
nym Nyamwezi 0 0
nyn Nyankole 0 0
nyo Nyoro 0 0
nzi Nzima 0 0
oci oc Occitan 1 1
oji oj Ojibwa 0 0
ori or Oriya 0 0
orm om Oromo 0 0
osa Osage 0 0
oss os Ossetian 0 0
ota Turkish, Ottoman (1500-1928) 0 0
oto Otomian languages 0 0
paa Papuan (Other) 0 0
pag Pangasinan 0 0
pal Pahlavi 0 0
pam Pampanga 0 0
pan pa Panjabi 0 0
pap Papiamento 0 0
pau Palauan 0 0
peo Persian, Old (ca.600-400 B.C.) 0 0
per fa Persian 1 1
phi Philippine (Other) 0 0
phn Phoenician 0 0
pli pi Pali 0 0
pol pl Polish 1 1
pon Pohnpeian 0 0
por pt Portuguese 1 1
pra Prakrit languages 0 0
pro Provençal, Old (to 1500) 0 0
pus ps Pushto 0 0
que qu Quechua 0 0
raj Rajasthani 0 0
rap Rapanui 0 0
rar Rarotongan 0 0
roa Romance (Other) 0 0
roh rm Raeto-Romance 0 0
rom Romany 0 0
run rn Rundi 0 0
rup Aromanian 0 0
rus ru Russian 1 1
sad Sandawe 0 0
sag sg Sango 0 0
sah Yakut 0 0
sai South American Indian (Other) 0 0
sal Salishan languages 0 0
sam Samaritan Aramaic 0 0
san sa Sanskrit 0 0
sas Sasak 0 0
sat Santali 0 0
scc sr Serbian 1 1
scn Sicilian 0 0
sco Scots 0 0
sel Selkup 0 0
sem Semitic (Other) 0 0
sga Irish, Old (to 900) 0 0
sgn Sign Languages 0 0
shn Shan 0 0
sid Sidamo 0 0
sin si Sinhalese 1 1
sio Siouan languages 0 0
sit Sino-Tibetan (Other) 0 0
sla Slavic (Other) 0 0
slo sk Slovak 1 1
slv sl Slovenian 1 1
sma Southern Sami 0 0
sme se Northern Sami 0 0
smi Sami languages (Other) 0 0
smj Lule Sami 0 0
smn Inari Sami 0 0
smo sm Samoan 0 0
sms Skolt Sami 0 0
sna sn Shona 0 0
snd sd Sindhi 0 0
snk Soninke 0 0
sog Sogdian 0 0
som so Somali 0 0
son Songhai 0 0
sot st Sotho, Southern 0 0
spa es Spanish 1 1
srd sc Sardinian 0 0
srr Serer 0 0
ssa Nilo-Saharan (Other) 0 0
ssw ss Swati 0 0
suk Sukuma 0 0
sun su Sundanese 0 0
sus Susu 0 0
sux Sumerian 0 0
swa sw Swahili 1 0
swe sv Swedish 1 1
syr Syriac 1 0
tah ty Tahitian 0 0
tai Tai (Other) 0 0
tam ta Tamil 0 0
tat tt Tatar 0 0
tel te Telugu 0 0
tem Timne 0 0
ter Tereno 0 0
tet Tetum 0 0
tgk tg Tajik 0 0
tgl tl Tagalog 1 1
tha th Thai 1 1
tib bo Tibetan 0 0
tig Tigre 0 0
tir ti Tigrinya 0 0
tiv Tiv 0 0
tkl Tokelau 0 0
tlh Klingon 0 0
tli Tlingit 0 0
tmh Tamashek 0 0
tog Tonga (Nyasa) 0 0
ton to Tonga (Tonga Islands) 0 0
tpi Tok Pisin 0 0
tsi Tsimshian 0 0
tsn tn Tswana 0 0
tso ts Tsonga 0 0
tuk tk Turkmen 0 0
tum Tumbuka 0 0
tup Tupi languages 0 0
tur tr Turkish 1 1
tut Altaic (Other) 0 0
tvl Tuvalu 0 0
twi tw Twi 0 0
tyv Tuvinian 0 0
udm Udmurt 0 0
uga Ugaritic 0 0
uig ug Uighur 0 0
ukr uk Ukrainian 1 1
umb Umbundu 0 0
und Undetermined 0 0
urd ur Urdu 1 0
uzb uz Uzbek 0 0
vai Vai 0 0
ven ve Venda 0 0
vie vi Vietnamese 1 1
vol vo Volapük 0 0
vot Votic 0 0
wak Wakashan languages 0 0
wal Walamo 0 0
war Waray 0 0
was Washo 0 0
wel cy Welsh 0 0
wen Sorbian languages 0 0
wln wa Walloon 0 0
wol wo Wolof 0 0
xal Kalmyk 0 0
xho xh Xhosa 0 0
yao Yao 0 0
yap Yapese 0 0
yid yi Yiddish 0 0
yor yo Yoruba 0 0
ypk Yupik languages 0 0
zap Zapotec 0 0
zen Zenaga 0 0
zha za Zhuang 0 0
znd Zande 0 0
zul zu Zulu 0 0
zun Zuni 0 0
rum ro Romanian 1 1
pob pb Brazilian 1 1
@@ -1,50 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2014 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from guessit.test.guessittest import *
class TestApi(TestGuessit):
def test_api(self):
movie_path = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv'
movie_info = guessit.guess_movie_info(movie_path)
video_info = guessit.guess_video_info(movie_path)
episode_info = guessit.guess_episode_info(movie_path)
file_info = guessit.guess_file_info(movie_path)
assert guessit.guess_file_info(movie_path, type='movie') == movie_info
assert guessit.guess_file_info(movie_path, type='video') == video_info
assert guessit.guess_file_info(movie_path, type='episode') == episode_info
assert guessit.guess_file_info(movie_path, options={'type': 'movie'}) == movie_info
assert guessit.guess_file_info(movie_path, options={'type': 'video'}) == video_info
assert guessit.guess_file_info(movie_path, options={'type': 'episode'}) == episode_info
# kwargs priority other options
assert guessit.guess_file_info(movie_path, options={'type': 'episode'}, type='movie') == episode_info
movie_path_name_only = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD'
file_info_name_only = guessit.guess_file_info(movie_path_name_only, options={"name_only": True})
assert 'container' not in file_info_name_only
assert 'container' in file_info
@@ -1,39 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from guessit.test.guessittest import *
class TestAutoDetect(TestGuessit):
def testEmpty(self):
result = guessit.guess_file_info('')
assert result == {}
result = guessit.guess_file_info('___-__')
assert result == {}
result = guessit.guess_file_info('__-.avc')
assert result == {'type': 'unknown', 'extension': 'avc'}
def testAutoDetect(self):
self.checkMinimumFieldsCorrect(filename='autodetect.yaml',
remove_type=False)
@@ -1,40 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from guessit.test.guessittest import *
IGNORE_EPISODES = []
IGNORE_MOVIES = []
class TestAutoDetectAll(TestGuessit):
def testAutoMatcher(self):
self.checkMinimumFieldsCorrect(filename='autodetect.yaml',
remove_type=False)
def testAutoMatcherMovies(self):
self.checkMinimumFieldsCorrect(filename='movies.yaml',
exclude_files=IGNORE_MOVIES)
def testAutoMatcherEpisodes(self):
self.checkMinimumFieldsCorrect(filename='episodes.yaml',
exclude_files=IGNORE_EPISODES)
@@ -1,29 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import, division, print_function, unicode_literals
from guessit.test.guessittest import *
class TestEpisode(TestGuessit):
def testEpisodes(self):
self.checkMinimumFieldsCorrect(filetype='episode',
filename='episodes.yaml')

Some files were not shown because too many files have changed in this diff Show More