panni
945 lines
38 KiB
Python
Executable File
945 lines
38 KiB
Python
Executable File
# coding=utf-8
|
|
import glob
|
|
import os
|
|
import datetime
|
|
import operator
|
|
import traceback
|
|
from urllib2 import URLError
|
|
|
|
from subliminal_patch.score import compute_score
|
|
from subliminal_patch.core import download_subtitles
|
|
from subliminal import list_subtitles as list_all_subtitles, region as subliminal_cache_region
|
|
from subzero.language import Language
|
|
from subzero.video import refine_video
|
|
|
|
from missing_subtitles import items_get_all_missing_subs, refresh_item
|
|
from scheduler import scheduler
|
|
from storage import save_subtitles, get_subtitle_storage
|
|
from support.config import config
|
|
from support.items import get_recent_items, get_item, is_wanted, get_item_title
|
|
from support.helpers import track_usage, get_title_for_video_metadata, cast_bool, PartUnknownException
|
|
from support.plex_media import get_plex_metadata
|
|
from support.extract import agent_extract_embedded
|
|
from support.scanning import scan_videos
|
|
from support.i18n import _
|
|
from download import download_best_subtitles, pre_download_hook, post_download_hook, language_hook
|
|
|
|
|
|
class Task(object):
|
|
name = None
|
|
scheduler = None
|
|
periodic = False
|
|
running = False
|
|
time_start = None
|
|
data = None
|
|
|
|
PROVIDER_SLACK = 30
|
|
DL_PROVIDER_SLACK = 30
|
|
|
|
stored_attributes = ("last_run", "last_run_time", "running")
|
|
default_data = {"last_run": None, "last_run_time": None, "running": False, "data": {}}
|
|
|
|
# task ready for being status-displayed?
|
|
ready_for_display = False
|
|
|
|
def __init__(self):
|
|
self.name = self.get_class_name()
|
|
self.ready_for_display = False
|
|
self.time_start = None
|
|
self.setup_defaults()
|
|
|
|
self.running = False
|
|
|
|
def get_class_name(self):
|
|
return getattr(getattr(self, "__class__"), "__name__")
|
|
|
|
def __getattribute__(self, name):
|
|
if name in object.__getattribute__(self, "stored_attributes"):
|
|
return Dict["tasks"].get(self.name, {}).get(name, None)
|
|
|
|
return object.__getattribute__(self, name)
|
|
|
|
def __setattr__(self, name, value):
|
|
if name in object.__getattribute__(self, "stored_attributes"):
|
|
Dict["tasks"][self.name][name] = value
|
|
Dict.Save()
|
|
return
|
|
|
|
object.__setattr__(self, name, value)
|
|
|
|
def setup_defaults(self):
|
|
if self.name not in Dict["tasks"]:
|
|
Dict["tasks"][self.name] = self.default_data.copy()
|
|
return
|
|
|
|
sd = Dict["tasks"][self.name]
|
|
|
|
# forward-migration
|
|
for key, def_value in self.default_data.iteritems():
|
|
hasval = key in sd
|
|
if not hasval:
|
|
sd[key] = def_value
|
|
|
|
def signal(self, *args, **kwargs):
|
|
raise NotImplementedError
|
|
|
|
def prepare(self, *args, **kwargs):
|
|
return
|
|
|
|
def run(self):
|
|
Log.Info(u"Task: running: %s", self.name)
|
|
self.time_start = datetime.datetime.now()
|
|
|
|
def post_run(self, data_holder):
|
|
self.running = False
|
|
self.last_run = datetime.datetime.now()
|
|
if self.time_start and self.last_run:
|
|
self.last_run_time = self.last_run - self.time_start
|
|
self.time_start = None
|
|
Log.Info(u"Task: ran: %s", self.name)
|
|
|
|
|
|
class SubtitleListingMixin(object):
|
|
def list_subtitles(self, rating_key, item_type, part_id, language, skip_wrong_fps=True, metadata=None,
|
|
scanned_parts=None, air_date_cutoff=None):
|
|
|
|
if not metadata:
|
|
metadata = get_plex_metadata(rating_key, part_id, item_type)
|
|
|
|
if not metadata:
|
|
return
|
|
|
|
providers = config.get_providers(media_type="series" if item_type == "episode" else "movies")
|
|
if not scanned_parts:
|
|
scanned_parts = scan_videos([metadata], ignore_all=True, providers=providers)
|
|
if not scanned_parts:
|
|
Log.Error(u"%s: Couldn't list available subtitles for %s", self.name, rating_key)
|
|
return
|
|
|
|
video, plex_part = scanned_parts.items()[0]
|
|
refine_video(video, refiner_settings=config.refiner_settings)
|
|
|
|
if air_date_cutoff is not None and metadata["item"].year and \
|
|
metadata["item"].year + air_date_cutoff < datetime.date.today().year:
|
|
Log.Debug("Skipping searching for subtitles: %s, it aired over %s year(s) ago.", rating_key,
|
|
air_date_cutoff)
|
|
return
|
|
|
|
config.init_subliminal_patches()
|
|
|
|
provider_settings = config.provider_settings
|
|
if not skip_wrong_fps:
|
|
provider_settings["opensubtitles"]["skip_wrong_fps"] = False
|
|
|
|
if item_type == "episode":
|
|
min_score = 240
|
|
if video.is_special:
|
|
min_score = 180
|
|
else:
|
|
min_score = 60
|
|
|
|
languages = {Language.fromietf(language)}
|
|
|
|
available_subs = list_all_subtitles([video], languages,
|
|
providers=providers,
|
|
provider_configs=provider_settings,
|
|
pool_class=config.provider_pool,
|
|
throttle_callback=config.provider_throttle,
|
|
language_hook=language_hook)
|
|
|
|
use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in ("prefer", "force HI")
|
|
|
|
# sort subtitles by score
|
|
unsorted_subtitles = []
|
|
for s in available_subs[video]:
|
|
Log.Debug(u"%s: Starting score computation for %s", self.name, s)
|
|
try:
|
|
matches = s.get_matches(video)
|
|
except AttributeError:
|
|
Log.Error(u"%s: Match computation failed for %s: %s", self.name, s, traceback.format_exc())
|
|
continue
|
|
|
|
# skip wrong season/episodes
|
|
if item_type == "episode":
|
|
can_verify_series = True
|
|
if not s.hash_verifiable and "hash" in matches:
|
|
can_verify_series = False
|
|
|
|
if can_verify_series and not {"series", "season", "episode"}.issubset(matches):
|
|
if "series" not in matches:
|
|
s.wrong_series = True
|
|
else:
|
|
s.wrong_season_ep = True
|
|
|
|
orig_matches = matches.copy()
|
|
score, score_without_hash = compute_score(matches, s, video, hearing_impaired=use_hearing_impaired)
|
|
|
|
unsorted_subtitles.append(
|
|
(s, score, score_without_hash, matches, orig_matches))
|
|
scored_subtitles = sorted(unsorted_subtitles, key=operator.itemgetter(1, 2), reverse=True)
|
|
|
|
subtitles = []
|
|
for subtitle, score, score_without_hash, matches, orig_matches in scored_subtitles:
|
|
# check score
|
|
if score < min_score and not subtitle.wrong_series:
|
|
Log.Info(u'%s: Score %d is below min_score (%d)', self.name, score, min_score)
|
|
continue
|
|
subtitle.score = score
|
|
subtitle.matches = matches
|
|
subtitle.part_id = part_id
|
|
subtitle.item_type = item_type
|
|
subtitles.append(subtitle)
|
|
return subtitles
|
|
|
|
|
|
class DownloadSubtitleMixin(object):
|
|
def download_subtitle(self, subtitle, rating_key, mode="m"):
|
|
from interface.menu_helpers import set_refresh_menu_state
|
|
|
|
item_type = subtitle.item_type
|
|
part_id = subtitle.part_id
|
|
metadata = get_plex_metadata(rating_key, part_id, item_type)
|
|
providers = config.get_providers(media_type="series" if item_type == "episode" else "movies")
|
|
scanned_parts = scan_videos([metadata], ignore_all=True, providers=providers)
|
|
video, plex_part = scanned_parts.items()[0]
|
|
|
|
pre_download_hook(subtitle)
|
|
|
|
# downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
|
|
download_subtitles([subtitle], providers=providers,
|
|
provider_configs=config.provider_settings,
|
|
pool_class=config.provider_pool, throttle_callback=config.provider_throttle)
|
|
|
|
post_download_hook(subtitle)
|
|
|
|
# may be redundant
|
|
subtitle.pack_data = None
|
|
|
|
download_successful = False
|
|
|
|
if subtitle.content:
|
|
try:
|
|
save_subtitles(scanned_parts, {video: [subtitle]}, mode=mode, mods=config.default_mods)
|
|
if mode == "m":
|
|
Log.Debug(u"%s: Manually downloaded subtitle for: %s", self.name, rating_key)
|
|
track_usage("Subtitle", "manual", "download", 1)
|
|
elif mode == "b":
|
|
Log.Debug(u"%s: Downloaded better subtitle for: %s", self.name, rating_key)
|
|
track_usage("Subtitle", "better", "download", 1)
|
|
download_successful = True
|
|
refresh_item(rating_key)
|
|
|
|
except:
|
|
Log.Error(u"%s: Something went wrong when downloading specific subtitle: %s",
|
|
self.name, traceback.format_exc())
|
|
finally:
|
|
set_refresh_menu_state(None)
|
|
|
|
if download_successful:
|
|
# store item in history
|
|
from support.history import get_history
|
|
item_title = get_title_for_video_metadata(metadata, add_section_title=False)
|
|
history = get_history()
|
|
history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
|
|
thumb=video.plexapi_metadata["super_thumb"],
|
|
subtitle=subtitle,
|
|
mode=mode)
|
|
history.destroy()
|
|
|
|
# clear missing subtitles menu data
|
|
if not scheduler.is_task_running("MissingSubtitles"):
|
|
scheduler.clear_task_data("MissingSubtitles")
|
|
else:
|
|
set_refresh_menu_state(_(u"%(class_name)s: Subtitle download failed (%(item_id)s)",
|
|
class_name=self.name,
|
|
item_id=rating_key))
|
|
return download_successful
|
|
|
|
|
|
class AvailableSubsForItem(SubtitleListingMixin, Task):
|
|
item_type = None
|
|
part_id = None
|
|
language = None
|
|
rating_key = None
|
|
|
|
def prepare(self, *args, **kwargs):
|
|
self.item_type = kwargs.get("item_type")
|
|
self.part_id = kwargs.get("part_id")
|
|
self.language = kwargs.get("language")
|
|
self.rating_key = kwargs.get("rating_key")
|
|
|
|
def setup_defaults(self):
|
|
super(AvailableSubsForItem, self).setup_defaults()
|
|
|
|
# reset any previous data
|
|
Dict["tasks"][self.name]["data"] = {}
|
|
|
|
def run(self):
|
|
super(AvailableSubsForItem, self).run()
|
|
self.running = True
|
|
try:
|
|
track_usage("Subtitle", "manual", "list", 1)
|
|
except:
|
|
Log.Error("Something went wrong with track_usage: %s", traceback.format_exc())
|
|
|
|
Log.Debug("Listing available subtitles for: %s", self.rating_key)
|
|
subs = self.list_subtitles(self.rating_key, self.item_type, self.part_id, self.language, skip_wrong_fps=False)
|
|
if not subs:
|
|
self.data = "found_none"
|
|
return
|
|
|
|
# we can't have nasty unpicklable stuff like ZipFile, BytesIO etc in self.data
|
|
self.data = [s.make_picklable() for s in subs]
|
|
|
|
def post_run(self, task_data):
|
|
super(AvailableSubsForItem, self).post_run(task_data)
|
|
# clean old data
|
|
for key in task_data.keys():
|
|
if key != self.rating_key:
|
|
del task_data[key]
|
|
task_data.update({self.rating_key: {self.language: self.data}})
|
|
|
|
|
|
class DownloadSubtitleForItem(DownloadSubtitleMixin, Task):
|
|
subtitle = None
|
|
rating_key = None
|
|
|
|
def prepare(self, *args, **kwargs):
|
|
self.subtitle = kwargs["subtitle"]
|
|
self.rating_key = kwargs["rating_key"]
|
|
|
|
def run(self):
|
|
super(DownloadSubtitleForItem, self).run()
|
|
self.running = True
|
|
self.download_subtitle(self.subtitle, self.rating_key)
|
|
self.running = False
|
|
|
|
|
|
class MissingSubtitles(Task):
|
|
rating_key = None
|
|
item_type = None
|
|
part_id = None
|
|
language = None
|
|
|
|
def run(self):
|
|
super(MissingSubtitles, self).run()
|
|
self.running = True
|
|
self.data = []
|
|
recent_items = get_recent_items()
|
|
if recent_items:
|
|
self.data = items_get_all_missing_subs(recent_items)
|
|
|
|
def post_run(self, task_data):
|
|
super(MissingSubtitles, self).post_run(task_data)
|
|
task_data["missing_subtitles"] = self.data
|
|
|
|
|
|
class SearchAllRecentlyAddedMissing(Task):
|
|
periodic = True
|
|
|
|
items_done = None
|
|
items_searching = None
|
|
percentage = 0
|
|
|
|
def __init__(self):
|
|
super(SearchAllRecentlyAddedMissing, self).__init__()
|
|
self.items_done = None
|
|
self.items_searching = None
|
|
self.percentage = 0
|
|
|
|
def signal_updated_metadata(self, *args, **kwargs):
|
|
return True
|
|
|
|
def prepare(self):
|
|
self.items_done = 0
|
|
self.items_searching = 0
|
|
self.percentage = 0
|
|
self.ready_for_display = True
|
|
|
|
def run(self):
|
|
super(SearchAllRecentlyAddedMissing, self).run()
|
|
|
|
self.running = True
|
|
self.prepare()
|
|
|
|
from support.history import get_history
|
|
history = get_history()
|
|
|
|
now = datetime.datetime.now()
|
|
min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
|
|
min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
|
|
series_providers = config.get_providers(media_type="series")
|
|
movie_providers = config.get_providers(media_type="movies")
|
|
|
|
is_recent_str = Prefs["scheduler.item_is_recent_age"]
|
|
num, ident = is_recent_str.split()
|
|
|
|
max_search_days = 0
|
|
if ident == "days":
|
|
max_search_days = int(num)
|
|
elif ident == "weeks":
|
|
max_search_days = int(num) * 7
|
|
|
|
subtitle_storage = get_subtitle_storage()
|
|
recent_files = subtitle_storage.get_recent_files(age_days=max_search_days)
|
|
|
|
self.items_searching = len(recent_files)
|
|
|
|
download_count = 0
|
|
videos_with_downloads = 0
|
|
|
|
config.init_subliminal_patches()
|
|
|
|
Log.Info(u"%s: Searching for subtitles for %s items", self.name, self.items_searching)
|
|
|
|
def skip_item():
|
|
self.items_searching = self.items_searching - 1
|
|
self.percentage = int(self.items_done * 100 / self.items_searching) if self.items_searching > 0 else 100
|
|
|
|
# search for subtitles in viable items
|
|
try:
|
|
for fn in recent_files:
|
|
stored_subs = subtitle_storage.load(filename=fn)
|
|
if not stored_subs:
|
|
Log.Debug("Skipping item %s because storage is empty", fn)
|
|
skip_item()
|
|
continue
|
|
|
|
video_id = stored_subs.video_id
|
|
|
|
# added_date <= max_search_days?
|
|
if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
|
|
Log.Debug("Skipping item %s because it's too old", video_id)
|
|
skip_item()
|
|
continue
|
|
|
|
if stored_subs.item_type == "episode":
|
|
min_score = min_score_series
|
|
providers = series_providers
|
|
else:
|
|
min_score = min_score_movies
|
|
providers = movie_providers
|
|
|
|
parts = []
|
|
plex_item = get_item(video_id)
|
|
|
|
if not plex_item:
|
|
Log.Info(u"%s: Item %s unknown, skipping", self.name, video_id)
|
|
skip_item()
|
|
continue
|
|
|
|
if not is_wanted(video_id, item=plex_item):
|
|
skip_item()
|
|
continue
|
|
|
|
for media in plex_item.media:
|
|
parts += media.parts
|
|
|
|
downloads_per_video = 0
|
|
hit_providers = False
|
|
for part in parts:
|
|
part_id = part.id
|
|
|
|
try:
|
|
metadata = get_plex_metadata(video_id, part_id, stored_subs.item_type)
|
|
except PartUnknownException:
|
|
Log.Info(u"%s: Part %s:%s unknown, skipping", self.name, video_id, part_id)
|
|
continue
|
|
|
|
if not metadata:
|
|
Log.Info(u"%s: Part %s:%s unknown, skipping", self.name, video_id, part_id)
|
|
continue
|
|
|
|
Log.Debug(u"%s: Looking for missing subtitles: %s", self.name, get_item_title(plex_item))
|
|
scanned_parts = scan_videos([metadata], providers=providers)
|
|
|
|
# auto extract embedded
|
|
if config.embedded_auto_extract:
|
|
if config.plex_transcoder:
|
|
ts = agent_extract_embedded(scanned_parts, set_as_existing=True)
|
|
if ts:
|
|
Log.Debug("Waiting for %i extraction threads to finish" % len(ts))
|
|
for t in ts:
|
|
t.join()
|
|
else:
|
|
Log.Warn("Plex Transcoder not found, can't auto extract")
|
|
|
|
downloaded_subtitles = download_best_subtitles(scanned_parts, min_score=min_score,
|
|
providers=providers)
|
|
hit_providers = downloaded_subtitles is not None
|
|
download_successful = False
|
|
|
|
if downloaded_subtitles:
|
|
downloaded_any = any(downloaded_subtitles.values())
|
|
if not downloaded_any:
|
|
continue
|
|
|
|
try:
|
|
save_subtitles(scanned_parts, downloaded_subtitles, mode="a", mods=config.default_mods)
|
|
Log.Debug(u"%s: Downloaded subtitle for item with missing subs: %s", self.name, video_id)
|
|
download_successful = True
|
|
refresh_item(video_id)
|
|
track_usage("Subtitle", "manual", "download", 1)
|
|
except:
|
|
Log.Error(u"%s: Something went wrong when downloading specific subtitle: %s", self.name,
|
|
traceback.format_exc())
|
|
finally:
|
|
scanned_parts = None
|
|
try:
|
|
item_title = get_title_for_video_metadata(metadata, add_section_title=False)
|
|
if download_successful:
|
|
# store item in history
|
|
for video, video_subtitles in downloaded_subtitles.items():
|
|
if not video_subtitles:
|
|
continue
|
|
|
|
for subtitle in video_subtitles:
|
|
downloads_per_video += 1
|
|
history.add(item_title, video.id, section_title=metadata["section"],
|
|
thumb=video.plexapi_metadata["super_thumb"],
|
|
subtitle=subtitle,
|
|
mode="a")
|
|
|
|
downloaded_subtitles = None
|
|
except:
|
|
Log.Error(u"%s: DEBUG HIT: %s", self.name, traceback.format_exc())
|
|
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK)
|
|
Thread.Sleep(self.PROVIDER_SLACK)
|
|
|
|
download_count += downloads_per_video
|
|
|
|
if downloads_per_video:
|
|
videos_with_downloads += 1
|
|
|
|
self.items_done = self.items_done + 1
|
|
self.percentage = int(self.items_done * 100 / self.items_searching) if self.items_searching > 0 else 100
|
|
|
|
stored_subs = None
|
|
|
|
if downloads_per_video:
|
|
Log.Debug(u"%s: Subtitles have been downloaded, "
|
|
u"waiting %s seconds before continuing", self.name, self.DL_PROVIDER_SLACK)
|
|
Thread.Sleep(self.DL_PROVIDER_SLACK)
|
|
else:
|
|
if hit_providers:
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK)
|
|
Thread.Sleep(self.PROVIDER_SLACK)
|
|
finally:
|
|
subtitle_storage.destroy()
|
|
history.destroy()
|
|
|
|
if download_count:
|
|
Log.Debug(u"%s: done. Missing subtitles found for %s/%s items (%s subs downloaded)", self.name,
|
|
videos_with_downloads, self.items_searching, download_count)
|
|
else:
|
|
Log.Debug(u"%s: done. No subtitles found for %s items", self.name, self.items_searching)
|
|
|
|
def post_run(self, task_data):
|
|
super(SearchAllRecentlyAddedMissing, self).post_run(task_data)
|
|
self.ready_for_display = False
|
|
self.percentage = 0
|
|
self.items_done = None
|
|
self.items_searching = None
|
|
|
|
|
|
class LegacySearchAllRecentlyAddedMissing(Task):
|
|
periodic = True
|
|
frequency = "never"
|
|
items_done = None
|
|
items_searching = None
|
|
items_searching_ids = None
|
|
items_failed = None
|
|
percentage = 0
|
|
|
|
stall_time = 30
|
|
|
|
def __init__(self):
|
|
super(LegacySearchAllRecentlyAddedMissing, self).__init__()
|
|
self.items_done = None
|
|
self.items_searching = None
|
|
self.items_searching_ids = None
|
|
self.items_failed = None
|
|
self.percentage = 0
|
|
|
|
def signal(self, signal_name, *args, **kwargs):
|
|
handler = getattr(self, "signal_%s" % signal_name)
|
|
return handler(*args, **kwargs) if handler else None
|
|
|
|
def signal_updated_metadata(self, *args, **kwargs):
|
|
item_id = int(args[0])
|
|
|
|
if self.items_searching_ids is not None and item_id in self.items_searching_ids:
|
|
self.items_done.append(item_id)
|
|
return True
|
|
|
|
def prepare(self, *args, **kwargs):
|
|
self.items_done = []
|
|
recent_items = get_recent_items()
|
|
missing = items_get_all_missing_subs(recent_items, sleep_after_request=0.2)
|
|
ids = set([id for added_at, id, title, item, missing_languages in missing if is_wanted(id, item=item)])
|
|
self.items_searching = missing
|
|
self.items_searching_ids = ids
|
|
self.items_failed = []
|
|
self.percentage = 0
|
|
self.ready_for_display = True
|
|
|
|
def run(self):
|
|
super(LegacySearchAllRecentlyAddedMissing, self).run()
|
|
self.running = True
|
|
missing_count = len(self.items_searching)
|
|
items_done_count = 0
|
|
|
|
for added_at, item_id, title, item, missing_languages in self.items_searching:
|
|
Log.Debug(u"Task: %s, triggering refresh for %s (%s)", self.name, title, item_id)
|
|
try:
|
|
refresh_item(item_id)
|
|
except URLError:
|
|
# timeout
|
|
pass
|
|
search_started = datetime.datetime.now()
|
|
tries = 1
|
|
while 1:
|
|
if item_id in self.items_done:
|
|
items_done_count += 1
|
|
self.percentage = int(items_done_count * 100 / missing_count) if missing_count > 0 else 100
|
|
Log.Debug(u"Task: %s, item %s done (%s%%, %s/%s)", self.name, item_id, self.percentage,
|
|
items_done_count, missing_count)
|
|
break
|
|
|
|
# item considered stalled after self.stall_time seconds passed after last refresh
|
|
if (datetime.datetime.now() - search_started).total_seconds() > self.stall_time:
|
|
if tries > 3:
|
|
self.items_failed.append(item_id)
|
|
Log.Debug(u"Task: %s, item stalled for %s times: %s, skipping", self.name, tries, item_id)
|
|
break
|
|
|
|
Log.Debug(u"Task: %s, item stalled for %s seconds: %s, retrying", self.name, self.stall_time,
|
|
item_id)
|
|
tries += 1
|
|
try:
|
|
refresh_item(item_id)
|
|
except URLError:
|
|
pass
|
|
search_started = datetime.datetime.now()
|
|
Thread.Sleep(1)
|
|
Thread.Sleep(0.1)
|
|
# we can't hammer the PMS, otherwise requests will be stalled
|
|
Thread.Sleep(5)
|
|
|
|
Log.Debug("Task: %s, done (%s%%, %s/%s). Failed items: %s", self.name, self.percentage,
|
|
items_done_count, missing_count, self.items_failed)
|
|
|
|
def post_run(self, task_data):
|
|
super(LegacySearchAllRecentlyAddedMissing, self).post_run(task_data)
|
|
self.ready_for_display = False
|
|
self.percentage = 0
|
|
self.items_done = None
|
|
self.items_failed = None
|
|
self.items_searching = None
|
|
self.items_searching_ids = None
|
|
|
|
|
|
class FindBetterSubtitles(DownloadSubtitleMixin, SubtitleListingMixin, Task):
|
|
periodic = True
|
|
|
|
# TV: episode, format, series, year, season, video_codec, release_group, hearing_impaired, resolution
|
|
series_cutoff = 357
|
|
|
|
# movies: format, title, release_group, year, video_codec, resolution, hearing_impaired
|
|
movies_cutoff = 117
|
|
|
|
def signal_updated_metadata(self, *args, **kwargs):
|
|
return True
|
|
|
|
def run(self):
|
|
super(FindBetterSubtitles, self).run()
|
|
self.running = True
|
|
better_found = 0
|
|
try:
|
|
max_search_days = int(Prefs["scheduler.tasks.FindBetterSubtitles.max_days_after_added"].strip())
|
|
except ValueError:
|
|
Log.Error(u"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting")
|
|
return
|
|
else:
|
|
if max_search_days > 30:
|
|
Log.Error(u"%s: FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.", self.name)
|
|
return
|
|
|
|
now = datetime.datetime.now()
|
|
min_score_series = int(Prefs["subtitles.search.minimumTVScore2"].strip())
|
|
min_score_movies = int(Prefs["subtitles.search.minimumMovieScore2"].strip())
|
|
min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
|
|
min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 112
|
|
overwrite_manually_modified = cast_bool(
|
|
Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
|
|
overwrite_manually_selected = cast_bool(
|
|
Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"])
|
|
|
|
air_date_cutoff_pref = Prefs["scheduler.tasks.FindBetterSubtitles.air_date_cutoff"]
|
|
if air_date_cutoff_pref == "don't limit":
|
|
air_date_cutoff = None
|
|
else:
|
|
air_date_cutoff = int(air_date_cutoff_pref.split()[0])
|
|
|
|
subtitle_storage = get_subtitle_storage()
|
|
viable_item_count = 0
|
|
|
|
try:
|
|
for fn in subtitle_storage.get_recent_files(age_days=max_search_days):
|
|
stored_subs = subtitle_storage.load(filename=fn)
|
|
if not stored_subs:
|
|
continue
|
|
|
|
video_id = stored_subs.video_id
|
|
|
|
if stored_subs.item_type == "episode":
|
|
cutoff = self.series_cutoff
|
|
min_score = min_score_series
|
|
min_score_extracted = min_score_extracted_series
|
|
else:
|
|
cutoff = self.movies_cutoff
|
|
min_score = min_score_movies
|
|
min_score_extracted = min_score_extracted_movies
|
|
|
|
# don't search for better subtitles until at least 30 minutes have passed
|
|
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
|
|
Log.Debug(u"%s: Item %s too new, skipping", self.name, video_id)
|
|
continue
|
|
|
|
# added_date <= max_search_days?
|
|
if stored_subs.added_at + datetime.timedelta(days=max_search_days) <= now:
|
|
continue
|
|
|
|
viable_item_count += 1
|
|
ditch_parts = []
|
|
|
|
# look through all stored subtitle data
|
|
for part_id, languages in stored_subs.parts.iteritems():
|
|
part_id = str(part_id)
|
|
|
|
# all languages
|
|
for language, current_subs in languages.iteritems():
|
|
current_key = current_subs.get("current")
|
|
current = current_subs.get(current_key)
|
|
|
|
# currently got subtitle?
|
|
# fixme: check for existence
|
|
if not current:
|
|
continue
|
|
current_score = current.score
|
|
current_mode = current.mode
|
|
|
|
# late cutoff met? skip
|
|
if current_score >= cutoff:
|
|
Log.Debug(u"%s: Skipping finding better subs, "
|
|
u"cutoff met (current: %s, cutoff: %s): %s (%s)",
|
|
self.name, current_score, cutoff, stored_subs.title, video_id)
|
|
continue
|
|
|
|
# got manual subtitle but don't want to touch those?
|
|
if current_mode == "m" and not overwrite_manually_selected:
|
|
Log.Debug(u"%s: Skipping finding better subs, "
|
|
u"had manual: %s (%s)", self.name, stored_subs.title, video_id)
|
|
continue
|
|
|
|
# subtitle modifications different from default
|
|
if not overwrite_manually_modified and current.mods \
|
|
and set(current.mods).difference(set(config.default_mods)):
|
|
Log.Debug(u"%s: Skipping finding better subs, it has manual modifications: %s (%s)",
|
|
self.name, stored_subs.title, video_id)
|
|
continue
|
|
|
|
try:
|
|
subs = self.list_subtitles(video_id, stored_subs.item_type, part_id, language,
|
|
air_date_cutoff=air_date_cutoff)
|
|
except PartUnknownException:
|
|
Log.Info(u"%s: Part %s unknown/gone; ditching subtitle info", self.name, part_id)
|
|
ditch_parts.append(part_id)
|
|
continue
|
|
|
|
hit_providers = subs is not None
|
|
|
|
if subs:
|
|
# subs are already sorted by score
|
|
better_downloaded = False
|
|
better_tried_download = 0
|
|
better_visited = 0
|
|
for sub in subs:
|
|
if sub.score > current_score and sub.score > min_score:
|
|
if current.provider_name == "embedded" and sub.score < min_score_extracted:
|
|
Log.Debug(u"%s: Not downloading subtitle for %s, we've got an active extracted "
|
|
u"embedded sub and the min score %s isn't met (%s).",
|
|
self.name, video_id, min_score_extracted, sub.score)
|
|
better_visited += 1
|
|
break
|
|
|
|
Log.Debug(u"%s: Better subtitle found for %s, downloading", self.name, video_id)
|
|
better_tried_download += 1
|
|
ret = self.download_subtitle(sub, video_id, mode="b")
|
|
if ret:
|
|
better_found += 1
|
|
better_downloaded = True
|
|
break
|
|
else:
|
|
Log.Debug(u"%s: Couldn't download/save subtitle. "
|
|
u"Continuing to the next one", self.name)
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing",
|
|
self.name, self.DL_PROVIDER_SLACK)
|
|
Thread.Sleep(self.DL_PROVIDER_SLACK)
|
|
better_visited += 1
|
|
|
|
if better_tried_download and not better_downloaded:
|
|
Log.Debug(u"%s: Tried downloading better subtitle for %s, "
|
|
u"but every try failed.", self.name, video_id)
|
|
|
|
elif better_downloaded:
|
|
Log.Debug(u"%s: Better subtitle downloaded for %s", self.name, video_id)
|
|
|
|
if better_tried_download or better_downloaded:
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.DL_PROVIDER_SLACK)
|
|
Thread.Sleep(self.DL_PROVIDER_SLACK)
|
|
|
|
elif better_visited:
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK)
|
|
Thread.Sleep(self.PROVIDER_SLACK)
|
|
|
|
subs = None
|
|
|
|
elif hit_providers:
|
|
# hit the providers but didn't try downloading? wait.
|
|
Log.Debug(u"%s: Waiting %s seconds before continuing", self.name, self.PROVIDER_SLACK)
|
|
Thread.Sleep(self.PROVIDER_SLACK)
|
|
|
|
if ditch_parts:
|
|
for part_id in ditch_parts:
|
|
try:
|
|
del stored_subs.parts[part_id]
|
|
except KeyError:
|
|
pass
|
|
subtitle_storage.save(stored_subs)
|
|
ditch_parts = None
|
|
|
|
stored_subs = None
|
|
|
|
Thread.Sleep(1)
|
|
finally:
|
|
subtitle_storage.destroy()
|
|
|
|
if better_found:
|
|
Log.Debug(u"%s: done. Better subtitles found for %s/%s items", self.name, better_found,
|
|
viable_item_count)
|
|
else:
|
|
Log.Debug(u"%s: done. No better subtitles found for %s items", self.name, viable_item_count)
|
|
|
|
|
|
class SubtitleStorageMaintenance(Task):
|
|
periodic = True
|
|
frequency = "every 7 days"
|
|
|
|
def run(self):
|
|
super(SubtitleStorageMaintenance, self).run()
|
|
self.running = True
|
|
Log.Info(u"%s: Running subtitle storage maintenance", self.name)
|
|
storage = get_subtitle_storage()
|
|
try:
|
|
deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list))
|
|
except OSError:
|
|
deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list),
|
|
scandir_generic=True)
|
|
|
|
if deleted_items:
|
|
Log.Info(u"%s: Subtitle information for %d non-existant videos have been cleaned up",
|
|
self.name, len(deleted_items))
|
|
Log.Debug(u"%s: Videos: %s", self.name, deleted_items)
|
|
else:
|
|
Log.Info(u"%s: Nothing to do", self.name)
|
|
|
|
storage.destroy()
|
|
|
|
|
|
class MenuHistoryMaintenance(Task):
|
|
periodic = True
|
|
frequency = "every 7 days"
|
|
|
|
def run(self):
|
|
super(MenuHistoryMaintenance, self).run()
|
|
self.running = True
|
|
Log.Info(u"%s: Running menu history maintenance", self.name)
|
|
now = datetime.datetime.now()
|
|
if "menu_history" in Dict:
|
|
for key, timeout in Dict["menu_history"].copy().items():
|
|
if now > timeout:
|
|
try:
|
|
del Dict["menu_history"][key]
|
|
except:
|
|
pass
|
|
|
|
|
|
class MigrateSubtitleStorage(Task):
|
|
periodic = False
|
|
frequency = None
|
|
|
|
def run(self):
|
|
super(MigrateSubtitleStorage, self).run()
|
|
self.running = True
|
|
Log.Info(u"%s: Running subtitle storage migration", self.name)
|
|
storage = get_subtitle_storage()
|
|
|
|
def migrate(scandir_generic=False):
|
|
for fn in storage.get_all_files(scandir_generic=scandir_generic):
|
|
if fn.endswith(".json.gz"):
|
|
continue
|
|
Log.Debug(u"%s: Migrating %s", self.name, fn)
|
|
storage.load(None, fn)
|
|
|
|
try:
|
|
migrate()
|
|
except OSError:
|
|
migrate(scandir_generic=True)
|
|
|
|
storage.destroy()
|
|
|
|
|
|
class CacheMaintenance(Task):
|
|
periodic = True
|
|
frequency = "every 1 days"
|
|
|
|
main_cache_validity = 14 # days
|
|
pack_cache_validity = 4 # days
|
|
|
|
def run(self):
|
|
super(CacheMaintenance, self).run()
|
|
self.running = True
|
|
Log.Info(u"%s: Running cache maintenance", self.name)
|
|
now = datetime.datetime.now()
|
|
|
|
def remove_expired(path, expiry):
|
|
mtime = datetime.datetime.fromtimestamp(os.path.getmtime(path))
|
|
if mtime + datetime.timedelta(days=expiry) < now:
|
|
try:
|
|
os.remove(path)
|
|
except (IOError, OSError):
|
|
Log.Debug("Couldn't remove cache file: %s", os.path.basename(path))
|
|
|
|
# main cache
|
|
if config.new_style_cache:
|
|
for fn in subliminal_cache_region.backend.all_filenames:
|
|
remove_expired(fn, self.main_cache_validity)
|
|
|
|
# archive cache
|
|
for fn in glob.iglob(os.path.join(config.pack_cache_dir, "*.archive")):
|
|
remove_expired(fn, self.pack_cache_validity)
|
|
|
|
|
|
scheduler.register(LegacySearchAllRecentlyAddedMissing)
|
|
scheduler.register(SearchAllRecentlyAddedMissing)
|
|
scheduler.register(AvailableSubsForItem)
|
|
scheduler.register(DownloadSubtitleForItem)
|
|
scheduler.register(MissingSubtitles)
|
|
scheduler.register(FindBetterSubtitles)
|
|
scheduler.register(SubtitleStorageMaintenance)
|
|
scheduler.register(MigrateSubtitleStorage)
|
|
scheduler.register(MenuHistoryMaintenance)
|
|
scheduler.register(CacheMaintenance)
|