From 96360498f894fe05057e4fb46f76292fa3609c8b Mon Sep 17 00:00:00 2001 From: panni Date: Sat, 31 Oct 2015 04:07:33 +0100 Subject: [PATCH] rewrite task scheduling; keep track of missing subtitles search task --- Contents/Code/__init__.py | 4 ++ Contents/Code/interface/menu.py | 6 +- Contents/Code/support/background.py | 61 ++++++++++-------- Contents/Code/support/missing_subtitles.py | 20 ++++-- Contents/Code/support/tasks.py | 73 +++++++++++++++++++++- 5 files changed, 127 insertions(+), 37 deletions(-) diff --git a/Contents/Code/__init__.py b/Contents/Code/__init__.py index 4acc52a5..75086d54 100644 --- a/Contents/Code/__init__.py +++ b/Contents/Code/__init__.py @@ -199,6 +199,10 @@ class SubZeroAgent(object): updateLocalMedia(media, media_type=self.agent_type) + # notify any running tasks about our finished update + for video in videos.keys(): + scheduler.signal("updated_metadata", video.id) + def update_movies(self, metadata, media, lang): videos = scanMovieMedia(media) subtitles = downloadBestSubtitles(videos, min_score=int(Prefs["subtitles.search.minimumMovieScore"])) diff --git a/Contents/Code/interface/menu.py b/Contents/Code/interface/menu.py index cbb553e5..19d86d90 100644 --- a/Contents/Code/interface/menu.py +++ b/Contents/Code/interface/menu.py @@ -7,7 +7,7 @@ from support.helpers import pad_title, encode_message, decode_message from support.auth import refresh_plex_token from support.storage import resetStorage from support.items import getRecentlyAddedItems, getOnDeckItems, refreshItem -from support.missing_subtitles import searchAllRecentlyAddedMissing +from support.missing_subtitles import getAllRecentlyAddedMissing, searchMissing from support.background import scheduler from support.lib import Plex @@ -95,7 +95,7 @@ def RefreshItem(rating_key=None, came_from="/recent", force=False): @route(PREFIX + '/missing/refresh') def RefreshMissing(): - Thread.CreateTimer(1.0, searchAllRecentlyAddedMissing) + Thread.CreateTimer(1.0, lambda: scheduler.run_task("searchAllRecentlyAddedMissing")) return ObjectContainer(message="Refresh of recently added items with missing subtitles triggered") @route(PREFIX + '/advanced') @@ -124,7 +124,7 @@ def AdvancedMenu(): def ValidatePrefs(): Log.Debug("Validate Prefs called.") config.initialize() - scheduler.discover_tasks() + scheduler.setup_tasks() return @route(PREFIX + '/advanced/restart/trigger') diff --git a/Contents/Code/support/background.py b/Contents/Code/support/background.py index ddc6c9e9..38605385 100644 --- a/Contents/Code/support/background.py +++ b/Contents/Code/support/background.py @@ -4,9 +4,6 @@ import datetime import logging import traceback -import tasks - - def parse_frequency(s): if s == "never": return None, None @@ -14,9 +11,14 @@ def parse_frequency(s): return int(num), unit class DefaultScheduler(object): + thread = None + running = False + registry = None + def __init__(self): self.thread = None self.running = False + self.registry = [] self.tasks = {} if not "tasks" in Dict: @@ -30,16 +32,17 @@ class DefaultScheduler(object): Dict["tasks"] = {} Dict.Save() - self.discover_tasks() + def register(self, task): + self.registry.append(task) - def discover_tasks(self): + def setup_tasks(self): # discover tasks; todo: add registry - for item in dir(tasks): - if item.startswith("task_"): - task_name = item.split("task_")[1] - self.tasks[task_name] = {"task": getattr(tasks, item), "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task_name])} + for cls in self.registry: + task = cls(self) + self.tasks[task.name] = {"task": task, "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task.name])} def run(self): + self.setup_tasks() self.running = True self.thread = Thread.Create(self.worker) @@ -47,9 +50,9 @@ class DefaultScheduler(object): self.running = False def last_run(self, task): - if task not in Dict["tasks"]: + if task not in self.tasks: return None - return Dict["tasks"][task]["last_run"] + return self.tasks[task]["task"].last_run def next_run(self, task): if task not in self.tasks: @@ -57,13 +60,30 @@ class DefaultScheduler(object): frequency_num, frequency_key = self.tasks[task]["frequency"] if not frequency_num: return None - last = self.last_run(task) + last = self.tasks[task]["task"].last_run use_date = last now = datetime.datetime.now() if not use_date: use_date = now return max(use_date + datetime.timedelta(**{frequency_key: frequency_num}), now) + def run_task(self, name): + task = self.tasks[name]["task"] + task.running = True + try: + task.run() + except Exception, e: + Log.Error("Something went wrong when running %s: %s", name, traceback.format_exc()) + finally: + task.last_run = datetime.datetime.now() + task.running = False + + def signal(self, name, *args, **kwargs): + for task_name, info in self.tasks.iteritems(): + task = info["task"] + if task.running: + task.signal(name, *args, **kwargs) + def worker(self): while 1: if not self.running: @@ -71,31 +91,22 @@ class DefaultScheduler(object): for name, info in self.tasks.iteritems(): now = datetime.datetime.now() + task = info["task"] if name not in Dict["tasks"]: Dict["tasks"][name] = {"last_run": None, "running": False} Dict.Save() continue - task_state = Dict["tasks"][name] - last_run, task_running = task_state["last_run"], task_state["running"] - if task_running: + if task.running: continue frequency_num, frequency_key = info["frequency"] if not frequency_num: continue - if not last_run or last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now: - task_state["running"] = True - try: - info["task"]() - except Exception, e: - Log.Error("Something went wrong when running %s: %s", name, traceback.format_exc()) - finally: - task_state["last_run"] = now - task_state["running"] = False - Dict.Save() + if not task.last_run or task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now: + self.run_task(name) Thread.Sleep(10.0) diff --git a/Contents/Code/support/missing_subtitles.py b/Contents/Code/support/missing_subtitles.py index fcda2155..bd17da97 100644 --- a/Contents/Code/support/missing_subtitles.py +++ b/Contents/Code/support/missing_subtitles.py @@ -8,7 +8,7 @@ from support.config import config from support.helpers import format_video from lib import Plex -def itemSearchMissing(rating_key, kind="episode", internal=False, external=True, languages=[], section_blacklist=[], series_blacklist=[], item_blacklist=[], dry_run=False): +def itemDiscoverMissing(rating_key, kind="episode", internal=False, external=True, languages=[], section_blacklist=[], series_blacklist=[], item_blacklist=[]): existing_subs = {"internal": [], "external": [], "count": 0} item_id = int(rating_key) @@ -58,14 +58,13 @@ def itemSearchMissing(rating_key, kind="episode", internal=False, external=True, Log.Info(u"Subs still missing for '%s': %s", item_title, missing) if missing: - Log.Info("Triggering refresh for '%s'", item_title) - if not dry_run: - Plex["library/metadata"].refresh(item_id) + return item_id, item_title -def searchAllRecentlyAddedMissing(): +def getAllRecentlyAddedMissing(): items = getRecentlyAddedItems() + missing = [] for kind, title, item in items: - itemSearchMissing( + state = itemDiscoverMissing( item.rating_key, kind=kind, languages=config.langList, @@ -75,3 +74,12 @@ def searchAllRecentlyAddedMissing(): series_blacklist=config.scheduler_series_blacklist, item_blacklist=config.scheduler_item_blacklist ) + if state: + # (item_id, title) + missing.append(state) + return missing + +def searchMissing(items): + for item, title in items: + Log.Info("Triggering refresh for '%s'", title) + Plex["library/metadata"].refresh(item) diff --git a/Contents/Code/support/tasks.py b/Contents/Code/support/tasks.py index 324d1a12..8500e119 100644 --- a/Contents/Code/support/tasks.py +++ b/Contents/Code/support/tasks.py @@ -1,7 +1,74 @@ # coding=utf-8 -from support.missing_subtitles import searchAllRecentlyAddedMissing +import datetime -def task_searchAllRecentlyAddedMissing(): - searchAllRecentlyAddedMissing() +from missing_subtitles import getAllRecentlyAddedMissing, searchMissing +from background import scheduler +class Task(object): + name = None + scheduler = None + + stored_attributes = ("last_run", "running", "last_run_time") + + def __init__(self, scheduler): + self.scheduler = scheduler + + def __getattribute__(self, name): + if name in object.__getattribute__(self, "stored_attributes"): + return Dict["tasks"][self.name].get(name, None) + + return object.__getattribute__(self, name) + + def __setattr__(self, name, value): + if name in object.__getattribute__(self, "stored_attributes"): + Dict["tasks"][self.name][name] = value + Dict.Save() + return + + object.__setattr__(self, name, value) + + def signal(self, *args, **kwargs): + raise NotImplementedError + + def prepare(self): + raise NotImplementedError + + def run(self): + raise NotImplementedError + + +class SearchAllRecentlyAddedMissing(Task): + name = "searchAllRecentlyAddedMissing" + items_done = None + items_searching = None + percentage = 0 + + def signal(self, signal_name, *args, **kwargs): + if signal_name == "updated_metadata": + item_id = int(args[0]) + self.items_done.append(item_id) + + def run(self): + self.items_done = [] + missing = getAllRecentlyAddedMissing() + ids = set([id for id, title in missing]) + self.items_searching = ids + missing_count = len(ids) + + # dispatch all searches + time_start = datetime.datetime.now() + searchMissing(missing) + + while 1: + if ids == set(self.items_done): + Log.Debug("Task: %s, all items done", self.name) + break + self.percentage = int(round(len(self.items_done) * 100 / missing_count)) + Thread.Sleep(1.0) + + self.last_run_time = datetime.datetime.now() - time_start + self.percentage = 0 + + +scheduler.register(SearchAllRecentlyAddedMissing) \ No newline at end of file