rewrite task scheduling; keep track of missing subtitles search task

This commit is contained in:
panni committed 2015-10-31 04:07:33 +01:00
1 parent abc26bbba2
commit 96360498f8
5 files changed
+127 -37

No files matched your search

+4
View File
@@ -199,6 +199,10 @@ class SubZeroAgent(object):
updateLocalMedia(media, media_type=self.agent_type)
# notify any running tasks about our finished update
for video in videos.keys():
scheduler.signal("updated_metadata", video.id)
def update_movies(self, metadata, media, lang):
videos = scanMovieMedia(media)
subtitles = downloadBestSubtitles(videos, min_score=int(Prefs["subtitles.search.minimumMovieScore"]))
+3 -3
View File
@@ -7,7 +7,7 @@ from support.helpers import pad_title, encode_message, decode_message
from support.auth import refresh_plex_token
from support.storage import resetStorage
from support.items import getRecentlyAddedItems, getOnDeckItems, refreshItem
from support.missing_subtitles import searchAllRecentlyAddedMissing
from support.missing_subtitles import getAllRecentlyAddedMissing, searchMissing
from support.background import scheduler
from support.lib import Plex
@@ -95,7 +95,7 @@ def RefreshItem(rating_key=None, came_from="/recent", force=False):
@route(PREFIX + '/missing/refresh')
def RefreshMissing():
Thread.CreateTimer(1.0, searchAllRecentlyAddedMissing)
Thread.CreateTimer(1.0, lambda: scheduler.run_task("searchAllRecentlyAddedMissing"))
return ObjectContainer(message="Refresh of recently added items with missing subtitles triggered")
@route(PREFIX + '/advanced')
@@ -124,7 +124,7 @@ def AdvancedMenu():
def ValidatePrefs():
Log.Debug("Validate Prefs called.")
config.initialize()
scheduler.discover_tasks()
scheduler.setup_tasks()
return
@route(PREFIX + '/advanced/restart/trigger')
+36 -25
View File
@@ -4,9 +4,6 @@ import datetime
import logging
import traceback
import tasks
def parse_frequency(s):
if s == "never":
return None, None
@@ -14,9 +11,14 @@ def parse_frequency(s):
return int(num), unit
class DefaultScheduler(object):
thread = None
running = False
registry = None
def __init__(self):
self.thread = None
self.running = False
self.registry = []
self.tasks = {}
if not "tasks" in Dict:
@@ -30,16 +32,17 @@ class DefaultScheduler(object):
Dict["tasks"] = {}
Dict.Save()
self.discover_tasks()
def register(self, task):
self.registry.append(task)
def discover_tasks(self):
def setup_tasks(self):
# discover tasks; todo: add registry
for item in dir(tasks):
if item.startswith("task_"):
task_name = item.split("task_")[1]
self.tasks[task_name] = {"task": getattr(tasks, item), "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task_name])}
for cls in self.registry:
task = cls(self)
self.tasks[task.name] = {"task": task, "frequency": parse_frequency(Prefs["scheduler.tasks.%s" % task.name])}
def run(self):
self.setup_tasks()
self.running = True
self.thread = Thread.Create(self.worker)
@@ -47,9 +50,9 @@ class DefaultScheduler(object):
self.running = False
def last_run(self, task):
if task not in Dict["tasks"]:
if task not in self.tasks:
return None
return Dict["tasks"][task]["last_run"]
return self.tasks[task]["task"].last_run
def next_run(self, task):
if task not in self.tasks:
@@ -57,13 +60,30 @@ class DefaultScheduler(object):
frequency_num, frequency_key = self.tasks[task]["frequency"]
if not frequency_num:
return None
last = self.last_run(task)
last = self.tasks[task]["task"].last_run
use_date = last
now = datetime.datetime.now()
if not use_date:
use_date = now
return max(use_date + datetime.timedelta(**{frequency_key: frequency_num}), now)
def run_task(self, name):
task = self.tasks[name]["task"]
task.running = True
try:
task.run()
except Exception, e:
Log.Error("Something went wrong when running %s: %s", name, traceback.format_exc())
finally:
task.last_run = datetime.datetime.now()
task.running = False
def signal(self, name, *args, **kwargs):
for task_name, info in self.tasks.iteritems():
task = info["task"]
if task.running:
task.signal(name, *args, **kwargs)
def worker(self):
while 1:
if not self.running:
@@ -71,31 +91,22 @@ class DefaultScheduler(object):
for name, info in self.tasks.iteritems():
now = datetime.datetime.now()
task = info["task"]
if name not in Dict["tasks"]:
Dict["tasks"][name] = {"last_run": None, "running": False}
Dict.Save()
continue
task_state = Dict["tasks"][name]
last_run, task_running = task_state["last_run"], task_state["running"]
if task_running:
if task.running:
continue
frequency_num, frequency_key = info["frequency"]
if not frequency_num:
continue
if not last_run or last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now:
task_state["running"] = True
try:
info["task"]()
except Exception, e:
Log.Error("Something went wrong when running %s: %s", name, traceback.format_exc())
finally:
task_state["last_run"] = now
task_state["running"] = False
Dict.Save()
if not task.last_run or task.last_run + datetime.timedelta(**{frequency_key: frequency_num}) <= now:
self.run_task(name)
Thread.Sleep(10.0)
+14 -6
View File
@@ -8,7 +8,7 @@ from support.config import config
from support.helpers import format_video
from lib import Plex
def itemSearchMissing(rating_key, kind="episode", internal=False, external=True, languages=[], section_blacklist=[], series_blacklist=[], item_blacklist=[], dry_run=False):
def itemDiscoverMissing(rating_key, kind="episode", internal=False, external=True, languages=[], section_blacklist=[], series_blacklist=[], item_blacklist=[]):
existing_subs = {"internal": [], "external": [], "count": 0}
item_id = int(rating_key)
@@ -58,14 +58,13 @@ def itemSearchMissing(rating_key, kind="episode", internal=False, external=True,
Log.Info(u"Subs still missing for '%s': %s", item_title, missing)
if missing:
Log.Info("Triggering refresh for '%s'", item_title)
if not dry_run:
Plex["library/metadata"].refresh(item_id)
return item_id, item_title
def searchAllRecentlyAddedMissing():
def getAllRecentlyAddedMissing():
items = getRecentlyAddedItems()
missing = []
for kind, title, item in items:
itemSearchMissing(
state = itemDiscoverMissing(
item.rating_key,
kind=kind,
languages=config.langList,
@@ -75,3 +74,12 @@ def searchAllRecentlyAddedMissing():
series_blacklist=config.scheduler_series_blacklist,
item_blacklist=config.scheduler_item_blacklist
)
if state:
# (item_id, title)
missing.append(state)
return missing
def searchMissing(items):
for item, title in items:
Log.Info("Triggering refresh for '%s'", title)
Plex["library/metadata"].refresh(item)
+70 -3
View File
@@ -1,7 +1,74 @@
# coding=utf-8
from support.missing_subtitles import searchAllRecentlyAddedMissing
import datetime
def task_searchAllRecentlyAddedMissing():
searchAllRecentlyAddedMissing()
from missing_subtitles import getAllRecentlyAddedMissing, searchMissing
from background import scheduler
class Task(object):
name = None
scheduler = None
stored_attributes = ("last_run", "running", "last_run_time")
def __init__(self, scheduler):
self.scheduler = scheduler
def __getattribute__(self, name):
if name in object.__getattribute__(self, "stored_attributes"):
return Dict["tasks"][self.name].get(name, None)
return object.__getattribute__(self, name)
def __setattr__(self, name, value):
if name in object.__getattribute__(self, "stored_attributes"):
Dict["tasks"][self.name][name] = value
Dict.Save()
return
object.__setattr__(self, name, value)
def signal(self, *args, **kwargs):
raise NotImplementedError
def prepare(self):
raise NotImplementedError
def run(self):
raise NotImplementedError
class SearchAllRecentlyAddedMissing(Task):
name = "searchAllRecentlyAddedMissing"
items_done = None
items_searching = None
percentage = 0
def signal(self, signal_name, *args, **kwargs):
if signal_name == "updated_metadata":
item_id = int(args[0])
self.items_done.append(item_id)
def run(self):
self.items_done = []
missing = getAllRecentlyAddedMissing()
ids = set([id for id, title in missing])
self.items_searching = ids
missing_count = len(ids)
# dispatch all searches
time_start = datetime.datetime.now()
searchMissing(missing)
while 1:
if ids == set(self.items_done):
Log.Debug("Task: %s, all items done", self.name)
break
self.percentage = int(round(len(self.items_done) * 100 / missing_count))
Thread.Sleep(1.0)
self.last_run_time = datetime.datetime.now() - time_start
self.percentage = 0
scheduler.register(SearchAllRecentlyAddedMissing)