diff --git a/changedetectionio/api/operations.py b/changedetectionio/api/operations.py index 4f1acc477..5f4dc5ac5 100644 --- a/changedetectionio/api/operations.py +++ b/changedetectionio/api/operations.py @@ -150,13 +150,18 @@ def create_watch(datastore, data): raise ApiError(f"Watch limit reached ({current_watch_count}/{datastore.watch_limit} watches). " f"Cannot add more watches.", 429) - new_uuid = datastore.add_watch(url=url, extras=extras, tag=tags) + # With processor config, add it paused until the config is saved: the scheduler picks up a new watch on its + # next pass, and a first check before the config file exists would run with the processor's defaults + paused_for_config = bool(processor_config_data) and not extras.get('paused') + new_uuid = datastore.add_watch(url=url, extras=dict(extras, paused=True) if paused_for_config else extras, tag=tags) if not new_uuid: raise ApiError("Invalid or unsupported URL") # Save processor config to separate JSON file if processor_config_data: processors.save_processor_config(datastore, new_uuid, processor_config_data) + if paused_for_config: + datastore.update_watch(uuid=new_uuid, update_obj={'paused': False}) return new_uuid diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index bc3b4044f..6eb8c4c25 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1913,13 +1913,18 @@ def ticker_thread_check_time_launch_checks(): # Record the last used time proxy_last_called_time[watch_proxy] = int(time.time()) + # running_uuids / queued_uuids are from the start of this pass: a worker may have taken it since + if worker_pool.is_watch_running(uuid) or uuid in update_q.get_queued_uuids(): + continue + # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it. priority = int(time.time()) - # Into the queue with you + # Into the queue with you. scheduled_at lets a worker drop it when the watch was checked since + # (a worker between taking it off the queue and claiming it is in neither list above) queued_successfully = worker_pool.queue_item_async_safe( update_q, - queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid}), + queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'scheduled_at': now}), ) if queued_successfully: logger.debug( diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 13cdccb88..9a0f9cc6c 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -48,6 +48,16 @@ BROTLI_COMPRESS_SIZE_THRESHOLD = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRE # Keyed by data_dir so it survives Watch object recreation, deepcopy, and concurrent requests. # Invalidated explicitly when the favicon is saved or the watch history is cleared. _FAVICON_FILENAME_CACHE: dict = {} +# Held while looking a favicon up and while forgetting one, so a lookup that globbed before a save can't store its +# stale "no favicon" after the save cleared the cache +import threading as _threading +_FAVICON_FILENAME_LOCK = _threading.Lock() +_MISSING = object() + + +def _forget_favicon_filename(data_dir): + with _FAVICON_FILENAME_LOCK: + _FAVICON_FILENAME_CACHE.pop(data_dir, None) minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} @@ -436,7 +446,7 @@ class model(EntityPersistenceMixin, watch_base): # A save or check running now may rename its .tmp file away between glob() and here item.unlink(missing_ok=True) - _FAVICON_FILENAME_CACHE.pop(self.data_dir, None) + _forget_favicon_filename(self.data_dir) # Force the attr to recalculate bump = self.history @@ -1036,7 +1046,7 @@ class model(EntityPersistenceMixin, watch_base): logger.debug(f"UUID: {self.get('uuid')} could not remove {stale}: {e}") # Invalidate module-level favicon filename cache for this watch - _FAVICON_FILENAME_CACHE.pop(self.data_dir, None) + _forget_favicon_filename(self.data_dir) # A signal that could trigger the socket server to update the browser also watch_check_update = signal('watch_favicon_bump') @@ -1060,7 +1070,7 @@ class model(EntityPersistenceMixin, watch_base): os.unlink(fname) except OSError as e: logger.warning(f"UUID: {self.get('uuid')} could not remove favicon {fname}: {e}") - _FAVICON_FILENAME_CACHE.pop(self.data_dir, None) + _forget_favicon_filename(self.data_dir) def get_favicon_filename(self) -> str | None: """ @@ -1073,13 +1083,15 @@ class model(EntityPersistenceMixin, watch_base): Returns: str: Basename of the favicon file, or None if not found. """ - if self.data_dir in _FAVICON_FILENAME_CACHE: - return _FAVICON_FILENAME_CACHE[self.data_dir] + cached = _FAVICON_FILENAME_CACHE.get(self.data_dir, _MISSING) + if cached is not _MISSING: + return cached import glob - files = glob.glob(os.path.join(self.data_dir, "favicon.*")) - fname = os.path.basename(files[0]) if files else None - _FAVICON_FILENAME_CACHE[self.data_dir] = fname + with _FAVICON_FILENAME_LOCK: + files = glob.glob(os.path.join(self.data_dir, "favicon.*")) + fname = os.path.basename(files[0]) if files else None + _FAVICON_FILENAME_CACHE[self.data_dir] = fname return fname def get_screenshot_as_thumbnail(self, max_age=3200): diff --git a/changedetectionio/tests/test_favicon_url_change.py b/changedetectionio/tests/test_favicon_url_change.py index 79a438b4d..e99997f6d 100644 --- a/changedetectionio/tests/test_favicon_url_change.py +++ b/changedetectionio/tests/test_favicon_url_change.py @@ -9,7 +9,7 @@ import json from flask import url_for -from .util import set_original_response, delete_all_watches +from .util import set_original_response, delete_all_watches, wait_for_all_checks SVG_BASE64 = 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxIDEiLz4=' @@ -17,6 +17,8 @@ SVG_BASE64 = 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9Ij def _add_watch_with_favicon(client, url): datastore = client.application.config.get('DATASTORE') uuid = datastore.add_watch(url=url) + # Its first check runs now, let it finish so it isn't looking the favicon up while it's being saved + wait_for_all_checks(client) watch = datastore.data['watching'][uuid] watch.ensure_data_dir_exists() watch.bump_favicon(url="favicon.svg", favicon_base_64=SVG_BASE64) diff --git a/changedetectionio/tests/test_queue_ui.py b/changedetectionio/tests/test_queue_ui.py index a632593ed..99d99fa5f 100644 --- a/changedetectionio/tests/test_queue_ui.py +++ b/changedetectionio/tests/test_queue_ui.py @@ -8,10 +8,24 @@ so update_q is a single instance. """ import time + +import pytest from flask import url_for -def _quiesce(live_server, timeout=3.0): +@pytest.fixture(autouse=True) +def restore_workers(live_server): + """These tests stop every worker. Put them back afterwards, or the next test on this pytest worker starts with + none until the scheduler's health check (every 60s) restarts them.""" + from changedetectionio import worker_pool + before = worker_pool.get_worker_count() + yield + if before and worker_pool.get_worker_count() != before: + live_server.app.set_workers(0) + live_server.app.set_workers(before) + + +def _quiesce(live_server, monkeypatch, timeout=3.0): """Pause scheduler, shut down all workers, wait for in-flight async workers to fully drain, clear queue. Returns (update_q, worker_pool, queuedWatchMetaData). @@ -24,6 +38,11 @@ def _quiesce(live_server, timeout=3.0): from changedetectionio.flask_app import update_q live_server.app.config['DATASTORE'].data['settings']['application']['all_paused'] = True + # The scheduler's health check (every 60s, paused or not) restarts workers up to FETCH_WORKERS / the workers + # setting; if it ran during the test the workers came back and took the sentinel, "Could not get queue to a + # quiescent state". Expect none while these tests run, restore_workers puts them back after. + monkeypatch.setenv('FETCH_WORKERS', '0') + monkeypatch.setitem(live_server.app.config['DATASTORE'].data['settings']['requests'], 'workers', 0) live_server.app.set_workers(0) deadline = time.time() + timeout @@ -62,9 +81,9 @@ def _push(update_q, worker_pool, qmeta, uuid, priority=1): assert ok, f"queue_item_async_safe returned falsy for {uuid}" -def test_queue_page_renders_when_empty(client, live_server, measure_memory_usage, datastore_path): +def test_queue_page_renders_when_empty(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """Empty queue: HTML 200, JSON shape matches contract.""" - _quiesce(live_server) + _quiesce(live_server, monkeypatch) res = client.get(url_for("ui.ui_queue.queue_page")) assert res.status_code == 200 @@ -83,13 +102,13 @@ def test_queue_page_renders_when_empty(client, live_server, measure_memory_usage assert data["running_count"] == 0 -def test_queue_state_full_lifecycle(client, live_server, measure_memory_usage, datastore_path): +def test_queue_state_full_lifecycle(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """One consolidated test of pushed items, priority labels, deleted-watch handling, and Clear Queue. Combined into one test deliberately — having separate tests revealed cross-test state leakage (workers occasionally surviving teardown) that this single test avoids. """ - update_q, worker_pool, qmeta = _quiesce(live_server) + update_q, worker_pool, qmeta = _quiesce(live_server, monkeypatch) running_uuid = "11111111-1111-1111-1111-111111111111" immediate_uuid = "22222222-2222-2222-2222-222222222222" @@ -158,9 +177,9 @@ def test_queue_state_full_lifecycle(client, live_server, measure_memory_usage, d update_q.clear() -def test_queue_includes_timestamps(client, live_server, measure_memory_usage, datastore_path): +def test_queue_includes_timestamps(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """Queue items carry enqueued_at; running items carry started_at.""" - update_q, worker_pool, qmeta = _quiesce(live_server) + update_q, worker_pool, qmeta = _quiesce(live_server, monkeypatch) # Push something — enqueued_at should be ~now before = time.time() @@ -194,9 +213,9 @@ def test_queue_includes_timestamps(client, live_server, measure_memory_usage, da update_q.clear() -def test_queue_pagination(client, live_server, measure_memory_usage, datastore_path): +def test_queue_pagination(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """Page math: per_page from settings.pager_size, total_pages, page slicing.""" - update_q, worker_pool, qmeta = _quiesce(live_server) + update_q, worker_pool, qmeta = _quiesce(live_server, monkeypatch) # Override pager_size for this test so we don't have to push 50+ items live_server.app.config['DATASTORE'].data['settings']['application']['pager_size'] = 2 @@ -229,9 +248,9 @@ def test_queue_pagination(client, live_server, measure_memory_usage, datastore_p update_q.clear() -def test_cancel_running_uuid_helper(client, live_server, measure_memory_usage, datastore_path): +def test_cancel_running_uuid_helper(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """worker_pool.cancel_running_uuid() drops tracking and returns cancelled flag.""" - update_q, worker_pool, qmeta = _quiesce(live_server) + update_q, worker_pool, qmeta = _quiesce(live_server, monkeypatch) synth = "cccccccc-cccc-cccc-cccc-cccccccccccc" assert worker_pool.claim_uuid_for_processing(synth, worker_id=999) is True @@ -253,9 +272,9 @@ def test_cancel_running_uuid_helper(client, live_server, measure_memory_usage, d assert result2["worker_id"] is None -def test_cancel_running_endpoint(client, live_server, measure_memory_usage, datastore_path): +def test_cancel_running_endpoint(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """POST /queue/cancel-running: 200 on success, 400 on empty uuid, 404 on unknown uuid.""" - update_q, worker_pool, qmeta = _quiesce(live_server) + update_q, worker_pool, qmeta = _quiesce(live_server, monkeypatch) synth = "dddddddd-dddd-dddd-dddd-dddddddddddd" assert worker_pool.claim_uuid_for_processing(synth, worker_id=999) is True diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index a07d87d21..860af98ff 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -239,6 +239,26 @@ def wait_for_all_checks(client=None): """ from changedetectionio.flask_app import update_q as global_update_q from changedetectionio import worker_pool + done = worker_pool.wait_for_all_checks(global_update_q, timeout=150) + if client is None: + return done + + # A monitor that was just added (an import, the API) isn't queued by that request, the scheduler finds it on its + # next pass, up to a second later. Until then the queue is empty and the workers idle, so the wait above can return + # before its first check has even started, and the test changes the page under it. Wait for the scheduler to pick + # up every unpaused monitor that was never checked, a few seconds at most (a time schedule may keep one waiting). + datastore = client.application.config.get('DATASTORE') + if not datastore or datastore.data['settings']['application'].get('all_paused'): + return done + deadline = time.time() + 5 + while time.time() < deadline: + unchecked = [w for w in list(datastore.data['watching'].values()) + if not w.get('paused') and not w.get('last_checked')] + if not unchecked: + break + time.sleep(0.2) + else: + return done return worker_pool.wait_for_all_checks(global_update_q, timeout=150) diff --git a/changedetectionio/worker.py b/changedetectionio/worker.py index 89bfb6e93..3914e098d 100644 --- a/changedetectionio/worker.py +++ b/changedetectionio/worker.py @@ -101,6 +101,16 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec worker_pool.queue_item_async_safe(q, deferred_item, silent=True) continue + # A scheduled check (the ticker stamps scheduled_at) for a monitor that was checked since it was scheduled: + # the ticker can queue it again while a worker has taken it off the queue but not claimed it yet, and that + # second copy is also re-queued above until the first check is done. One check is enough. + scheduled_at = queued_item_data.item.get('scheduled_at') + checked_watch = datastore.data['watching'].get(uuid) + if scheduled_at and checked_watch is not None and checked_watch.get('last_checked', 0) >= int(scheduled_at): + logger.debug(f"Worker {worker_id} skipping scheduled check of {uuid}, already checked since it was queued") + worker_pool.release_uuid_from_processing(uuid, worker_id=worker_id) + continue + except asyncio.TimeoutError: # No jobs available - check if we should restart based on time while idle runtime = time.time() - start_time diff --git a/docs/THIRD-PARTY-LICENSES.md b/docs/THIRD-PARTY-LICENSES.md index bf1f915c9..bad78b993 100644 --- a/docs/THIRD-PARTY-LICENSES.md +++ b/docs/THIRD-PARTY-LICENSES.md @@ -1,10 +1,10 @@ # Third-party licences -Generated 2026-11-04 from the built container. +Generated 2026-11-05 from the built container. **Do not edit by hand** - run `.github/scripts/generate-third-party-licenses.sh`. Every third-party Python package bundled in the distributed image -(176 packages). This exists to satisfy attribution requirements and to +(177 packages). This exists to satisfy attribution requirements and to answer procurement and due-diligence questions without a manual audit. For copyright ownership and our licence elections, see `NOTICE`. @@ -22,14 +22,14 @@ For copyright ownership and our licence elections, see `NOTICE`. | [apprise](https://appriseit.com) | 2.1.0 | BSD-2-Clause | | [arrow](https://github.com/arrow-py/arrow) | 1.4.0 | Apache Software License | | [attrs](https://www.attrs.org/en/stable/changelog.html) | 26.1.0 | MIT | -| [Authlib](https://github.com/authlib/authlib) | 1.9.4 | BSD License | +| [Authlib](https://github.com/authlib/authlib) | 1.9.5 | BSD License | | [babel](https://babel.pocoo.org/) | 2.18.0 | BSD License | | [backports.zstd](https://github.com/rogdham/backports.zstd) | 1.8.0 | PSF-2.0 | | [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) | 4.15.0 | MIT License | | [bidict](https://github.com/jab/bidict) | 0.24.1 | MPL-2.0 | | [blinker](https://github.com/pallets-eco/blinker/) | 1.9.0 | MIT License | -| [boto3](https://github.com/boto/boto3) | 1.43.123 | Apache-2.0 | -| [botocore](https://github.com/boto/botocore) | 1.43.123 | Apache-2.0 | +| [boto3](https://github.com/boto/boto3) | 1.43.124 | Apache-2.0 | +| [botocore](https://github.com/boto/botocore) | 1.43.124 | Apache-2.0 | | [brotli](https://github.com/google/brotli) | 1.2.1 | MIT | | [certifi](https://github.com/certifi/python-certifi) | 2026.10.23 | Mozilla Public License 2.0 (MPL 2.0) | | [cffi](https://cffi.readthedocs.io/en/latest/whatsnew.html) | 2.1.1 | MIT-0 | @@ -56,6 +56,7 @@ For copyright ownership and our licence elections, see `NOTICE`. | [Flask-RESTful](https://www.github.com/flask-restful/flask-restful/) | 0.3.10 | BSD License | | [Flask-SocketIO](https://code.miguelgrinberg.com/miguelgrinberg/flask-socketio) | 5.7.2 | MIT | | [Flask-WTF](https://flask-wtf.readthedocs.io/) | 1.3.0 | BSD License | +| [frostwork](https://github.com/scrapy/frostwork) | 0.2.2 | Apache-2.0 | | [frozenlist](https://github.com/aio-libs/frozenlist) | 1.8.0 | Apache-2.0 | | [fsspec](https://github.com/fsspec/filesystem_spec) | 2026.9.0 | BSD-3-Clause | | [gevent](http://www.gevent.org/) | 26.9.0 | MIT | @@ -103,7 +104,7 @@ For copyright ownership and our licence elections, see `NOTICE`. | [mypy_extensions](https://github.com/python/mypy_extensions) | 1.2.0 | MIT | | [numpy](https://numpy.org) | 2.4.6 | BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0 | | [oauthlib](https://github.com/oauthlib/oauthlib) | 4.0.1 | BSD-3-Clause | -| [openai](https://github.com/openai/openai-python) | 3.45.0 | Apache-2.0 | +| [openai](https://github.com/openai/openai-python) | 3.46.0 | Apache-2.0 | | [openapi-core](https://github.com/python-openapi/openapi-core) | 0.23.1 | BSD License | | [openapi-schema-validator](https://github.com/python-openapi/openapi-schema-validator) | 0.8.1 | BSD License | | [openapi-spec-validator](https://github.com/python-openapi/openapi-spec-validator) | 0.8.5 | Apache-2.0 | @@ -114,7 +115,7 @@ For copyright ownership and our licence elections, see `NOTICE`. | [outcome](https://github.com/python-trio/outcome) | 1.3.0.post0 | Apache Software License; MIT License | | [packaging](https://github.com/pypa/packaging) | 26.3 | Apache-2.0 OR BSD-2-Clause | | [packaging](https://github.com/pypa/packaging) | 26.3 | Apache-2.0 OR BSD-2-Clause | -| [paho-mqtt](http://eclipse.org/paho) | 2.1.0 | EPL-2.0 OR BSD-3-Clause | +| [paho-mqtt](https://eclipse.org/paho) | 2.2.0 | EPL-2.0 OR BSD-3-Clause | | [panzi-json-logic](https://github.com/panzi/panzi-json-logic) | 1.0.1 | MIT License | | [pathable](https://github.com/p1c2u/pathable) | 0.5.0 | Apache Software License | | [pillow](https://python-pillow.org) | 13.0.0 | MIT-CMU | @@ -125,9 +126,9 @@ For copyright ownership and our licence elections, see `NOTICE`. | [psutil](https://github.com/giampaolo/psutil) | 7.2.2 | BSD-3-Clause | | [puremagic](https://github.com/cdgriffith/puremagic) | 1.30 | MIT License | | [pycparser](https://github.com/eliben/pycparser) | 3.12 | BSD-3-Clause | -| [pydantic](https://github.com/pydantic/pydantic) | 2.14.2 | MIT | +| [pydantic](https://github.com/pydantic/pydantic) | 2.14.3 | MIT | | [pydantic-settings](https://github.com/pydantic/pydantic-settings) | 2.16.2 | MIT | -| [pydantic_core](https://github.com/pydantic) | 2.50.2 | MIT | +| [pydantic_core](https://github.com/pydantic) | 2.50.4 | MIT | | [pyee](https://github.com/jfhbrook/pyee) | 13.0.1 | MIT License | | [Pygments](https://pygments.org) | 2.21.0 | BSD-2-Clause | | [pyparsing](https://github.com/pyparsing/pyparsing/) | 3.3.3 | MIT | @@ -162,7 +163,7 @@ For copyright ownership and our licence elections, see `NOTICE`. | [six](https://github.com/benjaminp/six) | 1.17.0 | MIT License | | [sniffio](https://github.com/python-trio/sniffio) | 1.3.1 | Apache Software License; MIT License | | [sortedcontainers](http://www.grantjenks.com/docs/sortedcontainers/) | 2.4.0 | Apache Software License | -| [soupsieve](https://github.com/facelessuser/soupsieve) | 4.0 | MIT | +| [soupsieve](https://github.com/facelessuser/soupsieve) | 4.0.1 | MIT | | [tiktoken](https://github.com/openai/tiktoken) | 0.14.0 | MIT License Copyright (c) 2022 OpenAI, Shantanu Jain Permission is hereby granted, free... | | [timeago](https://github.com/hustcc/timeago) | 1.0.16 | MIT | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.23.3 | Apache Software License | @@ -171,12 +172,12 @@ For copyright ownership and our licence elections, see `NOTICE`. | [trio-websocket](https://github.com/python-trio/trio-websocket) | 0.12.2 | MIT License | | [truststore](https://github.com/sethmlarson/truststore) | 0.10.5 | MIT | | [typing-inspect](https://github.com/ilevkivskyi/typing_inspect) | 0.5.0 | MIT License | -| [typing-inspection](https://github.com/pydantic/typing-inspection) | 0.4.4 | MIT | +| [typing-inspection](https://github.com/pydantic/typing-inspection) | 0.4.5 | MIT | | [typing_extensions](https://github.com/python/typing_extensions) | 4.16.0 | PSF-2.0 | | [tzdata](https://github.com/python/tzdata) | 2026.6 | Apache-2.0 | | [urllib3](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) | 2.8.0 | MIT | | [validators](https://python-validators.github.io/validators) | 0.36.0 | MIT License | -| [w3lib](https://github.com/scrapy/w3lib) | 2.5.0 | BSD-3-Clause | +| [w3lib](https://github.com/scrapy/w3lib) | 2.6.0 | BSD-3-Clause | | [webencodings](https://github.com/CourtBouillon/webencodings) | 0.6.1 | BSD License | | [websocket-client](https://github.com/websocket-client/websocket-client) | 1.9.2 | Apache-2.0 | | [websockets](https://github.com/aaugustin/websockets) | 10.4 | BSD License |