mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-11-18 20:54:26 +00:00
Example plugins - API fetcher and check stats, watch list notes hook, update_finalize fix (#4622)
* Example plugins - API fetcher and check stats examples, watch list notes hook, update_finalize fix New hook watch_list_extra_text(watch) (ported from browser-ui): short plain-text notes under each watch in the watch list, escaped when rendered. New example plugins, each `pip install .`-able and importing only from plugin_api: - fetcher/api_fetcher: register_content_fetcher, plugin_settings_tab, fetcher_status_icon, plugin_static_path - lifecycle/check_stats: update_handler_alter, update_finalize, watch_list_extra_text, ui_edit_stats_extras, get_html_head_extras, plugin_static_path - processor/wordcounter: also shows "Found: N words" in the watch list via watch_list_extra_text Fixes found by the examples: - update_finalize always got update_handler=None after a successful check (the worker deleted the handler before the finally block captured it), so update_handler_alter + update_finalize couldn't be paired - The API rejected plugin fetchers in 'fetch_backend' (hardcoded pattern), now built from installed fetchers - fetcher_status_icon docstring said it returns HTML, it returns a dict plugin_api 0.3.0: exports Fetcher and load_plugin_settings. Tests check each example end to end, including that it's listed under Settings > Info. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Example plugins - Rename the fetcher example to fetcher/example_browser_fetcher "Example browser fetcher": an extra fetcher that gets pages through its own API. Package cdio_example_browser_fetcher, fetch method html_example_browser_fetcher. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Example browser fetcher - Return a screenshot (random noise PNG, standard library only) Shows the screenshot side of a fetcher: supports_screenshots = True and self.screenshot set to PNG bytes, saved as last-screenshot.png. The test checks it's served as a valid 240x160 PNG. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c0b9320ef3
commit
b40193b080
22 files changed
+618
-30
No files matched your search
@@ -28,6 +28,17 @@ def build_merged_spec_dict():
|
||||
with open(spec_path, 'r', encoding='utf-8') as f:
|
||||
spec_dict = yaml.safe_load(f)
|
||||
|
||||
# The static spec's fetch_backend pattern only knows the built-in fetchers, accept every installed one (plugins too)
|
||||
try:
|
||||
import re
|
||||
from changedetectionio import content_fetchers
|
||||
fetcher_names = sorted(name for name, _description in content_fetchers.available_fetchers())
|
||||
if fetcher_names:
|
||||
spec_dict['components']['schemas']['WatchBase']['properties']['fetch_backend']['pattern'] = \
|
||||
'^(system|extra_browser_.+|' + '|'.join(re.escape(n) for n in fetcher_names) + ')$'
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to add installed fetchers to the API spec: {e}")
|
||||
|
||||
try:
|
||||
from changedetectionio.processors import find_processors, get_parent_module
|
||||
# The static spec only lists the built-ins, accept every installed processor (plugins too)
|
||||
|
||||
@@ -87,6 +87,9 @@
|
||||
<a href="{{url_for('watchlist.index', tag=watch_tag_uuid) }}" class="watch-tag-list tag-{{ watch_tag.title|sanitize_tag_class }}">{{ watch_tag.title }}</a>
|
||||
{%- endfor -%}
|
||||
<div class="error-text" style="display:none;">{{ error_texts|safe }}</div>
|
||||
{%- for note in watch_list_extra_text(watch) %}
|
||||
<div class="extra-watch-text">{{ note }}</div>
|
||||
{%- endfor %}
|
||||
{%- if watch['processor'] == 'text_json_diff' -%}
|
||||
{%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data'] -%}
|
||||
<div class="ldjson-price-track-offer">{{ _('Switch to Restock & Price watch mode?') }} <button type="submit" formmethod="post" formaction="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">{{ _('Yes') }}</button> <button type="submit" formmethod="post" formaction="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="bare-btn bare-btn--link">{{ _('No') }}</button></div>
|
||||
|
||||
@@ -5,7 +5,7 @@ active until you install it.
|
||||
|
||||
To install one, run `pip install .` from its directory, in the same Python environment that runs
|
||||
changedetection.io, then restart changedetection.io. Plugins are picked up through the `changedetectionio`
|
||||
entry point in their `pyproject.toml`. They need plugin API 0.2.0 or newer
|
||||
entry point in their `pyproject.toml`. They need plugin API 0.3.0 or newer
|
||||
(`changedetectionio.plugin_api.PLUGIN_API_VERSION`).
|
||||
|
||||
Plugins import only from `changedetectionio.plugin_api`, everything else in the package is internal and
|
||||
@@ -21,6 +21,7 @@ the page and reports a change when the count changes. The `min_word_length` sett
|
||||
**Hooks**
|
||||
- `register_processor` - adds `word_counter` as a processor
|
||||
- `get_api_watch_processor_state` - shows the count as `processor_state_word_counter` in the API
|
||||
- `watch_list_extra_text` - "Found: 5 words" under the watch in the watch list
|
||||
|
||||
**Also shows** `processor.py` (the check itself) and `api.yaml` (documents `processor_config_word_counter`
|
||||
and `processor_state_word_counter` in the API spec).
|
||||
@@ -43,10 +44,60 @@ curl "http://localhost:5000/api/v1/watch/YOUR-UUID" -H "x-api-key: YOUR_API_KEY"
|
||||
|
||||
---
|
||||
|
||||
## Example browser fetcher - `fetcher/example_browser_fetcher`
|
||||
|
||||
A new fetch method, "Example browser fetcher", that gets pages through its own API instead of fetching
|
||||
them directly, for example a remote browser or rendering service.
|
||||
|
||||
**Hooks**
|
||||
- `register_content_fetcher` - adds the `html_example_browser_fetcher` fetch method
|
||||
- `plugin_settings_tab` - an "Example browser fetcher" tab in Settings for the API URL and key
|
||||
- `fetcher_status_icon` - the icon next to watches using it in the watch list
|
||||
- `plugin_static_path` - serves that icon
|
||||
|
||||
**Also shows** `fetcher.py`, a `Fetcher` subclass, reading its settings with `load_plugin_settings` and
|
||||
returning a screenshot (random noise here, a real browser service would send its own).
|
||||
|
||||
**Install**
|
||||
```bash
|
||||
cd changedetectionio/example_plugins/fetcher/example_browser_fetcher
|
||||
pip install .
|
||||
```
|
||||
|
||||
**Try it:** in Settings > "Example browser fetcher", set the API URL with `{url}` where the page URL goes,
|
||||
for example `https://api.example.com/fetch?url={url}`, and your key (sent as `X-Api-Key`). Then pick
|
||||
"Example browser fetcher" as a watch's fetch method, or `"fetch_backend": "html_example_browser_fetcher"`
|
||||
in the API.
|
||||
|
||||
---
|
||||
|
||||
## Check stats - `lifecycle/check_stats`
|
||||
|
||||
Times every check. Shows "Last check took 1.23s" under each watch in the watch list, and a small table on
|
||||
the edit page's Stats tab. Stats are kept in memory, so they start again after a restart.
|
||||
|
||||
**Hooks**
|
||||
- `update_handler_alter` - runs as a check starts, notes the time
|
||||
- `update_finalize` - runs when the check is over (worked or not), records how long it took
|
||||
- `watch_list_extra_text` - the note under the watch in the watch list
|
||||
- `ui_edit_stats_extras` - the table on the edit page's Stats tab
|
||||
- `get_html_head_extras` - links its CSS on every page
|
||||
- `plugin_static_path` - serves that CSS
|
||||
|
||||
**Install**
|
||||
```bash
|
||||
cd changedetectionio/example_plugins/lifecycle/check_stats
|
||||
pip install .
|
||||
```
|
||||
|
||||
**Try it:** open the watch list after the next checks have run.
|
||||
|
||||
---
|
||||
|
||||
## Writing your own
|
||||
|
||||
Copy an example's directory, then rename the package directory and its `name` / entry point in
|
||||
`pyproject.toml`. For a processor, also change `PROCESSOR_NAME` and the two schema names in `api.yaml`.
|
||||
`pyproject.toml`. Once installed, it's listed under Settings > Info with the first line of its docstring. For a processor, also change `PROCESSOR_NAME` and the two schema names in `api.yaml`.
|
||||
Every hook and its arguments are documented on `ChangeDetectionSpec` in
|
||||
[`pluggy_interface.py`](../pluggy_interface.py).
|
||||
|
||||
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Example plugin - an extra browser fetcher that gets pages through its own API.
|
||||
|
||||
Configure it under Settings > "Example browser fetcher", then pick "Example browser fetcher" as a
|
||||
watch's fetch method. Install it with `pip install .` from the directory above this one, see
|
||||
example_plugins/README.md. It is not active until installed (or registered).
|
||||
"""
|
||||
import os
|
||||
|
||||
from changedetectionio.plugin_api import hookimpl
|
||||
|
||||
# Names your settings file ({datastore}/cdio_example_browser_fetcher.json) and the settings tab
|
||||
PLUGIN_ID = 'cdio_example_browser_fetcher'
|
||||
# Fetcher names must start with 'html_', this is what's stored in a watch's 'fetch_backend'
|
||||
FETCHER_NAME = 'html_example_browser_fetcher'
|
||||
|
||||
# Set at startup by changedetection.io (inject_datastore_into_plugins), the fetcher reads its
|
||||
# settings with it
|
||||
datastore = None
|
||||
|
||||
|
||||
@hookimpl
|
||||
def register_content_fetcher():
|
||||
from .fetcher import fetcher
|
||||
return (FETCHER_NAME, fetcher)
|
||||
|
||||
|
||||
@hookimpl
|
||||
def plugin_settings_tab():
|
||||
from wtforms import Form, StringField
|
||||
|
||||
class SettingsForm(Form):
|
||||
# These share the settings page form with everything else, so give them unique names
|
||||
example_browser_fetcher_url = StringField(
|
||||
'API URL',
|
||||
description='Use {url} where the page URL goes, for example https://api.example.com/fetch?url={url}')
|
||||
example_browser_fetcher_key = StringField(
|
||||
'API key', description='Sent to the API in the X-Api-Key header')
|
||||
|
||||
return {'plugin_id': PLUGIN_ID, 'tab_label': 'Example browser fetcher', 'form_class': SettingsForm}
|
||||
|
||||
|
||||
@hookimpl
|
||||
def fetcher_status_icon(fetcher_name):
|
||||
"""The icon next to watches using this fetcher in the watch list."""
|
||||
if fetcher_name != FETCHER_NAME:
|
||||
return None
|
||||
return {'filename': 'cdio_example_browser_fetcher.svg', 'alt': 'Example browser fetcher', 'title': 'Fetched with the example browser fetcher',
|
||||
'group': 'plugin', 'style': 'height: 1em;'}
|
||||
|
||||
|
||||
@hookimpl
|
||||
def plugin_static_path():
|
||||
"""Files in here are served at /static/plugin/<filename>."""
|
||||
return os.path.join(os.path.dirname(__file__), 'static')
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
import asyncio
|
||||
from urllib.parse import quote
|
||||
|
||||
from changedetectionio.plugin_api import Fetcher, load_plugin_settings
|
||||
|
||||
|
||||
def noise_png(width=240, height=160):
|
||||
"""A PNG of random noise, standing in for a real page screenshot. Standard library only."""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
|
||||
# Each row: filter type 0, then width RGB pixels
|
||||
pixels = b''.join(b'\x00' + os.urandom(width * 3) for _ in range(height))
|
||||
return (b'\x89PNG\r\n\x1a\n'
|
||||
+ chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) # 8 bit RGB
|
||||
+ chunk(b'IDAT', zlib.compress(pixels))
|
||||
+ chunk(b'IEND', b''))
|
||||
|
||||
|
||||
class fetcher(Fetcher):
|
||||
fetcher_description = 'Example browser fetcher'
|
||||
# Tells changedetection.io this fetcher returns a screenshot (see _fetch), so it's shown and saved
|
||||
supports_screenshots = True
|
||||
|
||||
def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def run(self, url=None, timeout=None, request_headers=None, **kwargs):
|
||||
# requests is blocking, keep it off the event loop
|
||||
await asyncio.get_event_loop().run_in_executor(None, lambda: self._fetch(url, timeout, request_headers))
|
||||
|
||||
def _fetch(self, url, timeout, request_headers):
|
||||
import requests
|
||||
from . import PLUGIN_ID, datastore
|
||||
|
||||
settings = load_plugin_settings(datastore.datastore_path, PLUGIN_ID) if datastore else {}
|
||||
api_url = (settings.get('example_browser_fetcher_url') or '').strip()
|
||||
if '{url}' not in api_url:
|
||||
raise Exception("Example browser fetcher isn't set up, add an API URL with {url} in Settings")
|
||||
|
||||
headers = dict(request_headers or {})
|
||||
if settings.get('example_browser_fetcher_key'):
|
||||
headers['X-Api-Key'] = settings['example_browser_fetcher_key']
|
||||
|
||||
r = requests.get(api_url.replace('{url}', quote(url, safe='')), headers=headers, timeout=timeout or 60)
|
||||
if not r.ok:
|
||||
raise Exception(f"Example browser fetcher API returned HTTP {r.status_code}")
|
||||
|
||||
# What a fetcher must set for the processor
|
||||
self.status_code = r.status_code
|
||||
self.headers = dict(r.headers)
|
||||
self.content = r.text
|
||||
self.raw_content = r.content
|
||||
# A real browser service would return its screenshot of the page, saved as last-screenshot.png
|
||||
# (so return PNG bytes). This example just makes some noise.
|
||||
self.screenshot = noise_png()
|
||||
|
||||
async def quit(self, watch=None):
|
||||
pass
|
||||
|
||||
def get_error(self):
|
||||
return self.error
|
||||
|
||||
def get_last_status_code(self):
|
||||
return self.status_code
|
||||
|
||||
def screenshot_step(self, step_n):
|
||||
return None
|
||||
|
||||
def is_ready(self):
|
||||
return True
|
||||
+1
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 10h-1.26A8 8 0 1 0 9 20h9a5 5 0 0 0 0-10z"/></svg>
|
||||
|
After Width: | Height: | Size: 222 B |
@@ -0,0 +1,23 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=77"] # 77+ for the SPDX license string
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "cdio-example-browser-fetcher"
|
||||
version = "0.1.0"
|
||||
description = "Example changedetection.io plugin: an extra browser fetcher that gets pages through its own API"
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.11"
|
||||
# No dependency on changedetection.io itself: install this into the same environment that runs it
|
||||
# (it needs plugin API 0.3.0 or newer, changedetectionio.plugin_api.PLUGIN_API_VERSION)
|
||||
dependencies = []
|
||||
|
||||
# This is what makes changedetection.io load the plugin at startup
|
||||
[project.entry-points.changedetectionio]
|
||||
cdio_example_browser_fetcher = "cdio_example_browser_fetcher"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["cdio_example_browser_fetcher"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
cdio_example_browser_fetcher = ["static/*"]
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
Example plugin - times every check and shows the result in the watch list and on the edit page.
|
||||
|
||||
Install it with `pip install .` from the directory above this one, see example_plugins/README.md.
|
||||
It is not active until installed (or registered), nothing scans example_plugins/ for plugins.
|
||||
|
||||
The stats are kept in memory only, so they start empty again after a restart.
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
from markupsafe import escape
|
||||
|
||||
from changedetectionio.plugin_api import hookimpl
|
||||
|
||||
# Per watch uuid: {'duration': seconds, 'ok': bool, 'error': str or None, 'finished': unix time, 'count': int}
|
||||
STATS = {}
|
||||
|
||||
# Where the handler keeps its start time between the two check hooks below
|
||||
_STARTED_ATTR = '_cdio_example_check_stats_started'
|
||||
|
||||
|
||||
@hookimpl
|
||||
def update_handler_alter(update_handler, watch, datastore):
|
||||
"""Called once a check is about to start, before the page is fetched."""
|
||||
setattr(update_handler, _STARTED_ATTR, time.monotonic())
|
||||
# None = keep the handler as it is, you could also return a wrapped one
|
||||
return None
|
||||
|
||||
|
||||
@hookimpl
|
||||
def update_finalize(update_handler, watch, datastore, processing_exception):
|
||||
"""Called once the check is over, whether it worked or not."""
|
||||
started = getattr(update_handler, _STARTED_ATTR, None) if update_handler else None
|
||||
if started is None or not watch:
|
||||
return
|
||||
previous = STATS.get(watch.get('uuid'), {})
|
||||
STATS[watch.get('uuid')] = {
|
||||
'duration': time.monotonic() - started,
|
||||
'ok': processing_exception is None,
|
||||
'error': str(processing_exception) if processing_exception else None,
|
||||
'finished': time.time(),
|
||||
'count': previous.get('count', 0) + 1,
|
||||
}
|
||||
|
||||
|
||||
@hookimpl
|
||||
def watch_list_extra_text(watch):
|
||||
"""A short note under the watch in the watch list. Called for every row, so in memory only."""
|
||||
stats = STATS.get(watch.get('uuid'))
|
||||
if not stats:
|
||||
return None
|
||||
if stats['ok']:
|
||||
return f"Last check took {stats['duration']:.2f}s"
|
||||
return f"Last check failed after {stats['duration']:.2f}s"
|
||||
|
||||
|
||||
@hookimpl
|
||||
def ui_edit_stats_extras(watch):
|
||||
"""A small table on the edit page's Stats tab. This is raw HTML, so escape anything you include."""
|
||||
stats = STATS.get(watch.get('uuid'))
|
||||
if not stats:
|
||||
return '<p class="cdio-example-check-stats">No checks timed since the last restart.</p>'
|
||||
rows = [
|
||||
('Checks timed since restart', stats['count']),
|
||||
('Last check took', f"{stats['duration']:.2f}s"),
|
||||
('Last check', 'OK' if stats['ok'] else f"Failed: {stats['error']}"),
|
||||
]
|
||||
body = ''.join(f'<tr><td>{escape(label)}</td><td>{escape(value)}</td></tr>' for label, value in rows)
|
||||
return f'<table class="pure-table cdio-example-check-stats"><tbody>{body}</tbody></table>'
|
||||
|
||||
|
||||
@hookimpl
|
||||
def plugin_static_path():
|
||||
"""Files in here are served at /static/plugin/<filename>."""
|
||||
return os.path.join(os.path.dirname(__file__), 'static')
|
||||
|
||||
|
||||
@hookimpl
|
||||
def get_html_head_extras():
|
||||
"""Added inside <head> on every page."""
|
||||
from flask import url_for
|
||||
css = url_for('static_content', group='plugin', filename='cdio_example_check_stats.css')
|
||||
return f'<link rel="stylesheet" href="{css}">'
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
/* Served by the plugin's plugin_static_path(), linked from get_html_head_extras() */
|
||||
.cdio-example-check-stats td:first-child {
|
||||
font-weight: bold;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=77"] # 77+ for the SPDX license string
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "cdio-example-check-stats"
|
||||
version = "0.1.0"
|
||||
description = "Example changedetection.io plugin: times every check, shows it in the watch list and edit page"
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.11"
|
||||
# No dependency on changedetection.io itself: install this into the same environment that runs it
|
||||
# (it needs plugin API 0.3.0 or newer, changedetectionio.plugin_api.PLUGIN_API_VERSION)
|
||||
dependencies = []
|
||||
|
||||
# This is what makes changedetection.io load the plugin at startup
|
||||
[project.entry-points.changedetectionio]
|
||||
cdio_example_check_stats = "cdio_example_check_stats"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["cdio_example_check_stats"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
cdio_example_check_stats = ["static/*"]
|
||||
+9
@@ -25,6 +25,15 @@ def register_processor():
|
||||
}
|
||||
|
||||
|
||||
@hookimpl
|
||||
def watch_list_extra_text(watch):
|
||||
"""A note under the watch in the watch list. Called for every row, so in memory only."""
|
||||
if watch.get('processor') != PROCESSOR_NAME:
|
||||
return None
|
||||
word_count = (watch.get(f'processor_state_{PROCESSOR_NAME}') or {}).get('word_count')
|
||||
return f"Found: {word_count} words" if word_count is not None else None
|
||||
|
||||
|
||||
@hookimpl
|
||||
def get_api_watch_processor_state(watch, processor_name, context):
|
||||
"""processor_state_word_counter in GET /api/v1/watch/<uuid> and GET /api/v1/watch, see api.yaml"""
|
||||
|
||||
@@ -627,6 +627,13 @@ def _jinja2_filter_format_duration(seconds):
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@app.template_global('watch_list_extra_text')
|
||||
def _watch_list_extra_text(watch):
|
||||
from changedetectionio.pluggy_interface import collect_watch_list_extra_text
|
||||
|
||||
return collect_watch_list_extra_text(watch)
|
||||
|
||||
|
||||
@app.template_filter('fetcher_status_icons')
|
||||
def _jinja2_filter_fetcher_status_icons(fetcher_name):
|
||||
"""Get status icon HTML for a given fetcher.
|
||||
|
||||
@@ -60,14 +60,20 @@ class ChangeDetectionSpec:
|
||||
|
||||
@hookspec
|
||||
def fetcher_status_icon(fetcher_name):
|
||||
"""Return status icon HTML attributes for a content fetcher.
|
||||
"""Return the status icon shown in the watch list for a content fetcher.
|
||||
|
||||
Args:
|
||||
fetcher_name: The name of the fetcher (e.g., 'html_webdriver', 'html_js_zyte')
|
||||
|
||||
Returns:
|
||||
str: HTML string containing <img> tags or other status icon elements
|
||||
Empty string if no custom status icon is needed
|
||||
dict or None: None if fetcher_name isn't yours, otherwise
|
||||
{
|
||||
'filename': 'my-icon.svg', # served from the static 'group' below
|
||||
'alt': 'Alt text',
|
||||
'title': 'Tooltip text',
|
||||
'group': 'plugin', # optional, 'plugin' = your plugin_static_path() directory
|
||||
'style': 'height: 1em;', # optional inline CSS
|
||||
}
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -265,6 +271,21 @@ class ChangeDetectionSpec:
|
||||
"""
|
||||
pass
|
||||
|
||||
@hookspec
|
||||
def watch_list_extra_text(watch):
|
||||
"""Short plain-text notes shown under the watch in the watch list (class extra-watch-text).
|
||||
|
||||
This runs for every watch on every watch list render, so stay in memory: no disk,
|
||||
network or other slow work.
|
||||
|
||||
Args:
|
||||
watch: The Watch model instance
|
||||
|
||||
Returns:
|
||||
str, a list of str, or None. Escaped when rendered, so HTML shows as text.
|
||||
"""
|
||||
pass
|
||||
|
||||
@hookspec(firstresult=True)
|
||||
def get_api_watch_processor_state(watch, processor_name, context):
|
||||
"""Return the current processor state of a watch for the REST API.
|
||||
@@ -557,6 +578,19 @@ def register_builtin_api_plugins():
|
||||
# 'builtin_' names are hidden from the active plugins list, see get_active_plugins()
|
||||
plugin_manager.register(api_state, 'builtin_restock_api_state')
|
||||
|
||||
def collect_watch_list_extra_text(watch):
|
||||
"""All plugins' notes for this watch in the watch list, as a list of strings."""
|
||||
notes = []
|
||||
try:
|
||||
for result in reversed(plugin_manager.hook.watch_list_extra_text(watch=watch)):
|
||||
if not result:
|
||||
continue
|
||||
notes.extend([result] if isinstance(result, str) else [str(r) for r in result if r])
|
||||
except Exception as e:
|
||||
logger.error(f"watch_list_extra_text failed: {e}")
|
||||
return notes
|
||||
|
||||
|
||||
# Helper function to collect UI stats extras from all plugins
|
||||
def collect_ui_edit_stats_extras(watch):
|
||||
"""Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
|
||||
|
||||
@@ -38,7 +38,7 @@ hookimpl = pluggy.HookimplMarker("changedetectionio")
|
||||
|
||||
# Version of this public API, not of changedetection.io. Below 1.0 it can still change between
|
||||
# minor versions; from 1.0 on, breaking changes mean a major version bump.
|
||||
PLUGIN_API_VERSION = "0.2.0"
|
||||
PLUGIN_API_VERSION = "0.3.0"
|
||||
|
||||
|
||||
async def await_me_maybe(value):
|
||||
@@ -67,13 +67,19 @@ async def gather_hook_results(results):
|
||||
# difference_detection_processor - base class of a processor plugin's perform_site_check
|
||||
# get_resolved_processor_config - (config, source) for a watch's processor, tag override wins
|
||||
# html_to_text - HTML to plain text, the same conversion the built-in processors use
|
||||
# Fetcher - base class of a register_content_fetcher fetcher
|
||||
# load_plugin_settings - (datastore_path, plugin_id) -> the dict saved from your plugin_settings_tab
|
||||
if TYPE_CHECKING:
|
||||
# Never runs, only tells linters/IDEs/type checkers what the lazy exports below are
|
||||
from changedetectionio.content_fetchers.base import Fetcher
|
||||
from changedetectionio.html_tools import html_to_text
|
||||
from changedetectionio.pluggy_interface import load_plugin_settings
|
||||
from changedetectionio.processors import get_resolved_processor_config
|
||||
from changedetectionio.processors.base import difference_detection_processor
|
||||
|
||||
_LAZY_EXPORTS = {
|
||||
'Fetcher': 'changedetectionio.content_fetchers.base',
|
||||
'load_plugin_settings': 'changedetectionio.pluggy_interface',
|
||||
'difference_detection_processor': 'changedetectionio.processors.base',
|
||||
'get_resolved_processor_config': 'changedetectionio.processors',
|
||||
'html_to_text': 'changedetectionio.html_tools',
|
||||
@@ -90,6 +96,7 @@ def __getattr__(name):
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Fetcher',
|
||||
'PLUGIN_API_VERSION',
|
||||
'await_me_maybe',
|
||||
'difference_detection_processor',
|
||||
@@ -97,4 +104,5 @@ __all__ = [
|
||||
'get_resolved_processor_config',
|
||||
'hookimpl',
|
||||
'html_to_text',
|
||||
'load_plugin_settings',
|
||||
]
|
||||
@@ -262,6 +262,13 @@ body.watch-selection-active #checkbox-operations {
|
||||
}
|
||||
}
|
||||
|
||||
/* notes from plugins (hook watch_list_extra_text) */
|
||||
.extra-watch-text {
|
||||
color: var(--color-text-input-description);
|
||||
font-size: 0.8em;
|
||||
margin-top: 0.15em;
|
||||
}
|
||||
|
||||
&.has-error {
|
||||
.error-text {
|
||||
display: block !important;
|
||||
|
||||
@@ -1 +1 @@
|
||||
.header{color:var(--color-text-menu-heading)}.header a{color:var(--color-text-menu-heading)}.header .bare-btn{color:var(--color-text-menu-heading)}ul#top-right-menu{list-style:none;margin-left:auto;padding:0;margin-top:0;margin-right:0;margin-bottom:0;display:grid;gap:1.1rem;grid-auto-flow:column;grid-auto-columns:max-content;align-items:center}ul#top-right-menu .toggle-button{padding:0}.current-diff-url{flex:1 1 auto;min-width:0;overflow-x:auto;overflow-y:hidden;scrollbar-width:none;white-space:nowrap;text-align:left;margin:0 .55rem;-webkit-mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent)}.current-diff-url::-webkit-scrollbar{display:none}.current-diff-url.js-fades{-webkit-mask-image:none;mask-image:none}.current-diff-url.js-fades.fade-right{-webkit-mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent)}.current-diff-url.js-fades.fade-left{-webkit-mask-image:linear-gradient(to right, transparent, #000 2.5em);mask-image:linear-gradient(to right, transparent, #000 2.5em)}.current-diff-url.js-fades.fade-left.fade-right{-webkit-mask-image:linear-gradient(to right, transparent, #000 2.5em, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, transparent, #000 2.5em, #000 calc(100% - 2.5em), transparent)}.current-diff-url span{overflow:visible;white-space:nowrap}.pure-menu-horizontal{padding:.55rem;display:flex;justify-content:space-between;align-items:center}.pure-menu-horizontal svg{height:1.3rem}.fi{height:1.3rem;cursor:pointer}#pure-menu-horizontal-spinner{height:2px;background:linear-gradient(-75deg, #ff6000, #ff8f00, #ffdd00, #ed0000);background-size:400% 400%;animation:gradient 200s ease infinite;opacity:.8;position:fixed;bottom:0;left:0;width:100%;z-index:100;pointer-events:none}.status-pill{display:inline-flex;align-items:center;gap:8px;height:30px;padding:0 12px;border-radius:var(--common-round-border);border:1px solid hsla(0,0%,100%,.25);background:hsla(0,0%,100%,.1);color:var(--color-text-menu-heading);font-size:.78rem;font-weight:600;white-space:nowrap;text-decoration:none;font-family:inherit;cursor:pointer}.status-pill:hover{background:hsla(0,0%,100%,.18)}.status-pill .live-dot{width:8px;height:8px;border-radius:50%;background:#42dd53;box-shadow:0 0 0 3px rgba(66,221,83,.3);animation:status-pill-pulse 2s infinite}.status-pill.paused .live-dot{background:#e83d3d;box-shadow:0 0 0 3px rgba(232,61,61,.3);animation:none;width:12px;height:12px}.status-pill .action-icon{width:16px;height:16px}.status-pill.muted{opacity:.8}.status-pill.muted .action-icon{color:#e8a33d}@keyframes status-pill-pulse{0%,100%{opacity:1}50%{opacity:.4}}.menu-pop-wrap{position:relative}.icon-btn{display:inline-flex;align-items:center;justify-content:center;width:36px;height:36px;border:0;border-radius:var(--common-round-border);background:rgba(0,0,0,0);color:var(--color-text-menu-heading);cursor:pointer}.icon-btn svg{width:18px;height:18px;fill:none;stroke:currentColor}.icon-btn:hover{background:hsla(0,0%,100%,.14)}.menu-pop{display:none;position:absolute;top:calc(100% + 8px);right:0;min-width:220px;padding:6px;border:1px solid var(--color-border-table-cell);border-radius:12px;background:var(--color-background);box-shadow:0 18px 50px rgba(0,0,0,.18);z-index:80}.menu-pop.open{display:block}.menu-pop .mi{display:flex;align-items:center;gap:10px;width:100%;padding:9px 10px;border:0;border-radius:8px;background:rgba(0,0,0,0);color:var(--color-text);font-size:.85rem;text-align:left;text-decoration:none;white-space:nowrap;cursor:pointer}.menu-pop .mi:hover{background:var(--color-background-menu-link-hover)}.menu-pop .mi .ico{display:inline-flex;color:var(--color-text-input-description)}.menu-pop .mi .ico svg{width:16px;height:16px;fill:none;stroke:currentColor}.menu-pop .mi .right{margin-left:auto;font-size:.75rem;color:var(--color-text-input-description)}.top-menu-list .action-label{color:var(--color-text-menu-heading)}@media only screen and (max-width: 768px){.top-menu-list .action-label{display:none}}#inline-menu-extras-group{list-style:none;margin:0;padding:0;display:grid;gap:.55rem;grid-auto-flow:column;grid-auto-columns:max-content;align-items:center}@media only screen and (max-width: 980px){#nav-menu #menu-settings span{display:none}}:root{--body-main-text-size: 0.9rem;--color-white: #fff;--color-grey-50: #111;--color-grey-100: #262626;--color-grey-200: #333;--color-grey-300: #444;--color-grey-325: #555;--color-grey-350: #565d64;--color-grey-400: #666;--color-grey-500: #777;--color-grey-600: #999;--color-grey-700: #cbcbcb;--color-grey-750: #ddd;--color-grey-800: #e0e0e0;--color-grey-850: #eee;--color-grey-900: #f2f2f2;--color-black: #000;--color-dark-red: #a00;--color-light-red: #dd0000;--color-background-page: var(--color-grey-100);--color-background-gradient-first: #5ad8f7;--color-background-gradient-second: #2f5Line truncated
|
||||
.header{color:var(--color-text-menu-heading)}.header a{color:var(--color-text-menu-heading)}.header .bare-btn{color:var(--color-text-menu-heading)}ul#top-right-menu{list-style:none;margin-left:auto;padding:0;margin-top:0;margin-right:0;margin-bottom:0;display:grid;gap:1.1rem;grid-auto-flow:column;grid-auto-columns:max-content;align-items:center}ul#top-right-menu .toggle-button{padding:0}.current-diff-url{flex:1 1 auto;min-width:0;overflow-x:auto;overflow-y:hidden;scrollbar-width:none;white-space:nowrap;text-align:left;margin:0 .55rem;-webkit-mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent)}.current-diff-url::-webkit-scrollbar{display:none}.current-diff-url.js-fades{-webkit-mask-image:none;mask-image:none}.current-diff-url.js-fades.fade-right{-webkit-mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, #000 calc(100% - 2.5em), transparent)}.current-diff-url.js-fades.fade-left{-webkit-mask-image:linear-gradient(to right, transparent, #000 2.5em);mask-image:linear-gradient(to right, transparent, #000 2.5em)}.current-diff-url.js-fades.fade-left.fade-right{-webkit-mask-image:linear-gradient(to right, transparent, #000 2.5em, #000 calc(100% - 2.5em), transparent);mask-image:linear-gradient(to right, transparent, #000 2.5em, #000 calc(100% - 2.5em), transparent)}.current-diff-url span{overflow:visible;white-space:nowrap}.pure-menu-horizontal{padding:.55rem;display:flex;justify-content:space-between;align-items:center}.pure-menu-horizontal svg{height:1.3rem}.fi{height:1.3rem;cursor:pointer}#pure-menu-horizontal-spinner{height:2px;background:linear-gradient(-75deg, #ff6000, #ff8f00, #ffdd00, #ed0000);background-size:400% 400%;animation:gradient 200s ease infinite;opacity:.8;position:fixed;bottom:0;left:0;width:100%;z-index:100;pointer-events:none}.status-pill{display:inline-flex;align-items:center;gap:8px;height:30px;padding:0 12px;border-radius:var(--common-round-border);border:1px solid hsla(0,0%,100%,.25);background:hsla(0,0%,100%,.1);color:var(--color-text-menu-heading);font-size:.78rem;font-weight:600;white-space:nowrap;text-decoration:none;font-family:inherit;cursor:pointer}.status-pill:hover{background:hsla(0,0%,100%,.18)}.status-pill .live-dot{width:8px;height:8px;border-radius:50%;background:#42dd53;box-shadow:0 0 0 3px rgba(66,221,83,.3);animation:status-pill-pulse 2s infinite}.status-pill.paused .live-dot{background:#e83d3d;box-shadow:0 0 0 3px rgba(232,61,61,.3);animation:none;width:12px;height:12px}.status-pill .action-icon{width:16px;height:16px}.status-pill.muted{opacity:.8}.status-pill.muted .action-icon{color:#e8a33d}@keyframes status-pill-pulse{0%,100%{opacity:1}50%{opacity:.4}}.menu-pop-wrap{position:relative}.icon-btn{display:inline-flex;align-items:center;justify-content:center;width:36px;height:36px;border:0;border-radius:var(--common-round-border);background:rgba(0,0,0,0);color:var(--color-text-menu-heading);cursor:pointer}.icon-btn svg{width:18px;height:18px;fill:none;stroke:currentColor}.icon-btn:hover{background:hsla(0,0%,100%,.14)}.menu-pop{display:none;position:absolute;top:calc(100% + 8px);right:0;min-width:220px;padding:6px;border:1px solid var(--color-border-table-cell);border-radius:12px;background:var(--color-background);box-shadow:0 18px 50px rgba(0,0,0,.18);z-index:80}.menu-pop.open{display:block}.menu-pop .mi{display:flex;align-items:center;gap:10px;width:100%;padding:9px 10px;border:0;border-radius:8px;background:rgba(0,0,0,0);color:var(--color-text);font-size:.85rem;text-align:left;text-decoration:none;white-space:nowrap;cursor:pointer}.menu-pop .mi:hover{background:var(--color-background-menu-link-hover)}.menu-pop .mi .ico{display:inline-flex;color:var(--color-text-input-description)}.menu-pop .mi .ico svg{width:16px;height:16px;fill:none;stroke:currentColor}.menu-pop .mi .right{margin-left:auto;font-size:.75rem;color:var(--color-text-input-description)}.top-menu-list .action-label{color:var(--color-text-menu-heading)}@media only screen and (max-width: 768px){.top-menu-list .action-label{display:none}}#inline-menu-extras-group{list-style:none;margin:0;padding:0;display:grid;gap:.55rem;grid-auto-flow:column;grid-auto-columns:max-content;align-items:center}@media only screen and (max-width: 980px){#nav-menu #menu-settings span{display:none}}:root{--body-main-text-size: 0.9rem;--color-white: #fff;--color-grey-50: #111;--color-grey-100: #262626;--color-grey-200: #333;--color-grey-300: #444;--color-grey-325: #555;--color-grey-350: #565d64;--color-grey-400: #666;--color-grey-500: #777;--color-grey-600: #999;--color-grey-700: #cbcbcb;--color-grey-750: #ddd;--color-grey-800: #e0e0e0;--color-grey-850: #eee;--color-grey-900: #f2f2f2;--color-black: #000;--color-dark-red: #a00;--color-light-red: #dd0000;--color-background-page: var(--color-grey-100);--color-background-gradient-first: #5ad8f7;--color-background-gradient-second: #2f5Line truncated
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The example_browser_fetcher example plugin (example_plugins/fetcher/example_browser_fetcher): register_content_fetcher
|
||||
adds a fetcher that gets pages through its own API (here the test server), plugin_settings_tab gives it a
|
||||
settings tab for the API URL and key, and fetcher_status_icon + plugin_static_path give it an icon."""
|
||||
import json
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from flask import url_for
|
||||
|
||||
from .util import wait_for_all_checks, delete_all_watches, import_example_plugin, clear_plugin_discovery_caches
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_browser_fetcher_plugin(live_server):
|
||||
from changedetectionio import content_fetchers
|
||||
from changedetectionio.pluggy_interface import plugin_manager
|
||||
|
||||
plugin = import_example_plugin('fetcher/example_browser_fetcher', 'cdio_example_browser_fetcher')
|
||||
plugin_manager.register(plugin, 'cdio_example_browser_fetcher')
|
||||
# What changedetection.io does at startup: collect plugin fetchers, inject the datastore
|
||||
content_fetchers._plugin_fetchers.update(content_fetchers.get_plugin_fetchers())
|
||||
plugin.datastore = live_server.app.config['DATASTORE']
|
||||
clear_plugin_discovery_caches()
|
||||
yield plugin
|
||||
plugin.datastore = None
|
||||
content_fetchers._plugin_fetchers.pop(plugin.FETCHER_NAME, None)
|
||||
delattr(content_fetchers, plugin.FETCHER_NAME)
|
||||
plugin_manager.unregister(plugin)
|
||||
clear_plugin_discovery_caches()
|
||||
|
||||
|
||||
def test_example_browser_fetcher_plugin(client, live_server, measure_memory_usage, datastore_path, example_browser_fetcher_plugin):
|
||||
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
|
||||
headers = {'content-type': 'application/json', 'x-api-key': api_key}
|
||||
|
||||
# The settings tab, and the plugin listed under Settings > Info
|
||||
res = client.get(url_for("settings.settings_page"))
|
||||
assert b'Example browser fetcher' in res.data
|
||||
assert b'name="example_browser_fetcher_url"' in res.data
|
||||
assert b'<li><strong>cdio_example_browser_fetcher</strong> - Example plugin - an extra browser fetcher' in res.data
|
||||
|
||||
# Save the settings through the settings page, the "API" echoes the request headers back
|
||||
res = client.post(url_for("settings.settings_page"), data={
|
||||
"requests-time_between_check-minutes": 180,
|
||||
"application-fetch_backend": "html_requests",
|
||||
"example_browser_fetcher_url": url_for('test_headers', _external=True) + '?url={url}',
|
||||
"example_browser_fetcher_key": "secret-key-123",
|
||||
}, follow_redirects=True)
|
||||
assert b"Settings updated." in res.data
|
||||
with open(os.path.join(datastore_path, 'cdio_example_browser_fetcher.json')) as f:
|
||||
assert json.load(f)['example_browser_fetcher_key'] == 'secret-key-123'
|
||||
|
||||
watch_url = 'https://example.com/product?id=1'
|
||||
res = client.post(url_for("createwatch"), headers=headers,
|
||||
data=json.dumps({"url": watch_url, "fetch_backend": "html_example_browser_fetcher"}))
|
||||
assert res.status_code == 201, res.data
|
||||
uuid = res.json.get('uuid')
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Installed fetchers are accepted in 'fetch_backend', unknown ones still aren't
|
||||
res = client.post(url_for("createwatch"), headers=headers,
|
||||
data=json.dumps({"url": watch_url, "fetch_backend": "html_no_such_fetcher"}))
|
||||
assert res.status_code == 400
|
||||
|
||||
# Fetched through the API, which got the key in the X-Api-Key header
|
||||
res = client.get(url_for("watchsinglehistory", uuid=uuid, timestamp='latest'), headers=headers)
|
||||
assert res.status_code == 200, "The check should have fetched through the plugin's API"
|
||||
assert b'X-Api-Key:secret-key-123' in res.data
|
||||
|
||||
# The fetcher's screenshot (random noise) is saved and served as a valid PNG
|
||||
import struct
|
||||
res = client.get(url_for('static_content', group='screenshot', filename=uuid))
|
||||
assert res.status_code == 200
|
||||
assert res.data.startswith(b'\x89PNG\r\n\x1a\n') and res.data[12:16] == b'IHDR'
|
||||
assert struct.unpack('>II', res.data[16:24]) == (240, 160)
|
||||
|
||||
# The status icon in the watch list, served from the plugin's static directory
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'/static/plugin/cdio_example_browser_fetcher.svg' in res.data
|
||||
assert client.get('/static/plugin/cdio_example_browser_fetcher.svg').status_code == 200
|
||||
|
||||
# The watch URL is passed to the API URL-encoded
|
||||
from changedetectionio.pluggy_interface import save_plugin_settings
|
||||
save_plugin_settings(datastore_path, 'cdio_example_browser_fetcher',
|
||||
{"example_browser_fetcher_url": url_for('test_return_query', _external=True) + '?url={url}'})
|
||||
client.get(url_for("watch", uuid=uuid, recheck=1), headers=headers)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("watchsinglehistory", uuid=uuid, timestamp='latest'), headers=headers)
|
||||
assert quote(watch_url, safe='').encode() in res.data
|
||||
|
||||
delete_all_watches(client)
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The check_stats example plugin (example_plugins/lifecycle/check_stats): update_handler_alter and
|
||||
update_finalize time a check, watch_list_extra_text and ui_edit_stats_extras show it, and
|
||||
get_html_head_extras + plugin_static_path serve its CSS."""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from flask import url_for
|
||||
|
||||
from .util import wait_for_all_checks, delete_all_watches, import_example_plugin
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def check_stats_plugin():
|
||||
from changedetectionio.pluggy_interface import plugin_manager
|
||||
|
||||
plugin = import_example_plugin('lifecycle/check_stats', 'cdio_example_check_stats')
|
||||
plugin.STATS.clear()
|
||||
plugin_manager.register(plugin, 'cdio_example_check_stats')
|
||||
yield plugin
|
||||
plugin_manager.unregister(plugin)
|
||||
|
||||
|
||||
def test_example_check_stats_plugin(client, live_server, measure_memory_usage, datastore_path, check_stats_plugin):
|
||||
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
||||
f.write("<html><body>Hello</body></html>")
|
||||
|
||||
client.post(url_for("ui.ui_views.form_quick_watch_add"),
|
||||
data={"url": url_for('test_endpoint', _external=True), "tags": ''}, follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
datastore = client.application.config.get('DATASTORE')
|
||||
uuid = next(iter(datastore.data['watching']))
|
||||
stats = check_stats_plugin.STATS.get(uuid)
|
||||
assert stats and stats['ok'] and stats['count'] >= 1, f"The check should have been timed, got {stats}"
|
||||
|
||||
# Watch list note
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'<div class="extra-watch-text">Last check took' in res.data
|
||||
|
||||
# Edit page Stats tab
|
||||
res = client.get(url_for("ui.ui_edit.edit_page", uuid=uuid))
|
||||
assert b'cdio-example-check-stats' in res.data
|
||||
assert b'Checks timed since restart' in res.data
|
||||
|
||||
# CSS linked in <head> and served from the plugin's static directory
|
||||
assert b'/static/plugin/cdio_example_check_stats.css' in res.data
|
||||
res = client.get('/static/plugin/cdio_example_check_stats.css')
|
||||
assert res.status_code == 200
|
||||
assert b'.cdio-example-check-stats' in res.data
|
||||
|
||||
# Listed under Settings > Info as an active plugin, with its docstring's first line
|
||||
res = client.get(url_for("settings.settings_page"))
|
||||
assert b'<li><strong>cdio_example_check_stats</strong> - Example plugin - times every check' in res.data
|
||||
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_watch_list_extra_text_is_escaped(client, live_server, measure_memory_usage, datastore_path):
|
||||
"""watch_list_extra_text is plain text, a plugin can't inject HTML into the watch list."""
|
||||
from changedetectionio.pluggy_interface import plugin_manager, hookimpl
|
||||
|
||||
class Notes:
|
||||
@hookimpl
|
||||
def watch_list_extra_text(self, watch):
|
||||
return ['<b>bold?</b>', 'second note']
|
||||
|
||||
plugin_manager.register(Notes(), 'test_watch_list_notes')
|
||||
try:
|
||||
client.post(url_for("ui.ui_views.form_quick_watch_add"),
|
||||
data={"url": url_for('test_endpoint', _external=True), "tags": '', "paused": "y"}, follow_redirects=True)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'<div class="extra-watch-text"><b>bold?</b></div>' in res.data
|
||||
assert b'<div class="extra-watch-text">second note</div>' in res.data
|
||||
assert b'<b>bold?</b>' not in res.data
|
||||
finally:
|
||||
plugin_manager.unregister(name='test_watch_list_notes')
|
||||
delete_all_watches(client)
|
||||
@@ -8,39 +8,22 @@ import os
|
||||
import pytest
|
||||
from flask import url_for
|
||||
|
||||
from .util import wait_for_all_checks, delete_all_watches
|
||||
|
||||
|
||||
def _clear_processor_and_spec_caches():
|
||||
# Processor discovery and the merged API spec are cached at first use, a plugin registered
|
||||
# afterwards (like here) is only seen once these are cleared
|
||||
from changedetectionio import processors, api
|
||||
for f in (processors.find_processors, processors.get_plugin_processor_metadata, processors._available_processors_cached,
|
||||
api.build_merged_spec_dict, api.get_openapi_spec, api.get_openapi_schema_dict,
|
||||
api._resolve_schema_properties, api.get_watch_schema_properties, api.get_tag_schema_properties):
|
||||
f.cache_clear()
|
||||
from .util import wait_for_all_checks, delete_all_watches, import_example_plugin, clear_plugin_discovery_caches
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wordcounter_plugin():
|
||||
"""What `pip install .` + the entry point would do: import cdio_example_wordcounter as its own
|
||||
top-level package and register it."""
|
||||
import importlib
|
||||
import sys
|
||||
from changedetectionio.pluggy_interface import plugin_manager
|
||||
|
||||
example_dir = os.path.join(os.path.dirname(__file__), '..', 'example_plugins', 'processor', 'wordcounter')
|
||||
sys.path.insert(0, os.path.abspath(example_dir))
|
||||
try:
|
||||
wordcounter = importlib.import_module('cdio_example_wordcounter')
|
||||
finally:
|
||||
sys.path.pop(0)
|
||||
wordcounter = import_example_plugin('processor/wordcounter', 'cdio_example_wordcounter')
|
||||
|
||||
plugin_manager.register(wordcounter, 'cdio_example_wordcounter')
|
||||
_clear_processor_and_spec_caches()
|
||||
clear_plugin_discovery_caches()
|
||||
yield wordcounter
|
||||
plugin_manager.unregister(wordcounter)
|
||||
_clear_processor_and_spec_caches()
|
||||
clear_plugin_discovery_caches()
|
||||
|
||||
|
||||
def test_example_wordcounter_processor_plugin(client, live_server, measure_memory_usage, datastore_path, wordcounter_plugin):
|
||||
@@ -71,11 +54,19 @@ def test_example_wordcounter_processor_plugin(client, live_server, measure_memor
|
||||
assert res.json.get('processor_config_word_counter_source') == 'watch'
|
||||
assert 'processor_state_restock_diff' not in res.json
|
||||
|
||||
# Listed under Settings > Info as an active plugin, with its docstring's first line
|
||||
res = client.get(url_for("settings.settings_page"))
|
||||
assert b'<li><strong>cdio_example_wordcounter</strong> - Example processor plugin - counts the words' in res.data
|
||||
|
||||
res = client.get(url_for("createwatch"), headers=headers)
|
||||
assert res.json[uuid].get('processor') == 'word_counter'
|
||||
assert res.json[uuid].get('processor_state_word_counter') == {'word_count': 5}
|
||||
assert 'processor_config_word_counter' not in res.json[uuid]
|
||||
|
||||
# The note under the watch in the watch list
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert res.data.count(b'<div class="extra-watch-text">Found: 5 words</div>') == 1
|
||||
|
||||
# The snapshot is what the processor returned
|
||||
res = client.get(url_for("watchsinglehistory", uuid=uuid, timestamp='latest'), headers=headers)
|
||||
assert res.data == b'Words: 5'
|
||||
|
||||
@@ -55,6 +55,10 @@ def test_plugin_api_exports():
|
||||
assert plugin_api.difference_detection_processor is difference_detection_processor
|
||||
assert plugin_api.get_resolved_processor_config is get_resolved_processor_config
|
||||
assert plugin_api.html_to_text is html_to_text
|
||||
from changedetectionio.content_fetchers.base import Fetcher
|
||||
from changedetectionio.pluggy_interface import load_plugin_settings
|
||||
assert plugin_api.Fetcher is Fetcher
|
||||
assert plugin_api.load_plugin_settings is load_plugin_settings
|
||||
assert set(plugin_api.__all__) >= {'difference_detection_processor', 'get_resolved_processor_config', 'html_to_text'}
|
||||
with pytest.raises(AttributeError):
|
||||
plugin_api.no_such_export
|
||||
@@ -68,7 +72,8 @@ def test_plugin_api_import_stays_cheap():
|
||||
import sys
|
||||
import changedetectionio
|
||||
code = ("import sys, changedetectionio.plugin_api; "
|
||||
"print(','.join(m for m in ('changedetectionio.processors.base', 'changedetectionio.html_tools') if m in sys.modules))")
|
||||
"print(','.join(m for m in ('changedetectionio.processors.base', 'changedetectionio.html_tools', "
|
||||
"'changedetectionio.content_fetchers.base') if m in sys.modules))")
|
||||
# Run from the directory containing the package, CI runs pytest from inside changedetectionio/
|
||||
# where a fresh interpreter can't import it
|
||||
package_parent = os.path.dirname(os.path.dirname(os.path.abspath(changedetectionio.__file__)))
|
||||
|
||||
@@ -13,6 +13,29 @@ import threading
|
||||
_test_endpoint_content_lock = threading.Lock()
|
||||
_test_endpoint_content = {}
|
||||
|
||||
def import_example_plugin(example_dir, package):
|
||||
"""Import an example_plugins/<example_dir> package as its own top-level package, the way
|
||||
`pip install .` makes it importable, e.g. import_example_plugin('processor/wordcounter', 'cdio_example_wordcounter')"""
|
||||
import importlib
|
||||
import sys
|
||||
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'example_plugins', example_dir))
|
||||
sys.path.insert(0, path)
|
||||
try:
|
||||
return importlib.import_module(package)
|
||||
finally:
|
||||
sys.path.remove(path)
|
||||
|
||||
|
||||
def clear_plugin_discovery_caches():
|
||||
"""Processor discovery and the merged API spec are cached at first use, a plugin registered
|
||||
afterwards (like in a test) is only seen once these are cleared."""
|
||||
from changedetectionio import processors, api
|
||||
for f in (processors.find_processors, processors.get_plugin_processor_metadata, processors._available_processors_cached,
|
||||
api.build_merged_spec_dict, api.get_openapi_spec, api.get_openapi_schema_dict,
|
||||
api._resolve_schema_properties, api.get_watch_schema_properties, api.get_tag_schema_properties):
|
||||
f.cache_clear()
|
||||
|
||||
|
||||
def write_test_file_and_sync(filepath, content, mode='w'):
|
||||
"""
|
||||
Write test data to file and ensure it's synced to disk.
|
||||
|
||||
@@ -78,6 +78,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
|
||||
|
||||
while not app.config.exit.is_set():
|
||||
update_handler = None
|
||||
finalize_handler = None # The handler update_finalize gets, see below
|
||||
watch = None
|
||||
processing_exception = None # Reset at start of each iteration to prevent state bleeding
|
||||
|
||||
@@ -183,6 +184,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
|
||||
|
||||
# Allow plugins to modify/wrap the update_handler
|
||||
update_handler = apply_update_handler_alter(update_handler, watch, datastore)
|
||||
# Kept for the update_finalize hook: on success update_handler is deleted before the
|
||||
# finally block below, so capturing it there would hand plugins None
|
||||
finalize_handler = update_handler
|
||||
|
||||
set_watch_minitext_status(watch, "Fetching...")
|
||||
|
||||
@@ -683,7 +687,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
|
||||
if uuid:
|
||||
# Capture references for plugin finalize hook BEFORE cleanup
|
||||
# (cleanup may delete these variables, but plugins need the original references)
|
||||
finalize_handler = update_handler # Capture now, before cleanup deletes it
|
||||
finalize_watch = watch # Capture now, before any modifications
|
||||
|
||||
# Call quit() as backup (Puppeteer/Playwright have internal cleanup, but this acts as safety net)
|
||||
|
||||
Reference in new issue
Block a user