Files
changedetection.io/changedetectionio/pluggy_interface.py
T
dgtlmoonandClaude Opus 5.5 c525d2aa56 MCP plugins - their own permission rows (register_mcp_permission_rows), a tool 'area', and open_world (#4679)
Plugin tools always counted as the Monitors row, so a tool that isn't about monitors (e.g. a sign-up tool on a
read-only demo) had to pretend to be read-only. Now:
- register_mcp_permission_rows hook: a plugin adds a row to the None/Read/Write grid (tokens and the OAuth
  consent screen), None for every token until the owner allows it, offering only the levels its tools need.
- a plugin tool's 'area' puts it in a core row or a plugin row; 'open_world': True sets openWorldHint.
- refusals name the row by its grid label; the token list shows plugin rows, and the Browser row's label,
  which was missing.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 23:16:35 +01:00

1285 lines
55 KiB
Python

import pluggy
import os
import importlib
import inspect
import sys
import threading
import time
from functools import wraps
from loguru import logger
# Global plugin namespace for changedetection.io
PLUGIN_NAMESPACE = "changedetectionio"
# Conditions plugins used to have their own PluginManager under this name. Their hooks now live on
# the main manager, but implementations marked with the old namespace are still picked up (see
# ChangeDetectionPluginManager.parse_hookimpl_opts) and the old entry-point group is still loaded.
LEGACY_CONDITIONS_NAMESPACE = "changedetectionio_conditions"
hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
# The marker lives in plugin_api (the public plugin module), see the comment there
from changedetectionio.plugin_api import hookimpl
# A plugin that raises is logged and counted, and the hook carries on with the other plugins.
# After this many failures in a row the plugin is reported as "errored" (it is still called).
PLUGIN_CONSECUTIVE_FAILURES_ERRORED = int(os.getenv('PLUGIN_CONSECUTIVE_FAILURES_ERRORED', 5))
# Upper bound for an async hook implementation, so a hung plugin can't stall a check forever.
PLUGIN_ASYNC_HOOK_TIMEOUT = float(os.getenv('PLUGIN_ASYNC_HOOK_TIMEOUT', 120))
# Hooks whose exceptions must reach the caller (after being logged and counted), for example a
# future fetch hook that raises EmptyReply on purpose. Every other hook swallows plugin exceptions.
HOOKS_THAT_PROPAGATE_EXCEPTIONS = set()
class ChangeDetectionSpec:
"""Hook specifications for extending changedetection.io functionality."""
@hookspec
def ui_edit_stats_extras(watch):
"""Return HTML content to add to the stats tab in the edit view.
Args:
watch: The watch object being edited
Returns:
str: HTML content to be inserted in the stats tab
"""
pass
@hookspec
def register_content_fetcher(self):
"""Return a tuple of (fetcher_name, fetcher_class) for content fetcher plugins.
The fetcher_name should start with 'html_' and the fetcher_class
should inherit from changedetectionio.content_fetchers.base.Fetcher
Returns:
tuple: (str: fetcher_name, class: fetcher_class)
"""
pass
@hookspec
def select_fetch_backend(watch, datastore, backend_name, failed_fetch, tried):
"""Offer a different content fetcher for this one check, without changing the watch's settings.
Plugin API 0.6.0+.
Asked before fetching (failed_fetch=None, swap the fetcher before it runs), and again each time a fetch
raised (retry straight away with another one). Every plugin can offer, core tries the offers in priority
order, lowest first, skipping fetchers already tried in this check, up to FETCH_FALLBACK_MAX_ATTEMPTS
retries (default 3). So fallbacks chain: a preferred scraping service at priority 1, another at 2 when
the first fails or is out of quota. If every attempt fails, the last error is what the watch shows.
Don't offer what can't work, for example over your own daily quota, so the next fetcher gets the
attempt. See example_plugins/fetcher/fallback_fetcher.
Args:
watch: The watch being checked
datastore: The application datastore
backend_name: The fetcher that would be used (or just failed, when failed_fetch is set),
e.g. 'html_requests'
failed_fetch: None before the fetch, the exception after a failed one. A Non200ErrorCodeReceived
has the site's answer in .status_code
tried: Fetcher names already tried in this check, in order
Returns:
None, a registered fetcher name (starting with 'html_'), or {'backend': name, 'priority': int}.
A plain name has priority 0. Unknown names, fetchers already tried, and fetchers that can't
fetch a PDF watch are skipped.
"""
pass
@hookspec(firstresult=True)
def recheck_seconds_override(watch, datastore, seconds):
"""Override how often a watch is rechecked, the last word after system settings -> group -> watch.
Plugin API 0.6.0+.
For a plugin that has taken a watch "under its control", for example one that moved it to a paid
fetcher after repeated 403s and wants it checked every 12 hours instead of every 30 minutes.
Explain it to the user with watch_list_extra_text.
Called by the scheduler for every watch on every pass (about once a second), so stay in memory:
no disk, network or other slow work. The first plugin to return a number wins.
Args:
watch: The watch
datastore: The application datastore
seconds: The recheck time it would have without plugins
Returns:
int or None: Seconds between checks for this watch, or None to leave it.
"""
pass
@hookspec
def fetcher_status_icon(fetcher_name):
"""Return the status icon shown in the watch list for a content fetcher.
Args:
fetcher_name: The name of the fetcher (e.g., 'html_webdriver', 'html_js_zyte')
Returns:
dict or None: None if fetcher_name isn't yours, otherwise
{
'filename': 'my-icon.svg', # served from the static 'group' below
'alt': 'Alt text',
'title': 'Tooltip text',
'group': 'plugin', # optional, 'plugin' = your plugin_static_path() directory
'style': 'height: 1em;', # optional inline CSS
}
"""
pass
@hookspec
def plugin_static_path(self):
"""Return the path to the plugin's static files directory.
Returns:
str: Absolute path to the plugin's static directory, or None if no static files
"""
pass
@hookspec(firstresult=True)
def get_itemprop_availability_override(self, content, fetcher_name, fetcher_instance, url, llm_intent, text):
"""Provide custom implementation of get_itemprop_availability for a specific fetcher.
This hook allows plugins to provide their own product availability detection
when their fetcher is being used. This is called as a fallback when the built-in
method doesn't find good data.
The first implementation to return a non-None result wins and the rest are skipped,
so return None (not an empty dict) when you have nothing. The built-in LLM restock
plugin (litellm) is registered trylast, so your own plugin, for example one that
POSTs to an LLM with plain `requests`, always gets the first go and the built-in
one is never billed when yours already answered. See plugins/llm_restock.py for
the prompt, cache and reply-parsing helpers you can reuse without importing litellm.
Args:
content: The HTML content that was just fetched
fetcher_name: The name of the fetcher being used (e.g., 'html_js_zyte')
fetcher_instance: The fetcher instance that generated the content
url: The URL being watched/checked
llm_intent: Optional user-supplied intent string (e.g. "alert when price drops below $300")
text: The same content converted with html_to_text(). Only computed when an
implementation declares this argument, so leave it out if you don't need it.
Declare the arguments you want WITHOUT default values (`text`, not `text=None`),
pluggy silently never passes an argument that has a default.
Returns:
dict or None: Dictionary with availability data:
{
'price': float or None,
'availability': str or None, # e.g., 'in stock', 'out of stock'
'currency': str or None, # e.g., 'USD', 'EUR'
}
Or None if this plugin doesn't handle this fetcher or couldn't extract data
"""
pass
@hookspec
def plugin_settings_tab(self):
"""Return settings tab information for this plugin.
This hook allows plugins to add their own settings tab to the settings page.
Settings will be saved to a separate JSON file in the datastore directory.
Returns:
dict or None: Dictionary with settings tab information:
{
'plugin_id': str, # Unique identifier (e.g., 'zyte_fetcher')
'tab_label': str, # Display name for tab (e.g., 'Zyte Fetcher')
'form_class': Form, # WTForms Form class for the settings
'template_path': str, # Optional: path to Jinja2 template (relative to plugin)
# If not provided, a default form renderer will be used
}
Or None if this plugin doesn't provide settings
"""
pass
@hookspec
def register_processor(self):
"""Register an external processor plugin.
External packages can implement this hook to register custom processors
that will be discovered alongside built-in processors.
Returns:
dict or None: Dictionary with processor information:
{
'processor_name': str, # Machine name (e.g., 'osint_recon')
'processor_module': module, # Module containing processor.py
'processor_class': class, # The perform_site_check class
'metadata': { # Optional metadata
'name': str, # Display name
'description': str, # Description
'processor_weight': int,# Sort weight (lower = higher priority)
'list_badge_text': str, # Badge text for UI
}
}
Return None if this plugin doesn't provide a processor
"""
pass
@hookspec
def update_handler_alter(update_handler, watch, datastore):
"""Modify or wrap the update_handler before it processes a watch.
This hook is called after the update_handler (perform_site_check instance) is created
but before it calls call_browser() and run_changedetection(). Plugins can use this to:
- Wrap the handler to add logging/metrics
- Modify handler configuration
- Add custom preprocessing logic
Args:
update_handler: The perform_site_check instance that will process the watch
watch: The watch dict being processed
datastore: The application datastore
Returns:
object or None: Return a modified/wrapped handler, or None to keep the original.
If multiple plugins return handlers, they are chained in registration order.
"""
pass
@hookspec
def update_finalize(update_handler, watch, datastore, processing_exception):
"""Called after watch processing completes (success or failure).
This hook is called in the finally block after all processing is complete,
allowing plugins to perform cleanup, update metrics, or log final status.
The plugin can access update_handler.last_logging_insert_id if it was stored
during update_handler_alter, and use processing_exception to determine if
the processing succeeded or failed.
Args:
update_handler: The perform_site_check instance (may be None if creation failed)
watch: The watch dict that was processed (may be None if not loaded)
datastore: The application datastore
processing_exception: The exception from the main processing block, or None if successful.
This does NOT include cleanup exceptions - only exceptions from
the actual watch processing (fetch, diff, etc).
Returns:
None: This hook doesn't return a value
"""
pass
@hookspec
def register_sidebar_items(datastore):
"""Return links to add to the left action sidebar, after the built-in ones. Plugin API 0.5.0+.
Called on every page render inside a request (so url_for() and gettext work), keep it
in memory: no disk or network.
Return a dict, a list of dicts, or None. Each dict:
{
'label': str, # shown next to the icon, already translated
'endpoint': str, # Flask endpoint for url_for(), or give 'url' instead
'url': str, # used when there's no 'endpoint'
'icon': str, # a Feather icon name, e.g. 'cpu', https://feathericons.com
'svg': str, # or your own inline <svg class="action-icon" ...>, used instead of 'icon'.
# Trusted markup from your plugin's code only, never user input
# (it's rendered as-is). Draw with stroke="currentColor" to follow
# the theme and the active state
'title': str, # optional tooltip, defaults to the label
'active_prefix': str, # optional, highlight when request.endpoint starts with it
'weight': int, # optional, lower comes first among plugin items (default 0)
'disabled': bool, # optional, shown greyed out and not clickable (no endpoint/url needed),
# like the built-in "Browsers (coming soon)" item
'badge': str, # optional, a small label after the text, e.g. 'soon', already translated
}
Prefer 'endpoint' over a hard-coded 'url' so sub-path deployments (USE_X_SETTINGS /
X-Forwarded-Prefix) work.
Args:
datastore: The ChangeDetectionStore instance
"""
pass
@hookspec
def get_html_head_extras():
"""Return HTML to inject into the <head> of every page via base.html.
Plugins can use this to add <script>, <style>, or <link> tags that should
be present on all pages. Return a raw HTML string or None.
IMPORTANT: Always use Flask's url_for() for any src/href URLs so that
sub-path deployments (nginx reverse proxy with USE_X_SETTINGS / X-Forwarded-Prefix)
work correctly. This hook is called inside a request context so url_for() is
always available.
For small amounts of CSS/JS, return them inline — no file-serving needed::
from changedetectionio.pluggy_interface import hookimpl
@hookimpl
def get_html_head_extras(self):
return (
'<style>.my-module-banner { color: red; }</style>\\n'
'<script>console.log("my_module_content loaded");</script>'
)
For larger assets, register your own lightweight Flask routes in the plugin
module and point to them with url_for() so the sub-path prefix is handled
automatically::
from flask import url_for, Response
from changedetectionio.pluggy_interface import hookimpl
from changedetectionio.flask_app import app as _app
MY_CSS = ".my-module-example { color: red; }"
MY_JS = "console.log('my_module_content loaded');"
@_app.route('/my_module_content/css')
def my_module_content_css():
return Response(MY_CSS, mimetype='text/css',
headers={'Cache-Control': 'max-age=3600'})
@_app.route('/my_module_content/js')
def my_module_content_js():
return Response(MY_JS, mimetype='application/javascript',
headers={'Cache-Control': 'max-age=3600'})
@hookimpl
def get_html_head_extras(self):
css = url_for('my_module_content_css')
js = url_for('my_module_content_js')
return (
f'<link rel="stylesheet" href="{css}">\\n'
f'<script src="{js}" defer></script>'
)
Returns:
str or None: Raw HTML string to inject inside <head>, or None
"""
pass
@hookspec
def register_blueprints(datastore):
"""Return Flask blueprints to mount in the app, called once at startup. Plugin API 0.4.0+.
Return a Blueprint, a dict, or a list of either. The dict form sets the options:
{'blueprint': bp, 'url_prefix': '/my-plugin', 'login_required': True}
url_prefix defaults to the one the Blueprint was created with. Routes are protected
by the same optional password login as the rest of the UI unless you pass
'login_required': False (for example an endpoint that does its own API-key auth),
in which case securing them is up to you. CSRF protection applies to POST routes as
usual, exempt a blueprint with `from changedetectionio.flask_app import csrf;
csrf.exempt(bp)` only if it authenticates some other way.
Prefix your blueprint name and URLs with your plugin name: a blueprint whose name is
already registered is skipped with an error in the log.
Args:
datastore: The ChangeDetectionStore instance
Returns:
Blueprint, dict, a list of either, or None
"""
pass
@hookspec
def watch_list_extra_text(watch):
"""Short plain-text notes shown under the watch in the watch list (class extra-watch-text).
This runs for every watch on every watch list render, so stay in memory: no disk,
network or other slow work.
Args:
watch: The Watch model instance
Returns:
str, a list of str, or None. Escaped when rendered, so HTML shows as text.
"""
pass
@hookspec(firstresult=True)
def get_api_watch_processor_state(watch, processor_name, context):
"""Return the current processor state of a watch for the REST API.
Core calls this with the watch's own processor name and adds your dict to the response
as `processor_state_{processor_name}` on:
- GET /api/v1/watch/<uuid> (context='get')
- GET /api/v1/watch (context='list', called once per watch in the listing)
You never pick the key, core does, so a plugin can't overwrite other watch fields.
Return None when processor_name isn't yours, the first non-None result wins.
Document the shape as components.schemas.processor_state_{processor_name} in your
processor's api.yaml, it is merged into the Watch schema as read-only.
With context='list' this runs for every watch on every listing, so stay in memory:
no disk, network or other slow work.
@IMPORTANT-TODO: where a processor keeps this state on the watch has no convention yet,
see store update_watch().
Example (the built-in restock_diff processor):
@hookimpl
def get_api_watch_processor_state(watch, processor_name, context):
if processor_name != 'restock_diff':
return None
restock = watch.get('restock') or {}
return {'in_stock': restock.get('in_stock'), 'price': restock.get('price')}
Args:
watch: The Watch model instance
processor_name: The watch's processor, e.g. 'restock_diff'
context: 'get' or 'list'
Returns:
dict or None: JSON-serialisable state for this watch, or None
"""
pass
@hookspec
def register_mcp_tools():
"""Add tools to the MCP server (/mcp/v1), typically ones only a processor can answer.
Return a list of tool dicts. Core adds them to tools/list next to its own and wraps each one, so a
plugin never deals with MCP itself:
- the arguments are checked against `input_schema` before the handler runs;
- read-only mode and ?readonly=1 offer only the 'read' tools, from `kind`;
- a `watch` argument is resolved (uuid, or part of the title or URL) and, with `processor` set,
refused with a readable error when that monitor uses another processor;
- a ToolError (changedetectionio.blueprint.mcp.tools.ToolError) becomes an error the assistant
reads and can correct, any other exception an internal error;
- the text is capped in length.
The tool list is the same for every connection (MCP clients cache it), so list your tools whenever
the plugin is installed, not only while some monitor uses your processor.
Each dict:
name: a-z, 0-9 and _, at most 64 characters, starting with a letter. Prefix it with
your processor's name, e.g. 'restock_status'. A name core already uses is skipped.
title: short, shown to the user, e.g. 'Stock and prices'
description: what it answers, written for the assistant
kind: 'read' (changes nothing), 'write' or 'destructive' (can't be undone)
input_schema: JSON Schema properties, e.g. {'watch': tools.WATCH_ARGUMENT}. Unknown
arguments are refused.
required: optional list of required argument names
processor: optional processor name its `watch` argument must use, e.g. 'restock_diff'
area: optional row of the None / Read / Write permission grid the tool belongs to:
one of core's ('monitors', the default, 'tags', 'notifications', 'browser',
'system') or one a plugin adds with register_mcp_permission_rows. An access
token only gets the tool with Read (for 'read' tools) or Write on that row.
open_world: optional, True when it reaches outside this instance (another service, a
payment provider...), so clients may ask the user first (MCP openWorldHint)
handler: handler(ctx, args, watch) -> (text, structured_dict). ctx.datastore and
ctx.update_q, args as validated, watch the resolved Watch or None.
An access token can be limited to some tags (groups): list monitors with
ctx.watches() (or check ctx.in_scope(watch)), never datastore.data['watching']
directly, and resolve tags with tools._resolve_tag(ctx, ...). The `watch`
argument is already resolved within the token's groups.
Text that comes from monitored pages should start with tools.UNTRUSTED_NOTE.
Example (the built-in restock tools, processors/restock_diff/plugins/mcp_tools.py):
@hookimpl
def register_mcp_tools():
return [{'name': 'restock_status', 'title': 'Stock and prices', 'kind': 'read',
'description': 'Which product monitors are in stock, and their prices.',
'input_schema': {'tag': {'type': 'string'}}, 'handler': restock_status}]
Returns:
list of dict, or None
"""
pass
@hookspec
def register_mcp_permission_rows():
"""Add rows to the MCP None / Read / Write permission grid (access tokens and the OAuth consent screen),
for plugin tools that aren't about monitors, e.g. an 'account' row for a sign-up tool. A tool joins a row
with its `area` (see register_mcp_tools).
A row is None for every existing token until the owner changes it, and only offers the levels its
tools need: Read when one of them is 'read', Write when one changes something. A row no tool uses is
left out.
Each dict:
name: a-z, 0-9 and _, at most 32 characters, starting with a letter; not one of core's rows
label: shown in the grid, e.g. 'Account & billing'
description: optional, one line under the label: what Read / Write let an assistant do
Returns:
list of dict, or None
"""
pass
# --- Conditions (JSON Logic rules on the watch edit page) ---
@hookspec
def register_operators():
"""Return a dict of extra JSON Logic operators, {"operator_name": callable}."""
pass
@hookspec
def register_operator_choices():
"""Return a list of (operator_name, label) tuples to offer in the conditions UI."""
pass
@hookspec
def register_field_choices():
"""Return a list of (field_name, label) tuples to offer in the conditions UI."""
pass
@hookspec
def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
"""Return a dict of extra facts the conditions are evaluated against.
Each implementation is given 10 seconds, see conditions.execute_ruleset_against_all_plugins().
"""
pass
_plugin_health = {}
_plugin_health_lock = threading.Lock()
def _record_hook_success(plugin_name):
with _plugin_health_lock:
health = _plugin_health.get(plugin_name)
if health:
health['consecutive_failures'] = 0
def _record_hook_failure(plugin_name, hook_name, exception):
with _plugin_health_lock:
health = _plugin_health.setdefault(plugin_name, {'consecutive_failures': 0, 'total_failures': 0})
health['consecutive_failures'] += 1
health['total_failures'] += 1
health['last_hook'] = hook_name
health['last_error'] = f"{type(exception).__name__}: {exception}"
health['last_error_time'] = time.time()
logger.opt(exception=exception).error(f"Plugin '{plugin_name}' raised in hook '{hook_name}': {exception}")
def get_plugin_health(plugin_name=None):
"""Failure counters per plugin, {name: {'consecutive_failures', 'total_failures', 'last_hook',
'last_error', 'last_error_time', 'errored'}}. Plugins that never failed are not listed."""
with _plugin_health_lock:
snapshot = {name: dict(h, errored=h['consecutive_failures'] >= PLUGIN_CONSECUTIVE_FAILURES_ERRORED)
for name, h in _plugin_health.items()}
if plugin_name is not None:
return snapshot.get(plugin_name)
return snapshot
def _isolate_hookimpl(function, plugin_name, hook_name):
"""Wrap one hook implementation so its exceptions are logged and counted instead of breaking
the hook call (and with it the check, page or worker that made it).
pluggy calls hookimpl.function(*args) with the arguments the implementation declared, and it
has already read those names (HookImpl.argnames) from the original function, so the wrapper
only has to pass everything through.
"""
propagate = hook_name in HOOKS_THAT_PROPAGATE_EXCEPTIONS
if inspect.iscoroutinefunction(function):
import asyncio
@wraps(function)
async def isolated_async(*args, **kwargs):
try:
result = await asyncio.wait_for(function(*args, **kwargs), timeout=PLUGIN_ASYNC_HOOK_TIMEOUT)
except Exception as e:
_record_hook_failure(plugin_name, hook_name, e)
if propagate:
raise
return None
_record_hook_success(plugin_name)
return result
return isolated_async
@wraps(function)
def isolated(*args, **kwargs):
try:
result = function(*args, **kwargs)
except Exception as e:
_record_hook_failure(plugin_name, hook_name, e)
if propagate:
raise
return None
_record_hook_success(plugin_name)
return result
return isolated
class ChangeDetectionPluginManager(pluggy.PluginManager):
"""pluggy.PluginManager with failure isolation and the legacy conditions hookimpl marker."""
_warned_legacy_marker = set()
def parse_hookimpl_opts(self, plugin, name):
opts = super().parse_hookimpl_opts(plugin, name)
if opts is None:
method = getattr(plugin, name, None)
if inspect.isroutine(method):
legacy = getattr(method, LEGACY_CONDITIONS_NAMESPACE + "_impl", None)
if isinstance(legacy, dict):
plugin_label = getattr(plugin, '__name__', type(plugin).__name__)
if plugin_label not in self._warned_legacy_marker:
self._warned_legacy_marker.add(plugin_label)
logger.warning(f"Plugin '{plugin_label}' uses the deprecated "
f"HookimplMarker('{LEGACY_CONDITIONS_NAMESPACE}'), "
f"use 'from changedetectionio.plugin_api import hookimpl' instead")
opts = dict(legacy)
return opts
def register(self, plugin, name=None):
plugin_name = super().register(plugin, name)
if plugin_name is None:
return None
for hook_caller in self.get_hookcallers(plugin) or []:
for impl in hook_caller.get_hookimpls():
# Wrappers are generators that pluggy drives itself, leave them alone
if impl.plugin is plugin and not (impl.wrapper or impl.hookwrapper):
impl.function = _isolate_hookimpl(impl.function, plugin_name, hook_caller.name)
return plugin_name
# Set up Plugin Manager
plugin_manager = ChangeDetectionPluginManager(PLUGIN_NAMESPACE)
# Register hookspecs
plugin_manager.add_hookspecs(ChangeDetectionSpec)
# Load plugins from subdirectories
def load_plugins_from_directories():
# The built-in conditions operators (starts_with, length_min, ...), first so its choices lead the lists
from changedetectionio.conditions import default_plugin
if not plugin_manager.is_registered(default_plugin):
plugin_manager.register(default_plugin, 'conditions_default_plugin')
# List of (python_package_prefix, filesystem_path) pairs to scan for plugins.
# NOTE: processors/restock_diff/plugins is intentionally excluded here — those
# plugins are registered via register_builtin_restock_plugins() to avoid the
# circular import: restock_diff/__init__.py → model.Watch → content_fetchers → pluggy_interface.
plugin_dirs = [
(
'changedetectionio.conditions.plugins',
os.path.join(os.path.dirname(__file__), 'conditions', 'plugins'),
),
]
for module_prefix, dir_path in plugin_dirs:
if not os.path.exists(dir_path):
continue
# Get all Python files (excluding __init__.py)
for filename in os.listdir(dir_path):
if filename.endswith(".py") and filename != "__init__.py":
module_name = filename[:-3] # Remove .py extension
module_path = f"{module_prefix}.{module_name}"
try:
module = importlib.import_module(module_path)
# Register the plugin with pluggy
plugin_manager.register(module, module_name)
except (ImportError, AttributeError) as e:
print(f"Error loading plugin {module_name}: {e}")
# Load plugins
load_plugins_from_directories()
# Discover installed plugins from external packages (if any)
plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)
# Conditions plugins published before the hook namespaces were merged
plugin_manager.load_setuptools_entrypoints(LEGACY_CONDITIONS_NAMESPACE)
# Function to inject datastore into plugins that need it
def inject_datastore_into_plugins(datastore):
"""Inject the global datastore into plugins that need access to settings.
This should be called after plugins are loaded and datastore is initialized.
Args:
datastore: The global ChangeDetectionStore instance
"""
for plugin_name, plugin_obj in plugin_manager.list_name_plugin():
# Check if plugin has datastore attribute and it's not set
if hasattr(plugin_obj, 'datastore'):
if plugin_obj.datastore is None:
plugin_obj.datastore = datastore
logger.debug(f"Injected datastore into plugin: {plugin_name}")
# Function to register built-in fetchers - called later from content_fetchers/__init__.py
def register_builtin_fetchers():
"""Register built-in content fetchers as internal plugins
This is called from content_fetchers/__init__.py after all fetchers are imported
to avoid circular import issues.
"""
from changedetectionio.content_fetchers import requests, playwright, puppeteer, webdriver_selenium
# Register each built-in fetcher plugin
if hasattr(requests, 'requests_plugin'):
plugin_manager.register(requests.requests_plugin, 'builtin_requests')
if hasattr(playwright, 'playwright_plugin'):
plugin_manager.register(playwright.playwright_plugin, 'builtin_playwright')
if hasattr(puppeteer, 'puppeteer_plugin'):
plugin_manager.register(puppeteer.puppeteer_plugin, 'builtin_puppeteer')
if hasattr(webdriver_selenium, 'webdriver_selenium_plugin'):
plugin_manager.register(webdriver_selenium.webdriver_selenium_plugin, 'builtin_webdriver_selenium')
def register_builtin_restock_plugins():
"""Register built-in restock processor plugins after all imports are complete.
Only called from changedetectionio.main() (so not when the app is built directly, e.g. in tests),
deferred to avoid the circular import that occurs when loading via load_plugins_from_directories()
(restock_diff/__init__.py → model.Watch → content_fetchers → pluggy_interface).
"""
import importlib
module_path = 'changedetectionio.processors.restock_diff.plugins.llm_restock'
try:
module = importlib.import_module(module_path)
if not plugin_manager.is_registered(module):
plugin_manager.register(module, 'llm_restock')
logger.debug("Registered built-in restock plugin: llm_restock")
except Exception as e:
logger.error(f"Failed to register llm_restock plugin: {e}")
def register_builtin_api_plugins():
"""Register the built-in get_api_watch_processor_state and register_mcp_tools implementations.
Called from flask_app.changedetection_app() where the API resources are set up, so it runs
however the app is started. Can't happen at module load for the same circular import reason
as register_builtin_restock_plugins().
"""
from changedetectionio.processors.restock_diff.plugins import api_state, mcp_tools
if not plugin_manager.is_registered(api_state):
# 'builtin_' names are hidden from the active plugins list, see get_active_plugins()
plugin_manager.register(api_state, 'builtin_restock_api_state')
# The restock MCP tools (register_mcp_tools). Light to import: what they need is imported when they run
if not plugin_manager.is_registered(mcp_tools):
plugin_manager.register(mcp_tools, 'builtin_restock_mcp_tools')
def _require_login_before_request():
from changedetectionio.auth_decorator import login_optionally_required
# None lets the request through, otherwise it's the login redirect/401
return login_optionally_required(lambda: None)()
def register_plugin_blueprints(app, datastore):
"""Mount every blueprint returned by the register_blueprints hook on `app`.
One bad entry (wrong type, blueprint name already taken) is logged and skipped, it never
stops the app from starting. Returns the names of the blueprints that were registered.
"""
from flask import Blueprint
registered = []
for result in plugin_manager.hook.register_blueprints(datastore=datastore):
if not result:
continue
for entry in (result if isinstance(result, (list, tuple)) else [result]):
options = entry if isinstance(entry, dict) else {'blueprint': entry}
blueprint = options.get('blueprint')
if not isinstance(blueprint, Blueprint):
logger.error(f"register_blueprints: expected a Blueprint or dict with 'blueprint', got {entry!r}")
continue
try:
if options.get('login_required', True):
blueprint.before_request(_require_login_before_request)
kwargs = {'url_prefix': options['url_prefix']} if 'url_prefix' in options else {}
app.register_blueprint(blueprint, **kwargs)
except Exception as e:
logger.error(f"register_blueprints: could not register blueprint '{blueprint.name}': {e}")
continue
registered.append(blueprint.name)
logger.debug(f"Registered plugin blueprint '{blueprint.name}'")
return registered
def collect_sidebar_items(datastore):
"""Every plugin's sidebar links (register_sidebar_items), ready for sidebar-nav.html.
[{'label', 'url', 'icon', 'svg', 'title', 'active', 'disabled', 'badge', 'source'}], sorted by weight, 'source' being the module path of
the plugin that added it. A bad entry is logged and skipped.
"""
from flask import request, url_for
from markupsafe import Markup
items = []
# Each implementation is called on its own (still through the failure-isolation wrapper) so every
# item knows which plugin added it, sidebar-nav.html writes that as <!-- blueprint:module.path -->
for impl in plugin_manager.hook.register_sidebar_items.get_hookimpls():
result = impl.function(*[{'datastore': datastore}.get(name) for name in impl.argnames])
if not result:
continue
plugin = impl.plugin
source = getattr(plugin, '__name__', None) if inspect.ismodule(plugin) else \
f"{type(plugin).__module__}.{type(plugin).__qualname__}"
# Goes into an HTML comment, '--' would end it early
source = (source or impl.plugin_name).replace('--', '-')
for entry in (result if isinstance(result, (list, tuple)) else [result]):
if not isinstance(entry, dict) or not entry.get('label'):
logger.error(f"register_sidebar_items: expected a dict with a 'label', got {entry!r}")
continue
disabled = bool(entry.get('disabled'))
if disabled:
# Not clickable, nothing to link to
url = '#'
else:
try:
url = url_for(entry['endpoint']) if entry.get('endpoint') else entry.get('url')
except Exception as e:
logger.error(f"register_sidebar_items: no URL for '{entry.get('label')}': {e}")
continue
if not url:
continue
prefix = entry.get('active_prefix')
items.append({
'label': str(entry['label']),
'url': url,
'icon': entry.get('icon') or 'circle',
'svg': Markup(entry['svg']) if entry.get('svg') else None,
'title': str(entry.get('title') or entry['label']),
'active': not disabled and bool(prefix and (request.endpoint or '').startswith(prefix)),
'disabled': disabled,
'badge': str(entry['badge']) if entry.get('badge') else None,
'weight': entry.get('weight', 0),
'source': source,
})
return sorted(items, key=lambda i: i['weight'])
def collect_watch_list_extra_text(watch):
"""All plugins' notes for this watch in the watch list, as a list of strings."""
notes = []
try:
for result in reversed(plugin_manager.hook.watch_list_extra_text(watch=watch)):
if not result:
continue
notes.extend([result] if isinstance(result, str) else [str(r) for r in result if r])
except Exception as e:
logger.error(f"watch_list_extra_text failed: {e}")
return notes
# Helper function to collect UI stats extras from all plugins
def collect_ui_edit_stats_extras(watch):
"""Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
extras_content = []
# Get all plugins that implement the ui_edit_stats_extras hook
results = plugin_manager.hook.ui_edit_stats_extras(watch=watch)
# If we have results, add them to our content
if results:
for result in results:
if result: # Skip empty results
extras_content.append(result)
return "\n".join(extras_content) if extras_content else ""
def collect_fetcher_status_icons(fetcher_name):
"""Collect status icon data from all plugins
Args:
fetcher_name: The name of the fetcher (e.g., 'html_webdriver', 'html_js_zyte')
Returns:
dict or None: Icon data dictionary from first matching plugin, or None
"""
# Get status icon data from plugins
results = plugin_manager.hook.fetcher_status_icon(fetcher_name=fetcher_name)
# Return first non-None result
if results:
for result in results:
if result and isinstance(result, dict):
return result
return None
def get_itemprop_availability_from_plugin(content, fetcher_name, fetcher_instance, url, llm_intent=None):
"""Get itemprop availability data from plugins as a fallback.
This is called when the built-in get_itemprop_availability doesn't find good data.
Args:
content: The HTML/text content to parse
fetcher_name: The name of the fetcher being used (e.g., 'html_js_zyte')
fetcher_instance: The fetcher instance that generated the content
url: The URL being watched (watch.link - includes Jinja2 evaluation)
llm_intent: Optional user-supplied intent string passed through to plugins
Returns:
dict or None: Availability data dictionary from first matching plugin, or None
"""
hook = plugin_manager.hook.get_itemprop_availability_override
# html_to_text() isn't free on a large product page, so only run it when some
# implementation actually asks for `text` (pluggy only passes the args an impl declares).
text = None
if content and any('text' in impl.argnames for impl in hook.get_hookimpls()):
try:
from changedetectionio.html_tools import html_to_text
text = html_to_text(content)
except Exception as e:
logger.warning(f"Could not convert {url} to text for restock plugins: {e}")
# firstresult=True, so this is the first non-None answer
result = hook(
content=content,
fetcher_name=fetcher_name,
fetcher_instance=fetcher_instance,
url=url,
llm_intent=llm_intent,
text=text,
)
# Only accept it if it carries actual data
if result and isinstance(result, dict):
if result.get('price') is not None or result.get('availability'):
return result
return None
def get_active_plugins():
"""Get a list of active plugins with their descriptions.
Returns:
list: List of dictionaries with plugin information:
[
{'name': 'plugin_name', 'description': 'Plugin description'},
...
]
"""
active_plugins = []
# Get all registered plugins
for plugin_name, plugin_obj in plugin_manager.list_name_plugin():
# Skip built-in plugins (they start with 'builtin_')
if plugin_name.startswith('builtin_'):
continue
# Get plugin description if available
description = None
if hasattr(plugin_obj, '__doc__') and plugin_obj.__doc__:
description = plugin_obj.__doc__.strip().split('\n')[0] # First line only
elif hasattr(plugin_obj, 'description'):
description = plugin_obj.description
# Try to get a friendly name from the plugin
friendly_name = plugin_name
if hasattr(plugin_obj, 'name'):
friendly_name = plugin_obj.name
active_plugins.append({
'name': friendly_name,
'description': description or 'No description available'
})
return active_plugins
def get_fetcher_capabilities(watch, datastore):
"""Get capability flags for a watch's fetcher.
Args:
watch: The watch object/dict
datastore: The datastore to resolve 'system' fetcher
Returns:
dict: Dictionary with capability flags:
{
'supports_browser_steps': bool,
'supports_screenshots': bool,
'supports_xpath_element_data': bool
}
"""
# Get the fetcher name from watch
fetcher_name = watch.get('fetch_backend', 'system')
# Resolve 'system' to actual fetcher
if fetcher_name == 'system':
fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
# Get the fetcher class
from changedetectionio import content_fetchers
# Try built-in fetchers first, then plugin-provided fetchers
fetcher_class = None
if hasattr(content_fetchers, fetcher_name):
fetcher_class = getattr(content_fetchers, fetcher_name)
else:
# Query all plugins for registered fetchers
for fetcher_registration in plugin_manager.hook.register_content_fetcher():
if fetcher_registration and fetcher_registration[0] == fetcher_name:
fetcher_class = fetcher_registration[1]
break
# Log + build from the shared FetcherCapabilities model. Return a plain dict so
# callers (e.g. blueprint/ui/edit.py) can still add their own extra capability keys.
caps = content_fetchers._log_fetcher_capabilities(fetcher_class, fetcher_name, uuid=watch.get('uuid'))
return caps.model_dump()
def get_plugin_settings_tabs():
"""Get all plugin settings tabs.
Returns:
list: List of dictionaries with plugin settings tab information:
[
{
'plugin_id': str,
'tab_label': str,
'form_class': Form,
'description': str
},
...
]
"""
tabs = []
results = plugin_manager.hook.plugin_settings_tab()
for result in results:
if result and isinstance(result, dict):
# Validate required fields
if 'plugin_id' in result and 'tab_label' in result and 'form_class' in result:
tabs.append(result)
else:
logger.warning(f"Invalid plugin settings tab spec: {result}")
return tabs
def load_plugin_settings(datastore_path, plugin_id):
"""Load settings for a specific plugin from JSON file.
Args:
datastore_path: Path to the datastore directory
plugin_id: Unique identifier for the plugin (e.g., 'zyte_fetcher')
Returns:
dict: Plugin settings, or empty dict if file doesn't exist
"""
import json
settings_file = os.path.join(datastore_path, f"{plugin_id}.json")
if not os.path.exists(settings_file):
return {}
try:
with open(settings_file, 'r', encoding='utf-8') as f:
return json.load(f)
except Exception as e:
logger.error(f"Failed to load settings for plugin '{plugin_id}': {e}")
return {}
def save_plugin_settings(datastore_path, plugin_id, settings):
"""Save settings for a specific plugin to JSON file.
Args:
datastore_path: Path to the datastore directory
plugin_id: Unique identifier for the plugin (e.g., 'zyte_fetcher')
settings: Dictionary of settings to save
Returns:
bool: True if save was successful, False otherwise
"""
import json
settings_file = os.path.join(datastore_path, f"{plugin_id}.json")
try:
with open(settings_file, 'w', encoding='utf-8') as f:
json.dump(settings, f, indent=2, ensure_ascii=False)
logger.info(f"Saved settings for plugin '{plugin_id}' to {settings_file}")
return True
except Exception as e:
logger.error(f"Failed to save settings for plugin '{plugin_id}': {e}")
return False
def get_plugin_template_paths():
"""Get list of plugin template directories for Jinja2 loader.
Scans both external pluggy plugins and built-in processor plugins.
Returns:
list: List of absolute paths to plugin template directories
"""
template_paths = []
# Add the base processors/templates directory (as absolute path)
processors_templates_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processors', 'templates')
if os.path.isdir(processors_templates_dir):
template_paths.append(processors_templates_dir)
logger.debug(f"Added base processors template path: {processors_templates_dir}")
# Scan built-in processor plugins
from changedetectionio.processors import find_processors
processor_list = find_processors()
for processor_module, processor_name in processor_list:
# Each processor is a module, check if it has a templates directory
if hasattr(processor_module, '__file__'):
processor_file = processor_module.__file__
if processor_file:
# Get the processor directory (e.g., processors/image_ssim_diff/)
processor_dir = os.path.dirname(os.path.abspath(processor_file))
templates_dir = os.path.join(processor_dir, 'templates')
if os.path.isdir(templates_dir):
template_paths.append(templates_dir)
logger.debug(f"Added processor template path: {templates_dir}")
# Get all registered external pluggy plugins
for plugin_name, plugin_obj in plugin_manager.list_name_plugin():
# Check if plugin has a templates directory
if hasattr(plugin_obj, '__file__'):
plugin_file = plugin_obj.__file__
elif hasattr(plugin_obj, '__module__'):
# Get the module file
module = sys.modules.get(plugin_obj.__module__)
if module and hasattr(module, '__file__'):
plugin_file = module.__file__
else:
continue
else:
continue
if plugin_file:
plugin_dir = os.path.dirname(os.path.abspath(plugin_file))
templates_dir = os.path.join(plugin_dir, 'templates')
if os.path.isdir(templates_dir):
template_paths.append(templates_dir)
logger.debug(f"Added plugin template path: {templates_dir}")
return template_paths
async def select_fetch_backends(watch, datastore, backend_name, failed_fetch=None, tried=()):
"""The fetcher names plugins offered for this check (see the select_fetch_backend hook), best first."""
from changedetectionio.plugin_api import gather_hook_results
offers = await gather_hook_results(plugin_manager.hook.select_fetch_backend(
watch=watch, datastore=datastore, backend_name=backend_name, failed_fetch=failed_fetch, tried=list(tried)))
ranked = []
for offer in offers:
if isinstance(offer, str):
name, priority = offer, 0
elif isinstance(offer, dict) and isinstance(offer.get('backend'), str):
name = offer['backend']
try:
priority = int(offer.get('priority') or 0)
except (TypeError, ValueError):
priority = 0
else:
logger.warning(f"select_fetch_backend returned {offer!r}, expected a fetcher name or {{'backend': .., 'priority': ..}}")
continue
if name != backend_name and name not in tried:
ranked.append((priority, name))
# sorted() is stable, equal priorities keep the plugins' order
names = []
for _, name in sorted(ranked, key=lambda r: r[0]):
if name not in names:
names.append(name)
return names
def apply_recheck_seconds_override(watch, datastore, seconds):
"""The recheck time for this watch after plugins had their say (recheck_seconds_override hook)."""
hook = plugin_manager.hook.recheck_seconds_override
# The scheduler asks for every watch every second, skip the hook machinery when no plugin implements it
if not hook.get_hookimpls():
return seconds
override = hook(watch=watch, datastore=datastore, seconds=seconds)
if override is None:
return seconds
try:
override = int(override)
except (TypeError, ValueError):
logger.warning(f"recheck_seconds_override returned {override!r}, expected seconds")
return seconds
return override if override > 0 else seconds
def apply_update_handler_alter(update_handler, watch, datastore):
"""Apply update_handler_alter hooks from all plugins.
Allows plugins to wrap or modify the update_handler before it processes a watch.
Multiple plugins can chain modifications - each plugin receives the result from
the previous plugin.
Args:
update_handler: The perform_site_check instance to potentially modify
watch: The watch dict being processed
datastore: The application datastore
Returns:
object: The (potentially modified/wrapped) update_handler
"""
# Get all plugins that implement the update_handler_alter hook
results = plugin_manager.hook.update_handler_alter(
update_handler=update_handler,
watch=watch,
datastore=datastore
)
# Chain results - each plugin gets the result from the previous one
current_handler = update_handler
if results:
for result in results:
if result is not None:
logger.debug(f"Plugin modified update_handler for watch {watch.get('uuid')}")
current_handler = result
return current_handler
def apply_update_finalize(update_handler, watch, datastore, processing_exception):
"""Apply update_finalize hooks from all plugins.
Called in the finally block after watch processing completes, allowing plugins
to perform cleanup, update metrics, or log final status.
Args:
update_handler: The perform_site_check instance (may be None)
watch: The watch dict that was processed (may be None)
datastore: The application datastore
processing_exception: The exception from processing, or None if successful
Returns:
None
"""
try:
# Call all plugins that implement the update_finalize hook
plugin_manager.hook.update_finalize(
update_handler=update_handler,
watch=watch,
datastore=datastore,
processing_exception=processing_exception
)
except Exception as e:
# Don't let plugin errors crash the worker
logger.error(f"Error in update_finalize hook: {e}")
logger.exception(f"update_finalize hook exception details:")
def collect_html_head_extras():
"""Collect and combine HTML head extras from all plugins.
Called from a Flask template global so it always runs inside a request context.
This means url_for() works correctly in plugin implementations, including when the
app is deployed under a sub-path via USE_X_SETTINGS / X-Forwarded-Prefix (ProxyFix
sets SCRIPT_NAME so url_for() automatically prepends the prefix).
Returns:
str: Combined HTML string to inject inside <head>, or empty string
"""
results = plugin_manager.hook.get_html_head_extras()
parts = [r for r in results if r]
return "\n".join(parts) if parts else ""