From 075f1905942b23219d9920dd67bafb752a567464 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 16 Nov 2026 01:42:24 +0100 Subject: [PATCH] Puppeteer fetcher - A page that navigates on from its load event is not an EmptyReply (#4740) A parked domain answers with an empty document whose onload sets location.href. goto() returns no response for it, which was taken as "the page did not fetch": three retries in two seconds, then "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code None", for a page that goes on to load fine. Use the main-frame response that was already seen, as the interstitial case does, and carry on to the content-ready wait where the page it lands on is picked up. Co-authored-by: Claude Fable 5.2 --- .../content_fetchers/puppeteer.py | 9 ++++++ .../tests/fetchers/test_renavigation.py | 29 +++++++++++++++++++ changedetectionio/tests/util.py | 11 +++++++ 3 files changed, 49 insertions(+) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index c4a6929b3..ce1154713 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -670,6 +670,15 @@ class fetcher(Fetcher): if goto_task in done: try: response = goto_task.result() + if response is None and navigation_response.get('response'): + # goto() hands back nothing when the page starts its next navigation + # from its own load event (a parked domain: an empty document whose + # onload sets location.href). That is not "the page did not fetch" - + # retrying gets the same answer three times in a row and ends as an + # EmptyReply, on a page that goes on to load perfectly well. + response = navigation_response['response'] + logger.debug(f"goto() returned no response (the page navigated again as it loaded), " + f"continuing with the one seen for {response.url} - status {response.status}") except Exception as e: if 'ERR_HTTP_RESPONSE_CODE_FAILURE' not in str(e) or not navigation_response: raise diff --git a/changedetectionio/tests/fetchers/test_renavigation.py b/changedetectionio/tests/fetchers/test_renavigation.py index 3a974ceee..71fe6d3d0 100644 --- a/changedetectionio/tests/fetchers/test_renavigation.py +++ b/changedetectionio/tests/fetchers/test_renavigation.py @@ -60,6 +60,35 @@ def test_interstitial_redirect_is_followed(client, live_server, measure_memory_u client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True) +def test_redirect_from_the_load_event_is_followed(client, live_server, measure_memory_usage, datastore_path): + """An empty page that navigates on from its own load event (a parked domain) is not an EmptyReply""" + assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" + + res = client.post( + url_for("settings.settings_page"), + data={ + "application-empty_pages_are_a_change": "", + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_webdriver", + }, + follow_redirects=True + ) + assert b"Settings updated." in res.data + + datastore = live_server.app.config['DATASTORE'] + uuid = datastore.add_watch(url=_cdio(url_for('test_onload_redirect', _external=True))) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + watch = datastore.data['watching'][uuid] + assert not watch.get('last_error'), \ + f"The page it was sent on to loaded fine, but the fetch failed: {watch.get('last_error')}" + assert watch.history_n >= 1, "Fetch succeeded but no snapshot was stored" + assert 'This domain is for sale' in watch.get_history_snapshot(list(watch.history.keys())[-1]) + + client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True) + + def test_har_record_env_writes_har_zip(client, live_server, measure_memory_usage, datastore_path, monkeypatch): """HAR_UUID_RECORD= saves the network traffic of that watch's fetch to har-record.zip""" import json diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 6925207ef..b95996dbb 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -329,6 +329,17 @@ def new_live_server_setup(live_server): # not seen recently (reported against fotokoch.de, which answers 503 + meta refresh and then # serves a 200). The browser follows the refresh, so the fetch has to be judged on the # document we actually end up extracting rather than on the interstitial. + # What a parked domain serves: an empty document that sends the browser on from its load event + @live_server.app.route('/test-onload-redirect') + def test_onload_redirect(): + landing = url_for('test_onload_redirect_landing') + return ('') + + @live_server.app.route('/test-onload-redirect-landing') + def test_onload_redirect_landing(): + return '

This domain is for sale

' + # Keyed on last-seen time rather than a hit count, so EVERY fresh check starts out gated - # a counter would serve a clean 200 to the second check and let the test pass without the fix. _interstitial_last_seen = {}