Puppeteer fetcher - A page that navigates on from its load event is not an EmptyReply (#4740)

A parked domain answers with an empty document whose onload sets
location.href. goto() returns no response for it, which was taken as "the page
did not fetch": three retries in two seconds, then "EmptyReply - try increasing
'Wait seconds before extracting text', Status Code None", for a page that goes
on to load fine.

Use the main-frame response that was already seen, as the interstitial case
does, and carry on to the content-ready wait where the page it lands on is
picked up.

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
dgtlmoonandClaude Fable 5.2 authored and GitHub committed 2026-11-16 01:42:24 +01:00
1 parent 8fa39dfe60
commit 075f190594
3 files changed
+49

No files matched your search

@@ -670,6 +670,15 @@ class fetcher(Fetcher):
if goto_task in done:
try:
response = goto_task.result()
if response is None and navigation_response.get('response'):
# goto() hands back nothing when the page starts its next navigation
# from its own load event (a parked domain: an empty document whose
# onload sets location.href). That is not "the page did not fetch" -
# retrying gets the same answer three times in a row and ends as an
# EmptyReply, on a page that goes on to load perfectly well.
response = navigation_response['response']
logger.debug(f"goto() returned no response (the page navigated again as it loaded), "
f"continuing with the one seen for {response.url} - status {response.status}")
except Exception as e:
if 'ERR_HTTP_RESPONSE_CODE_FAILURE' not in str(e) or not navigation_response:
raise
@@ -60,6 +60,35 @@ def test_interstitial_redirect_is_followed(client, live_server, measure_memory_u
client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
def test_redirect_from_the_load_event_is_followed(client, live_server, measure_memory_usage, datastore_path):
"""An empty page that navigates on from its own load event (a parked domain) is not an EmptyReply"""
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
res = client.post(
url_for("settings.settings_page"),
data={
"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_webdriver",
},
follow_redirects=True
)
assert b"Settings updated." in res.data
datastore = live_server.app.config['DATASTORE']
uuid = datastore.add_watch(url=_cdio(url_for('test_onload_redirect', _external=True)))
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
watch = datastore.data['watching'][uuid]
assert not watch.get('last_error'), \
f"The page it was sent on to loaded fine, but the fetch failed: {watch.get('last_error')}"
assert watch.history_n >= 1, "Fetch succeeded but no snapshot was stored"
assert 'This domain is for sale' in watch.get_history_snapshot(list(watch.history.keys())[-1])
client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
def test_har_record_env_writes_har_zip(client, live_server, measure_memory_usage, datastore_path, monkeypatch):
"""HAR_UUID_RECORD=<uuid> saves the network traffic of that watch's fetch to har-record.zip"""
import json
+11
View File
@@ -329,6 +329,17 @@ def new_live_server_setup(live_server):
# not seen recently (reported against fotokoch.de, which answers 503 + meta refresh and then
# serves a 200). The browser follows the refresh, so the fetch has to be judged on the
# document we actually end up extracting rather than on the interstitial.
# What a parked domain serves: an empty document that sends the browser on from its load event
@live_server.app.route('/test-onload-redirect')
def test_onload_redirect():
landing = url_for('test_onload_redirect_landing')
return ('<!DOCTYPE html><html><head><script>window.onload=function(){window.location.href="'
+ landing + '"}</script></head></html>')
@live_server.app.route('/test-onload-redirect-landing')
def test_onload_redirect_landing():
return '<html><body><h1>This domain is for sale</h1></body></html>'
# Keyed on last-seen time rather than a hit count, so EVERY fresh check starts out gated -
# a counter would serve a clean 200 to the second check and let the test pass without the fix.
_interstitial_last_seen = {}