mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-11-18 20:54:26 +00:00
Puppeteer fetcher - A page that navigates on from its load event is not an EmptyReply (#4740)
A parked domain answers with an empty document whose onload sets location.href. goto() returns no response for it, which was taken as "the page did not fetch": three retries in two seconds, then "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code None", for a page that goes on to load fine. Use the main-frame response that was already seen, as the interstitial case does, and carry on to the content-ready wait where the page it lands on is picked up. Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
1 parent
8fa39dfe60
commit
075f190594
3 files changed
+49
No files matched your search
@@ -670,6 +670,15 @@ class fetcher(Fetcher):
|
||||
if goto_task in done:
|
||||
try:
|
||||
response = goto_task.result()
|
||||
if response is None and navigation_response.get('response'):
|
||||
# goto() hands back nothing when the page starts its next navigation
|
||||
# from its own load event (a parked domain: an empty document whose
|
||||
# onload sets location.href). That is not "the page did not fetch" -
|
||||
# retrying gets the same answer three times in a row and ends as an
|
||||
# EmptyReply, on a page that goes on to load perfectly well.
|
||||
response = navigation_response['response']
|
||||
logger.debug(f"goto() returned no response (the page navigated again as it loaded), "
|
||||
f"continuing with the one seen for {response.url} - status {response.status}")
|
||||
except Exception as e:
|
||||
if 'ERR_HTTP_RESPONSE_CODE_FAILURE' not in str(e) or not navigation_response:
|
||||
raise
|
||||
|
||||
@@ -60,6 +60,35 @@ def test_interstitial_redirect_is_followed(client, live_server, measure_memory_u
|
||||
client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_redirect_from_the_load_event_is_followed(client, live_server, measure_memory_usage, datastore_path):
|
||||
"""An empty page that navigates on from its own load event (a parked domain) is not an EmptyReply"""
|
||||
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
|
||||
|
||||
res = client.post(
|
||||
url_for("settings.settings_page"),
|
||||
data={
|
||||
"application-empty_pages_are_a_change": "",
|
||||
"requests-time_between_check-minutes": 180,
|
||||
'application-fetch_backend': "html_webdriver",
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Settings updated." in res.data
|
||||
|
||||
datastore = live_server.app.config['DATASTORE']
|
||||
uuid = datastore.add_watch(url=_cdio(url_for('test_onload_redirect', _external=True)))
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
watch = datastore.data['watching'][uuid]
|
||||
assert not watch.get('last_error'), \
|
||||
f"The page it was sent on to loaded fine, but the fetch failed: {watch.get('last_error')}"
|
||||
assert watch.history_n >= 1, "Fetch succeeded but no snapshot was stored"
|
||||
assert 'This domain is for sale' in watch.get_history_snapshot(list(watch.history.keys())[-1])
|
||||
|
||||
client.post(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_har_record_env_writes_har_zip(client, live_server, measure_memory_usage, datastore_path, monkeypatch):
|
||||
"""HAR_UUID_RECORD=<uuid> saves the network traffic of that watch's fetch to har-record.zip"""
|
||||
import json
|
||||
|
||||
@@ -329,6 +329,17 @@ def new_live_server_setup(live_server):
|
||||
# not seen recently (reported against fotokoch.de, which answers 503 + meta refresh and then
|
||||
# serves a 200). The browser follows the refresh, so the fetch has to be judged on the
|
||||
# document we actually end up extracting rather than on the interstitial.
|
||||
# What a parked domain serves: an empty document that sends the browser on from its load event
|
||||
@live_server.app.route('/test-onload-redirect')
|
||||
def test_onload_redirect():
|
||||
landing = url_for('test_onload_redirect_landing')
|
||||
return ('<!DOCTYPE html><html><head><script>window.onload=function(){window.location.href="'
|
||||
+ landing + '"}</script></head></html>')
|
||||
|
||||
@live_server.app.route('/test-onload-redirect-landing')
|
||||
def test_onload_redirect_landing():
|
||||
return '<html><body><h1>This domain is for sale</h1></body></html>'
|
||||
|
||||
# Keyed on last-seen time rather than a hit count, so EVERY fresh check starts out gated -
|
||||
# a counter would serve a clean 200 to the second check and let the test pass without the fix.
|
||||
_interstitial_last_seen = {}
|
||||
|
||||
Reference in new issue
Block a user