From ae532c82e81e499dd97def6598045c1e54e2f4ce Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 28 Oct 2025 21:32:25 +0100 Subject: [PATCH] Run all pytests in parallel (#3569) --- .../actions/extract-memory-report/action.yml | 41 +++--- .github/workflows/containers.yml | 8 ++ .github/workflows/test-container-build.yml | 8 ++ .../test-stack-reusable-workflow.yml | 14 ++- Dockerfile | 2 + changedetectionio/model/App.py | 4 +- changedetectionio/processors/__init__.py | 2 + changedetectionio/processors/magic.py | 2 +- .../processors/text_json_diff/__init__.py | 17 +-- changedetectionio/pytest.ini | 2 +- changedetectionio/run_basic_tests.sh | 11 +- changedetectionio/run_proxy_tests.sh | 28 +++-- changedetectionio/run_socks_proxy_tests.sh | 10 +- changedetectionio/store.py | 23 ++-- changedetectionio/tests/conftest.py | 119 +++++++++++++++--- .../test_custom_browser_url.py | 4 +- .../tests/fetchers/test_content.py | 2 +- .../fetchers/test_custom_js_before_content.py | 2 +- .../tests/proxy_list/test_multiple_proxy.py | 2 +- .../tests/proxy_list/test_noproxy.py | 2 +- .../tests/proxy_list/test_proxy.py | 2 +- .../tests/proxy_list/test_proxy_noconnect.py | 2 +- .../proxy_list/test_select_custom_proxy.py | 4 +- .../tests/proxy_socks5/test_socks5_proxy.py | 8 +- .../proxy_socks5/test_socks5_proxy_sources.py | 8 +- .../tests/restock/test_restock.py | 25 ++-- .../tests/smtp/test_notification_smtp.py | 73 ++++++----- .../tests/test_access_control.py | 2 +- .../tests/test_add_replace_remove_filter.py | 34 ++--- changedetectionio/tests/test_api.py | 29 ++--- .../tests/test_api_notifications.py | 2 +- changedetectionio/tests/test_api_openapi.py | 16 +-- changedetectionio/tests/test_api_search.py | 2 +- changedetectionio/tests/test_api_tags.py | 5 +- changedetectionio/tests/test_auth.py | 2 +- .../test_automatic_follow_ldjson_price.py | 23 ++-- changedetectionio/tests/test_backend.py | 27 ++-- changedetectionio/tests/test_backup.py | 8 +- .../tests/test_basic_socketio.py | 15 +-- .../tests/test_block_while_text_present.py | 24 ++-- changedetectionio/tests/test_clone.py | 5 +- changedetectionio/tests/test_conditions.py | 41 +++--- changedetectionio/tests/test_css_selector.py | 25 ++-- .../tests/test_datastore_isolation.py | 19 +++ .../tests/test_element_removal.py | 23 ++-- changedetectionio/tests/test_encoding.py | 13 +- changedetectionio/tests/test_errorhandling.py | 26 ++-- changedetectionio/tests/test_extract_csv.py | 7 +- changedetectionio/tests/test_extract_regex.py | 27 ++-- .../tests/test_filter_exist_changes.py | 26 ++-- .../tests/test_filter_failure_notification.py | 55 ++++---- changedetectionio/tests/test_group.py | 42 +++---- .../tests/test_history_consistency.py | 10 +- changedetectionio/tests/test_ignore.py | 13 +- changedetectionio/tests/test_ignore_text.py | 37 +++--- .../tests/test_ignorehyperlinks.py | 17 +-- .../tests/test_ignorestatuscode.py | 21 ++-- .../tests/test_ignorewhitespace.py | 17 +-- changedetectionio/tests/test_import.py | 12 +- changedetectionio/tests/test_jinja2.py | 8 +- .../tests/test_jsonpath_jq_selector.py | 93 +++++++------- changedetectionio/tests/test_live_preview.py | 9 +- .../tests/test_nonrenderable_pages.py | 19 +-- changedetectionio/tests/test_notification.py | 82 ++++++------ .../tests/test_notification_errors.py | 10 +- changedetectionio/tests/test_obfuscations.py | 9 +- changedetectionio/tests/test_pdf.py | 17 +-- .../tests/test_preview_endpoints.py | 11 +- changedetectionio/tests/test_request.py | 31 ++--- .../tests/test_restock_itemprop.py | 88 ++++++------- changedetectionio/tests/test_rss.py | 32 ++--- .../tests/test_rss_reader_mode.py | 14 ++- changedetectionio/tests/test_scheduler.py | 8 +- changedetectionio/tests/test_search.py | 4 +- changedetectionio/tests/test_security.py | 18 +-- changedetectionio/tests/test_share_watch.py | 5 +- changedetectionio/tests/test_source.py | 12 +- changedetectionio/tests/test_trigger.py | 21 ++-- changedetectionio/tests/test_trigger_regex.py | 13 +- .../tests/test_trigger_regex_with_filter.py | 13 +- changedetectionio/tests/test_ui.py | 21 ++-- changedetectionio/tests/test_unique_lines.py | 35 +++--- .../tests/test_watch_fields_storage.py | 5 +- .../tests/test_xpath_selector.py | 69 +++++----- changedetectionio/tests/util.py | 46 ++++--- .../tests/visualselector/test_fetch_data.py | 14 +-- requirements.txt | 2 + 87 files changed, 986 insertions(+), 773 deletions(-) create mode 100644 changedetectionio/tests/test_datastore_isolation.py diff --git a/.github/actions/extract-memory-report/action.yml b/.github/actions/extract-memory-report/action.yml index a6f4d685..c7729218 100644 --- a/.github/actions/extract-memory-report/action.yml +++ b/.github/actions/extract-memory-report/action.yml @@ -23,26 +23,29 @@ runs: - name: Dump container log shell: bash run: | - docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout" - docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr" + echo "Disabled for now" +# return +# docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout" +# docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr" - name: Extract and display memory test report shell: bash run: | - echo "Extracting test-memory.log from container..." - docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container" - - echo "=== Top 10 Highest Peak Memory Tests ===" - if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then - grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \ - sed 's/.*Peak memory: //' | \ - paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \ - sort -t'|' -k1 -nr | \ - cut -d'|' -f2 | \ - head -10 - echo "" - echo "=== Full Memory Test Report ===" - cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log - else - echo "No memory log available" - fi + echo "Disabled for now" +# echo "Extracting test-memory.log from container..." +# docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container" +# +# echo "=== Top 10 Highest Peak Memory Tests ===" +# if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then +# grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \ +# sed 's/.*Peak memory: //' | \ +# paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \ +# sort -t'|' -k1 -nr | \ +# cut -d'|' -f2 | \ +# head -10 +# echo "" +# echo "=== Full Memory Test Report ===" +# cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log +# else +# echo "No memory log available" +# fi diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 6c591cff..37a7ae43 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -45,6 +45,14 @@ jobs: with: python-version: 3.11 + - name: Cache pip packages + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index 48838e82..7b3f3772 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -50,6 +50,14 @@ jobs: with: python-version: 3.11 + - name: Cache pip packages + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + # Just test that the build works, some libraries won't compile on ARM/rPi etc - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml index bb27b721..e5fb006a 100644 --- a/.github/workflows/test-stack-reusable-workflow.yml +++ b/.github/workflows/test-stack-reusable-workflow.yml @@ -28,6 +28,15 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Cache pip packages + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}- + ${{ runner.os }}-pip- + - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }} run: | echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----" @@ -289,15 +298,18 @@ jobs: docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest - - name: Test proxy squid style interaction + - name: Test proxy Squid style interaction run: | cd changedetectionio ./run_proxy_tests.sh + docker ps + cd .. - name: Test proxy SOCKS5 style interaction run: | cd changedetectionio ./run_socks_proxy_tests.sh + cd .. # Custom browser URL tests custom-browser-tests: diff --git a/Dockerfile b/Dockerfile index 3abf0bbc..bdc22907 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl" ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 RUN --mount=type=cache,target=/tmp/pip-cache \ pip install \ + --prefer-binary \ --extra-index-url https://www.piwheels.org/simple \ --extra-index-url https://pypi.anaconda.org/ARM-software/simple \ --cache-dir=/tmp/pip-cache \ @@ -47,6 +48,7 @@ RUN --mount=type=cache,target=/tmp/pip-cache \ # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported) RUN --mount=type=cache,target=/tmp/pip-cache \ pip install \ + --prefer-binary \ --cache-dir=/tmp/pip-cache \ --target=/dependencies \ playwright~=1.48.0 \ diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index d3dc474b..084e957a 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -1,4 +1,5 @@ from os import getenv +from copy import deepcopy from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES @@ -74,7 +75,8 @@ class model(dict): def __init__(self, *arg, **kw): super(model, self).__init__(*arg, **kw) - self.update(self.base_config) + # CRITICAL: deepcopy to avoid sharing mutable objects between instances + self.update(deepcopy(self.base_config)) def parse_headers_from_text_file(filepath): diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index e5aa9b23..4aa87761 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -91,6 +91,8 @@ class difference_detection_processor(): else: logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ") + logger.debug(f"Using proxy '{proxy_url}' for {self.watch['uuid']}") + # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need. # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc) self.fetcher = fetcher_obj(proxy_override=proxy_url, diff --git a/changedetectionio/processors/magic.py b/changedetectionio/processors/magic.py index 5eebe93d..2a0ef68f 100644 --- a/changedetectionio/processors/magic.py +++ b/changedetectionio/processors/magic.py @@ -88,7 +88,7 @@ class guess_stream_type(): magic_content_header = mime except Exception as e: - logger.error(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection") + logger.warning(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection") # Content-based detection (most reliable for text formats) # Check for HTML patterns first - if found, override magic's text/plain diff --git a/changedetectionio/processors/text_json_diff/__init__.py b/changedetectionio/processors/text_json_diff/__init__.py index 14053374..8dc10490 100644 --- a/changedetectionio/processors/text_json_diff/__init__.py +++ b/changedetectionio/processors/text_json_diff/__init__.py @@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data): '''Used by @app.route("/edit//preview-rendered", methods=['POST'])''' from changedetectionio import forms, html_tools from changedetectionio.model.Watch import model as watch_model - from concurrent.futures import ProcessPoolExecutor + from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from flask import request import brotli @@ -76,13 +76,16 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data): update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type') # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk - # Do this as a parallel process because it could take some time - with ProcessPoolExecutor(max_workers=2) as executor: - future1 = executor.submit(_task, tmp_watch, update_handler) - future2 = executor.submit(_task, blank_watch_no_filters, update_handler) + # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects + try: + with ThreadPoolExecutor(max_workers=2) as executor: + future1 = executor.submit(_task, tmp_watch, update_handler) + future2 = executor.submit(_task, blank_watch_no_filters, update_handler) - text_after_filter = future1.result() - text_before_filter = future2.result() + text_after_filter = future1.result() + text_before_filter = future2.result() + except Exception as e: + x=1 try: trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter, diff --git a/changedetectionio/pytest.ini b/changedetectionio/pytest.ini index af2b409a..38ae2db8 100644 --- a/changedetectionio/pytest.ini +++ b/changedetectionio/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --no-start-live-server --live-server-port=5005 +addopts = --no-start-live-server --live-server-port=0 #testpaths = tests pytest_invenio #live_server_scope = function diff --git a/changedetectionio/run_basic_tests.sh b/changedetectionio/run_basic_tests.sh index eb0db6ec..420f1f60 100755 --- a/changedetectionio/run_basic_tests.sh +++ b/changedetectionio/run_basic_tests.sh @@ -11,19 +11,16 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -find tests/test_*py -type f|while read test_name -do - echo "TEST RUNNING $test_name" - # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser - REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name -done +# REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser +REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load tests/test_*.py +#time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py echo "RUNNING WITH BASE_URL SET" # Now re-run some tests with BASE_URL enabled # Re #65 - Ability to include a link back to the installation, in the notification. export BASE_URL="https://really-unique-domain.io" -REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py +REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv --maxfail=1 tests/test_notification.py # Re-run with HIDE_REFERER set - could affect login diff --git a/changedetectionio/run_proxy_tests.sh b/changedetectionio/run_proxy_tests.sh index fe28bb2f..5a157c27 100755 --- a/changedetectionio/run_proxy_tests.sh +++ b/changedetectionio/run_proxy_tests.sh @@ -5,8 +5,6 @@ set -e # enable debug set -x -docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network - # Test proxy list handling, starting two squids on different ports # Each squid adds a different header to the response, which is the main thing we test for. docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge @@ -21,12 +19,13 @@ docker run --network changedet-network -d \ -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \ ubuntu/squid:4.13-21.10_edge - +sleep 5 ## 2nd test actually choose the preferred proxy from proxies.json +# This will force a request via "proxy-two" docker run --network changedet-network \ - -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \ + -v `pwd`/tests/proxy_list/proxies.json-example:/tmp/proxies.json \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py' + bash -c 'cd changedetectionio && pytest -s tests/proxy_list/test_multiple_proxy.py --datastore-path /tmp' set +e echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here" @@ -50,8 +49,10 @@ fi # Test the UI configurable proxies docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py' + bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py --datastore-path /tmp' +# Give squid proxies a moment to flush their logs +sleep 2 # Should see a request for one.changedetection.io in there echo "- Looking for .changedetection.io request in squid-custom" @@ -65,7 +66,10 @@ fi # Test "no-proxy" option docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py' + bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py --datastore-path /tmp' + +# Give squid proxies a moment to flush their logs +sleep 2 # We need to handle grep returning 1 set +e @@ -82,6 +86,8 @@ for c in $(echo "squid-one squid-two squid-custom"); do fi done +echo "docker ps output" +docker ps docker kill squid-one squid-two squid-custom @@ -90,19 +96,19 @@ docker kill squid-one squid-two squid-custom # Requests docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py' + bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp' # Playwright docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py' + bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp' # Puppeteer fast docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py' + bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp' # Selenium docker run --network changedet-network \ test-changedetectionio \ - bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py' + bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp' diff --git a/changedetectionio/run_socks_proxy_tests.sh b/changedetectionio/run_socks_proxy_tests.sh index 3bcc6852..813cc57d 100755 --- a/changedetectionio/run_socks_proxy_tests.sh +++ b/changedetectionio/run_socks_proxy_tests.sh @@ -15,13 +15,13 @@ docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p echo "---------------------------------- SOCKS5 -------------------" # SOCKS5 related - test from proxies.json docker run --network changedet-network \ - -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \ + -v `pwd`/tests/proxy_socks5/proxies.json-example:/tmp/proxies.json \ --rm \ -e "FLASK_SERVER_NAME=cdio" \ --hostname cdio \ -e "SOCKSTEST=proxiesjson" \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy_sources.py' + bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp' # SOCKS5 related - by manually entering in UI docker run --network changedet-network \ @@ -30,18 +30,18 @@ docker run --network changedet-network \ --hostname cdio \ -e "SOCKSTEST=manual" \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy.py' + bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy.py --datastore-path /tmp' # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY docker run --network changedet-network \ -e "SOCKSTEST=manual-playwright" \ --hostname cdio \ -e "FLASK_SERVER_NAME=cdio" \ - -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \ + -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/tmp/proxies.json \ -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \ --rm \ test-changedetectionio \ - bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy_sources.py' + bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004 -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp' echo "socks5 server logs" docker logs socks5proxy diff --git a/changedetectionio/store.py b/changedetectionio/store.py index ec0ba823..d581db17 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -42,17 +42,24 @@ class ChangeDetectionStore: needs_write_urgent = False __version_check = True + save_data_thread = None def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): # Should only be active for docker # logging.basicConfig(filename='/dev/stdout', level=logging.INFO) - self.__data = App.model() - self.datastore_path = datastore_path - self.json_store_path = os.path.join(self.datastore_path, "url-watches.json") - logger.info(f"Datastore path is '{self.json_store_path}'") + self.needs_write = False self.start_time = time.time() self.stop_thread = False + self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag) + + + def reload_state(self, datastore_path, include_default_watches, version_tag): + logger.info(f"Datastore path is '{datastore_path}'") + + self.__data = App.model() + self.datastore_path = datastore_path + self.json_store_path = os.path.join(self.datastore_path, "url-watches.json") # Base definition for all watchers # deepcopy part of #569 - not sure why its needed exactly self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={})) @@ -145,7 +152,10 @@ class ChangeDetectionStore: self.needs_write = True # Finally start the thread that will manage periodic data saves to JSON - save_data_thread = threading.Thread(target=self.save_datastore).start() + # Only start if thread is not already running (reload_state might be called multiple times) + if not self.save_data_thread or not self.save_data_thread.is_alive(): + self.save_data_thread = threading.Thread(target=self.save_datastore) + self.save_data_thread.start() def rehydrate_entity(self, uuid, entity, processor_override=None): """Set the dict back to the dict Watch object""" @@ -411,7 +421,6 @@ class ChangeDetectionStore: self.sync_to_json() return else: - try: # Re #286 - First write to a temp file, then confirm it looks OK and rename it # This is a fairly basic strategy to deal with the case that the file is corrupted, @@ -441,7 +450,7 @@ class ChangeDetectionStore: logger.remove() logger.add(sys.stderr) - logger.critical("Shutting down datastore thread") + logger.info(f"Shutting down datastore '{self.datastore_path}' thread") return if self.needs_write or self.needs_write_urgent: diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py index 7bfe15f1..24984a27 100644 --- a/changedetectionio/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -11,6 +11,7 @@ import os import sys from loguru import logger +from changedetectionio.flask_app import init_app_secret from changedetectionio.tests.util import live_server_setup, new_live_server_setup # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py @@ -87,7 +88,6 @@ def measure_memory_usage(request): def cleanup(datastore_path): import glob # Unlink test output files - for g in ["*.txt", "*.json", "*.pdf"]: files = glob.glob(os.path.join(datastore_path, g)) for f in files: @@ -97,34 +97,121 @@ def cleanup(datastore_path): if os.path.isfile(f): os.unlink(f) -@pytest.fixture(scope='function', autouse=True) -def prepare_test_function(live_server): +def pytest_addoption(parser): + """Add custom command-line options for pytest. + Provides --datastore-path option for specifying custom datastore location. + Note: Cannot use -d short option as it's reserved by pytest for debug mode. + """ + parser.addoption( + "--datastore-path", + action="store", + default=None, + help="Custom datastore path for tests" + ) + +@pytest.fixture(scope='session') +def datastore_path(tmp_path_factory, request): + """Provide datastore path unique to this worker. + + Supports custom path via --datastore-path/-d flag (mirrors main app). + + CRITICAL for xdist isolation: + - Each WORKER gets its own directory + - Tests on same worker run SEQUENTIALLY and cleanup between tests + - No subdirectories needed since tests don't overlap on same worker + - Example: /tmp/test-datastore-gw0/ for worker gw0 + """ + # Check for custom path first (mirrors main app's -d flag) + custom_path = request.config.getoption("--datastore-path") + if custom_path: + # Ensure the directory exists + os.makedirs(custom_path, exist_ok=True) + logger.info(f"Using custom datastore path: {custom_path}") + return custom_path + + # Otherwise use default tmp_path_factory logic + worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master') + if worker_id == 'master': + path = tmp_path_factory.mktemp("test-datastore") + else: + path = tmp_path_factory.mktemp(f"test-datastore-{worker_id}") + return str(path) + + +@pytest.fixture(scope='function', autouse=True) +def prepare_test_function(live_server, datastore_path): + """Prepare each test with complete isolation. + + CRITICAL for xdist per-test isolation: + - Reuses the SAME datastore instance (so blueprint references stay valid) + - Clears all watches and state for a clean slate + - First watch will get uuid="first" + """ routes = [rule.rule for rule in live_server.app.url_map.iter_rules()] if '/test-random-content-endpoint' not in routes: logger.debug("Setting up test URL routes") new_live_server_setup(live_server) + # CRITICAL: Point app to THIS test's unique datastore directory + live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path + + # CRITICAL: Get datastore and stop it from writing stale data + datastore = live_server.app.config.get('DATASTORE') + + # Prevent background thread from writing during cleanup/reload + datastore.needs_write = False + datastore.needs_write_urgent = False + + # CRITICAL: Clean up any files from previous tests + # This ensures a completely clean directory + cleanup(datastore_path) + + # CRITICAL: Reload the EXISTING datastore instead of creating a new one + # This keeps blueprint references valid (they capture datastore at construction) + # reload_state() completely resets the datastore to a clean state + + # Reload state with clean data (no default watches) + datastore.reload_state( + datastore_path=datastore_path, + include_default_watches=False, + version_tag=datastore.data.get('version_tag', '0.0.0') + ) + live_server.app.secret_key = init_app_secret(datastore_path) + logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}") + logger.debug(f"prepare_test_function: Path {datastore.datastore_path}") yield - # Then cleanup/shutdown - live_server.app.config['DATASTORE'].data['watching']={} - time.sleep(0.3) - live_server.app.config['DATASTORE'].data['watching']={} + + # Cleanup: Clear watches again after test + try: + datastore.data['watching'] = {} + datastore.needs_write = True + except Exception as e: + logger.warning(f"Error during datastore cleanup: {e}") + + +# So the app can also know which test name it was +@pytest.fixture(autouse=True) +def set_test_name(request): + """Automatically set TEST_NAME env var for every test""" + test_name = request.node.name + os.environ['PYTEST_CURRENT_TEST'] = test_name + yield + # Cleanup if needed @pytest.fixture(scope='session') -def app(request): - """Create application for the tests.""" - datastore_path = "./test-datastore" +def app(request, datastore_path): + """Create application once per worker (session). + Note: Actual per-test isolation is handled by: + - prepare_test_function() recreates datastore and cleans directory + - All tests on same worker use same directory (cleaned between tests) + """ # So they don't delay in fetching os.environ["MINIMUM_SECONDS_RECHECK_TIME"] = "0" - try: - os.mkdir(datastore_path) - except FileExistsError: - pass - + logger.debug(f"Testing with datastore_path={datastore_path}") cleanup(datastore_path) app_config = {'datastore_path': datastore_path, 'disable_checkver' : True} @@ -147,6 +234,8 @@ def app(request): # Disable CSRF while running tests app.config['WTF_CSRF_ENABLED'] = False app.config['STOP_THREADS'] = True + # Store datastore_path so Flask routes can access it + app.config['TEST_DATASTORE_PATH'] = datastore_path def teardown(): # Stop all threads and services diff --git a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py index 96be7d59..102adbe6 100644 --- a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py +++ b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py @@ -73,13 +73,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False): # Requires playwright to be installed -def test_request_via_custom_browser_url(client, live_server, measure_memory_usage): +def test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # We do this so we can grep the logs of the custom container and see if the request actually went through that container do_test(client, live_server, make_test_use_extra_browser=True) -def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage): +def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # We do this so we can grep the logs of the custom container and see if the request actually went through that container do_test(client, live_server, make_test_use_extra_browser=False) diff --git a/changedetectionio/tests/fetchers/test_content.py b/changedetectionio/tests/fetchers/test_content.py index 37ee6461..c169dc52 100644 --- a/changedetectionio/tests/fetchers/test_content.py +++ b/changedetectionio/tests/fetchers/test_content.py @@ -8,7 +8,7 @@ import logging # Requires playwright to be installed -def test_fetch_webdriver_content(client, live_server, measure_memory_usage): +def test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function ##################### diff --git a/changedetectionio/tests/fetchers/test_custom_js_before_content.py b/changedetectionio/tests/fetchers/test_custom_js_before_content.py index ec41759f..76c89ce2 100644 --- a/changedetectionio/tests/fetchers/test_custom_js_before_content.py +++ b/changedetectionio/tests/fetchers/test_custom_js_before_content.py @@ -3,7 +3,7 @@ from flask import url_for from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client -def test_execute_custom_js(client, live_server, measure_memory_usage): +def test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" diff --git a/changedetectionio/tests/proxy_list/test_multiple_proxy.py b/changedetectionio/tests/proxy_list/test_multiple_proxy.py index be67a9df..b81cd816 100644 --- a/changedetectionio/tests/proxy_list/test_multiple_proxy.py +++ b/changedetectionio/tests/proxy_list/test_multiple_proxy.py @@ -5,7 +5,7 @@ from flask import url_for from ..util import live_server_setup, wait_for_all_checks -def test_preferred_proxy(client, live_server, measure_memory_usage): +def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function url = "http://chosen.changedetection.io" diff --git a/changedetectionio/tests/proxy_list/test_noproxy.py b/changedetectionio/tests/proxy_list/test_noproxy.py index bbdd75dc..c3b4c3e4 100644 --- a/changedetectionio/tests/proxy_list/test_noproxy.py +++ b/changedetectionio/tests/proxy_list/test_noproxy.py @@ -5,7 +5,7 @@ from flask import url_for from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client -def test_noproxy_option(client, live_server, measure_memory_usage): +def test_noproxy_option(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Run by run_proxy_tests.sh # Call this URL then scan the containers that it never went through them diff --git a/changedetectionio/tests/proxy_list/test_proxy.py b/changedetectionio/tests/proxy_list/test_proxy.py index bda17d1b..e9cd08c2 100644 --- a/changedetectionio/tests/proxy_list/test_proxy.py +++ b/changedetectionio/tests/proxy_list/test_proxy.py @@ -5,7 +5,7 @@ from flask import url_for from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client # just make a request, we will grep in the docker logs to see it actually got called -def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage): +def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function res = client.post( url_for("imports.import_page"), diff --git a/changedetectionio/tests/proxy_list/test_proxy_noconnect.py b/changedetectionio/tests/proxy_list/test_proxy_noconnect.py index b53d2777..93b7e8cb 100644 --- a/changedetectionio/tests/proxy_list/test_proxy_noconnect.py +++ b/changedetectionio/tests/proxy_list/test_proxy_noconnect.py @@ -12,7 +12,7 @@ from ... import strtobool # FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py # WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py -def test_proxy_noconnect_custom(client, live_server, measure_memory_usage): +def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Goto settings, add our custom one diff --git a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py index eb29db6b..bddeff27 100644 --- a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py +++ b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py @@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks import os # just make a request, we will grep in the docker logs to see it actually got called -def test_select_custom(client, live_server, measure_memory_usage): +def test_select_custom(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Goto settings, add our custom one @@ -50,7 +50,7 @@ def test_select_custom(client, live_server, measure_memory_usage): # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default -def test_custom_proxy_validation(client, live_server, measure_memory_usage): +def test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Goto settings, add our custom one diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py index 547f6099..850474bb 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py @@ -5,7 +5,7 @@ from flask import url_for from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches -def set_response(): +def set_response(datastore_path): import time data = """ @@ -15,13 +15,13 @@ def set_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) time.sleep(1) -def test_socks5(client, live_server, measure_memory_usage): +def test_socks5(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_response() + set_response(datastore_path) # Setup a proxy res = client.post( diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py index 0893da42..c605fef6 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py @@ -4,7 +4,7 @@ from flask import url_for from changedetectionio.tests.util import live_server_setup, wait_for_all_checks -def set_response(): +def set_response(datastore_path): import time data = """ @@ -14,15 +14,15 @@ def set_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) time.sleep(1) # should be proxies.json mounted from run_proxy_tests.sh already # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json -def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage): +def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_response() + set_response(datastore_path) # Because the socks server should connect back to us test_url = url_for('test_endpoint', _external=True) + f"?socks-test-tag={os.getenv('SOCKSTEST', '')}" test_url = test_url.replace('localhost.localdomain', 'cdio') diff --git a/changedetectionio/tests/restock/test_restock.py b/changedetectionio/tests/restock/test_restock.py index ecee00fa..aaaf5605 100644 --- a/changedetectionio/tests/restock/test_restock.py +++ b/changedetectionio/tests/restock/test_restock.py @@ -11,7 +11,7 @@ from changedetectionio.notification import ( ) -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ @@ -26,13 +26,13 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_back_in_stock_response(): +def set_back_in_stock_response(datastore_path): test_return_data = """ Some initial text
@@ -45,14 +45,14 @@ def set_back_in_stock_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready -def test_restock_detection(client, live_server, measure_memory_usage): +def test_restock_detection(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" # live_server_setup(live_server) # Setup on conftest per function ##################### @@ -88,24 +88,25 @@ def test_restock_detection(client, live_server, measure_memory_usage): assert b'not-in-stock' in res.data # should be out of stock # Is it correctly shown as in stock - set_back_in_stock_response() + set_back_in_stock_response(datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) assert b'not-in-stock' not in res.data # We should have a notification - wait_for_notification_endpoint_output() - assert os.path.isfile("test-datastore/notification.txt"), "Notification received" - os.unlink("test-datastore/notification.txt") + notification_file = os.path.join(datastore_path, "notification.txt") + wait_for_notification_endpoint_output(datastore_path=datastore_path) + assert os.path.isfile(notification_file), "Notification received" + os.unlink(notification_file) # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK # So here there should be no file, because we go IN STOCK -> OUT OF STOCK - set_original_response() + set_original_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(5) - assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default" + assert not os.path.isfile(notification_file), "No notification should have fired when it went OUT OF STOCK by default" # BUT we should see that it correctly shows "not in stock" res = client.get(url_for("watchlist.index")) diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index a963cdd5..02482b9b 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -40,9 +40,10 @@ def get_last_message_from_smtp_server(): # Requires running the test SMTP server -def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage): +def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path): ## live_server_setup(live_server) # Setup on conftest per function - set_original_response() + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -71,7 +72,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas assert b"Watch added" in res.data wait_for_all_checks(client) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) time.sleep(2) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -112,8 +113,9 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas delete_all_watches(client) -def test_check_notification_plaintext_format(client, live_server, measure_memory_usage): - set_original_response() +def test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -138,7 +140,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(2) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -164,8 +166,9 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory -def test_check_notification_html_color_format(client, live_server, measure_memory_usage): - set_original_response() +def test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -195,7 +198,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor assert b"Watch added" in res.data wait_for_all_checks(client) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) time.sleep(2) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -235,8 +238,9 @@ def test_check_notification_html_color_format(client, live_server, measure_memor assert 'some text
' in html_content delete_all_watches(client) -def test_check_notification_markdown_format(client, live_server, measure_memory_usage): - set_original_response() +def test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -266,7 +270,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ assert b"Watch added" in res.data wait_for_all_checks(client) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) time.sleep(2) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -305,12 +309,13 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ delete_all_watches(client) # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent -def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage): +def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path): # HTML problems? see this # https://github.com/caronc/apprise/issues/633 - set_original_response() + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' notification_body = f""" @@ -350,7 +355,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ########## wait_for_all_checks(client) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) time.sleep(2) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -375,7 +380,8 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL #################### - set_original_response() + set_original_response(datastore_path=datastore_path) + # Now override as HTML format res = client.post( url_for("ui.ui_edit.edit_page", uuid="first"), @@ -424,10 +430,11 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv delete_all_watches(client) -def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage): +def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path): """When following a plaintext document, notification in Plain Text format is sent correctly""" + import os - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Some nice plain text\nwhich we add some extra data\nover here\n") notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -454,7 +461,7 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve wait_for_all_checks(client) # Change the content - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about tags\nover here\n") @@ -476,10 +483,11 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve assert '<pre' not in body delete_all_watches(client) -def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage): +def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path): """When following a plaintext document, notification in Plain Text format is sent correctly""" + import os - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(" Some nice plain text\nwhich we add some extra data\nover here\n") notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -506,7 +514,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur wait_for_all_checks(client) # Change the content - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(" Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n") @@ -554,10 +562,11 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur delete_all_watches(client) -def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage): +def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path): """When following a plaintext document, notification in Plain Text format is sent correctly""" + import os - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Some nice plain text\nwhich we add some extra data\nover here\n") notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -585,7 +594,7 @@ def test_check_plaintext_document_html_color_notifications(client, live_server, wait_for_all_checks(client) # Change the content - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n") time.sleep(1) @@ -626,10 +635,11 @@ def test_check_plaintext_document_html_color_notifications(client, live_server, assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py delete_all_watches(client) -def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage): +def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path): """When following a HTML document, notification in Plain Text format is sent correctly""" + import os - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>") notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' @@ -656,7 +666,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br><tag></body></html>") time.sleep(0.1) @@ -682,9 +692,10 @@ def test_check_html_document_plaintext_notification(client, live_server, measure delete_all_watches(client) -def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage): +def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path): ## live_server_setup(live_server) # Setup on conftest per function - set_original_response() + set_original_response(datastore_path=datastore_path) + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html' @@ -713,7 +724,7 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server assert b"Watch added" in res.data wait_for_all_checks(client) - set_longer_modified_response() + set_longer_modified_response(datastore_path=datastore_path) time.sleep(2) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index a3c62309..6a2b680b 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -2,7 +2,7 @@ from .util import live_server_setup, wait_for_all_checks from flask import url_for import time -def test_check_access_control(app, client, live_server, measure_memory_usage): +def test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path): # Still doesnt work, but this is closer. # live_server_setup(live_server) # Setup on conftest per function diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index ef38b9ad..5f715e33 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os.path +import os from flask import url_for from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches @@ -9,7 +10,7 @@ import time from ..diff import ADDED_PLACEMARKER_OPEN -def set_original(excluding=None, add_line=None): +def set_original(datastore_path, excluding=None, add_line=None): test_return_data = """<html> <body> <p>Some initial text</p> @@ -35,16 +36,16 @@ def set_original(excluding=None, add_line=None): test_return_data = output - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage): +def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path): # Give the endpoint time to spin up - set_original() + set_original(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -64,9 +65,10 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory "time_between_check_use_default": "y"}, follow_redirects=True ) + assert b"Updated watch." in res.data wait_for_all_checks(client) - set_original(excluding='Something irrelevant') + set_original(excluding='Something irrelevant', datastore_path=datastore_path) # A line thats not the trigger should not trigger anything res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -77,7 +79,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory assert b'has-unread-changes' not in res.data # The trigger line is REMOVED, this should trigger - set_original(excluding='The golden line') + set_original(excluding='The golden line', datastore_path=datastore_path) # Check in the processor here what's going on, its triggering empty-reply and no change. client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -92,7 +94,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory time.sleep(0.2) time.sleep(1) - set_original(excluding=None) + set_original(excluding=None, datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(1) @@ -100,7 +102,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory assert b'has-unread-changes' not in res.data # Remove it again, and we should get a trigger - set_original(excluding='The golden line') + set_original(excluding='The golden line', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -109,7 +111,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory delete_all_watches(client) -def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage): +def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path): delete_all_watches(client) time.sleep(1) @@ -132,7 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa ) assert b'Settings updated' in res.data - set_original() + set_original(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -155,7 +157,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa ) assert b"Updated watch." in res.data wait_for_all_checks(client) - set_original(excluding='Something irrelevant') + set_original(excluding='Something irrelevant', datastore_path=datastore_path) # A line thats not the trigger should not trigger anything res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -166,7 +168,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa assert b'has-unread-changes' not in res.data # The trigger line is ADDED, this should trigger - set_original(add_line='<p>Oh yes please</p>') + set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -174,9 +176,9 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa assert b'has-unread-changes' in res.data # Takes a moment for apprise to fire - wait_for_notification_endpoint_output() - assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file" - with open("test-datastore/notification.txt", 'rb') as f: + wait_for_notification_endpoint_output(datastore_path=datastore_path) + assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification fired because I can see the output file" + with open(os.path.join(datastore_path, "notification.txt"), 'rb') as f: response = f.read() assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response # _apply_diff_filtering shouldnt add something here assert b'-Oh yes please' in response diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index 2e48395b..5b239605 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -3,12 +3,13 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches +import os import json import uuid -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """<html> <body> Some initial text<br> @@ -21,12 +22,12 @@ def set_original_response(): </html> """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """<html> <body> Some initial text<br> @@ -39,7 +40,7 @@ def set_modified_response(): </html> """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None @@ -52,17 +53,17 @@ def is_valid_uuid(val): return False -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_api_simple(client, live_server, measure_memory_usage): +def test_api_simple(client, live_server, measure_memory_usage, datastore_path): api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') # Create a watch - set_original_response() + set_original_response(datastore_path=datastore_path) # Validate bad URL test_url = url_for('test_endpoint', _external=True ) @@ -111,7 +112,7 @@ def test_api_simple(client, live_server, measure_memory_usage): time.sleep(1) wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger recheck of all ?recheck_all=1 client.get( url_for("createwatch", recheck_all='1'), @@ -244,7 +245,7 @@ def test_api_simple(client, live_server, measure_memory_usage): ) assert len(res.json) == 0, "Watch list should be empty" -def test_access_denied(client, live_server, measure_memory_usage): +def test_access_denied(client, live_server, measure_memory_usage, datastore_path): # `config_api_token_enabled` Should be On by default res = client.get( url_for("createwatch") @@ -289,11 +290,11 @@ def test_access_denied(client, live_server, measure_memory_usage): ) assert b"Settings updated." in res.data -def test_api_watch_PUT_update(client, live_server, measure_memory_usage): +def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path): api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') # Create a watch - set_original_response() + set_original_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) # Create new @@ -398,7 +399,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_api_import(client, live_server, measure_memory_usage): +def test_api_import(client, live_server, measure_memory_usage, datastore_path): api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -420,7 +421,7 @@ def test_api_import(client, live_server, measure_memory_usage): res = client.get(url_for('tags.tags_overview_page')) assert b'import-test' in res.data -def test_api_conflict_UI_password(client, live_server, measure_memory_usage): +def test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path): api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -438,7 +439,7 @@ def test_api_conflict_UI_password(client, live_server, measure_memory_usage): assert b"Password protection enabled." in res.data # Create a watch - set_original_response() + set_original_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) # Create new diff --git a/changedetectionio/tests/test_api_notifications.py b/changedetectionio/tests/test_api_notifications.py index d5b97952..d75b7e66 100644 --- a/changedetectionio/tests/test_api_notifications.py +++ b/changedetectionio/tests/test_api_notifications.py @@ -4,7 +4,7 @@ from flask import url_for from .util import live_server_setup import json -def test_api_notifications_crud(client, live_server, measure_memory_usage): +def test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') diff --git a/changedetectionio/tests/test_api_openapi.py b/changedetectionio/tests/test_api_openapi.py index 22f440dd..2adc2fbc 100644 --- a/changedetectionio/tests/test_api_openapi.py +++ b/changedetectionio/tests/test_api_openapi.py @@ -12,7 +12,7 @@ from flask import url_for from .util import live_server_setup, wait_for_all_checks -def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage): +def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path): """Test that creating a watch with invalid content-type triggers OpenAPI validation error.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -29,7 +29,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" -def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage): +def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path): """Test that creating a watch without required URL field triggers OpenAPI validation error.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -46,7 +46,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" -def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage): +def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path): """Test that including invalid fields triggers OpenAPI validation error.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -83,7 +83,7 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties" -def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage): +def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path): """Test that import endpoint with wrong content-type triggers OpenAPI validation error.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -100,7 +100,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" -def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage): +def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path): """Test that import endpoint with correct content-type succeeds (positive test).""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -117,7 +117,7 @@ def test_openapi_validation_import_correct_content_type_succeeds(client, live_se assert len(res.json) == 2, "Should import 2 URLs" -def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage): +def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path): """Test that GET requests bypass OpenAPI validation entirely.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -141,7 +141,7 @@ def test_openapi_validation_get_requests_bypass_validation(client, live_server, assert isinstance(res.json, dict), "Should return JSON dictionary for watch list" -def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage): +def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path): """Test that creating a tag without required title triggers OpenAPI validation error.""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') @@ -158,7 +158,7 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" -def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage): +def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path): """Test that watch updates allow partial updates without requiring all fields (positive test).""" api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') diff --git a/changedetectionio/tests/test_api_search.py b/changedetectionio/tests/test_api_search.py index 1863e3bf..845c7ac8 100644 --- a/changedetectionio/tests/test_api_search.py +++ b/changedetectionio/tests/test_api_search.py @@ -6,7 +6,7 @@ import time from .util import live_server_setup, wait_for_all_checks -def test_api_search(client, live_server, measure_memory_usage): +def test_api_search(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') diff --git a/changedetectionio/tests/test_api_tags.py b/changedetectionio/tests/test_api_tags.py index fcae1546..2404c6d1 100644 --- a/changedetectionio/tests/test_api_tags.py +++ b/changedetectionio/tests/test_api_tags.py @@ -5,13 +5,14 @@ from .util import live_server_setup, wait_for_all_checks, set_original_response import json import time -def test_api_tags_listing(client, live_server, measure_memory_usage): +def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') tag_title = 'Test Tag' - set_original_response() + set_original_response(datastore_path=datastore_path) + res = client.get( url_for("tags"), diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py index 33266a5b..4ac30ad8 100644 --- a/changedetectionio/tests/test_auth.py +++ b/changedetectionio/tests/test_auth.py @@ -5,7 +5,7 @@ from flask import url_for from .util import live_server_setup, wait_for_all_checks # test pages with http://username@password:foobar.com/ work -def test_basic_auth(client, live_server, measure_memory_usage): +def test_basic_auth(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function diff --git a/changedetectionio/tests/test_automatic_follow_ldjson_price.py b/changedetectionio/tests/test_automatic_follow_ldjson_price.py index b4b674cc..264fc2f0 100644 --- a/changedetectionio/tests/test_automatic_follow_ldjson_price.py +++ b/changedetectionio/tests/test_automatic_follow_ldjson_price.py @@ -3,9 +3,10 @@ import time from flask import url_for from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks +import os -def set_response_with_ldjson(): +def set_response_with_ldjson(datastore_path): test_return_data = """<html> <body> Some initial text<br> @@ -55,11 +56,11 @@ def set_response_with_ldjson(): </html> """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_response_without_ldjson(): +def set_response_without_ldjson(datastore_path): test_return_data = """<html> <body> Some initial text<br> @@ -72,17 +73,17 @@ def set_response_without_ldjson(): </html> """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # actually only really used by the distll.io importer, but could be handy too -def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage): +def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage, datastore_path): - set_response_with_ldjson() + set_response_with_ldjson(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -121,7 +122,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage ########################################################################################## # And we shouldnt see the offer - set_response_without_ldjson() + set_response_without_ldjson(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -151,7 +152,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_ client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage): +def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage, datastore_path): test_return_data = """ <html> @@ -181,7 +182,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa <div class="yes">Some extra stuff</div> </body></html> """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True) @@ -215,7 +216,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa # <div class="yes">Some extra stuff</div> # </body></html> # """ - # with open("test-datastore/endpoint-content.txt", "w") as f: + # with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: # f.write(test_return_data) # # _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False) diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index bd56f91e..e2faaf19 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import os import time from flask import url_for @@ -16,8 +17,8 @@ def test_inscriptus(): assert stripped_text_from_html == 'test!\nok man' -def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage): - set_original_response() +def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # live_server_setup(live_server) # Setup on conftest per function # Add our URL to the import page @@ -60,7 +61,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure assert b'foobar-detection' not in res.data # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Force recheck res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -121,7 +122,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure assert b'test-endpoint' in res.data # Recheck it but only with a title change, content wasnt changed - set_original_response(extra_title=" and more") + set_original_response(datastore_path=datastore_path, extra_title=" and more") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -167,7 +168,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that -def test_requests_timeout(client, live_server, measure_memory_usage): +def test_requests_timeout(client, live_server, measure_memory_usage, datastore_path): delay = 2 test_url = url_for('test_endpoint', delay=delay, _external=True) @@ -205,7 +206,7 @@ def test_requests_timeout(client, live_server, measure_memory_usage): res = client.get(url_for("watchlist.index")) assert b'Read timed out' not in res.data -def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage): +def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path): """ https://github.com/dgtlmoon/changedetection.io/issues/3434 @@ -220,7 +221,7 @@ def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage): :param measure_memory_usage: :return: """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("""some random text that should be split by line and not parsed with html_to_text this way we know that it correctly parsed as plain text @@ -264,7 +265,7 @@ got it\r\n delete_all_watches(client) -def test_standard_text_plain(client, live_server, measure_memory_usage): +def test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path): """ https://github.com/dgtlmoon/changedetection.io/issues/3434 @@ -279,7 +280,7 @@ def test_standard_text_plain(client, live_server, measure_memory_usage): :param measure_memory_usage: :return: """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("""some random text that should be split by line and not parsed with html_to_text <title>Even this title should stay because we are just plain text @@ -325,9 +326,9 @@ got it\r\n delete_all_watches(client) # Server says its plaintext, we should always treat it as plaintext -def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage): +def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path): - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(""" @@ -353,10 +354,10 @@ def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage delete_all_watches(client) # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that -def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage): +def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path): - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(""" diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py index ca1a0c6f..e7bbc7a1 100644 --- a/changedetectionio/tests/test_backup.py +++ b/changedetectionio/tests/test_backup.py @@ -8,13 +8,11 @@ import re import time -def test_backup(client, live_server, measure_memory_usage): +def test_backup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_original_response() + set_original_response(datastore_path=datastore_path) - # Give the endpoint time to spin up - time.sleep(1) # Add our URL to the import page res = client.post( @@ -31,7 +29,7 @@ def test_backup(client, live_server, measure_memory_usage): url_for("backups.request_backup"), follow_redirects=True ) - time.sleep(2) + time.sleep(4) res = client.get( url_for("backups.index"), diff --git a/changedetectionio/tests/test_basic_socketio.py b/changedetectionio/tests/test_basic_socketio.py index 120907dd..daf6ba3e 100644 --- a/changedetectionio/tests/test_basic_socketio.py +++ b/changedetectionio/tests/test_basic_socketio.py @@ -10,11 +10,12 @@ from .util import ( ) from loguru import logger -def run_socketio_watch_update_test(client, live_server, password_mode=""): +def run_socketio_watch_update_test(client, live_server, password_mode="", datastore_path=""): """Test that the socketio emits a watch update event when content changes""" # Set up the test server - set_original_response() + set_original_response(datastore_path=datastore_path) + # Get the SocketIO instance from the app from changedetectionio.flask_app import app @@ -47,7 +48,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""): socketio_test_client.get_received() # Make a change to trigger an update - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Force recheck res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -105,11 +106,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""): # Clean up client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_everything(live_server, client): +def test_everything(live_server, client, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client) + run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client, datastore_path=datastore_path) ############################ Password required auth check ############################## @@ -124,7 +125,7 @@ def test_everything(live_server, client): assert b"Password protection enabled." in res.data - run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client) + run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client, datastore_path=datastore_path) res = client.post( url_for("login"), data={"password": "foobar"}, @@ -133,4 +134,4 @@ def test_everything(live_server, client): # Yes we are correctly logged in assert b"LOG OUT" in res.data - run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client) + run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client, datastore_path=datastore_path) diff --git a/changedetectionio/tests/test_block_while_text_present.py b/changedetectionio/tests/test_block_while_text_present.py index 9580d727..485732ed 100644 --- a/changedetectionio/tests/test_block_while_text_present.py +++ b/changedetectionio/tests/test_block_while_text_present.py @@ -4,8 +4,9 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches from changedetectionio import html_tools +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -17,11 +18,11 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_original_ignore_response(): +def set_modified_original_ignore_response(datastore_path): test_return_data = """ Some NEW nice initial text
@@ -36,12 +37,12 @@ def set_modified_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text -def set_modified_response_minus_block_text(): +def set_modified_response_minus_block_text(datastore_path): test_return_data = """ Some NEW nice initial text
@@ -56,16 +57,16 @@ def set_modified_response_minus_block_text(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage): +def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Use a mix of case in ZzZ to prove it works case-insensitive. ignore_text = "out of stoCk\r\nfoobar" - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page @@ -109,7 +110,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu assert b'/test-endpoint' in res.data # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change - set_modified_original_ignore_response() + set_modified_original_ignore_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -123,7 +124,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu # 2548 # Going back to the ORIGINAL should NOT trigger a change - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -131,10 +132,11 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu # Now we set a change where the text is gone AND its different content, it should now trigger - set_modified_response_minus_block_text() + set_modified_response_minus_block_text(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) + assert b'has-unread-changes' in res.data diff --git a/changedetectionio/tests/test_clone.py b/changedetectionio/tests/test_clone.py index 970b43c7..02231301 100644 --- a/changedetectionio/tests/test_clone.py +++ b/changedetectionio/tests/test_clone.py @@ -3,12 +3,13 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks +import os -def test_clone_functionality(client, live_server, measure_memory_usage): +def test_clone_functionality(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Some content") test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index ac3deb7f..9bebd6ef 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import json import time +import os from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT -def set_original_response(number="50"): +def set_original_response(datastore_path, number="50"): test_return_data = f"""

Test Page for Conditions

@@ -17,10 +18,10 @@ def set_original_response(number="50"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_number_in_range_response(number="75"): +def set_number_in_range_response(datastore_path, number="75"): test_return_data = f"""

Test Page for Conditions

@@ -30,10 +31,10 @@ def set_number_in_range_response(number="75"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_number_out_of_range_response(number="150"): +def set_number_out_of_range_response(datastore_path, number="150"): test_return_data = f"""

Test Page for Conditions

@@ -43,18 +44,18 @@ def set_number_out_of_range_response(number="150"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): """Test that both text and number conditions work together with AND logic.""" # live_server_setup(live_server) # Setup on conftest per function -def test_conditions_with_text_and_number(client, live_server, measure_memory_usage): +def test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path): """Test that both text and number conditions work together with AND logic.""" - set_original_response("50") + set_original_response(datastore_path=datastore_path, number="50") test_url = url_for('test_endpoint', _external=True) @@ -114,7 +115,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa wait_for_all_checks(client) # Case 1 - set_number_in_range_response("70.5") + set_number_in_range_response(datastore_path=datastore_path, number="70.5") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -129,7 +130,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) time.sleep(0.2) - set_number_out_of_range_response("150.5") + set_number_out_of_range_response(datastore_path=datastore_path, number="150.5") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -142,9 +143,9 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa delete_all_watches(client) # The 'validate' button next to each rule row -def test_condition_validate_rule_row(client, live_server, measure_memory_usage): +def test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path): - set_original_response("50") + set_original_response(datastore_path=datastore_path, number="50") test_url = url_for('test_endpoint', _external=True) @@ -203,7 +204,7 @@ def test_condition_validate_rule_row(client, live_server, measure_memory_usage): # If there was only a change in the whitespacing, then we shouldnt have a change detected -def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage): +def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path): test_return_data = """ @@ -216,7 +217,7 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # Add our URL to the import page @@ -242,10 +243,10 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage): ) # If there was only a change in the whitespacing, then we shouldnt have a change detected -def test_lev_conditions_plugin(client, live_server, measure_memory_usage): - +def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path): + # This should break.. - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(""" Some initial text
@@ -297,7 +298,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage): ############### Now change it a LITTLE bit... - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(""" Some initial text
@@ -326,7 +327,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 1 watch for rechecking.' in res.data diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index 09442490..46605f0c 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -3,12 +3,13 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks +import os from ..html_tools import * -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ Some initial text
@@ -21,11 +22,11 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ Some initial text
@@ -38,7 +39,7 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None @@ -69,12 +70,12 @@ def test_include_filters_output(): # Tests the whole stack works with the CSS Filter -def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage): +def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path): sleep_time_for_fetch_thread = 3 include_filters = "#sametext" - set_original_response() + set_original_response(datastore_path=datastore_path) # Give the endpoint time to spin up time.sleep(1) @@ -105,7 +106,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -119,11 +120,11 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m # Tests the whole stack works with the CSS Filter -def test_check_multiple_filters(client, live_server, measure_memory_usage): +def test_check_multiple_filters(client, live_server, measure_memory_usage, datastore_path): include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]" - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("""
Blob A
Blob B
@@ -168,12 +169,12 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage): # The filter exists, but did not contain anything useful # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text -def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage): +def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage, datastore_path): include_filters = "#blob-a" - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("""
@@ -216,7 +217,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa ### Just an empty selector, no image - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("""
diff --git a/changedetectionio/tests/test_datastore_isolation.py b/changedetectionio/tests/test_datastore_isolation.py new file mode 100644 index 00000000..3eef00f9 --- /dev/null +++ b/changedetectionio/tests/test_datastore_isolation.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Test to verify client and live_server share the same datastore""" + +def test_client_and_live_server_share_datastore(client, live_server): + """Verify that client and live_server use the same app and datastore.""" + + # They should be the SAME object + assert client.application is live_server.app, "client.application and live_server.app should be the SAME object!" + + # They should share the same datastore + client_datastore = client.application.config.get('DATASTORE') + server_datastore = live_server.app.config.get('DATASTORE') + + assert client_datastore is server_datastore, \ + f"Datastores are DIFFERENT objects! client={hex(id(client_datastore))} server={hex(id(server_datastore))}" + + print(f"✓ client.application and live_server.app are the SAME object") + print(f"✓ Both use the same DATASTORE at {hex(id(client_datastore))}") + print(f"✓ Datastore path: {client_datastore.datastore_path}") diff --git a/changedetectionio/tests/test_element_removal.py b/changedetectionio/tests/test_element_removal.py index d06c468c..bc9503be 100644 --- a/changedetectionio/tests/test_element_removal.py +++ b/changedetectionio/tests/test_element_removal.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import time +import os from flask import url_for @@ -10,7 +11,7 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches -def set_response_with_multiple_index(): +def set_response_with_multiple_index(datastore_path): data= """ @@ -36,11 +37,11 @@ def set_response_with_multiple_index(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """

Header

@@ -65,11 +66,11 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """

Header changed

@@ -94,7 +95,7 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) @@ -146,10 +147,10 @@ across multiple lines ) -def test_element_removal_full(client, live_server, measure_memory_usage): +def test_element_removal_full(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) # Add our URL to the import page @@ -194,7 +195,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage): client.get(url_for("ui.ui_views.diff_history_page", uuid="first")) # Make a change to header/footer/nav - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -208,9 +209,9 @@ def test_element_removal_full(client, live_server, measure_memory_usage): assert b"unviewed" not in res.data # Re #2752 -def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage): +def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage, datastore_path): - set_response_with_multiple_index() + set_response_with_multiple_index(datastore_path=datastore_path) subtractive_selectors_data = [ ### css style ### """body > table > tr:nth-child(1) > th:nth-child(2) diff --git a/changedetectionio/tests/test_encoding.py b/changedetectionio/tests/test_encoding.py index 35704797..8a050ea4 100644 --- a/changedetectionio/tests/test_encoding.py +++ b/changedetectionio/tests/test_encoding.py @@ -5,26 +5,27 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client import pytest +import os -def set_html_response(): +def set_html_response(datastore_path): test_return_data = """        铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。 """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None # In the case the server does not issue a charset= or doesnt have content_type header set -def test_check_encoding_detection(client, live_server, measure_memory_usage): - set_html_response() +def test_check_encoding_detection(client, live_server, measure_memory_usage, datastore_path): + set_html_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', content_type="text/html", _external=True) @@ -51,8 +52,8 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage): # In the case the server does not issue a charset= or doesnt have content_type header set -def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage): - set_html_response() +def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage, datastore_path): + set_html_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_errorhandling.py b/changedetectionio/tests/test_errorhandling.py index 52cc5055..66438e3d 100644 --- a/changedetectionio/tests/test_errorhandling.py +++ b/changedetectionio/tests/test_errorhandling.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import time +import os from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches @@ -8,9 +9,9 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches -def _runner_test_http_errors(client, live_server, http_code, expected_text): +def _runner_test_http_errors(client, live_server, http_code, expected_text, datastore_path): - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Now you going to get a {} error code\n".format(http_code)) @@ -46,17 +47,15 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text): delete_all_watches(client) -def test_http_error_handler(client, live_server, measure_memory_usage): - _runner_test_http_errors(client, live_server, 403, 'Access denied') - _runner_test_http_errors(client, live_server, 404, 'Page not found') - _runner_test_http_errors(client, live_server, 500, '(Internal server error) received') - _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400') +def test_http_error_handler(client, live_server, measure_memory_usage, datastore_path): + _runner_test_http_errors(client, live_server, 403, 'Access denied', datastore_path=datastore_path) + _runner_test_http_errors(client, live_server, 404, 'Page not found', datastore_path=datastore_path) + _runner_test_http_errors(client, live_server, 500, '(Internal server error) received', datastore_path=datastore_path) + _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400', datastore_path=datastore_path) delete_all_watches(client) # Just to be sure error text is properly handled -def test_DNS_errors(client, live_server, measure_memory_usage): - # Give the endpoint time to spin up - time.sleep(1) +def test_DNS_errors(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page res = client.post( @@ -84,12 +83,9 @@ def test_DNS_errors(client, live_server, measure_memory_usage): delete_all_watches(client) # Re 1513 -def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage): - - # Give the endpoint time to spin up - time.sleep(1) +def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage, datastore_path): - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("
Hello world
") # Add our URL to the import page diff --git a/changedetectionio/tests/test_extract_csv.py b/changedetectionio/tests/test_extract_csv.py index edbb415b..1ca4f83b 100644 --- a/changedetectionio/tests/test_extract_csv.py +++ b/changedetectionio/tests/test_extract_csv.py @@ -4,14 +4,15 @@ import time from flask import url_for from urllib.request import urlopen from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks +import os sleep_time_for_fetch_thread = 3 -def test_check_extract_text_from_diff(client, live_server, measure_memory_usage): +def test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path): import time - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time()))) # live_server_setup(live_server) # Setup on conftest per function @@ -33,7 +34,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage) # Give the thread time to pick it up print("Bumping snapshot and checking.. ", n) last_date = str(time.time()) - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("Now it's {} seconds since epoch, time flies!".format(last_date)) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_extract_regex.py b/changedetectionio/tests/test_extract_regex.py index b398de9c..1983b8fa 100644 --- a/changedetectionio/tests/test_extract_regex.py +++ b/changedetectionio/tests/test_extract_regex.py @@ -3,11 +3,12 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches +import os from ..html_tools import * -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ Some initial text
@@ -20,12 +21,12 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ Some initial text
@@ -39,13 +40,13 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_multiline_response(): +def set_multiline_response(datastore_path): test_return_data = """ @@ -61,18 +62,18 @@ def set_multiline_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_check_filter_multiline(client, live_server, measure_memory_usage): +def test_check_filter_multiline(client, live_server, measure_memory_usage, datastore_path): ## live_server_setup(live_server) # Setup on conftest per function - set_multiline_response() + set_multiline_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -119,11 +120,11 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage): # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking) assert b'aaand something lines' not in res.data -def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage): +def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage, datastore_path): include_filters = ".changetext" - set_original_response() + set_original_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -159,7 +160,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag assert b'not at the start of the expression' not in res.data # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -198,7 +199,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag -def test_regex_error_handling(client, live_server, measure_memory_usage): +def test_regex_error_handling(client, live_server, measure_memory_usage, datastore_path): diff --git a/changedetectionio/tests/test_filter_exist_changes.py b/changedetectionio/tests/test_filter_exist_changes.py index 157dc786..d5dccd72 100644 --- a/changedetectionio/tests/test_filter_exist_changes.py +++ b/changedetectionio/tests/test_filter_exist_changes.py @@ -8,7 +8,7 @@ from .util import set_original_response, live_server_setup, wait_for_notificatio from changedetectionio.model import App -def set_response_without_filter(): +def set_response_without_filter(datastore_path): test_return_data = """ Some initial text
@@ -20,12 +20,12 @@ def set_response_without_filter(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_response_with_filter(): +def set_response_with_filter(datastore_path): test_return_data = """ Some initial text
@@ -37,11 +37,11 @@ def set_response_with_filter(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage): +def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage, datastore_path): # Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again # And the page has that filter available # Then I should get a notification @@ -50,7 +50,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se # Give the endpoint time to spin up time.sleep(1) - set_response_without_filter() + set_response_without_filter(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -105,20 +105,20 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se follow_redirects=True ) assert b"Updated watch." in res.data - wait_for_notification_endpoint_output() + wait_for_notification_endpoint_output(datastore_path=datastore_path) # Shouldn't exist, shouldn't have fired - assert not os.path.isfile("test-datastore/notification.txt") + assert not os.path.isfile(os.path.join(datastore_path, "notification.txt")) # Now the filter should exist - set_response_with_filter() + set_response_with_filter(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) - wait_for_notification_endpoint_output() + wait_for_notification_endpoint_output(datastore_path=datastore_path) - assert os.path.isfile("test-datastore/notification.txt") + assert os.path.isfile(os.path.join(datastore_path, "notification.txt")) - with open("test-datastore/notification.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: notification = f.read() assert 'Ticket now on sale' in notification - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) diff --git a/changedetectionio/tests/test_filter_failure_notification.py b/changedetectionio/tests/test_filter_failure_notification.py index 875e0b21..99290b1d 100644 --- a/changedetectionio/tests/test_filter_failure_notification.py +++ b/changedetectionio/tests/test_filter_failure_notification.py @@ -5,7 +5,7 @@ from .util import set_original_response, wait_for_all_checks, wait_for_notifica from ..notification import valid_notification_formats -def set_response_with_filter(): +def set_response_with_filter(datastore_path): test_return_data = """ Some initial text
@@ -17,14 +17,14 @@ def set_response_with_filter(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def run_filter_test(client, live_server, content_filter, app_notification_format): +def run_filter_test(client, live_server, content_filter, app_notification_format, datastore_path): # Response WITHOUT the filter ID element - set_original_response() + set_original_response(datastore_path=datastore_path) live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format # Goto the edit page, add our ignore text @@ -38,10 +38,16 @@ def run_filter_test(client, live_server, content_filter, app_notification_format url_for("ui.form_delete", uuid="all"), follow_redirects=True ) - if os.path.isfile("test-datastore/notification.txt"): - os.unlink("test-datastore/notification.txt") + notification_file = os.path.join(datastore_path, "notification.txt") + if os.path.isfile(notification_file): + os.unlink(notification_file) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) + res = client.get(url_for("watchlist.index")) + + assert b'No website watches configured' not in res.data + + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -79,6 +85,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format data=watch_data, follow_redirects=True ) + assert b"Updated watch." in res.data wait_for_all_checks(client) assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure" @@ -95,7 +102,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format # It should have checked once so far and given this error (because we hit SAVE) wait_for_all_checks(client) - assert not os.path.isfile("test-datastore/notification.txt") + assert not os.path.isfile(notification_file) # Hitting [save] would have triggered a recheck, and we have a filter, so this would be ONE failure assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 1, "Should have been checked once" @@ -110,20 +117,20 @@ def run_filter_test(client, live_server, content_filter, app_notification_format wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) assert b'Warning, no filters were found' in res.data - assert not os.path.isfile("test-datastore/notification.txt") + assert not os.path.isfile(notification_file) time.sleep(1) - + assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5 time.sleep(2) # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - wait_for_notification_endpoint_output() + wait_for_notification_endpoint_output(datastore_path=datastore_path) # Now it should exist and contain our "filter not found" alert - assert os.path.isfile("test-datastore/notification.txt") - with open("test-datastore/notification.txt", 'r') as f: + assert os.path.isfile(notification_file) + with open(notification_file, 'r') as f: notification = f.read() assert 'Your configured CSS/xPath filters' in notification @@ -146,19 +153,19 @@ def run_filter_test(client, live_server, content_filter, app_notification_format # Remove it and prove that it doesn't trigger when not expected # It should register a change, but no 'filter not found' - os.unlink("test-datastore/notification.txt") - set_response_with_filter() + os.unlink(notification_file) + set_response_with_filter(datastore_path) # Try several times, it should NOT have 'filter not found' for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2): client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - wait_for_notification_endpoint_output() + wait_for_notification_endpoint_output(datastore_path=datastore_path) # It should have sent a notification, but.. - assert os.path.isfile("test-datastore/notification.txt") + assert os.path.isfile(notification_file) # but it should not contain the info about a failed filter (because there was none in this case) - with open("test-datastore/notification.txt", 'r') as f: + with open(notification_file, 'r') as f: notification = f.read() assert not 'CSS/xPath filter was not present in the page' in notification @@ -170,22 +177,22 @@ def run_filter_test(client, live_server, content_filter, app_notification_format url_for("ui.form_delete", uuid="all"), follow_redirects=True ) - os.unlink("test-datastore/notification.txt") + os.unlink(notification_file) -def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage): +def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage, datastore_path): # # live_server_setup(live_server) # Setup on conftest per function - run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor')) + run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path) # Check markup send conversion didnt affect plaintext preference - run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text')) + run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'), datastore_path=datastore_path) -def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage): +def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage, datastore_path): # # live_server_setup(live_server) # Setup on conftest per function - run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor')) + run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path) # Test that notification is never sent -def test_basic_markup_from_text(client, live_server, measure_memory_usage): +def test_basic_markup_from_text(client, live_server, measure_memory_usage, datastore_path): # Test the notification error templates convert to HTML if needed (link activate) from ..notification.handler import markup_text_links_to_html x = markup_text_links_to_html("hello https://google.com") diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py index 186a641c..64524f37 100644 --- a/changedetectionio/tests/test_group.py +++ b/changedetectionio/tests/test_group.py @@ -6,10 +6,10 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from import os -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ Some initial text
@@ -20,11 +20,11 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ Some initial text
@@ -35,13 +35,13 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def test_setup_group_tag(client, live_server, measure_memory_usage): +def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) # Add a tag with some config, import a tag and it should roughly work res = client.post( @@ -116,7 +116,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage): ) assert b"1 Imported" in res.data wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) rss_token = extract_rss_token_from_UI(client) @@ -129,7 +129,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage): assert b"first-imported=1" in res.data delete_all_watches(client) -def test_tag_import_singular(client, live_server, measure_memory_usage): +def test_tag_import_singular(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_endpoint', _external=True) @@ -148,7 +148,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage): assert res.data.count(b'test-tag') == 1 delete_all_watches(client) -def test_tag_add_in_ui(client, live_server, measure_memory_usage): +def test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path): # res = client.post( @@ -164,9 +164,9 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_group_tag_notification(client, live_server, measure_memory_usage): +def test_group_tag_notification(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) res = client.post( @@ -207,16 +207,16 @@ def test_group_tag_notification(client, live_server, measure_memory_usage): wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(3) - assert os.path.isfile("test-datastore/notification.txt") + assert os.path.isfile(os.path.join(datastore_path, "notification.txt")) # Verify what was sent as a notification, this file should exist - with open("test-datastore/notification.txt", "r") as f: + with open(os.path.join(datastore_path, "notification.txt"), "r") as f: notification_submission = f.read() - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) # Did we see the URL that had a change, in the notification? # Diff was correctly executed @@ -231,7 +231,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage): #@todo Test that each of multiple notifications with different settings delete_all_watches(client) -def test_limit_tag_ui(client, live_server, measure_memory_usage): +def test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_random_content_endpoint', _external=True) @@ -269,7 +269,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage): res = client.get(url_for("tags.delete_all"), follow_redirects=True) assert b'All tags deleted' in res.data -def test_clone_tag_on_import(client, live_server, measure_memory_usage): +def test_clone_tag_on_import(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_endpoint', _external=True) res = client.post( @@ -294,7 +294,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage): assert res.data.count(b'another-tag') == 3 delete_all_watches(client) -def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage): +def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_endpoint', _external=True) @@ -324,7 +324,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa res = client.get(url_for("tags.delete_all"), follow_redirects=True) assert b'All tags deleted' in res.data -def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage): +def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage, datastore_path): # Add a tag with some config, import a tag and it should roughly work res = client.post( @@ -378,7 +378,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(d) test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_history_consistency.py b/changedetectionio/tests/test_history_consistency.py index 280ad131..16baed39 100644 --- a/changedetectionio/tests/test_history_consistency.py +++ b/changedetectionio/tests/test_history_consistency.py @@ -7,7 +7,7 @@ from flask import url_for from .util import wait_for_all_checks, delete_all_watches from urllib.parse import urlparse, parse_qs -def test_consistent_history(client, live_server, measure_memory_usage): +def test_consistent_history(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function workers = int(os.getenv("FETCH_WORKERS", 10)) r = range(1, 10+workers) @@ -80,9 +80,9 @@ def test_consistent_history(client, live_server, measure_memory_usage): assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved" -def test_check_text_history_view(client, live_server, measure_memory_usage): +def test_check_text_history_view(client, live_server, measure_memory_usage, datastore_path): - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("test-one") # Add our URL to the import page @@ -94,7 +94,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage): wait_for_all_checks(client) # Set second version, Make a change - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("test-two") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -105,7 +105,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage): assert b'test-two' in res.data # Set third version, Make a change - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("test-three") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_ignore.py b/changedetectionio/tests/test_ignore.py index ac456ec7..a85f4ee5 100644 --- a/changedetectionio/tests/test_ignore.py +++ b/changedetectionio/tests/test_ignore.py @@ -5,8 +5,9 @@ from flask import url_for from .util import live_server_setup, wait_for_all_checks from changedetectionio import html_tools from . util import extract_UUID_from_client +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -19,13 +20,13 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_ignore(client, live_server, measure_memory_usage): +def test_ignore(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_original_ignore_response() + set_original_ignore_response(datastore_path) test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -55,9 +56,9 @@ def test_ignore(client, live_server, measure_memory_usage): assert b'csrftoken' in res.data -def test_strip_ignore_lines(client, live_server, measure_memory_usage): +def test_strip_ignore_lines(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_original_ignore_response() + set_original_ignore_response(datastore_path) # Goto the settings page, add our ignore text diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index e575f866..f8003b59 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -4,6 +4,7 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches from changedetectionio import html_tools +import os @@ -31,7 +32,7 @@ def test_strip_text_func(): stripped_content = html_tools.strip_ignore_text(test_content, ignore) assert stripped_content == "Some initial text\n\nWhich is across multiple lines\n\n\n\nSo let's see what happens." -def set_original_ignore_response(ver_stamp="123"): +def set_original_ignore_response(datastore_path, ver_stamp="123"): test_return_data = f""" Some initial text
@@ -44,11 +45,11 @@ def set_original_ignore_response(ver_stamp="123"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_original_ignore_response(ver_stamp="123"): +def set_modified_original_ignore_response(datastore_path, ver_stamp="123"): test_return_data = f""" Some NEW nice initial text
@@ -63,12 +64,12 @@ def set_modified_original_ignore_response(ver_stamp="123"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text -def set_modified_ignore_response(ver_stamp="123"): +def set_modified_ignore_response(datastore_path, ver_stamp="123"): test_return_data = f""" Some initial text
@@ -82,17 +83,17 @@ def set_modified_ignore_response(ver_stamp="123"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # Ignore text now just removes it entirely, is a LOT more simpler code this way -def test_check_ignore_text_functionality(client, live_server, measure_memory_usage): +def test_check_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path): # Use a mix of case in ZzZ to prove it works case-insensitive. ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff" - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page @@ -130,7 +131,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa assert b'/test-endpoint' in res.data # Make a change - set_modified_ignore_response() + set_modified_ignore_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -145,7 +146,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa # Just to be sure.. set a regular modified change.. - set_modified_original_ignore_response() + set_modified_original_ignore_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -162,10 +163,10 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa delete_all_watches(client) # When adding some ignore text, it should not trigger a change, even if something else on that line changes -def _run_test_global_ignore(client, as_source=False, extra_ignore=""): +def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignore=""): ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\n"+extra_ignore - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Goto the settings page, add our ignore text res = client.post( @@ -222,7 +223,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""): # Make a change which includes the ignore text, it should be ignored and no 'change' triggered # It adds text with "ZZZZzzzz" and "ZZZZ" is in the ignore list # And tweaks the ver_stamp which should be picked up by global regex ignore - set_modified_ignore_response(ver_stamp=time.time()) + set_modified_ignore_response(ver_stamp=time.time(), datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -236,7 +237,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""): assert b'/test-endpoint' in res.data # Just to be sure.. set a regular modified change that will trigger it - set_modified_original_ignore_response() + set_modified_original_ignore_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -244,10 +245,10 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""): delete_all_watches(client) -def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage): +def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path): - _run_test_global_ignore(client, as_source=False) + _run_test_global_ignore(client, as_source=False, datastore_path=datastore_path) -def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage): +def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage, datastore_path): - _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/') + _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/', datastore_path=datastore_path) diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py index a3412af5..9a811c53 100644 --- a/changedetectionio/tests/test_ignorehyperlinks.py +++ b/changedetectionio/tests/test_ignorehyperlinks.py @@ -4,9 +4,10 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -17,13 +18,13 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -# Should be the same as set_original_ignore_response() but with a different +# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a different # link -def set_modified_ignore_response(): +def set_modified_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -34,10 +35,10 @@ def set_modified_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage): +def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path): """Testing that the link changes are detected when render_anchor_tag_content setting is set to true""" sleep_time_for_fetch_thread = 3 @@ -46,7 +47,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag time.sleep(1) # set original html text - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content") res = client.post( @@ -72,7 +73,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # set a new html text with a modified link - set_modified_ignore_response() + set_modified_ignore_response(datastore_path=datastore_path) wait_for_all_checks(client) # Trigger a check diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 37e7fb38..c93083fb 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -3,12 +3,13 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks +import os -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ Some initial text
@@ -19,11 +20,11 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_some_changed_response(): +def set_some_changed_response(datastore_path): test_return_data = """ Some initial text
@@ -34,17 +35,17 @@ def set_some_changed_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage): +def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path): # Give the endpoint time to spin up time.sleep(1) - set_original_response() + set_original_response(datastore_path=datastore_path) # Goto the settings page, add our ignore text res = client.post( @@ -65,7 +66,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me wait_for_all_checks(client) - set_some_changed_response() + set_some_changed_response(datastore_path=datastore_path) wait_for_all_checks(client) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -80,10 +81,10 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me # Tests the whole stack works with staus codes ignored -def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage): +def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path): sleep_time_for_fetch_thread = 3 - set_original_response() + set_original_response(datastore_path=datastore_path) # Give the endpoint time to spin up time.sleep(1) @@ -109,7 +110,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu wait_for_all_checks(client) # Make a change - set_some_changed_response() + set_some_changed_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py index c507885c..ed0a53b6 100644 --- a/changedetectionio/tests/test_ignorewhitespace.py +++ b/changedetectionio/tests/test_ignorewhitespace.py @@ -3,12 +3,13 @@ import time from flask import url_for from . util import live_server_setup +import os -# Should be the same as set_original_ignore_response() but with a little more whitespacing -def set_original_ignore_response_but_with_whitespace(): +# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a little more whitespacing +def set_original_ignore_response_but_with_whitespace(datastore_path): test_return_data = """ Some initial text
@@ -26,11 +27,11 @@ def set_original_ignore_response_but_with_whitespace(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -42,19 +43,19 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # If there was only a change in the whitespacing, then we shouldnt have a change detected -def test_check_ignore_whitespace(client, live_server, measure_memory_usage): +def test_check_ignore_whitespace(client, live_server, measure_memory_usage, datastore_path): sleep_time_for_fetch_thread = 3 # Give the endpoint time to spin up time.sleep(1) - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Goto the settings page, add our ignore text res = client.post( @@ -77,7 +78,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage): # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) - set_original_ignore_response_but_with_whitespace() + set_original_ignore_response_but_with_whitespace(datastore_path) time.sleep(sleep_time_for_fetch_thread) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_import.py b/changedetectionio/tests/test_import.py index 0d9de7c8..401e9e41 100644 --- a/changedetectionio/tests/test_import.py +++ b/changedetectionio/tests/test_import.py @@ -8,10 +8,10 @@ from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_import(client, live_server, measure_memory_usage): +def test_import(client, live_server, measure_memory_usage, datastore_path): # Give the endpoint time to spin up wait_for_all_checks(client) @@ -34,7 +34,7 @@ https://example.com tag1, other tag""" res = client.get( url_for("watchlist.index")) res = client.get( url_for("watchlist.index")) -def xtest_import_skip_url(client, live_server, measure_memory_usage): +def xtest_import_skip_url(client, live_server, measure_memory_usage, datastore_path): # Give the endpoint time to spin up @@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage): # Clear flask alerts res = client.get( url_for("watchlist.index")) -def test_import_distillio(client, live_server, measure_memory_usage): +def test_import_distillio(client, live_server, measure_memory_usage, datastore_path): distill_data=''' { @@ -123,7 +123,7 @@ def test_import_distillio(client, live_server, measure_memory_usage): # Clear flask alerts res = client.get(url_for("watchlist.index")) -def test_import_custom_xlsx(client, live_server, measure_memory_usage): +def test_import_custom_xlsx(client, live_server, measure_memory_usage, datastore_path): """Test can upload a excel spreadsheet and the watches are created correctly""" @@ -171,7 +171,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_import_watchete_xlsx(client, live_server, measure_memory_usage): +def test_import_watchete_xlsx(client, live_server, measure_memory_usage, datastore_path): """Test can upload a excel spreadsheet and the watches are created correctly""" diff --git a/changedetectionio/tests/test_jinja2.py b/changedetectionio/tests/test_jinja2.py index e2992471..09612a69 100644 --- a/changedetectionio/tests/test_jinja2.py +++ b/changedetectionio/tests/test_jinja2.py @@ -7,11 +7,11 @@ from .util import live_server_setup, wait_for_all_checks from ..jinja2_custom import render -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # # live_server_setup(live_server) # Setup on conftest per function # If there was only a change in the whitespacing, then we shouldnt have a change detected -def test_jinja2_in_url_query(client, live_server, measure_memory_usage): +def test_jinja2_in_url_query(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page @@ -36,7 +36,7 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage): assert b'date=2' in res.data # Test for issue #1493 - jinja2-time offset functionality -def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage): +def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage, datastore_path): """Test that jinja2 time offset expressions work in watch URLs (issue #1493).""" # Add our URL to the import page with time offset expression @@ -66,7 +66,7 @@ def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usa # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456 -def test_jinja2_security_url_query(client, live_server, measure_memory_usage): +def test_jinja2_security_url_query(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page test_url = url_for('test_return_query', _external=True) diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 342f77af..7ea5b90c 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -6,6 +6,7 @@ from flask import url_for from markupsafe import escape from . util import live_server_setup, wait_for_all_checks, delete_all_watches import pytest +import os jq_support = True try: @@ -92,7 +93,7 @@ def test_unittest_inline_extract_body(): text = html_tools.extract_json_as_string(content, "json:$.testKey") assert text == '42' -def set_original_ext_response(): +def set_original_ext_response(datastore_path): data = """ [ { @@ -109,11 +110,11 @@ def set_original_ext_response(): ] """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) return None -def set_modified_ext_response(): +def set_modified_ext_response(datastore_path): # This should get reformatted data = """ [ { "isPriceLowered": false, "status": "Sold", "statusOrig": "sold" }, { "_id": "5e7b3e1fb3262d306323ff1e", @@ -124,11 +125,11 @@ def set_modified_ext_response(): ] """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) return None -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ { "employees": [ @@ -149,12 +150,12 @@ def set_original_response(): "available": true } """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_json_response_with_html(): +def set_json_response_with_html(datastore_path): test_return_data = """ { "test": [ @@ -164,11 +165,11 @@ def set_json_response_with_html(): ] } """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ { "employees": [ @@ -190,15 +191,15 @@ def set_modified_response(): } """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def test_check_json_without_filter(client, live_server, measure_memory_usage): +def test_check_json_without_filter(client, live_server, measure_memory_usage, datastore_path): # Request a JSON document from a application/json source containing HTML # and be sure it doesn't get chewed up by instriptis - set_json_response_with_html() + set_json_response_with_html(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', content_type="application/json", _external=True) @@ -219,8 +220,8 @@ def test_check_json_without_filter(client, live_server, measure_memory_usage): delete_all_watches(client) -def check_json_filter(json_filter, client, live_server): - set_original_response() +def check_json_filter(json_filter, client, live_server, datastore_path): + set_original_response(datastore_path=datastore_path) # Add our URL to the import page @@ -240,7 +241,7 @@ def check_json_filter(json_filter, client, live_server): # Give the thread time to pick it up wait_for_all_checks(client) # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -260,19 +261,19 @@ def check_json_filter(json_filter, client, live_server): delete_all_watches(client) -def test_check_jsonpath_filter(client, live_server, measure_memory_usage): - check_json_filter('json:boss.name', client, live_server) +def test_check_jsonpath_filter(client, live_server, measure_memory_usage, datastore_path): + check_json_filter('json:boss.name', client, live_server, datastore_path=datastore_path) -def test_check_jq_filter(client, live_server, measure_memory_usage): +def test_check_jq_filter(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_filter('jq:.boss.name', client, live_server) + check_json_filter('jq:.boss.name', client, live_server, datastore_path=datastore_path) -def test_check_jqraw_filter(client, live_server, measure_memory_usage): +def test_check_jqraw_filter(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_filter('jqraw:.boss.name', client, live_server) + check_json_filter('jqraw:.boss.name', client, live_server, datastore_path=datastore_path) -def check_json_filter_bool_val(json_filter, client, live_server): - set_original_response() +def check_json_filter_bool_val(json_filter, client, live_server, datastore_path): + set_original_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', content_type="application/json", _external=True) @@ -281,7 +282,7 @@ def check_json_filter_bool_val(json_filter, client, live_server): wait_for_all_checks(client) # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -294,24 +295,24 @@ def check_json_filter_bool_val(json_filter, client, live_server): delete_all_watches(client) -def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage): - check_json_filter_bool_val("json:$['available']", client, live_server) +def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage, datastore_path): + check_json_filter_bool_val("json:$['available']", client, live_server, datastore_path=datastore_path) -def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage): +def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_filter_bool_val("jq:.available", client, live_server) + check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path) -def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage): +def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_filter_bool_val("jq:.available", client, live_server) + check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path) # Re #265 - Extended JSON selector test # Stuff to consider here # - Selector should be allowed to return empty when it doesnt match (people might wait for some condition) # - The 'diff' tab could show the old and new content # - Form should let us enter a selector that doesnt (yet) match anything -def check_json_ext_filter(json_filter, client, live_server): - set_original_ext_response() +def check_json_ext_filter(json_filter, client, live_server, datastore_path): + set_original_ext_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', content_type="application/json", _external=True) @@ -343,7 +344,7 @@ def check_json_ext_filter(json_filter, client, live_server): # Give the thread time to pick it up wait_for_all_checks(client) # Make a change - set_modified_ext_response() + set_modified_ext_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -376,10 +377,10 @@ def check_json_ext_filter(json_filter, client, live_server): delete_all_watches(client) -def test_ignore_json_order(client, live_server, measure_memory_usage): +def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_path): # A change in order shouldn't trigger a notification - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write('{"hello" : 123, "world": 123}') @@ -390,7 +391,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage): wait_for_all_checks(client) - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write('{"world" : 123, "hello": 123}') # Trigger a check @@ -401,7 +402,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage): assert b'has-unread-changes' not in res.data # Just to be sure it still works - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write('{"world" : 123, "hello": 124}') # Trigger a check @@ -413,10 +414,10 @@ def test_ignore_json_order(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_correct_header_detect(client, live_server, measure_memory_usage): +def test_correct_header_detect(client, live_server, measure_memory_usage, datastore_path): # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593 # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write('{ "world": 123, "hello" : 123}') # Add our URL to the import page @@ -450,18 +451,18 @@ def test_correct_header_detect(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage): - check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server) +def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage, datastore_path): + check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server, datastore_path=datastore_path) -def test_check_jq_ext_filter(client, live_server, measure_memory_usage): +def test_check_jq_ext_filter(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server) + check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path) -def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage): +def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage, datastore_path): if jq_support: - check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server) + check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path) -def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage): +def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage, datastore_path): from .. import html_tools # JSON string with BOM and correct double-quoted keys diff --git a/changedetectionio/tests/test_live_preview.py b/changedetectionio/tests/test_live_preview.py index 2927098f..8c8f874b 100644 --- a/changedetectionio/tests/test_live_preview.py +++ b/changedetectionio/tests/test_live_preview.py @@ -2,9 +2,10 @@ from flask import url_for from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches +import os -def set_response(): +def set_response(datastore_path): data = """ Awesome, you made it
@@ -15,12 +16,12 @@ something to trigger
""" - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(data) -def test_content_filter_live_preview(client, live_server, measure_memory_usage): +def test_content_filter_live_preview(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_response() + set_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_nonrenderable_pages.py b/changedetectionio/tests/test_nonrenderable_pages.py index 85fad316..299f5412 100644 --- a/changedetectionio/tests/test_nonrenderable_pages.py +++ b/changedetectionio/tests/test_nonrenderable_pages.py @@ -3,9 +3,10 @@ from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, delete_all_watches import time +import os -def set_nonrenderable_response(): +def set_nonrenderable_response(datastore_path): test_return_data = """ modified head title @@ -13,20 +14,20 @@ def set_nonrenderable_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) time.sleep(1) return None -def set_zero_byte_response(): - with open("test-datastore/endpoint-content.txt", "w") as f: +def set_zero_byte_response(datastore_path): + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("") time.sleep(1) return None -def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage): - set_original_response() +def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # live_server_setup(live_server) # Setup on conftest per function # Add our URL to the import page @@ -55,7 +56,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure ) # this should not trigger a change, because no good text could be converted from the HTML - set_nonrenderable_response() + set_nonrenderable_response(datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -84,7 +85,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure 'application-fetch_backend': "html_requests"}, follow_redirects=True ) - set_modified_response() + set_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -100,7 +101,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # A totally zero byte (#2528) response should also not trigger an error - set_zero_byte_response() + set_zero_byte_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # 2877 diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index b9e418a2..233e0652 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -21,9 +21,9 @@ from ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH # Hard to just add more live server URLs when one test is already running (I think) # So we add our test here (was in a different file) -def test_check_notification(client, live_server, measure_memory_usage): +def test_check_notification(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) # Re 360 - new install should have defaults set res = client.get(url_for("settings.settings_page")) @@ -83,8 +83,7 @@ def test_check_notification(client, live_server, measure_memory_usage): uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) - datastore = 'test-datastore' - with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f: + with open(os.path.join(datastore_path, str(uuid), 'last-screenshot.png'), 'wb') as f: f.write(base64.b64decode(testimage_png)) # Goto the edit page, add our ignore text @@ -138,7 +137,7 @@ def test_check_notification(client, live_server, measure_memory_usage): ## Now recheck, and it should have sent the notification wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -151,9 +150,9 @@ def test_check_notification(client, live_server, measure_memory_usage): # Verify what was sent as a notification, this file should exist - with open("test-datastore/notification.txt", "r") as f: + with open(os.path.join(datastore_path, "notification.txt"), "r") as f: notification_submission = f.read() - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) # Did we see the URL that had a change, in the notification? # Diff was correctly executed @@ -197,18 +196,18 @@ def test_check_notification(client, live_server, measure_memory_usage): # This should insert the {current_snapshot} - set_more_modified_response() + set_more_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(3) # Verify what was sent as a notification, this file should exist - with open("test-datastore/notification.txt", "r") as f: + with open(os.path.join(datastore_path, "notification.txt"), "r") as f: notification_submission = f.read() assert "Ohh yeah awesome" in notification_submission # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing # https://github.com/dgtlmoon/changedetection.io/discussions/192 - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -217,13 +216,13 @@ def test_check_notification(client, live_server, measure_memory_usage): wait_for_all_checks(client) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - assert os.path.exists("test-datastore/notification.txt") == False + assert os.path.exists(os.path.join(datastore_path, "notification.txt")) == False res = client.get(url_for("settings.notification_logs")) # be sure we see it in the output log assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data - set_original_response() + set_original_response(datastore_path=datastore_path) res = client.post( url_for("ui.ui_edit.edit_page", uuid="first"), data={ @@ -243,7 +242,7 @@ def test_check_notification(client, live_server, measure_memory_usage): time.sleep(2) # Verify what was sent as a notification, this file should exist - with open("test-datastore/notification.txt", "r") as f: + with open(os.path.join(datastore_path, "notification.txt"), "r") as f: notification_submission = f.read() assert "fallback-title" in notification_submission assert "fallback-body" in notification_submission @@ -254,7 +253,7 @@ def test_check_notification(client, live_server, measure_memory_usage): follow_redirects=True ) -def test_notification_validation(client, live_server, measure_memory_usage): +def test_notification_validation(client, live_server, measure_memory_usage, datastore_path): time.sleep(1) @@ -292,7 +291,7 @@ def test_notification_validation(client, live_server, measure_memory_usage): ) -def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage): +def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage, datastore_path): # # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation @@ -314,14 +313,14 @@ def test_notification_urls_jinja2_apprise_integration(client, live_server, measu assert b'Settings updated' in res.data -def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage): +def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage, datastore_path): # test_endpoint - that sends the contents of a file # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt) # CUSTOM JSON BODY CHECK for POST:// - set_original_response() + set_original_response(datastore_path=datastore_path) # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&watch_uuid={{ watch_uuid }}&xxx={{ watch_url }}&now={% now 'Europe/London', '%Y-%m-%d' %}&+custom-header=123&+second=hello+world%20%22space%22" @@ -352,7 +351,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -364,7 +363,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me res = client.get(url_for("watchlist.index")) assert b'notification-error' not in res.data - with open("test-datastore/notification.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: x = f.read() j = json.loads(x) assert j['url'].startswith('http://localhost') @@ -373,8 +372,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me # URL check, this will always be converted to lowercase - assert os.path.isfile("test-datastore/notification-url.txt") - with open("test-datastore/notification-url.txt", 'r') as f: + assert os.path.isfile(os.path.join(datastore_path, "notification-url.txt")) + with open(os.path.join(datastore_path, "notification-url.txt"), 'r') as f: notification_url = f.read() assert 'xxx=http' in notification_url # apprise style headers should be stripped @@ -385,18 +384,18 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me assert f'watch_uuid={watch_uuid}' in notification_url - with open("test-datastore/notification-headers.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification-headers.txt"), 'r') as f: notification_headers = f.read() assert 'custom-header: 123' in notification_headers.lower() assert 'second: hello world "space"' in notification_headers.lower() # Should always be automatically detected as JSON content type even when we set it as 'Plain Text' (default) - assert os.path.isfile("test-datastore/notification-content-type.txt") - with open("test-datastore/notification-content-type.txt", 'r') as f: + assert os.path.isfile(os.path.join(datastore_path, "notification-content-type.txt")) + with open(os.path.join(datastore_path, "notification-content-type.txt"), 'r') as f: assert 'application/json' in f.read() - os.unlink("test-datastore/notification-url.txt") + os.unlink(os.path.join(datastore_path, "notification-url.txt")) client.get( url_for("ui.form_delete", uuid="all"), @@ -405,12 +404,12 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me #2510 -def test_global_send_test_notification(client, live_server, measure_memory_usage): +def test_global_send_test_notification(client, live_server, measure_memory_usage, datastore_path): - set_original_response() - if os.path.isfile("test-datastore/notification.txt"): - os.unlink("test-datastore/notification.txt") \ + set_original_response(datastore_path=datastore_path) + if os.path.isfile(os.path.join(datastore_path, "notification.txt")): + os.unlink(os.path.join(datastore_path, "notification.txt")) \ # 1995 UTF-8 content should be encoded test_body = 'change detection is cool 网站监测 内容更新了' @@ -451,11 +450,11 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage assert res.status_code != 400 assert res.status_code != 500 - with open("test-datastore/notification.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: x = f.read() assert test_body in x - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) ######### Test group/tag settings res = client.post( @@ -470,7 +469,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage # Give apprise time to fire time.sleep(4) - with open("test-datastore/notification.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: x = f.read() # Should come from notification.py default handler when there is no notification body to pull from assert 'change detection is cool 网站监测 内容更新了' in x @@ -511,12 +510,12 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage -def _test_color_notifications(client, notification_body_token): +def _test_color_notifications(client, notification_body_token, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) - if os.path.isfile("test-datastore/notification.txt"): - os.unlink("test-datastore/notification.txt") + if os.path.isfile(os.path.join(datastore_path, "notification.txt")): + os.unlink(os.path.join(datastore_path, "notification.txt")) test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123" @@ -548,7 +547,7 @@ def _test_color_notifications(client, notification_body_token): wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -557,7 +556,7 @@ def _test_color_notifications(client, notification_body_token): wait_for_all_checks(client) time.sleep(3) - with open("test-datastore/notification.txt", 'r') as f: + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: x = f.read() s = f'Which is across multiple lines' assert s in x @@ -569,6 +568,7 @@ def _test_color_notifications(client, notification_body_token): ) # Just checks the format of the colour notifications was correct -def test_html_color_notifications(client, live_server, measure_memory_usage): - _test_color_notifications(client, '{{diff}}') - _test_color_notifications(client, '{{diff_full}}') +def test_html_color_notifications(client, live_server, measure_memory_usage, datastore_path): + _test_color_notifications(client, '{{diff}}',datastore_path=datastore_path) + _test_color_notifications(client, '{{diff_full}}',datastore_path=datastore_path) + diff --git a/changedetectionio/tests/test_notification_errors.py b/changedetectionio/tests/test_notification_errors.py index 38967001..f09fdc9a 100644 --- a/changedetectionio/tests/test_notification_errors.py +++ b/changedetectionio/tests/test_notification_errors.py @@ -4,10 +4,10 @@ from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks import logging -def test_check_notification_error_handling(client, live_server, measure_memory_usage): +def test_check_notification_error_handling(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_original_response() + set_original_response(datastore_path=datastore_path) # Set a URL and fetch it, then set a notification URL which is going to give errors test_url = url_for('test_endpoint', _external=True) @@ -19,7 +19,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u assert b"Watch added" in res.data wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') broken_notification_url = "jsons://broken-url-xxxxxxxx123/test" @@ -73,9 +73,9 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u assert found_name_resolution_error # And the working one, which is after the 'broken' one should still have fired - with open("test-datastore/notification.txt", "r") as f: + with open(os.path.join(datastore_path, "notification.txt"), "r") as f: notification_submission = f.read() - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) assert 'xxxxx' in notification_submission client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) diff --git a/changedetectionio/tests/test_obfuscations.py b/changedetectionio/tests/test_obfuscations.py index 53034669..e1ef0df2 100644 --- a/changedetectionio/tests/test_obfuscations.py +++ b/changedetectionio/tests/test_obfuscations.py @@ -3,9 +3,10 @@ import time from flask import url_for from .util import live_server_setup +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ The price is$90.74 @@ -14,12 +15,12 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_obfuscations(client, live_server, measure_memory_usage): - set_original_ignore_response() +def test_obfuscations(client, live_server, measure_memory_usage, datastore_path): + set_original_ignore_response(datastore_path) # live_server_setup(live_server) # Setup on conftest per function time.sleep(1) # Add our URL to the import page diff --git a/changedetectionio/tests/test_pdf.py b/changedetectionio/tests/test_pdf.py index 64212bae..6442d4c6 100644 --- a/changedetectionio/tests/test_pdf.py +++ b/changedetectionio/tests/test_pdf.py @@ -3,15 +3,16 @@ import time from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks +import os # `subtractive_selectors` should still work in `source:` type requests -def test_fetch_pdf(client, live_server, measure_memory_usage): +def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path): import shutil import os - shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf") - first_version_size = os.path.getsize("test-datastore/endpoint-test.pdf") + shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf")) + first_version_size = os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf")) test_url = url_for('test_pdf_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -35,14 +36,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage): # So we know if the file changes in other ways import hashlib - original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() + original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper() # We should have one assert len(original_md5) >0 # And it's going to be in the document assert f'Document checksum - {original_md5}' in snapshot_contents - shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf") - changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() + shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf")) + changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper() res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 1 watch for rechecking.' in res.data @@ -76,9 +77,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage): # new snapshot was also OK, no HTML snapshot_contents = watch.get_history_snapshot(dates[1]) assert 'html' not in snapshot_contents.lower() - assert f'Original file size - {os.path.getsize("test-datastore/endpoint-test.pdf")}' in snapshot_contents + assert f'Original file size - {os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))}' in snapshot_contents assert f'here is a change' in snapshot_contents - assert os.path.getsize("test-datastore/endpoint-test.pdf") != first_version_size # And the disk change worked + assert os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf")) != first_version_size # And the disk change worked \ No newline at end of file diff --git a/changedetectionio/tests/test_preview_endpoints.py b/changedetectionio/tests/test_preview_endpoints.py index 6044bf9f..9ae72839 100644 --- a/changedetectionio/tests/test_preview_endpoints.py +++ b/changedetectionio/tests/test_preview_endpoints.py @@ -3,12 +3,13 @@ import time from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks +import os # `subtractive_selectors` should still work in `source:` type requests -def test_fetch_pdf(client, live_server, measure_memory_usage): +def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path): import shutil - shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf") + shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf")) # live_server_setup(live_server) # Setup on conftest per function test_url = url_for('test_pdf_endpoint', _external=True) @@ -29,14 +30,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage): # So we know if the file changes in other ways import hashlib - original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() + original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper() # We should have one assert len(original_md5) > 0 # And it's going to be in the document assert b'Document checksum - ' + bytes(str(original_md5).encode('utf-8')) in res.data - shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf") - changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() + shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf")) + changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper() res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 1 watch for rechecking.' in res.data diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 34a9c8df..5e080405 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -8,7 +8,7 @@ from . util import set_original_response, set_modified_response, live_server_set # Hard to just add more live server URLs when one test is already running (I think) # So we add our test here (was in a different file) -def test_headers_in_request(client, live_server, measure_memory_usage): +def test_headers_in_request(client, live_server, measure_memory_usage, datastore_path): #ve_server_setup(live_server) # Add our URL to the import page test_url = url_for('test_headers', _external=True) @@ -76,7 +76,8 @@ def test_headers_in_request(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_body_in_request(client, live_server, measure_memory_usage): +def test_body_in_request(client, live_server, measure_memory_usage, datastore_path): + import os # Add our URL to the import page test_url = url_for('test_body', _external=True) @@ -141,7 +142,7 @@ def test_body_in_request(client, live_server, measure_memory_usage): client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) watches_with_body = 0 - with open('test-datastore/url-watches.json') as f: + with open(os.path.join(datastore_path, 'url-watches.json')) as f: app_struct = json.load(f) for uuid in app_struct['watching']: if app_struct['watching'][uuid]['body']==body_value: @@ -165,7 +166,8 @@ def test_body_in_request(client, live_server, measure_memory_usage): assert b"Body must be empty when Request Method is set to GET" in res.data delete_all_watches(client) -def test_method_in_request(client, live_server, measure_memory_usage): +def test_method_in_request(client, live_server, measure_memory_usage, datastore_path): + import os # Add our URL to the import page test_url = url_for('test_method', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): @@ -223,7 +225,7 @@ def test_method_in_request(client, live_server, measure_memory_usage): wait_for_all_checks(client) watches_with_method = 0 - with open('test-datastore/url-watches.json') as f: + with open(os.path.join(datastore_path, 'url-watches.json')) as f: app_struct = json.load(f) for uuid in app_struct['watching']: if app_struct['watching'][uuid]['method'] == 'PATCH': @@ -235,7 +237,7 @@ def test_method_in_request(client, live_server, measure_memory_usage): delete_all_watches(client) # Re #2408 - user-agent override test, also should handle case-insensitive header deduplication -def test_ua_global_override(client, live_server, measure_memory_usage): +def test_ua_global_override(client, live_server, measure_memory_usage, datastore_path): ## live_server_setup(live_server) # Setup on conftest per function test_url = url_for('test_headers', _external=True) @@ -286,8 +288,9 @@ def test_ua_global_override(client, live_server, measure_memory_usage): assert b"html-requests-user-agent" not in res.data delete_all_watches(client) -def test_headers_textfile_in_request(client, live_server, measure_memory_usage): - +def test_headers_textfile_in_request(client, live_server, measure_memory_usage, datastore_path): + import os + # Add our URL to the import page webdriver_ua = "Hello fancy webdriver UA 1.0" @@ -343,14 +346,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): assert b"Updated watch." in res.data wait_for_all_checks(client) - with open('test-datastore/headers-testtag.txt', 'w') as f: + with open(os.path.join(datastore_path, 'headers-testtag.txt'), 'w') as f: f.write("tag-header: test\r\nurl-header: http://example.com") - with open('test-datastore/headers.txt', 'w') as f: + with open(os.path.join(datastore_path, 'headers.txt'), 'w') as f: f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global") uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) - with open(f'test-datastore/{uuid}/headers.txt', 'w') as f: + with open(os.path.join(datastore_path, uuid, 'headers.txt'), 'w') as f: f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch") wait_for_all_checks(client) @@ -368,8 +371,8 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): assert b"Extra headers file found and will be added to this watch" in res.data # Not needed anymore - os.unlink('test-datastore/headers.txt') - os.unlink('test-datastore/headers-testtag.txt') + os.unlink(os.path.join(datastore_path, 'headers.txt')) + os.unlink(os.path.join(datastore_path, 'headers-testtag.txt')) # The service should echo back the request verb res = client.get( @@ -395,7 +398,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): # unlink headers.txt on start/stop delete_all_watches(client) -def test_headers_validation(client, live_server, measure_memory_usage): +def test_headers_validation(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_headers', _external=True) diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index 62588188..b265a357 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -21,8 +21,7 @@ out_of_stock_props = [ '' ] -def set_original_response(props_markup='', price="121.95"): - +def set_original_response(datastore_path, props_markup='', price="121.95"): props_markup=props_markup.replace('$$PRICE$$', price) test_return_data = f""" @@ -36,28 +35,19 @@ def set_original_response(props_markup='', price="121.95"): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) time.sleep(1) return None - - - -# def test_setup(client, live_server, measure_memory_usage): - - # live_server_setup(live_server) # Setup on conftest per function - -def test_restock_itemprop_basic(client, live_server, measure_memory_usage): - - +def test_restock_itemprop_basic(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_endpoint', _external=True) # By default it should enable ('in_stock_processing') == 'all_changes' for p in instock_props: - set_original_response(props_markup=p) + set_original_response(props_markup=p, datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, @@ -73,7 +63,7 @@ def test_restock_itemprop_basic(client, live_server, measure_memory_usage): for p in out_of_stock_props: - set_original_response(props_markup=p) + set_original_response(props_markup=p, datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": '', 'processor': 'restock_diff'}, @@ -86,13 +76,13 @@ def test_restock_itemprop_basic(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_itemprop_price_change(client, live_server, measure_memory_usage): +def test_itemprop_price_change(client, live_server, measure_memory_usage, datastore_path): # Out of the box 'Follow price changes' should be ON test_url = url_for('test_endpoint', _external=True) - set_original_response(props_markup=instock_props[0], price="190.95") + set_original_response(props_markup=instock_props[0], price="190.95", datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, @@ -105,7 +95,7 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage): assert b'190.95' in res.data # basic price change, look for notification - set_original_response(props_markup=instock_props[0], price='180.45') + set_original_response(props_markup=instock_props[0], price='180.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -116,7 +106,7 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage): # turning off price change trigger, but it should show the new price, with no change notification - set_original_response(props_markup=instock_props[0], price='120.45') + set_original_response(props_markup=instock_props[0], price='120.45', datastore_path=datastore_path) res = client.post( url_for("ui.ui_edit.edit_page", uuid="first"), data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"}, @@ -132,13 +122,13 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage): delete_all_watches(client) -def _run_test_minmax_limit(client, extra_watch_edit_form): +def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path): delete_all_watches(client) test_url = url_for('test_endpoint', _external=True) - set_original_response(props_markup=instock_props[0], price="950.95") + set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, @@ -166,7 +156,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): client.get(url_for("ui.mark_all_viewed")) # price changed to something greater than min (900), BUT less than max (1100).. should be no change - set_original_response(props_markup=instock_props[0], price='1000.45') + set_original_response(props_markup=instock_props[0], price='1000.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -177,7 +167,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): assert b'has-unread-changes' not in res.data # price changed to something LESS than min (900), SHOULD be a change - set_original_response(props_markup=instock_props[0], price='890.45') + set_original_response(props_markup=instock_props[0], price='890.45', datastore_path=datastore_path) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 1 watch for rechecking.' in res.data @@ -190,7 +180,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again! - set_original_response(props_markup=instock_props[0], price='820.45') + set_original_response(props_markup=instock_props[0], price='820.45', datastore_path=datastore_path) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) @@ -200,7 +190,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): client.get(url_for("ui.mark_all_viewed")) # price changed to something MORE than max (1100.10), SHOULD be a change - set_original_response(props_markup=instock_props[0], price='1890.45') + set_original_response(props_markup=instock_props[0], price='1890.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -211,16 +201,16 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): delete_all_watches(client) -def test_restock_itemprop_minmax(client, live_server, measure_memory_usage): +def test_restock_itemprop_minmax(client, live_server, measure_memory_usage, datastore_path): extras = { "restock_settings-follow_price_changes": "y", "restock_settings-price_change_min": 900.0, "restock_settings-price_change_max": 1100.10 } - _run_test_minmax_limit(client, extra_watch_edit_form=extras) + _run_test_minmax_limit(client, extra_watch_edit_form=extras, datastore_path=datastore_path) -def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage): +def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage, datastore_path): res = client.post( @@ -245,18 +235,18 @@ def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage): "tags": "test-tag" } - _run_test_minmax_limit(client, extra_watch_edit_form=extras) + _run_test_minmax_limit(client, extra_watch_edit_form=extras,datastore_path=datastore_path) -def test_itemprop_percent_threshold(client, live_server, measure_memory_usage): +def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, datastore_path): delete_all_watches(client) test_url = url_for('test_endpoint', _external=True) - set_original_response(props_markup=instock_props[0], price="950.95") + set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, @@ -283,7 +273,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage): # Basic change should not trigger - set_original_response(props_markup=instock_props[0], price='960.45') + set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -291,7 +281,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage): assert b'has-unread-changes' not in res.data # Bigger INCREASE change than the threshold should trigger - set_original_response(props_markup=instock_props[0], price='1960.45') + set_original_response(props_markup=instock_props[0], price='1960.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -301,7 +291,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage): # Small decrease should NOT trigger client.get(url_for("ui.mark_all_viewed")) - set_original_response(props_markup=instock_props[0], price='1950.45') + set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) @@ -315,14 +305,14 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage): -def test_change_with_notification_values(client, live_server, measure_memory_usage): +def test_change_with_notification_values(client, live_server, measure_memory_usage, datastore_path): - if os.path.isfile("test-datastore/notification.txt"): - os.unlink("test-datastore/notification.txt") + if os.path.isfile(os.path.join(datastore_path, "notification.txt")): + os.unlink(os.path.join(datastore_path, "notification.txt")) test_url = url_for('test_endpoint', _external=True) - set_original_response(props_markup=instock_props[0], price='960.45') + set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path) notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') @@ -363,34 +353,34 @@ def test_change_with_notification_values(client, live_server, measure_memory_usa assert b"Settings updated." in res.data - set_original_response(props_markup=instock_props[0], price='960.45') + set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path) # A change in price, should trigger a change by default - set_original_response(props_markup=instock_props[0], price='1950.45') + set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) - wait_for_notification_endpoint_output() - assert os.path.isfile("test-datastore/notification.txt"), "Notification received" - with open("test-datastore/notification.txt", 'r') as f: + wait_for_notification_endpoint_output(datastore_path=datastore_path) + assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification received" + with open(os.path.join(datastore_path, "notification.txt"), 'r') as f: notification = f.read() assert "new price 1950.45" in notification assert "title new price 1950.45" in notification ## Now test the "SEND TEST NOTIFICATION" is working - os.unlink("test-datastore/notification.txt") + os.unlink(os.path.join(datastore_path, "notification.txt")) uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) res = client.post(url_for("ui.ui_notification.ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True) time.sleep(5) - assert os.path.isfile("test-datastore/notification.txt"), "Notification received" + assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification received" -def test_data_sanity(client, live_server, measure_memory_usage): +def test_data_sanity(client, live_server, measure_memory_usage, datastore_path): delete_all_watches(client) test_url = url_for('test_endpoint', _external=True) test_url2 = url_for('test_endpoint2', _external=True) - set_original_response(props_markup=instock_props[0], price="950.95") + set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path) client.post( url_for("ui.ui_views.form_quick_watch_add"), data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'}, @@ -429,7 +419,7 @@ def test_data_sanity(client, live_server, measure_memory_usage): delete_all_watches(client) # All examples should give a prive of 666.66 -def test_special_prop_examples(client, live_server, measure_memory_usage): +def test_special_prop_examples(client, live_server, measure_memory_usage, datastore_path): import glob @@ -439,7 +429,7 @@ def test_special_prop_examples(client, live_server, measure_memory_usage): assert files for test_example_filename in files: with open(test_example_filename, 'r') as example_f: - with open("test-datastore/endpoint-content.txt", "w") as test_f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as test_f: test_f.write(f"{example_f.read()}") # Now fetch it and check the price worked diff --git a/changedetectionio/tests/test_rss.py b/changedetectionio/tests/test_rss.py index 9874632a..2a1e4c0e 100644 --- a/changedetectionio/tests/test_rss.py +++ b/changedetectionio/tests/test_rss.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 - +import os import time from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \ extract_UUID_from_client, delete_all_watches -def set_original_cdata_xml(): +def set_original_cdata_xml(datastore_path): test_return_data = """ Gizi @@ -45,12 +45,12 @@ def set_original_cdata_xml(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_html_content(content): +def set_html_content(datastore_path, content): test_return_data = f""" Some initial text
@@ -62,16 +62,16 @@ def set_html_content(content): """ # Write as UTF-8 encoded bytes - with open("test-datastore/endpoint-content.txt", "wb") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "wb") as f: f.write(test_return_data.encode('utf-8')) -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_rss_and_token(client, live_server, measure_memory_usage): +def test_rss_and_token(client, live_server, measure_memory_usage, datastore_path): # # live_server_setup(live_server) # Setup on conftest per function - set_original_response() + set_original_response(datastore_path=datastore_path) rss_token = extract_rss_token_from_UI(client) # Add our URL to the import page @@ -84,7 +84,7 @@ def test_rss_and_token(client, live_server, measure_memory_usage): assert b"1 Imported" in res.data wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) time.sleep(1) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -106,10 +106,10 @@ def test_rss_and_token(client, live_server, measure_memory_usage): client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage): +def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage, datastore_path): - set_original_cdata_xml() + set_original_cdata_xml(datastore_path) # Rarely do endpoints give the right header, usually just text/xml, so we check also for Security Bulletins on wetscale @@ -40,13 +42,13 @@ def set_original_cdata_xml(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_rss_reader_mode(client, live_server, measure_memory_usage): - set_original_cdata_xml() +def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_path): + set_original_cdata_xml(datastore_path=datastore_path) # Rarely do endpoints give the right header, usually just text/xml, so we check also for head title @@ -20,11 +20,11 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def test_bad_access(client, live_server, measure_memory_usage): +def test_bad_access(client, live_server, measure_memory_usage, datastore_path): res = client.post( url_for("imports.import_page"), @@ -111,7 +111,7 @@ def _runner_test_various_file_slash(client, file_uri): delete_all_watches(client) -def test_file_slash_access(client, live_server, measure_memory_usage): +def test_file_slash_access(client, live_server, measure_memory_usage, datastore_path): # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check @@ -121,7 +121,7 @@ def test_file_slash_access(client, live_server, measure_memory_usage): # _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}") # _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509 -def test_xss(client, live_server, measure_memory_usage): +def test_xss(client, live_server, measure_memory_usage, datastore_path): from changedetectionio.notification import ( default_notification_format @@ -142,12 +142,12 @@ def test_xss(client, live_server, measure_memory_usage): assert b"<img" in res.data # Check that even forcing an update directly still doesnt get to the frontend - set_original_response() + set_original_response(datastore_path=datastore_path) XSS_HACK = 'javascript:alert(document.domain)' uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True)) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -162,8 +162,8 @@ def test_xss(client, live_server, measure_memory_usage): assert XSS_HACK.encode('utf-8') not in res.data and res.status_code == 200 -def test_xss_watch_last_error(client, live_server, measure_memory_usage): - set_original_response() +def test_xss_watch_last_error(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # Add our URL to the import page res = client.post( url_for("imports.import_page"), diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py index a1dd3c32..76cdc1cc 100644 --- a/changedetectionio/tests/test_share_watch.py +++ b/changedetectionio/tests/test_share_watch.py @@ -9,8 +9,9 @@ import re sleep_time_for_fetch_thread = 3 -def test_share_watch(client, live_server, measure_memory_usage): - set_original_response() +def test_share_watch(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + # live_server_setup(live_server) # Setup on conftest per function test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_source.py b/changedetectionio/tests/test_source.py index 4d4ef45a..77232dc0 100644 --- a/changedetectionio/tests/test_source.py +++ b/changedetectionio/tests/test_source.py @@ -9,8 +9,9 @@ sleep_time_for_fetch_thread = 3 -def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage): - set_original_response() +def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + test_url = 'source:'+url_for('test_endpoint', _external=True) # Add our URL to the import page uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -30,7 +31,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server, assert b'foobar-detection' in res.data # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Force recheck res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -52,8 +53,9 @@ def test_check_basic_change_detection_functionality_source(client, live_server, # `subtractive_selectors` should still work in `source:` type requests -def test_check_ignore_elements(client, live_server, measure_memory_usage): - set_original_response() +def test_check_ignore_elements(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + time.sleep(1) test_url = 'source:'+url_for('test_endpoint', _external=True) # Add our URL to the import page diff --git a/changedetectionio/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py index d44e7929..1213e789 100644 --- a/changedetectionio/tests/test_trigger.py +++ b/changedetectionio/tests/test_trigger.py @@ -3,9 +3,10 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -17,11 +18,11 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_original_ignore_response(): +def set_modified_original_ignore_response(datastore_path): test_return_data = """ Some NEW nice initial text
@@ -33,11 +34,11 @@ def set_modified_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_with_trigger_text_response(): +def set_modified_with_trigger_text_response(datastore_path): test_return_data = """ Some NEW nice initial text
@@ -51,16 +52,16 @@ def set_modified_with_trigger_text_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_trigger_functionality(client, live_server, measure_memory_usage): +def test_trigger_functionality(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function trigger_text = "Add to cart" - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page @@ -106,7 +107,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): assert b'/test-endpoint' in res.data # Make a change - set_modified_original_ignore_response() + set_modified_original_ignore_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -117,7 +118,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): assert b'has-unread-changes' not in res.data # Now set the content which contains the trigger text - set_modified_with_trigger_text_response() + set_modified_with_trigger_text_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_trigger_regex.py b/changedetectionio/tests/test_trigger_regex.py index 4f10a7de..acadd10b 100644 --- a/changedetectionio/tests/test_trigger_regex.py +++ b/changedetectionio/tests/test_trigger_regex.py @@ -3,9 +3,10 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -17,16 +18,16 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_trigger_regex_functionality(client, live_server, measure_memory_usage): +def test_trigger_regex_functionality(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -53,7 +54,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage): # so that we set the state to 'has-unread-changes' after all the edits client.get(url_for("ui.ui_views.diff_history_page", uuid="first")) - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("some new noise") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -63,7 +64,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage): res = client.get(url_for("watchlist.index")) assert b'has-unread-changes' not in res.data - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("regex test123
\nsomething 123") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_trigger_regex_with_filter.py b/changedetectionio/tests/test_trigger_regex_with_filter.py index a2f9d255..33908551 100644 --- a/changedetectionio/tests/test_trigger_regex_with_filter.py +++ b/changedetectionio/tests/test_trigger_regex_with_filter.py @@ -3,9 +3,10 @@ import time from flask import url_for from . util import live_server_setup, delete_all_watches +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """ Some initial text
@@ -17,17 +18,17 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage): +def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function sleep_time_for_fetch_thread = 3 - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Give the endpoint time to spin up time.sleep(1) @@ -57,7 +58,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me client.get(url_for("ui.ui_views.diff_history_page", uuid="first")) # Check that we have the expected text.. but it's not in the css filter we want - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("some new noise with cool stuff2 ok") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -68,7 +69,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me assert b'has-unread-changes' not in res.data # now this should trigger something - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write("some new noise with cool stuff6 ok") client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) diff --git a/changedetectionio/tests/test_ui.py b/changedetectionio/tests/test_ui.py index 9f7715fe..7243a837 100644 --- a/changedetectionio/tests/test_ui.py +++ b/changedetectionio/tests/test_ui.py @@ -5,7 +5,7 @@ from .util import set_original_response, set_modified_response, live_server_setu from ..forms import REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT, REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT -def test_recheck_time_field_validation_global_settings(client, live_server, measure_memory_usage): +def test_recheck_time_field_validation_global_settings(client, live_server, measure_memory_usage, datastore_path): """ Tests that the global settings time field has atleast one value for week/day/hours/minute/seconds etc entered class globalSettingsRequestForm(Form): @@ -27,7 +27,7 @@ def test_recheck_time_field_validation_global_settings(client, live_server, meas assert REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT.encode('utf-8') in res.data -def test_recheck_time_field_validation_single_watch(client, live_server, measure_memory_usage): +def test_recheck_time_field_validation_single_watch(client, live_server, measure_memory_usage, datastore_path): """ Tests that the global settings time field has atleast one value for week/day/hours/minute/seconds etc entered class globalSettingsRequestForm(Form): @@ -95,9 +95,10 @@ def test_recheck_time_field_validation_single_watch(client, live_server, measure assert b"Updated watch." in res.data assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') not in res.data -def test_checkbox_open_diff_in_new_tab(client, live_server, measure_memory_usage): +def test_checkbox_open_diff_in_new_tab(client, live_server, measure_memory_usage, datastore_path): - set_original_response() + set_original_response(datastore_path=datastore_path) + # Add our URL to the import page res = client.post( url_for("imports.import_page"), @@ -109,7 +110,7 @@ def test_checkbox_open_diff_in_new_tab(client, live_server, measure_memory_usage wait_for_all_checks(client) # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Test case 1 - checkbox is enabled in settings res = client.post( @@ -168,9 +169,9 @@ def test_checkbox_open_diff_in_new_tab(client, live_server, measure_memory_usage # Cleanup everything delete_all_watches(client) -def test_page_title_listing_behaviour(client, live_server, measure_memory_usage): +def test_page_title_listing_behaviour(client, live_server, measure_memory_usage, datastore_path): - set_original_response(extra_title="custom html") + set_original_response(extra_title="custom html", datastore_path=datastore_path) # either the manually entered title/description or the page link should be visible res = client.post( @@ -243,11 +244,11 @@ def test_page_title_listing_behaviour(client, live_server, measure_memory_usage) assert b"head titlecustom html" in res.data -def test_ui_viewed_unread_flag(client, live_server, measure_memory_usage): +def test_ui_viewed_unread_flag(client, live_server, measure_memory_usage, datastore_path): import time - set_original_response(extra_title="custom html") + set_original_response(datastore_path=datastore_path, extra_title="custom html") # Add our URL to the import page res = client.post( @@ -259,7 +260,7 @@ def test_ui_viewed_unread_flag(client, live_server, measure_memory_usage): assert b"2 Imported" in res.data wait_for_all_checks(client) - set_modified_response() + set_modified_response(datastore_path=datastore_path) res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) assert b'Queued 2 watches for rechecking.' in res.data wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_unique_lines.py b/changedetectionio/tests/test_unique_lines.py index cdec0744..285d3117 100644 --- a/changedetectionio/tests/test_unique_lines.py +++ b/changedetectionio/tests/test_unique_lines.py @@ -3,9 +3,10 @@ import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, delete_all_watches +import os -def set_original_ignore_response(): +def set_original_ignore_response(datastore_path): test_return_data = """

Some initial text

@@ -17,12 +18,12 @@ def set_original_ignore_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) # The same but just re-ordered the text -def set_modified_swapped_lines(): +def set_modified_swapped_lines(datastore_path): # Re-ordered and with some whitespacing, should get stripped() too. test_return_data = """ @@ -33,10 +34,10 @@ def set_modified_swapped_lines(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_swapped_lines_with_extra_text_for_sorting(): +def set_modified_swapped_lines_with_extra_text_for_sorting(datastore_path): test_return_data = """

 Which is across multiple lines

@@ -50,11 +51,11 @@ def set_modified_swapped_lines_with_extra_text_for_sorting(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -def set_modified_with_trigger_text_response(): +def set_modified_with_trigger_text_response(datastore_path): test_return_data = """

Some initial text

@@ -65,17 +66,17 @@ def set_modified_with_trigger_text_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function -def test_unique_lines_functionality(client, live_server, measure_memory_usage): +def test_unique_lines_functionality(client, live_server, measure_memory_usage, datastore_path): - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -96,7 +97,7 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage): assert b'has-unread-changes' not in res.data # Make a change - set_modified_swapped_lines() + set_modified_swapped_lines(datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -109,17 +110,17 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage): assert b'has-unread-changes' not in res.data # Now set the content which contains the new text and re-ordered existing text - set_modified_with_trigger_text_response() + set_modified_with_trigger_text_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) assert b'has-unread-changes' in res.data delete_all_watches(client) -def test_sort_lines_functionality(client, live_server, measure_memory_usage): +def test_sort_lines_functionality(client, live_server, measure_memory_usage, datastore_path): - set_modified_swapped_lines_with_extra_text_for_sorting() + set_modified_swapped_lines_with_extra_text_for_sorting(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -162,10 +163,10 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_extra_filters(client, live_server, measure_memory_usage): +def test_extra_filters(client, live_server, measure_memory_usage, datastore_path): - set_original_ignore_response() + set_original_ignore_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py index 1cfd6030..9cb32ecf 100644 --- a/changedetectionio/tests/test_watch_fields_storage.py +++ b/changedetectionio/tests/test_watch_fields_storage.py @@ -4,8 +4,9 @@ from urllib.request import urlopen from . util import set_original_response, set_modified_response, live_server_setup -def test_check_watch_field_storage(client, live_server, measure_memory_usage): - set_original_response() +def test_check_watch_field_storage(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) + # live_server_setup(live_server) # Setup on conftest per function test_url = "http://somerandomsitewewatch.com" diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index 105bcaf3..e8eabf0e 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -4,9 +4,10 @@ from flask import url_for from .util import wait_for_all_checks, delete_all_watches from ..processors.magic import RSS_XML_CONTENT_TYPES +import os -def set_rss_atom_feed_response(header=''): +def set_rss_atom_feed_response(datastore_path, header='', ): test_return_data = f"""{header} @@ -33,14 +34,14 @@ def set_rss_atom_feed_response(header=''): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_original_response(): +def set_original_response(datastore_path): test_return_data = """ Some initial text
@@ -53,12 +54,12 @@ def set_original_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ Some initial text
@@ -71,14 +72,14 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613 -def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage): +def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage, datastore_path): filter = '//item/*[self::description]' d = ''' @@ -108,7 +109,7 @@ def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage): ''' - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(d) # Add our URL to the import page @@ -129,7 +130,7 @@ def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage): # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613 -def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage): +def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage, datastore_path): filter = '//item/title/text()' d = ''' @@ -157,7 +158,7 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag ''' - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(d) # Add our URL to the import page @@ -187,10 +188,10 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag delete_all_watches(client) -def test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage): +def test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage, datastore_path): xpath_filter = "//*[contains(@class, 'sametext')]" - set_original_response() + set_original_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -216,7 +217,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo client.get(url_for("ui.ui_views.diff_history_page", uuid="first"), follow_redirects=True) # Make a change - set_modified_response() + set_modified_response(datastore_path=datastore_path) # Trigger a check client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -228,7 +229,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo delete_all_watches(client) -def test_xpath_validation(client, live_server, measure_memory_usage): +def test_xpath_validation(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -244,7 +245,7 @@ def test_xpath_validation(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_xpath23_prefix_validation(client, live_server, measure_memory_usage): +def test_xpath23_prefix_validation(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -259,7 +260,7 @@ def test_xpath23_prefix_validation(client, live_server, measure_memory_usage): assert b"is not a valid XPath expression" in res.data delete_all_watches(client) -def test_xpath1_lxml(client, live_server, measure_memory_usage): +def test_xpath1_lxml(client, live_server, measure_memory_usage, datastore_path): d = ''' @@ -287,7 +288,7 @@ def test_xpath1_lxml(client, live_server, measure_memory_usage): '''.encode('utf-8') - with open("test-datastore/endpoint-content.txt", "wb") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "wb") as f: f.write(d) @@ -319,7 +320,7 @@ def test_xpath1_lxml(client, live_server, measure_memory_usage): ##### -def test_xpath1_validation(client, live_server, measure_memory_usage): +def test_xpath1_validation(client, live_server, measure_memory_usage, datastore_path): # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) @@ -336,10 +337,10 @@ def test_xpath1_validation(client, live_server, measure_memory_usage): # actually only really used by the distll.io importer, but could be handy too -def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage): +def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage, datastore_path): delete_all_watches(client) - set_original_response() + set_original_response(datastore_path=datastore_path) wait_for_all_checks(client) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -368,10 +369,10 @@ def test_check_with_prefix_include_filters(client, live_server, measure_memory_u client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_various_rules(client, live_server, measure_memory_usage): +def test_various_rules(client, live_server, measure_memory_usage, datastore_path): # Just check these don't error ## live_server_setup(live_server) # Setup on conftest per function - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(""" Some initial text
@@ -412,13 +413,13 @@ def test_various_rules(client, live_server, measure_memory_usage): delete_all_watches(client) -def test_xpath_20(client, live_server, measure_memory_usage): +def test_xpath_20(client, live_server, measure_memory_usage, datastore_path): test_url = url_for('test_endpoint', _external=True) uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - set_original_response() + set_original_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) res = client.post( @@ -446,8 +447,8 @@ def test_xpath_20(client, live_server, measure_memory_usage): client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_xpath_20_function_count(client, live_server, measure_memory_usage): - set_original_response() +def test_xpath_20_function_count(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -479,8 +480,8 @@ def test_xpath_20_function_count(client, live_server, measure_memory_usage): client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_xpath_20_function_count2(client, live_server, measure_memory_usage): - set_original_response() +def test_xpath_20_function_count2(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -512,8 +513,8 @@ def test_xpath_20_function_count2(client, live_server, measure_memory_usage): client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage): - set_original_response() +def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage, datastore_path): + set_original_response(datastore_path=datastore_path) # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) @@ -546,7 +547,7 @@ def test_xpath_20_function_string_join_matches(client, live_server, measure_memo client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) -def _subtest_xpath_rss(client, content_type='text/html'): +def _subtest_xpath_rss(client, datastore_path, content_type='text/html'): # Add our URL to the import page test_url = url_for('test_endpoint', content_type=content_type, _external=True) @@ -584,8 +585,8 @@ def _subtest_xpath_rss(client, content_type='text/html'): client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) # Be sure all-in-the-wild types of RSS feeds work with xpath -def test_rss_xpath(client, live_server, measure_memory_usage): +def test_rss_xpath(client, live_server, measure_memory_usage, datastore_path): for feed_header in ['', '']: - set_rss_atom_feed_response(header=feed_header) + set_rss_atom_feed_response(header=feed_header, datastore_path=datastore_path) for content_type in RSS_XML_CONTENT_TYPES: - _subtest_xpath_rss(client, content_type=content_type) + _subtest_xpath_rss(client, content_type=content_type, datastore_path=datastore_path) diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 403a6bc8..4692b251 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 from operator import truediv -from flask import make_response, request +from flask import make_response, request, current_app from flask import url_for import logging import time +import os -def set_original_response(extra_title=''): +def set_original_response(datastore_path, extra_title=''): test_return_data = f""" head title{extra_title} @@ -19,11 +20,11 @@ def set_original_response(extra_title=''): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_modified_response(): +def set_modified_response(datastore_path): test_return_data = """ modified head title @@ -35,11 +36,11 @@ def set_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_longer_modified_response(): +def set_longer_modified_response(datastore_path): test_return_data = """ modified head title @@ -49,16 +50,17 @@ def set_longer_modified_response(): So let's see what happens.
So let's see what happens.
So let's see what happens.
- So let's see what happens.
+ So let's see what happens.
""" - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_more_modified_response(): + +def set_more_modified_response(datastore_path): test_return_data = """ modified head title @@ -71,27 +73,28 @@ def set_more_modified_response(): """ - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def set_empty_text_response(): +def set_empty_text_response(datastore_path): test_return_data = """""" - with open("test-datastore/endpoint-content.txt", "w") as f: + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: f.write(test_return_data) return None -def wait_for_notification_endpoint_output(): +def wait_for_notification_endpoint_output(datastore_path): '''Apprise can take a few seconds to fire''' #@todo - could check the apprise object directly instead of looking for this file from os.path import isfile + notification_file = os.path.join(datastore_path, "notification.txt") for i in range(1, 20): time.sleep(1) - if isfile("test-datastore/notification.txt"): + if isfile(notification_file): return True return False @@ -207,7 +210,8 @@ def new_live_server_setup(live_server): return resp # Tried using a global var here but didn't seem to work, so reading from a file instead. - with open("test-datastore/endpoint-content.txt", "rb") as f: + datastore_path = current_app.config.get('TEST_DATASTORE_PATH', 'test-datastore') + with open(os.path.join(datastore_path, "endpoint-content.txt"), "rb") as f: resp = make_response(f.read(), status_code) if uppercase_headers: resp.headers['CONTENT-TYPE'] = ctype if ctype else 'text/html' @@ -246,21 +250,22 @@ def new_live_server_setup(live_server): # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 ) @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET']) def test_notification_endpoint(): + datastore_path = current_app.config.get('TEST_DATASTORE_PATH', 'test-datastore') - with open("test-datastore/notification.txt", "wb") as f: + with open(os.path.join(datastore_path, "notification.txt"), "wb") as f: # Debug method, dump all POST to file also, used to prove #65 data = request.stream.read() if data != None: f.write(data) - with open("test-datastore/notification-url.txt", "w") as f: + with open(os.path.join(datastore_path, "notification-url.txt"), "w") as f: f.write(request.url) - with open("test-datastore/notification-headers.txt", "w") as f: + with open(os.path.join(datastore_path, "notification-headers.txt"), "w") as f: f.write(str(request.headers)) if request.content_type: - with open("test-datastore/notification-content-type.txt", "w") as f: + with open(os.path.join(datastore_path, "notification-content-type.txt"), "w") as f: f.write(request.content_type) print("\n>> Test notification endpoint was hit.\n", data) @@ -285,9 +290,10 @@ def new_live_server_setup(live_server): @live_server.app.route('/endpoint-test.pdf') def test_pdf_endpoint(): + datastore_path = current_app.config.get('TEST_DATASTORE_PATH', 'test-datastore') # Tried using a global var here but didn't seem to work, so reading from a file instead. - with open("test-datastore/endpoint-test.pdf", "rb") as f: + with open(os.path.join(datastore_path, "endpoint-test.pdf"), "rb") as f: resp = make_response(f.read(), 200) resp.headers['Content-Type'] = 'application/pdf' return resp diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index dedef9a3..2354e051 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -4,12 +4,12 @@ import os from flask import url_for from ..util import live_server_setup, wait_for_all_checks -# def test_setup(client, live_server, measure_memory_usage): +# def test_setup(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready -def test_visual_selector_content_ready(client, live_server, measure_memory_usage): +def test_visual_selector_content_ready(client, live_server, measure_memory_usage, datastore_path): import os import json @@ -54,11 +54,11 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage assert b"user-agent: mycustomagent" in res.data - assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist" - assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.deflate')), "xpath elements.deflate data should exist" + assert os.path.isfile(os.path.join(datastore_path, uuid, 'last-screenshot.png')), "last-screenshot.png should exist" + assert os.path.isfile(os.path.join(datastore_path, uuid, 'elements.deflate')), "xpath elements.deflate data should exist" # Open it and see if it roughly looks correct - with open(os.path.join('test-datastore', uuid, 'elements.deflate'), 'rb') as f: + with open(os.path.join(datastore_path, uuid, 'elements.deflate'), 'rb') as f: import zlib compressed_data = f.read() decompressed_data = zlib.decompress(compressed_data) @@ -86,7 +86,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage follow_redirects=True ) -def test_basic_browserstep(client, live_server, measure_memory_usage): +def test_basic_browserstep(client, live_server, measure_memory_usage, datastore_path): assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test" @@ -142,7 +142,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage): assert b"testheader: yes" in res.data assert b"user-agent: mycustomagent" in res.data -def test_non_200_errors_report_browsersteps(client, live_server, measure_memory_usage): +def test_non_200_errors_report_browsersteps(client, live_server, measure_memory_usage, datastore_path): four_o_four_url = url_for('test_endpoint', status_code=404, _external=True) diff --git a/requirements.txt b/requirements.txt index c17d3546..3da266be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -144,3 +144,5 @@ pre_commit >= 4.2.0 # For events between checking and socketio updates blinker + +pytest-xdist