Files
Tianlang (Jackie) ChenandTianlang Chen 984112f7af fix(file): preserve semicolons in quoted remote filenames (#944)
Parse Content-Disposition parameters with the standard library instead of splitting on every semicolon. Quoted filenames such as `report; final.pdf` must retain the semicolon and extension. Keep the existing filename sanitization and URL fallback, with regression coverage for extra parameters, equals signs, traversal-like prefixes and empty filenames.

Signed-off-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com>
Co-authored-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com>
2026-11-12 07:49:17 +01:00

596 lines
21 KiB
Python

"""Test the pydantic models in package utils."""
import json
import socket
from io import BytesIO
from pathlib import Path
import pytest
from pydantic import AnyHttpUrl, Field
from requests import Response, Session
from docling_core.types.doc import DocItemLabel, DoclingDocument, TableData
from docling_core.types.doc.document import GroupLabel, RichTableCell
from docling_core.utils.alias import AliasModel
from docling_core.utils.file import (
_MAX_REDIRECTS,
_is_safe_url,
_sanitize_filename,
resolve_remote_filename,
resolve_source_to_path,
resolve_source_to_stream,
)
from .test_data_gen_flag import GEN_TEST_DATA
def _closeable_response(status_code=200, content=b"", headers=None) -> Response:
"""Build a Response with a real, closeable ``raw`` like a streamed response.
``resolve_source_to_stream`` closes the response (via ``with``) to release the
streamed connection, which a bare ``Response()`` (``raw is None``) cannot do.
"""
r = Response()
r.status_code = status_code
r._content = content
r._content_consumed = True
r.raw = BytesIO(content)
if headers:
r.headers.update(headers)
return r
def build_single_cell_rich_table_doc(text: str) -> DoclingDocument:
"""Build a doc with a single-cell table whose cell is a RichTableCell ref to a text group."""
doc = DoclingDocument(name="single_cell_rich_table")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
wrapper = doc.add_group(parent=table, label=GroupLabel.UNSPECIFIED)
doc.add_text(parent=wrapper, label=DocItemLabel.TEXT, text=text)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
ref=wrapper.get_ref(),
text="",
),
)
return doc
def assert_or_generate_ground_truth(
result_text: str,
exp_path: Path,
error_msg: str = "Serialized text should match expected output",
is_json: bool = False,
) -> None:
"""Helper function to either generate ground truth or assert against it.
This function supports the GEN_TEST_DATA pattern for test maintenance:
- When GEN_TEST_DATA=1, it writes the result to the ground truth file
- Otherwise, it reads the ground truth file and asserts equality
Args:
result_text: The serialized result text to compare or save
exp_path: Path to the expected/ground truth file
error_msg: Error message to display if assertion fails
is_json: If True, compare as JSON objects instead of raw strings
"""
if GEN_TEST_DATA:
with open(exp_path, "w", encoding="utf-8") as f:
f.write(result_text)
else:
with open(exp_path, encoding="utf-8") as f:
if is_json:
expected = json.load(f)
actual = json.loads(result_text)
assert actual == expected, error_msg
else:
expected = f.read()
assert result_text == expected, error_msg
def assert_or_generate_json_ground_truth(
result_data: dict | list,
exp_path: Path | str,
error_msg: str = "JSON data should match expected output",
) -> None:
"""Helper function to either generate JSON ground truth or assert against it.
This is a convenience wrapper for JSON data that handles serialization with
proper formatting (indent=4, trailing newline).
Args:
result_data: The data structure (dict or list) to compare or save
exp_path: Path to the expected/ground truth JSON file
error_msg: Error message to display if assertion fails
"""
exp_path = Path(exp_path) if isinstance(exp_path, str) else exp_path
if GEN_TEST_DATA:
with open(exp_path, "w", encoding="utf-8") as f:
json.dump(result_data, f, indent=4)
f.write("\n")
else:
with open(exp_path, encoding="utf-8") as f:
expected = json.load(f)
assert result_data == expected, error_msg
def test_alias_model() -> None:
"""Test the functionality of AliasModel."""
class AliasModelChild(AliasModel):
foo: str = Field(alias="boo")
data = {"foo": "lorem ipsum"}
data_alias = {"boo": "lorem ipsum"}
# data validated from dict, JSON, and constructor can use field names or aliases
AliasModelChild.model_validate(data_alias)
AliasModelChild.model_validate(data)
AliasModelChild.model_validate_json(json.dumps(data_alias))
AliasModelChild.model_validate_json(json.dumps(data))
AliasModelChild(boo="lorem ipsum") # type: ignore[call-arg]
AliasModelChild(foo="lorem ipsum")
# children classes will also inherite the populate_by_name
class AliasModelGrandChild(AliasModelChild):
var: int
AliasModelGrandChild(boo="lorem ipsum", var=3) # type: ignore[call-arg]
AliasModelGrandChild(foo="lorem ipsum", var=3)
# serialized data will always use aliases
obj = AliasModelChild.model_validate(data_alias)
assert obj.model_dump() == data_alias
assert obj.model_dump() != data
assert obj.model_dump_json() == json.dumps(data_alias, separators=(",", ":"))
assert obj.model_dump_json() != json.dumps(data, separators=(",", ":"))
def test_resolve_source_to_path_url_wout_path(monkeypatch):
expected_str = "foo"
expected_bytes = bytes(expected_str, "utf-8")
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr(Session, "get", lambda *args, **kwargs: _closeable_response(content=expected_bytes))
path = resolve_source_to_path("https://pypi.org")
with open(path, encoding="utf-8") as f:
text = f.read()
assert text == expected_str
def test_resolve_source_to_stream_url_wout_path(monkeypatch):
expected_str = "foo"
expected_bytes = bytes(expected_str, "utf-8")
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr(Session, "get", lambda *args, **kwargs: _closeable_response(content=expected_bytes))
doc_stream = resolve_source_to_stream("https://pypi.org")
assert doc_stream.name == "file"
text = doc_stream.stream.read().decode("utf8")
assert text == expected_str
def test_sanitize_filename_paths():
"""Test filename sanitization for path-like inputs."""
assert _sanitize_filename("../../etc/config.txt") == "config.txt"
assert _sanitize_filename("/etc/config.txt") == "config.txt"
assert _sanitize_filename("..\\..\\windows\\system32\\config") == "config"
assert _sanitize_filename("C:\\Windows\\System32\\config") == "config"
assert _sanitize_filename("../../../etc\\config.txt") == "config.txt"
assert _sanitize_filename("document.pdf") == "document.pdf"
assert _sanitize_filename("my-file_123.txt") == "my-file_123.txt"
assert _sanitize_filename("") is None
assert _sanitize_filename(".") is None
assert _sanitize_filename("..") is None
def _make_getaddrinfo(mapping: dict):
"""Return a getaddrinfo stub that resolves hostnames from a fixed mapping.
Each mapping value is a list of (family, address) tuples. Raises
socket.gaierror for hosts not in the mapping.
"""
def _getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
if host not in mapping:
raise socket.gaierror(f"Name or service not known: {host}")
return [(fam, socket.SOCK_STREAM, 0, "", (addr, port or 0)) for fam, addr in mapping[host]]
return _getaddrinfo
def test_is_safe_url_rejects_private_networks(monkeypatch):
"""Test URL filtering for non-public network ranges."""
monkeypatch.setattr(
socket,
"getaddrinfo",
_make_getaddrinfo(
{
"example.com": [(socket.AF_INET, "93.184.216.34")],
"github.com": [(socket.AF_INET, "140.82.121.4")],
"localhost": [(socket.AF_INET, "127.0.0.1")],
}
),
)
assert not _is_safe_url("http://10.0.0.1/file")
assert not _is_safe_url("http://172.16.0.1/file")
assert not _is_safe_url("http://192.168.1.1/file")
assert not _is_safe_url("http://127.0.0.1/file")
assert not _is_safe_url("http://localhost/file")
assert not _is_safe_url("http://169.254.169.254/latest/meta-data/")
assert not _is_safe_url("http://[::1]/file")
assert not _is_safe_url("http://[fe80::1]/file")
assert _is_safe_url("http://8.8.8.8/file")
assert _is_safe_url("https://example.com/file")
assert _is_safe_url("https://github.com/github/file")
def test_is_safe_url_rejects_dual_stack_with_private_ipv6(monkeypatch):
"""A host with a public A record but a private AAAA record must be rejected."""
monkeypatch.setattr(
socket,
"getaddrinfo",
_make_getaddrinfo(
{
# Public IPv4, loopback IPv6 — the AAAA record is private.
"dual.example": [
(socket.AF_INET6, "::1"),
(socket.AF_INET, "1.1.1.1"),
],
# Public IPv4, IPv4-mapped loopback AAAA record.
"mapped.example": [
(socket.AF_INET, "1.1.1.1"),
(socket.AF_INET6, "::ffff:127.0.0.1"),
],
}
),
)
assert not _is_safe_url("http://dual.example/file")
assert not _is_safe_url("http://mapped.example/file")
def test_is_safe_url_accepts_ipv6_only_host(monkeypatch):
"""A host with only a public AAAA record must be accepted."""
monkeypatch.setattr(
socket,
"getaddrinfo",
_make_getaddrinfo(
{
"ipv6only.example": [(socket.AF_INET6, "2606:4700:4700::1111")],
}
),
)
assert _is_safe_url("https://ipv6only.example/file")
def test_is_safe_url_rejects_ipv4_mapped_loopback_literal():
"""A literal IPv4-mapped loopback address in the URL must be rejected."""
assert not _is_safe_url("http://[::ffff:127.0.0.1]/file")
assert not _is_safe_url("http://[::ffff:10.0.0.1]/file")
def test_is_safe_url_allowlist_applies_to_ipv6_mapped_address(monkeypatch):
"""An allowlisted IPv4 address reached via an IPv4-mapped AAAA record is permitted."""
from docling_core.utils.settings import settings
monkeypatch.setattr(settings, "allowed_private_ips", ["10.0.0.1"])
monkeypatch.setattr(
socket,
"getaddrinfo",
_make_getaddrinfo(
{
"internal.example": [(socket.AF_INET6, "::ffff:10.0.0.1")],
}
),
)
assert _is_safe_url("http://internal.example/file")
def test_ip_in_allowlist():
import ipaddress
from docling_core.utils.file import _ip_in_allowlist
assert _ip_in_allowlist(ipaddress.ip_address("127.0.0.1"), ["127.0.0.1"])
assert not _ip_in_allowlist(ipaddress.ip_address("127.0.0.1"), ["127.0.0.2"])
assert _ip_in_allowlist(ipaddress.ip_address("10.20.30.40"), ["10.0.0.0/8"])
assert not _ip_in_allowlist(ipaddress.ip_address("10.21.0.1"), ["10.20.0.0/16"])
assert _ip_in_allowlist(ipaddress.ip_address("192.168.1.100"), ["192.168.1.0/24"])
assert _ip_in_allowlist(ipaddress.ip_address("192.168.1.100"), ["192.168.1.5/24"])
assert _ip_in_allowlist(ipaddress.ip_address("127.0.0.1"), ["garbage", "127.0.0.1"])
assert not _ip_in_allowlist(ipaddress.ip_address("127.0.0.1"), ["garbage", "300.300.300.300"])
def test_allowed_private_ips(monkeypatch):
import pytest
from docling_core.utils.file import resolve_source_to_stream
from docling_core.utils.settings import settings
monkeypatch.setattr(Session, "get", lambda *args, **kwargs: _closeable_response(content=b"ok"))
monkeypatch.setattr(settings, "allowed_private_ips", [])
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://10.0.0.1/file")
monkeypatch.setattr(settings, "allowed_private_ips", [""])
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://10.0.0.1/file")
monkeypatch.setattr(settings, "allowed_private_ips", ["127.0.0.1"])
assert resolve_source_to_stream("http://127.0.0.1/file").stream.read() == b"ok"
monkeypatch.setattr(settings, "allowed_private_ips", ["192.168.1.0/24"])
assert resolve_source_to_stream("http://192.168.1.42/doc").stream.read() == b"ok"
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://192.168.2.1/doc")
def test_resolve_remote_filename_sanitizes_content_disposition(monkeypatch):
"""Test filename normalization from Content-Disposition."""
def get_response(*args, **kwargs):
return _closeable_response(
content=b"test content",
headers={"Content-Disposition": 'attachment; filename="../../etc/config.txt"'},
)
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr("requests.Session.get", get_response)
doc_stream = resolve_source_to_stream("https://example.com/file")
assert doc_stream.name == "config.txt"
def test_resolve_source_rejects_non_public_urls(monkeypatch):
"""Test that non-public URLs are rejected."""
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://127.0.0.1/file")
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://10.0.0.1/file")
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://192.168.1.1/file")
with pytest.raises(ValueError, match="URL is not allowed"):
resolve_source_to_stream("http://169.254.169.254/latest/meta-data/")
def test_resolve_source_rejects_unsupported_scheme():
"""Test that unsupported URL schemes are rejected before file fallback."""
with pytest.raises(ValueError, match="Unsupported URL scheme"):
resolve_source_to_stream("ftp://some-server/file.pdf")
def test_resolve_source_to_path_sanitizes_filename(monkeypatch, tmp_path):
"""Test that saved filenames stay within the target directory."""
def get_response(*args, **kwargs):
return _closeable_response(
content=b"test content",
headers={"Content-Disposition": 'attachment; filename="../../../../tmp/output.txt"'},
)
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr("requests.Session.get", get_response)
cache_dir = tmp_path / "cache"
cache_dir.mkdir()
result_path = resolve_source_to_path("https://example.com/file", workdir=cache_dir)
assert result_path.parent == cache_dir
assert result_path.name == "output.txt"
assert result_path.exists()
assert not (tmp_path.parent.parent.parent / "tmp" / "output.txt").exists()
def test_redirect_limit_enforced(monkeypatch):
"""Test that redirect limits are configured on the session."""
session_created = []
original_init = Session.__init__
def track_session_init(self, *args, **kwargs):
original_init(self, *args, **kwargs)
session_created.append(self)
monkeypatch.setattr(Session, "__init__", track_session_init)
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr(Session, "get", lambda *args, **kwargs: _closeable_response(content=b"test"))
try:
resolve_source_to_stream("https://example.com/file")
except Exception:
pass
assert len(session_created) > 0
session = session_created[0]
assert session.max_redirects == _MAX_REDIRECTS
def test_redirect_to_non_public_ip_rejected(monkeypatch):
"""Test that redirects to non-public addresses are rejected."""
def mock_get_with_redirect(self, *args, **kwargs):
r = Response()
r.status_code = 302
r.headers["location"] = "http://192.168.1.1/private-file"
r.url = args[0] if args else kwargs.get("url", "http://example.com")
if hasattr(self, "hooks") and "response" in self.hooks:
for hook in self.hooks["response"]:
hook(r)
return r
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: url != "http://192.168.1.1/private-file")
monkeypatch.setattr(Session, "get", mock_get_with_redirect)
with pytest.raises(ValueError, match="Redirect target is not allowed"):
resolve_source_to_stream("https://example.com/redirect")
def test_resolve_source_rejects_large_content_length(monkeypatch):
def mock_get(self, *args, **kwargs):
return _closeable_response(content=b"ignored", headers={"Content-Length": "10"})
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr(Session, "get", mock_get)
with pytest.raises(ValueError, match="maximum allowed size"):
resolve_source_to_stream("https://example.com/file", max_file_size=5)
def test_resolve_source_stops_when_stream_exceeds_limit(monkeypatch):
def mock_get(self, *args, **kwargs):
r = _closeable_response(headers={"Content-Length": "4"})
def iter_content(chunk_size=1):
yield b"abcd"
yield b"ef"
r.iter_content = iter_content
return r
monkeypatch.setattr("docling_core.utils.file._is_safe_url", lambda url: True)
monkeypatch.setattr(Session, "get", mock_get)
with pytest.raises(ValueError, match="maximum allowed size"):
resolve_source_to_stream("https://example.com/file", max_file_size=5)
def test_connect_time_validation_blocks_dns_rebinding(monkeypatch):
"""A DNS rebinding attempt must be rejected at connect time.
The first getaddrinfo call (made by _is_safe_url) returns a public IP so
the pre-flight check passes. The second call (made by urllib3's
create_connection) returns loopback, simulating a DNS server that gives
different answers to successive queries.
"""
call_count = 0
def rebinding_getaddrinfo(host, port, *args, **kwargs):
nonlocal call_count
call_count += 1
addr = "1.1.1.1" if call_count == 1 else "127.0.0.1"
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (addr, port or 0))]
monkeypatch.setattr(socket, "getaddrinfo", rebinding_getaddrinfo)
with pytest.raises(ValueError, match="Connect-time address is not allowed"):
resolve_source_to_stream("https://example.com/file")
def test_connect_time_validation_connects_to_validated_address(monkeypatch):
"""The socket must go to the validated address, not to a later DNS answer.
The resolver answers with a public IP for the pre-flight and connect-time
lookups, and with loopback afterwards. A loopback-only server must never be
reached.
"""
import http.server
import threading
hits = []
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
hits.append(self.path)
self.send_response(200)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, *args):
pass
server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
port = server.server_address[1]
threading.Thread(target=server.serve_forever, daemon=True).start()
call_count = 0
real_getaddrinfo = socket.getaddrinfo
def rebinding_getaddrinfo(host, port, *args, **kwargs):
nonlocal call_count
if host != "rebind.example":
return real_getaddrinfo(host, port, *args, **kwargs)
call_count += 1
addr = "192.0.2.1" if call_count <= 2 else "127.0.0.1"
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (addr, port or 0))]
monkeypatch.setattr(socket, "getaddrinfo", rebinding_getaddrinfo)
# 192.0.2.0/24 is non-routable in practice; patch it as safe and make the
# connect fail fast instead of waiting for a timeout.
monkeypatch.setattr("docling_core.utils.file._is_address_safe", lambda ip: str(ip) == "192.0.2.1")
real_connect = socket.socket.connect
def fast_fail_connect(self, sockaddr):
if sockaddr[0] == "192.0.2.1":
raise ConnectionRefusedError("unreachable in test")
return real_connect(self, sockaddr)
monkeypatch.setattr(socket.socket, "connect", fast_fail_connect)
try:
with pytest.raises(Exception):
resolve_source_to_stream(f"http://rebind.example:{port}/file")
finally:
server.shutdown()
assert hits == []
@pytest.mark.parametrize(
("disposition", "expected"),
[
('attachment; filename="report; final.pdf"', "report; final.pdf"),
('attachment; filename="a=b; c.pdf"; size=123', "a=b; c.pdf"),
('attachment; filename="report; final.pdf"; creation-date="today"', "report; final.pdf"),
('attachment; filename="../../report; final.pdf"', "report; final.pdf"),
('attachment; filename=""', "download.pdf"),
],
)
def test_resolve_remote_filename_quoted_parameters(disposition: str, expected: str) -> None:
"""Quoted semicolons belong to the filename, not to the parameter list."""
assert (
resolve_remote_filename(
AnyHttpUrl("https://example.com/download.pdf"),
{"Content-Disposition": disposition},
)
== expected
)