mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
Parse Content-Disposition parameters with the standard library instead of splitting on every semicolon. Quoted filenames such as `report; final.pdf` must retain the semicolon and extension. Keep the existing filename sanitization and URL fallback, with regression coverage for extra parameters, equals signs, traversal-like prefixes and empty filenames. Signed-off-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com> Co-authored-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com>
452 lines
16 KiB
Python
452 lines
16 KiB
Python
"""File-related utilities."""
|
|
|
|
import ipaddress
|
|
import logging
|
|
import re
|
|
import socket
|
|
import tempfile
|
|
from email.message import Message
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import Optional, Union
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
from pydantic import AnyHttpUrl, TypeAdapter, ValidationError
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.utils import select_proxy
|
|
from typing_extensions import deprecated
|
|
|
|
from docling_core.types.doc.utils import relative_path
|
|
from docling_core.types.io import DocumentStream
|
|
from docling_core.utils.settings import settings
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
_MAX_REDIRECTS = 5
|
|
|
|
# Chunk size for streaming remote downloads. Sized for document payloads (PDFs
|
|
# and similar), which are either small or in the multi-megabyte range, so a
|
|
# large chunk keeps the read loop short without wasting memory.
|
|
_DOWNLOAD_CHUNK_SIZE = 512 * 1024
|
|
|
|
|
|
class FileSizeLimitExceededError(ValueError):
|
|
"""Raised when a remote file exceeds the configured download size limit."""
|
|
|
|
def __init__(self, filename: str, size: int, limit: int):
|
|
self.filename = filename
|
|
self.size = size
|
|
self.limit = limit
|
|
super().__init__(f"Remote file exceeds the maximum allowed size ({size} > {limit} bytes).")
|
|
|
|
|
|
def _ip_in_allowlist(ip: ipaddress.IPv4Address | ipaddress.IPv6Address, allowlist: list[str]) -> bool:
|
|
"""Return whether a ip matches any IP addresss or CIDR entry in allowlist."""
|
|
for entry in allowlist:
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
try:
|
|
network = ipaddress.ip_network(entry, strict=False)
|
|
normalized = str(network)
|
|
if normalized != entry:
|
|
_logger.warning(
|
|
f"DOCLINGCORE_ALLOWED_PRIVATE_IPS entry {entry!r} was normalized to "
|
|
f"{normalized!r}. Consider using the explicit network address."
|
|
)
|
|
if ip in network:
|
|
return True
|
|
except ValueError:
|
|
_logger.warning(f"Skipping malformed entry in DOCLINGCORE_ALLOWED_PRIVATE_IPS: {entry!r} ")
|
|
return False
|
|
|
|
|
|
def _is_address_safe(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
|
"""Check whether a single resolved IP address is globally routable.
|
|
|
|
IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1) are unwrapped and
|
|
evaluated as their underlying IPv4 address.
|
|
|
|
Args:
|
|
ip: The IP address to evaluate.
|
|
|
|
Returns:
|
|
True if the address is safe to connect to, False otherwise.
|
|
"""
|
|
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
|
ip = ip.ipv4_mapped
|
|
|
|
if settings.allowed_private_ips and _ip_in_allowlist(ip, settings.allowed_private_ips):
|
|
return True
|
|
|
|
return ip.is_global and not (
|
|
ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
|
|
)
|
|
|
|
|
|
def _is_safe_url(url: str) -> bool:
|
|
"""Check whether every address a URL's hostname resolves to is globally routable.
|
|
|
|
Args:
|
|
url: The URL to validate.
|
|
|
|
Returns:
|
|
True if the URL is safe to fetch, False otherwise.
|
|
"""
|
|
try:
|
|
parsed = urlparse(url)
|
|
hostname = parsed.hostname
|
|
|
|
if not hostname:
|
|
return False
|
|
|
|
try:
|
|
ip = ipaddress.ip_address(hostname)
|
|
return _is_address_safe(ip)
|
|
except ValueError:
|
|
pass
|
|
|
|
try:
|
|
results = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
|
except (socket.gaierror, socket.herror):
|
|
return False
|
|
|
|
if not results:
|
|
return False
|
|
|
|
for _family, _type, _proto, _canonname, sockaddr in results:
|
|
try:
|
|
ip = ipaddress.ip_address(sockaddr[0])
|
|
except ValueError:
|
|
return False
|
|
if not _is_address_safe(ip):
|
|
return False
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _sanitize_filename(filename: str) -> str | None:
|
|
"""Return a basename-safe filename, or None if no usable basename remains."""
|
|
normalized = filename.replace("\\", "/")
|
|
basename = Path(normalized).name
|
|
|
|
if not basename or basename in (".", "..") or "/" in basename:
|
|
return None
|
|
|
|
return basename
|
|
|
|
|
|
def resolve_remote_filename(
|
|
http_url: AnyHttpUrl,
|
|
response_headers: dict[str, str],
|
|
fallback_filename="file",
|
|
) -> str:
|
|
"""Resolves the filename from a remote url and its response headers.
|
|
|
|
Args:
|
|
source AnyHttpUrl: The source http url.
|
|
response_headers Dict: Headers received while fetching the remote file.
|
|
fallback_filename str: Filename to use in case none can be determined.
|
|
|
|
Returns:
|
|
str: The actual filename of the remote url.
|
|
"""
|
|
raw_fname = None
|
|
if cont_disp := response_headers.get("Content-Disposition"):
|
|
message = Message()
|
|
message["Content-Disposition"] = cont_disp
|
|
raw_fname = message.get_filename() or None
|
|
|
|
if raw_fname is None:
|
|
raw_fname = Path(http_url.path or "").name or fallback_filename
|
|
|
|
if fname := _sanitize_filename(raw_fname):
|
|
return fname
|
|
|
|
if fname := _sanitize_filename(fallback_filename):
|
|
return fname
|
|
|
|
raise ValueError("Could not derive a safe filename")
|
|
|
|
|
|
def _resolve_safe_addresses(host: str, port: int | None) -> list[str]:
|
|
"""Resolve a host once and return its addresses, rejecting it if any is not allowed.
|
|
|
|
Args:
|
|
host: The hostname or IP address to resolve.
|
|
port: The port to resolve for.
|
|
|
|
Returns:
|
|
The resolved addresses, in resolver order.
|
|
"""
|
|
try:
|
|
results = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
|
except (socket.gaierror, socket.herror) as exc:
|
|
raise ValueError(f"Could not resolve host at connect time: {host}") from exc
|
|
|
|
addresses = []
|
|
for _family, _type, _proto, _canonname, sockaddr in results:
|
|
addr_str = str(sockaddr[0])
|
|
try:
|
|
ip = ipaddress.ip_address(addr_str)
|
|
except ValueError:
|
|
raise ValueError(f"Unexpected address at connect time: {addr_str}") from None
|
|
if not _is_address_safe(ip):
|
|
raise ValueError(f"Connect-time address is not allowed: {addr_str}")
|
|
addresses.append(addr_str)
|
|
|
|
if not addresses:
|
|
raise ValueError(f"No addresses returned for host: {host}")
|
|
return addresses
|
|
|
|
|
|
class _SafeConnectionAdapter(HTTPAdapter):
|
|
"""HTTPAdapter that connects directly to a validated address.
|
|
|
|
Prevents DNS rebinding: the host is resolved and validated once, and the
|
|
connection pool is opened to the validated IP itself, so no later DNS
|
|
lookup can change the target. The original hostname is kept for the
|
|
``Host`` header and for TLS (SNI and certificate verification).
|
|
|
|
Only the first resolved address is used, with no fallback to the others.
|
|
A dual-stack host whose first address is unreachable (e.g. IPv6 without
|
|
a working route) fails instead of falling back to another address family.
|
|
|
|
Proxied requests connect to the proxy, which resolves the target itself,
|
|
so they rely on the pre-flight check alone.
|
|
"""
|
|
|
|
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
|
if select_proxy(request.url, proxies):
|
|
return super().get_connection_with_tls_context(request, verify, proxies, cert)
|
|
|
|
host_params, pool_kwargs = self.build_connection_pool_key_attributes(request, verify, cert)
|
|
host = host_params["host"]
|
|
address = _resolve_safe_addresses(host, host_params["port"])[0]
|
|
if host_params["scheme"] == "https":
|
|
pool_kwargs = {**pool_kwargs, "server_hostname": host, "assert_hostname": host}
|
|
request.headers["Host"] = urlparse(request.url).netloc.rpartition("@")[2]
|
|
|
|
return self.poolmanager.connection_from_host(
|
|
host=address,
|
|
port=host_params["port"],
|
|
scheme=host_params["scheme"],
|
|
pool_kwargs=pool_kwargs,
|
|
)
|
|
|
|
|
|
def resolve_source_to_stream(
|
|
source: Path | AnyHttpUrl | str,
|
|
headers: dict[str, str] | None = None,
|
|
max_file_size: int | None = None,
|
|
) -> DocumentStream:
|
|
"""Resolves the source (URL, path) of a file to a binary stream.
|
|
|
|
Args:
|
|
source: The file input source. Can be a path or URL.
|
|
headers: Optional set of headers to use for fetching the remote URL.
|
|
max_file_size: Optional maximum size, in bytes, for a remote download.
|
|
When set, the download is rejected upfront if the declared
|
|
``Content-Length`` exceeds it, and aborted while streaming as soon as
|
|
the received bytes exceed it.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
FileSizeLimitExceededError: If a remote download exceeds ``max_file_size``.
|
|
|
|
Returns:
|
|
DocumentStream: The resolved file loaded as a stream.
|
|
"""
|
|
try:
|
|
http_url: AnyHttpUrl = TypeAdapter(AnyHttpUrl).validate_python(source)
|
|
url_str = str(http_url)
|
|
|
|
if not _is_safe_url(url_str):
|
|
raise ValueError(f"URL is not allowed: {url_str}")
|
|
|
|
_headers = headers or {}
|
|
req_headers = {k.lower(): v for k, v in _headers.items()}
|
|
if "user-agent" not in req_headers:
|
|
try:
|
|
import importlib.metadata
|
|
|
|
agent_name = f"docling-core/{importlib.metadata.version('docling-core')}"
|
|
except Exception:
|
|
agent_name = "docling-core"
|
|
req_headers["user-agent"] = agent_name
|
|
|
|
google_doc_id = re.search(
|
|
r"google\.com\/(file|document|spreadsheets|presentation)\/d\/([\w-]+)",
|
|
url_str,
|
|
)
|
|
if google_doc_id:
|
|
doc_type = google_doc_id.group(1)
|
|
doc_id = google_doc_id.group(2)
|
|
|
|
if doc_type == "file":
|
|
url_str = f"https://drive.google.com/uc?export=download&id={doc_id}"
|
|
elif doc_type == "document":
|
|
url_str = f"https://docs.google.com/document/d/{doc_id}/export?format=docx"
|
|
elif doc_type == "spreadsheets":
|
|
url_str = f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=xlsx"
|
|
elif doc_type == "presentation":
|
|
url_str = f"https://docs.google.com/presentation/d/{doc_id}/export?format=pptx"
|
|
else:
|
|
raise ValueError(f"Unexpected Google doc type: {doc_type}")
|
|
|
|
http_url = TypeAdapter(AnyHttpUrl).validate_python(url_str)
|
|
|
|
def _check_redirect_safety(response, *args, **kwargs):
|
|
"""Validate each redirect target before following it."""
|
|
if response.is_redirect or response.is_permanent_redirect:
|
|
redirect_url = response.headers.get("location")
|
|
if redirect_url:
|
|
if not redirect_url.startswith(("http://", "https://")):
|
|
from urllib.parse import urljoin
|
|
|
|
redirect_url = urljoin(response.url, redirect_url)
|
|
|
|
if not _is_safe_url(redirect_url):
|
|
raise ValueError(f"Redirect target is not allowed: {redirect_url}")
|
|
|
|
with requests.Session() as session:
|
|
session.max_redirects = _MAX_REDIRECTS
|
|
session.hooks["response"].append(_check_redirect_safety)
|
|
session.mount("https://", _SafeConnectionAdapter())
|
|
session.mount("http://", _SafeConnectionAdapter())
|
|
|
|
with session.get(
|
|
url_str,
|
|
stream=True,
|
|
headers=req_headers,
|
|
allow_redirects=True,
|
|
) as res:
|
|
res.raise_for_status()
|
|
|
|
response_headers = dict(res.headers)
|
|
fname = resolve_remote_filename(http_url=http_url, response_headers=response_headers)
|
|
|
|
if max_file_size is not None:
|
|
content_length = res.headers.get("Content-Length")
|
|
if content_length is not None:
|
|
try:
|
|
content_length_value = int(content_length)
|
|
except ValueError:
|
|
content_length_value = None
|
|
|
|
if content_length_value is not None and content_length_value > max_file_size:
|
|
raise FileSizeLimitExceededError(
|
|
filename=fname,
|
|
size=content_length_value,
|
|
limit=max_file_size,
|
|
)
|
|
|
|
stream = BytesIO()
|
|
downloaded = 0
|
|
for chunk in res.iter_content(chunk_size=_DOWNLOAD_CHUNK_SIZE):
|
|
if not chunk:
|
|
continue
|
|
downloaded += len(chunk)
|
|
if max_file_size is not None and downloaded > max_file_size:
|
|
raise FileSizeLimitExceededError(
|
|
filename=fname,
|
|
size=downloaded,
|
|
limit=max_file_size,
|
|
)
|
|
stream.write(chunk)
|
|
stream.seek(0)
|
|
doc_stream = DocumentStream(name=fname, stream=stream)
|
|
except ValidationError:
|
|
if isinstance(source, str) and "://" in source:
|
|
scheme = source.split("://", 1)[0].lower()
|
|
if scheme not in ("http", "https"):
|
|
raise ValueError(f"Unsupported URL scheme: '{scheme}'. Only http:// and https:// are supported.")
|
|
try:
|
|
local_path = TypeAdapter(Path).validate_python(source)
|
|
stream = BytesIO(local_path.read_bytes())
|
|
doc_stream = DocumentStream(name=local_path.name, stream=stream)
|
|
except ValidationError:
|
|
raise ValueError(f"Unexpected source type encountered: {type(source)}")
|
|
return doc_stream
|
|
|
|
|
|
def _resolve_source_to_path(
|
|
source: Path | AnyHttpUrl | str,
|
|
headers: dict[str, str] | None = None,
|
|
workdir: Path | None = None,
|
|
) -> Path:
|
|
doc_stream = resolve_source_to_stream(source=source, headers=headers)
|
|
|
|
# use a temporary directory if not specified
|
|
if workdir is None:
|
|
workdir = Path(tempfile.mkdtemp())
|
|
|
|
# create the parent workdir if it doesn't exist
|
|
workdir.mkdir(exist_ok=True, parents=True)
|
|
|
|
# save result to a local file
|
|
local_path = workdir / doc_stream.name
|
|
with local_path.open("wb") as f:
|
|
f.write(doc_stream.stream.read())
|
|
|
|
return local_path
|
|
|
|
|
|
def resolve_source_to_path(
|
|
source: Path | AnyHttpUrl | str,
|
|
headers: dict[str, str] | None = None,
|
|
workdir: Path | None = None,
|
|
) -> Path:
|
|
"""Resolves the source (URL, path) of a file to a local file path.
|
|
|
|
If a URL is provided, the content is first downloaded to a local file, located in
|
|
the provided workdir or in a temporary directory if no workdir provided.
|
|
|
|
Args:
|
|
source (Path | AnyHttpUrl | str): The file input source. Can be a path or URL.
|
|
headers (Optional[dict[str, str]]): Optional set of headers to use for fetching
|
|
the remote URL.
|
|
workdir (Optional[Path]): If set, the work directory where the file will
|
|
be downloaded, otherwise a temp dir will be used.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
|
|
Returns:
|
|
Path: The local file path.
|
|
"""
|
|
return _resolve_source_to_path(
|
|
source=source,
|
|
headers=headers,
|
|
workdir=workdir,
|
|
)
|
|
|
|
|
|
@deprecated("Use `resolve_source_to_path()` or `resolve_source_to_stream()` instead")
|
|
def resolve_file_source(
|
|
source: Path | AnyHttpUrl | str,
|
|
headers: dict[str, str] | None = None,
|
|
) -> Path:
|
|
"""Resolves the source (URL, path) of a file to a local file path.
|
|
|
|
If a URL is provided, the content is first downloaded to a temporary local file.
|
|
|
|
Args:
|
|
source (Path | AnyHttpUrl | str): The file input source. Can be a path or URL.
|
|
headers (Optional[dict[str, str]]): Optional set of headers to use for fetching
|
|
the remote URL.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
|
|
Returns:
|
|
Path: The local file path.
|
|
"""
|
|
return _resolve_source_to_path(
|
|
source=source,
|
|
headers=headers,
|
|
)
|