mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* Early-abort HTTP source fetch when it is too large Signed-off-by: Christoph Auer <cau@zurich.ibm.com> * Address review comments Signed-off-by: Christoph Auer <cau@zurich.ibm.com> --------- Signed-off-by: Christoph Auer <cau@zurich.ibm.com>
325 lines
11 KiB
Python
325 lines
11 KiB
Python
"""File-related utilities."""
|
|
|
|
import ipaddress
|
|
import re
|
|
import tempfile
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import Optional, Union
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
from pydantic import AnyHttpUrl, TypeAdapter, ValidationError
|
|
from typing_extensions import deprecated
|
|
|
|
from docling_core.types.doc.utils import relative_path
|
|
from docling_core.types.io import DocumentStream
|
|
|
|
_MAX_REDIRECTS = 5
|
|
|
|
# Chunk size for streaming remote downloads. Sized for document payloads (PDFs
|
|
# and similar), which are either small or in the multi-megabyte range, so a
|
|
# large chunk keeps the read loop short without wasting memory.
|
|
_DOWNLOAD_CHUNK_SIZE = 512 * 1024
|
|
|
|
|
|
class FileSizeLimitExceededError(ValueError):
|
|
"""Raised when a remote file exceeds the configured download size limit."""
|
|
|
|
def __init__(self, filename: str, size: int, limit: int):
|
|
self.filename = filename
|
|
self.size = size
|
|
self.limit = limit
|
|
super().__init__(f"Remote file exceeds the maximum allowed size ({size} > {limit} bytes).")
|
|
|
|
|
|
def _is_safe_url(url: str) -> bool:
|
|
"""Return whether a URL resolves to a globally routable address."""
|
|
try:
|
|
parsed = urlparse(url)
|
|
hostname = parsed.hostname
|
|
|
|
if not hostname:
|
|
return False
|
|
|
|
try:
|
|
ip = ipaddress.ip_address(hostname)
|
|
except ValueError:
|
|
import socket
|
|
|
|
try:
|
|
ip_str = socket.gethostbyname(hostname)
|
|
ip = ipaddress.ip_address(ip_str)
|
|
except (socket.gaierror, socket.herror):
|
|
return False
|
|
|
|
return ip.is_global and not (
|
|
ip.is_private
|
|
or ip.is_loopback
|
|
or ip.is_link_local
|
|
or ip.is_reserved
|
|
or ip.is_multicast
|
|
or ip.is_unspecified
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _sanitize_filename(filename: str) -> Optional[str]:
|
|
"""Return a basename-safe filename, or None if no usable basename remains."""
|
|
normalized = filename.replace("\\", "/")
|
|
basename = Path(normalized).name
|
|
|
|
if not basename or basename in (".", "..") or "/" in basename:
|
|
return None
|
|
|
|
return basename
|
|
|
|
|
|
def resolve_remote_filename(
|
|
http_url: AnyHttpUrl,
|
|
response_headers: dict[str, str],
|
|
fallback_filename="file",
|
|
) -> str:
|
|
"""Resolves the filename from a remote url and its response headers.
|
|
|
|
Args:
|
|
source AnyHttpUrl: The source http url.
|
|
response_headers Dict: Headers received while fetching the remote file.
|
|
fallback_filename str: Filename to use in case none can be determined.
|
|
|
|
Returns:
|
|
str: The actual filename of the remote url.
|
|
"""
|
|
raw_fname = None
|
|
if cont_disp := response_headers.get("Content-Disposition"):
|
|
for par in cont_disp.strip().split(";"):
|
|
if (split := par.split("=")) and split[0].strip() == "filename":
|
|
raw_fname = "=".join(split[1:]).strip().strip("'\"") or None
|
|
break
|
|
|
|
if raw_fname is None:
|
|
raw_fname = Path(http_url.path or "").name or fallback_filename
|
|
|
|
if fname := _sanitize_filename(raw_fname):
|
|
return fname
|
|
|
|
if fname := _sanitize_filename(fallback_filename):
|
|
return fname
|
|
|
|
raise ValueError("Could not derive a safe filename")
|
|
|
|
|
|
def resolve_source_to_stream(
|
|
source: Union[Path, AnyHttpUrl, str],
|
|
headers: Optional[dict[str, str]] = None,
|
|
max_file_size: Optional[int] = None,
|
|
) -> DocumentStream:
|
|
"""Resolves the source (URL, path) of a file to a binary stream.
|
|
|
|
Args:
|
|
source: The file input source. Can be a path or URL.
|
|
headers: Optional set of headers to use for fetching the remote URL.
|
|
max_file_size: Optional maximum size, in bytes, for a remote download.
|
|
When set, the download is rejected upfront if the declared
|
|
``Content-Length`` exceeds it, and aborted while streaming as soon as
|
|
the received bytes exceed it.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
FileSizeLimitExceededError: If a remote download exceeds ``max_file_size``.
|
|
|
|
Returns:
|
|
DocumentStream: The resolved file loaded as a stream.
|
|
"""
|
|
try:
|
|
http_url: AnyHttpUrl = TypeAdapter(AnyHttpUrl).validate_python(source)
|
|
url_str = str(http_url)
|
|
|
|
if not _is_safe_url(url_str):
|
|
raise ValueError(f"URL is not allowed: {url_str}")
|
|
|
|
_headers = headers or {}
|
|
req_headers = {k.lower(): v for k, v in _headers.items()}
|
|
if "user-agent" not in req_headers:
|
|
try:
|
|
import importlib.metadata
|
|
|
|
agent_name = f"docling-core/{importlib.metadata.version('docling-core')}"
|
|
except Exception:
|
|
agent_name = "docling-core"
|
|
req_headers["user-agent"] = agent_name
|
|
|
|
google_doc_id = re.search(
|
|
r"google\.com\/(file|document|spreadsheets|presentation)\/d\/([\w-]+)",
|
|
url_str,
|
|
)
|
|
if google_doc_id:
|
|
doc_type = google_doc_id.group(1)
|
|
doc_id = google_doc_id.group(2)
|
|
|
|
if doc_type == "file":
|
|
url_str = f"https://drive.google.com/uc?export=download&id={doc_id}"
|
|
elif doc_type == "document":
|
|
url_str = f"https://docs.google.com/document/d/{doc_id}/export?format=docx"
|
|
elif doc_type == "spreadsheets":
|
|
url_str = f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=xlsx"
|
|
elif doc_type == "presentation":
|
|
url_str = f"https://docs.google.com/presentation/d/{doc_id}/export?format=pptx"
|
|
else:
|
|
raise ValueError(f"Unexpected Google doc type: {doc_type}")
|
|
|
|
http_url = TypeAdapter(AnyHttpUrl).validate_python(url_str)
|
|
|
|
def _check_redirect_safety(response, *args, **kwargs):
|
|
"""Validate each redirect target before following it."""
|
|
if response.is_redirect or response.is_permanent_redirect:
|
|
redirect_url = response.headers.get("location")
|
|
if redirect_url:
|
|
if not redirect_url.startswith(("http://", "https://")):
|
|
from urllib.parse import urljoin
|
|
|
|
redirect_url = urljoin(response.url, redirect_url)
|
|
|
|
if not _is_safe_url(redirect_url):
|
|
raise ValueError(f"Redirect target is not allowed: {redirect_url}")
|
|
|
|
# Use context managers so the session and the streamed response are
|
|
# always released, including on the size-limit abort paths below where
|
|
# the response body is left unconsumed.
|
|
with requests.Session() as session:
|
|
session.max_redirects = _MAX_REDIRECTS
|
|
session.hooks["response"].append(_check_redirect_safety)
|
|
|
|
with session.get(
|
|
url_str,
|
|
stream=True,
|
|
headers=req_headers,
|
|
allow_redirects=True,
|
|
) as res:
|
|
res.raise_for_status()
|
|
|
|
response_headers = dict(res.headers)
|
|
fname = resolve_remote_filename(http_url=http_url, response_headers=response_headers)
|
|
|
|
if max_file_size is not None:
|
|
content_length = res.headers.get("Content-Length")
|
|
if content_length is not None:
|
|
try:
|
|
content_length_value = int(content_length)
|
|
except ValueError:
|
|
content_length_value = None
|
|
|
|
if content_length_value is not None and content_length_value > max_file_size:
|
|
raise FileSizeLimitExceededError(
|
|
filename=fname,
|
|
size=content_length_value,
|
|
limit=max_file_size,
|
|
)
|
|
|
|
stream = BytesIO()
|
|
downloaded = 0
|
|
for chunk in res.iter_content(chunk_size=_DOWNLOAD_CHUNK_SIZE):
|
|
if not chunk:
|
|
continue
|
|
downloaded += len(chunk)
|
|
if max_file_size is not None and downloaded > max_file_size:
|
|
raise FileSizeLimitExceededError(
|
|
filename=fname,
|
|
size=downloaded,
|
|
limit=max_file_size,
|
|
)
|
|
stream.write(chunk)
|
|
stream.seek(0)
|
|
doc_stream = DocumentStream(name=fname, stream=stream)
|
|
except ValidationError:
|
|
if isinstance(source, str) and "://" in source:
|
|
scheme = source.split("://", 1)[0].lower()
|
|
if scheme not in ("http", "https"):
|
|
raise ValueError(f"Unsupported URL scheme: '{scheme}'. Only http:// and https:// are supported.")
|
|
try:
|
|
local_path = TypeAdapter(Path).validate_python(source)
|
|
stream = BytesIO(local_path.read_bytes())
|
|
doc_stream = DocumentStream(name=local_path.name, stream=stream)
|
|
except ValidationError:
|
|
raise ValueError(f"Unexpected source type encountered: {type(source)}")
|
|
return doc_stream
|
|
|
|
|
|
def _resolve_source_to_path(
|
|
source: Union[Path, AnyHttpUrl, str],
|
|
headers: Optional[dict[str, str]] = None,
|
|
workdir: Optional[Path] = None,
|
|
) -> Path:
|
|
doc_stream = resolve_source_to_stream(source=source, headers=headers)
|
|
|
|
# use a temporary directory if not specified
|
|
if workdir is None:
|
|
workdir = Path(tempfile.mkdtemp())
|
|
|
|
# create the parent workdir if it doesn't exist
|
|
workdir.mkdir(exist_ok=True, parents=True)
|
|
|
|
# save result to a local file
|
|
local_path = workdir / doc_stream.name
|
|
with local_path.open("wb") as f:
|
|
f.write(doc_stream.stream.read())
|
|
|
|
return local_path
|
|
|
|
|
|
def resolve_source_to_path(
|
|
source: Union[Path, AnyHttpUrl, str],
|
|
headers: Optional[dict[str, str]] = None,
|
|
workdir: Optional[Path] = None,
|
|
) -> Path:
|
|
"""Resolves the source (URL, path) of a file to a local file path.
|
|
|
|
If a URL is provided, the content is first downloaded to a local file, located in
|
|
the provided workdir or in a temporary directory if no workdir provided.
|
|
|
|
Args:
|
|
source (Path | AnyHttpUrl | str): The file input source. Can be a path or URL.
|
|
headers (Optional[dict[str, str]]): Optional set of headers to use for fetching
|
|
the remote URL.
|
|
workdir (Optional[Path]): If set, the work directory where the file will
|
|
be downloaded, otherwise a temp dir will be used.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
|
|
Returns:
|
|
Path: The local file path.
|
|
"""
|
|
return _resolve_source_to_path(
|
|
source=source,
|
|
headers=headers,
|
|
workdir=workdir,
|
|
)
|
|
|
|
|
|
@deprecated("Use `resolve_source_to_path()` or `resolve_source_to_stream()` instead")
|
|
def resolve_file_source(
|
|
source: Union[Path, AnyHttpUrl, str],
|
|
headers: Optional[dict[str, str]] = None,
|
|
) -> Path:
|
|
"""Resolves the source (URL, path) of a file to a local file path.
|
|
|
|
If a URL is provided, the content is first downloaded to a temporary local file.
|
|
|
|
Args:
|
|
source (Path | AnyHttpUrl | str): The file input source. Can be a path or URL.
|
|
headers (Optional[dict[str, str]]): Optional set of headers to use for fetching
|
|
the remote URL.
|
|
|
|
Raises:
|
|
ValueError: If source is of unexpected type.
|
|
|
|
Returns:
|
|
Path: The local file path.
|
|
"""
|
|
return _resolve_source_to_path(
|
|
source=source,
|
|
headers=headers,
|
|
)
|