Files
docling-core/tests/test_serialization.py
laoyyuandlaoyyu 3faf821870 fix(webvtt): keep a sub-millisecond cue serializable (#1014)
* fix(webvtt): keep a sub-millisecond cue serializable

TrackSource accepts any end_time greater than start_time, but a WebVTT
timestamp only carries milliseconds. Rounding both ends of a sub-millisecond
segment onto the same timestamp produced a zero-length WebVTTCueTimings, whose
validator rejects it, so exporting the document raised ValidationError instead
of emitting the cue.

Build the cue timings through one helper that extends the end by the smallest
representable step when rounding collapsed the range.

Signed-off-by: laoyyu <laoyyu@users.noreply.github.com>

* fix(webvtt): keep a sub-millisecond cue serializable

TrackSource accepts any end_time greater than start_time, but a WebVTT
timestamp only carries milliseconds. Rounding both ends of a sub-millisecond
segment onto the same timestamp produced a zero-length WebVTTCueTimings, whose
validator rejects it, so exporting the document raised ValidationError instead
of emitting the cue.

Build the cue timings through one helper that extends the end by the smallest
representable step when rounding collapsed the range, and log a warning when
that happens, so the loss of precision is visible instead of the timings being
silently rewritten.

Signed-off-by: laoyyu <laoyyu@foxmail.com>

---------

Signed-off-by: laoyyu <laoyyu@users.noreply.github.com>
Signed-off-by: laoyyu <laoyyu@foxmail.com>
Co-authored-by: laoyyu <laoyyu@users.noreply.github.com>
2026-11-14 07:52:54 +01:00

2749 lines
103 KiB
Python

"""Test serialization."""
import logging
import threading
from html.parser import HTMLParser
from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
from typing import TypeAlias, Union
from unittest.mock import MagicMock, patch
from xml.etree import ElementTree as ET
import pytest
from PIL import Image
from pydantic import AnyUrl
from docling_core.transforms.chunker.hierarchical_chunker import TripletTableSerializer
from docling_core.transforms.serializer.common import _DEFAULT_LABELS
from docling_core.transforms.serializer.html import (
HTMLDocSerializer,
HTMLMetaSerializer,
HTMLOutputStyle,
HTMLParams,
HTMLTableSerializer,
)
from docling_core.transforms.serializer.latex import LaTeXDocSerializer
from docling_core.transforms.serializer.markdown import (
CaptionPlacement,
MarkdownDocSerializer,
MarkdownParams,
MarkdownPictureSerializer,
MarkdownTableSerializer,
OrigListItemMarkerMode,
_cell_content_has_table,
)
from docling_core.transforms.serializer.plain_text import PlainTextDocSerializer, PlainTextParams
from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams
from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer
from docling_core.types.doc import DoclingDocument, ImageRef, TrackSource
from docling_core.types.doc.base import BoundingBox, CoordOrigin, FormattingFallbackMode, ImageRefMode, Size
from docling_core.types.doc.document import (
BaseMeta,
CharSpan,
DescriptionAnnotation,
EntitiesMetaField,
EntityMention,
Formatting,
LanguageMetaField,
PictureClassificationMetaField,
PictureClassificationPrediction,
PictureMeta,
ProvenanceItem,
RefItem,
RichTableCell,
Script,
SummaryMetaField,
TableCell,
TableData,
TextItem,
)
from docling_core.types.doc.labels import DocItemLabel
from .test_data_gen_flag import GEN_TEST_DATA
def verify(exp_file: Path, actual: str):
if GEN_TEST_DATA:
with open(exp_file, "w", encoding="utf-8") as f:
f.write(f"{actual}\n")
else:
with open(exp_file, encoding="utf-8") as f:
expected = f.read().rstrip()
# Normalize platform-dependent quote escaping for DocTags outputs
name = exp_file.name
if name.endswith((".dt", ".idt", ".idt.xml")):
def _normalize_quotes(s: str) -> str:
return s.replace("&quot;", '"').replace("&#34;", '"')
expected = _normalize_quotes(expected)
actual = _normalize_quotes(actual)
assert actual == expected
# ===============================
# Markdown tests
# ===============================
def test_md_cross_page_list_page_break():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder="<!-- page break -->",
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
def test_md_checkboxes():
src = Path("./tests/data/doc/checkboxes.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder="<!-- page break -->",
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}.gt.md", actual=actual)
def test_md_cross_page_list_page_break_none():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder=None,
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_pb_none.gt.md", actual=actual)
def test_md_cross_page_list_page_break_empty():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder="",
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_pb_empty.gt.md", actual=actual)
def test_md_cross_page_list_page_break_non_empty():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder="<!-- page-break -->",
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_pb_non_empty.gt.md", actual=actual)
def test_md_cross_page_list_page_break_p2():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
page_break_placeholder=None,
pages={2},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_p2.gt.md", actual=actual)
_PAGE_BREAK = "<!-- page break -->"
# builds a document whose items carry the given (text, page number) provenance
def _make_paged_doc(entries: list[tuple[str, int]], *, as_list: bool, num_pages: int) -> DoclingDocument:
doc = DoclingDocument(name="paged")
for page_no in range(1, num_pages + 1):
doc.add_page(page_no=page_no, size=Size(width=595, height=842), image=None)
group = doc.add_list_group(name="list") if as_list else None
for text, page_no in entries:
prov = ProvenanceItem(
page_no=page_no,
bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT),
charspan=(0, len(text)),
)
if group is not None:
doc.add_list_item(text=text, parent=group, prov=prov)
else:
doc.add_text(label=DocItemLabel.TEXT, text=text, prov=prov)
return doc
def test_md_cross_page_list_page_break_matches_per_page_export():
doc = _make_paged_doc(
[("item one", 1), ("item two", 1), ("item three", 2), ("item four", 2)],
as_list=True,
num_pages=2,
)
globally = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK)
per_page = f"\n\n{_PAGE_BREAK}\n\n".join(doc.export_to_markdown(page_no=n) for n in (1, 2))
assert globally == per_page
def test_md_cross_page_text_page_break_matches_per_page_export():
# the control: outside a list scope this already held, and must keep holding
doc = _make_paged_doc(
[("item one", 1), ("item two", 1), ("item three", 2), ("item four", 2)],
as_list=False,
num_pages=2,
)
globally = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK)
per_page = f"\n\n{_PAGE_BREAK}\n\n".join(doc.export_to_markdown(page_no=n) for n in (1, 2))
assert globally == per_page
def test_md_page_break_spacing_independent_of_scope():
entries = [("item one", 1), ("item two", 1), ("item three", 2)]
in_list = _make_paged_doc(entries, as_list=True, num_pages=2).export_to_markdown(page_break_placeholder=_PAGE_BREAK)
at_doc_scope = _make_paged_doc(entries, as_list=False, num_pages=2).export_to_markdown(
page_break_placeholder=_PAGE_BREAK
)
for actual in (in_list, at_doc_scope):
assert f"\n\n{_PAGE_BREAK}\n\n" in actual
def test_md_cross_page_list_page_break_empty_placeholder():
doc = _make_paged_doc([("item one", 1), ("item two", 2)], as_list=True, num_pages=2)
# an empty placeholder still occupies the blank lines the marker would have
assert doc.export_to_markdown(page_break_placeholder="") == "- item one\n\n\n\n- item two"
def test_md_page_break_at_document_bounds():
doc = _make_paged_doc([("item 1", 1), ("item 2", 2), ("item 3", 3)], as_list=True, num_pages=3)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(page_break_placeholder=_PAGE_BREAK, pages={2}),
)
# a break opening or closing the export gains no blank line on the outer side
assert ser.serialize().text == f"{_PAGE_BREAK}\n\n- item 2\n\n{_PAGE_BREAK}"
def test_md_consecutive_page_breaks():
doc = _make_paged_doc([(f"item {n}", n) for n in range(1, 7)], as_list=True, num_pages=6)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(page_break_placeholder=_PAGE_BREAK, pages={3, 4, 6}),
)
actual = ser.serialize().text
assert f"{_PAGE_BREAK}\n\n{_PAGE_BREAK}" in actual
assert "\n\n\n" not in actual
def test_md_nested_list_page_break_keeps_indent():
doc = DoclingDocument(name="nested")
for page_no in (1, 2):
doc.add_page(page_no=page_no, size=Size(width=595, height=842), image=None)
def _prov(page_no: int, text: str) -> ProvenanceItem:
return ProvenanceItem(
page_no=page_no,
bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT),
charspan=(0, len(text)),
)
outer = doc.add_list_group(name="outer")
doc.add_list_item(text="outer one", parent=outer, prov=_prov(1, "outer one"))
inner = doc.add_list_group(name="inner", parent=outer)
doc.add_list_item(text="inner a", parent=inner, prov=_prov(1, "inner a"))
doc.add_list_item(text="inner b", parent=inner, prov=_prov(2, "inner b"))
doc.add_list_item(text="outer two", parent=outer, prov=_prov(2, "outer two"))
# the marker keeps the indentation that holds it inside the outer item
assert f"\n\n {_PAGE_BREAK}\n\n" in doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK)
def test_md_page_break_between_list_and_table():
doc = _make_paged_doc([("item one", 1), ("item two", 1)], as_list=True, num_pages=2)
doc.add_table(
data=TableData(
num_rows=1,
num_cols=1,
table_cells=[
TableCell(
text="cell",
row_span=1,
col_span=1,
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
)
],
),
prov=ProvenanceItem(
page_no=2,
bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT),
charspan=(0, 4),
),
)
actual = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK)
assert f"- item two\n\n{_PAGE_BREAK}\n\n|" in actual
def test_md_charts():
src = Path("./tests/data/doc/barchart.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
def test_md_inline_and_formatting():
src = Path("./tests/data/doc/inline_and_formatting.yaml")
doc = DoclingDocument.load_from_yaml(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
@pytest.mark.parametrize(
("mode", "expected"),
[
(FormattingFallbackMode.NONE, "Water is H2O, energy is E=mc2, &lt;tag&gt;"),
(
FormattingFallbackMode.HTML,
"Water is H<sub>2</sub>O, energy is E=mc<sup>2</sup>, <u>&lt;tag&gt;</u>",
),
],
)
def test_md_formatting_fallback_mode(mode: FormattingFallbackMode, expected: str, tmp_path: Path):
"""Formatting without Markdown syntax is dropped or rendered as HTML."""
doc = DoclingDocument(name="test")
inline = doc.add_inline_group()
doc.add_text(label=DocItemLabel.TEXT, text="Water is H", parent=inline)
doc.add_text(label=DocItemLabel.TEXT, text="2", formatting=Formatting(script=Script.SUB), parent=inline)
doc.add_text(label=DocItemLabel.TEXT, text="O, energy is E=mc", parent=inline)
doc.add_text(label=DocItemLabel.TEXT, text="2", formatting=Formatting(script=Script.SUPER), parent=inline)
doc.add_text(label=DocItemLabel.TEXT, text=", ", parent=inline)
doc.add_text(label=DocItemLabel.TEXT, text="<tag>", formatting=Formatting(underline=True), parent=inline)
params = MarkdownParams(formatting_fallback_mode=mode)
assert MarkdownDocSerializer(doc=doc, params=params).serialize().text == expected
assert doc.export_to_markdown(formatting_fallback_mode=mode) == expected
md_path = tmp_path / "doc.md"
doc.save_as_markdown(md_path, formatting_fallback_mode=mode)
assert md_path.read_text(encoding="utf-8") == expected
plain_params = PlainTextParams(formatting_fallback_mode=mode)
plain = PlainTextDocSerializer(doc=doc, params=plain_params).serialize().text
assert plain == "Water is H2O, energy is E=mc2, <tag>"
def test_md_pb_placeholder_and_page_filter():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
# NOTE ambiguous case
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
page_break_placeholder="<!-- page break -->",
pages={3, 4, 6},
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
def test_md_list_item_markers(sample_doc):
root_dir = Path("./tests/data/doc")
for mode in OrigListItemMarkerMode:
for valid in [False, True]:
ser = MarkdownDocSerializer(
doc=sample_doc,
params=MarkdownParams(
orig_list_item_marker_mode=mode,
ensure_valid_list_item_marker=valid,
),
)
actual = ser.serialize().text
verify(
root_dir / f"constructed_mode_{str(mode.value).lower()}_valid_{str(valid).lower()}.gt.md",
actual=actual,
)
@pytest.mark.parametrize(
("marker", "expected"),
[
pytest.param("a)", "- a) item", id="ascii-letter"),
pytest.param("第九条", "- 第九条 item", id="cjk-article"),
pytest.param("\uff08一\uff09", "- \uff08一\uff09 item", id="cjk-fullwidth-parens"),
pytest.param("①", "- ① item", id="enclosed-circle"),
pytest.param("壹、", "- 壹、 item", id="cjk-legal"),
pytest.param("•", "- item", id="bullet-glyph"),
],
)
def test_md_list_item_marker_auto_non_ascii(marker, expected):
doc = DoclingDocument(name="markers")
group = doc.add_list_group()
doc.add_list_item(text="item", enumerated=True, marker=marker, parent=group)
actual = MarkdownDocSerializer(doc=doc).serialize().text
assert actual == expected
def test_md_mark_meta_true():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
mark_meta=True,
pages={1, 5},
),
)
actual = ser.serialize().text
verify(
exp_file=src.parent / f"{src.stem}_p1_mark_meta_true.gt.md",
actual=actual,
)
def test_md_mark_meta_false():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
mark_meta=False,
pages={1, 5},
),
)
actual = ser.serialize().text
verify(
exp_file=src.parent / f"{src.stem}_p1_mark_meta_false.gt.md",
actual=actual,
)
def test_md_legacy_annotations_mark_true(sample_doc):
exp_file = Path("./tests/data/doc/constructed_legacy_annot_mark_true.gt.md")
with pytest.warns(DeprecationWarning):
sample_doc.tables[0].annotations.append(
DescriptionAnnotation(text="This is a description of table 1.", provenance="foo")
)
ser = MarkdownDocSerializer(
doc=sample_doc,
params=MarkdownParams(
mark_annotations=True,
),
)
actual = ser.serialize().text
verify(
exp_file=exp_file,
actual=actual,
)
def test_md_legacy_annotations_mark_false(sample_doc):
exp_file = Path("./tests/data/doc/constructed_legacy_annot_mark_false.gt.md")
with pytest.warns(DeprecationWarning):
sample_doc.tables[0].annotations.append(
DescriptionAnnotation(text="This is a description of table 1.", provenance="foo")
)
ser = MarkdownDocSerializer(
doc=sample_doc,
params=MarkdownParams(
mark_annotations=False,
),
)
actual = ser.serialize().text
verify(
exp_file=exp_file,
actual=actual,
)
def test_md_nested_lists():
src = Path("./tests/data/doc/polymers.json")
doc = DoclingDocument.load_from_json(src)
ser = MarkdownDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
def test_md_rich_table(rich_table_doc):
exp_file = Path("./tests/data/doc/rich_table.gt.md")
ser = MarkdownDocSerializer(doc=rich_table_doc)
actual = ser.serialize().text
verify(exp_file=exp_file, actual=actual)
def test_md_single_row_table():
exp_file = Path("./tests/data/doc/single_row_table.gt.md")
words = ["foo", "bar"]
doc = DoclingDocument(name="")
row_idx = 0
table = doc.add_table(data=TableData(num_rows=1, num_cols=len(words)))
for col_idx, word in enumerate(words):
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row_idx,
end_row_offset_idx=row_idx + 1,
start_col_offset_idx=col_idx,
end_col_offset_idx=col_idx + 1,
text=word,
),
)
ser = MarkdownDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=exp_file, actual=actual)
def test_md_field_region():
exp_file = Path("./tests/data/doc/field_region.gt.md")
doc = DoclingDocument(name="")
field_region = doc.add_field_region()
field_item = doc.add_field_item(parent=field_region)
doc.add_field_key(text="Name:", parent=field_item)
doc.add_field_value(text="John Doe", parent=field_item)
ser = MarkdownDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=exp_file, actual=actual)
def _table_with_rows(rows: list[list[str]], num_header_rows: int) -> DoclingDocument:
doc = DoclingDocument(name="")
table = doc.add_table(data=TableData(num_rows=len(rows), num_cols=len(rows[0])))
for row_idx, row in enumerate(rows):
for col_idx, text in enumerate(row):
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row_idx,
end_row_offset_idx=row_idx + 1,
start_col_offset_idx=col_idx,
end_col_offset_idx=col_idx + 1,
text=text,
column_header=row_idx < num_header_rows,
),
)
return doc
def test_md_table_stacked_header_is_flattened():
doc = _table_with_rows(
[["Product", "Tier 1", "Tier 2"], ["", "0-99", "100+"], ["CAT-001", "10.00", "9.00"]],
num_header_rows=2,
)
actual = MarkdownDocSerializer(doc=doc).serialize().text
assert actual == (
"| Product | Tier 1 - 0-99 | Tier 2 - 100+ |\n"
"|-----------|-----------------|-----------------|\n"
"| CAT-001 | 10.00 | 9.00 |"
)
def test_md_table_stacked_header_drops_repeated_span_text():
doc = _table_with_rows(
[["human", "MRCNN"], ["human", "R50"], ["84-89", "68.4"]],
num_header_rows=2,
)
actual = MarkdownDocSerializer(doc=doc).serialize().text
assert actual.splitlines()[0] == "| human | MRCNN - R50 |"
def _cell(text: str, row: int, col: int, *, row_span: int = 1, column_header: bool = False) -> TableCell:
return TableCell(
text=text,
start_row_offset_idx=row,
end_row_offset_idx=row + row_span,
start_col_offset_idx=col,
end_col_offset_idx=col + 1,
row_span=row_span,
column_header=column_header,
)
@pytest.mark.parametrize(
("num_rows", "num_cols", "cells", "md_table", "df_columns", "df_values"),
[
# "10.00" sits in the leading column, where unflagged text does not end the
# header block, so only the "header starts on this row" rule keeps row 1 out.
pytest.param(
2,
2,
[
_cell("Price", 0, 0, column_header=True),
_cell("Category", 0, 1, row_span=2, column_header=True),
_cell("10.00", 1, 0),
],
"| Price | Category |\n| - | - |\n| 10.00 | Category |",
["Price", "Category"],
[["10.00", "Category"]],
id="rows-under-row-spanning-header-stay-in-body",
),
# "2025" is deliberately flagged as column_header, as the HTML backend did
# before docling-project/docling#4216, to model documents serialized then.
pytest.param(
3,
3,
[
_cell("Year", 0, 0, column_header=True),
_cell("Month", 0, 1, column_header=True),
_cell("Revenue", 0, 2, column_header=True),
_cell("2025", 1, 0, row_span=2, column_header=True),
_cell("January", 1, 1),
_cell("$134", 1, 2),
_cell("February", 2, 1),
_cell("$150", 2, 2),
],
"| Year | Month | Revenue |\n| - | - | - |\n| 2025 | January | $134 |\n| 2025 | February | $150 |",
["Year", "Month", "Revenue"],
[["2025", "January", "$134"], ["2025", "February", "$150"]],
id="row-header-flagged-as-column-header-stays-in-body",
),
pytest.param(
3,
3,
[
_cell("", 0, 0, column_header=True),
_cell("% of Total", 0, 1, column_header=True),
_cell("% of Total", 0, 2, column_header=True),
_cell("class label", 1, 0),
_cell("Train", 1, 1, column_header=True),
_cell("Test", 1, 2, column_header=True),
_cell("Caption", 2, 0),
_cell("2.5", 2, 1),
_cell("2.4", 2, 2),
],
"| class label | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.5 | 2.4 |",
["class label", "% of Total.Train", "% of Total.Test"],
[["Caption", "2.5", "2.4"]],
id="stacked-header-with-unmarked-row-label",
),
pytest.param(
2,
2,
[
_cell("Name", 0, 0, column_header=True),
_cell("Value", 0, 1),
_cell("A", 1, 0),
_cell("1", 1, 1),
],
"| Name | Value |\n| - | - |\n| A | 1 |",
["Name", "Value"],
[["A", "1"]],
id="partially-flagged-first-row-stays-header",
),
# Without any column_header flag, markdown promotes row 0 to the header since
# GFM requires one, while the dataframe keeps every row as data with default
# integer column names.
pytest.param(
2,
2,
[
_cell("Name", 0, 0),
_cell("Value", 0, 1),
_cell("A", 1, 0),
_cell("1", 1, 1),
],
"| Name | Value |\n| - | - |\n| A | 1 |",
[0, 1],
[["Name", "Value"], ["A", "1"]],
id="no-flags-markdown-promotes-row-0-dataframe-does-not",
),
# Row headers flagged as column_header in every row, with no header row, as
# the HTML backend produced before docling-project/docling#4216 for a body
# `<th>`. The leading-column flag on row 1 shows that row 0 holds a row
# header too, so row 0 is subject to the body-text check and no row is
# promoted: markdown keeps an empty header and the dataframe keeps every row.
pytest.param(
2,
2,
[
_cell("2025", 0, 0, column_header=True),
_cell("$134", 0, 1),
_cell("2026", 1, 0, column_header=True),
_cell("$150", 1, 1),
],
"| | |\n| - | - |\n| 2025 | $134 |\n| 2026 | $150 |",
[0, 1],
[["2025", "$134"], ["2026", "$150"]],
id="row-headers-only-flagged-as-column-header-stay-in-body",
),
],
)
def test_table_header_rows_in_markdown_and_dataframe(
num_rows: int,
num_cols: int,
cells: list[TableCell],
md_table: str,
df_columns: list[str | int],
df_values: list[list[str]],
):
"""Markdown and dataframe exports agree on which leading rows form the column header."""
doc = DoclingDocument(name="")
table = doc.add_table(data=TableData(num_rows=num_rows, num_cols=num_cols, table_cells=cells))
md_ser = MarkdownDocSerializer(doc=doc, params=MarkdownParams(compact_tables=True))
assert md_ser.serialize().text == md_table
dataframe = table.export_to_dataframe(doc)
assert list(dataframe.columns) == df_columns
assert dataframe.values.tolist() == df_values
def test_md_table_with_header_flags_below_row_zero_keeps_every_row_as_data():
"""No leading header block, so nothing is promoted and no row is dropped."""
doc = DoclingDocument(name="")
rows = [["title", ""], ["Product", "Price"], ["CAT-001", "10.00"]]
table = doc.add_table(data=TableData(num_rows=len(rows), num_cols=2))
for row_idx, row in enumerate(rows):
for col_idx, text in enumerate(row):
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row_idx,
end_row_offset_idx=row_idx + 1,
start_col_offset_idx=col_idx,
end_col_offset_idx=col_idx + 1,
text=text,
column_header=row_idx == 1,
),
)
actual = MarkdownDocSerializer(doc=doc).serialize().text
assert actual.splitlines()[0] == "| | |"
assert "| title | |" in actual
assert "| CAT-001 | 10.00 |" in actual
def test_md_table_without_header_flags_keeps_first_row_as_header():
"""Backends that never set column_header keep the pre-existing behavior."""
doc = _table_with_rows([["foo", "bar"], ["baz", "qux"]], num_header_rows=0)
actual = MarkdownDocSerializer(doc=doc).serialize().text
assert actual.splitlines()[0] == "| foo | bar |"
def test_md_table_single_header_row_with_flags():
"""A single flagged header row is used as-is without joining or deduplication."""
doc = _table_with_rows(
[["Product", "Price"], ["CAT-001", "10.00"], ["CAT-002", "9.00"]],
num_header_rows=1,
)
actual = MarkdownDocSerializer(doc=doc).serialize().text
lines = actual.splitlines()
assert lines[0] == "| Product | Price |"
assert lines[2] == "| CAT-001 | 10.00 |"
assert lines[3] == "| CAT-002 | 9.00 |"
def test_md_pipe_in_table():
doc = DoclingDocument(name="Pipe in Table")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
# TODO: properly handle nested tables, for now just escape the pipe
doc.add_table_cell(
table,
TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="Fruits | Veggies",
),
)
ser = doc.export_to_markdown()
assert ser == "| Fruits &#124; Veggies |\n|-------------------------|"
def test_md_heading_in_rich_table_cell_renders_as_plain_text():
"""Test headings inside RichTableCell must not emit `#` markers.
According to the Markdown spec, heading syntax is invalid inside tables.
A SectionHeaderItem or TitleItem referenced by a RichTableCell must be
serialized as plain text, not as `## Heading`.
"""
doc = DoclingDocument(name="heading_in_table")
table = doc.add_table(data=TableData(num_rows=2, num_cols=2))
section_header = doc.add_heading(text="Section Heading", level=1, parent=table)
title = doc.add_title(text="Document Title", parent=table)
doc.add_table_cell(
table,
RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
ref=section_header.get_ref(),
),
)
doc.add_table_cell(
table,
RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
ref=title.get_ref(),
),
)
doc.add_table_cell(
table,
TableCell(
start_row_offset_idx=1,
end_row_offset_idx=2,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="value A",
),
)
doc.add_table_cell(
table,
TableCell(
start_row_offset_idx=1,
end_row_offset_idx=2,
start_col_offset_idx=1,
end_col_offset_idx=2,
text="value B",
),
)
md = doc.export_to_markdown()
assert "Section Heading" in md
assert "Document Title" in md
table_lines = [line for line in md.splitlines() if line.startswith("|")]
for line in table_lines:
assert "#" not in line, f"Heading marker leaked into table cell: {line!r}\nFull output:\n{md}"
def test_cell_content_has_table_detects_descendant_table():
"""Ensure nested tables are detected through non-table parent nodes."""
doc = DoclingDocument(name="descendant_table")
wrapper = doc.add_group()
nested_table = doc.add_table(data=TableData(num_rows=1, num_cols=1), parent=wrapper)
doc.add_table_cell(
nested_table,
TableCell(
text="inner",
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
),
)
assert _cell_content_has_table(wrapper, doc)
def _build_nested_rich_table_doc(depth: int) -> DoclingDocument:
"""Build a document with `depth` levels of nested RichTableCell tables.
Each level is a 1x2 table whose first cell is a RichTableCell referencing
the next-level table, and whose second cell is a plain TableCell.
This is the structure produced by the HTML backend for Wikipedia clade tables.
"""
doc = DoclingDocument(name="nested_tables")
def _add_level(parent, remaining: int):
table = doc.add_table(data=TableData(num_rows=1, num_cols=2), parent=parent)
if remaining > 0:
nested = _add_level(table, remaining - 1)
rich_cell: TableCell = RichTableCell(
ref=nested.get_ref(),
text="rich",
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
)
else:
rich_cell = TableCell(
text="leaf",
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
)
doc.add_table_cell(table, rich_cell)
doc.add_table_cell(
table,
TableCell(
text="plain",
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
),
)
return table
_add_level(doc.body, depth)
return doc
def test_md_nested_rich_table_no_hang():
"""Regression: export_to_markdown() must not hang on nested RichTableCells.
When a RichTableCell's content contains a nested table, the
``_nested_in_table`` flag passed through kwargs causes
MarkdownTableSerializer to flatten the inner table instead of
re-entering the full table serializer recursively. Without this
guard every level of nesting re-enters the table serializer, causing
exponential string growth and an indefinite hang.
"""
doc = _build_nested_rich_table_doc(depth=5)
result: list[str] = []
def _run() -> None:
result.append(doc.export_to_markdown())
t = threading.Thread(target=_run, daemon=True)
t.start()
t.join(timeout=5.0)
assert not t.is_alive(), "export_to_markdown() hung on a document with nested RichTableCells."
assert result, "export_to_markdown() produced no output"
# The outer table must be a valid 2-column markdown table.
# Without the pipe-escaping fix, inner-table pipes would leak into the outer
# table and produce dozens of phantom columns.
table_rows = [line for line in result[0].splitlines() if line.startswith("|")]
assert table_rows, "Expected at least one markdown table row in output"
col_counts = {line.count("|") - 1 for line in table_rows}
assert col_counts == {2}, f"Outer table must have exactly 2 columns throughout; got column counts: {col_counts}"
def test_md_compact_table():
"""Test compact table format removes padding and uses minimal separators."""
# Test the _compact_table method directly
padded_table = """| item | qty | description |
| ------ | ----: | :-------------------: |
| spam | 42 | A canned meat product |
| eggs | 451 | Fresh farm eggs |
| bacon | 0 | Out of stock |"""
expected_compact = """| item | qty | description |
| - | -: | :-: |
| spam | 42 | A canned meat product |
| eggs | 451 | Fresh farm eggs |
| bacon | 0 | Out of stock |"""
compact_result = MarkdownTableSerializer._compact_table(padded_table)
assert compact_result == expected_compact
# Verify space savings
assert len(compact_result) < len(padded_table)
def test_md_numeric_precision_preserved():
"""Test that numeric values in tables preserve their full precision.
Regression test for issue where tabulate's numparse would silently
truncate numeric strings to ~6 significant figures.
"""
doc = DoclingDocument(name="Numeric Precision Test")
precise_values = [
"225.8183",
"24797.34",
"20896.7184",
"17358.138",
"123.456789",
]
table = doc.add_table(data=TableData(num_rows=len(precise_values) + 1, num_cols=2))
# Add header row
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="Description",
),
)
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
text="Value",
),
)
# Add data rows with precise numeric values
for row_idx, value in enumerate(precise_values, start=1):
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row_idx,
end_row_offset_idx=row_idx + 1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text=f"Item {row_idx}",
),
)
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row_idx,
end_row_offset_idx=row_idx + 1,
start_col_offset_idx=1,
end_col_offset_idx=2,
text=value,
),
)
markdown_output = doc.export_to_markdown()
for value in precise_values:
assert value in markdown_output, (
f"Numeric value '{value}' was not preserved in markdown output. "
"This indicates precision loss during table serialization."
)
def test_md_traverse_pictures():
"""Test traverse_pictures parameter to include text inside PictureItems."""
doc = DoclingDocument(name="Test Document")
doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text before picture")
picture = doc.add_picture()
# Manually add a text item as child of picture
text_in_picture = TextItem(
self_ref=f"#/texts/{len(doc.texts)}",
parent=RefItem(cref=picture.self_ref),
label=DocItemLabel.PARAGRAPH,
text="Text inside picture",
orig="Text inside picture",
)
doc.texts.append(text_in_picture)
picture.children.append(RefItem(cref=text_in_picture.self_ref))
doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text after picture")
# Test with traverse_pictures=False (default)
ser_no_traverse = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
traverse_pictures=False,
),
)
result_no_traverse = ser_no_traverse.serialize().text
# Should NOT contain text inside picture
assert "Text before picture" in result_no_traverse
assert "Text after picture" in result_no_traverse
assert "Text inside picture" not in result_no_traverse
assert "<!-- image -->" in result_no_traverse
# Test with traverse_pictures=True
ser_with_traverse = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
image_mode=ImageRefMode.PLACEHOLDER,
image_placeholder="<!-- image -->",
traverse_pictures=True,
),
)
result_with_traverse = ser_with_traverse.serialize().text
# Should contain text inside picture
assert "Text before picture" in result_with_traverse
assert "Text after picture" in result_with_traverse
assert "Text inside picture" in result_with_traverse
assert "<!-- image -->" in result_with_traverse
def test_md_deep_section_headers_stay_headings():
"""Section headers deeper than level 5 are capped at six hashes."""
doc = DoclingDocument(name="deep")
for level in (4, 5, 6, 9):
doc.add_heading(text=f"Level {level}", level=level)
assert MarkdownDocSerializer(doc=doc).serialize().text == (
"##### Level 4\n\n###### Level 5\n\n###### Level 6\n\n###### Level 9"
)
def test_md_line_breaks():
"""Newlines inside TextItem and its subclasses are serialized per GFM spec.
Single `\\n` in body text and list items becomes a hard line break (` \\n`).
Double `\\n\\n` is preserved as a paragraph break.
Heading newlines are collapsed to a space (headings cannot span multiple lines).
Table cell newlines are replaced with a space (table syntax requires single lines).
"""
# Single \n in a TextItem -> GFM hard line break (two trailing spaces).
doc = DoclingDocument(name="lb")
doc.add_text(label=DocItemLabel.TEXT, text="Hello\nWorld")
assert MarkdownDocSerializer(doc=doc).serialize().text == "Hello \nWorld"
# \n\n in a TextItem -> paragraph break, no trailing spaces added.
doc = DoclingDocument(name="lb")
doc.add_text(label=DocItemLabel.TEXT, text="Para one\n\nPara two")
assert MarkdownDocSerializer(doc=doc).serialize().text == "Para one\n\nPara two"
# Mix of single and double newlines in one TextItem.
doc = DoclingDocument(name="lb")
doc.add_text(label=DocItemLabel.TEXT, text="A\nB\n\nC\nD")
assert MarkdownDocSerializer(doc=doc).serialize().text == "A \nB\n\nC \nD"
# \n in a SectionHeaderItem -> space (headings cannot span multiple lines;
# "Hello\nWorld" must become "### Hello World", not "### HelloWorld").
doc = DoclingDocument(name="lb")
doc.add_heading(text="Hello\nWorld", level=2)
assert MarkdownDocSerializer(doc=doc).serialize().text == "### Hello World"
# \n in a ListItem -> GFM hard line break.
doc = DoclingDocument(name="lb")
lg = doc.add_list_group()
doc.add_list_item(text="Line one\nLine two", parent=lg)
assert MarkdownDocSerializer(doc=doc).serialize().text == "- Line one \nLine two"
# \n in a plain table cell -> space (keeps the table row on a single line).
doc = DoclingDocument(name="lb")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="Line one\nLine two",
),
)
result = MarkdownDocSerializer(doc=doc).serialize().text
assert "\n" not in result.split("|")[1].strip()
# ===============================
# HTML tests
# ===============================
def test_html_table_serializer_get_header_and_body_lines():
"""Test HTMLTableSerializer.get_header_and_body_lines() method."""
serializer = HTMLTableSerializer()
# Test 1: Valid HTML with headers and data
valid_html = "<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Data1</td><td>Data2</td></tr></table>"
headers, body = serializer.get_header_and_body_lines(table_text=valid_html)
assert len(headers) > 0, "Should have headers"
assert len(body) > 0, "Should have body rows"
# Test 2: Row without closing </tr> tag
# Parser will find the row, but when we search for </tr> it won't be found
no_close_tr = "<tr><th>Header</th></tr><tr><td>Data1"
headers, body = serializer.get_header_and_body_lines(table_text=no_close_tr)
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 3: Data rows with incomplete closing tags
# When collecting remaining rows, some </tr> tags are missing
incomplete_data = "<tr><th>H1</th></tr><tr><td>D1</td></tr><tr><td>D2"
headers, body = serializer.get_header_and_body_lines(table_text=incomplete_data)
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 4: Force exception in parser
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
mock_parser = MagicMock()
mock_parser.feed.side_effect = Exception("Parser error")
mock_parser_class.return_value = mock_parser
broken_html = "<tr><th>Header</th></tr><tr><td>Data</td></tr>"
headers, body = serializer.get_header_and_body_lines(table_text=broken_html)
# Should use fallback logic
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 5: Parser returns more rows than exist in HTML
# Mock parser to return extra rows that don't exist in the HTML
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
mock_parser = MagicMock()
# Create fake row data - more rows than actually exist in HTML
mock_parser.rows = [
{"th_cells": ["H1"], "td_cells": []},
{"th_cells": ["H2"], "td_cells": []},
{"th_cells": ["H3"], "td_cells": []}, # This row doesn't exist in HTML
{"th_cells": [], "td_cells": ["D1"]},
]
mock_parser_class.return_value = mock_parser
# HTML with only 2 rows, but parser claims 4
limited_html = "<tr><th>H1</th></tr><tr><th>H2</th></tr>"
headers, body = serializer.get_header_and_body_lines(table_text=limited_html)
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 6: Specific case for line 485 - row_start found but row_end not found
# Create HTML where parser finds a row, but the actual HTML has <tr without </tr>
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
mock_parser = MagicMock()
# Parser reports a header row exists
mock_parser.rows = [
{"th_cells": ["Header"], "td_cells": []},
]
mock_parser_class.return_value = mock_parser
# But the HTML has <tr without matching </tr>
html_no_close = "<tr><th>Header"
headers, body = serializer.get_header_and_body_lines(table_text=html_no_close)
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 7: Specific case for line 504 - data collection finds <tr but no </tr>
# Create HTML where we start collecting data rows but encounter incomplete row
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
mock_parser = MagicMock()
# Parser reports header then data rows
mock_parser.rows = [
{"th_cells": ["H"], "td_cells": []},
{"th_cells": [], "td_cells": ["D1"]}, # This triggers data collection
]
mock_parser_class.return_value = mock_parser
# HTML has complete header but incomplete data row
html_incomplete_data = "<tr><th>H</th></tr><tr><td>D1</td></tr><tr><td>D2"
headers, body = serializer.get_header_and_body_lines(table_text=html_incomplete_data)
assert isinstance(headers, list)
assert isinstance(body, list)
# Test 8: Table with footer content
with_footer = "<tr><th>H</th></tr><tr><td>D</td></tr>Footer content"
headers, body = serializer.get_header_and_body_lines(table_text=with_footer)
assert isinstance(headers, list)
assert isinstance(body, list)
# Footer should be in body
assert "Footer" in str(body)
def test_html_charts():
src = Path("./tests/data/doc/barchart.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
def test_html_cross_page_list_page_break():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
),
)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
def test_html_cross_page_list_page_break_p1():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
pages={1},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_p1.gt.html", actual=actual)
def test_html_cross_page_list_page_break_p2():
src = Path("./tests/data/doc/activities.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
pages={2},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_p2.gt.html", actual=actual)
def test_html_split_page():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.EMBEDDED,
output_style=HTMLOutputStyle.SPLIT_PAGE,
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_split.gt.html", actual=actual)
def test_html_split_page_p2():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.EMBEDDED,
output_style=HTMLOutputStyle.SPLIT_PAGE,
pages={2},
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_split_p2.gt.html", actual=actual)
def test_html_split_page_p2_with_visualizer():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.EMBEDDED,
output_style=HTMLOutputStyle.SPLIT_PAGE,
pages={2},
),
)
ser_res = ser.serialize(
visualizer=LayoutVisualizer(),
)
actual = ser_res.text
# pinning the result with visualizer appeared flaky, so at least ensure it contains
# a figure (for the page) and that it is different than without visualizer:
assert '<figure><img src="data:image/png;base64' in actual
file_without_viz = src.parent / f"{src.stem}_split_p2.gt.html"
with open(file_without_viz) as f:
data_without_viz = f.read()
assert actual.strip() != data_without_viz.strip()
def test_html_split_page_no_page_breaks():
src = Path("./tests/data/doc/2408.09869_p1.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.EMBEDDED,
output_style=HTMLOutputStyle.SPLIT_PAGE,
),
)
actual = ser.serialize().text
verify(exp_file=src.parent / f"{src.stem}_split.gt.html", actual=actual)
def test_html_include_annotations_false():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
include_annotations=False,
pages={1},
html_head="<head></head>", # keeping test output minimal
),
)
actual = ser.serialize().text
verify(
exp_file=src.parent / f"{src.stem}_p1_include_annotations_false.gt.html",
actual=actual,
)
def test_html_include_annotations_true():
src = Path("./tests/data/doc/2408.09869v3_enriched.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(
doc=doc,
params=HTMLParams(
image_mode=ImageRefMode.PLACEHOLDER,
include_annotations=True,
pages={1},
html_head="<head></head>", # keeping test output minimal
),
)
actual = ser.serialize().text
verify(
exp_file=src.parent / f"{src.stem}_p1_include_annotations_true.gt.html",
actual=actual,
)
def test_html_list_item_markers(sample_doc):
root_dir = Path("./tests/data/doc")
for orig in [False, True]:
ser = HTMLDocSerializer(
doc=sample_doc,
params=HTMLParams(
show_original_list_item_marker=orig,
),
)
actual = ser.serialize().text
verify(
root_dir / f"constructed_orig_{str(orig).lower()}.gt.html",
actual=actual,
)
def test_html_nested_lists():
src = Path("./tests/data/doc/polymers.json")
doc = DoclingDocument.load_from_json(src)
ser = HTMLDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
def test_html_rich_table(rich_table_doc):
exp_file = Path("./tests/data/doc/rich_table.gt.html")
ser = HTMLDocSerializer(doc=rich_table_doc)
actual = ser.serialize().text
verify(exp_file=exp_file, actual=actual)
def test_html_rich_cell_textitem_ref_subtree_inside_and_not_outside():
"""Descendants of a RichTableCell's TextItem ref render inside the table.
With HTMLTextSerializer recursing into its item's children, the parent text
(CELL-TEXT) and its child (DEEP-LEAK) both render as siblings inside the
rich cell. The outer document iteration must not re-emit either of them as
standalone content after the table.
"""
doc = DoclingDocument(name="rich_cell_textitem_subtree")
table = doc.add_table(data=TableData(num_rows=1, num_cols=2))
rich_ref = doc.add_text(label=DocItemLabel.TEXT, text="CELL-TEXT", parent=table)
doc.add_text(label=DocItemLabel.TEXT, text="DEEP-LEAK", parent=rich_ref)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
ref=rich_ref.get_ref(),
text="cell 0,0",
),
)
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
text="plain",
),
)
out = HTMLDocSerializer(doc=doc).serialize().text
body = out[out.find("<body>") : out.find("</body>") + len("</body>")]
table_end = body.find("</table>") + len("</table>")
inside_table = body[:table_end]
after_table = body[table_end:]
assert "CELL-TEXT" in inside_table, f"RichTableCell ref content missing from the table:\n{body}"
assert "DEEP-LEAK" in inside_table, f"RichTableCell ref descendant missing from the table:\n{body}"
assert "CELL-TEXT" not in after_table, f"RichTableCell ref content leaked outside the table:\n{body}"
assert "DEEP-LEAK" not in after_table, f"RichTableCell ref descendant leaked outside the table:\n{body}"
def test_html_textitem_with_children_at_document_level():
"""Doc-level parity: TextItem with TextItem child renders both exactly once.
Once HTMLTextSerializer recurses into children, the child is rendered by
the parent's serialize call instead of by the outer iteration. Both must
still appear in document order, exactly once each.
"""
doc = DoclingDocument(name="textitem_with_children")
parent = doc.add_text(label=DocItemLabel.TEXT, text="PARENT-TEXT")
doc.add_text(label=DocItemLabel.TEXT, text="CHILD-TEXT", parent=parent)
out = HTMLDocSerializer(doc=doc).serialize().text
body = out[out.find("<body>") : out.find("</body>") + len("</body>")]
assert body.count("PARENT-TEXT") == 1, f"PARENT-TEXT should appear exactly once:\n{body}"
assert body.count("CHILD-TEXT") == 1, f"CHILD-TEXT should appear exactly once:\n{body}"
assert body.find("PARENT-TEXT") < body.find("CHILD-TEXT"), f"PARENT-TEXT should appear before CHILD-TEXT:\n{body}"
def test_html_inline_and_formatting():
src = Path("./tests/data/doc/inline_and_formatting.yaml")
doc = DoclingDocument.load_from_yaml(src)
ser = HTMLDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
@pytest.mark.parametrize(
("formatting", "hyperlink", "expected"),
[
(Formatting(bold=True), None, "<p><strong>Some text</strong></p>"),
(Formatting(italic=True), None, "<p><em>Some text</em></p>"),
(Formatting(underline=True), None, "<p><u>Some text</u></p>"),
(Formatting(strikethrough=True), None, "<p><del>Some text</del></p>"),
(Formatting(script=Script.SUB), None, "<p><sub>Some text</sub></p>"),
(Formatting(script=Script.SUPER), None, "<p><sup>Some text</sup></p>"),
(None, AnyUrl("https://example.com/page"), '<p><a href="https://example.com/page">Some text</a></p>'),
(
Formatting(bold=True, italic=True),
AnyUrl("https://example.com/page"),
'<p><a href="https://example.com/page"><em><strong>Some text</strong></em></a></p>',
),
],
)
def test_html_text_formatting_inside_paragraph(formatting, hyperlink, expected):
"""Formatting and hyperlinks of a text item go inside its <p> tag."""
doc = DoclingDocument(name="formatted_text")
item = doc.add_text(label=DocItemLabel.TEXT, text="Some text", formatting=formatting, hyperlink=hyperlink)
assert HTMLDocSerializer(doc=doc).serialize(item=item).text == expected
def test_html_heading_formatting_inside_heading_tag():
"""Formatting and hyperlinks of a title or section header go inside its <hN> tag."""
doc = DoclingDocument(name="formatted_headings")
title = doc.add_title(text="A title", formatting=Formatting(italic=True))
heading = doc.add_heading(
text="A section",
level=1,
formatting=Formatting(bold=True),
hyperlink=AnyUrl("https://example.com/section"),
)
ser = HTMLDocSerializer(doc=doc)
assert ser.serialize(item=title).text == "<h1><em>A title</em></h1>"
assert ser.serialize(item=heading).text == (
'<h2><a href="https://example.com/section"><strong>A section</strong></a></h2>'
)
def test_html_rich_table_cell_formatting_inside_paragraph():
"""A bold-only rich table cell keeps the <strong> tag inside the <p> tag of the cell."""
doc = DoclingDocument(name="bold_rich_cells")
table = doc.add_table(data=TableData(num_rows=2, num_cols=2))
for row, text in enumerate(["Bold header", "Bold cell"]):
bold_text = doc.add_text(label=DocItemLabel.TEXT, text=text, parent=table, formatting=Formatting(bold=True))
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=row,
end_row_offset_idx=row + 1,
start_col_offset_idx=0,
end_col_offset_idx=1,
column_header=row == 0,
ref=bold_text.get_ref(),
text=text,
),
)
for row, text in enumerate(["Plain header", "Plain cell"]):
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=row,
end_row_offset_idx=row + 1,
start_col_offset_idx=1,
end_col_offset_idx=2,
column_header=row == 0,
text=text,
),
)
assert HTMLDocSerializer(doc=doc).serialize(item=table).text == (
"<table><tbody>"
"<tr><th><p><strong>Bold header</strong></p></th><th>Plain header</th></tr>"
"<tr><td><p><strong>Bold cell</strong></p></td><td>Plain cell</td></tr>"
"</tbody></table>"
)
class _HTMLProbe(HTMLParser):
"""Collects the elements, attributes and per-element text of a parsed HTML document."""
_VOID_TAGS = frozenset({"img", "br", "meta", "link"})
def __init__(self) -> None:
super().__init__()
self.elements: list[tuple[str, dict[str, str | None]]] = []
self._open: list[str] = []
self._text: dict[str, list[str]] = {}
@property
def tags(self) -> set[str]:
return {tag for tag, _ in self.elements}
def text_of(self, tag: str) -> str:
"""Returns the text directly contained in the elements with the given tag."""
return "".join(self._text.get(tag, []))
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
self.elements.append((tag, dict(attrs)))
if tag not in self._VOID_TAGS:
self._open.append(tag)
def handle_endtag(self, tag: str) -> None:
if self._open and self._open[-1] == tag:
self._open.pop()
def handle_data(self, data: str) -> None:
if self._open:
self._text.setdefault(self._open[-1], []).append(data)
def _parse_html(markup: str) -> _HTMLProbe:
probe = _HTMLProbe()
probe.feed(markup)
return probe
@pytest.mark.parametrize(
"code",
[
'conda create -n <CONDA_NAME>\nDict<str> x = "<div>";',
"if a && b:\n print('&amp; stays literal')",
],
)
def test_html_code_item_text_is_escaped(code: str):
doc = DoclingDocument(name="code-escape")
doc.add_code(text=code)
probe = _parse_html(HTMLDocSerializer(doc=doc).serialize().text)
assert probe.text_of("code") == code
def test_html_formula_without_mathml_keeps_text():
doc = DoclingDocument(name="formula-fallback")
doc.add_formula(text="a<b && c>d")
actual = HTMLDocSerializer(doc=doc, params=HTMLParams(formula_to_mathml=False)).serialize().text
assert "Formula not decoded" not in actual
assert _parse_html(actual).text_of("pre") == "a<b && c>d"
def test_html_inline_formula_without_mathml_keeps_text():
doc = DoclingDocument(name="formula-inline-fallback")
heading = doc.add_heading(text="", level=1)
ig = doc.add_inline_group(parent=heading)
doc.add_formula(text="x<y", parent=ig)
actual = HTMLDocSerializer(doc=doc, params=HTMLParams(formula_to_mathml=False)).serialize().text
assert _parse_html(actual).text_of("code") == "x<y"
def test_html_formula_without_mathml_does_not_warn_about_mathml(caplog: pytest.LogCaptureFixture):
doc = DoclingDocument(name="formula-no-mathml-warning")
doc.add_formula(text="a<b")
with caplog.at_level("WARNING", logger="docling_core.transforms.serializer.html"):
HTMLDocSerializer(doc=doc, params=HTMLParams(formula_to_mathml=False)).serialize()
assert not caplog.records
def test_html_formula_mathml_failure_falls_back_to_escaped_pre():
doc = DoclingDocument(name="formula-mathml-failure")
doc.add_formula(text="a<b & c")
with patch(
"docling_core.transforms.serializer.html.latex2mathml.converter.convert_to_element",
side_effect=ValueError("unsupported LaTeX"),
):
actual = HTMLDocSerializer(doc=doc).serialize().text
assert "Formula not decoded" not in actual
assert _parse_html(actual).text_of("pre") == "a<b & c"
@pytest.mark.parametrize(
"formula",
[
"a<b && c>d",
r"\text{<svg/onload=alert(1)>}",
"x</annotation></math><img src=x onerror=alert(1)>",
],
)
def test_html_mathml_annotation_is_escaped(formula: str):
doc = DoclingDocument(name="formula-mathml-annotation")
doc.add_formula(text=formula)
probe = _parse_html(HTMLDocSerializer(doc=doc).serialize().text)
assert probe.tags.isdisjoint({"img", "svg", "b"})
assert probe.text_of("annotation") == formula
def test_html_mathml_keeps_symbol_character_references():
# Escaping formula source must not double-escape MathML symbol references.
doc = DoclingDocument(name="formula-mathml-symbols")
doc.add_formula(text=r"\alpha \leq \beta")
actual = HTMLDocSerializer(doc=doc).serialize().text
assert "<mi>&#x003B1;</mi><mo>&#x02264;</mo><mi>&#x003B2;</mi>" in actual
assert "&amp;#x" not in actual
def test_html_formula_image_fallback_escapes_alt_attribute(doc_with_handwritten: DoclingDocument):
doc = doc_with_handwritten
doc.pages[1].image = ImageRef.from_pil(Image.new("RGB", (100, 100), "white"), dpi=72)
orig = 'x" onerror="alert(1)" y="<b>'
doc.add_formula(text="", orig=orig, prov=doc.texts[1].prov[0])
actual = HTMLDocSerializer(doc=doc, params=HTMLParams(image_mode=ImageRefMode.EMBEDDED)).serialize().text
probe = _parse_html(actual)
imgs = [attrs for tag, attrs in probe.elements if tag == "img"]
assert len(imgs) == 1
assert imgs[0]["alt"] == orig
assert "onerror" not in imgs[0]
assert "b" not in probe.tags
@pytest.mark.parametrize(
("inline_scope", "expected"),
[
(False, "<pre><span class='inline-group'>a&lt;b x</span></pre>"),
(True, "<code><span class='inline-group'>a&lt;b x</span></code>"),
],
)
def test_html_formula_with_inline_group_is_not_double_escaped(inline_scope: bool, expected: str):
doc = DoclingDocument(name="formula-inline-group")
parent = None
if inline_scope:
parent = doc.add_inline_group(parent=doc.add_heading(text="", level=1))
formula = doc.add_formula(text="", parent=parent)
group = doc.add_inline_group(parent=formula)
doc.add_text(label=DocItemLabel.TEXT, text="a<b ", parent=group)
doc.add_text(label=DocItemLabel.TEXT, text="x", parent=group)
actual = HTMLDocSerializer(doc=doc, params=HTMLParams(formula_to_mathml=False)).serialize().text
assert expected in actual
@pytest.mark.parametrize(
("marker", "expected_style"),
[
('x" onmouseover="alert(1)" y="', 'list-style-type: \'x" onmouseover="alert(1)" y=" \';'),
("it's", "list-style-type: 'it\\'s ';"),
("'; color: red; content: '", "list-style-type: '\\'; color: red; content: \\' ';"),
],
)
def test_html_list_item_marker_cannot_break_out_of_style_attribute(marker: str, expected_style: str):
doc = DoclingDocument(name="list-marker")
group = doc.add_list_group()
doc.add_list_item(text="item", parent=group, marker=marker)
probe = _parse_html(HTMLDocSerializer(doc=doc).serialize().text)
(li_attrs,) = [attrs for tag, attrs in probe.elements if tag == "li"]
assert li_attrs == {"style": expected_style}
def test_html_path_hyperlink_cannot_break_out_of_href_attribute():
doc = DoclingDocument(name="path-hyperlink")
doc.add_text(label=DocItemLabel.TEXT, text="link", hyperlink=Path('a" onmouseover="alert(1)'))
probe = _parse_html(HTMLDocSerializer(doc=doc).serialize().text)
(anchor,) = [attrs for tag, attrs in probe.elements if tag == "a"]
assert anchor == {"href": 'a" onmouseover="alert(1)'}
def test_html_url_hyperlink_keeps_query_separator():
doc = DoclingDocument(name="url-hyperlink")
doc.add_text(label=DocItemLabel.TEXT, text="link", hyperlink=AnyUrl("https://example.com/p?a=1&b=2"))
probe = _parse_html(HTMLDocSerializer(doc=doc).serialize().text)
(anchor,) = [attrs for tag, attrs in probe.elements if tag == "a"]
assert anchor == {"href": "https://example.com/p?a=1&b=2"}
def test_html_embedded_data_uri_cannot_break_out_of_src_attribute():
doc = DoclingDocument(name="data-uri-picture")
uri = AnyUrl('data:image/png;base64,AAA"onerror="alert(1)')
doc.add_picture(image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=1, height=1), uri=uri))
actual = HTMLDocSerializer(doc=doc, params=HTMLParams(image_mode=ImageRefMode.EMBEDDED)).serialize().text
(img,) = [attrs for tag, attrs in _parse_html(actual).elements if tag == "img"]
assert set(img) == {"src"}
def test_html_split_page_data_uri_cannot_break_out_of_src_attribute():
doc = DoclingDocument(name="data-uri-page")
uri = AnyUrl('data:image/png;base64,AAA"onerror="alert(1)')
doc.add_page(
page_no=1,
size=Size(width=10, height=10),
image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=10, height=10), uri=uri),
)
doc.add_text(
label=DocItemLabel.TEXT,
text="text",
prov=ProvenanceItem(page_no=1, bbox=BoundingBox(l=0, t=0, r=1, b=1), charspan=(0, 4)),
)
params = HTMLParams(image_mode=ImageRefMode.EMBEDDED, output_style=HTMLOutputStyle.SPLIT_PAGE)
actual = HTMLDocSerializer(doc=doc, params=params).serialize().text
(img,) = [attrs for tag, attrs in _parse_html(actual).elements if tag == "img"]
assert set(img) == {"src"}
# ===============================
# WebVTT tests
# ===============================
@pytest.mark.parametrize(
"example_num",
[1, 2, 3, 4, 5],
)
def test_webvtt(example_num):
src = Path(f"tests/data/doc/webvtt_example_{example_num:02d}.json")
doc = DoclingDocument.load_from_json(src)
ser = WebVTTDocSerializer(doc=doc)
actual = ser.serialize().text
verify(exp_file=src.with_suffix(".gt.vtt"), actual=actual)
def test_webvtt_params():
"""Test WebVTT serialization with WebVTTParams."""
src = Path("./tests/data/doc/webvtt_example_01.json")
doc = DoclingDocument.load_from_json(src)
# Test with omit_hours_if_zero=True
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_hours_if_zero=True))
actual = ser.serialize().text
assert "00:11.000 --> 00:13.000" in actual
# Test with omit_voice_end=True
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_voice_end=True))
actual = ser.serialize().text
assert "</v>" not in actual
# Test with both parameters enabled
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_hours_if_zero=True, omit_voice_end=True))
actual = ser.serialize().text
assert "00:11.000 --> 00:13.000" in actual
assert "</v>" not in actual
ser_default = WebVTTDocSerializer(doc=doc, params=WebVTTParams())
actual_default = ser_default.serialize().text
assert len(actual) <= len(actual_default) or actual != actual_default
def test_webvtt_cue_shorter_than_a_millisecond_stays_serializable(caplog):
"""A sub-millisecond track segment must not abort the whole serialization.
TrackSource accepts any end_time greater than start_time, but WebVTT
timestamps carry milliseconds only, so both ends used to round onto the
same timestamp and WebVTTCueTimings rejected the zero-length cue. The cue is
kept at the smallest representable span, and that loss of precision is
reported in the logs instead of silently dropping the segment.
"""
doc = DoclingDocument(name="sub-millisecond-cue")
doc.add_text(
label=DocItemLabel.TEXT,
text="short cue",
source=TrackSource(start_time=11.00001, end_time=11.00002),
)
doc.add_text(
label=DocItemLabel.TEXT,
text="normal cue",
source=TrackSource(start_time=12.0, end_time=13.5),
)
with caplog.at_level(logging.WARNING, logger="docling_core.transforms.serializer.webvtt"):
actual = WebVTTDocSerializer(doc=doc).serialize().text
assert "00:00:11.000 --> 00:00:11.001" in actual
assert "11.00001..11.00002" in caplog.text
assert "short cue" in actual
assert "00:00:12.000 --> 00:00:13.500" in actual
assert "normal cue" in actual
def test_html_meta_emits_xhtml_compatible_attributes():
"""Test that metadata attributes are name=value pairs."""
doc = DoclingDocument(name="x")
plain = TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="",
)
table = doc.add_table(data=TableData(num_rows=1, num_cols=1, table_cells=[plain]))
pic = doc.add_picture(parent=table)
pic.meta = PictureMeta(
classification=PictureClassificationMetaField(
predictions=[PictureClassificationPrediction(class_name="other", confidence=1.0)]
)
)
table.data.table_cells = [
RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="",
ref=pic.get_ref(),
)
]
text = doc.add_text(
label=DocItemLabel.TEXT, text="Output of HTML serializer must be parseable by a strict XML parser"
)
text.meta = BaseMeta(
summary=SummaryMetaField(text="XHTML-compliant"),
language=LanguageMetaField(code="en"),
entities=EntitiesMetaField(
mentions=[
EntityMention(
text="HTML serializer",
label="software",
charspan=CharSpan((10, 25)),
),
EntityMention(
text="XML parser",
label="software",
charspan=CharSpan((56, 66)),
),
]
),
)
html_out = table.export_to_html(doc)
# No bare valueless attribute like `data-meta-classification` (must be
# followed by `=` to be XHTML-compliant).
assert "data-meta-classification>" not in html_out
assert 'data-meta-name="classification"' in html_out
html_out = doc.export_to_html()
print(html_out)
assert 'data-meta-name="language"' in html_out
assert 'data-meta-name="entities"' in html_out
# Output must be parseable by a strict XML parser.
ET.fromstring(html_out)
# A link destination to encode, paired with its expected encoding.
_EscapeCase: TypeAlias = tuple[AnyUrl | PurePath, str]
_ESCAPE_PATH_CASES: list[_EscapeCase] = [
# A relative path with nothing to encode: the hash-based names generated by
# `_with_pictures_refs` must pass through untouched.
(PurePosixPath("doc_artifacts/image_000001_ab12.png"), "doc_artifacts/image_000001_ab12.png"),
# A relative artifacts dir is derived from the output filename, so it can hold
# spaces. An unencoded space ends the link destination.
(PurePosixPath("My Report_artifacts/img.png"), "My%20Report_artifacts/img.png"),
# Parentheses are only valid in a destination while balanced; encoding them keeps
# an odd one in a filename from ending the link early.
(PurePosixPath("artifacts/img (1).png"), "artifacts/img%20%281%29.png"),
# In a path these are literal characters, not URI delimiters. "%" is kept as-is so
# that an already-encoded URI is not encoded twice.
(PurePosixPath("100%_scale/a#b?c.png"), "100%_scale/a%23b%3Fc.png"),
# A root-relative POSIX path stays root-relative.
(PurePosixPath("/home/a b/img.png"), "/home/a%20b/img.png"),
# Backslash separators must become forward slashes: "\" is an escape character in
# Markdown, so "dir\img.png" would render as "dirimg.png".
(PureWindowsPath("My Report_artifacts/img.png"), "My%20Report_artifacts/img.png"),
# An absolute Windows path becomes an RFC 8089 file:// URL, so that "C:" cannot be
# read as a URL scheme.
(PureWindowsPath("C:/Users/me/My Docs/img.png"), "file:///C:/Users/me/My%20Docs/img.png"),
# A UNC share becomes the authority of the file:// URL, so that the leading "//"
# cannot be read as a scheme-relative URL.
(PureWindowsPath("//server/share/My Docs/img.png"), "file://server/share/My%20Docs/img.png"),
]
_ESCAPE_URL_CASES: list[_EscapeCase] = [
# A URL keeps its scheme, authority, and delimiters; only the components get
# encoded. Escapes pydantic already applied must not become "%2520".
(AnyUrl("file:///home/a b/img.png"), "file:///home/a%20b/img.png"),
(AnyUrl("s3://bucket/My Report_artifacts/img.png"), "s3://bucket/My%20Report_artifacts/img.png"),
(AnyUrl("https://example.com:8080/a b.png?w=1&h=2#frag"), "https://example.com:8080/a%20b.png?w=1&h=2#frag"),
# pydantic leaves parentheses as-is, so the encoding still has work to do here.
(AnyUrl("https://example.com/img (1).png"), "https://example.com/img%20%281%29.png"),
]
def _as_destination(dest: str) -> AnyUrl | PurePath:
"""Re-read an encoded destination the way a caller would supply it."""
return AnyUrl(dest) if "://" in dest else PurePosixPath(dest)
@pytest.mark.parametrize(("value", "expected"), _ESCAPE_PATH_CASES + _ESCAPE_URL_CASES)
def test_escape_uri_path(value: AnyUrl | PurePath, expected: str):
"""Test encoding of link destinations for both URLs and local paths."""
assert MarkdownPictureSerializer._escape_uri_path(value) == expected
@pytest.mark.parametrize(("value", "expected"), _ESCAPE_PATH_CASES + _ESCAPE_URL_CASES)
def test_escape_uri_path_is_idempotent(value: AnyUrl | PurePath, expected: str):
"""Test that re-encoding an encoded destination is a no-op (no double-encoding)."""
assert MarkdownPictureSerializer._escape_uri_path(_as_destination(expected)) == expected
@pytest.mark.parametrize(
"posix_spelling",
[
"doc_artifacts/image_000001_ab12.png",
"My Report_artifacts/img.png",
"C:/Users/me/My Docs/img.png",
"//server/share/My Docs/img.png",
],
)
def test_escape_uri_path_is_flavour_independent(posix_spelling: str):
"""Test that a path encodes the same way however its separators arrive.
A document authored on Windows is re-read as a POSIX path on another host, and PRs
#641 and #663 make `_with_pictures_refs` store the POSIX spelling on Windows. Neither
may change the destination.
"""
escape = MarkdownPictureSerializer._escape_uri_path
windows = PureWindowsPath(posix_spelling)
assert "\\" in str(windows) # guard: the fixture really has separators to convert
assert escape(PurePosixPath(posix_spelling)) == escape(windows)
assert escape(PurePosixPath(str(windows))) == escape(windows)
def test_escape_uri_path_file_scheme_is_only_for_absolute_windows_paths():
"""Test that only an absolute Windows path is given a file:// scheme."""
escape = MarkdownPictureSerializer._escape_uri_path
# A drive letter and a UNC host are the two absolute Windows forms.
assert escape(PureWindowsPath("C:/Users/me/img.png")) == "file:///C:/Users/me/img.png"
assert escape(PureWindowsPath("//server/share/img.png")) == "file://server/share/img.png"
# Nothing else acquires the scheme, a root-relative POSIX path included.
for value in (
PurePosixPath("artifacts/img.png"),
PurePosixPath("/home/me/img.png"),
PureWindowsPath("artifacts/img.png"),
AnyUrl("https://example.com/img.png"),
AnyUrl("s3://bucket/img.png"),
):
assert not escape(value).startswith("file:")
# A URL that already carries the scheme keeps it: dropping it would turn an absolute
# filesystem reference into a root-relative URL.
assert escape(AnyUrl("file:///home/me/img.png")) == "file:///home/me/img.png"
@pytest.mark.parametrize(
("uri", "expected"),
[
(Path("doc_artifacts/image_000001_ab12.png"), "doc_artifacts/image_000001_ab12.png"),
(Path("My Report (final)_artifacts/img.png"), "My%20Report%20%28final%29_artifacts/img.png"),
# A Windows-authored separator resolves the same way on either exporting host.
(Path("My Report_artifacts\\img.png"), "My%20Report_artifacts/img.png"),
# An AnyUrl was already percent-encoded by pydantic on parse: its escapes must
# not be encoded a second time, nor its scheme mangled into "file%3A".
(AnyUrl("file:///home/a b/img.png"), "file:///home/a%20b/img.png"),
# `_save_image_and_resolve_uri` returns an AnyUrl for remote artifact dirs.
(AnyUrl("s3://bucket/My Report_artifacts/img.png"), "s3://bucket/My%20Report_artifacts/img.png"),
# Query delimiters must stay functional.
(AnyUrl("https://example.com/img.png?w=1&h=2"), "https://example.com/img.png?w=1&h=2"),
],
)
def test_referenced_image_uri_is_encoded(uri, expected: str):
"""Test that `_serialize_image_part` encodes the URI it emits."""
doc = DoclingDocument(name="x")
doc.add_picture(image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=10, height=10), uri=uri))
assert doc.export_to_markdown(image_mode=ImageRefMode.REFERENCED) == f"![Image]({expected})"
@pytest.mark.parametrize(
("uri", "expected"),
[
(Path("doc_artifacts/image_000001_ab12.png"), "doc_artifacts/image_000001_ab12.png"),
(Path("My Report (final)_artifacts/img.png"), "My%20Report%20%28final%29_artifacts/img.png"),
# A Windows-authored separator must reach the page as "/", not as "%5C".
(Path("My Report_artifacts\\img.png"), "My%20Report_artifacts/img.png"),
# An absolute Windows path becomes a file:// URL, as in the Markdown output.
(Path("C:/Users/me/My Docs/img.png"), "file:///C:/Users/me/My%20Docs/img.png"),
# URLs keep their scheme and delimiters, and pydantic's escapes are not encoded twice.
(AnyUrl("file:///home/a b/img.png"), "file:///home/a%20b/img.png"),
(AnyUrl("s3://bucket/My Report_artifacts/img.png"), "s3://bucket/My%20Report_artifacts/img.png"),
# In an HTML attribute, "&" is written as "&amp;".
(AnyUrl("https://example.com/img.png?w=1&h=2"), "https://example.com/img.png?w=1&amp;h=2"),
],
)
def test_html_referenced_image_uri_is_encoded(uri, expected: str):
"""Test that the HTML serializer encodes a referenced image URI the same way as Markdown."""
doc = DoclingDocument(name="x")
doc.add_picture(image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=10, height=10), uri=uri))
assert f'<img src="{expected}">' in doc.export_to_html(image_mode=ImageRefMode.REFERENCED)
def test_referenced_image_uri_encoding_only_applies_to_referenced_mode():
"""Test that PLACEHOLDER mode still emits the placeholder, not an encoded URI."""
doc = DoclingDocument(name="x")
uri = Path("My Report_artifacts/img.png")
doc.add_picture(image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=10, height=10), uri=uri))
assert doc.export_to_markdown(image_mode=ImageRefMode.PLACEHOLDER) == "<!-- image -->"
def test_referenced_image_data_uri_is_not_encoded():
"""Test that a data URI still falls back to the placeholder in REFERENCED mode."""
doc = DoclingDocument(name="x")
uri = AnyUrl("data:image/png;base64,iVBORw0KGgo=")
doc.add_picture(image=ImageRef(mimetype="image/png", dpi=72, size=Size(width=10, height=10), uri=uri))
assert doc.export_to_markdown(image_mode=ImageRefMode.REFERENCED) == "<!-- image -->"
@pytest.mark.parametrize(
("hyperlink", "expected"),
[
# The plain relative link from the report (native and POSIX spelling).
(Path("sub/next.html"), "sub/next.html"),
(PurePosixPath("sub/next.html"), "sub/next.html"),
# A Windows spelling normalizes on every host -- a native POSIX Path
# treats '\' as a legal filename character, so the foreign spelling
# must be expressed with PureWindowsPath to stay host-independent.
(PureWindowsPath("sub\\next.html"), "sub/next.html"),
# Fragment and query delimiters are part of the URL, not of the path, and must
# not be percent-encoded the way an image path would be.
(Path("sub/next.html#section"), "sub/next.html#section"),
(Path("sub/next.html?x=1#section"), "sub/next.html?x=1#section"),
# Spaces and existing escapes stay as they are.
(Path("dir with space/x.html"), "dir with space/x.html"),
(Path("a%20b/x.html"), "a%20b/x.html"),
# A URL is untouched, including its own backslash-free spelling.
(AnyUrl("https://example.com/a/b?q=1#f"), "https://example.com/a/b?q=1#f"),
],
)
def test_hyperlink_uri_is_portable(hyperlink: AnyUrl | PurePath, expected: str):
"""Test that `hyperlink_uri` emits a portable destination on every host."""
from docling_core.transforms.serializer.common import hyperlink_uri
assert hyperlink_uri(hyperlink) == expected
@pytest.mark.parametrize(
("hyperlink", "expected_md", "expected_html"),
[
(Path("sub/next.html"), "[next page](sub/next.html)", '<a href="sub/next.html">'),
(
Path("sub/next.html#section"),
"[next page](sub/next.html#section)",
'<a href="sub/next.html#section">',
),
(
Path("sub/next.html?x=1#section"),
"[next page](sub/next.html?x=1#section)",
'<a href="sub/next.html?x=1#section">',
),
],
)
def test_relative_hyperlink_export_uses_forward_slashes(hyperlink: AnyUrl | Path, expected_md: str, expected_html: str):
"""Test that Markdown and HTML export a relative hyperlink with `/` separators."""
doc = DoclingDocument(name="x")
doc.add_text(label=DocItemLabel.TEXT, text="next page", hyperlink=hyperlink)
assert doc.export_to_markdown().strip() == expected_md
html = doc.export_to_html(image_mode=ImageRefMode.PLACEHOLDER)
assert expected_html in html
def test_relative_hyperlink_export_doclang_uses_forward_slashes():
"""Test that DocLang exports a relative hyperlink with `/` separators."""
doc = DoclingDocument(name="x")
doc.add_text(label=DocItemLabel.TEXT, text="next page", hyperlink=Path("sub/next.html"))
doclang = doc.export_to_doclang()
assert '<href uri="sub/next.html"/>' in doclang
assert "sub\\next.html" not in doclang
@pytest.mark.xfail(reason="LaTeX backslash-escapes '#' inside href URL args; pre-existing, out of scope")
def test_relative_hyperlink_export_latex_keeps_fragment():
"""LaTeX should keep the '#' fragment delimiter in a hyperlink destination."""
doc = DoclingDocument(name="x")
doc.add_text(label=DocItemLabel.TEXT, text="next page", hyperlink=Path("sub/next.html#section"))
tex = LaTeXDocSerializer(doc=doc).serialize().text
assert "\\href{sub/next.html#section}" in tex
_BREAKOUT = Path('x">\n<script>1</script>\n<a href="')
def _doc_with_hyperlink(hyperlink: AnyUrl | Path) -> DoclingDocument:
doc = DoclingDocument(name="x")
doc.add_text(label=DocItemLabel.TEXT, text="c", hyperlink=hyperlink)
return doc
class _TagCollector(HTMLParser):
def __init__(self):
super().__init__()
self.tags: list[tuple[str, dict[str, str | None]]] = []
def handle_starttag(self, tag, attrs):
self.tags.append((tag, dict(attrs)))
def test_html_hyperlink_cannot_break_out_of_attribute():
"""An untrusted hyperlink stays inside the href attribute and injects no elements."""
collector = _TagCollector()
collector.feed(_doc_with_hyperlink(_BREAKOUT).export_to_html())
assert "script" not in [tag for tag, _ in collector.tags]
assert [attrs["href"] for tag, attrs in collector.tags if tag == "a"] == [_BREAKOUT.as_posix()]
@pytest.mark.parametrize(
"hyperlink",
[
Path("javascript:alert(1)"),
AnyUrl("javascript:alert(1)"),
AnyUrl("JaVaScRiPt:alert(1)"),
Path(" javascript:alert(1)"),
Path("java\tscript:alert(1)"),
AnyUrl("data:text/html,<script>1</script>"),
Path("vbscript:msgbox(1)"),
],
)
def test_unsafe_hyperlink_scheme_is_dropped(hyperlink: AnyUrl | Path):
"""Hyperlinks with an active scheme keep the link text but are not emitted as links."""
doc = _doc_with_hyperlink(hyperlink)
assert doc.export_to_markdown().strip() == "c"
html = doc.export_to_html()
assert "<a " not in html
assert "<p>c</p>" in html
assert "\\href" not in LaTeXDocSerializer(doc=doc).serialize().text
@pytest.mark.parametrize(
("hyperlink", "expected_md", "expected_html"),
[
(AnyUrl("https://example.com/x"), "[c](https://example.com/x)", '<a href="https://example.com/x">'),
(AnyUrl("mailto:a@example.com"), "[c](mailto:a@example.com)", '<a href="mailto:a@example.com">'),
(Path("sub/next.html"), "[c](sub/next.html)", '<a href="sub/next.html">'),
(Path("C:/docs/a.pdf"), "[c](C:/docs/a.pdf)", '<a href="C:/docs/a.pdf">'),
(Path("#anchor"), "[c](#anchor)", '<a href="#anchor">'),
],
)
def test_safe_hyperlink_is_preserved(hyperlink: AnyUrl | Path, expected_md: str, expected_html: str):
"""Relative references and allowlisted schemes are still emitted as links."""
doc = _doc_with_hyperlink(hyperlink)
assert doc.export_to_markdown().strip() == expected_md
assert expected_html in doc.export_to_html()
def test_malformed_hyperlink_does_not_fail_export():
"""A malformed URL, which makes `urlsplit` raise, must not abort the export."""
doc = _doc_with_hyperlink(Path("//[::1/x"))
assert doc.export_to_markdown().strip() == "[c](//[::1/x)"
assert '<a href="//[::1/x">' in doc.export_to_html()
assert "\\href{" in LaTeXDocSerializer(doc=doc).serialize().text
@pytest.mark.parametrize(
("hyperlink", "expected"),
[
(_BREAKOUT, "x%22%3E%0A%3Cscript%3E1%3C/script%3E%0A%3Ca%20href=%22"),
(Path("a(b)c d"), "a%28b%29c%20d"),
# Markdown renderers decode entities in link destinations, which would
# rebuild a scheme the allowlist has already rejected.
(Path("&#106;avascript:alert(1)"), "&amp;#106;avascript:alert%281%29"),
(Path("a?x=1&y=2"), "a?x=1&y=2"),
],
)
def test_markdown_hyperlink_destination_is_encoded(hyperlink: Path, expected: str):
"""Characters that end or alter a Markdown link destination are encoded."""
md = _doc_with_hyperlink(hyperlink).export_to_markdown().strip()
assert md == f"[c]({expected})"
def test_doclang_hyperlink_stays_inside_href_attribute():
"""The DocLang output remains well-formed XML and round-trips the destination."""
payload = Path('x"><script>1</script><a href="')
root = ET.fromstring(_doc_with_hyperlink(payload).export_to_doclang())
assert [el.get("uri") for el in root.iter("href")] == [payload.as_posix()]
assert not list(root.iter("script"))
_DATA_URI_BREAKOUT = AnyUrl('data:image/png;base64,x"><script>1</script>)<b>(')
def _image_ref(uri: AnyUrl | Path) -> ImageRef:
return ImageRef(mimetype="image/png", dpi=72, size=Size(width=1, height=1), uri=uri)
def test_html_embedded_data_uri_stays_inside_src_attribute():
"""A data URI from the document stays inside the src attribute of picture and page images."""
doc = DoclingDocument(name="x")
doc.add_page(page_no=1, size=Size(width=1, height=1), image=_image_ref(_DATA_URI_BREAKOUT))
prov = ProvenanceItem(page_no=1, bbox=BoundingBox(l=0, t=0, r=1, b=1), charspan=(0, 1))
doc.add_picture(image=_image_ref(_DATA_URI_BREAKOUT), prov=prov)
for html in (
doc.export_to_html(image_mode=ImageRefMode.EMBEDDED),
doc.export_to_html(image_mode=ImageRefMode.EMBEDDED, split_page_view=True),
):
collector = _TagCollector()
collector.feed(html)
srcs = [attrs["src"] for tag, attrs in collector.tags if tag == "img"]
assert srcs and set(srcs) == {str(_DATA_URI_BREAKOUT)}
assert "script" not in [tag for tag, _ in collector.tags]
def test_markdown_embedded_data_uri_is_encoded():
"""A data URI from the document cannot end the Markdown image destination early."""
doc = DoclingDocument(name="x")
doc.add_picture(image=_image_ref(_DATA_URI_BREAKOUT))
md = doc.export_to_markdown(image_mode=ImageRefMode.EMBEDDED).strip()
assert md == "![Image](data:image/png;base64,x%22%3E%3Cscript%3E1%3C/script%3E%29%3Cb%3E%28)"
def test_doclang_src_stays_inside_attribute():
"""The DocLang output remains well-formed XML and round-trips the image URI."""
uri = Path('a"<b>.png')
doc = DoclingDocument(name="x")
doc.add_picture(image=_image_ref(uri))
root = ET.fromstring(doc.export_to_doclang(image_mode=ImageRefMode.REFERENCED))
assert [el.get("uri") for el in root.iter("src")] == [str(uri)]
def test_export_to_markdown_image_dir_saves_and_references_images(sample_doc, tmp_path):
"""export_to_markdown with image_dir saves images and references the portable URI."""
image_dir = tmp_path / "images"
md = sample_doc.export_to_markdown(
image_mode=ImageRefMode.REFERENCED,
image_dir=image_dir,
image_uri_prefix="images/",
)
saved = sorted(image_dir.glob("*.png"))
assert saved, "expected at least one image to be saved"
for img in saved:
assert f"images/{img.name}" in md
assert str(image_dir.resolve()) not in md
def test_export_to_html_image_dir_saves_and_references_images(sample_doc, tmp_path):
"""export_to_html with image_dir saves images and references the portable URI."""
image_dir = tmp_path / "images"
html = sample_doc.export_to_html(
image_mode=ImageRefMode.REFERENCED,
image_dir=image_dir,
image_uri_prefix="images/",
)
saved = sorted(image_dir.glob("*.png"))
assert saved, "expected at least one image to be saved"
for img in saved:
assert f"images/{img.name}" in html
def test_export_to_markdown_image_dir_without_referenced_raises(sample_doc, tmp_path):
"""Passing image_dir without REFERENCED mode raises ValueError."""
with pytest.raises(ValueError, match=r"ImageRefMode\.REFERENCED"):
sample_doc.export_to_markdown(image_dir=tmp_path / "images")
def test_export_to_html_image_dir_without_referenced_raises(sample_doc, tmp_path):
"""Passing image_dir without REFERENCED mode raises ValueError."""
with pytest.raises(ValueError, match=r"ImageRefMode\.REFERENCED"):
sample_doc.export_to_html(image_dir=tmp_path / "images")
def _make_captioned_doc(
*,
kind: str,
cap_bbox: BoundingBox | None,
obj_bbox: BoundingBox | None,
page_no: int = 1,
) -> DoclingDocument:
doc = DoclingDocument(name="cap_order")
doc.add_page(page_no=1, size=Size(width=100, height=200))
def prov(bbox: BoundingBox | None) -> ProvenanceItem | None:
if bbox is None:
return None
return ProvenanceItem(page_no=page_no, bbox=bbox, charspan=(0, 0))
caption = doc.add_text(label=DocItemLabel.CAPTION, text="THE CAPTION", prov=prov(cap_bbox))
if kind == "table":
table_data = TableData(num_rows=1, num_cols=1)
table_data.add_row(["cell"])
table_data.num_rows = 1
doc.add_table(data=table_data, caption=caption, prov=prov(obj_bbox))
else:
doc.add_picture(caption=caption, prov=prov(obj_bbox))
return doc
def _tl(t: float, b: float) -> BoundingBox:
return BoundingBox(l=0, r=10, t=t, b=b, coord_origin=CoordOrigin.TOPLEFT)
def _bl(t: float, b: float) -> BoundingBox:
return BoundingBox(l=0, r=10, t=t, b=b, coord_origin=CoordOrigin.BOTTOMLEFT)
def _serialize_md(doc: DoclingDocument, placement: CaptionPlacement) -> str:
return MarkdownDocSerializer(doc=doc, params=MarkdownParams(caption_placement=placement)).serialize().text
@pytest.mark.parametrize("kind, marker", [("table", "cell"), ("picture", "<!-- image -->")])
@pytest.mark.parametrize(
"cap_bbox, obj_bbox, layout_caption_first",
[
(None, None, True), # no bboxes: falls back to standard order (caption first)
(_tl(150, 170), None, True), # object without bbox: falls back to standard order
(_tl(10, 30), _tl(50, 100), True), # caption above
(_tl(150, 170), _tl(50, 100), False), # caption below
(_bl(190, 170), _bl(150, 100), True), # caption above, BOTTOMLEFT origin
(_bl(50, 30), _bl(150, 100), False), # caption below, BOTTOMLEFT origin
(_tl(150, 170), _bl(150, 100), False), # mixed origins: caption (y=160) below object (y=75)
(_tl(10, 30), _bl(150, 100), True), # mixed origins: caption (y=20) above object (y=75)
],
)
def test_markdown_caption_placement(kind, marker, cap_bbox, obj_bbox, layout_caption_first):
doc = _make_captioned_doc(kind=kind, cap_bbox=cap_bbox, obj_bbox=obj_bbox)
# standard ignores bboxes: tables and pictures always have the caption first
md = _serialize_md(doc, CaptionPlacement.STANDARD)
assert md.index("THE CAPTION") < md.index(marker)
assert md == doc.export_to_markdown()
md = _serialize_md(doc, CaptionPlacement.LAYOUT)
if layout_caption_first:
assert md.index("THE CAPTION") < md.index(marker)
else:
assert md.index("THE CAPTION") > md.index(marker)
def test_markdown_caption_placement_layout_other_page_falls_back_to_standard():
doc = DoclingDocument(name="cap_pages")
doc.add_page(page_no=1, size=Size(width=100, height=200))
doc.add_page(page_no=2, size=Size(width=100, height=200))
caption = doc.add_text(
label=DocItemLabel.CAPTION,
text="THE CAPTION",
prov=ProvenanceItem(page_no=2, bbox=_tl(150, 170), charspan=(0, 0)),
)
doc.add_picture(caption=caption, prov=ProvenanceItem(page_no=1, bbox=_tl(50, 100), charspan=(0, 0)))
md = _serialize_md(doc, CaptionPlacement.LAYOUT)
assert md.index("THE CAPTION") < md.index("<!-- image -->")
@pytest.mark.parametrize(
"cap_bbox, placement, caption_first",
[
(None, CaptionPlacement.STANDARD, False), # standard: code captions come after
(_tl(10, 30), CaptionPlacement.STANDARD, False), # standard ignores bboxes
(None, CaptionPlacement.LAYOUT, False), # no bboxes: falls back to standard
(_tl(10, 30), CaptionPlacement.LAYOUT, True), # caption above the code
(_tl(150, 170), CaptionPlacement.LAYOUT, False), # caption below the code
],
)
def test_markdown_caption_placement_code(cap_bbox, placement, caption_first):
doc = DoclingDocument(name="cap_code")
doc.add_page(page_no=1, size=Size(width=100, height=200))
def prov(bbox):
return None if bbox is None else ProvenanceItem(page_no=1, bbox=bbox, charspan=(0, 0))
caption = doc.add_text(label=DocItemLabel.CAPTION, text="THE CAPTION", prov=prov(cap_bbox))
code = doc.add_code(text="print(1)", prov=prov(_tl(50, 100) if cap_bbox else None))
code.captions.append(caption.get_ref())
md = _serialize_md(doc, placement)
assert (md.index("THE CAPTION") < md.index("print(1)")) == caption_first
@pytest.mark.parametrize("placement", [CaptionPlacement.LAYOUT, "layout"])
def test_export_and_save_markdown_caption_placement(tmp_path, placement):
doc = _make_captioned_doc(kind="picture", cap_bbox=_tl(150, 170), obj_bbox=_tl(50, 100))
md = doc.export_to_markdown(caption_placement=placement)
assert md.index("THE CAPTION") > md.index("<!-- image -->")
out = tmp_path / "out.md"
doc.save_as_markdown(out, caption_placement=placement)
assert out.read_text(encoding="utf-8") == md
# default is unchanged
assert doc.export_to_markdown().index("THE CAPTION") < doc.export_to_markdown().index("<!-- image -->")
assert doc.export_to_markdown() == doc.export_to_markdown(caption_placement="standard")
def test_md_column_spanning_rich_cell_keeps_text_in_every_column(col_span_rich_table_doc):
params = MarkdownParams(compact_tables=True)
actual = MarkdownDocSerializer(doc=col_span_rich_table_doc, params=params).serialize().text
assert actual == "| | Partner | Partner |\n| - | - | - |\n| Status | Done | Open |"
# same table with the spanning header as a plain TableCell
plain_doc = col_span_rich_table_doc.model_copy(deep=True)
plain_doc.tables[0].data.table_cells = [
TableCell(**cell.model_dump(exclude={"ref", "text"}), text="Partner")
if isinstance(cell, RichTableCell)
else cell
for cell in plain_doc.tables[0].data.table_cells
]
plain_doc.delete_items(node_items=[ref.resolve(plain_doc) for ref in plain_doc.tables[0].children])
assert actual == MarkdownDocSerializer(doc=plain_doc, params=params).serialize().text
def test_triplet_column_spanning_rich_cell_keeps_text_in_every_column(col_span_rich_table_doc):
ser = MarkdownDocSerializer(doc=col_span_rich_table_doc, table_serializer=TripletTableSerializer())
actual = ser.serialize().text
assert actual == "Status, Partner = Done. Status, Partner = Open"
@pytest.mark.parametrize(
("code", "expected"),
[
("print(1)", "```\nprint(1)\n```"),
("x = '```'\n```\nprint(1)", "````\nx = '```'\n```\nprint(1)\n````"),
("a\n`````\nb", "``````\na\n`````\nb\n``````"),
],
)
def test_markdown_code_block_fence_outlasts_backticks_in_code(code: str, expected: str):
doc = DoclingDocument(name="code")
doc.add_code(text=code)
assert doc.export_to_markdown() == expected
@pytest.mark.parametrize(
("code", "expected"),
[
("x", "`x`"),
("a`b", "``a`b``"),
("`tick", "`` `tick ``"),
],
)
def test_markdown_inline_code_span_outlasts_backticks_in_code(code: str, expected: str):
doc = DoclingDocument(name="code")
group = doc.add_inline_group()
doc.add_text(label=DocItemLabel.TEXT, text="see ", parent=group)
doc.add_code(text=code, parent=group)
assert doc.export_to_markdown() == f"see {expected}"
@pytest.mark.parametrize(
("code", "expected"),
[
(" x ", "` x `"),
(" x ", "` x `"),
(" ", "` `"),
(" ", "` `"),
(" x", "` x`"),
("x ", "`x `"),
(" \t ", "` \t `"),
(" \u00a0 ", "` \u00a0 `"),
("\tx\t", "`\tx\t`"),
("\u00a0x\u00a0", "`\u00a0x\u00a0`"),
(" `x` ", "`` `x` ``"),
("\nx ", "` \nx `"),
(" x\n", "` x\n `"),
("\nx\n", "` \nx\n `"),
("\r\nx\r\n", "` \r\nx\r\n `"),
("\rx\r", "` \rx\r `"),
(" x\ny ", "` x\ny `"),
(" \n ", "` \n `"),
(" \r\n ", "` \r\n `"),
],
)
def test_markdown_inline_code_span_preserves_spaces(code: str, expected: str):
doc = DoclingDocument(name="code")
group = doc.add_inline_group()
doc.add_text(label=DocItemLabel.TEXT, text="see ", parent=group)
doc.add_code(text=code, parent=group)
assert doc.export_to_markdown() == f"see {expected}"
def test_md_html_table_footnotes():
"""Real conversion of the PDF from #496: a table with two footnotes."""
src = Path("./tests/data/doc/table_footnotes_496.json")
doc = DoclingDocument.load_from_json(src)
actual = MarkdownDocSerializer(doc=doc).serialize().text
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
actual = HTMLDocSerializer(doc=doc).serialize().text
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
def test_table_footnotes_serialized_to_markdown_and_html():
"""Table footnotes must not be dropped by the Markdown and HTML serializers."""
doc = DoclingDocument(name="t")
td = TableData(num_rows=0, num_cols=2)
td.add_row(["Header 1", "Header 2"])
td.add_row(["Data 1", "Data 2"])
table = doc.add_table(data=td)
for text in ["First footnote.", "Second footnote.", "a < b & c > d"]:
table.footnotes.append(doc.add_text(label=DocItemLabel.FOOTNOTE, text=text).get_ref())
# one paragraph per footnote after the table
md_out = doc.export_to_markdown()
assert "|\n\nFirst footnote.\n\nSecond footnote.\n" in md_out
# footnotes are HTML-escaped divs after the table, with nothing between them
html_out = doc.export_to_html()
assert (
'</table><div class="footnote">First footnote.</div>'
'<div class="footnote">Second footnote.</div>'
'<div class="footnote">a &lt; b &amp; c &gt; d</div>'
) in html_out
assert "a < b & c > d" not in html_out
def test_footnotes_kept_when_owner_label_is_excluded():
"""Footnotes are serialized like captions when only the owner's label is excluded."""
doc = DoclingDocument(name="t")
td = TableData(num_rows=0, num_cols=1)
td.add_row(["Cell"])
table = doc.add_table(data=td)
table.footnotes.append(doc.add_text(label=DocItemLabel.FOOTNOTE, text="Table note.").get_ref())
pic = doc.add_picture()
pic.footnotes.append(doc.add_text(label=DocItemLabel.FOOTNOTE, text="Picture note.").get_ref())
labels = {DocItemLabel.TEXT, DocItemLabel.FOOTNOTE}
assert doc.export_to_markdown(labels=labels) == "Table note.\n\nPicture note."
html_out = doc.export_to_html(labels=labels)
assert "Cell" not in html_out
assert '<div class="footnote">Table note.</div>' in html_out
assert '<div class="footnote">Picture note.</div>' in html_out
def test_picture_footnotes_serialized_to_markdown_and_html():
"""Picture footnotes are skipped by the doc serializer like table ones."""
doc = DoclingDocument(name="t")
pic = doc.add_picture()
pic.footnotes.append(doc.add_text(label=DocItemLabel.FOOTNOTE, text="Picture note.").get_ref())
assert "Picture note." in doc.export_to_markdown()
# inside <figure>, so the footnote stays with its picture
assert '<div class="footnote">Picture note.</div></figure>' in doc.export_to_html()
def test_code_footnotes_serialized_to_markdown_and_html():
"""Code footnotes are skipped by the doc serializer like table ones."""
doc = DoclingDocument(name="t")
code = doc.add_code(text="print(1)")
code.footnotes.append(doc.add_text(label=DocItemLabel.FOOTNOTE, text="Code note.").get_ref())
assert "Code note." in doc.export_to_markdown()
# wrapped in <figure> like a picture, so the footnote stays with its code
assert (
'<figure><pre><code>print(1)</code></pre><div class="footnote">Code note.</div></figure>'
in doc.export_to_html()
)