mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* chore: update pre-commit hooks and development dependencies Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * docs: update CONTRIBUTING after pre-commit hook changes Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
1106 lines
36 KiB
Python
1106 lines
36 KiB
Python
"""Test serialization."""
|
|
|
|
import threading
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
from xml.etree import ElementTree as ET
|
|
|
|
import pytest
|
|
|
|
from docling_core.transforms.serializer.common import _DEFAULT_LABELS
|
|
from docling_core.transforms.serializer.html import (
|
|
HTMLDocSerializer,
|
|
HTMLMetaSerializer,
|
|
HTMLOutputStyle,
|
|
HTMLParams,
|
|
HTMLTableSerializer,
|
|
)
|
|
from docling_core.transforms.serializer.markdown import (
|
|
MarkdownDocSerializer,
|
|
MarkdownParams,
|
|
MarkdownTableSerializer,
|
|
OrigListItemMarkerMode,
|
|
_cell_content_has_table,
|
|
)
|
|
from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams
|
|
from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer
|
|
from docling_core.types.doc import DoclingDocument
|
|
from docling_core.types.doc.base import ImageRefMode
|
|
from docling_core.types.doc.document import (
|
|
BaseMeta,
|
|
CharSpan,
|
|
DescriptionAnnotation,
|
|
EntitiesMetaField,
|
|
EntityMention,
|
|
LanguageMetaField,
|
|
PictureClassificationMetaField,
|
|
PictureClassificationPrediction,
|
|
PictureMeta,
|
|
RefItem,
|
|
RichTableCell,
|
|
SummaryMetaField,
|
|
TableCell,
|
|
TableData,
|
|
TextItem,
|
|
)
|
|
from docling_core.types.doc.labels import DocItemLabel
|
|
|
|
from .test_data_gen_flag import GEN_TEST_DATA
|
|
|
|
|
|
def verify(exp_file: Path, actual: str):
|
|
if GEN_TEST_DATA:
|
|
with open(exp_file, "w", encoding="utf-8") as f:
|
|
f.write(f"{actual}\n")
|
|
else:
|
|
with open(exp_file, encoding="utf-8") as f:
|
|
expected = f.read().rstrip()
|
|
|
|
# Normalize platform-dependent quote escaping for DocTags outputs
|
|
name = exp_file.name
|
|
if name.endswith((".dt", ".idt", ".idt.xml")):
|
|
|
|
def _normalize_quotes(s: str) -> str:
|
|
return s.replace(""", '"').replace(""", '"')
|
|
|
|
expected = _normalize_quotes(expected)
|
|
actual = _normalize_quotes(actual)
|
|
|
|
assert actual == expected
|
|
|
|
|
|
# ===============================
|
|
# Markdown tests
|
|
# ===============================
|
|
|
|
|
|
def test_md_cross_page_list_page_break():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder="<!-- page break -->",
|
|
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
|
|
|
|
|
|
def test_md_checkboxes():
|
|
src = Path("./test/data/doc/checkboxes.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder="<!-- page break -->",
|
|
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}.gt.md", actual=actual)
|
|
|
|
|
|
def test_md_cross_page_list_page_break_none():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder=None,
|
|
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_pb_none.gt.md", actual=actual)
|
|
|
|
|
|
def test_md_cross_page_list_page_break_empty():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder="",
|
|
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_pb_empty.gt.md", actual=actual)
|
|
|
|
|
|
def test_md_cross_page_list_page_break_non_empty():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder="<!-- page-break -->",
|
|
labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_pb_non_empty.gt.md", actual=actual)
|
|
|
|
|
|
def test_md_cross_page_list_page_break_p2():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
page_break_placeholder=None,
|
|
pages={2},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_p2.gt.md", actual=actual)
|
|
|
|
|
|
def test_md_charts():
|
|
src = Path("./test/data/doc/barchart.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
|
|
|
|
|
|
def test_md_inline_and_formatting():
|
|
src = Path("./test/data/doc/inline_and_formatting.yaml")
|
|
doc = DoclingDocument.load_from_yaml(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
|
|
|
|
|
|
def test_md_pb_placeholder_and_page_filter():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
# NOTE ambiguous case
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
page_break_placeholder="<!-- page break -->",
|
|
pages={3, 4, 6},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
|
|
|
|
|
|
def test_md_list_item_markers(sample_doc):
|
|
root_dir = Path("./test/data/doc")
|
|
for mode in OrigListItemMarkerMode:
|
|
for valid in [False, True]:
|
|
ser = MarkdownDocSerializer(
|
|
doc=sample_doc,
|
|
params=MarkdownParams(
|
|
orig_list_item_marker_mode=mode,
|
|
ensure_valid_list_item_marker=valid,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
root_dir / f"constructed_mode_{str(mode.value).lower()}_valid_{str(valid).lower()}.gt.md",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_md_mark_meta_true():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
mark_meta=True,
|
|
pages={1, 5},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=src.parent / f"{src.stem}_p1_mark_meta_true.gt.md",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_md_mark_meta_false():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
mark_meta=False,
|
|
pages={1, 5},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=src.parent / f"{src.stem}_p1_mark_meta_false.gt.md",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_md_legacy_annotations_mark_true(sample_doc):
|
|
exp_file = Path("./test/data/doc/constructed_legacy_annot_mark_true.gt.md")
|
|
with pytest.warns(DeprecationWarning):
|
|
sample_doc.tables[0].annotations.append(
|
|
DescriptionAnnotation(text="This is a description of table 1.", provenance="foo")
|
|
)
|
|
ser = MarkdownDocSerializer(
|
|
doc=sample_doc,
|
|
params=MarkdownParams(
|
|
mark_annotations=True,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=exp_file,
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_md_legacy_annotations_mark_false(sample_doc):
|
|
exp_file = Path("./test/data/doc/constructed_legacy_annot_mark_false.gt.md")
|
|
with pytest.warns(DeprecationWarning):
|
|
sample_doc.tables[0].annotations.append(
|
|
DescriptionAnnotation(text="This is a description of table 1.", provenance="foo")
|
|
)
|
|
ser = MarkdownDocSerializer(
|
|
doc=sample_doc,
|
|
params=MarkdownParams(
|
|
mark_annotations=False,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=exp_file,
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_md_nested_lists():
|
|
src = Path("./test/data/doc/polymers.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = MarkdownDocSerializer(doc=doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.md"), actual=actual)
|
|
|
|
|
|
def test_md_rich_table(rich_table_doc):
|
|
exp_file = Path("./test/data/doc/rich_table.gt.md")
|
|
|
|
ser = MarkdownDocSerializer(doc=rich_table_doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=exp_file, actual=actual)
|
|
|
|
|
|
def test_md_single_row_table():
|
|
exp_file = Path("./test/data/doc/single_row_table.gt.md")
|
|
words = ["foo", "bar"]
|
|
doc = DoclingDocument(name="")
|
|
row_idx = 0
|
|
table = doc.add_table(data=TableData(num_rows=1, num_cols=len(words)))
|
|
for col_idx, word in enumerate(words):
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=row_idx,
|
|
end_row_offset_idx=row_idx + 1,
|
|
start_col_offset_idx=col_idx,
|
|
end_col_offset_idx=col_idx + 1,
|
|
text=word,
|
|
),
|
|
)
|
|
|
|
ser = MarkdownDocSerializer(doc=doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=exp_file, actual=actual)
|
|
|
|
|
|
def test_md_pipe_in_table():
|
|
doc = DoclingDocument(name="Pipe in Table")
|
|
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
|
|
# TODO: properly handle nested tables, for now just escape the pipe
|
|
doc.add_table_cell(
|
|
table,
|
|
TableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
text="Fruits | Veggies",
|
|
),
|
|
)
|
|
ser = doc.export_to_markdown()
|
|
assert ser == "| Fruits | Veggies |\n|-------------------------|"
|
|
|
|
|
|
def test_cell_content_has_table_detects_descendant_table():
|
|
"""Ensure nested tables are detected through non-table parent nodes."""
|
|
doc = DoclingDocument(name="descendant_table")
|
|
wrapper = doc.add_group()
|
|
nested_table = doc.add_table(data=TableData(num_rows=1, num_cols=1), parent=wrapper)
|
|
doc.add_table_cell(
|
|
nested_table,
|
|
TableCell(
|
|
text="inner",
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
),
|
|
)
|
|
|
|
assert _cell_content_has_table(wrapper, doc)
|
|
|
|
|
|
def _build_nested_rich_table_doc(depth: int) -> DoclingDocument:
|
|
"""Build a document with `depth` levels of nested RichTableCell tables.
|
|
|
|
Each level is a 1x2 table whose first cell is a RichTableCell referencing
|
|
the next-level table, and whose second cell is a plain TableCell.
|
|
This is the structure produced by the HTML backend for Wikipedia clade tables.
|
|
"""
|
|
doc = DoclingDocument(name="nested_tables")
|
|
|
|
def _add_level(parent, remaining: int):
|
|
table = doc.add_table(data=TableData(num_rows=1, num_cols=2), parent=parent)
|
|
if remaining > 0:
|
|
nested = _add_level(table, remaining - 1)
|
|
rich_cell: TableCell = RichTableCell(
|
|
ref=nested.get_ref(),
|
|
text="rich",
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
)
|
|
else:
|
|
rich_cell = TableCell(
|
|
text="leaf",
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
)
|
|
doc.add_table_cell(table, rich_cell)
|
|
doc.add_table_cell(
|
|
table,
|
|
TableCell(
|
|
text="plain",
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
),
|
|
)
|
|
return table
|
|
|
|
_add_level(doc.body, depth)
|
|
return doc
|
|
|
|
|
|
def test_md_nested_rich_table_no_hang():
|
|
"""Regression: export_to_markdown() must not hang on nested RichTableCells.
|
|
|
|
When a RichTableCell's content contains a nested table, the
|
|
``_nested_in_table`` flag passed through kwargs causes
|
|
MarkdownTableSerializer to flatten the inner table instead of
|
|
re-entering the full table serializer recursively. Without this
|
|
guard every level of nesting re-enters the table serializer, causing
|
|
exponential string growth and an indefinite hang.
|
|
"""
|
|
doc = _build_nested_rich_table_doc(depth=5)
|
|
|
|
result: list[str] = []
|
|
|
|
def _run() -> None:
|
|
result.append(doc.export_to_markdown())
|
|
|
|
t = threading.Thread(target=_run, daemon=True)
|
|
t.start()
|
|
t.join(timeout=5.0)
|
|
|
|
assert not t.is_alive(), "export_to_markdown() hung on a document with nested RichTableCells."
|
|
assert result, "export_to_markdown() produced no output"
|
|
|
|
# The outer table must be a valid 2-column markdown table.
|
|
# Without the pipe-escaping fix, inner-table pipes would leak into the outer
|
|
# table and produce dozens of phantom columns.
|
|
table_rows = [line for line in result[0].splitlines() if line.startswith("|")]
|
|
assert table_rows, "Expected at least one markdown table row in output"
|
|
col_counts = {line.count("|") - 1 for line in table_rows}
|
|
assert col_counts == {2}, f"Outer table must have exactly 2 columns throughout; got column counts: {col_counts}"
|
|
|
|
|
|
def test_md_compact_table():
|
|
"""Test compact table format removes padding and uses minimal separators."""
|
|
# Test the _compact_table method directly
|
|
padded_table = """| item | qty | description |
|
|
| ------ | ----: | :-------------------: |
|
|
| spam | 42 | A canned meat product |
|
|
| eggs | 451 | Fresh farm eggs |
|
|
| bacon | 0 | Out of stock |"""
|
|
|
|
expected_compact = """| item | qty | description |
|
|
| - | -: | :-: |
|
|
| spam | 42 | A canned meat product |
|
|
| eggs | 451 | Fresh farm eggs |
|
|
| bacon | 0 | Out of stock |"""
|
|
|
|
compact_result = MarkdownTableSerializer._compact_table(padded_table)
|
|
assert compact_result == expected_compact
|
|
|
|
# Verify space savings
|
|
assert len(compact_result) < len(padded_table)
|
|
|
|
|
|
def test_md_numeric_precision_preserved():
|
|
"""Test that numeric values in tables preserve their full precision.
|
|
|
|
Regression test for issue where tabulate's numparse would silently
|
|
truncate numeric strings to ~6 significant figures.
|
|
"""
|
|
doc = DoclingDocument(name="Numeric Precision Test")
|
|
precise_values = [
|
|
"225.8183",
|
|
"24797.34",
|
|
"20896.7184",
|
|
"17358.138",
|
|
"123.456789",
|
|
]
|
|
table = doc.add_table(data=TableData(num_rows=len(precise_values) + 1, num_cols=2))
|
|
|
|
# Add header row
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
text="Description",
|
|
),
|
|
)
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
text="Value",
|
|
),
|
|
)
|
|
|
|
# Add data rows with precise numeric values
|
|
for row_idx, value in enumerate(precise_values, start=1):
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=row_idx,
|
|
end_row_offset_idx=row_idx + 1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
text=f"Item {row_idx}",
|
|
),
|
|
)
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=row_idx,
|
|
end_row_offset_idx=row_idx + 1,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
text=value,
|
|
),
|
|
)
|
|
|
|
markdown_output = doc.export_to_markdown()
|
|
for value in precise_values:
|
|
assert value in markdown_output, (
|
|
f"Numeric value '{value}' was not preserved in markdown output. "
|
|
"This indicates precision loss during table serialization."
|
|
)
|
|
|
|
|
|
def test_md_traverse_pictures():
|
|
"""Test traverse_pictures parameter to include text inside PictureItems."""
|
|
doc = DoclingDocument(name="Test Document")
|
|
doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text before picture")
|
|
picture = doc.add_picture()
|
|
|
|
# Manually add a text item as child of picture
|
|
text_in_picture = TextItem(
|
|
self_ref=f"#/texts/{len(doc.texts)}",
|
|
parent=RefItem(cref=picture.self_ref),
|
|
label=DocItemLabel.PARAGRAPH,
|
|
text="Text inside picture",
|
|
orig="Text inside picture",
|
|
)
|
|
doc.texts.append(text_in_picture)
|
|
picture.children.append(RefItem(cref=text_in_picture.self_ref))
|
|
doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text after picture")
|
|
|
|
# Test with traverse_pictures=False (default)
|
|
ser_no_traverse = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
traverse_pictures=False,
|
|
),
|
|
)
|
|
result_no_traverse = ser_no_traverse.serialize().text
|
|
|
|
# Should NOT contain text inside picture
|
|
assert "Text before picture" in result_no_traverse
|
|
assert "Text after picture" in result_no_traverse
|
|
assert "Text inside picture" not in result_no_traverse
|
|
assert "<!-- image -->" in result_no_traverse
|
|
|
|
# Test with traverse_pictures=True
|
|
ser_with_traverse = MarkdownDocSerializer(
|
|
doc=doc,
|
|
params=MarkdownParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
image_placeholder="<!-- image -->",
|
|
traverse_pictures=True,
|
|
),
|
|
)
|
|
result_with_traverse = ser_with_traverse.serialize().text
|
|
|
|
# Should contain text inside picture
|
|
assert "Text before picture" in result_with_traverse
|
|
assert "Text after picture" in result_with_traverse
|
|
assert "Text inside picture" in result_with_traverse
|
|
assert "<!-- image -->" in result_with_traverse
|
|
|
|
|
|
# ===============================
|
|
# HTML tests
|
|
# ===============================
|
|
|
|
|
|
def test_html_table_serializer_get_header_and_body_lines():
|
|
"""Test HTMLTableSerializer.get_header_and_body_lines() method."""
|
|
serializer = HTMLTableSerializer()
|
|
|
|
# Test 1: Valid HTML with headers and data
|
|
valid_html = "<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Data1</td><td>Data2</td></tr></table>"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=valid_html)
|
|
assert len(headers) > 0, "Should have headers"
|
|
assert len(body) > 0, "Should have body rows"
|
|
|
|
# Test 2: Row without closing </tr> tag
|
|
# Parser will find the row, but when we search for </tr> it won't be found
|
|
no_close_tr = "<tr><th>Header</th></tr><tr><td>Data1"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=no_close_tr)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 3: Data rows with incomplete closing tags
|
|
# When collecting remaining rows, some </tr> tags are missing
|
|
incomplete_data = "<tr><th>H1</th></tr><tr><td>D1</td></tr><tr><td>D2"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=incomplete_data)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 4: Force exception in parser
|
|
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
|
|
mock_parser = MagicMock()
|
|
mock_parser.feed.side_effect = Exception("Parser error")
|
|
mock_parser_class.return_value = mock_parser
|
|
|
|
broken_html = "<tr><th>Header</th></tr><tr><td>Data</td></tr>"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=broken_html)
|
|
# Should use fallback logic
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 5: Parser returns more rows than exist in HTML
|
|
# Mock parser to return extra rows that don't exist in the HTML
|
|
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
|
|
mock_parser = MagicMock()
|
|
# Create fake row data - more rows than actually exist in HTML
|
|
mock_parser.rows = [
|
|
{"th_cells": ["H1"], "td_cells": []},
|
|
{"th_cells": ["H2"], "td_cells": []},
|
|
{"th_cells": ["H3"], "td_cells": []}, # This row doesn't exist in HTML
|
|
{"th_cells": [], "td_cells": ["D1"]},
|
|
]
|
|
mock_parser_class.return_value = mock_parser
|
|
|
|
# HTML with only 2 rows, but parser claims 4
|
|
limited_html = "<tr><th>H1</th></tr><tr><th>H2</th></tr>"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=limited_html)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 6: Specific case for line 485 - row_start found but row_end not found
|
|
# Create HTML where parser finds a row, but the actual HTML has <tr without </tr>
|
|
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
|
|
mock_parser = MagicMock()
|
|
# Parser reports a header row exists
|
|
mock_parser.rows = [
|
|
{"th_cells": ["Header"], "td_cells": []},
|
|
]
|
|
mock_parser_class.return_value = mock_parser
|
|
|
|
# But the HTML has <tr without matching </tr>
|
|
html_no_close = "<tr><th>Header"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=html_no_close)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 7: Specific case for line 504 - data collection finds <tr but no </tr>
|
|
# Create HTML where we start collecting data rows but encounter incomplete row
|
|
with patch("docling_core.transforms.serializer.html._SimpleHTMLTableParser") as mock_parser_class:
|
|
mock_parser = MagicMock()
|
|
# Parser reports header then data rows
|
|
mock_parser.rows = [
|
|
{"th_cells": ["H"], "td_cells": []},
|
|
{"th_cells": [], "td_cells": ["D1"]}, # This triggers data collection
|
|
]
|
|
mock_parser_class.return_value = mock_parser
|
|
|
|
# HTML has complete header but incomplete data row
|
|
html_incomplete_data = "<tr><th>H</th></tr><tr><td>D1</td></tr><tr><td>D2"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=html_incomplete_data)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
|
|
# Test 8: Table with footer content
|
|
with_footer = "<tr><th>H</th></tr><tr><td>D</td></tr>Footer content"
|
|
headers, body = serializer.get_header_and_body_lines(table_text=with_footer)
|
|
assert isinstance(headers, list)
|
|
assert isinstance(body, list)
|
|
# Footer should be in body
|
|
assert "Footer" in str(body)
|
|
|
|
|
|
def test_html_charts():
|
|
src = Path("./test/data/doc/barchart.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
|
|
|
|
|
|
def test_html_cross_page_list_page_break():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
|
|
|
|
|
|
def test_html_cross_page_list_page_break_p1():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
pages={1},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_p1.gt.html", actual=actual)
|
|
|
|
|
|
def test_html_cross_page_list_page_break_p2():
|
|
src = Path("./test/data/doc/activities.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
pages={2},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_p2.gt.html", actual=actual)
|
|
|
|
|
|
def test_html_split_page():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
output_style=HTMLOutputStyle.SPLIT_PAGE,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_split.gt.html", actual=actual)
|
|
|
|
|
|
def test_html_split_page_p2():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
output_style=HTMLOutputStyle.SPLIT_PAGE,
|
|
pages={2},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_split_p2.gt.html", actual=actual)
|
|
|
|
|
|
def test_html_split_page_p2_with_visualizer():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
output_style=HTMLOutputStyle.SPLIT_PAGE,
|
|
pages={2},
|
|
),
|
|
)
|
|
ser_res = ser.serialize(
|
|
visualizer=LayoutVisualizer(),
|
|
)
|
|
actual = ser_res.text
|
|
|
|
# pinning the result with visualizer appeared flaky, so at least ensure it contains
|
|
# a figure (for the page) and that it is different than without visualizer:
|
|
assert '<figure><img src="data:image/png;base64' in actual
|
|
file_without_viz = src.parent / f"{src.stem}_split_p2.gt.html"
|
|
with open(file_without_viz) as f:
|
|
data_without_viz = f.read()
|
|
assert actual.strip() != data_without_viz.strip()
|
|
|
|
|
|
def test_html_split_page_no_page_breaks():
|
|
src = Path("./test/data/doc/2408.09869_p1.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
output_style=HTMLOutputStyle.SPLIT_PAGE,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.parent / f"{src.stem}_split.gt.html", actual=actual)
|
|
|
|
|
|
def test_html_include_annotations_false():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
include_annotations=False,
|
|
pages={1},
|
|
html_head="<head></head>", # keeping test output minimal
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=src.parent / f"{src.stem}_p1_include_annotations_false.gt.html",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_html_include_annotations_true():
|
|
src = Path("./test/data/doc/2408.09869v3_enriched.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(
|
|
doc=doc,
|
|
params=HTMLParams(
|
|
image_mode=ImageRefMode.PLACEHOLDER,
|
|
include_annotations=True,
|
|
pages={1},
|
|
html_head="<head></head>", # keeping test output minimal
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
exp_file=src.parent / f"{src.stem}_p1_include_annotations_true.gt.html",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_html_list_item_markers(sample_doc):
|
|
root_dir = Path("./test/data/doc")
|
|
for orig in [False, True]:
|
|
ser = HTMLDocSerializer(
|
|
doc=sample_doc,
|
|
params=HTMLParams(
|
|
show_original_list_item_marker=orig,
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify(
|
|
root_dir / f"constructed_orig_{str(orig).lower()}.gt.html",
|
|
actual=actual,
|
|
)
|
|
|
|
|
|
def test_html_nested_lists():
|
|
src = Path("./test/data/doc/polymers.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = HTMLDocSerializer(doc=doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
|
|
|
|
|
|
def test_html_rich_table(rich_table_doc):
|
|
exp_file = Path("./test/data/doc/rich_table.gt.html")
|
|
|
|
ser = HTMLDocSerializer(doc=rich_table_doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=exp_file, actual=actual)
|
|
|
|
|
|
def test_html_rich_cell_textitem_ref_subtree_inside_and_not_outside():
|
|
"""Descendants of a RichTableCell's TextItem ref render inside the table.
|
|
|
|
With HTMLTextSerializer recursing into its item's children, the parent text
|
|
(CELL-TEXT) and its child (DEEP-LEAK) both render as siblings inside the
|
|
rich cell. The outer document iteration must not re-emit either of them as
|
|
standalone content after the table.
|
|
"""
|
|
doc = DoclingDocument(name="rich_cell_textitem_subtree")
|
|
|
|
table = doc.add_table(data=TableData(num_rows=1, num_cols=2))
|
|
rich_ref = doc.add_text(label=DocItemLabel.TEXT, text="CELL-TEXT", parent=table)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="DEEP-LEAK", parent=rich_ref)
|
|
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=RichTableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
ref=rich_ref.get_ref(),
|
|
text="cell 0,0",
|
|
),
|
|
)
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
text="plain",
|
|
),
|
|
)
|
|
|
|
out = HTMLDocSerializer(doc=doc).serialize().text
|
|
body = out[out.find("<body>") : out.find("</body>") + len("</body>")]
|
|
table_end = body.find("</table>") + len("</table>")
|
|
inside_table = body[:table_end]
|
|
after_table = body[table_end:]
|
|
|
|
assert "CELL-TEXT" in inside_table, f"RichTableCell ref content missing from the table:\n{body}"
|
|
assert "DEEP-LEAK" in inside_table, f"RichTableCell ref descendant missing from the table:\n{body}"
|
|
assert "CELL-TEXT" not in after_table, f"RichTableCell ref content leaked outside the table:\n{body}"
|
|
assert "DEEP-LEAK" not in after_table, f"RichTableCell ref descendant leaked outside the table:\n{body}"
|
|
|
|
|
|
def test_html_textitem_with_children_at_document_level():
|
|
"""Doc-level parity: TextItem with TextItem child renders both exactly once.
|
|
|
|
Once HTMLTextSerializer recurses into children, the child is rendered by
|
|
the parent's serialize call instead of by the outer iteration. Both must
|
|
still appear in document order, exactly once each.
|
|
"""
|
|
doc = DoclingDocument(name="textitem_with_children")
|
|
parent = doc.add_text(label=DocItemLabel.TEXT, text="PARENT-TEXT")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="CHILD-TEXT", parent=parent)
|
|
|
|
out = HTMLDocSerializer(doc=doc).serialize().text
|
|
body = out[out.find("<body>") : out.find("</body>") + len("</body>")]
|
|
|
|
assert body.count("PARENT-TEXT") == 1, f"PARENT-TEXT should appear exactly once:\n{body}"
|
|
assert body.count("CHILD-TEXT") == 1, f"CHILD-TEXT should appear exactly once:\n{body}"
|
|
assert body.find("PARENT-TEXT") < body.find("CHILD-TEXT"), f"PARENT-TEXT should appear before CHILD-TEXT:\n{body}"
|
|
|
|
|
|
def test_html_inline_and_formatting():
|
|
src = Path("./test/data/doc/inline_and_formatting.yaml")
|
|
doc = DoclingDocument.load_from_yaml(src)
|
|
|
|
ser = HTMLDocSerializer(doc=doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.html"), actual=actual)
|
|
|
|
|
|
# ===============================
|
|
# WebVTT tests
|
|
# ===============================
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"example_num",
|
|
[1, 2, 3, 4, 5],
|
|
)
|
|
def test_webvtt(example_num):
|
|
src = Path(f"test/data/doc/webvtt_example_{example_num:02d}.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
ser = WebVTTDocSerializer(doc=doc)
|
|
actual = ser.serialize().text
|
|
verify(exp_file=src.with_suffix(".gt.vtt"), actual=actual)
|
|
|
|
|
|
def test_webvtt_params():
|
|
"""Test WebVTT serialization with WebVTTParams."""
|
|
src = Path("./test/data/doc/webvtt_example_01.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
# Test with omit_hours_if_zero=True
|
|
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_hours_if_zero=True))
|
|
actual = ser.serialize().text
|
|
assert "00:11.000 --> 00:13.000" in actual
|
|
|
|
# Test with omit_voice_end=True
|
|
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_voice_end=True))
|
|
actual = ser.serialize().text
|
|
assert "</v>" not in actual
|
|
|
|
# Test with both parameters enabled
|
|
ser = WebVTTDocSerializer(doc=doc, params=WebVTTParams(omit_hours_if_zero=True, omit_voice_end=True))
|
|
actual = ser.serialize().text
|
|
|
|
assert "00:11.000 --> 00:13.000" in actual
|
|
assert "</v>" not in actual
|
|
|
|
ser_default = WebVTTDocSerializer(doc=doc, params=WebVTTParams())
|
|
actual_default = ser_default.serialize().text
|
|
assert len(actual) <= len(actual_default) or actual != actual_default
|
|
|
|
|
|
def test_html_meta_emits_xhtml_compatible_attributes():
|
|
"""Test that metadata attributes are name=value pairs."""
|
|
doc = DoclingDocument(name="x")
|
|
|
|
plain = TableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
text="",
|
|
)
|
|
table = doc.add_table(data=TableData(num_rows=1, num_cols=1, table_cells=[plain]))
|
|
|
|
pic = doc.add_picture(parent=table)
|
|
pic.meta = PictureMeta(
|
|
classification=PictureClassificationMetaField(
|
|
predictions=[PictureClassificationPrediction(class_name="other", confidence=1.0)]
|
|
)
|
|
)
|
|
|
|
table.data.table_cells = [
|
|
RichTableCell(
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
text="",
|
|
ref=pic.get_ref(),
|
|
)
|
|
]
|
|
|
|
text = doc.add_text(
|
|
label=DocItemLabel.TEXT, text="Output of HTML serializer must be parseable by a strict XML parser"
|
|
)
|
|
text.meta = BaseMeta(
|
|
summary=SummaryMetaField(text="XHTML-compliant"),
|
|
language=LanguageMetaField(code="en"),
|
|
entities=EntitiesMetaField(
|
|
mentions=[
|
|
EntityMention(
|
|
text="HTML serializer",
|
|
label="software",
|
|
charspan=CharSpan((10, 25)),
|
|
),
|
|
EntityMention(
|
|
text="XML parser",
|
|
label="software",
|
|
charspan=CharSpan((56, 66)),
|
|
),
|
|
]
|
|
),
|
|
)
|
|
|
|
html_out = table.export_to_html(doc)
|
|
# No bare valueless attribute like `data-meta-classification` (must be
|
|
# followed by `=` to be XHTML-compliant).
|
|
assert "data-meta-classification>" not in html_out
|
|
assert 'data-meta-name="classification"' in html_out
|
|
|
|
html_out = doc.export_to_html()
|
|
print(html_out)
|
|
assert 'data-meta-name="language"' in html_out
|
|
assert 'data-meta-name="entities"' in html_out
|
|
# Output must be parseable by a strict XML parser.
|
|
ET.fromstring(html_out)
|