import re
from collections.abc import Callable
from pathlib import Path
import pytest
from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer
from docling_core.transforms.serializer.doclang import (
DocLangDocSerializer,
DocLangParams,
EscapeMode,
LabelMode,
)
from docling_core.types.doc import (
BoundingBox,
CoordOrigin,
DescriptionMetaField,
DocItemLabel,
DoclingDocument,
Formatting,
ImageRefMode,
MoleculeMetaField,
PictureClassificationMetaField,
PictureClassificationPrediction,
PictureItem,
PictureMeta,
ProvenanceItem,
RichTableCell,
Size,
SummaryMetaField,
TableCell,
TableData,
TableItem,
TabularChartMetaField,
)
from docling_core.types.doc.document import GroupLabel
from docling_core.types.doc.labels import CodeLanguageLabel, PictureClassificationLabel
from test.doclang_validation import assert_valid_dclg_xml, doclang_validator
from test.test_data_gen_flag import GEN_TEST_DATA
from test.test_serialization_doclang import (
_doc_cross_column_list,
_doc_cross_page_list,
_doc_cross_page_paragraph,
_doc_cross_page_table,
_doc_multi_prov_text,
_verify_doc,
add_list_section,
add_texts_section,
verify_doclang,
)
from test.test_serialization_doctag import verify
DO_PRINT: bool = False
def _serialize(doc: DoclingDocument) -> str:
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False),
)
text = ser.serialize().text
if not GEN_TEST_DATA:
assert_valid_dclg_xml(text)
return text
def _deserialize(text: str, *, validate: bool = True) -> DoclingDocument:
if validate and not GEN_TEST_DATA:
assert_valid_dclg_xml(text)
return DocLangDocDeserializer().deserialize_str(text)
def _add_default_page(doc: DoclingDocument):
doc.add_page(page_no=0, size=Size(width=1000, height=1000))
def _default_prov() -> ProvenanceItem:
return ProvenanceItem(
page_no=0,
bbox=BoundingBox(l=100, t=100, r=300, b=200),
charspan=(0, 0),
)
class _VirtualTextMixedBboxFactory:
"""Assign distinct page bboxes for the virtual-text mixed fixture."""
def __init__(self, *, page_no: int = 0, page_w: float = 1000, page_h: float = 1000) -> None:
self.page_no = page_no
self.page_w = page_w
self.page_h = page_h
self._slot = 0
def next_prov(self) -> ProvenanceItem:
row, col = divmod(self._slot, 4)
self._slot += 1
cell_w = self.page_w / 5
cell_h = self.page_h / 20
left = 20 + col * cell_w
top = self.page_h - 40 - row * cell_h
return ProvenanceItem(
page_no=self.page_no,
bbox=BoundingBox.from_tuple(
(left, top - cell_h, left + cell_w - 10, top),
origin=CoordOrigin.BOTTOMLEFT,
),
charspan=(0, 0),
)
def next_bbox(self) -> BoundingBox:
return self.next_prov().bbox
def _serialize_virtual_text_mixed(doc: DoclingDocument, *, add_location: bool = True) -> str:
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, add_table_cell_location=True, add_location=add_location),
)
text = ser.serialize().text
if not GEN_TEST_DATA:
assert_valid_dclg_xml(text)
return text
def test_roundtrip_text():
doc = DoclingDocument(name="t")
doc.add_text(label=DocItemLabel.TEXT, text="Hello world")
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
Hello world
"""
assert dt2.strip() == exp_dt.strip()
def test_deserialize_include_namespace_and_version():
"""Deserialize DocLang XML with namespace and version, then roundtrip."""
exp_file = Path("./test/data/doc/deserialize_include_namespace_and_version.gt.dclg.xml")
xml = exp_file.read_text(encoding="utf-8")
doc = _deserialize(xml)
assert len(doc.texts) == 1
assert doc.texts[0].text == "Hello world"
reserialized = (
DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_namespace=True, include_version=True),
)
.serialize()
.text
)
verify_doclang(exp_file=exp_file, actual=reserialized)
def test_roundtrip_title():
doc = DoclingDocument(name="t")
doc.add_title(text="My Title")
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
My Title
"""
assert dt2.strip() == exp_dt.strip()
def test_roundtrip_heading():
doc = DoclingDocument(name="t")
doc.add_heading(text="Section A", level=2)
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
Section A
"""
assert dt2.strip() == exp_dt.strip()
def test_roundtrip_caption():
doc = DoclingDocument(name="t")
doc.add_text(label=DocItemLabel.CAPTION, text="Cap text")
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
Cap text
"""
assert dt2.strip() == exp_dt.strip()
def test_roundtrip_footnote():
doc = DoclingDocument(name="t")
doc.add_text(label=DocItemLabel.FOOTNOTE, text="Foot note")
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.FOOTNOTE
assert doc2.texts[0].text == "Foot note"
def test_roundtrip_page_header():
doc = DoclingDocument(name="t")
doc.add_text(label=DocItemLabel.PAGE_HEADER, text="Header")
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.PAGE_HEADER
assert doc2.texts[0].text == "Header"
def test_roundtrip_page_footer():
doc = DoclingDocument(name="t")
doc.add_text(label=DocItemLabel.PAGE_FOOTER, text="Footer")
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.PAGE_FOOTER
assert doc2.texts[0].text == "Footer"
def test_roundtrip_code():
doc = DoclingDocument(name="t")
doc.add_code(text="print('hi')", code_language=CodeLanguageLabel.PYTHON)
doc.add_code(text='disp("Hello world!")', code_language=CodeLanguageLabel.OCTAVE)
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
"""
assert dt2.strip() == exp_dt.strip()
def test_roundtrip_code_python_and_unknown():
doc = DoclingDocument(name="t")
doc.add_code(text="x = 1", code_language=CodeLanguageLabel.PYTHON)
doc.add_code(text="y = 2") # CodeLanguageLabel.UNKNOWN by default
xml = _serialize(doc)
assert ' ' in xml
assert ' ' not in xml
assert ' ' not in xml
assert ' ' not in xml
doc2 = _deserialize(xml)
codes = [t for t in doc2.texts if t.label == DocItemLabel.CODE]
assert len(codes) == 2
assert codes[0].code_language == CodeLanguageLabel.PYTHON
assert codes[1].code_language == CodeLanguageLabel.UNKNOWN
assert codes[0].text.strip() == "x = 1"
assert codes[1].text.strip() == "y = 2"
@pytest.mark.parametrize(
("docling_lang", "linguist_label", "roundtrip_lang"),
[
(CodeLanguageLabel.BASH, "Shell", CodeLanguageLabel.BASH),
(CodeLanguageLabel.LATEX, "TeX", CodeLanguageLabel.LATEX),
(CodeLanguageLabel.LISP, "Common Lisp", CodeLanguageLabel.LISP),
(CodeLanguageLabel.OBJECTIVEC, "Objective-C", CodeLanguageLabel.OBJECTIVEC),
(CodeLanguageLabel.SML, "Standard ML", CodeLanguageLabel.SML),
(CodeLanguageLabel.VISUALBASIC, "Visual Basic .NET", CodeLanguageLabel.VISUALBASIC),
(CodeLanguageLabel.OCTAVE, "MATLAB", CodeLanguageLabel.MATLAB),
(CodeLanguageLabel.BC, "other", CodeLanguageLabel.UNKNOWN),
(CodeLanguageLabel.DOCLANG, "XML", CodeLanguageLabel.XML),
],
)
def test_code_language_linguist_mapping(docling_lang, linguist_label, roundtrip_lang):
doc = DoclingDocument(name="t")
doc.add_code(text="snippet", code_language=docling_lang)
xml = DocLangDocSerializer(doc=doc, params=DocLangParams(include_version=False)).serialize().text
assert f' ' in xml
doc2 = _deserialize(xml)
assert doc2.texts[0].code_language == roundtrip_lang
def test_roundtrip_code_unknown_as_other_when_enabled():
doc = DoclingDocument(name="t")
doc.add_code(text="y = 2")
xml = (
DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, interpret_code_unknown_as_other=True),
)
.serialize()
.text
)
assert ' ' in xml
doc2 = _deserialize(xml)
assert doc2.texts[0].code_language == CodeLanguageLabel.UNKNOWN
def test_roundtrip_picture_other_and_unknown_labels():
doc = DoclingDocument(name="t")
classified = doc.add_picture()
classified.meta = PictureMeta(
classification=PictureClassificationMetaField(
predictions=[
PictureClassificationPrediction(
class_name=PictureClassificationLabel.OTHER.value,
confidence=1.0,
)
]
)
)
doc.add_picture()
xml = _serialize(doc)
assert ' ' in xml
assert ' ' not in xml
doc2 = _deserialize(xml)
pics = [item for item, _ in doc2.iterate_items() if isinstance(item, PictureItem)]
assert pics[0].meta.classification.get_main_prediction().class_name == "other"
assert pics[1].meta is None or pics[1].meta.classification is None
xml_always = (
DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, label_mode=LabelMode.ALWAYS, add_location=False),
)
.serialize()
.text
)
assert xml_always.count(' ') == 1
def test_code_language_unmapped_linguist_deserializes_to_unknown():
xml = ' foo '
doc = _deserialize(xml, validate=False)
assert doc.texts[0].code_language == CodeLanguageLabel.UNKNOWN
def test_roundtrip_formula():
doc = DoclingDocument(name="t")
doc.add_formula(text="E=mc^2")
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.FORMULA
assert doc2.texts[0].text == "E=mc^2"
def test_roundtrip_list_unordered():
doc = DoclingDocument(name="t")
lg = doc.add_list_group()
doc.add_list_item("A", parent=lg, enumerated=False)
doc.add_list_item("B", parent=lg, enumerated=False)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Ensure group created and items present
assert len(doc2.groups) == 1
assert len(doc2.texts) == 2
assert [it.text for it in doc2.texts] == ["A", "B"]
def test_roundtrip_list_ordered():
doc = DoclingDocument(name="t")
lg = doc.add_list_group()
doc.add_list_item("1", parent=lg, enumerated=True)
doc.add_list_item("2", parent=lg, enumerated=True)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.groups) == 1
assert len(doc2.texts) == 2
assert [it.text for it in doc2.texts] == ["1", "2"]
def test_roundtrip_picture_with_caption():
doc = DoclingDocument(name="t")
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Fig 1")
doc.add_picture(caption=cap)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.pictures) == 1
pic = doc2.pictures[0]
assert len(pic.captions) == 1
cap_item = pic.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION and cap_item.text == "Fig 1"
def test_roundtrip_table_simple():
doc = DoclingDocument(name="t")
td = TableData(num_rows=0, num_cols=2)
td.add_row(["H1", "H2"]) # header row semantics not required here
td.add_row(["C1", "C2"]) # data row
doc.add_table(data=td)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.tables) == 1
t2 = doc2.tables[0].data
assert t2.num_rows == 2 and t2.num_cols == 2
grid_texts = [[cell.text for cell in row] for row in t2.grid]
assert grid_texts == [["H1", "H2"], ["C1", "C2"]]
def test_roundtrip_table_with_caption():
doc = DoclingDocument(name="t")
# Create a caption and a simple 2x2 table
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Tbl 1")
td = TableData(num_rows=0, num_cols=2)
td.add_row(["H1", "H2"]) # header row
td.add_row(["C1", "C2"]) # data row
doc.add_table(data=td, caption=cap)
dt = _serialize(doc)
if DO_PRINT:
print(dt)
doc2 = _deserialize(dt)
# One table reconstructed with same grid
assert len(doc2.tables) == 1
t2 = doc2.tables[0]
assert t2.data.num_rows == 2 and t2.data.num_cols == 2
grid_texts = [[cell.text for cell in row] for row in t2.data.grid]
assert grid_texts == [["H1", "H2"], ["C1", "C2"]]
# Caption preserved and linked to the table
assert len(t2.captions) == 1
cap_item = t2.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION and cap_item.text == "Tbl 1"
def test_roundtrip_text_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_text(label=DocItemLabel.TEXT, text="Hello world", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
if DO_PRINT:
print("\n", dt2)
print(f"`{doc2.texts[0].text}`")
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.TEXT
assert doc2.texts[0].text == "Hello world"
def test_roundtrip_title_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_title(
text="My Title",
prov=_default_prov(),
)
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
exp_dt = """
My Title
"""
assert dt2.strip() == exp_dt.strip()
def test_roundtrip_heading_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_heading(text="Section A", level=2, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
h = doc2.texts[0]
assert h.label == DocItemLabel.SECTION_HEADER and getattr(h, "level", 0) == 2
assert h.text == "Section A"
def test_roundtrip_caption_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_text(label=DocItemLabel.TEXT, text="Cap text", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.TEXT
assert doc2.texts[0].text == "Cap text"
def test_roundtrip_footnote_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_text(label=DocItemLabel.FOOTNOTE, text="Foot note", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.FOOTNOTE
assert doc2.texts[0].text == "Foot note"
def test_roundtrip_page_header_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_text(label=DocItemLabel.PAGE_HEADER, text="Header", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.PAGE_HEADER
assert doc2.texts[0].text == "Header"
def test_roundtrip_page_footer_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_text(label=DocItemLabel.PAGE_FOOTER, text="Footer", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.PAGE_FOOTER
assert doc2.texts[0].text == "Footer"
def test_roundtrip_code_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_code(text="print('hi')", code_language=CodeLanguageLabel.PYTHON, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.CODE
assert doc2.texts[0].text.strip() == "print('hi')"
assert getattr(doc2.texts[0], "code_language", None) == CodeLanguageLabel.PYTHON
def test_roundtrip_formula_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
doc.add_formula(text="E=mc^2", prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.texts) == 1
assert doc2.texts[0].label == DocItemLabel.FORMULA
assert doc2.texts[0].text == "E=mc^2"
def test_roundtrip_list_unordered_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
lg = doc.add_list_group()
prov = _default_prov()
doc.add_list_item("A", parent=lg, prov=prov)
doc.add_list_item("B", parent=lg, prov=prov)
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
assert dt2.strip() == dt.strip()
def test_roundtrip_list_ordered_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
lg = doc.add_list_group()
prov = _default_prov()
doc.add_list_item("1", parent=lg, enumerated=True, prov=prov)
doc.add_list_item("2", parent=lg, enumerated=True, prov=prov)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.groups) == 1
assert len(doc2.texts) == 2
assert [it.text for it in doc2.texts] == ["1", "2"]
def test_roundtrip_picture_with_caption_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Fig 1", prov=_default_prov())
doc.add_picture(caption=cap, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.pictures) == 1
pic = doc2.pictures[0]
assert len(pic.captions) == 1
cap_item = pic.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION and cap_item.text == "Fig 1"
def test_roundtrip_table_simple_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
td = TableData(num_rows=0, num_cols=2)
td.add_row(["H1", "H2"]) # header row semantics not required here
td.add_row(["C1", "C2"]) # data row
doc.add_table(data=td, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.tables) == 1
t2 = doc2.tables[0].data
assert t2.num_rows == 2 and t2.num_cols == 2
grid_texts = [[cell.text for cell in row] for row in t2.grid]
assert grid_texts == [["H1", "H2"], ["C1", "C2"]]
def test_roundtrip_table_with_caption_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Tbl 1", prov=_default_prov())
td = TableData(num_rows=0, num_cols=2)
td.add_row(["H1", "H2"]) # header row
td.add_row(["C1", "C2"]) # data row
doc.add_table(data=td, caption=cap, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print(dt)
doc2 = _deserialize(dt)
assert len(doc2.tables) == 1
t2 = doc2.tables[0]
assert t2.data.num_rows == 2 and t2.data.num_cols == 2
grid_texts = [[cell.text for cell in row] for row in t2.data.grid]
assert grid_texts == [["H1", "H2"], ["C1", "C2"]]
assert len(t2.captions) == 1
cap_item = t2.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION and cap_item.text == "Tbl 1"
def test_roundtrip_complex_table_with_caption_prov():
doc = DoclingDocument(name="t")
_add_default_page(doc)
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Tbl 1", prov=_default_prov())
td = TableData(num_rows=3, num_cols=4)
cell_0_0 = TableCell(
row_span=1,
col_span=1,
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="H 0-0",
)
td.table_cells.append(cell_0_0)
cell_0_1 = TableCell(
row_span=1,
col_span=3,
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=4,
text="H 1-4",
)
td.table_cells.append(cell_0_1)
cell_1_0 = TableCell(
row_span=2,
col_span=1,
start_row_offset_idx=1,
end_row_offset_idx=3,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="R 1-3",
)
td.table_cells.append(cell_1_0)
cell_1_1 = TableCell(
row_span=2,
col_span=3,
start_row_offset_idx=1,
end_row_offset_idx=3,
start_col_offset_idx=1,
end_col_offset_idx=4,
text="R 2-2",
)
td.table_cells.append(cell_1_1)
doc.add_table(data=td, caption=cap, prov=_default_prov())
dt = _serialize(doc)
if DO_PRINT:
print(dt)
doc2 = _deserialize(dt)
assert len(doc2.tables) == 1
t2 = doc2.tables[0]
assert t2.data.num_rows == 3 and t2.data.num_cols == 4
grid_texts = [[cell.text for cell in row] for row in t2.data.grid]
assert grid_texts == [
["H 0-0", "H 1-4", "H 1-4", "H 1-4"],
["R 1-3", "R 2-2", "R 2-2", "R 2-2"],
["R 1-3", "R 2-2", "R 2-2", "R 2-2"],
]
assert len(t2.captions) == 1
cap_item = t2.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION and cap_item.text == "Tbl 1"
def test_roundtrip_nested_list_unordered_in_unordered():
"""Test nested unordered list within unordered list."""
doc = DoclingDocument(name="t")
lg_outer = doc.add_list_group()
doc.add_list_item("Outer Item 1", parent=lg_outer, enumerated=False)
# Create nested list
lg_inner = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Inner Item 1.1", parent=lg_inner, enumerated=False)
doc.add_list_item("Inner Item 1.2", parent=lg_inner, enumerated=False)
doc.add_list_item("Outer Item 2", parent=lg_outer, enumerated=False)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Should have 2 groups (outer and inner)
assert len(doc2.groups) == 2
# Should have 4 text items total (1 ListItem for "Outer Item 1" + 2 inner ListItems + 1 outer ListItem)
assert len(doc2.texts) == 4
# Verify text content
text_contents = [it.text for it in doc2.texts]
assert "Outer Item 1" in text_contents
assert "Inner Item 1.1" in text_contents
assert "Inner Item 1.2" in text_contents
assert "Outer Item 2" in text_contents
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_nested_list_ordered_in_ordered():
"""Test nested ordered list within ordered list."""
doc = DoclingDocument(name="t")
lg_outer = doc.add_list_group()
doc.add_list_item("Step 1", parent=lg_outer, enumerated=False)
# Create nested ordered list
lg_inner = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Step 1.1", parent=lg_inner, enumerated=True)
doc.add_list_item("Step 1.2", parent=lg_inner, enumerated=True)
doc.add_list_item("Step 2", parent=lg_outer, enumerated=True)
dt = _serialize(doc)
doc2 = _deserialize(dt)
dt2 = _serialize(doc2)
assert dt2.strip() == dt.strip()
def test_roundtrip_nested_list_ordered_in_unordered():
"""Test nested ordered list within unordered list."""
doc = DoclingDocument(name="t")
lg_outer = doc.add_list_group()
doc.add_list_item("Bullet A", parent=lg_outer, enumerated=False)
# Create nested ordered list
lg_inner = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Numbered 1", parent=lg_inner, enumerated=True)
doc.add_list_item("Numbered 2", parent=lg_inner, enumerated=True)
doc.add_list_item("Bullet B", parent=lg_outer, enumerated=False)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.groups) == 2
assert len(doc2.texts) == 4
text_contents = [it.text for it in doc2.texts]
assert "Bullet A" in text_contents
assert "Numbered 1" in text_contents
assert "Numbered 2" in text_contents
assert "Bullet B" in text_contents
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_nested_list_unordered_in_ordered():
"""Test nested unordered list within ordered list."""
doc = DoclingDocument(name="t")
lg_outer = doc.add_list_group()
doc.add_list_item("Step 1", parent=lg_outer, enumerated=True)
# Create nested unordered list
lg_inner = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Bullet point", parent=lg_inner, enumerated=False)
doc.add_list_item("Another bullet", parent=lg_inner, enumerated=False)
doc.add_list_item("Step 2", parent=lg_outer, enumerated=True)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
assert len(doc2.groups) == 2
assert len(doc2.texts) == 4
text_contents = [it.text for it in doc2.texts]
assert "Step 1" in text_contents
assert "Bullet point" in text_contents
assert "Another bullet" in text_contents
assert "Step 2" in text_contents
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_deeply_nested_list():
"""Test deeply nested lists (3 levels)."""
doc = DoclingDocument(name="t")
# Level 1
lg_level1 = doc.add_list_group()
doc.add_list_item("Level 1 Item 1", parent=lg_level1, enumerated=False)
# Level 2 (nested in level 1)
lg_level2 = doc.add_list_group(parent=lg_level1)
doc.add_list_item("Level 2 Item 1", parent=lg_level2, enumerated=False)
# Level 3 (nested in level 2)
lg_level3 = doc.add_list_group(parent=lg_level2)
doc.add_list_item("Level 3 Item 1", parent=lg_level3, enumerated=False)
doc.add_list_item("Level 3 Item 2", parent=lg_level3, enumerated=False)
doc.add_list_item("Level 2 Item 2", parent=lg_level2, enumerated=False)
doc.add_list_item("Level 1 Item 2", parent=lg_level1, enumerated=False)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Should have 3 groups (3 levels)
assert len(doc2.groups) == 3
# Should have 6 text items total
assert len(doc2.texts) == 6
text_contents = [it.text for it in doc2.texts]
assert "Level 1 Item 1" in text_contents
assert "Level 2 Item 1" in text_contents
assert "Level 3 Item 1" in text_contents
assert "Level 3 Item 2" in text_contents
assert "Level 2 Item 2" in text_contents
assert "Level 1 Item 2" in text_contents
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_multiple_nested_lists_same_level():
"""Test multiple nested lists at the same level."""
doc = DoclingDocument(name="t")
lg_outer = doc.add_list_group()
doc.add_list_item("Item 1", parent=lg_outer, enumerated=False)
# First nested list
lg_inner1 = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Nested 1.1", parent=lg_inner1, enumerated=False)
doc.add_list_item("Nested 1.2", parent=lg_inner1, enumerated=False)
doc.add_list_item("Item 2", parent=lg_outer, enumerated=False)
# Second nested list
lg_inner2 = doc.add_list_group(parent=lg_outer)
doc.add_list_item("Nested 2.1", parent=lg_inner2, enumerated=False)
doc.add_list_item("Nested 2.2", parent=lg_inner2, enumerated=False)
doc.add_list_item("Item 3", parent=lg_outer, enumerated=False)
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Should have 3 groups (1 outer, 2 inner)
assert len(doc2.groups) == 3
# Should have 7 text items total
assert len(doc2.texts) == 7
text_contents = [it.text for it in doc2.texts]
assert "Item 1" in text_contents
assert "Nested 1.1" in text_contents
assert "Nested 1.2" in text_contents
assert "Item 2" in text_contents
assert "Nested 2.1" in text_contents
assert "Nested 2.2" in text_contents
assert "Item 3" in text_contents
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_list_item_with_inline_group():
doc = DoclingDocument(name="t")
add_texts_section(doc)
add_list_section(doc)
dt = _serialize(doc)
exp_ser_file = Path(__file__).parent / "data" / "doc" / "roundtrip_list_item_with_inline_serialized.dclg.xml"
verify(exp_ser_file, dt)
doc2 = _deserialize(dt)
deser_yaml_file = Path(__file__).parent / "data" / "doc" / "roundtrip_list_item_with_inline_deserialized.yaml"
doc2.save_as_yaml(deser_yaml_file)
dt2 = _serialize(doc2)
exp_dt2_file = Path(__file__).parent / "data" / "doc" / "roundtrip_list_item_with_inline_reserialized.dclg.xml"
verify(exp_dt2_file, dt2)
def test_deserialize_bare_picture():
dt = " "
doc = _deserialize(dt)
assert len(doc.pictures) == 1
def test_deserialize_bare_table():
dt = " "
doc = _deserialize(dt)
assert len(doc.tables) == 1
assert doc.tables[0].data.num_rows == 0
assert doc.tables[0].data.num_cols == 0
def test_roundtrip_table_with_caption_and_footnotes():
"""Test table with caption and multiple footnotes."""
doc = DoclingDocument(name="t")
# Create caption and footnotes
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Table 1: Sample Data")
# Create a simple 2x2 table
td = TableData(num_rows=0, num_cols=2)
td.add_row(["Header 1", "Header 2"])
td.add_row(["Data 1", "Data 2"])
# Add table with caption
table = doc.add_table(data=td, caption=cap)
footnote1 = doc.add_text(label=DocItemLabel.FOOTNOTE, text="First footnote")
footnote2 = doc.add_text(label=DocItemLabel.FOOTNOTE, text="Second footnote")
footnote3 = doc.add_text(label=DocItemLabel.FOOTNOTE, text="Third footnote")
# Add footnotes to the table
table.footnotes.append(footnote1.get_ref())
table.footnotes.append(footnote2.get_ref())
table.footnotes.append(footnote3.get_ref())
# Serialize and deserialize
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt2:", dt2)
# Verify table structure
assert len(doc2.tables) == 1
t2 = doc2.tables[0]
assert t2.data.num_rows == 2 and t2.data.num_cols == 2
grid_texts = [[cell.text for cell in row] for row in t2.data.grid]
assert grid_texts == [["Header 1", "Header 2"], ["Data 1", "Data 2"]]
# Verify caption
assert len(t2.captions) == 1
cap_item = t2.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION
assert cap_item.text == "Table 1: Sample Data"
# Verify footnotes
assert len(t2.footnotes) == 3
footnote_texts = [fn.resolve(doc2).text for fn in t2.footnotes]
assert footnote_texts == ["First footnote", "Second footnote", "Third footnote"]
# Verify all footnotes have the correct label
for fn_ref in t2.footnotes:
fn_item = fn_ref.resolve(doc2)
assert fn_item.label == DocItemLabel.FOOTNOTE
assert dt2 == dt
def test_roundtrip_picture_with_caption_and_footnotes():
"""Test picture with caption and multiple footnotes."""
doc = DoclingDocument(name="t")
# Create caption and footnotes
cap = doc.add_text(label=DocItemLabel.CAPTION, text="Figure 1: Sample Image")
footnote1 = doc.add_text(label=DocItemLabel.FOOTNOTE, text="Image source: Dataset A")
footnote2 = doc.add_text(label=DocItemLabel.FOOTNOTE, text="Resolution: 1024x768")
# Add picture with caption
picture = doc.add_picture(caption=cap)
# Add footnotes to the picture
picture.footnotes.append(footnote1.get_ref())
picture.footnotes.append(footnote2.get_ref())
# Serialize and deserialize
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Verify picture exists
assert len(doc2.pictures) == 1
pic = doc2.pictures[0]
# Verify caption
assert len(pic.captions) == 1
cap_item = pic.captions[0].resolve(doc2)
assert cap_item.label == DocItemLabel.CAPTION
assert cap_item.text == "Figure 1: Sample Image"
# Verify footnotes
assert len(pic.footnotes) == 2
footnote_texts = [fn.resolve(doc2).text for fn in pic.footnotes]
assert footnote_texts == ["Image source: Dataset A", "Resolution: 1024x768"]
# Verify all footnotes have the correct label
for fn_ref in pic.footnotes:
fn_item = fn_ref.resolve(doc2)
assert fn_item.label == DocItemLabel.FOOTNOTE
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def test_roundtrip_table_with_rich_cells():
"""Test table with RichTableCells containing paragraphs, lists, and nested tables."""
doc = DoclingDocument(name="t")
# Create the main table (3x2 grid)
table = doc.add_table(data=TableData(num_rows=3, num_cols=2))
# Create content for rich cells
# Cell (0, 0): Multiple paragraphs
para1 = doc.add_text(parent=table, label=DocItemLabel.TEXT, text="First paragraph in cell.")
doc.add_text(parent=table, label=DocItemLabel.TEXT, text="Second paragraph in cell.")
# Cell (1, 0): A list
list_group = doc.add_list_group(parent=table)
doc.add_list_item(parent=list_group, text="List item 1", enumerated=False)
doc.add_list_item(parent=list_group, text="List item 2", enumerated=False)
doc.add_list_item(parent=list_group, text="List item 3", enumerated=False)
# Cell (2, 1): A nested table
nested_table_data = TableData(num_rows=0, num_cols=2)
nested_table_data.add_row(["A1", "A2"])
nested_table_data.add_row(["B1", "B2"])
nested_table = doc.add_table(data=nested_table_data, parent=table)
# Create the table cells
# Row 0: Rich cell with paragraphs in (0,0), simple cell in (0,1)
rich_cell_0_0 = RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
ref=para1.get_ref(), # Note: In a real scenario, we'd want a group containing both paragraphs
text="cell 0,0",
)
doc.add_table_cell(table_item=table, cell=rich_cell_0_0)
simple_cell_0_1 = TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=1,
text="Simple cell 0,1",
)
doc.add_table_cell(table_item=table, cell=simple_cell_0_1)
# Row 1: Rich cell with list in (1,0), simple cell in (1,1)
rich_cell_1_0 = RichTableCell(
start_row_offset_idx=1,
end_row_offset_idx=2,
start_col_offset_idx=0,
end_col_offset_idx=1,
ref=list_group.get_ref(),
text="cell 1,0",
)
doc.add_table_cell(table_item=table, cell=rich_cell_1_0)
simple_cell_1_1 = TableCell(
start_row_offset_idx=1,
end_row_offset_idx=2,
start_col_offset_idx=1,
end_col_offset_idx=1,
text="Simple cell 1,1",
)
doc.add_table_cell(table_item=table, cell=simple_cell_1_1)
# Row 2: Simple cell in (2,0), rich cell with nested table in (2,1)
simple_cell_2_0 = TableCell(
start_row_offset_idx=2,
end_row_offset_idx=3,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="Simple cell 2,0",
)
doc.add_table_cell(table_item=table, cell=simple_cell_2_0)
rich_cell_2_1 = RichTableCell(
start_row_offset_idx=2,
end_row_offset_idx=3,
start_col_offset_idx=1,
end_col_offset_idx=1,
ref=nested_table.get_ref(),
text="cell 2,1",
)
doc.add_table_cell(table_item=table, cell=rich_cell_2_1)
# Serialize and deserialize
dt = _serialize(doc)
if DO_PRINT:
print("\n", dt)
doc2 = _deserialize(dt)
# Verify main table structure
assert len(doc2.tables) >= 1
main_table = doc2.tables[0]
assert main_table.data.num_rows == 3
assert main_table.data.num_cols == 2
# Verify that we have rich table cells
rich_cells = [cell for cell in main_table.data.table_cells if isinstance(cell, RichTableCell)]
assert len(rich_cells) >= 1 # At least one rich cell should be preserved
# Verify round-trip serialization
dt2 = _serialize(doc2)
if DO_PRINT:
print("\ndt:", dt)
print("\ndt2:", dt2)
assert dt2 == dt
def _create_virtual_text_list_doc() -> DoclingDocument:
"""Document exercising virtual vs explicit ```` in list items."""
doc = DoclingDocument(name="virtual_text_list")
_add_default_page(doc)
bbox = _VirtualTextMixedBboxFactory()
lg = doc.add_list_group()
doc.add_list_item(text="plain list item", parent=lg, prov=bbox.next_prov())
li_inline = doc.add_list_item(text="", parent=lg, prov=bbox.next_prov())
inline = doc.add_inline_group(parent=li_inline)
doc.add_text(label=DocItemLabel.TEXT, text="this is a ", parent=inline)
doc.add_text(
label=DocItemLabel.TEXT,
text="bold",
parent=inline,
formatting=Formatting(bold=True),
)
doc.add_text(label=DocItemLabel.TEXT, text=" text", parent=inline)
li_nested = doc.add_list_item(
text="list item with nested list",
parent=lg,
prov=bbox.next_prov(),
)
lg_sub = doc.add_list_group(parent=li_nested)
doc.add_list_item(text="nested list item", parent=lg_sub, prov=bbox.next_prov())
li_picture = doc.add_list_item(
text="list item with picture",
parent=lg,
prov=bbox.next_prov(),
)
doc.add_picture(parent=li_picture, prov=bbox.next_prov())
return doc
def _assert_virtual_text_list_dclg(dclg: str) -> None:
"""Sanity-check virtual vs explicit ```` in list output."""
assert " " in dclg
assert "plain list item" in dclg
assert "plain list item " not in dclg
assert "" in dclg
assert "this is a " in dclg
assert "bold " in dclg
assert " text" in dclg
assert re.search(
r" \s*(?:]*/>\s*)*this is a \s*bold \s* text ",
dclg,
)
assert not re.search(
r"\s*(?:]*/>\s*)*this is a \s*bold ",
dclg,
)
assert "list item with nested list" in dclg
assert "list item with picture" in dclg
assert re.search(
r"\s*(?:]*/>\s*)*list item with picture\s* ",
dclg,
)
assert "" in dt
doc2 = _deserialize(dt)
_verify_doc(doc=doc2, exp_json=deserialized_json)
dt2 = _serialize(doc2)
verify(reserialized_dclg, dt2)
assert_valid_dclg_xml(dt2)
def _create_multi_page_roundtrip_doc() -> DoclingDocument:
"""Build a three-page document with title, index, and body paragraphs."""
page_size = Size(width=512, height=512)
doc = DoclingDocument(name="multi_page_roundtrip")
for page_no in (1, 2, 3):
doc.add_page(page_no=page_no, size=page_size, image=None)
doc.add_title(
text="Document Title",
prov=_page_prov(page_no=1, bbox=(10, 10, 200, 40)),
)
index_data = TableData(num_cols=2)
index_data.add_row(["Chapter", "Page"])
index_data.add_row(["Intro", "1"])
index_data.add_row(["Body", "3"])
doc.add_table(
data=index_data,
label=DocItemLabel.DOCUMENT_INDEX,
prov=_page_prov(page_no=2, bbox=(10, 10, 400, 120)),
)
for i, text in enumerate(["First paragraph.", "Second paragraph.", "Third paragraph."], start=1):
doc.add_text(
label=DocItemLabel.TEXT,
text=text,
prov=_page_prov(page_no=3, bbox=(10, 40 + i * 30, 400, 60 + i * 30)),
)
return doc
def _page_prov(*, page_no: int, bbox: tuple[float, float, float, float]) -> ProvenanceItem:
return ProvenanceItem(
page_no=page_no,
bbox=BoundingBox.from_tuple(bbox, origin=CoordOrigin.TOPLEFT),
charspan=(0, 0),
)
def _create_virtual_text_table_doc() -> DoclingDocument:
"""Document exercising virtual vs explicit ```` in table cells."""
doc = DoclingDocument(name="virtual_text_table")
_add_default_page(doc)
bbox = _VirtualTextMixedBboxFactory()
def _add_row_table(*, label: DocItemLabel) -> None:
table = doc.add_table(data=TableData(num_rows=1, num_cols=3), label=label, prov=bbox.next_prov())
cell_group = doc.add_group(parent=table, label=GroupLabel.UNSPECIFIED)
doc.add_text(
label=DocItemLabel.TEXT,
text="cell with list:",
parent=cell_group,
prov=bbox.next_prov(),
)
cell_list = doc.add_list_group(parent=cell_group)
doc.add_list_item(text="nested in cell", parent=cell_list, prov=bbox.next_prov())
rich_group = doc.add_group(parent=table, label=GroupLabel.UNSPECIFIED)
doc.add_text(
label=DocItemLabel.TEXT,
text="group text",
parent=rich_group,
prov=bbox.next_prov(),
)
doc.add_picture(parent=rich_group, prov=bbox.next_prov())
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="plain cell",
bbox=bbox.next_bbox(),
),
)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
ref=cell_group.get_ref(),
text="cell with list:",
bbox=bbox.next_bbox(),
),
)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=2,
end_col_offset_idx=3,
ref=rich_group.get_ref(),
text="group cell",
bbox=bbox.next_bbox(),
),
)
_add_row_table(label=DocItemLabel.TABLE)
return doc
def _create_virtual_text_index_doc() -> DoclingDocument:
"""Document exercising virtual vs explicit ```` in index cells."""
doc = DoclingDocument(name="virtual_text_index")
_add_default_page(doc)
bbox = _VirtualTextMixedBboxFactory()
table = doc.add_table(
data=TableData(num_rows=1, num_cols=3),
label=DocItemLabel.DOCUMENT_INDEX,
prov=bbox.next_prov(),
)
cell_group = doc.add_group(parent=table, label=GroupLabel.UNSPECIFIED)
doc.add_text(
label=DocItemLabel.TEXT,
text="cell with list:",
parent=cell_group,
prov=bbox.next_prov(),
)
cell_list = doc.add_list_group(parent=cell_group)
doc.add_list_item(text="nested in cell", parent=cell_list, prov=bbox.next_prov())
rich_group = doc.add_group(parent=table, label=GroupLabel.UNSPECIFIED)
doc.add_text(
label=DocItemLabel.TEXT,
text="group text",
parent=rich_group,
prov=bbox.next_prov(),
)
doc.add_picture(parent=rich_group, prov=bbox.next_prov())
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="plain cell",
bbox=bbox.next_bbox(),
),
)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=1,
end_col_offset_idx=2,
ref=cell_group.get_ref(),
text="cell with list:",
bbox=bbox.next_bbox(),
),
)
doc.add_table_cell(
table_item=table,
cell=RichTableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=2,
end_col_offset_idx=3,
ref=rich_group.get_ref(),
text="group cell",
bbox=bbox.next_bbox(),
),
)
return doc
def _assert_virtual_text_table_dclg(dclg: str) -> None:
host_block = dclg.split("", 1)[0]
assert "plain cell" in host_block
assert "plain cell " not in host_block
assert "cell with list:" in host_block
assert "" in host_block
assert "group text" in host_block
assert "", 1)[1].split("", 1)[0]
assert "plain cell" in host_block
assert "plain cell " not in host_block
assert "cell with list:" in host_block
assert "" in host_block
assert "group text" in host_block
assert " None:
"""Verify picture/table caption refs are preserved in the document model."""
assert len(doc.pictures) == 1
pic = doc.pictures[0]
assert len(pic.captions) == 1
pic_cap = pic.captions[0].resolve(doc)
assert pic_cap.label == DocItemLabel.CAPTION
assert pic_cap.text == "Figure 1"
assert len(doc.tables) == 1
tbl = doc.tables[0]
assert len(tbl.captions) == 1
tbl_cap = tbl.captions[0].resolve(doc)
assert tbl_cap.label == DocItemLabel.CAPTION
assert tbl_cap.text == "Table 1"
grid_texts = [[cell.text for cell in row] for row in tbl.data.grid]
assert grid_texts == [["H1", "H2"], ["C1", "C2"]]
def _assert_referenced_caption_dclg(dclg: str, *, with_location: bool) -> None:
"""Sanity-check referenced captions appear in host element heads."""
pic_block = dclg.split("", 1)[1].split(" ", 1)[0]
assert "Figure 1" in pic_block
assert re.search(r"\s*(?:]*/>\s*)*Figure 1\s* ", pic_block)
tbl_block = dclg.split("", 1)[0]
assert "Table 1" in tbl_block
assert re.search(r"\s*(?:]*/>\s*)*Table 1\s* ", tbl_block)
assert "H1" in tbl_block and "C2" in tbl_block
if with_location:
assert "
simple
leading
trailing
w/prov simple
w/prov leading
w/prov trailing
"""
doc = _deserialize(dt)
dt2 = _serialize(doc)
assert dt2.strip() == dt.strip()
def test_rich_table_cells():
dt = """
foo
text in italic
inner cell 0,0
inner cell 0,1
inner cell 1,0
inner cell 1,1
in bold
bar
"""
doc = _deserialize(dt)
dt2 = _serialize(doc)
assert dt2.strip() == dt.strip()
def test_picture_tabular_chart_content_cdata_cells():
"""Deserializer must extract text from in OTSL cells."""
doclang = """ """
doc = _deserialize(doclang)
first_cell_text = doc.pictures[0].meta.tabular_chart.chart_data.grid[0][0].text
assert first_cell_text == "Characteristic"
assert doc.pictures[0].meta.tabular_chart.chart_data.grid[0][1].text == "Player expenses in million U.S. dollars"
assert doc.pictures[0].meta.tabular_chart.chart_data.grid[1][0].text == "19/20"
assert doc.pictures[0].meta.tabular_chart.chart_data.grid[1][1].text == "111"
@pytest.mark.parametrize("escape_mode", [EscapeMode.AUTO, EscapeMode.ALWAYS])
def test_roundtrip_otsl_xml_sensitive_cell_content(escape_mode: EscapeMode):
"""OTSL cell text remains exact when DOM fragments are reparsed."""
cell_texts = [
'Parsing (Structure&Semantic) "quoted"',
"ampersand & less < greater >",
"\"quotes\" and 'apostrophes'",
"literal entity-like text: &",
"CDATA terminator: ]]>",
"multiline code:\nif a < b:\n return a & b",
"Unicode and math: café ∀x ∈ ℝ, x² ≥ 0", # noqa: RUF001
]
doc = DoclingDocument(name="otsl-xml-sensitive")
table_data = TableData(num_rows=0, num_cols=1)
for text in cell_texts:
table_data.add_row([text])
doc.add_table(data=table_data)
serialized = (
DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, escape_mode=escape_mode),
)
.serialize()
.text
)
roundtripped = _deserialize(serialized)
assert [row[0].text for row in roundtripped.tables[0].data.grid] == cell_texts
def test_otsl_xml_sensitive_virtual_and_explicit_text_cells():
"""Both virtual and explicit rich OTSL cell bodies preserve safe text."""
doclang = """\
"quoted" 'apostrophe']]>
]]> bold & styled
"""
doc = _deserialize(doclang)
assert doc.tables[0].data.grid[0][0].text == "virtual & \"quoted\" 'apostrophe'"
assert doc.tables[0].data.grid[0][1].text == "nested & bold & styled"
assert isinstance(doc.tables[0].data.grid[0][1], RichTableCell)
def test_picture_body_table_is_semantic_content_not_chart_tabular():
"""```` after the preamble is nested picture content, not ``meta.tabular_chart``."""
doclang = (
''
" Chart 1 "
""
" "
)
doc = _deserialize(doclang)
pic = doc.pictures[0]
assert pic.meta is not None
assert pic.meta.tabular_chart.chart_data.grid[0][0].text == "Chart"
assert len(pic.children) == 1
nested = pic.children[0].resolve(doc)
assert isinstance(nested, TableItem)
assert nested.data.grid[0][0].text == "Nested"
# SMILES from test/data/doc/dummy_doc_with_meta.yaml (molecule_data annotation)
_EXAMPLE_MOLECULE_SMILES = "CC1=NNC(C2=CN3C=CN=C3C(CC3=CC(F)=CC(F)=C3)=N2)=N1"
_PICTURE_META_SUMMARY = "Picture meta summary"
_PICTURE_META_DESCRIPTION = "Picture meta description"
_PICTURE_META_CUSTOM_VALUE = "custom field on picture meta"
_CHART_TITLE = "Chart Title"
def _create_picture_molecule_doc() -> DoclingDocument:
doc = DoclingDocument(name="picture_molecule_meta")
pic = doc.add_picture()
pic.meta = PictureMeta(
summary=SummaryMetaField(text=_PICTURE_META_SUMMARY),
description=DescriptionMetaField(text=_PICTURE_META_DESCRIPTION),
classification=PictureClassificationMetaField(
predictions=[
PictureClassificationPrediction(
class_name=PictureClassificationLabel.PIE_CHART.value,
confidence=1.0,
)
]
),
molecule=MoleculeMetaField(smi=_EXAMPLE_MOLECULE_SMILES),
)
pic.meta.set_custom_field(
namespace="my_corp",
name="note",
value=_PICTURE_META_CUSTOM_VALUE,
)
# Tabular chart data (pattern from test_serialization_doclang._create_content_filtering_doc)
chart_data = TableData(num_cols=2)
chart_data.add_row(["Foo", "Bar"])
chart_data.add_row(["One", "Two"])
pic.meta.tabular_chart = TabularChartMetaField(
title=_CHART_TITLE,
chart_data=chart_data,
)
return doc
_KV_ANNOT_ROOT = Path(__file__).parent / "data" / "doc" / "kv"
# KV annot fixtures with lossless DocLang XML round-trip today.
_KV_ANNOT_XML_LOSSLESS = frozenset(
{
"01d07afe1cb54ecd23eedfe4d91b81dd88e61bf4e0dbe2467784db4177a6c691",
"08212053e2db1a70dd60a4f85650ceb33d7519af34f502e3ac894389d76663d6",
"1eac20e5ac5fac655a611343f86927d6a76277e170430c1eba741585437a2e90",
"ba4120cada21304563625490e9ad13911e96114d3f07df056a6bf62397a859e1",
}
)
def _kv_annot_fixture_dirs() -> list[Path]:
"""Return migrated KV annot fixture dirs with serialized DocLang goldens."""
return sorted(
p
for p in _KV_ANNOT_ROOT.iterdir()
if p.is_dir() and (p / "output.json").is_file() and (p / "output.dclg.xml").is_file()
)
def _serialize_kv_annot_fixture(doc: DoclingDocument) -> str:
text = DocLangDocSerializer(doc=doc, params=DocLangParams(include_version=False)).serialize().text
if not GEN_TEST_DATA:
assert_valid_dclg_xml(text)
return text
@pytest.mark.parametrize(
"fixture_dir",
_kv_annot_fixture_dirs(),
ids=[p.name for p in _kv_annot_fixture_dirs()],
)
def test_kv_annot_doclang_roundtrip(fixture_dir: Path):
"""Round-trip migrated KV annot fixtures through DocLang (see ``test/data/doc/kv/``)."""
output_json = fixture_dir / "output.json"
serialized_dclg = fixture_dir / "output.dclg.xml"
deserialized_json = fixture_dir / "deserialized.json"
reserialized_dclg = fixture_dir / "reserialized.dclg.xml"
doc = DoclingDocument.load_from_json(output_json)
dt = _serialize_kv_annot_fixture(doc)
verify_doclang(exp_file=serialized_dclg, actual=dt)
doc2 = _deserialize(dt)
_verify_doc(doc=doc2, exp_json=deserialized_json)
dt2 = _serialize_kv_annot_fixture(doc2)
verify_doclang(exp_file=reserialized_dclg, actual=dt2)
if fixture_dir.name in _KV_ANNOT_XML_LOSSLESS:
assert dt.strip() == dt2.strip()
def _serialize_field_region_fixture(doc: DoclingDocument, *, fixture_dir: str) -> str:
"""Serialize a field-region fixture doc (invoice uses placeholder pictures)."""
params = (
DocLangParams(include_version=False, image_mode=ImageRefMode.PLACEHOLDER)
if fixture_dir == "field_region_kv_invoice"
else DocLangParams(include_version=False)
)
text = DocLangDocSerializer(doc=doc, params=params).serialize().text
if not GEN_TEST_DATA:
assert_valid_dclg_xml(text)
return text
@pytest.mark.parametrize(
"fixture_dir",
[
"field_region_kv_migration",
"field_region_kv",
"field_region_kv_invoice",
],
)
def test_field_region_doclang_roundtrip(fixture_dir: str):
"""Round-trip field regions/items through DocLang deserialization."""
data_dir = Path(__file__).parent / "data" / "doc" / fixture_dir
input_json = data_dir / "input.json"
serialized_dclg = data_dir / "serialized.dclg.xml"
deserialized_json = data_dir / "deserialized.json"
reserialized_dclg = data_dir / "reserialized.dclg.xml"
doc = DoclingDocument.load_from_json(input_json)
_verify_doc(doc=doc, exp_json=input_json)
assert doc.field_regions
dt = _serialize_field_region_fixture(doc, fixture_dir=fixture_dir)
verify_doclang(exp_file=serialized_dclg, actual=dt)
doc2 = _deserialize(dt)
_verify_doc(doc=doc2, exp_json=deserialized_json)
assert doc2.field_regions
dt2 = _serialize_field_region_fixture(doc2, fixture_dir=fixture_dir)
verify_doclang(exp_file=reserialized_dclg, actual=dt2)
assert dt.strip() == dt2.strip()
def test_picture_molecule_meta_roundtrip():
"""Round-trip picture meta (molecule, chart/tabular, summary, description, custom) through DocLang."""
data_dir = Path(__file__).parent / "data" / "doc" / "picture_molecule_meta"
input_json = data_dir / "input.json"
serialized_dclg = data_dir / "serialized.dclg.xml"
deserialized_json = data_dir / "deserialized.json"
reserialized_dclg = data_dir / "reserialized.dclg.xml"
doc = _create_picture_molecule_doc()
_verify_doc(doc=doc, exp_json=input_json)
dt = _serialize(doc)
verify(serialized_dclg, dt)
assert f"{_PICTURE_META_SUMMARY} " in dt
assert f"{_PICTURE_META_DESCRIPTION} " in dt
assert f"{_EXAMPLE_MOLECULE_SMILES} " in dt
assert f"{_PICTURE_META_CUSTOM_VALUE} " in dt
assert 'class="chart"' in dt
assert 'value="pie_chart"' in dt
assert "" in dt
assert "Foo" in dt and "Bar" in dt and "One" in dt and "Two" in dt
doc2 = _deserialize(dt)
_verify_doc(doc=doc2, exp_json=deserialized_json)
dt2 = _serialize(doc2)
verify(reserialized_dclg, dt2)
def test_roundtrip_with_layers():
"""Test roundtrip with content layers."""
from docling_core.types.doc import ContentLayer
doc = DoclingDocument(name="t")
# Add items with different layers
doc.add_text(label=DocItemLabel.PAGE_HEADER, text="Header", content_layer=ContentLayer.FURNITURE)
doc.add_text(label=DocItemLabel.TEXT, text="Body text", content_layer=ContentLayer.BODY)
doc.add_text(label=DocItemLabel.PAGE_FOOTER, text="Footer", content_layer=ContentLayer.FURNITURE)
# Serialize with ALWAYS mode to ensure layers are included
from docling_core.transforms.serializer.doclang import LayerMode
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, layer_mode=LayerMode.ALWAYS),
)
dt = ser.serialize().text
# Deserialize
doc2 = _deserialize(dt)
# Verify layers are preserved
assert len(doc2.body.children) == 3
items = [doc2.body.children[i].resolve(doc2) for i in range(3)]
assert items[0].content_layer == ContentLayer.FURNITURE
assert items[1].content_layer == ContentLayer.BODY
assert items[2].content_layer == ContentLayer.FURNITURE
def test_roundtrip_with_newlines():
"""Test that newlines in survive deserialization and reserialization."""
doclang_str = """
foo
bar
zoo
zen
"""
doc = _deserialize(doclang_str)
dt2 = _serialize(doc)
assert dt2.strip() == doclang_str.strip()
def test_roundtrip_document_index_table():
"""Test that DOCUMENT_INDEX label is preserved through serialization/deserialization."""
doc = DoclingDocument(name="test")
_add_default_page(doc)
# Add a regular table
table_data = TableData(num_cols=2)
table_data.add_row(["Header 1", "Header 2"])
table_data.grid[0][0].column_header = True
table_data.grid[0][1].column_header = True
table_data.add_row(["Data 1", "Data 2"])
doc.add_table(data=table_data, label=DocItemLabel.TABLE, prov=_default_prov())
# Add a DOCUMENT_INDEX table
index_data = TableData(num_cols=2)
index_data.add_row(["Index 1", "Page 1"])
index_data.add_row(["Index 2", "Page 2"])
doc.add_table(data=index_data, label=DocItemLabel.DOCUMENT_INDEX, prov=_default_prov())
# Serialize
xml_str = _serialize(doc)
assert "" in xml_str
# Deserialize
doc2 = _deserialize(xml_str)
# Verify we have 2 tables
assert len(doc2.tables) == 2
# Verify labels are preserved
assert doc2.tables[0].label == DocItemLabel.TABLE
assert doc2.tables[1].label == DocItemLabel.DOCUMENT_INDEX
# Verify table data is preserved
assert doc2.tables[0].data.num_rows == 2
assert doc2.tables[0].data.num_cols == 2
assert doc2.tables[1].data.num_rows == 2
assert doc2.tables[1].data.num_cols == 2
def _thread_roundtrip(
*,
name: str,
doc_factory: Callable[[], DoclingDocument],
page_breaks: int = 0,
) -> None:
"""Round-trip a threaded document fixture through serialize → deserialize → reserialize."""
data_dir = Path(__file__).parent / "data" / "doc" / name
input_json = data_dir / "input.json"
serialized_dclg = data_dir / "serialized.dclg.xml"
deserialized_json = data_dir / "deserialized.json"
reserialized_dclg = data_dir / "reserialized.dclg.xml"
doc = doc_factory()
_verify_doc(doc=doc, exp_json=input_json)
dt = _serialize(doc)
verify(serialized_dclg, dt)
if page_breaks:
assert dt.count("`` is rejected (v0.5: use ```` for document indexes)."""
xml_str = """
"""
with pytest.raises(ValueError, match="table element must not have a class attribute"):
_deserialize(xml_str, validate=False)
def test_deserialize_rejects_oversize_xml() -> None:
"""Huge DocLang markup must fail closed before DOM construction grows unchecked."""
xml = "" + ("x" * 200) + " "
with pytest.raises(ValueError, match="exceeds size limit"):
DocLangDocDeserializer().deserialize_str(xml, max_xml_bytes=64)
def test_deserialize_rejects_over_deep_xml() -> None:
"""Extreme element nesting must fail closed (defusedxml does not bound depth)."""
depth = 20
xml = "".join(f"" for i in range(depth)) + "x" + "".join(f" " for i in reversed(range(depth)))
xml = f"{xml} "
with pytest.raises(ValueError, match="exceeds nesting depth limit"):
DocLangDocDeserializer().deserialize_str(xml, max_xml_depth=8)
def test_deserialize_rejects_too_many_elements() -> None:
"""Element floods must fail closed even when total byte size is modest."""
# 30 sibling elements under doclang — over a tight element budget.
children = "".join(f"t{i} " for i in range(30))
xml = f"{children} "
with pytest.raises(ValueError, match="exceeds element count limit"):
DocLangDocDeserializer().deserialize_str(xml, max_xml_elements=10)
def test_deserialize_accepts_document_within_budgets() -> None:
xml = "hello "
doc = DocLangDocDeserializer().deserialize_str(
xml,
max_xml_bytes=1024,
max_xml_depth=16,
max_xml_elements=100,
)
assert len(doc.texts) == 1
assert doc.texts[0].text == "hello"
| |