import re from collections.abc import Callable from pathlib import Path import pytest from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer from docling_core.transforms.serializer.doclang import ( DocLangDocSerializer, DocLangParams, EscapeMode, LabelMode, ) from docling_core.types.doc import ( BoundingBox, CoordOrigin, DescriptionMetaField, DocItemLabel, DoclingDocument, Formatting, ImageRefMode, MoleculeMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureItem, PictureMeta, ProvenanceItem, RichTableCell, Size, SummaryMetaField, TableCell, TableData, TableItem, TabularChartMetaField, ) from docling_core.types.doc.document import GroupLabel from docling_core.types.doc.labels import CodeLanguageLabel, PictureClassificationLabel from test.doclang_validation import assert_valid_dclg_xml, doclang_validator from test.test_data_gen_flag import GEN_TEST_DATA from test.test_serialization_doclang import ( _doc_cross_column_list, _doc_cross_page_list, _doc_cross_page_paragraph, _doc_cross_page_table, _doc_multi_prov_text, _verify_doc, add_list_section, add_texts_section, verify_doclang, ) from test.test_serialization_doctag import verify DO_PRINT: bool = False def _serialize(doc: DoclingDocument) -> str: ser = DocLangDocSerializer( doc=doc, params=DocLangParams(include_version=False), ) text = ser.serialize().text if not GEN_TEST_DATA: assert_valid_dclg_xml(text) return text def _deserialize(text: str, *, validate: bool = True) -> DoclingDocument: if validate and not GEN_TEST_DATA: assert_valid_dclg_xml(text) return DocLangDocDeserializer().deserialize_str(text) def _add_default_page(doc: DoclingDocument): doc.add_page(page_no=0, size=Size(width=1000, height=1000)) def _default_prov() -> ProvenanceItem: return ProvenanceItem( page_no=0, bbox=BoundingBox(l=100, t=100, r=300, b=200), charspan=(0, 0), ) class _VirtualTextMixedBboxFactory: """Assign distinct page bboxes for the virtual-text mixed fixture.""" def __init__(self, *, page_no: int = 0, page_w: float = 1000, page_h: float = 1000) -> None: self.page_no = page_no self.page_w = page_w self.page_h = page_h self._slot = 0 def next_prov(self) -> ProvenanceItem: row, col = divmod(self._slot, 4) self._slot += 1 cell_w = self.page_w / 5 cell_h = self.page_h / 20 left = 20 + col * cell_w top = self.page_h - 40 - row * cell_h return ProvenanceItem( page_no=self.page_no, bbox=BoundingBox.from_tuple( (left, top - cell_h, left + cell_w - 10, top), origin=CoordOrigin.BOTTOMLEFT, ), charspan=(0, 0), ) def next_bbox(self) -> BoundingBox: return self.next_prov().bbox def _serialize_virtual_text_mixed(doc: DoclingDocument, *, add_location: bool = True) -> str: ser = DocLangDocSerializer( doc=doc, params=DocLangParams(include_version=False, add_table_cell_location=True, add_location=add_location), ) text = ser.serialize().text if not GEN_TEST_DATA: assert_valid_dclg_xml(text) return text def test_roundtrip_text(): doc = DoclingDocument(name="t") doc.add_text(label=DocItemLabel.TEXT, text="Hello world") dt = _serialize(doc) doc2 = _deserialize(dt) dt2 = _serialize(doc2) exp_dt = """ Hello world """ assert dt2.strip() == exp_dt.strip() def test_deserialize_include_namespace_and_version(): """Deserialize DocLang XML with namespace and version, then roundtrip.""" exp_file = Path("./test/data/doc/deserialize_include_namespace_and_version.gt.dclg.xml") xml = exp_file.read_text(encoding="utf-8") doc = _deserialize(xml) assert len(doc.texts) == 1 assert doc.texts[0].text == "Hello world" reserialized = ( DocLangDocSerializer( doc=doc, params=DocLangParams(include_namespace=True, include_version=True), ) .serialize() .text ) verify_doclang(exp_file=exp_file, actual=reserialized) def test_roundtrip_title(): doc = DoclingDocument(name="t") doc.add_title(text="My Title") dt = _serialize(doc) doc2 = _deserialize(dt) dt2 = _serialize(doc2) exp_dt = """ My Title """ assert dt2.strip() == exp_dt.strip() def test_roundtrip_heading(): doc = DoclingDocument(name="t") doc.add_heading(text="Section A", level=2) dt = _serialize(doc) doc2 = _deserialize(dt) dt2 = _serialize(doc2) exp_dt = """ Section A """ assert dt2.strip() == exp_dt.strip() def test_roundtrip_caption(): doc = DoclingDocument(name="t") doc.add_text(label=DocItemLabel.CAPTION, text="Cap text") dt = _serialize(doc) doc2 = _deserialize(dt) dt2 = _serialize(doc2) exp_dt = """ Cap text """ assert dt2.strip() == exp_dt.strip() def test_roundtrip_footnote(): doc = DoclingDocument(name="t") doc.add_text(label=DocItemLabel.FOOTNOTE, text="Foot note") dt = _serialize(doc) if DO_PRINT: print("\n", dt) doc2 = _deserialize(dt) assert len(doc2.texts) == 1 assert doc2.texts[0].label == DocItemLabel.FOOTNOTE assert doc2.texts[0].text == "Foot note" def test_roundtrip_page_header(): doc = DoclingDocument(name="t") doc.add_text(label=DocItemLabel.PAGE_HEADER, text="Header") dt = _serialize(doc) if DO_PRINT: print("\n", dt) doc2 = _deserialize(dt) assert len(doc2.texts) == 1 assert doc2.texts[0].label == DocItemLabel.PAGE_HEADER assert doc2.texts[0].text == "Header" def test_roundtrip_page_footer(): doc = DoclingDocument(name="t") doc.add_text(label=DocItemLabel.PAGE_FOOTER, text="Footer") dt = _serialize(doc) if DO_PRINT: print("\n", dt) doc2 = _deserialize(dt) assert len(doc2.texts) == 1 assert doc2.texts[0].label == DocItemLabel.PAGE_FOOTER assert doc2.texts[0].text == "Footer" def test_roundtrip_code(): doc = DoclingDocument(name="t") doc.add_code(text="print('hi')", code_language=CodeLanguageLabel.PYTHON) doc.add_code(text='disp("Hello world!")', code_language=CodeLanguageLabel.OCTAVE) dt = _serialize(doc) doc2 = _deserialize(dt) dt2 = _serialize(doc2) exp_dt = """ """ assert dt2.strip() == exp_dt.strip() def test_roundtrip_code_python_and_unknown(): doc = DoclingDocument(name="t") doc.add_code(text="x = 1", code_language=CodeLanguageLabel.PYTHON) doc.add_code(text="y = 2") # CodeLanguageLabel.UNKNOWN by default xml = _serialize(doc) assert '