"""Unit tests for Doclang create_closing_token helper.""" import warnings import xml.etree.ElementTree as ET from itertools import chain from pathlib import Path from typing import Optional import pytest from pydantic import AnyUrl from docling_core.transforms.serializer._doclang_utils import ( _create_location_tokens_for_bbox, _quantize_to_resolution, ) from docling_core.transforms.serializer.doclang import ( ContentType, DocLangDocSerializer, DocLangParams, EscapeMode, LabelMode, LayerMode, WrapMode, ) from docling_core.types.doc import ( BaseMeta, BoundingBox, CodeLanguageLabel, CoordOrigin, DescriptionMetaField, DocItemLabel, DoclingDocument, Formatting, PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, ProvenanceItem, Script, Size, SummaryMetaField, TableData, TabularChartMetaField, ) from docling_core.types.doc.base import ImageRefMode from docling_core.types.doc.document import ( ContentLayer, GraphCell, GraphData, GraphLink, ImageRef, RichTableCell, TableCell, ) from docling_core.types.doc.labels import GraphCellLabel, GraphLinkLabel from test.doclang_validation import ( doclang_validator, validate_dclg_xml, ) from test.test_data_gen_flag import GEN_TEST_DATA from test.test_serialization import verify def verify_doclang(exp_file: Path, actual: str) -> None: """Golden compare plus reference DocLang validation.""" verify(exp_file=exp_file, actual=actual) if not GEN_TEST_DATA: validate_dclg_xml(actual) def add_texts_section(doc: DoclingDocument): doc.add_text(label=DocItemLabel.TEXT, text="Simple text") inline1 = doc.add_inline_group() doc.add_text( label=DocItemLabel.TEXT, text="Here a code snippet: ", parent=inline1, ) doc.add_code( text="help()", parent=inline1, code_language=CodeLanguageLabel.PYTHON, ) doc.add_text( label=DocItemLabel.TEXT, text=" (to be shown)", parent=inline1, ) def add_list_section(doc: DoclingDocument): doc.add_page(page_no=1, size=Size(width=100, height=100), image=None) prov = ProvenanceItem( page_no=1, bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT), charspan=(0, 2), ) lg = doc.add_list_group() doc.add_list_item(text="foo", parent=lg) doc.add_list_item(text="bar", parent=lg) # just inline group with a formula li = doc.add_list_item(text="", parent=lg) inline = doc.add_inline_group(parent=li) doc.add_text( label=DocItemLabel.TEXT, text="Here a formula: ", parent=inline, ) doc.add_formula(text="E=mc^2 ", parent=inline) doc.add_text( label=DocItemLabel.TEXT, text="in line", parent=inline, ) # just inline group with formatted span li = doc.add_list_item(text="", parent=lg) inline = doc.add_inline_group(parent=li) doc.add_text( label=DocItemLabel.TEXT, text="Here a ", parent=inline, ) doc.add_text( label=DocItemLabel.TEXT, text="bold", parent=inline, formatting=Formatting(bold=True), ) doc.add_text( label=DocItemLabel.TEXT, text=" text", parent=inline, ) li = doc.add_list_item(text="will contain sublist", parent=lg) lg_sub = doc.add_list_group(parent=li) doc.add_list_item(text="sublist item 1", parent=lg_sub) doc.add_list_item(text="sublist item 2", parent=lg_sub) li = doc.add_list_item(text="", parent=lg, prov=prov) inline = doc.add_inline_group(parent=li) doc.add_text( label=DocItemLabel.TEXT, text="Here a ", parent=inline, ) doc.add_text( label=DocItemLabel.TEXT, text="both bold and italicized", parent=inline, formatting=Formatting(bold=True, italic=True), ) doc.add_text( label=DocItemLabel.TEXT, text=" text and a sublist:", parent=inline, ) lg_sub = doc.add_list_group(parent=li) doc.add_list_item(text="sublist item a", parent=lg_sub) doc.add_list_item(text="sublist item b", parent=lg_sub) doc.add_list_item(text="final element", parent=lg) # =============================== # Doclang tests # =============================== def serialize_doclang(doc: DoclingDocument, params: Optional[DocLangParams] = None) -> str: ser = DocLangDocSerializer(doc=doc, params=params or DocLangParams(include_version=False)) text = ser.serialize().text if not GEN_TEST_DATA: validate_dclg_xml(text) return text def test_list_items_not_double_wrapped_when_no_content(): doc = DoclingDocument(name="t") lst = doc.add_list_group() doc.add_list_item("Item A", parent=lst) doc.add_list_item("Item B", parent=lst) txt = serialize_doclang(doc, params=DocLangParams(include_version=False, content_types=set())) exp_txt = """ """ assert txt.strip() == exp_txt.strip() def test_doclang(): src = Path("./test/data/doc/ddoc_0.json") doc = DoclingDocument.load_from_json(src) # Human readable, indented and with content params = DocLangParams(include_version=False) ser = DocLangDocSerializer(doc=doc, params=params) actual = ser.serialize().text verify_doclang(exp_file=src.with_suffix(".v0.gt.dclg.xml"), actual=actual) # Human readable, indented but without content ser = DocLangDocSerializer( doc=doc, params=DocLangParams( include_version=False, content_types={ContentType.TABLE}, ), ) actual = ser.serialize().text verify_doclang(exp_file=src.with_suffix(".v1.gt.dclg.xml"), actual=actual) # Machine readable, not indented and without content ser = DocLangDocSerializer( doc=doc, params=DocLangParams( include_version=False, pretty_indentation=None, content_types={ContentType.TABLE}, ), ) actual = ser.serialize().text verify_doclang(exp_file=src.with_suffix(".v2.gt.dclg.xml"), actual=actual) def test_doclang_meta(): src = Path("./test/data/doc/dummy_doc_with_meta.yaml") doc = DoclingDocument.load_from_yaml(src) ser = DocLangDocSerializer( doc=doc, params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED), ) actual = ser.serialize().text verify_doclang(exp_file=src.with_suffix(".gt.dclg.xml"), actual=actual) def test_doclang_crop_embedded(): src = Path("./test/data/doc/activities_simplified.yaml") doc = DoclingDocument.load_from_yaml(src) serializer = DocLangDocSerializer( doc=doc, params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED), ) actual = serializer.serialize().text # verifying everything except base64 data as the latter seems to be flaky across runs/platforms exp_prefix = """ """.strip() assert actual.endswith(exp_suffix) if not GEN_TEST_DATA: validate_dclg_xml(actual) def test_doclang_crop_placeholder(): src = Path("./test/data/doc/activities_simplified.yaml") doc = DoclingDocument.load_from_yaml(src) serializer = DocLangDocSerializer( doc=doc, params=DocLangParams(include_version=False, image_mode=ImageRefMode.PLACEHOLDER), ) actual = serializer.serialize().text exp_file = src.parent / f"{src.stem}_cropped_placeholder.dclg.xml" verify_doclang(exp_file=exp_file, actual=actual) def _create_escape_test_doc(inp_doc: DoclingDocument): doc = inp_doc.model_copy(deep=True) doc.add_text(label=DocItemLabel.TEXT, text="Simple text") doc.add_text(label=DocItemLabel.TEXT, text=" 4 leading spaces, 1 trailing ") doc.add_text(label=DocItemLabel.TEXT, text="Some 'single' quotes") doc.add_text(label=DocItemLabel.TEXT, text='Some "double" quotes') text_item = doc.add_text(label=DocItemLabel.TEXT, text="An ampersand: &") text_item.meta = BaseMeta( summary=SummaryMetaField(text="Summary with & ampersands"), ) pic = doc.add_picture() pic.meta = PictureMeta( description=DescriptionMetaField(text="Description content"), ) doc.add_code(text="0 == 0") doc.add_code(text=" 1 leading space, 4 trailing ") doc.add_code(text="0 < 1") doc.add_code(text="42 == 42", code_language=CodeLanguageLabel.PYTHON) doc.add_code(text="42 < 1337", code_language=CodeLanguageLabel.PYTHON) td = TableData(num_cols=2) td.add_row(["Foo", "Bar"]) td.add_row(["Header & Title", "Value > 100"]) td.add_row(["