"""Unit tests for Doclang create_closing_token helper."""
import warnings
import xml.etree.ElementTree as ET
from itertools import chain
from pathlib import Path
from typing import Optional
import pytest
from pydantic import AnyUrl
from docling_core.transforms.serializer._doclang_utils import (
_create_location_tokens_for_bbox,
_quantize_to_resolution,
)
from docling_core.transforms.serializer.doclang import (
ContentType,
DocLangDocSerializer,
DocLangParams,
EscapeMode,
LabelMode,
LayerMode,
WrapMode,
)
from docling_core.types.doc import (
BaseMeta,
BoundingBox,
CodeLanguageLabel,
CoordOrigin,
DescriptionMetaField,
DocItemLabel,
DoclingDocument,
Formatting,
PictureClassificationLabel,
PictureClassificationMetaField,
PictureClassificationPrediction,
PictureMeta,
ProvenanceItem,
Script,
Size,
SummaryMetaField,
TableData,
TabularChartMetaField,
)
from docling_core.types.doc.base import ImageRefMode
from docling_core.types.doc.document import (
ContentLayer,
GraphCell,
GraphData,
GraphLink,
ImageRef,
RichTableCell,
TableCell,
)
from docling_core.types.doc.labels import GraphCellLabel, GraphLinkLabel
from tests.doclang_validation import (
doclang_validator,
validate_dclg_xml,
)
from tests.test_data_gen_flag import GEN_TEST_DATA
from tests.test_serialization import verify
def verify_doclang(exp_file: Path, actual: str) -> None:
"""Golden compare plus reference DocLang validation."""
verify(exp_file=exp_file, actual=actual)
if not GEN_TEST_DATA:
validate_dclg_xml(actual)
def add_texts_section(doc: DoclingDocument):
doc.add_text(label=DocItemLabel.TEXT, text="Simple text")
inline1 = doc.add_inline_group()
doc.add_text(
label=DocItemLabel.TEXT,
text="Here a code snippet: ",
parent=inline1,
)
doc.add_code(
text="help()",
parent=inline1,
code_language=CodeLanguageLabel.PYTHON,
)
doc.add_text(
label=DocItemLabel.TEXT,
text=" (to be shown)",
parent=inline1,
)
def add_list_section(doc: DoclingDocument):
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
prov = ProvenanceItem(
page_no=1,
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
charspan=(0, 2),
)
lg = doc.add_list_group()
doc.add_list_item(text="foo", parent=lg)
doc.add_list_item(text="bar", parent=lg)
# just inline group with a formula
li = doc.add_list_item(text="", parent=lg)
inline = doc.add_inline_group(parent=li)
doc.add_text(
label=DocItemLabel.TEXT,
text="Here a formula: ",
parent=inline,
)
doc.add_formula(text="E=mc^2 ", parent=inline)
doc.add_text(
label=DocItemLabel.TEXT,
text="in line",
parent=inline,
)
# just inline group with formatted span
li = doc.add_list_item(text="", parent=lg)
inline = doc.add_inline_group(parent=li)
doc.add_text(
label=DocItemLabel.TEXT,
text="Here a ",
parent=inline,
)
doc.add_text(
label=DocItemLabel.TEXT,
text="bold",
parent=inline,
formatting=Formatting(bold=True),
)
doc.add_text(
label=DocItemLabel.TEXT,
text=" text",
parent=inline,
)
li = doc.add_list_item(text="will contain sublist", parent=lg)
lg_sub = doc.add_list_group(parent=li)
doc.add_list_item(text="sublist item 1", parent=lg_sub)
doc.add_list_item(text="sublist item 2", parent=lg_sub)
li = doc.add_list_item(text="", parent=lg, prov=prov)
inline = doc.add_inline_group(parent=li)
doc.add_text(
label=DocItemLabel.TEXT,
text="Here a ",
parent=inline,
)
doc.add_text(
label=DocItemLabel.TEXT,
text="both bold and italicized",
parent=inline,
formatting=Formatting(bold=True, italic=True),
)
doc.add_text(
label=DocItemLabel.TEXT,
text=" text and a sublist:",
parent=inline,
)
lg_sub = doc.add_list_group(parent=li)
doc.add_list_item(text="sublist item a", parent=lg_sub)
doc.add_list_item(text="sublist item b", parent=lg_sub)
doc.add_list_item(text="final element", parent=lg)
# ===============================
# Doclang tests
# ===============================
def serialize_doclang(doc: DoclingDocument, params: DocLangParams | None = None) -> str:
ser = DocLangDocSerializer(doc=doc, params=params or DocLangParams(include_version=False))
text = ser.serialize().text
if not GEN_TEST_DATA:
validate_dclg_xml(text)
return text
def test_list_items_not_double_wrapped_when_no_content():
doc = DoclingDocument(name="t")
lst = doc.add_list_group()
doc.add_list_item("Item A", parent=lst)
doc.add_list_item("Item B", parent=lst)
txt = serialize_doclang(doc, params=DocLangParams(include_version=False, content_types=set()))
exp_txt = """
"""
assert txt.strip() == exp_txt.strip()
def test_doclang():
src = Path("./tests/data/doc/ddoc_0.json")
doc = DoclingDocument.load_from_json(src)
# Human readable, indented and with content
params = DocLangParams(include_version=False)
ser = DocLangDocSerializer(doc=doc, params=params)
actual = ser.serialize().text
verify_doclang(exp_file=src.with_suffix(".v0.gt.dclg.xml"), actual=actual)
# Human readable, indented but without content
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(
include_version=False,
content_types={ContentType.TABLE},
),
)
actual = ser.serialize().text
verify_doclang(exp_file=src.with_suffix(".v1.gt.dclg.xml"), actual=actual)
# Machine readable, not indented and without content
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(
include_version=False,
pretty_indentation=None,
content_types={ContentType.TABLE},
),
)
actual = ser.serialize().text
verify_doclang(exp_file=src.with_suffix(".v2.gt.dclg.xml"), actual=actual)
def test_doclang_meta():
src = Path("./tests/data/doc/dummy_doc_with_meta.yaml")
doc = DoclingDocument.load_from_yaml(src)
ser = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED),
)
actual = ser.serialize().text
verify_doclang(exp_file=src.with_suffix(".gt.dclg.xml"), actual=actual)
def test_doclang_crop_embedded():
src = Path("./tests/data/doc/activities_simplified.yaml")
doc = DoclingDocument.load_from_yaml(src)
serializer = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED),
)
actual = serializer.serialize().text
# verifying everything except base64 data as the latter seems to be flaky across runs/platforms
exp_prefix = """
""".strip()
assert actual.endswith(exp_suffix)
if not GEN_TEST_DATA:
validate_dclg_xml(actual)
def test_doclang_crop_placeholder():
src = Path("./tests/data/doc/activities_simplified.yaml")
doc = DoclingDocument.load_from_yaml(src)
serializer = DocLangDocSerializer(
doc=doc,
params=DocLangParams(include_version=False, image_mode=ImageRefMode.PLACEHOLDER),
)
actual = serializer.serialize().text
exp_file = src.parent / f"{src.stem}_cropped_placeholder.dclg.xml"
verify_doclang(exp_file=exp_file, actual=actual)
def _create_escape_test_doc(inp_doc: DoclingDocument):
doc = inp_doc.model_copy(deep=True)
doc.add_text(label=DocItemLabel.TEXT, text="Simple text")
doc.add_text(label=DocItemLabel.TEXT, text=" 4 leading spaces, 1 trailing ")
doc.add_text(label=DocItemLabel.TEXT, text="Some 'single' quotes")
doc.add_text(label=DocItemLabel.TEXT, text='Some "double" quotes')
text_item = doc.add_text(label=DocItemLabel.TEXT, text="An ampersand: &")
text_item.meta = BaseMeta(
summary=SummaryMetaField(text="Summary with & ampersands"),
)
pic = doc.add_picture()
pic.meta = PictureMeta(
description=DescriptionMetaField(text="Description content"),
)
doc.add_code(text="0 == 0")
doc.add_code(text=" 1 leading space, 4 trailing ")
doc.add_code(text="0 < 1")
doc.add_code(text="42 == 42", code_language=CodeLanguageLabel.PYTHON)
doc.add_code(text="42 < 1337", code_language=CodeLanguageLabel.PYTHON)
td = TableData(num_cols=2)
td.add_row(["Foo", "Bar"])
td.add_row(["Header & Title", "Value > 100"])
td.add_row(["