mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* test(doclang): add serialization test for multiple captions & footnotes Add DoclingDocument fixtures with pictures, a table and a code block carrying multiple captions and footnotes: above, below and beside their host, interleaved, and without provenance. A mirrored fixture has them at body level instead of as children of their host. Add a golden-file test for their DocLang serialization. Multiple captions are currently merged into a single <caption> element with several locations, which fails DocLang schema validation and breaks the deserialize/reserialize roundtrip. Until that is fixed, the test compares against the golden file without schema validation, and the roundtrip steps are commented out. Signed-off-by: Panos Vagenas <pva@zurich.ibm.com> * fix(doclang): serialize multiple captions and footnotes of a float as satellites All captions of a picture/table were merged into its single <caption>, yielding invalid DocLang, while those of a code block were escaped into its content. Keep only the first caption in the element head and emit any further captions as <text> and footnotes as <footnote> next to the host, placed before or after it by bbox position, all wrapped in a <group>. On deserialization, read such a group back as the float with its captions and footnotes (a convention until DocLang supports this). Enable schema validation and the roundtrip in the corresponding test. Signed-off-by: Panos Vagenas <pva@zurich.ibm.com> --------- Signed-off-by: Panos Vagenas <pva@zurich.ibm.com>
2814 lines
94 KiB
Python
2814 lines
94 KiB
Python
"""Unit tests for Doclang create_closing_token helper."""
|
|
|
|
import warnings
|
|
import xml.etree.ElementTree as ET
|
|
from itertools import chain
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import pytest
|
|
from pydantic import AnyUrl
|
|
|
|
from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer
|
|
from docling_core.transforms.serializer._doclang_utils import (
|
|
_create_location_tokens_for_bbox,
|
|
_quantize_to_resolution,
|
|
)
|
|
from docling_core.transforms.serializer.doclang import (
|
|
ContentType,
|
|
DocLangDocSerializer,
|
|
DocLangParams,
|
|
EscapeMode,
|
|
LabelMode,
|
|
LayerMode,
|
|
WrapMode,
|
|
)
|
|
from docling_core.types.doc import (
|
|
BaseMeta,
|
|
BoundingBox,
|
|
CodeLanguageLabel,
|
|
CoordOrigin,
|
|
DescriptionMetaField,
|
|
DocItemLabel,
|
|
DoclingDocument,
|
|
Formatting,
|
|
PictureClassificationLabel,
|
|
PictureClassificationMetaField,
|
|
PictureClassificationPrediction,
|
|
PictureMeta,
|
|
ProvenanceItem,
|
|
Script,
|
|
Size,
|
|
SummaryMetaField,
|
|
TableData,
|
|
TabularChartMetaField,
|
|
)
|
|
from docling_core.types.doc.base import ImageRefMode
|
|
from docling_core.types.doc.document import (
|
|
ContentLayer,
|
|
GraphCell,
|
|
GraphData,
|
|
GraphLink,
|
|
ImageRef,
|
|
RichTableCell,
|
|
TableCell,
|
|
)
|
|
from docling_core.types.doc.labels import GraphCellLabel, GraphLinkLabel
|
|
from tests.doclang_validation import (
|
|
doclang_validator,
|
|
validate_dclg_xml,
|
|
)
|
|
from tests.test_data_gen_flag import GEN_TEST_DATA
|
|
from tests.test_serialization import verify
|
|
|
|
|
|
def verify_doclang(exp_file: Path, actual: str) -> None:
|
|
"""Golden compare plus reference DocLang validation."""
|
|
verify(exp_file=exp_file, actual=actual)
|
|
if not GEN_TEST_DATA:
|
|
validate_dclg_xml(actual)
|
|
|
|
|
|
def add_texts_section(doc: DoclingDocument):
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Simple text")
|
|
inline1 = doc.add_inline_group()
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Here a code snippet: ",
|
|
parent=inline1,
|
|
)
|
|
doc.add_code(
|
|
text="help()",
|
|
parent=inline1,
|
|
code_language=CodeLanguageLabel.PYTHON,
|
|
)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=" (to be shown)",
|
|
parent=inline1,
|
|
)
|
|
|
|
|
|
def add_list_section(doc: DoclingDocument):
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
lg = doc.add_list_group()
|
|
|
|
doc.add_list_item(text="foo", parent=lg)
|
|
doc.add_list_item(text="bar", parent=lg)
|
|
|
|
# just inline group with a formula
|
|
li = doc.add_list_item(text="", parent=lg)
|
|
inline = doc.add_inline_group(parent=li)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Here a formula: ",
|
|
parent=inline,
|
|
)
|
|
doc.add_formula(text="E=mc^2 ", parent=inline)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="in line",
|
|
parent=inline,
|
|
)
|
|
|
|
# just inline group with formatted span
|
|
li = doc.add_list_item(text="", parent=lg)
|
|
inline = doc.add_inline_group(parent=li)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Here a ",
|
|
parent=inline,
|
|
)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="bold",
|
|
parent=inline,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=" text",
|
|
parent=inline,
|
|
)
|
|
|
|
li = doc.add_list_item(text="will contain sublist", parent=lg)
|
|
lg_sub = doc.add_list_group(parent=li)
|
|
doc.add_list_item(text="sublist item 1", parent=lg_sub)
|
|
doc.add_list_item(text="sublist item 2", parent=lg_sub)
|
|
|
|
li = doc.add_list_item(text="", parent=lg, prov=prov)
|
|
inline = doc.add_inline_group(parent=li)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Here a ",
|
|
parent=inline,
|
|
)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="both bold and italicized",
|
|
parent=inline,
|
|
formatting=Formatting(bold=True, italic=True),
|
|
)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=" text and a sublist:",
|
|
parent=inline,
|
|
)
|
|
lg_sub = doc.add_list_group(parent=li)
|
|
doc.add_list_item(text="sublist item a", parent=lg_sub)
|
|
doc.add_list_item(text="sublist item b", parent=lg_sub)
|
|
|
|
doc.add_list_item(text="final element", parent=lg)
|
|
|
|
|
|
# ===============================
|
|
# Doclang tests
|
|
# ===============================
|
|
|
|
|
|
def serialize_doclang(doc: DoclingDocument, params: DocLangParams | None = None) -> str:
|
|
ser = DocLangDocSerializer(doc=doc, params=params or DocLangParams(include_version=False))
|
|
text = ser.serialize().text
|
|
if not GEN_TEST_DATA:
|
|
validate_dclg_xml(text)
|
|
return text
|
|
|
|
|
|
def test_list_items_not_double_wrapped_when_no_content():
|
|
doc = DoclingDocument(name="t")
|
|
lst = doc.add_list_group()
|
|
doc.add_list_item("Item A", parent=lst)
|
|
doc.add_list_item("Item B", parent=lst)
|
|
|
|
txt = serialize_doclang(doc, params=DocLangParams(include_version=False, content_types=set()))
|
|
exp_txt = """
|
|
<doclang>
|
|
<list>
|
|
<ldiv/>
|
|
<ldiv/>
|
|
</list>
|
|
</doclang>
|
|
"""
|
|
assert txt.strip() == exp_txt.strip()
|
|
|
|
|
|
def test_doclang():
|
|
src = Path("./tests/data/doc/ddoc_0.json")
|
|
doc = DoclingDocument.load_from_json(src)
|
|
|
|
# Human readable, indented and with content
|
|
params = DocLangParams(include_version=False)
|
|
|
|
ser = DocLangDocSerializer(doc=doc, params=params)
|
|
actual = ser.serialize().text
|
|
|
|
verify_doclang(exp_file=src.with_suffix(".v0.gt.dclg.xml"), actual=actual)
|
|
|
|
# Human readable, indented but without content
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_types={ContentType.TABLE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
|
|
verify_doclang(exp_file=src.with_suffix(".v1.gt.dclg.xml"), actual=actual)
|
|
|
|
# Machine readable, not indented and without content
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
pretty_indentation=None,
|
|
content_types={ContentType.TABLE},
|
|
),
|
|
)
|
|
actual = ser.serialize().text
|
|
|
|
verify_doclang(exp_file=src.with_suffix(".v2.gt.dclg.xml"), actual=actual)
|
|
|
|
|
|
def test_doclang_meta():
|
|
src = Path("./tests/data/doc/dummy_doc_with_meta.yaml")
|
|
doc = DoclingDocument.load_from_yaml(src)
|
|
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED),
|
|
)
|
|
actual = ser.serialize().text
|
|
verify_doclang(exp_file=src.with_suffix(".gt.dclg.xml"), actual=actual)
|
|
|
|
|
|
def test_doclang_crop_embedded():
|
|
src = Path("./tests/data/doc/activities_simplified.yaml")
|
|
doc = DoclingDocument.load_from_yaml(src)
|
|
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED),
|
|
)
|
|
actual = serializer.serialize().text
|
|
|
|
# verifying everything except base64 data as the latter seems to be flaky across runs/platforms
|
|
exp_prefix = """
|
|
<doclang>
|
|
<picture>
|
|
<label value="other"/>
|
|
<location value="43"/>
|
|
<location value="117"/>
|
|
<location value="172"/>
|
|
<location value="208"/>
|
|
<src uri="data:image/png;base64,
|
|
""".strip()
|
|
assert actual.startswith(exp_prefix)
|
|
|
|
exp_suffix = """
|
|
</picture>
|
|
</doclang>
|
|
""".strip()
|
|
assert actual.endswith(exp_suffix)
|
|
if not GEN_TEST_DATA:
|
|
validate_dclg_xml(actual)
|
|
|
|
|
|
def test_doclang_crop_placeholder():
|
|
src = Path("./tests/data/doc/activities_simplified.yaml")
|
|
doc = DoclingDocument.load_from_yaml(src)
|
|
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, image_mode=ImageRefMode.PLACEHOLDER),
|
|
)
|
|
actual = serializer.serialize().text
|
|
exp_file = src.parent / f"{src.stem}_cropped_placeholder.dclg.xml"
|
|
verify_doclang(exp_file=exp_file, actual=actual)
|
|
|
|
|
|
def _create_escape_test_doc(inp_doc: DoclingDocument):
|
|
doc = inp_doc.model_copy(deep=True)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Simple text")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" 4 leading spaces, 1 trailing ")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Some 'single' quotes")
|
|
doc.add_text(label=DocItemLabel.TEXT, text='Some "double" quotes')
|
|
text_item = doc.add_text(label=DocItemLabel.TEXT, text="An ampersand: &")
|
|
text_item.meta = BaseMeta(
|
|
summary=SummaryMetaField(text="Summary with <tags> & ampersands"),
|
|
)
|
|
pic = doc.add_picture()
|
|
pic.meta = PictureMeta(
|
|
description=DescriptionMetaField(text="Description content"),
|
|
)
|
|
doc.add_code(text="0 == 0")
|
|
doc.add_code(text=" 1 leading space, 4 trailing ")
|
|
doc.add_code(text="0 < 1")
|
|
doc.add_code(text="42 == 42", code_language=CodeLanguageLabel.PYTHON)
|
|
doc.add_code(text="42 < 1337", code_language=CodeLanguageLabel.PYTHON)
|
|
|
|
td = TableData(num_cols=2)
|
|
td.add_row(["Foo", "Bar"])
|
|
td.add_row(["Header & Title", "Value > 100"])
|
|
td.add_row(["<script>", "A & B"])
|
|
td.add_row(["Only", "<second>"])
|
|
doc.add_table(data=td)
|
|
|
|
# test combination of formatting and special characters
|
|
doc.add_text(label=DocItemLabel.TEXT, text="0 < 1")
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="0 < 42",
|
|
formatting=Formatting(bold=True, italic=True),
|
|
)
|
|
|
|
return doc
|
|
|
|
|
|
@pytest.mark.parametrize("pretty_indentation", [None, " "])
|
|
def test_doclang_replaces_xml_illegal_characters(pretty_indentation: str | None):
|
|
"""DocLang output remains valid when document text contains XML-illegal controls."""
|
|
doc = DoclingDocument(name="xml_illegal_character")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Before break\x0bAfter break")
|
|
|
|
result = (
|
|
DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
add_location=False,
|
|
pretty_indentation=pretty_indentation,
|
|
),
|
|
)
|
|
.serialize()
|
|
.text
|
|
)
|
|
|
|
ET.fromstring(result)
|
|
assert "Before break[U+000B]After break" in result
|
|
assert "\x0b" not in result
|
|
|
|
|
|
def test_cdata_always(sample_doc: DoclingDocument):
|
|
"""Test escape_mode=ALWAYS."""
|
|
doc = _create_escape_test_doc(sample_doc)
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
escape_mode=EscapeMode.ALWAYS,
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_res = serializer.serialize()
|
|
ser_txt = ser_res.text
|
|
|
|
exp_file = Path("./tests/data/doc/cdata_always.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_cdata_when_needed(sample_doc: DoclingDocument):
|
|
"""Test escape_mode=AUTO."""
|
|
doc = _create_escape_test_doc(sample_doc)
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
escape_mode=EscapeMode.AUTO,
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_res = serializer.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/cdata_when_needed.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_deep_section_header_levels_clamp_instead_of_raising():
|
|
"""Section headers deeper than the heading vocabulary allows clamp to level 6.
|
|
|
|
``SectionHeaderItem.level`` accepts up to 100 and docling's own heading
|
|
hierarchy inference emits level-6 headings by default, but the serializer
|
|
shifts by one (level 1 is the title), so level 6 used to overflow the
|
|
``heading`` token's [1, 6] range and raise. ``html.py`` already clamps the
|
|
same ``item.level + 1`` computation.
|
|
"""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_heading(text="Top", level=1)
|
|
doc.add_heading(text="Deep", level=6)
|
|
doc.add_heading(text="Deeper", level=42)
|
|
|
|
result = serialize_doclang(doc, params=DocLangParams(include_version=False, add_location=False))
|
|
|
|
assert '<heading level="2">Top</heading>' in result
|
|
assert '<heading level="6">Deep</heading>' in result
|
|
assert '<heading level="6">Deeper</heading>' in result
|
|
|
|
|
|
def test_strikethrough_formatting():
|
|
"""Test strikethrough formatting serialization."""
|
|
doc = DoclingDocument(name="test")
|
|
formatting = Formatting(strikethrough=True)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Strike text", formatting=formatting)
|
|
|
|
result = serialize_doclang(doc, params=DocLangParams(include_version=False, add_location=False))
|
|
assert "<strikethrough>Strike text</strikethrough>" in result
|
|
|
|
|
|
def test_subscript_formatting():
|
|
"""Test subscript formatting serialization."""
|
|
doc = DoclingDocument(name="test")
|
|
formatting = Formatting(script=Script.SUB)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="H2O", formatting=formatting)
|
|
|
|
result = serialize_doclang(doc, params=DocLangParams(include_version=False, add_location=False))
|
|
assert "<subscript>H2O</subscript>" in result
|
|
|
|
|
|
def test_superscript_formatting():
|
|
"""Test superscript formatting serialization."""
|
|
doc = DoclingDocument(name="test")
|
|
formatting = Formatting(script=Script.SUPER)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="x^2", formatting=formatting)
|
|
|
|
result = serialize_doclang(doc, params=DocLangParams(include_version=False, add_location=False))
|
|
assert "<superscript>x^2</superscript>" in result
|
|
|
|
|
|
def test_combined_formatting():
|
|
"""Test combined formatting (bold + italic)."""
|
|
doc = DoclingDocument(name="test")
|
|
formatting = Formatting(bold=True, italic=True)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Bold and italic", formatting=formatting)
|
|
|
|
result = serialize_doclang(doc, params=DocLangParams(include_version=False, add_location=False))
|
|
# When both bold and italic are applied, they should be nested
|
|
assert "<bold>" in result
|
|
assert "<italic>" in result
|
|
assert "Bold and italic" in result
|
|
|
|
|
|
def _create_content_filtering_doc(inp_doc: DoclingDocument):
|
|
doc = inp_doc.model_copy(deep=True)
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
pic = doc.add_picture(caption=doc.add_text(label=DocItemLabel.CAPTION, text="Picture Caption"))
|
|
pic.prov = [prov]
|
|
pic.meta = PictureMeta(
|
|
summary=SummaryMetaField(text="Picture Summary"),
|
|
description=DescriptionMetaField(text="Picture Description"),
|
|
)
|
|
|
|
chart = doc.add_picture(caption=doc.add_text(label=DocItemLabel.CAPTION, text="Picture Caption"))
|
|
chart.prov = [prov]
|
|
chart.meta = PictureMeta(
|
|
summary=SummaryMetaField(text="Picture Summary"),
|
|
description=DescriptionMetaField(text="Picture Description"),
|
|
classification=PictureClassificationMetaField(
|
|
predictions=[
|
|
PictureClassificationPrediction(
|
|
class_name=PictureClassificationLabel.PIE_CHART.value,
|
|
confidence=1.0,
|
|
)
|
|
]
|
|
),
|
|
)
|
|
chart_data = TableData(num_cols=2)
|
|
chart_data.add_row(["Foo", "Bar"])
|
|
chart_data.add_row(["One", "Two"])
|
|
chart.meta.tabular_chart = TabularChartMetaField(
|
|
title="Chart Title",
|
|
chart_data=chart_data,
|
|
)
|
|
doc.add_code(text="0 == 0")
|
|
doc.add_code(text="with location", prov=prov)
|
|
|
|
return doc
|
|
|
|
|
|
def test_handwritten_text_label(doc_with_handwritten: DoclingDocument):
|
|
result = serialize_doclang(doc_with_handwritten)
|
|
exp_file = Path("./tests/data/doc/handwritten_text.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=result)
|
|
|
|
|
|
def test_content_allow_all_types(sample_doc: DoclingDocument):
|
|
doc = _create_content_filtering_doc(sample_doc)
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_types=set(ContentType),
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/content_all.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_content_allow_no_types(sample_doc: DoclingDocument):
|
|
doc = _create_content_filtering_doc(sample_doc)
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_types=set(),
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/content_none.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_content_allow_specific_types(sample_doc: DoclingDocument):
|
|
doc = _create_content_filtering_doc(sample_doc)
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_types={
|
|
ContentType.PICTURE,
|
|
ContentType.TABLE,
|
|
ContentType.TABLE_CELL,
|
|
ContentType.REF_CAPTION,
|
|
ContentType.TEXT_CODE,
|
|
},
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/content_specific.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_content_block_specific_types(sample_doc: DoclingDocument):
|
|
doc = _create_content_filtering_doc(sample_doc)
|
|
blocked_types = {
|
|
ContentType.TABLE,
|
|
ContentType.TEXT_CODE,
|
|
}
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_types={ct for ct in ContentType if ct not in blocked_types},
|
|
image_mode=ImageRefMode.EMBEDDED,
|
|
),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/content_block_specific.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_inline_group():
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
|
|
parent_txt = doc.add_text(label=DocItemLabel.TEXT, text="", prov=prov)
|
|
simple_inline_gr = doc.add_inline_group(parent=parent_txt)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="One", parent=simple_inline_gr)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Two",
|
|
parent=simple_inline_gr,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Three", parent=simple_inline_gr)
|
|
|
|
li_inline_gr = doc.add_list_group()
|
|
doc.add_list_item(text="Item 1", parent=li_inline_gr)
|
|
li2 = doc.add_list_item(text="", parent=li_inline_gr)
|
|
li2_inline_gr = doc.add_inline_group(parent=li2)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Four", parent=li2_inline_gr)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="Five",
|
|
parent=li2_inline_gr,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Six", parent=li2_inline_gr)
|
|
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/inline_group.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_mini_inline():
|
|
doc = DoclingDocument(name="test")
|
|
ul = doc.add_list_group()
|
|
li = doc.add_list_item(text="", parent=ul)
|
|
inl = doc.add_inline_group(parent=li)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="foo", parent=inl)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="bar",
|
|
parent=inl,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/mini_inline.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _doc_picture_body_inline_and_formula() -> DoclingDocument:
|
|
doc = DoclingDocument(name="picture_body_children")
|
|
pic = doc.add_picture()
|
|
inline = doc.add_inline_group(parent=pic)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Hello ", parent=inline)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="world",
|
|
parent=inline,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
doc.add_formula(text=r"E=mc^2", parent=pic)
|
|
return doc
|
|
|
|
|
|
def test_picture_body_children_inline_group_and_formula():
|
|
"""Picture children serialize as picture body content after the preamble."""
|
|
doc = _doc_picture_body_inline_and_formula()
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False),
|
|
)
|
|
ser_txt = ser.serialize().text
|
|
exp_file = Path("./tests/data/doc/picture_body_children.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _create_wrapping_test_doc():
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="simple")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" leading")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="trailing ")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="< special")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" leading and < special")
|
|
|
|
doc.add_text(label=DocItemLabel.TEXT, text="w/prov simple", prov=prov)
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" w/prov leading", prov=prov)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="w/prov trailing ", prov=prov)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="w/prov < special", prov=prov)
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" w/prov leading and < special", prov=prov)
|
|
|
|
return doc
|
|
|
|
|
|
def test_content_wrapping_mode_when_needed():
|
|
doc = _create_wrapping_test_doc()
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_wrapping_mode=WrapMode.AUTO,
|
|
),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/wrapping_when_needed.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_content_wrapping_mode_always():
|
|
doc = _create_wrapping_test_doc()
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
content_wrapping_mode=WrapMode.ALWAYS,
|
|
),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/wrapping_always.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_default_mode():
|
|
doc = DoclingDocument(name="test")
|
|
add_texts_section(doc)
|
|
add_list_section(doc)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/default_mode.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_vlm_mode():
|
|
doc = DoclingDocument(name="test")
|
|
add_texts_section(doc)
|
|
add_list_section(doc)
|
|
doc.add_picture(
|
|
image=ImageRef(
|
|
mimetype="image/png",
|
|
uri="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAC0lEQVR4nGNgQAYAAA4AAamRc7EAAAAASUVORK5CYII=",
|
|
dpi=300,
|
|
size=Size(width=100, height=100),
|
|
),
|
|
caption=doc.add_text(
|
|
label=DocItemLabel.CAPTION,
|
|
text="""Picture
|
|
Caption""",
|
|
),
|
|
)
|
|
doc.add_code(text="0 == 0")
|
|
|
|
table_prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 60, 90, 90), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 0),
|
|
)
|
|
table = doc.add_table(data=TableData(num_rows=2, num_cols=2), prov=table_prov)
|
|
for i, row in enumerate((("A", "B"), ("C", "D"))):
|
|
for j, text in enumerate(row):
|
|
doc.add_table_cell(
|
|
table_item=table,
|
|
cell=TableCell(
|
|
start_row_offset_idx=i,
|
|
end_row_offset_idx=i + 1,
|
|
start_col_offset_idx=j,
|
|
end_col_offset_idx=j + 1,
|
|
text=text,
|
|
bbox=BoundingBox.from_tuple(
|
|
(10 + j * 40, 70 - i * 8, 45 + j * 40, 78 - i * 8),
|
|
origin=CoordOrigin.BOTTOMLEFT,
|
|
),
|
|
),
|
|
)
|
|
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
add_table_cell_location=True,
|
|
pretty_indentation=None,
|
|
escape_mode=EscapeMode.ALWAYS, # decide between ALWAYS and AUTO
|
|
content_wrapping_mode=WrapMode.ALWAYS, # decide between ALWAYS and AUTO
|
|
traverse_pictures=True,
|
|
include_namespace=False,
|
|
include_version=False,
|
|
use_virtual_text=True,
|
|
label_mode=LabelMode.ALWAYS,
|
|
interpret_code_unknown_as_other=True,
|
|
),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/vlm_mode.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_rich_cells(rich_table_doc):
|
|
ser = DocLangDocSerializer(
|
|
doc=rich_table_doc,
|
|
params=DocLangParams(include_version=False),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/rich_table.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _create_simple_prov_doc():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Hello", prov=prov)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="World", prov=prov)
|
|
return doc
|
|
|
|
|
|
def test_checkboxes():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_UNSELECTED, text="TODO")
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_SELECTED, text="DONE")
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/checkboxes.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_def_prov_512():
|
|
doc = _create_simple_prov_doc()
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
xsize=512,
|
|
ysize=512,
|
|
),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/simple_prov_res_512.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_quantize_clamp_upper_edge_without_warning() -> None:
|
|
with warnings.catch_warnings(record=True) as w:
|
|
warnings.simplefilter("always")
|
|
out = _quantize_to_resolution(512, resolution=512)
|
|
|
|
assert out == 511
|
|
assert not w
|
|
|
|
|
|
def test_quantize_clamp_warns_when_genuinely_out_of_range() -> None:
|
|
with pytest.warns(UserWarning, match=r"greater than 511"):
|
|
out = _quantize_to_resolution(518, resolution=512)
|
|
|
|
assert out == 511
|
|
|
|
|
|
def test_location_tokens_tolerates_exact_right_edge() -> None:
|
|
with warnings.catch_warnings(record=True) as w:
|
|
warnings.simplefilter("always")
|
|
_create_location_tokens_for_bbox(
|
|
bbox=(0.0, 0.0, 100.0, 10.0),
|
|
page_w=100.0,
|
|
page_h=100.0,
|
|
xres=512,
|
|
yres=512,
|
|
)
|
|
|
|
assert not w
|
|
|
|
|
|
def test_def_prov_256():
|
|
doc = _create_simple_prov_doc()
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
xsize=256,
|
|
ysize=256,
|
|
),
|
|
)
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/simple_prov_res_256.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_chart():
|
|
doc = DoclingDocument.load_from_json("./tests/data/doc/barchart.json")
|
|
ser = DocLangDocSerializer(doc=doc, params=DocLangParams(include_version=False))
|
|
ser_res = ser.serialize()
|
|
ser_txt = ser_res.text
|
|
exp_file = Path("./tests/data/doc/barchart.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"src",
|
|
[
|
|
# captions and footnotes as children of their picture/table (as produced by conversion)
|
|
Path("./tests/data/doc/multi_captions_footnotes.json"),
|
|
# captions and footnotes at body level
|
|
Path("./tests/data/doc/multi_captions_footnotes_top_level.json"),
|
|
],
|
|
ids=lambda p: p.stem,
|
|
)
|
|
def test_multiple_captions_and_footnotes(src: Path):
|
|
doc = DoclingDocument.load_from_json(src)
|
|
params = DocLangParams(include_version=False)
|
|
|
|
ser_txt = DocLangDocSerializer(doc=doc, params=params).serialize().text
|
|
verify_doclang(exp_file=src.with_suffix(".gt.dclg.xml"), actual=ser_txt)
|
|
|
|
doc2 = DocLangDocDeserializer().deserialize_str(ser_txt)
|
|
_verify_doc(doc=doc2, exp_json=src.with_suffix(".deserialized.gt.json"))
|
|
|
|
reser_txt = DocLangDocSerializer(doc=doc2, params=params).serialize().text
|
|
verify_doclang(exp_file=src.with_suffix(".reserialized.gt.dclg.xml"), actual=reser_txt)
|
|
assert reser_txt == ser_txt
|
|
|
|
|
|
def _verify_doc(doc: DoclingDocument, exp_json: Path):
|
|
if GEN_TEST_DATA:
|
|
doc.save_as_json(filename=exp_json)
|
|
else:
|
|
exp_doc = DoclingDocument.load_from_json(filename=exp_json)
|
|
assert doc == exp_doc
|
|
|
|
|
|
def test_kv():
|
|
doc = DoclingDocument(name="")
|
|
kvm = doc.add_field_region()
|
|
|
|
doc.add_field_heading(text="KV heading", parent=kvm)
|
|
|
|
kve = doc.add_field_item(parent=kvm)
|
|
doc.add_field_key(text="simple key", parent=kve)
|
|
doc.add_field_value(text="simple value", parent=kve)
|
|
|
|
doc.add_field_heading(level=2, text="KV sub-heading", parent=kvm)
|
|
|
|
# inlined key-value pair (outer is <text>...</text>)
|
|
# TODO: possibly support outer bounding box
|
|
inl = doc.add_inline_group(parent=kvm)
|
|
kve = doc.add_field_item(parent=inl)
|
|
doc.add_field_key(text="my inline key1: ", parent=kve)
|
|
doc.add_field_value(text="my inline value1", parent=kve, kind="fillable")
|
|
|
|
# # inlined key-value pair (outer is <kv_entry>...</kv_entry>)
|
|
# # TODO: possibly support outer bounding box
|
|
# kve = doc.add_field_item(parent=kvm)
|
|
# inl = doc.add_inline_group(parent=kve)
|
|
# doc.add_field_key(text="my inline key2: ", parent=inl)
|
|
# doc.add_field_value(text="my inline value2", parent=inl, kind="fillable")
|
|
|
|
kve = doc.add_field_item(parent=kvm)
|
|
doc.add_field_key(text="name", parent=kve)
|
|
doc.add_field_value(text="John Doe", parent=kve, kind="fillable")
|
|
doc.add_field_value(text="Max Mustermann", parent=kve, kind="fillable")
|
|
|
|
kk = doc.add_field_value(text="", parent=kve, kind="fillable")
|
|
opt_vis = doc.add_inline_group(parent=kk)
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_UNSELECTED, text="", parent=opt_vis)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Clark ", parent=opt_vis)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Kent", parent=opt_vis, formatting=Formatting(bold=True))
|
|
doc.add_field_hint(text="Select this if you are a Superman fan", parent=opt_vis)
|
|
|
|
doc.add_field_value(text="", parent=kve)
|
|
|
|
# inlined form inputs
|
|
# TODO: add support for outer bounding box
|
|
inl = doc.add_inline_group(parent=kve)
|
|
|
|
doc.add_text(label=DocItemLabel.TEXT, text="My first input ", parent=inl)
|
|
doc.add_field_value(text="", parent=inl, kind="fillable")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" and my second input ", parent=inl)
|
|
doc.add_field_value(text="m", parent=inl, kind="fillable")
|
|
|
|
kv_entry_3 = doc.add_field_item(parent=kvm)
|
|
doc.add_field_key(text="I am in the country as a: ", parent=kv_entry_3)
|
|
|
|
vis = doc.add_field_value(text="", parent=kv_entry_3, kind="fillable")
|
|
opt_vis = doc.add_inline_group(parent=vis)
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_UNSELECTED, text="Visitor", parent=opt_vis)
|
|
|
|
std = doc.add_field_value(text="", parent=kv_entry_3, kind="fillable")
|
|
opt_std = doc.add_inline_group(parent=std)
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_UNSELECTED, text=" Student", parent=opt_std)
|
|
|
|
oth = doc.add_field_value(text="", parent=kv_entry_3, kind="fillable")
|
|
opt_oth = doc.add_inline_group(parent=oth)
|
|
doc.add_text(label=DocItemLabel.CHECKBOX_UNSELECTED, text="Other (Specify)", parent=opt_oth)
|
|
|
|
doc.add_field_value(text="", parent=kv_entry_3, kind="fillable")
|
|
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Some final stuff.")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="The end.")
|
|
|
|
exp_json = Path("./tests/data/doc/kv.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/kv.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _create_kv_invoice_doc() -> DoclingDocument:
|
|
"""Helper to create a key-value invoice document with various field types."""
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
image_uri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAC0lEQVR4nGNgQAYAAA4AAamRc7EAAAAASUVORK5CYII="
|
|
|
|
# first key-value map
|
|
kvm = doc.add_field_region(prov=prov)
|
|
|
|
# inlined key-value pair
|
|
kve = doc.add_field_item(parent=kvm)
|
|
kvk = doc.add_field_key(text="", parent=kve)
|
|
doc.add_picture(
|
|
parent=kvk,
|
|
image=ImageRef(
|
|
mimetype="image/png",
|
|
uri=image_uri,
|
|
dpi=300,
|
|
size=Size(width=100, height=100),
|
|
),
|
|
)
|
|
doc.add_field_value(text="+123-456-7890", parent=kve)
|
|
|
|
# another inlined key-value pair
|
|
kve = doc.add_field_item(parent=kvm)
|
|
kvk = doc.add_field_key(text="", parent=kve)
|
|
doc.add_picture(
|
|
parent=kvk,
|
|
image=ImageRef(
|
|
mimetype="image/png",
|
|
uri=image_uri,
|
|
dpi=300,
|
|
size=Size(width=100, height=100),
|
|
),
|
|
)
|
|
doc.add_field_value(text="hello@example.com", parent=kve)
|
|
|
|
# second key-value map
|
|
kvm = doc.add_field_region()
|
|
|
|
# inlined key-value pair
|
|
inl_outer = doc.add_inline_group(parent=kvm)
|
|
kve = doc.add_field_item(parent=inl_outer)
|
|
doc.add_field_key(text="Invoice No: ", parent=kve)
|
|
doc.add_field_value(text="222", parent=kve)
|
|
|
|
# another inlined key-value pair
|
|
inl_outer = doc.add_inline_group(parent=kvm)
|
|
kve = doc.add_field_item(parent=inl_outer)
|
|
doc.add_field_key(text="Date: ", parent=kve)
|
|
doc.add_field_value(text="02 May, 2021", parent=kve)
|
|
|
|
# a last key-value map
|
|
kvm = doc.add_field_region()
|
|
kve = doc.add_field_item(parent=kvm)
|
|
doc.add_field_key(text="Administrator", parent=kve, prov=prov)
|
|
doc.add_field_value(text="John Doe", parent=kve, prov=prov)
|
|
|
|
return doc
|
|
|
|
|
|
def test_kv_invoice():
|
|
doc = _create_kv_invoice_doc()
|
|
|
|
exp_json = Path("./tests/data/doc/kv_invoice.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, image_mode=ImageRefMode.PLACEHOLDER),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/kv_invoice.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, image_mode=ImageRefMode.EMBEDDED),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/kv_invoice_embedded.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_kv_advanced_inline():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
|
|
# first key-value map
|
|
kvm = doc.add_field_region()
|
|
|
|
# inlined key-value pair
|
|
inl_outer = doc.add_inline_group(parent=kvm)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="This certificate applies to ", parent=inl_outer)
|
|
|
|
kve = doc.add_field_item(parent=inl_outer)
|
|
doc.add_field_value(text="", parent=kve, kind="fillable")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" percent of Buyer's purchases from ", parent=inl_outer)
|
|
|
|
kve = doc.add_field_item(parent=inl_outer)
|
|
doc.add_field_value(text="", parent=kve, kind="fillable")
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=" (name, address, and employer idenficiation number of seller) as follows (complete as applicable): ",
|
|
parent=inl_outer,
|
|
)
|
|
|
|
kve = doc.add_field_item(parent=inl_outer)
|
|
doc.add_field_value(text="", parent=kve, kind="fillable")
|
|
doc.add_text(label=DocItemLabel.TEXT, text=".", parent=inl_outer)
|
|
|
|
exp_json = Path("./tests/data/doc/kv_advanced_inline.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/kv_advanced_inline.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_kv_nested():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
# prov = None
|
|
|
|
# first key-value map
|
|
kvm = doc.add_field_region(prov=prov)
|
|
|
|
kve = doc.add_field_item(parent=kvm)
|
|
doc.add_field_key(text="A", parent=kve)
|
|
kvv = doc.add_field_value(text="", parent=kve)
|
|
kvm_inner = doc.add_field_region(parent=kvv)
|
|
kve_inner = doc.add_field_item(parent=kvm_inner)
|
|
doc.add_marker(text="1.", parent=kve_inner)
|
|
doc.add_field_key(text="AA", parent=kve_inner)
|
|
doc.add_field_hint(text="Some explanation for key AA", parent=kve_inner)
|
|
doc.add_field_value(text="AAA", parent=kve_inner)
|
|
doc.add_field_hint(text="Some explanation for value AAA", parent=kve_inner)
|
|
doc.add_field_value(text="AAB", parent=kve_inner)
|
|
doc.add_field_hint(text="Some explanation for value AAB", parent=kve_inner)
|
|
kve_inner = doc.add_field_item(parent=kvm_inner)
|
|
doc.add_marker(text="2.", parent=kve_inner)
|
|
doc.add_field_key(text="AB", parent=kve_inner)
|
|
doc.add_field_value(text="ABA", parent=kve_inner)
|
|
doc.add_field_value(text="ABB", parent=kve_inner)
|
|
|
|
exp_json = Path("./tests/data/doc/kv_nested.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/kv_nested.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_kv_form_with_table():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
|
|
# first key-value map
|
|
kvm = doc.add_field_region()
|
|
|
|
# table
|
|
|
|
table_vals = [
|
|
["Description of property", "Cost or other basis, plus improvements and expense of sale", "Gain or loss"],
|
|
["", "gain", "150,997"],
|
|
["", "loss", "114,676"],
|
|
]
|
|
num_rows = len(table_vals)
|
|
num_cols = len(table_vals[0])
|
|
table = doc.add_table(data=TableData(num_rows=num_rows, num_cols=num_cols), parent=kvm)
|
|
|
|
for i in range(num_rows):
|
|
for j in range(num_cols):
|
|
if i == 0: # headers
|
|
cell = TableCell(
|
|
start_row_offset_idx=i,
|
|
end_row_offset_idx=i + 1,
|
|
start_col_offset_idx=j,
|
|
end_col_offset_idx=j + 1,
|
|
text=table_vals[i][j],
|
|
column_header=True,
|
|
)
|
|
else:
|
|
kve = doc.add_field_item(parent=table)
|
|
doc.add_field_value(text=table_vals[i][j], parent=kve, kind="fillable")
|
|
cell = RichTableCell(
|
|
start_row_offset_idx=i,
|
|
end_row_offset_idx=i + 1,
|
|
start_col_offset_idx=j,
|
|
end_col_offset_idx=j + 1,
|
|
text="",
|
|
ref=kve.get_ref(),
|
|
)
|
|
doc.add_table_cell(table_item=table, cell=cell)
|
|
|
|
exp_json = Path("./tests/data/doc/kv_form_with_table.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
|
|
exp_file = Path("./tests/data/doc/kv_form_with_table.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_kv_migration_self_contained_scenario():
|
|
doc = DoclingDocument(name="")
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((1, 2, 3, 4), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 2),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Hello, world!")
|
|
doc.add_key_values(
|
|
graph=GraphData(
|
|
cells=[
|
|
# both TO_VALUE & TO_KEY links:
|
|
GraphCell(
|
|
label=GraphCellLabel.KEY,
|
|
cell_id=0,
|
|
text="Common name",
|
|
orig="Common name",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=1,
|
|
text="Duck",
|
|
orig="Duck",
|
|
),
|
|
# TO_PARENT & TO_CHILD links:
|
|
GraphCell(
|
|
label=GraphCellLabel.KEY,
|
|
cell_id=2,
|
|
text="Anatoidea",
|
|
orig="Anatoidea",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=3,
|
|
text="Anatidae",
|
|
orig="Anatidae",
|
|
),
|
|
# multiple TO_VALUE links:
|
|
GraphCell(
|
|
label=GraphCellLabel.KEY,
|
|
cell_id=4,
|
|
text="Distribution package",
|
|
orig="Distribution package",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=5,
|
|
text="docling",
|
|
orig="docling",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=6,
|
|
text="docling-core",
|
|
orig="docling-core",
|
|
prov=prov,
|
|
),
|
|
],
|
|
links=[
|
|
GraphLink(
|
|
label=GraphLinkLabel.TO_VALUE,
|
|
source_cell_id=0,
|
|
target_cell_id=1,
|
|
),
|
|
GraphLink(label=GraphLinkLabel.TO_KEY, source_cell_id=1, target_cell_id=0),
|
|
GraphLink(label=GraphLinkLabel.TO_CHILD, source_cell_id=2, target_cell_id=3),
|
|
GraphLink(label=GraphLinkLabel.TO_PARENT, source_cell_id=3, target_cell_id=2),
|
|
GraphLink(label=GraphLinkLabel.TO_VALUE, source_cell_id=4, target_cell_id=5),
|
|
GraphLink(label=GraphLinkLabel.TO_VALUE, source_cell_id=4, target_cell_id=6),
|
|
],
|
|
),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Some more text...", parent=doc.body)
|
|
doc.add_form(
|
|
graph=GraphData(
|
|
cells=[
|
|
# both TO_VALUE & TO_KEY links:
|
|
GraphCell(
|
|
label=GraphCellLabel.KEY,
|
|
cell_id=0,
|
|
text="Color",
|
|
orig="Color",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=1,
|
|
text="Orange",
|
|
orig="Orange",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=2,
|
|
text="Black",
|
|
orig="Black",
|
|
),
|
|
GraphCell(
|
|
label=GraphCellLabel.VALUE,
|
|
cell_id=3,
|
|
text="White",
|
|
orig="White",
|
|
),
|
|
],
|
|
links=[
|
|
GraphLink(label=GraphLinkLabel.TO_VALUE, source_cell_id=0, target_cell_id=1),
|
|
GraphLink(label=GraphLinkLabel.TO_VALUE, source_cell_id=0, target_cell_id=2),
|
|
GraphLink(label=GraphLinkLabel.TO_VALUE, source_cell_id=0, target_cell_id=3),
|
|
GraphLink(label=GraphLinkLabel.TO_KEY, source_cell_id=3, target_cell_id=0),
|
|
],
|
|
),
|
|
prov=prov,
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="The end.", parent=doc.body)
|
|
|
|
exp_json = Path("./tests/data/doc/kv_pre_migration.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
doc._migrate_to_field_regions()
|
|
|
|
exp_json = Path("./tests/data/doc/kv_post_migration.out.json")
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/kv_migration.out.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_kv_migration_annot_scenario():
|
|
roots = [
|
|
"./tests/data/doc/kv",
|
|
"./tests/data/doc/doclang_ref",
|
|
]
|
|
for subdir in chain.from_iterable([Path(root).iterdir() for root in roots]):
|
|
if not subdir.is_dir():
|
|
continue
|
|
input_json = subdir / "input.json"
|
|
if not input_json.exists():
|
|
continue
|
|
doc = DoclingDocument.load_from_json(input_json)
|
|
if GEN_TEST_DATA:
|
|
modes = {
|
|
# "ro": "reading_order",
|
|
"kv": "key_value",
|
|
}
|
|
for mode_kw in modes:
|
|
pages = doc.get_visualization(viz_mode=modes[mode_kw])
|
|
for page_no, page in pages.items():
|
|
page.save(str(subdir / f"input_{mode_kw}_p{page_no}.png"))
|
|
doc._migrate_to_field_regions()
|
|
exp_json = subdir / "output.json"
|
|
_verify_doc(doc=doc, exp_json=exp_json)
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = subdir / "output.dclg.xml"
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
ser = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
add_content=False,
|
|
),
|
|
)
|
|
ser_txt = ser.serialize().text
|
|
exp_file = subdir / "output_no_content.dclg.xml"
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
if GEN_TEST_DATA:
|
|
modes = {
|
|
# "ro": "reading_order",
|
|
"kv": "key_value",
|
|
}
|
|
for mode_kw in modes:
|
|
pages = doc.get_visualization(viz_mode=modes[mode_kw])
|
|
for page_no, page in pages.items():
|
|
page.save(str(subdir / f"output_{mode_kw}_p{page_no}.png"))
|
|
|
|
|
|
# ===============================
|
|
# suppress_empty_elements tests
|
|
# ===============================
|
|
|
|
_SUPPRESS_PARAMS = DocLangParams(
|
|
include_version=False,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
content_types=set(), # no content → forces items empty
|
|
)
|
|
|
|
|
|
def test_suppress_empty_text_item():
|
|
"""An empty text item is omitted when suppress_empty_elements is True."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="")
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
# The empty <text></text> must not appear
|
|
assert "<text" not in result
|
|
# Document root must still be present
|
|
assert "<doclang" in result
|
|
|
|
|
|
def test_empty_text_item_preserved_by_default():
|
|
"""Without suppress_empty_elements the empty tag pair is emitted."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="")
|
|
|
|
default_params = DocLangParams(
|
|
include_version=False,
|
|
add_location=False,
|
|
content_types=set(),
|
|
)
|
|
result = serialize_doclang(doc, params=default_params)
|
|
assert "<text></text>" in result
|
|
|
|
|
|
def test_suppress_empty_heading():
|
|
"""An empty heading is suppressed."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_heading(text="", level=2)
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<heading" not in result
|
|
assert "</heading>" not in result
|
|
|
|
|
|
def test_suppress_empty_code():
|
|
"""An empty code block is suppressed when ``UNKNOWN`` maps to ``undefined``."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_code(text="")
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<code" not in result
|
|
|
|
|
|
def test_suppress_empty_picture():
|
|
"""A picture with no content, no caption, no footnotes is suppressed."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_picture()
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<group" not in result
|
|
assert "<picture" not in result
|
|
|
|
|
|
def test_empty_picture_preserved_by_default():
|
|
"""Without suppress_empty_elements the empty picture is preserved."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_picture()
|
|
|
|
default_params = DocLangParams(
|
|
include_version=False,
|
|
add_location=False,
|
|
content_types=set(),
|
|
)
|
|
result = serialize_doclang(doc, params=default_params)
|
|
# After fix: empty pictures now emit <picture></picture> instead of <group></group>
|
|
assert "<picture></picture>" in result
|
|
assert "<group" not in result
|
|
|
|
|
|
def test_suppress_empty_table():
|
|
"""A table with no data, no caption, no footnotes is suppressed."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_table(data=TableData())
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<group" not in result
|
|
assert "<otsl" not in result
|
|
|
|
|
|
def test_empty_table_preserved_by_default():
|
|
"""Without suppress_empty_elements the empty table element is preserved."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_table(data=TableData())
|
|
|
|
default_params = DocLangParams(
|
|
include_version=False,
|
|
add_location=False,
|
|
content_types=set(),
|
|
)
|
|
result = serialize_doclang(doc, params=default_params)
|
|
assert "<table" in result
|
|
assert "<group" not in result
|
|
|
|
|
|
def test_document_index_serialization():
|
|
"""Test that DOCUMENT_INDEX tables are serialized as <index>."""
|
|
doc = DoclingDocument(name="test")
|
|
|
|
# Add a regular table
|
|
table_data = TableData(num_cols=2)
|
|
table_data.add_row(["Header 1", "Header 2"])
|
|
table_data.grid[0][0].column_header = True
|
|
table_data.grid[0][1].column_header = True
|
|
table_data.add_row(["Data 1", "Data 2"])
|
|
doc.add_table(data=table_data, label=DocItemLabel.TABLE)
|
|
|
|
# Add a DOCUMENT_INDEX table
|
|
index_data = TableData(num_cols=2)
|
|
index_data.add_row(["Index 1", "Page 1"])
|
|
index_data.add_row(["Index 2", "Page 2"])
|
|
doc.add_table(data=index_data, label=DocItemLabel.DOCUMENT_INDEX)
|
|
|
|
result = serialize_doclang(doc)
|
|
|
|
# Verify against expected output
|
|
exp_file = Path("./tests/data/doc/document_index.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=result)
|
|
|
|
|
|
def test_suppress_empty_inline_group():
|
|
"""An inline group whose children are all empty is suppressed."""
|
|
doc = DoclingDocument(name="test")
|
|
inl = doc.add_inline_group()
|
|
doc.add_text(label=DocItemLabel.TEXT, text="", parent=inl)
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
# The inline group emits <text> wrapper when unwrapped; both should vanish
|
|
assert "<text" not in result
|
|
|
|
|
|
def test_suppress_list_with_all_empty_children():
|
|
"""A list group whose children all produce empty text is auto-suppressed.
|
|
|
|
The list serializer already skips the <list> wrapper when no child text
|
|
is produced, so suppressing individual empty <list_text> items causes
|
|
the parent <list> to vanish too.
|
|
"""
|
|
doc = DoclingDocument(name="test")
|
|
lst = doc.add_list_group()
|
|
doc.add_list_item(text="", parent=lst)
|
|
doc.add_list_item(text="", parent=lst)
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<list" not in result
|
|
assert "<ldiv" not in result
|
|
|
|
|
|
def test_suppress_list_keeps_nonempty_items():
|
|
"""Only empty list items are suppressed; non-empty ones remain."""
|
|
doc = DoclingDocument(name="test")
|
|
lst = doc.add_list_group()
|
|
doc.add_list_item(text="", parent=lst)
|
|
doc.add_list_item(text="Keep me", parent=lst)
|
|
doc.add_list_item(text="", parent=lst)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
)
|
|
result = serialize_doclang(doc, params=params)
|
|
assert "<list>" in result
|
|
assert result.count("<ldiv/>") == 1
|
|
assert "Keep me" in result
|
|
|
|
|
|
def test_suppress_mixed_content():
|
|
"""A document with a mix of empty and non-empty items.
|
|
|
|
Empty items are suppressed, non-empty ones remain.
|
|
"""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="") # suppressed
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Visible paragraph") # kept
|
|
doc.add_picture() # suppressed (empty picture)
|
|
doc.add_heading(text="Visible Heading", level=1) # kept
|
|
doc.add_code(text="") # suppressed
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
)
|
|
result = serialize_doclang(doc, params=params)
|
|
assert result.count("<text>") == 1
|
|
assert "Visible paragraph" in result
|
|
assert "<group" not in result
|
|
assert '<heading level="2">' in result
|
|
assert "Visible Heading" in result
|
|
assert "<code" not in result
|
|
|
|
|
|
def test_suppress_does_not_affect_nonempty():
|
|
"""Suppression flag has no effect on items that carry content."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Hello")
|
|
doc.add_heading(text="World", level=1)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
)
|
|
result = serialize_doclang(doc, params=params)
|
|
assert "<text>Hello</text>" in result
|
|
assert '<heading level="2">World</heading>' in result
|
|
|
|
|
|
def test_suppress_nested_section_with_empty_children():
|
|
"""A section containing only empty elements should still emit the section
|
|
(sections are grouping tokens and not subject to content-level suppression),
|
|
but all its empty children should be suppressed.
|
|
"""
|
|
from docling_core.types.doc import GroupLabel
|
|
|
|
doc = DoclingDocument(name="test")
|
|
sec = doc.add_group(label=GroupLabel.SECTION, name="empty_sec")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="", parent=sec)
|
|
doc.add_code(text="", parent=sec)
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
# Section grouping wrapper may or may not remain (depends on serializer),
|
|
# but importantly no <text> or <code> tags appear
|
|
assert "<text" not in result
|
|
assert "<code" not in result
|
|
|
|
|
|
def test_suppress_empty_caption_and_footnote_on_picture():
|
|
"""A picture with an empty caption and empty footnote is suppressed when
|
|
suppress_empty_elements is True and there is no other content.
|
|
"""
|
|
doc = DoclingDocument(name="test")
|
|
cap = doc.add_text(label=DocItemLabel.CAPTION, text="")
|
|
doc.add_picture(caption=cap)
|
|
|
|
result = serialize_doclang(doc, params=_SUPPRESS_PARAMS)
|
|
assert "<group" not in result
|
|
assert "<picture" not in result
|
|
|
|
|
|
def test_suppress_empty_picture_with_nonempty_caption():
|
|
"""A picture with a non-empty caption should still be emitted even when
|
|
suppress_empty_elements is True, because the composed content is non-empty.
|
|
"""
|
|
doc = DoclingDocument(name="test")
|
|
cap = doc.add_text(label=DocItemLabel.CAPTION, text="My Figure")
|
|
doc.add_picture(caption=cap)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
)
|
|
result = serialize_doclang(doc, params=params)
|
|
assert "<picture" in result
|
|
assert "<caption" in result
|
|
assert "My Figure" in result
|
|
assert "<group" not in result
|
|
|
|
|
|
def test_layer_when_needed_mode(doc_with_layers):
|
|
"""Test AUTO mode omits default layer, includes non-default."""
|
|
params = DocLangParams(include_version=False, layer_mode=LayerMode.AUTO)
|
|
ser = DocLangDocSerializer(doc=doc_with_layers, params=params)
|
|
ser_txt = ser.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/layer_minimal_mode.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_layer_always_mode(doc_with_layers):
|
|
"""Test ALWAYS mode includes layer element for all items."""
|
|
params = DocLangParams(include_version=False, layer_mode=LayerMode.ALWAYS)
|
|
ser = DocLangDocSerializer(doc=doc_with_layers, params=params)
|
|
ser_txt = ser.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/layer_always_mode.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_layer_filter_body_only(doc_with_layers):
|
|
"""Test that layers parameter filters content to only show specified layers."""
|
|
# Serialize with only body layer
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
layers={ContentLayer.BODY},
|
|
)
|
|
ser = DocLangDocSerializer(doc=doc_with_layers, params=params)
|
|
ser_txt = ser.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/layer_only_body.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _doc_with_labeled_code_and_pictures() -> DoclingDocument:
|
|
doc = DoclingDocument(name="t")
|
|
doc.add_code(text="x = 1", code_language=CodeLanguageLabel.PYTHON)
|
|
doc.add_code(text="y = 2")
|
|
classified = doc.add_picture()
|
|
classified.meta = PictureMeta(
|
|
classification=PictureClassificationMetaField(
|
|
predictions=[
|
|
PictureClassificationPrediction(
|
|
class_name=PictureClassificationLabel.OTHER.value,
|
|
confidence=1.0,
|
|
)
|
|
]
|
|
)
|
|
)
|
|
doc.add_picture()
|
|
return doc
|
|
|
|
|
|
def test_label_mode_when_needed():
|
|
result = serialize_doclang(
|
|
_doc_with_labeled_code_and_pictures(),
|
|
params=DocLangParams(include_version=False, label_mode=LabelMode.AUTO, add_location=False),
|
|
)
|
|
assert result.count('<label value="Python"/>') == 1
|
|
assert result.count('<label value="other"/>') == 1
|
|
assert '<label value="undefined"/>' not in result
|
|
|
|
|
|
def test_label_mode_always():
|
|
result = serialize_doclang(
|
|
_doc_with_labeled_code_and_pictures(),
|
|
params=DocLangParams(include_version=False, label_mode=LabelMode.ALWAYS, add_location=False),
|
|
)
|
|
assert result.count('<label value="Python"/>') == 1
|
|
assert result.count('<label value="other"/>') == 1
|
|
assert result.count('<label value="undefined"/>') == 2
|
|
|
|
|
|
def test_label_mode_always_empty_code_emits_undefined_by_default():
|
|
doc = DoclingDocument(name="t")
|
|
doc.add_code(text="")
|
|
result = serialize_doclang(
|
|
doc,
|
|
params=DocLangParams(include_version=False, label_mode=LabelMode.ALWAYS, add_location=False),
|
|
)
|
|
assert '<label value="undefined"/>' in result
|
|
assert '<label value="other"/>' not in result
|
|
|
|
|
|
def test_interpret_code_unknown_as_other_true():
|
|
doc = DoclingDocument(name="t")
|
|
doc.add_code(text="x = 1")
|
|
result = serialize_doclang(
|
|
doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
interpret_code_unknown_as_other=True,
|
|
label_mode=LabelMode.AUTO,
|
|
add_location=False,
|
|
),
|
|
)
|
|
assert '<label value="other"/>' in result
|
|
assert '<label value="undefined"/>' not in result
|
|
|
|
|
|
def test_interpret_code_unknown_as_other_false():
|
|
doc = DoclingDocument(name="t")
|
|
doc.add_code(text="x = 1")
|
|
result = serialize_doclang(
|
|
doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
interpret_code_unknown_as_other=False,
|
|
label_mode=LabelMode.ALWAYS,
|
|
add_location=False,
|
|
),
|
|
)
|
|
assert '<label value="undefined"/>' in result
|
|
assert '<label value="other"/>' not in result
|
|
|
|
|
|
def test_label_mode_when_needed_suppresses_empty_unknown_code():
|
|
doc = DoclingDocument(name="t")
|
|
doc.add_code(text="")
|
|
result = serialize_doclang(
|
|
doc,
|
|
params=DocLangParams(
|
|
include_version=False,
|
|
label_mode=LabelMode.AUTO,
|
|
suppress_empty_elements=True,
|
|
add_location=False,
|
|
),
|
|
)
|
|
assert "<code" not in result
|
|
|
|
|
|
def test_newline_to_br():
|
|
code = """
|
|
|
|
|
|
import pytest
|
|
|
|
from docling_core.transforms.serializer.doclang import (
|
|
ContentType,
|
|
EscapeMode,
|
|
DocLangDocSerializer,
|
|
DocLangParams,
|
|
DocLangVocabulary,
|
|
LayerMode,
|
|
WrapMode,
|
|
)
|
|
"""
|
|
|
|
"""Test that newlines survive serialization and deserialization roundtrip."""
|
|
from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer
|
|
from docling_core.types.doc import TextItem
|
|
|
|
# Create a document with newlines
|
|
doc = DoclingDocument(name="")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="foo\nbar")
|
|
|
|
inl = doc.add_inline_group()
|
|
doc.add_text(label=DocItemLabel.TEXT, text="eins\n", parent=inl)
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" zwei\n ", parent=inl)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="drei", parent=inl, formatting=Formatting(bold=True))
|
|
|
|
doc.add_code(text=code)
|
|
|
|
# NOTE: this particular case seems bit brittle as to how it's preserved by XML tooling
|
|
doc.add_text(label=DocItemLabel.TEXT, text="\n")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/newline_to_br.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_list_item_with_code_child():
|
|
"""Test list item with empty text but CodeItem as child."""
|
|
doc = DoclingDocument(name="test")
|
|
lst = doc.add_list_group()
|
|
|
|
# List item with empty text but has a code child
|
|
li = doc.add_list_item(text="", parent=lst, marker="•")
|
|
doc.add_code(text="print('hello')", parent=li)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
exp_file = Path("./tests/data/doc/list_item_with_code.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_list_item_with_code_child_and_bbox():
|
|
"""Test list item with empty text, CodeItem child, and bounding box."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_page(page_no=0, size=Size(width=100, height=100), image=None)
|
|
lst = doc.add_list_group()
|
|
|
|
# List item with empty text but has a code child and provenance
|
|
li = doc.add_list_item(text="", parent=lst, marker="•")
|
|
doc.add_code(
|
|
text="print('hello')",
|
|
parent=li,
|
|
prov=ProvenanceItem(
|
|
page_no=0,
|
|
bbox=BoundingBox.from_tuple((10, 20, 80, 60), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 0),
|
|
),
|
|
)
|
|
|
|
serializer = DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, add_location=True, xsize=256, ysize=256),
|
|
)
|
|
ser_txt = serializer.serialize().text
|
|
exp_file = Path("./tests/data/doc/list_item_with_code_and_bbox.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def _create_virtual_text_test_doc(add_location: bool = False) -> DoclingDocument:
|
|
"""Helper to create a test document for virtual text testing.
|
|
|
|
Args:
|
|
add_location: If True, add provenance/location info to items.
|
|
|
|
Returns:
|
|
DoclingDocument with list items and table cells for testing.
|
|
"""
|
|
doc = DoclingDocument(name="test_virtual_texts")
|
|
|
|
# Add page if we need location
|
|
if add_location:
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
|
|
# Add a list with various item types
|
|
lg = doc.add_list_group()
|
|
|
|
# Regular list item with text
|
|
prov = None
|
|
if add_location:
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 20, 30, 25), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 12),
|
|
)
|
|
doc.add_list_item(text="Regular item", parent=lg, prov=prov)
|
|
|
|
# List item with empty text and CodeItem child
|
|
li_with_code = doc.add_list_item(text="", parent=lg)
|
|
doc.add_code(
|
|
text="print('hello')",
|
|
parent=li_with_code,
|
|
code_language=CodeLanguageLabel.PYTHON,
|
|
)
|
|
|
|
# List item with text
|
|
prov2 = None
|
|
if add_location:
|
|
prov2 = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 30, 30, 35), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 12),
|
|
)
|
|
doc.add_list_item(text="Another item", parent=lg, prov=prov2)
|
|
|
|
# Add a table with cells (mix of regular and rich cells)
|
|
# Add provenance to the table so cell locations can be serialized
|
|
table_prov = None
|
|
if add_location:
|
|
table_prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((2, 40, 90, 80), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 50),
|
|
)
|
|
table = doc.add_table(data=TableData(num_rows=2, num_cols=2), prov=table_prov)
|
|
|
|
cell: TableCell
|
|
# Add cells to the table
|
|
for i in range(2):
|
|
for j in range(2):
|
|
# Make cell (1,1) a RichTableCell with a formula
|
|
if i == 1 and j == 1:
|
|
formula_item = doc.add_formula(text="E=mc^2", parent=table)
|
|
cell = RichTableCell(
|
|
start_row_offset_idx=i,
|
|
end_row_offset_idx=i + 1,
|
|
start_col_offset_idx=j,
|
|
end_col_offset_idx=j + 1,
|
|
text="",
|
|
ref=formula_item.get_ref(),
|
|
)
|
|
else:
|
|
cell = TableCell(
|
|
start_row_offset_idx=i,
|
|
end_row_offset_idx=i + 1,
|
|
start_col_offset_idx=j,
|
|
end_col_offset_idx=j + 1,
|
|
text=f"Cell {i * 2 + j + 1}",
|
|
bbox=prov2.bbox if prov2 and i + j == 0 else None,
|
|
)
|
|
doc.add_table_cell(table_item=table, cell=cell)
|
|
|
|
return doc
|
|
|
|
|
|
def test_virtual_text_true_no_location():
|
|
"""Test use_virtual_text=True (default) without location info."""
|
|
doc = _create_virtual_text_test_doc(add_location=False)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
add_location=False,
|
|
)
|
|
serializer = DocLangDocSerializer(doc=doc, params=params)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/virtual_texts_true_no_loc.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_virtual_text_true_with_location():
|
|
"""Test use_virtual_text=True (default) with location info."""
|
|
doc = _create_virtual_text_test_doc(add_location=True)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
add_location=True,
|
|
add_table_cell_location=True,
|
|
)
|
|
serializer = DocLangDocSerializer(doc=doc, params=params)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/virtual_texts_true_with_loc.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_virtual_text_false_no_location():
|
|
"""Test use_virtual_text=False without location info."""
|
|
doc = _create_virtual_text_test_doc(add_location=False)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
use_virtual_text=False,
|
|
add_location=False,
|
|
)
|
|
serializer = DocLangDocSerializer(doc=doc, params=params)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/virtual_texts_false_no_loc.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_virtual_text_false_with_location():
|
|
"""Test use_virtual_text=False with location info."""
|
|
doc = _create_virtual_text_test_doc(add_location=True)
|
|
|
|
params = DocLangParams(
|
|
include_version=False,
|
|
use_virtual_text=False,
|
|
add_location=True,
|
|
add_table_cell_location=True,
|
|
)
|
|
serializer = DocLangDocSerializer(doc=doc, params=params)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/virtual_texts_false_with_loc.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
def test_include_namespace_and_version():
|
|
"""Test that include_namespace and include_version parameters work."""
|
|
doc = DoclingDocument(name="test")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Hello world")
|
|
|
|
params = DocLangParams(include_namespace=True, include_version=True)
|
|
serializer = DocLangDocSerializer(doc=doc, params=params)
|
|
ser_txt = serializer.serialize().text
|
|
|
|
exp_file = Path("./tests/data/doc/include_namespace_and_version.gt.dclg.xml")
|
|
verify_doclang(exp_file=exp_file, actual=ser_txt)
|
|
|
|
|
|
# ===============================
|
|
# Field region / key-value auto-wrapping
|
|
# ===============================
|
|
|
|
|
|
def _default_page(doc: DoclingDocument) -> None:
|
|
doc.add_page(page_no=1, size=Size(width=100, height=100), image=None)
|
|
|
|
|
|
def _doc_field_region_flat_key_value() -> DoclingDocument:
|
|
doc = DoclingDocument(name="field_region_flat_kv")
|
|
_default_page(doc)
|
|
region = doc.add_field_region()
|
|
doc.add_field_key(text="Key", parent=region)
|
|
doc.add_field_value(text="Value", parent=region, kind="read_only")
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_field_region_flat_key_value_wraps_field_item():
|
|
"""Key/value directly under field_region are auto-wrapped in field_item."""
|
|
doc = _doc_field_region_flat_key_value()
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/field_region_flat_key_value.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
def _doc_field_region_value_only() -> DoclingDocument:
|
|
doc = DoclingDocument(name="field_region_value_only")
|
|
_default_page(doc)
|
|
region = doc.add_field_region()
|
|
doc.add_field_value(text="42", parent=region, kind="read_only")
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_field_region_value_only_wraps_field_item():
|
|
"""Value directly under field_region is auto-wrapped in field_item."""
|
|
doc = _doc_field_region_value_only()
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/field_region_value_only.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
def _doc_field_item_outside_field_region() -> DoclingDocument:
|
|
doc = DoclingDocument(name="field_item_outside_region")
|
|
_default_page(doc)
|
|
item = doc.add_field_item(parent=doc.body)
|
|
doc.add_field_key(text="K", parent=item)
|
|
doc.add_field_value(text="V", parent=item, kind="read_only")
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_field_item_without_region_wraps_field_region():
|
|
"""Orphan field_item is auto-wrapped in <field_region>."""
|
|
doc = _doc_field_item_outside_field_region()
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/field_item_outside_field_region.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
# ===============================
|
|
# Hyperlinks
|
|
# ===============================
|
|
|
|
|
|
def _doc_text_with_hyperlink() -> DoclingDocument:
|
|
doc = DoclingDocument(name="hyperlink")
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="see repository",
|
|
hyperlink=AnyUrl("https://github.com/docling-project/docling"),
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_text_with_hyperlink():
|
|
r"""Text with hyperlink emits <href uri=\"...\"/> in element head."""
|
|
doc = _doc_text_with_hyperlink()
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/text_with_hyperlink.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
def _doc_text_with_hyperlink_and_layer() -> DoclingDocument:
|
|
doc = DoclingDocument(name="hyperlink_and_layer")
|
|
doc.add_text(
|
|
label=DocItemLabel.PAGE_HEADER,
|
|
text="linked header",
|
|
hyperlink=AnyUrl("https://example.com/doc"),
|
|
content_layer=ContentLayer.FURNITURE,
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_element_head_href_before_layer():
|
|
"""Element head follows spec order: href precedes layer."""
|
|
doc = _doc_text_with_hyperlink_and_layer()
|
|
ser_txt = (
|
|
DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, add_location=False, layer_mode=LayerMode.AUTO),
|
|
)
|
|
.serialize()
|
|
.text
|
|
)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/text_with_hyperlink_and_layer.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
assert ser_txt.index("<href") < ser_txt.index("<layer")
|
|
|
|
|
|
# ===============================
|
|
# Known spec gaps (validator-OK)
|
|
# ===============================
|
|
|
|
|
|
def _doc_multi_prov_text() -> DoclingDocument:
|
|
"""Single text item with two provenance spans (same page, two bboxes)."""
|
|
doc = DoclingDocument(name="multi_prov_thread")
|
|
# Page width must cover bbox right edges (810); height must cover bottom (50).
|
|
doc.add_page(page_no=1, size=Size(width=820, height=100), image=None)
|
|
part1 = "This paragraph starts on the left column and "
|
|
part2 = "continues in the right one."
|
|
text = part1 + part2
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, len(part1)),
|
|
)
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text=text, prov=prov)
|
|
item.orig = text
|
|
item.prov.append(
|
|
ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((420, 10, 810, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(len(part1), len(text)),
|
|
)
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_multi_prov_text_emits_thread():
|
|
"""Multi-prov text fragments share a ``thread_id`` in each element head.
|
|
|
|
Materializes ``input.json`` and ``serialized.dclg.xml`` under
|
|
``tests/data/doc/multi_prov_thread/``.
|
|
"""
|
|
data_dir = Path("./tests/data/doc/multi_prov_thread")
|
|
doc = _doc_multi_prov_text()
|
|
_verify_doc(doc=doc, exp_json=data_dir / "input.json")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(exp_file=data_dir / "serialized.dclg.xml", actual=ser_txt)
|
|
if not GEN_TEST_DATA:
|
|
assert ser_txt.count("<thread thread_id=") == 2
|
|
assert "<page_break" not in ser_txt
|
|
|
|
|
|
@doclang_validator
|
|
def test_thread_ids_unique_across_fragmented_components():
|
|
"""Each fragmented component receives a distinct document-scoped ``thread_id``."""
|
|
doc = _doc_cross_page_paragraph()
|
|
lg = doc.add_list_group(parent=doc.body)
|
|
doc.add_list_item(
|
|
text="First list item on page one.",
|
|
parent=lg,
|
|
marker="•",
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 60, 400, 100), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 30),
|
|
),
|
|
)
|
|
doc.add_list_item(
|
|
text="Second list item on page two.",
|
|
parent=lg,
|
|
marker="•",
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=2,
|
|
bbox=BoundingBox.from_tuple((10, 60, 400, 100), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 31),
|
|
),
|
|
)
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
assert ser_txt.count('thread_id="1"') == 2 # cross-page paragraph fragments
|
|
assert ser_txt.count('thread_id="2"') == 2 # cross-page list fragments
|
|
assert 'thread_id="3"' not in ser_txt
|
|
|
|
|
|
def _doc_cross_page_paragraph() -> DoclingDocument:
|
|
"""Single text item split across two pages via multi-prov charspans."""
|
|
doc = DoclingDocument(name="cross_page_paragraph")
|
|
for page_no in (1, 2):
|
|
doc.add_page(page_no=page_no, size=Size(width=512, height=512), image=None)
|
|
part1 = "This paragraph starts on page one and "
|
|
part2 = "continues on page two."
|
|
text = part1 + part2
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, len(part1)),
|
|
)
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text=text, prov=prov)
|
|
item.orig = text
|
|
item.prov.append(
|
|
ProvenanceItem(
|
|
page_no=2,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(len(part1), len(text)),
|
|
)
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_page_paragraph_emits_thread_and_page_break():
|
|
"""Cross-page paragraph: threaded ``<text>`` fragments with ``<page_break/>`` between pages.
|
|
|
|
Materializes ``input.json`` and ``serialized.dclg.xml`` under
|
|
``tests/data/doc/cross_page_paragraph/``.
|
|
"""
|
|
data_dir = Path("./tests/data/doc/cross_page_paragraph")
|
|
doc = _doc_cross_page_paragraph()
|
|
_verify_doc(doc=doc, exp_json=data_dir / "input.json")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(exp_file=data_dir / "serialized.dclg.xml", actual=ser_txt)
|
|
if not GEN_TEST_DATA:
|
|
assert ser_txt.count("<thread thread_id=") == 2
|
|
assert ser_txt.count("<page_break") == 1
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_page_paragraph_keeps_following_page_one_picture_before_break():
|
|
"""A picture following a cross-page paragraph remains on its provenance page."""
|
|
doc = DoclingDocument(name="cross_page_paragraph_with_picture")
|
|
for page_no in (10, 11):
|
|
doc.add_page(page_no=page_no, size=Size(width=612, height=792), image=None)
|
|
|
|
page_10_text = "Paragraph starts on page ten. "
|
|
page_11_text = "Paragraph continues on page eleven."
|
|
full_text = page_10_text + page_11_text
|
|
paragraph = doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=full_text,
|
|
orig=full_text,
|
|
prov=ProvenanceItem(
|
|
page_no=10,
|
|
bbox=BoundingBox.from_tuple((32, 88, 284, 38), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, len(page_10_text)),
|
|
),
|
|
)
|
|
paragraph.prov.append(
|
|
ProvenanceItem(
|
|
page_no=11,
|
|
bbox=BoundingBox.from_tuple((43, 726, 294, 676), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(len(page_10_text), len(full_text)),
|
|
)
|
|
)
|
|
|
|
picture = doc.add_picture(
|
|
prov=ProvenanceItem(
|
|
page_no=10,
|
|
bbox=BoundingBox.from_tuple((313, 353, 541, 98), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, 0),
|
|
)
|
|
)
|
|
caption_text = "Figure 7 caption"
|
|
caption = doc.add_text(
|
|
label=DocItemLabel.CAPTION,
|
|
text=caption_text,
|
|
parent=picture,
|
|
prov=ProvenanceItem(
|
|
page_no=10,
|
|
bbox=BoundingBox.from_tuple((302, 88, 553, 39), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, len(caption_text)),
|
|
),
|
|
)
|
|
picture.captions.append(caption.get_ref())
|
|
|
|
assert doc.body.children == [paragraph.get_ref(), picture.get_ref()]
|
|
assert [prov.page_no for prov in paragraph.prov] == [10, 11]
|
|
assert picture.prov[0].page_no == 10
|
|
assert caption.parent == picture.get_ref()
|
|
assert caption.prov[0].page_no == 10
|
|
|
|
root = ET.fromstring(serialize_doclang(doc))
|
|
assert [child.tag for child in root] == [
|
|
"text",
|
|
"picture",
|
|
"page_break",
|
|
"text",
|
|
]
|
|
caption_element = root.find("picture/caption")
|
|
assert caption_element is not None
|
|
assert "".join(caption_element.itertext()).strip() == caption_text
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_page_paragraph_keeps_grouped_page_six_footer_before_break():
|
|
"""A page-six footer in a transparent group stays before the page break."""
|
|
doc = DoclingDocument(name="cross_page_paragraph_with_grouped_footer")
|
|
for page_no in (6, 7):
|
|
doc.add_page(page_no=page_no, size=Size(width=612, height=792), image=None)
|
|
|
|
group = doc.add_group(name="section")
|
|
page_6_text = "Paragraph starts on page six. "
|
|
page_7_text = "Paragraph continues on page seven."
|
|
full_text = page_6_text + page_7_text
|
|
paragraph = doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text=full_text,
|
|
orig=full_text,
|
|
parent=group,
|
|
prov=ProvenanceItem(
|
|
page_no=6,
|
|
bbox=BoundingBox.from_tuple((32, 88, 284, 38), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, len(page_6_text)),
|
|
),
|
|
)
|
|
paragraph.prov.append(
|
|
ProvenanceItem(
|
|
page_no=7,
|
|
bbox=BoundingBox.from_tuple((43, 726, 294, 676), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(len(page_6_text), len(full_text)),
|
|
)
|
|
)
|
|
|
|
footer_text = "nnh: number of neuron in the hidden layer."
|
|
doc.add_text(
|
|
label=DocItemLabel.PAGE_FOOTER,
|
|
text=footer_text,
|
|
parent=group,
|
|
content_layer=ContentLayer.FURNITURE,
|
|
prov=ProvenanceItem(
|
|
page_no=6,
|
|
bbox=BoundingBox.from_tuple((32, 24, 284, 10), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, len(footer_text)),
|
|
),
|
|
)
|
|
header_text = "Header on page seven"
|
|
doc.add_text(
|
|
label=DocItemLabel.PAGE_HEADER,
|
|
text=header_text,
|
|
parent=group,
|
|
content_layer=ContentLayer.FURNITURE,
|
|
prov=ProvenanceItem(
|
|
page_no=7,
|
|
bbox=BoundingBox.from_tuple((32, 780, 284, 760), origin=CoordOrigin.BOTTOMLEFT),
|
|
charspan=(0, len(header_text)),
|
|
),
|
|
)
|
|
|
|
root = ET.fromstring(serialize_doclang(doc))
|
|
assert [child.tag for child in root] == [
|
|
"text",
|
|
"page_footer",
|
|
"page_break",
|
|
"text",
|
|
"page_header",
|
|
]
|
|
footer_element = root.find("page_footer")
|
|
assert footer_element is not None
|
|
assert "".join(footer_element.itertext()).strip() == footer_text
|
|
|
|
|
|
def _doc_cross_page_list() -> DoclingDocument:
|
|
"""List group with whole items on page 1 and page 2 (list-level fragmentation)."""
|
|
doc = DoclingDocument(name="cross_page_list")
|
|
for page_no in (1, 2):
|
|
doc.add_page(page_no=page_no, size=Size(width=512, height=512), image=None)
|
|
lg = doc.add_list_group(parent=doc.body)
|
|
doc.add_list_item(
|
|
text="First item on page one.",
|
|
parent=lg,
|
|
marker="•",
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 24),
|
|
),
|
|
)
|
|
doc.add_list_item(
|
|
text="Second item on page two.",
|
|
parent=lg,
|
|
marker="•",
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=2,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 25),
|
|
),
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_page_list_emits_thread_and_page_break():
|
|
"""Cross-page list: threaded ``<list>`` blocks with ``<page_break/>`` between pages."""
|
|
data_dir = Path("./tests/data/doc/cross_page_list")
|
|
doc = _doc_cross_page_list()
|
|
_verify_doc(doc=doc, exp_json=data_dir / "input.json")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(exp_file=data_dir / "serialized.dclg.xml", actual=ser_txt)
|
|
if not GEN_TEST_DATA:
|
|
assert ser_txt.count("<thread thread_id=") == 2
|
|
assert ser_txt.count("<page_break") == 1
|
|
assert ser_txt.count("<list") == 2
|
|
|
|
|
|
def _doc_cross_page_table() -> DoclingDocument:
|
|
"""Single table split across two pages via multi-prov bounding boxes."""
|
|
doc = DoclingDocument(name="cross_page_table")
|
|
for page_no in (1, 2):
|
|
doc.add_page(page_no=page_no, size=Size(width=512, height=512), image=None)
|
|
data = TableData(num_cols=2)
|
|
data.add_row(["H1", "H2"])
|
|
data.add_row(["A", "B"])
|
|
data.grid[0][0].column_header = True
|
|
data.grid[0][1].column_header = True
|
|
prov = ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 200), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 0),
|
|
)
|
|
item = doc.add_table(data=data, prov=prov)
|
|
item.prov.append(
|
|
ProvenanceItem(
|
|
page_no=2,
|
|
bbox=BoundingBox.from_tuple((10, 10, 400, 200), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 0),
|
|
)
|
|
)
|
|
return doc
|
|
|
|
|
|
def _doc_cross_column_list() -> DoclingDocument:
|
|
"""List group with one item per column on the same page (cross-column layout)."""
|
|
doc = DoclingDocument(name="cross_column_list")
|
|
doc.add_page(page_no=1, size=Size(width=512, height=512), image=None)
|
|
lg = doc.add_list_group(parent=doc.body)
|
|
doc.add_list_item(
|
|
text="Left column item.",
|
|
parent=lg,
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((10, 10, 200, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 18),
|
|
),
|
|
)
|
|
doc.add_list_item(
|
|
text="Right column item.",
|
|
parent=lg,
|
|
enumerated=False,
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox.from_tuple((220, 10, 400, 50), origin=CoordOrigin.TOPLEFT),
|
|
charspan=(0, 19),
|
|
),
|
|
)
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_page_table_emits_thread_and_page_break():
|
|
"""Cross-page table: threaded ``<table>`` fragments with ``<page_break/>`` between pages."""
|
|
data_dir = Path("./tests/data/doc/cross_page_table")
|
|
doc = _doc_cross_page_table()
|
|
_verify_doc(doc=doc, exp_json=data_dir / "input.json")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(exp_file=data_dir / "serialized.dclg.xml", actual=ser_txt)
|
|
if not GEN_TEST_DATA:
|
|
assert ser_txt.count("<thread thread_id=") == 2
|
|
assert ser_txt.count("<page_break") == 1
|
|
assert ser_txt.count("<table") == 2
|
|
|
|
|
|
@doclang_validator
|
|
def test_cross_column_list_same_page():
|
|
"""Cross-column list: both items on one page with column-aligned locations."""
|
|
data_dir = Path("./tests/data/doc/cross_column_list")
|
|
doc = _doc_cross_column_list()
|
|
_verify_doc(doc=doc, exp_json=data_dir / "input.json")
|
|
|
|
ser_txt = serialize_doclang(doc)
|
|
verify_doclang(exp_file=data_dir / "serialized.dclg.xml", actual=ser_txt)
|
|
if not GEN_TEST_DATA:
|
|
assert ser_txt.count("<list") == 1
|
|
assert "<thread" not in ser_txt
|
|
assert "<page_break" not in ser_txt
|
|
assert "<marker>" not in ser_txt
|
|
|
|
|
|
# ===============================
|
|
# DocLang vocabulary tokens: rtl, corn, xcel
|
|
# ===============================
|
|
|
|
|
|
def _doc_rtl_text() -> DoclingDocument:
|
|
doc = DoclingDocument(name="rtl_text")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="مرحبا")
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_rtl_text_formatting():
|
|
"""RTL script text is wrapped in <rtl> formatting."""
|
|
doc = _doc_rtl_text()
|
|
ser_txt = (
|
|
DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, add_location=False),
|
|
)
|
|
.serialize()
|
|
.text
|
|
)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/rtl_text.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
def _doc_table_xcel_span() -> DoclingDocument:
|
|
doc = DoclingDocument(name="table_xcel")
|
|
cells = [
|
|
TableCell(
|
|
text="A",
|
|
row_span=2,
|
|
col_span=2,
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=2,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=2,
|
|
),
|
|
]
|
|
doc.add_table(data=TableData(num_rows=2, num_cols=2, table_cells=cells))
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_table_xcel_span():
|
|
"""2D table spans emit <xcel> only at interior continuation positions.
|
|
|
|
OTSL rule: ``<xcel>`` marks a cell that continues both the left and upper
|
|
neighbor; it must not appear in the first row or first column of the grid.
|
|
"""
|
|
doc = _doc_table_xcel_span()
|
|
ser_txt = (
|
|
DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, add_location=False, add_table_cell_location=False),
|
|
)
|
|
.serialize()
|
|
.text
|
|
)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/table_xcel.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
if not GEN_TEST_DATA:
|
|
# Row 0: origin + horizontal continuation; row 1: vertical + cross.
|
|
assert "<lcel/>" in ser_txt
|
|
assert "<ucel/>" in ser_txt
|
|
assert ser_txt.index("<xcel/>") > ser_txt.index("<ucel/>")
|
|
|
|
|
|
# TODO: Extend DocLang/OTSL validation for advanced multispan rules (e.g. reject
|
|
# <xcel> in the first row or column, validate lcel/ucel/xcel placement against
|
|
# the OTSL grammar from the DocLang spec).
|
|
|
|
|
|
def _doc_table_corn_header() -> DoclingDocument:
|
|
doc = DoclingDocument(name="table_corn")
|
|
cells = [
|
|
TableCell(
|
|
text="",
|
|
row_span=1,
|
|
col_span=1,
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
column_header=True,
|
|
row_header=True,
|
|
),
|
|
TableCell(
|
|
text="Col1",
|
|
row_span=1,
|
|
col_span=1,
|
|
start_row_offset_idx=0,
|
|
end_row_offset_idx=1,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
column_header=True,
|
|
),
|
|
TableCell(
|
|
text="Row1",
|
|
row_span=1,
|
|
col_span=1,
|
|
start_row_offset_idx=1,
|
|
end_row_offset_idx=2,
|
|
start_col_offset_idx=0,
|
|
end_col_offset_idx=1,
|
|
row_header=True,
|
|
),
|
|
TableCell(
|
|
text="Data",
|
|
row_span=1,
|
|
col_span=1,
|
|
start_row_offset_idx=1,
|
|
end_row_offset_idx=2,
|
|
start_col_offset_idx=1,
|
|
end_col_offset_idx=2,
|
|
),
|
|
]
|
|
doc.add_table(data=TableData(num_rows=2, num_cols=2, table_cells=cells))
|
|
return doc
|
|
|
|
|
|
@doclang_validator
|
|
def test_table_corn_header():
|
|
"""Top-left header intersection emits <corn> OTSL token."""
|
|
doc = _doc_table_corn_header()
|
|
ser_txt = (
|
|
DocLangDocSerializer(
|
|
doc=doc,
|
|
params=DocLangParams(include_version=False, add_location=False, add_table_cell_location=False),
|
|
)
|
|
.serialize()
|
|
.text
|
|
)
|
|
verify_doclang(
|
|
exp_file=Path("./tests/data/doc/table_corn.gt.dclg.xml"),
|
|
actual=ser_txt,
|
|
)
|
|
|
|
|
|
def test_create_threading_token_emits_thread_id():
|
|
"""Continuation tokens use ``thread_id`` per DocLang v0.5."""
|
|
import pytest
|
|
|
|
from docling_core.transforms.serializer.doclang import DocLangVocabulary
|
|
|
|
assert DocLangVocabulary._create_threading_token(thread_id="42") == '<thread thread_id="42"/>'
|
|
with pytest.raises(ValueError, match="thread_id length"):
|
|
DocLangVocabulary._create_threading_token(thread_id="")
|
|
|
|
|
|
def test_inline_group_nested_when_parent_text_item_has_text():
|
|
"""Runs of an InlineGroup stay inside the parent element even if it has text.
|
|
|
|
Regression test for #750: when the parent ``TextItem`` carries text of its
|
|
own, the inline runs used to be emitted bare at the parent's sibling level,
|
|
which is invalid against the DocLang XSD (``group`` is not mixed) and drops
|
|
the plain-text run on deserialization.
|
|
"""
|
|
doc = DoclingDocument(name="t")
|
|
heading = doc.add_heading(text="Heading text", level=2)
|
|
inline = doc.add_inline_group(parent=heading)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="bold",
|
|
parent=inline,
|
|
formatting=Formatting(bold=True),
|
|
)
|
|
doc.add_text(label=DocItemLabel.TEXT, text=" plain ", parent=inline)
|
|
doc.add_text(
|
|
label=DocItemLabel.TEXT,
|
|
text="italic",
|
|
parent=inline,
|
|
formatting=Formatting(italic=True),
|
|
)
|
|
|
|
txt = serialize_doclang(doc)
|
|
|
|
root = ET.fromstring(txt)
|
|
assert [child.tag for child in root] == ["heading"]
|
|
heading_el = root[0]
|
|
assert [child.tag for child in heading_el] == ["bold", "content", "italic"]
|
|
assert "".join(heading_el.itertext()).replace("\n", " ").split() == [
|
|
"Heading",
|
|
"text",
|
|
"bold",
|
|
"plain",
|
|
"italic",
|
|
]
|
|
assert " plain " in txt
|
|
|
|
|
|
def test_inline_group_nested_preserves_checkbox_and_handwriting():
|
|
"""The checkbox token and handwriting wrap survive when text has an InlineGroup child.
|
|
|
|
Regression test: a ``CHECKBOX_SELECTED``/``HANDWRITTEN_TEXT`` item that has
|
|
both its own text and an ``InlineGroup`` child used to silently drop the
|
|
checkbox state token / ``<handwriting>`` wrap, since that label-specific
|
|
handling only lived in the branch that runs when there is no InlineGroup
|
|
child.
|
|
"""
|
|
doc = DoclingDocument(name="t")
|
|
checkbox = doc.add_text(label=DocItemLabel.CHECKBOX_SELECTED, text="Yes")
|
|
inline = doc.add_inline_group(parent=checkbox)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="confirmed", parent=inline, formatting=Formatting(bold=True))
|
|
|
|
txt = serialize_doclang(doc)
|
|
assert "<checkbox" in txt
|
|
assert "confirmed" in txt
|
|
|
|
doc2 = DoclingDocument(name="t")
|
|
handwritten = doc2.add_text(label=DocItemLabel.HANDWRITTEN_TEXT, text="Signed by")
|
|
inline2 = doc2.add_inline_group(parent=handwritten)
|
|
doc2.add_text(label=DocItemLabel.TEXT, text="John", parent=inline2, formatting=Formatting(bold=True))
|
|
|
|
txt2 = serialize_doclang(doc2)
|
|
assert "<handwriting>" in txt2
|
|
assert "John" in txt2
|
|
|
|
|
|
def test_inline_group_nested_word_boundary_when_minimized():
|
|
"""A word boundary is kept between own text and inline runs in minimized mode.
|
|
|
|
Regression test: joining the item's own text with the InlineGroup's runs
|
|
used to reuse the pretty-printing record delimiter, which is the empty
|
|
string when ``pretty_indentation=None`` (minimized serialization), running
|
|
the two words together with no separator.
|
|
"""
|
|
doc = DoclingDocument(name="t")
|
|
heading = doc.add_heading(text="Heading text", level=2)
|
|
inline = doc.add_inline_group(parent=heading)
|
|
doc.add_text(label=DocItemLabel.TEXT, text="bold", parent=inline, formatting=Formatting(bold=True))
|
|
|
|
txt = serialize_doclang(doc, params=DocLangParams(include_version=False, pretty_indentation=None))
|
|
assert "Heading textbold" not in txt
|
|
|
|
root = ET.fromstring(txt)
|
|
heading_el = root[0]
|
|
assert "".join(heading_el.itertext()).split() == ["Heading", "text", "bold"]
|