Files
Cesar Berrospi Ramis a2bf024b43 feat(serializer): add LaTeX meta serialization, fix legacy annotation warnings (#871)
* test: catch expected UserWarnings in unit tests

Catch intentional UserWarnings across chunker, expander, and document
metadata tests using  to assert that warnings are correctly
triggered and suppress warning noise during test runs

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* test(serialization): fix the wrong way to build lists

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(doctags): suppress deprecation warning when populating legacy annotations

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* feat(latex): add LaTeXMetaSerializer and migrate metadata handling

Introduce 'LaTeXMetaSerializer' to serialize 'item.meta' fields as LaTeX comments.
Update 'LaTeXPictureSerializer' and 'LaTeXTableSerializer' to prioritize 'item.meta'
and only fall back to legacy annotations when 'meta' is absent.
Update tabular data serialization on pictures to use 'item.meta.tabular_chart'.
Add unit tests covering metadata serialization, legacy annotations fallback, and
tabular chart export in LaTeX.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(serializer): repair _should_use_legacy_annotations warning handling

Two bugs prevented correct warning scoping:
- warnings.simplefilter('ignore', DeprecationWarning) passed DeprecationWarning as lineno
  rather than category, making the ignore filter a no-op. In Python 3.14, nested
  catch_warnings no longer isolates the outer filter stack, so the deprecation
  from accessing item.annotations leaked to callers unconditionally.
- The re-emit loop was inside the catch_warnings block, causing
  infinite recursing when the outer context had simplefilter('always') active.
Replace the 'catch_warnings' approach with '__dict__' access for both
deprecated fields ('include_annotations' and 'annotations') to bypass
their Pydantic property descriptors. Emit the deprecation explicitly
only when 'use_legacy' is True.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(serializer): correct include_annotations deprecation message

The previous message ("Use include_meta instead.") pointed to a field
that does not exist. Since the PR narrowed include_annotations to only
affect legacy annotations, update the message to accurately describe
the current behaviour and point to blocked_meta_names as the modern
alternative for suppressing specific meta fields.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* docs(latex): add documentation to option fields

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

---------

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
2026-10-15 13:21:17 +02:00

426 lines
17 KiB
Python

from pathlib import Path
from typing import Any
import pytest
from pydantic import BaseModel, ValidationError
from typing_extensions import override
from docling_core.transforms.serializer.base import SerializationResult
from docling_core.transforms.serializer.common import create_ser_result
from docling_core.transforms.serializer.html import HTMLDocSerializer, HTMLParams
from docling_core.transforms.serializer.markdown import (
MarkdownDocSerializer,
MarkdownMetaSerializer,
MarkdownParams,
)
from docling_core.types.doc import (
BaseMeta,
DocItem,
DocItemLabel,
DoclingDocument,
EntitiesMetaField,
EntityMention,
GroupLabel,
HumanLanguageLabel,
KeywordsMetaField,
LanguageMetaField,
MetaFieldName,
MetaUtils,
NodeItem,
PictureClassificationClass,
PictureClassificationData,
PictureClassificationMetaField,
PictureClassificationPrediction,
PictureMeta,
RefItem,
SummaryMetaField,
TopicsMetaField,
)
from .test_data_gen_flag import GEN_TEST_DATA
from .test_utils import assert_or_generate_ground_truth
class CustomCoordinates(BaseModel):
longitude: float
latitude: float
@pytest.fixture(scope="module")
def dummy_doc_with_meta() -> DoclingDocument:
"""Fixture that loads dummy_doc_with_meta.yaml once per module."""
src = Path("tests/data/doc/dummy_doc_with_meta.yaml")
return DoclingDocument.load_from_yaml(filename=src)
@pytest.fixture(scope="module")
def doc_with_group_with_metadata() -> DoclingDocument:
"""Fixture that creates a document with groups and metadata once per module."""
doc = DoclingDocument(name="")
doc.body.meta = BaseMeta(summary=SummaryMetaField(text="This document talks about various topics."))
grp1 = doc.add_group(name="1", label=GroupLabel.CHAPTER)
grp1.meta = BaseMeta(summary=SummaryMetaField(text="This chapter discusses foo and bar."))
doc.add_text(text="This is some introductory text.", label=DocItemLabel.TEXT, parent=grp1)
grp1a = doc.add_group(parent=grp1, name="1a", label=GroupLabel.SECTION)
grp1a.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about foo."))
grp1a.meta.set_custom_field(namespace="my_corp", name="test_1", value="custom field value 1")
txt1 = doc.add_text(text="Regarding foo...", label=DocItemLabel.TEXT, parent=grp1a)
txt1.meta = BaseMeta(summary=SummaryMetaField(text="This paragraph provides more details about foo."))
lst1a = doc.add_list_group(parent=grp1a)
lst1a.meta = BaseMeta(summary=SummaryMetaField(text="Here some foo specifics are listed."))
doc.add_list_item(text="lorem", parent=lst1a, enumerated=True)
doc.add_list_item(text="ipsum", parent=lst1a, enumerated=True)
grp1b = doc.add_group(parent=grp1, name="1b", label=GroupLabel.SECTION)
grp1b.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about bar."))
grp1b.meta.set_custom_field(namespace="my_corp", name="test_2", value="custom field value 2")
doc.add_text(text="Regarding bar...", label=DocItemLabel.TEXT, parent=grp1b)
return doc
def test_metadata_usage(dummy_doc_with_meta: DoclingDocument) -> None:
doc = dummy_doc_with_meta.model_copy(deep=True)
first_pic = doc.pictures[0]
assert first_pic.meta
assert first_pic.meta.classification
assert first_pic.meta.classification.predictions
assert first_pic.meta.classification.predictions[0].confidence == 0.78
example_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=doc)
assert example_item.meta is not None
# add a custom metadata object to the item
value = CustomCoordinates(longitude=47.3769, latitude=8.5417)
target_name = example_item.meta.set_custom_field(namespace="my_corp", name="coords", value=value)
assert target_name == "my_corp__coords"
# save the document
exp_file = Path("tests/data/doc/dummy_doc_with_meta_modified.yaml")
if GEN_TEST_DATA:
doc.save_as_yaml(filename=exp_file)
else:
expected = DoclingDocument.load_from_yaml(filename=exp_file)
assert doc.model_dump(mode="json") == expected.model_dump(mode="json")
# load back the document and read the custom metadata object
loaded_doc = DoclingDocument.load_from_yaml(filename=exp_file)
loaded_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=loaded_doc)
assert loaded_item.meta is not None
loaded_dict = loaded_item.meta.get_custom_part()[target_name]
loaded_value = CustomCoordinates.model_validate(loaded_dict)
# ensure the value is the same
assert loaded_value == value
def test_metadata_relaxed_migration() -> None:
src = Path("tests/data/doc/dummy_doc_with_meta_2.yaml")
with pytest.warns(UserWarning, match=r"outside page bounds.*clamping to"):
doc = DoclingDocument.load_from_yaml(filename=src)
first_pic = doc.pictures[0]
assert first_pic.meta
assert first_pic.meta.classification
assert first_pic.meta.classification.predictions
# check migration was skipped since respetive meta already present:
assert first_pic.meta.classification.predictions[0].confidence == 0.42
def test_namespace_absence_raises(dummy_doc_with_meta: DoclingDocument):
example_item = RefItem(cref="#/texts/2").resolve(doc=dummy_doc_with_meta)
with pytest.raises(ValueError):
example_item.meta.my_corp_programmaticaly_added_field = True
def test_ser_deser(doc_with_group_with_metadata: DoclingDocument):
doc = doc_with_group_with_metadata
# test dumping to and loading from YAML
exp_file = Path("tests/data/doc/group_with_metadata.yaml")
if GEN_TEST_DATA:
doc.save_as_yaml(filename=exp_file)
else:
expected = DoclingDocument.load_from_yaml(filename=exp_file)
assert doc == expected
def test_md_ser_default(doc_with_group_with_metadata: DoclingDocument):
# test exporting to Markdown
doc = doc_with_group_with_metadata
params = MarkdownParams()
ser = MarkdownDocSerializer(doc=doc, params=params)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_default.md")
assert_or_generate_ground_truth(actual, exp_file)
def test_md_ser_marked(doc_with_group_with_metadata: DoclingDocument):
# test exporting to Markdown
doc = doc_with_group_with_metadata
params = MarkdownParams(
mark_meta=True,
)
ser = MarkdownDocSerializer(doc=doc, params=params)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_marked.md")
if GEN_TEST_DATA:
with open(exp_file, "w", encoding="utf-8") as f:
f.write(actual)
else:
with open(exp_file, encoding="utf-8") as f:
expected = f.read()
assert actual == expected
def test_md_ser_allowed_meta_names(doc_with_group_with_metadata: DoclingDocument):
params = MarkdownParams(
allowed_meta_names={
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
},
mark_meta=True,
)
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_allowed_meta_names.md")
assert_or_generate_ground_truth(actual, exp_file)
def test_md_ser_blocked_meta_names(doc_with_group_with_metadata: DoclingDocument):
params = MarkdownParams(
blocked_meta_names={
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
MetaFieldName.SUMMARY.value,
},
mark_meta=True,
)
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_blocked_meta_names.md")
assert_or_generate_ground_truth(actual, exp_file)
def test_md_ser_without_non_meta(doc_with_group_with_metadata: DoclingDocument):
params = MarkdownParams(
include_non_meta=False,
mark_meta=True,
)
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_without_non_meta.md")
assert_or_generate_ground_truth(actual, exp_file)
def test_md_ser_include_picture_classification_meta_path():
doc = DoclingDocument(name="test")
pic = doc.add_picture()
pic.meta = PictureMeta(
classification=PictureClassificationMetaField(
predictions=[PictureClassificationPrediction(class_name="chart", confidence=0.9)]
)
)
default_text = doc.export_to_markdown()
assert "Chart" in default_text
suppressed_text = doc.export_to_markdown(include_picture_classification=False)
assert "Chart" not in suppressed_text
assert "<!-- image -->" in suppressed_text
def test_md_ser_include_picture_classification_legacy_path():
doc = DoclingDocument(name="test")
pic = doc.add_picture()
with pytest.warns(DeprecationWarning, match=r"Field `annotations` is deprecated"):
pic.annotations.append(
PictureClassificationData(
provenance="test",
predicted_classes=[PictureClassificationClass(class_name="chart", confidence=0.9)],
)
)
with pytest.warns(DeprecationWarning):
default_text = doc.export_to_markdown()
assert "chart" in default_text
with pytest.warns(DeprecationWarning):
suppressed_text = doc.export_to_markdown(include_picture_classification=False)
assert "chart" not in suppressed_text
assert "<!-- image -->" in suppressed_text
def test_ser_custom_meta_serializer(doc_with_group_with_metadata: DoclingDocument):
class SummaryMarkdownMetaSerializer(MarkdownMetaSerializer):
@override
def serialize(
self,
*,
item: NodeItem,
doc: DoclingDocument,
level: int | None = None,
**kwargs: Any,
) -> SerializationResult:
"""Serialize the item's meta."""
params = MarkdownParams(**kwargs)
return create_ser_result(
text="\n\n".join(
[
f"{' ' * (level or 0)}[{item.self_ref}] [{item.__class__.__name__}:{item.label.value}] {tmp}" # type:ignore[attr-defined]
for key in (list(item.meta.__class__.model_fields) + list(item.meta.get_custom_part()))
if (tmp := self._serialize_meta_field(item.meta, key, params.mark_meta))
]
if item.meta
else []
),
span_source=item if isinstance(item, DocItem) else [],
)
def _serialize_meta_field(self, meta: BaseMeta, name: str, mark_meta: bool, **kwargs: Any) -> str | None:
if (field_val := getattr(meta, name)) is not None and isinstance(field_val, SummaryMetaField):
txt = field_val.text
return f"[{self._humanize_text(name, title=True)}] {txt}" if mark_meta else txt
else:
return None
# test exporting to Markdown
params = MarkdownParams(
include_non_meta=False,
)
ser = MarkdownDocSerializer(
doc=doc_with_group_with_metadata, params=params, meta_serializer=SummaryMarkdownMetaSerializer()
)
ser_res = ser.serialize()
actual = ser_res.text
exp_file = Path("tests/data/doc/group_with_metadata_summaries.md")
assert_or_generate_ground_truth(actual, exp_file)
def test_document_level_metadata(dummy_doc_with_meta: DoclingDocument) -> None:
"""Test that document-level metadata can be loaded and accessed through 'body' field."""
# Verify document-level metadata exists
assert dummy_doc_with_meta.body.meta is not None
assert dummy_doc_with_meta.body.meta.summary is not None
assert (
dummy_doc_with_meta.body.meta.summary.text == "This is a document-level summary describing the entire document."
)
assert dummy_doc_with_meta.body.meta.summary.confidence == 0.98
# Verify custom metadata fields at document level
custom_part = dummy_doc_with_meta.body.meta.get_custom_part()
assert custom_part["my_corp__doc_category"] == "technical_report"
assert custom_part["my_corp__doc_version"] == "1.0"
# Verify that item-level metadata still works alongside document-level metadata
first_text = dummy_doc_with_meta.texts[1] # The title item
assert first_text.meta is not None
assert first_text.meta.summary is not None
assert first_text.meta.summary.text == "This is a title."
def test_semantic_base_meta_fields_roundtrip_and_html_rendering() -> None:
doc = DoclingDocument(name="semantic-meta")
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
item.meta = BaseMeta(
summary=SummaryMetaField(text="A short company/location statement."),
language=LanguageMetaField(code=HumanLanguageLabel.EN),
entities=EntitiesMetaField(
mentions=[
EntityMention(text="IBM", label="ORG", charspan=(0, 3)),
EntityMention(text="Zurich", label="LOC", charspan=(16, 22)),
]
),
keywords=KeywordsMetaField(values=["ibm", "zurich", "company"]),
topics=TopicsMetaField(values=["business", "geography"]),
)
roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json"))
meta = roundtrip.texts[0].meta
assert meta is not None
assert meta.language is not None
assert meta.language.code == HumanLanguageLabel.EN
assert meta.entities is not None
assert [mention.text for mention in meta.entities.mentions] == ["IBM", "Zurich"]
assert meta.keywords is not None and meta.keywords.values == ["ibm", "zurich", "company"]
assert meta.topics is not None and meta.topics.values == ["business", "geography"]
assert meta.has_content()
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
assert 'data-meta-name="language"' in html
assert 'data-meta-name="entities"' in html
assert 'data-meta-name="keywords"' in html
assert 'data-meta-name="topics"' in html
assert ">en<" in html
assert "IBM (ORG, [0,3]), Zurich (LOC, [16,22])" in html
assert "ibm, zurich, company" in html
assert ">business, geography<" in html
# duplicate values are removed without rejection
assert KeywordsMetaField(values=["ai", "ml", "ai"]).values == ["ai", "ml"]
assert TopicsMetaField(values=["nlp", "nlp"]).values == ["nlp"]
def test_html_escapes_entity_text() -> None:
doc = DoclingDocument(name="escaped-entity-meta")
item = doc.add_text(label=DocItemLabel.TEXT, text="A<B & C> appears here.")
item.meta = BaseMeta(
entities=EntitiesMetaField(
mentions=[
EntityMention(text="A<B & C>", label="TAG", charspan=(0, 7)),
]
),
)
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
assert "A&lt;B &amp; C&gt; (TAG, [0,7])" in html
def test_html_skips_empty_base_meta() -> None:
doc = DoclingDocument(name="empty-meta")
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
item.meta = BaseMeta()
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
assert '<details class="docling-meta">' not in html
assert "data-meta-entities" not in html
def test_html_escapes_keywords() -> None:
doc = DoclingDocument(name="kw-escape")
item = doc.add_text(label=DocItemLabel.TEXT, text="x")
item.meta = BaseMeta(keywords=KeywordsMetaField(values=["A<B & C>"]))
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
assert "A&lt;B &amp; C&gt;" in html
def test_md_marked_renders_keywords_and_topics() -> None:
doc = DoclingDocument(name="kw-md")
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
item.meta = BaseMeta(
keywords=KeywordsMetaField(values=["ibm", "zurich"]),
topics=TopicsMetaField(values=["business"]),
)
md = MarkdownDocSerializer(doc=doc, params=MarkdownParams(mark_meta=True)).serialize().text
assert "[Keywords] ibm, zurich" in md
assert "[Topics] business" in md
def test_keywords_topics_required_values() -> None:
with pytest.raises(ValidationError, match="at least 1 item"):
KeywordsMetaField(values=[])
with pytest.raises(ValidationError, match="at least 1 item"):
TopicsMetaField(values=[])
with pytest.raises(ValidationError, match="list of strings"):
TopicsMetaField(values=34)
with pytest.raises(ValidationError, match="valid string"):
TopicsMetaField(values=[34])