mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* test: catch expected UserWarnings in unit tests Catch intentional UserWarnings across chunker, expander, and document metadata tests using to assert that warnings are correctly triggered and suppress warning noise during test runs Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * test(serialization): fix the wrong way to build lists Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(doctags): suppress deprecation warning when populating legacy annotations Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * feat(latex): add LaTeXMetaSerializer and migrate metadata handling Introduce 'LaTeXMetaSerializer' to serialize 'item.meta' fields as LaTeX comments. Update 'LaTeXPictureSerializer' and 'LaTeXTableSerializer' to prioritize 'item.meta' and only fall back to legacy annotations when 'meta' is absent. Update tabular data serialization on pictures to use 'item.meta.tabular_chart'. Add unit tests covering metadata serialization, legacy annotations fallback, and tabular chart export in LaTeX. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(serializer): repair _should_use_legacy_annotations warning handling Two bugs prevented correct warning scoping: - warnings.simplefilter('ignore', DeprecationWarning) passed DeprecationWarning as lineno rather than category, making the ignore filter a no-op. In Python 3.14, nested catch_warnings no longer isolates the outer filter stack, so the deprecation from accessing item.annotations leaked to callers unconditionally. - The re-emit loop was inside the catch_warnings block, causing infinite recursing when the outer context had simplefilter('always') active. Replace the 'catch_warnings' approach with '__dict__' access for both deprecated fields ('include_annotations' and 'annotations') to bypass their Pydantic property descriptors. Emit the deprecation explicitly only when 'use_legacy' is True. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(serializer): correct include_annotations deprecation message The previous message ("Use include_meta instead.") pointed to a field that does not exist. Since the PR narrowed include_annotations to only affect legacy annotations, update the message to accurately describe the current behaviour and point to blocked_meta_names as the modern alternative for suppressing specific meta fields. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * docs(latex): add documentation to option fields Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
426 lines
17 KiB
Python
426 lines
17 KiB
Python
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import BaseModel, ValidationError
|
|
from typing_extensions import override
|
|
|
|
from docling_core.transforms.serializer.base import SerializationResult
|
|
from docling_core.transforms.serializer.common import create_ser_result
|
|
from docling_core.transforms.serializer.html import HTMLDocSerializer, HTMLParams
|
|
from docling_core.transforms.serializer.markdown import (
|
|
MarkdownDocSerializer,
|
|
MarkdownMetaSerializer,
|
|
MarkdownParams,
|
|
)
|
|
from docling_core.types.doc import (
|
|
BaseMeta,
|
|
DocItem,
|
|
DocItemLabel,
|
|
DoclingDocument,
|
|
EntitiesMetaField,
|
|
EntityMention,
|
|
GroupLabel,
|
|
HumanLanguageLabel,
|
|
KeywordsMetaField,
|
|
LanguageMetaField,
|
|
MetaFieldName,
|
|
MetaUtils,
|
|
NodeItem,
|
|
PictureClassificationClass,
|
|
PictureClassificationData,
|
|
PictureClassificationMetaField,
|
|
PictureClassificationPrediction,
|
|
PictureMeta,
|
|
RefItem,
|
|
SummaryMetaField,
|
|
TopicsMetaField,
|
|
)
|
|
|
|
from .test_data_gen_flag import GEN_TEST_DATA
|
|
from .test_utils import assert_or_generate_ground_truth
|
|
|
|
|
|
class CustomCoordinates(BaseModel):
|
|
longitude: float
|
|
latitude: float
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def dummy_doc_with_meta() -> DoclingDocument:
|
|
"""Fixture that loads dummy_doc_with_meta.yaml once per module."""
|
|
src = Path("tests/data/doc/dummy_doc_with_meta.yaml")
|
|
return DoclingDocument.load_from_yaml(filename=src)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def doc_with_group_with_metadata() -> DoclingDocument:
|
|
"""Fixture that creates a document with groups and metadata once per module."""
|
|
doc = DoclingDocument(name="")
|
|
doc.body.meta = BaseMeta(summary=SummaryMetaField(text="This document talks about various topics."))
|
|
grp1 = doc.add_group(name="1", label=GroupLabel.CHAPTER)
|
|
grp1.meta = BaseMeta(summary=SummaryMetaField(text="This chapter discusses foo and bar."))
|
|
doc.add_text(text="This is some introductory text.", label=DocItemLabel.TEXT, parent=grp1)
|
|
|
|
grp1a = doc.add_group(parent=grp1, name="1a", label=GroupLabel.SECTION)
|
|
grp1a.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about foo."))
|
|
grp1a.meta.set_custom_field(namespace="my_corp", name="test_1", value="custom field value 1")
|
|
txt1 = doc.add_text(text="Regarding foo...", label=DocItemLabel.TEXT, parent=grp1a)
|
|
txt1.meta = BaseMeta(summary=SummaryMetaField(text="This paragraph provides more details about foo."))
|
|
lst1a = doc.add_list_group(parent=grp1a)
|
|
lst1a.meta = BaseMeta(summary=SummaryMetaField(text="Here some foo specifics are listed."))
|
|
doc.add_list_item(text="lorem", parent=lst1a, enumerated=True)
|
|
doc.add_list_item(text="ipsum", parent=lst1a, enumerated=True)
|
|
|
|
grp1b = doc.add_group(parent=grp1, name="1b", label=GroupLabel.SECTION)
|
|
grp1b.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about bar."))
|
|
grp1b.meta.set_custom_field(namespace="my_corp", name="test_2", value="custom field value 2")
|
|
doc.add_text(text="Regarding bar...", label=DocItemLabel.TEXT, parent=grp1b)
|
|
|
|
return doc
|
|
|
|
|
|
def test_metadata_usage(dummy_doc_with_meta: DoclingDocument) -> None:
|
|
doc = dummy_doc_with_meta.model_copy(deep=True)
|
|
|
|
first_pic = doc.pictures[0]
|
|
assert first_pic.meta
|
|
assert first_pic.meta.classification
|
|
assert first_pic.meta.classification.predictions
|
|
assert first_pic.meta.classification.predictions[0].confidence == 0.78
|
|
|
|
example_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=doc)
|
|
assert example_item.meta is not None
|
|
|
|
# add a custom metadata object to the item
|
|
value = CustomCoordinates(longitude=47.3769, latitude=8.5417)
|
|
target_name = example_item.meta.set_custom_field(namespace="my_corp", name="coords", value=value)
|
|
assert target_name == "my_corp__coords"
|
|
|
|
# save the document
|
|
exp_file = Path("tests/data/doc/dummy_doc_with_meta_modified.yaml")
|
|
if GEN_TEST_DATA:
|
|
doc.save_as_yaml(filename=exp_file)
|
|
else:
|
|
expected = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
assert doc.model_dump(mode="json") == expected.model_dump(mode="json")
|
|
|
|
# load back the document and read the custom metadata object
|
|
loaded_doc = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
loaded_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=loaded_doc)
|
|
assert loaded_item.meta is not None
|
|
|
|
loaded_dict = loaded_item.meta.get_custom_part()[target_name]
|
|
loaded_value = CustomCoordinates.model_validate(loaded_dict)
|
|
|
|
# ensure the value is the same
|
|
assert loaded_value == value
|
|
|
|
|
|
def test_metadata_relaxed_migration() -> None:
|
|
src = Path("tests/data/doc/dummy_doc_with_meta_2.yaml")
|
|
with pytest.warns(UserWarning, match=r"outside page bounds.*clamping to"):
|
|
doc = DoclingDocument.load_from_yaml(filename=src)
|
|
|
|
first_pic = doc.pictures[0]
|
|
assert first_pic.meta
|
|
assert first_pic.meta.classification
|
|
assert first_pic.meta.classification.predictions
|
|
# check migration was skipped since respetive meta already present:
|
|
assert first_pic.meta.classification.predictions[0].confidence == 0.42
|
|
|
|
|
|
def test_namespace_absence_raises(dummy_doc_with_meta: DoclingDocument):
|
|
example_item = RefItem(cref="#/texts/2").resolve(doc=dummy_doc_with_meta)
|
|
|
|
with pytest.raises(ValueError):
|
|
example_item.meta.my_corp_programmaticaly_added_field = True
|
|
|
|
|
|
def test_ser_deser(doc_with_group_with_metadata: DoclingDocument):
|
|
doc = doc_with_group_with_metadata
|
|
|
|
# test dumping to and loading from YAML
|
|
exp_file = Path("tests/data/doc/group_with_metadata.yaml")
|
|
if GEN_TEST_DATA:
|
|
doc.save_as_yaml(filename=exp_file)
|
|
else:
|
|
expected = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
assert doc == expected
|
|
|
|
|
|
def test_md_ser_default(doc_with_group_with_metadata: DoclingDocument):
|
|
# test exporting to Markdown
|
|
doc = doc_with_group_with_metadata
|
|
params = MarkdownParams()
|
|
ser = MarkdownDocSerializer(doc=doc, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_default.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_marked(doc_with_group_with_metadata: DoclingDocument):
|
|
# test exporting to Markdown
|
|
doc = doc_with_group_with_metadata
|
|
params = MarkdownParams(
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_marked.md")
|
|
if GEN_TEST_DATA:
|
|
with open(exp_file, "w", encoding="utf-8") as f:
|
|
f.write(actual)
|
|
else:
|
|
with open(exp_file, encoding="utf-8") as f:
|
|
expected = f.read()
|
|
assert actual == expected
|
|
|
|
|
|
def test_md_ser_allowed_meta_names(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
allowed_meta_names={
|
|
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
|
|
},
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_allowed_meta_names.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_blocked_meta_names(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
blocked_meta_names={
|
|
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
|
|
MetaFieldName.SUMMARY.value,
|
|
},
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_blocked_meta_names.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_without_non_meta(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
include_non_meta=False,
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_without_non_meta.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_include_picture_classification_meta_path():
|
|
doc = DoclingDocument(name="test")
|
|
pic = doc.add_picture()
|
|
pic.meta = PictureMeta(
|
|
classification=PictureClassificationMetaField(
|
|
predictions=[PictureClassificationPrediction(class_name="chart", confidence=0.9)]
|
|
)
|
|
)
|
|
|
|
default_text = doc.export_to_markdown()
|
|
assert "Chart" in default_text
|
|
|
|
suppressed_text = doc.export_to_markdown(include_picture_classification=False)
|
|
assert "Chart" not in suppressed_text
|
|
assert "<!-- image -->" in suppressed_text
|
|
|
|
|
|
def test_md_ser_include_picture_classification_legacy_path():
|
|
doc = DoclingDocument(name="test")
|
|
pic = doc.add_picture()
|
|
with pytest.warns(DeprecationWarning, match=r"Field `annotations` is deprecated"):
|
|
pic.annotations.append(
|
|
PictureClassificationData(
|
|
provenance="test",
|
|
predicted_classes=[PictureClassificationClass(class_name="chart", confidence=0.9)],
|
|
)
|
|
)
|
|
|
|
with pytest.warns(DeprecationWarning):
|
|
default_text = doc.export_to_markdown()
|
|
assert "chart" in default_text
|
|
|
|
with pytest.warns(DeprecationWarning):
|
|
suppressed_text = doc.export_to_markdown(include_picture_classification=False)
|
|
assert "chart" not in suppressed_text
|
|
assert "<!-- image -->" in suppressed_text
|
|
|
|
|
|
def test_ser_custom_meta_serializer(doc_with_group_with_metadata: DoclingDocument):
|
|
class SummaryMarkdownMetaSerializer(MarkdownMetaSerializer):
|
|
@override
|
|
def serialize(
|
|
self,
|
|
*,
|
|
item: NodeItem,
|
|
doc: DoclingDocument,
|
|
level: int | None = None,
|
|
**kwargs: Any,
|
|
) -> SerializationResult:
|
|
"""Serialize the item's meta."""
|
|
params = MarkdownParams(**kwargs)
|
|
return create_ser_result(
|
|
text="\n\n".join(
|
|
[
|
|
f"{' ' * (level or 0)}[{item.self_ref}] [{item.__class__.__name__}:{item.label.value}] {tmp}" # type:ignore[attr-defined]
|
|
for key in (list(item.meta.__class__.model_fields) + list(item.meta.get_custom_part()))
|
|
if (tmp := self._serialize_meta_field(item.meta, key, params.mark_meta))
|
|
]
|
|
if item.meta
|
|
else []
|
|
),
|
|
span_source=item if isinstance(item, DocItem) else [],
|
|
)
|
|
|
|
def _serialize_meta_field(self, meta: BaseMeta, name: str, mark_meta: bool, **kwargs: Any) -> str | None:
|
|
if (field_val := getattr(meta, name)) is not None and isinstance(field_val, SummaryMetaField):
|
|
txt = field_val.text
|
|
return f"[{self._humanize_text(name, title=True)}] {txt}" if mark_meta else txt
|
|
else:
|
|
return None
|
|
|
|
# test exporting to Markdown
|
|
params = MarkdownParams(
|
|
include_non_meta=False,
|
|
)
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc_with_group_with_metadata, params=params, meta_serializer=SummaryMarkdownMetaSerializer()
|
|
)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("tests/data/doc/group_with_metadata_summaries.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_document_level_metadata(dummy_doc_with_meta: DoclingDocument) -> None:
|
|
"""Test that document-level metadata can be loaded and accessed through 'body' field."""
|
|
# Verify document-level metadata exists
|
|
assert dummy_doc_with_meta.body.meta is not None
|
|
assert dummy_doc_with_meta.body.meta.summary is not None
|
|
assert (
|
|
dummy_doc_with_meta.body.meta.summary.text == "This is a document-level summary describing the entire document."
|
|
)
|
|
assert dummy_doc_with_meta.body.meta.summary.confidence == 0.98
|
|
|
|
# Verify custom metadata fields at document level
|
|
custom_part = dummy_doc_with_meta.body.meta.get_custom_part()
|
|
assert custom_part["my_corp__doc_category"] == "technical_report"
|
|
assert custom_part["my_corp__doc_version"] == "1.0"
|
|
|
|
# Verify that item-level metadata still works alongside document-level metadata
|
|
first_text = dummy_doc_with_meta.texts[1] # The title item
|
|
assert first_text.meta is not None
|
|
assert first_text.meta.summary is not None
|
|
assert first_text.meta.summary.text == "This is a title."
|
|
|
|
|
|
def test_semantic_base_meta_fields_roundtrip_and_html_rendering() -> None:
|
|
doc = DoclingDocument(name="semantic-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta(
|
|
summary=SummaryMetaField(text="A short company/location statement."),
|
|
language=LanguageMetaField(code=HumanLanguageLabel.EN),
|
|
entities=EntitiesMetaField(
|
|
mentions=[
|
|
EntityMention(text="IBM", label="ORG", charspan=(0, 3)),
|
|
EntityMention(text="Zurich", label="LOC", charspan=(16, 22)),
|
|
]
|
|
),
|
|
keywords=KeywordsMetaField(values=["ibm", "zurich", "company"]),
|
|
topics=TopicsMetaField(values=["business", "geography"]),
|
|
)
|
|
|
|
roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json"))
|
|
meta = roundtrip.texts[0].meta
|
|
assert meta is not None
|
|
assert meta.language is not None
|
|
assert meta.language.code == HumanLanguageLabel.EN
|
|
assert meta.entities is not None
|
|
assert [mention.text for mention in meta.entities.mentions] == ["IBM", "Zurich"]
|
|
assert meta.keywords is not None and meta.keywords.values == ["ibm", "zurich", "company"]
|
|
assert meta.topics is not None and meta.topics.values == ["business", "geography"]
|
|
assert meta.has_content()
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert 'data-meta-name="language"' in html
|
|
assert 'data-meta-name="entities"' in html
|
|
assert 'data-meta-name="keywords"' in html
|
|
assert 'data-meta-name="topics"' in html
|
|
assert ">en<" in html
|
|
assert "IBM (ORG, [0,3]), Zurich (LOC, [16,22])" in html
|
|
assert "ibm, zurich, company" in html
|
|
assert ">business, geography<" in html
|
|
|
|
# duplicate values are removed without rejection
|
|
assert KeywordsMetaField(values=["ai", "ml", "ai"]).values == ["ai", "ml"]
|
|
assert TopicsMetaField(values=["nlp", "nlp"]).values == ["nlp"]
|
|
|
|
|
|
def test_html_escapes_entity_text() -> None:
|
|
doc = DoclingDocument(name="escaped-entity-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="A<B & C> appears here.")
|
|
item.meta = BaseMeta(
|
|
entities=EntitiesMetaField(
|
|
mentions=[
|
|
EntityMention(text="A<B & C>", label="TAG", charspan=(0, 7)),
|
|
]
|
|
),
|
|
)
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert "A<B & C> (TAG, [0,7])" in html
|
|
|
|
|
|
def test_html_skips_empty_base_meta() -> None:
|
|
doc = DoclingDocument(name="empty-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta()
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert '<details class="docling-meta">' not in html
|
|
assert "data-meta-entities" not in html
|
|
|
|
|
|
def test_html_escapes_keywords() -> None:
|
|
doc = DoclingDocument(name="kw-escape")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="x")
|
|
item.meta = BaseMeta(keywords=KeywordsMetaField(values=["A<B & C>"]))
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert "A<B & C>" in html
|
|
|
|
|
|
def test_md_marked_renders_keywords_and_topics() -> None:
|
|
doc = DoclingDocument(name="kw-md")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta(
|
|
keywords=KeywordsMetaField(values=["ibm", "zurich"]),
|
|
topics=TopicsMetaField(values=["business"]),
|
|
)
|
|
md = MarkdownDocSerializer(doc=doc, params=MarkdownParams(mark_meta=True)).serialize().text
|
|
assert "[Keywords] ibm, zurich" in md
|
|
assert "[Topics] business" in md
|
|
|
|
|
|
def test_keywords_topics_required_values() -> None:
|
|
with pytest.raises(ValidationError, match="at least 1 item"):
|
|
KeywordsMetaField(values=[])
|
|
with pytest.raises(ValidationError, match="at least 1 item"):
|
|
TopicsMetaField(values=[])
|
|
with pytest.raises(ValidationError, match="list of strings"):
|
|
TopicsMetaField(values=34)
|
|
with pytest.raises(ValidationError, match="valid string"):
|
|
TopicsMetaField(values=[34])
|