mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* chore: update pre-commit hooks and development dependencies Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * docs: update CONTRIBUTING after pre-commit hook changes Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
382 lines
15 KiB
Python
382 lines
15 KiB
Python
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import BaseModel, ValidationError
|
|
from typing_extensions import override
|
|
|
|
from docling_core.transforms.serializer.base import SerializationResult
|
|
from docling_core.transforms.serializer.common import create_ser_result
|
|
from docling_core.transforms.serializer.html import HTMLDocSerializer, HTMLParams
|
|
from docling_core.transforms.serializer.markdown import (
|
|
MarkdownDocSerializer,
|
|
MarkdownMetaSerializer,
|
|
MarkdownParams,
|
|
)
|
|
from docling_core.types.doc import (
|
|
BaseMeta,
|
|
DocItem,
|
|
DocItemLabel,
|
|
DoclingDocument,
|
|
EntitiesMetaField,
|
|
EntityMention,
|
|
GroupLabel,
|
|
HumanLanguageLabel,
|
|
KeywordsMetaField,
|
|
LanguageMetaField,
|
|
MetaFieldName,
|
|
MetaUtils,
|
|
NodeItem,
|
|
RefItem,
|
|
SummaryMetaField,
|
|
TopicsMetaField,
|
|
)
|
|
|
|
from .test_data_gen_flag import GEN_TEST_DATA
|
|
from .test_utils import assert_or_generate_ground_truth
|
|
|
|
|
|
class CustomCoordinates(BaseModel):
|
|
longitude: float
|
|
latitude: float
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def dummy_doc_with_meta() -> DoclingDocument:
|
|
"""Fixture that loads dummy_doc_with_meta.yaml once per module."""
|
|
src = Path("test/data/doc/dummy_doc_with_meta.yaml")
|
|
return DoclingDocument.load_from_yaml(filename=src)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def doc_with_group_with_metadata() -> DoclingDocument:
|
|
"""Fixture that creates a document with groups and metadata once per module."""
|
|
doc = DoclingDocument(name="")
|
|
doc.body.meta = BaseMeta(summary=SummaryMetaField(text="This document talks about various topics."))
|
|
grp1 = doc.add_group(name="1", label=GroupLabel.CHAPTER)
|
|
grp1.meta = BaseMeta(summary=SummaryMetaField(text="This chapter discusses foo and bar."))
|
|
doc.add_text(text="This is some introductory text.", label=DocItemLabel.TEXT, parent=grp1)
|
|
|
|
grp1a = doc.add_group(parent=grp1, name="1a", label=GroupLabel.SECTION)
|
|
grp1a.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about foo."))
|
|
grp1a.meta.set_custom_field(namespace="my_corp", name="test_1", value="custom field value 1")
|
|
txt1 = doc.add_text(text="Regarding foo...", label=DocItemLabel.TEXT, parent=grp1a)
|
|
txt1.meta = BaseMeta(summary=SummaryMetaField(text="This paragraph provides more details about foo."))
|
|
lst1a = doc.add_list_group(parent=grp1a)
|
|
lst1a.meta = BaseMeta(summary=SummaryMetaField(text="Here some foo specifics are listed."))
|
|
doc.add_list_item(text="lorem", parent=lst1a, enumerated=True)
|
|
doc.add_list_item(text="ipsum", parent=lst1a, enumerated=True)
|
|
|
|
grp1b = doc.add_group(parent=grp1, name="1b", label=GroupLabel.SECTION)
|
|
grp1b.meta = BaseMeta(summary=SummaryMetaField(text="This section talks about bar."))
|
|
grp1b.meta.set_custom_field(namespace="my_corp", name="test_2", value="custom field value 2")
|
|
doc.add_text(text="Regarding bar...", label=DocItemLabel.TEXT, parent=grp1b)
|
|
|
|
return doc
|
|
|
|
|
|
def test_metadata_usage(dummy_doc_with_meta: DoclingDocument) -> None:
|
|
doc = dummy_doc_with_meta.model_copy(deep=True)
|
|
|
|
first_pic = doc.pictures[0]
|
|
assert first_pic.meta
|
|
assert first_pic.meta.classification
|
|
assert first_pic.meta.classification.predictions
|
|
assert first_pic.meta.classification.predictions[0].confidence == 0.78
|
|
|
|
example_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=doc)
|
|
assert example_item.meta is not None
|
|
|
|
# add a custom metadata object to the item
|
|
value = CustomCoordinates(longitude=47.3769, latitude=8.5417)
|
|
target_name = example_item.meta.set_custom_field(namespace="my_corp", name="coords", value=value)
|
|
assert target_name == "my_corp__coords"
|
|
|
|
# save the document
|
|
exp_file = Path("test/data/doc/dummy_doc_with_meta_modified.yaml")
|
|
if GEN_TEST_DATA:
|
|
doc.save_as_yaml(filename=exp_file)
|
|
else:
|
|
expected = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
assert doc.model_dump(mode="json") == expected.model_dump(mode="json")
|
|
|
|
# load back the document and read the custom metadata object
|
|
loaded_doc = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
loaded_item: NodeItem = RefItem(cref="#/texts/2").resolve(doc=loaded_doc)
|
|
assert loaded_item.meta is not None
|
|
|
|
loaded_dict = loaded_item.meta.get_custom_part()[target_name]
|
|
loaded_value = CustomCoordinates.model_validate(loaded_dict)
|
|
|
|
# ensure the value is the same
|
|
assert loaded_value == value
|
|
|
|
|
|
def test_metadata_relaxed_migration() -> None:
|
|
src = Path("test/data/doc/dummy_doc_with_meta_2.yaml")
|
|
doc = DoclingDocument.load_from_yaml(filename=src)
|
|
|
|
first_pic = doc.pictures[0]
|
|
assert first_pic.meta
|
|
assert first_pic.meta.classification
|
|
assert first_pic.meta.classification.predictions
|
|
# check migration was skipped since respetive meta already present:
|
|
assert first_pic.meta.classification.predictions[0].confidence == 0.42
|
|
|
|
|
|
def test_namespace_absence_raises(dummy_doc_with_meta: DoclingDocument):
|
|
example_item = RefItem(cref="#/texts/2").resolve(doc=dummy_doc_with_meta)
|
|
|
|
with pytest.raises(ValueError):
|
|
example_item.meta.my_corp_programmaticaly_added_field = True
|
|
|
|
|
|
def test_ser_deser(doc_with_group_with_metadata: DoclingDocument):
|
|
doc = doc_with_group_with_metadata
|
|
|
|
# test dumping to and loading from YAML
|
|
exp_file = Path("test/data/doc/group_with_metadata.yaml")
|
|
if GEN_TEST_DATA:
|
|
doc.save_as_yaml(filename=exp_file)
|
|
else:
|
|
expected = DoclingDocument.load_from_yaml(filename=exp_file)
|
|
assert doc == expected
|
|
|
|
|
|
def test_md_ser_default(doc_with_group_with_metadata: DoclingDocument):
|
|
# test exporting to Markdown
|
|
doc = doc_with_group_with_metadata
|
|
params = MarkdownParams()
|
|
ser = MarkdownDocSerializer(doc=doc, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_default.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_marked(doc_with_group_with_metadata: DoclingDocument):
|
|
# test exporting to Markdown
|
|
doc = doc_with_group_with_metadata
|
|
params = MarkdownParams(
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_marked.md")
|
|
if GEN_TEST_DATA:
|
|
with open(exp_file, "w", encoding="utf-8") as f:
|
|
f.write(actual)
|
|
else:
|
|
with open(exp_file, encoding="utf-8") as f:
|
|
expected = f.read()
|
|
assert actual == expected
|
|
|
|
|
|
def test_md_ser_allowed_meta_names(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
allowed_meta_names={
|
|
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
|
|
},
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_allowed_meta_names.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_blocked_meta_names(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
blocked_meta_names={
|
|
MetaUtils.create_meta_field_name(namespace="my_corp", name="test_1"),
|
|
MetaFieldName.SUMMARY.value,
|
|
},
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_blocked_meta_names.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_md_ser_without_non_meta(doc_with_group_with_metadata: DoclingDocument):
|
|
params = MarkdownParams(
|
|
include_non_meta=False,
|
|
mark_meta=True,
|
|
)
|
|
ser = MarkdownDocSerializer(doc=doc_with_group_with_metadata, params=params)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_without_non_meta.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_ser_custom_meta_serializer(doc_with_group_with_metadata: DoclingDocument):
|
|
class SummaryMarkdownMetaSerializer(MarkdownMetaSerializer):
|
|
@override
|
|
def serialize(
|
|
self,
|
|
*,
|
|
item: NodeItem,
|
|
doc: DoclingDocument,
|
|
level: int | None = None,
|
|
**kwargs: Any,
|
|
) -> SerializationResult:
|
|
"""Serialize the item's meta."""
|
|
params = MarkdownParams(**kwargs)
|
|
return create_ser_result(
|
|
text="\n\n".join(
|
|
[
|
|
f"{' ' * (level or 0)}[{item.self_ref}] [{item.__class__.__name__}:{item.label.value}] {tmp}" # type:ignore[attr-defined]
|
|
for key in (list(item.meta.__class__.model_fields) + list(item.meta.get_custom_part()))
|
|
if (tmp := self._serialize_meta_field(item.meta, key, params.mark_meta))
|
|
]
|
|
if item.meta
|
|
else []
|
|
),
|
|
span_source=item if isinstance(item, DocItem) else [],
|
|
)
|
|
|
|
def _serialize_meta_field(self, meta: BaseMeta, name: str, mark_meta: bool) -> str | None:
|
|
if (field_val := getattr(meta, name)) is not None and isinstance(field_val, SummaryMetaField):
|
|
txt = field_val.text
|
|
return f"[{self._humanize_text(name, title=True)}] {txt}" if mark_meta else txt
|
|
else:
|
|
return None
|
|
|
|
# test exporting to Markdown
|
|
params = MarkdownParams(
|
|
include_non_meta=False,
|
|
)
|
|
ser = MarkdownDocSerializer(
|
|
doc=doc_with_group_with_metadata, params=params, meta_serializer=SummaryMarkdownMetaSerializer()
|
|
)
|
|
ser_res = ser.serialize()
|
|
actual = ser_res.text
|
|
exp_file = Path("test/data/doc/group_with_metadata_summaries.md")
|
|
assert_or_generate_ground_truth(actual, exp_file)
|
|
|
|
|
|
def test_document_level_metadata(dummy_doc_with_meta: DoclingDocument) -> None:
|
|
"""Test that document-level metadata can be loaded and accessed through 'body' field."""
|
|
# Verify document-level metadata exists
|
|
assert dummy_doc_with_meta.body.meta is not None
|
|
assert dummy_doc_with_meta.body.meta.summary is not None
|
|
assert (
|
|
dummy_doc_with_meta.body.meta.summary.text == "This is a document-level summary describing the entire document."
|
|
)
|
|
assert dummy_doc_with_meta.body.meta.summary.confidence == 0.98
|
|
|
|
# Verify custom metadata fields at document level
|
|
custom_part = dummy_doc_with_meta.body.meta.get_custom_part()
|
|
assert custom_part["my_corp__doc_category"] == "technical_report"
|
|
assert custom_part["my_corp__doc_version"] == "1.0"
|
|
|
|
# Verify that item-level metadata still works alongside document-level metadata
|
|
first_text = dummy_doc_with_meta.texts[1] # The title item
|
|
assert first_text.meta is not None
|
|
assert first_text.meta.summary is not None
|
|
assert first_text.meta.summary.text == "This is a title."
|
|
|
|
|
|
def test_semantic_base_meta_fields_roundtrip_and_html_rendering() -> None:
|
|
doc = DoclingDocument(name="semantic-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta(
|
|
summary=SummaryMetaField(text="A short company/location statement."),
|
|
language=LanguageMetaField(code=HumanLanguageLabel.EN),
|
|
entities=EntitiesMetaField(
|
|
mentions=[
|
|
EntityMention(text="IBM", label="ORG", charspan=(0, 3)),
|
|
EntityMention(text="Zurich", label="LOC", charspan=(16, 22)),
|
|
]
|
|
),
|
|
keywords=KeywordsMetaField(values=["ibm", "zurich", "company"]),
|
|
topics=TopicsMetaField(values=["business", "geography"]),
|
|
)
|
|
|
|
roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json"))
|
|
meta = roundtrip.texts[0].meta
|
|
assert meta is not None
|
|
assert meta.language is not None
|
|
assert meta.language.code == HumanLanguageLabel.EN
|
|
assert meta.entities is not None
|
|
assert [mention.text for mention in meta.entities.mentions] == ["IBM", "Zurich"]
|
|
assert meta.keywords is not None and meta.keywords.values == ["ibm", "zurich", "company"]
|
|
assert meta.topics is not None and meta.topics.values == ["business", "geography"]
|
|
assert meta.has_content()
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert 'data-meta-name="language"' in html
|
|
assert 'data-meta-name="entities"' in html
|
|
assert 'data-meta-name="keywords"' in html
|
|
assert 'data-meta-name="topics"' in html
|
|
assert ">en<" in html
|
|
assert "IBM (ORG, [0,3]), Zurich (LOC, [16,22])" in html
|
|
assert "ibm, zurich, company" in html
|
|
assert ">business, geography<" in html
|
|
|
|
# duplicate values are removed without rejection
|
|
assert KeywordsMetaField(values=["ai", "ml", "ai"]).values == ["ai", "ml"]
|
|
assert TopicsMetaField(values=["nlp", "nlp"]).values == ["nlp"]
|
|
|
|
|
|
def test_html_escapes_entity_text() -> None:
|
|
doc = DoclingDocument(name="escaped-entity-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="A<B & C> appears here.")
|
|
item.meta = BaseMeta(
|
|
entities=EntitiesMetaField(
|
|
mentions=[
|
|
EntityMention(text="A<B & C>", label="TAG", charspan=(0, 7)),
|
|
]
|
|
),
|
|
)
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert "A<B & C> (TAG, [0,7])" in html
|
|
|
|
|
|
def test_html_skips_empty_base_meta() -> None:
|
|
doc = DoclingDocument(name="empty-meta")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta()
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert '<details class="docling-meta">' not in html
|
|
assert "data-meta-entities" not in html
|
|
|
|
|
|
def test_html_escapes_keywords() -> None:
|
|
doc = DoclingDocument(name="kw-escape")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="x")
|
|
item.meta = BaseMeta(keywords=KeywordsMetaField(values=["A<B & C>"]))
|
|
|
|
html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text
|
|
assert "A<B & C>" in html
|
|
|
|
|
|
def test_md_marked_renders_keywords_and_topics() -> None:
|
|
doc = DoclingDocument(name="kw-md")
|
|
item = doc.add_text(label=DocItemLabel.TEXT, text="IBM is based in Zurich.")
|
|
item.meta = BaseMeta(
|
|
keywords=KeywordsMetaField(values=["ibm", "zurich"]),
|
|
topics=TopicsMetaField(values=["business"]),
|
|
)
|
|
md = MarkdownDocSerializer(doc=doc, params=MarkdownParams(mark_meta=True)).serialize().text
|
|
assert "[Keywords] ibm, zurich" in md
|
|
assert "[Topics] business" in md
|
|
|
|
|
|
def test_keywords_topics_required_values() -> None:
|
|
with pytest.raises(ValidationError, match="at least 1 item"):
|
|
KeywordsMetaField(values=[])
|
|
with pytest.raises(ValidationError, match="at least 1 item"):
|
|
TopicsMetaField(values=[])
|
|
with pytest.raises(ValidationError, match="list of strings"):
|
|
TopicsMetaField(values=34)
|
|
with pytest.raises(ValidationError, match="valid string"):
|
|
TopicsMetaField(values=[34])
|