mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
perf: contextualize function() skip excluded fields (#581)
* contextualize function() skip excluded fields This change proposes that we use model_dump to skip/avoid serializing fields that are in excluded_embed. This should reduce unnecesary computation and is also more precisely what I believe we are intending to do here in that we do not need to unnecessarily filter the results below in the for-loop. Signed-off-by: Jordan White <jordan.page.white@gmail.com> * style(chunker): fix style issue C416 Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(chunker): resolve excluded_embed to field names model_dump(exclude=...) requires Python attribute names, not aliases. excluded_embed stores alias keys, which happen to match today but would silently fail to exclude fields in subclasses where they differ, leaking excluded fields into the embedding context. Adds BaseMeta._excluded_embed_field_names() to resolve each entry through the alias-to-field-name map. contextualize() now calls this helper instead of using excluded_embed directly. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(chunker): fix pydantic deprecation Deprecated in pydantic v2.11, removed in v3: model_fields can only be accessed from class object, not the instance. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * test(chunker): add test to verify contextualize() excludes the right fields Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * chore: update uv.lock dependencies Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Jordan White <jordan.page.white@gmail.com> Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> Co-authored-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
This commit is contained in:
1 parent
f756342105
commit
ec0069c6ec
3 files changed
+996
-734
No files matched your search
@@ -20,6 +20,21 @@ class BaseMeta(BaseModel):
|
||||
excluded_embed: ClassVar[list[str]] = []
|
||||
excluded_llm: ClassVar[list[str]] = []
|
||||
|
||||
def _excluded_embed_field_names(self) -> set[str]:
|
||||
"""Return the Python attribute names for all fields listed in `excluded_embed`.
|
||||
|
||||
`excluded_embed` conventionally stores alias keys (the serialised field
|
||||
names). `model_dump(exclude=...)` requires Python attribute names, not
|
||||
aliases. This helper resolves each entry to its canonical attribute name
|
||||
so that exclusion is correct even when a subclass assigns an alias that
|
||||
differs from the Python attribute name.
|
||||
|
||||
Returns:
|
||||
Python attribute names to exclude from embedding serialization.
|
||||
"""
|
||||
alias_to_name = {(field_info.alias or name): name for name, field_info in type(self).model_fields.items()}
|
||||
return {alias_to_name.get(key, key) for key in self.excluded_embed}
|
||||
|
||||
def export_json_dict(self) -> dict[str, Any]:
|
||||
"""Helper method for exporting non-None keys to JSON mode.
|
||||
|
||||
@@ -73,15 +88,19 @@ class BaseChunker(BaseModel, ABC):
|
||||
Returns:
|
||||
str: the serialized form of the chunk
|
||||
"""
|
||||
meta = chunk.meta.export_json_dict()
|
||||
meta = chunk.meta.model_dump(
|
||||
mode="json",
|
||||
by_alias=True,
|
||||
exclude_none=True,
|
||||
exclude=chunk.meta._excluded_embed_field_names(),
|
||||
)
|
||||
|
||||
items = []
|
||||
for k in meta:
|
||||
if k not in chunk.meta.excluded_embed:
|
||||
if isinstance(meta[k], list):
|
||||
items.append(self.delim.join([d if isinstance(d, str) else json.dumps(d) for d in meta[k]]))
|
||||
else:
|
||||
items.append(json.dumps(meta[k]))
|
||||
if isinstance(meta[k], list):
|
||||
items.append(self.delim.join([d if isinstance(d, str) else json.dumps(d) for d in meta[k]]))
|
||||
else:
|
||||
items.append(json.dumps(meta[k]))
|
||||
items.append(chunk.text)
|
||||
|
||||
return self.delim.join(items)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from docling_core.transforms.chunker import HierarchicalChunker
|
||||
from docling_core.transforms.chunker.base import BaseChunk, BaseChunker, BaseMeta
|
||||
from docling_core.transforms.chunker.hierarchical_chunker import (
|
||||
ChunkingDocSerializer,
|
||||
ChunkingSerializerProvider,
|
||||
@@ -293,3 +297,32 @@ def test_chunk_rich_table_custom_serializer(rich_table_doc: DoclingDocument):
|
||||
act_data = dict(root=[DocChunk.model_validate(n).export_json_dict() for n in chunks])
|
||||
|
||||
assert_or_generate_json_ground_truth(act_data, "test/data/chunker/0c_out_chunks.json")
|
||||
|
||||
|
||||
def test_contextualize_excludes_fields_when_alias_differs_from_attribute_name():
|
||||
"""contextualize() must not leak excluded fields even when alias != attribute name.
|
||||
|
||||
This guards against the assumption that excluded_embed entries are Python
|
||||
attribute names. model_dump(exclude=...) requires attribute names; if a
|
||||
subclass uses an alias that differs, the raw alias would be silently ignored
|
||||
and the field would appear in the embedding context.
|
||||
"""
|
||||
|
||||
class AliasedMeta(BaseMeta):
|
||||
visible: str = Field(default="keep me", alias="visibleAlias")
|
||||
hidden: str = Field(default="drop me", alias="hiddenAlias")
|
||||
excluded_embed: ClassVar[list[str]] = ["hiddenAlias"]
|
||||
|
||||
class AliasedChunk(BaseChunk):
|
||||
meta: AliasedMeta
|
||||
|
||||
class _ConcreteChunker(BaseChunker):
|
||||
def chunk(self, dl_doc, **kwargs):
|
||||
return iter([])
|
||||
|
||||
chunk = AliasedChunk(text="body", meta=AliasedMeta())
|
||||
result = _ConcreteChunker().contextualize(chunk)
|
||||
|
||||
assert "drop me" not in result
|
||||
assert "keep me" in result
|
||||
assert "body" in result
|
||||
Reference in new issue
Block a user