perf: contextualize function() skip excluded fields (#581)

* contextualize function() skip excluded fields

This change proposes that we use model_dump to skip/avoid serializing
fields that are in excluded_embed. This should reduce unnecesary
computation and is also more precisely what I believe we are intending
to do here in that we do not need to unnecessarily filter the results
below in the for-loop.

Signed-off-by: Jordan White <jordan.page.white@gmail.com>

* style(chunker): fix style issue C416

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(chunker): resolve excluded_embed to field names

model_dump(exclude=...) requires Python attribute names, not
aliases. excluded_embed stores alias keys, which happen to
match today but would silently fail to exclude fields in
subclasses where they differ, leaking excluded fields into the
embedding context.

Adds BaseMeta._excluded_embed_field_names() to resolve each
entry through the alias-to-field-name map. contextualize() now
calls this helper instead of using excluded_embed directly.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(chunker): fix pydantic deprecation

Deprecated in pydantic v2.11, removed in v3: model_fields can only be accessed
from class object, not the instance.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* test(chunker): add test to verify contextualize() excludes the right fields

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* chore: update uv.lock dependencies

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

---------

Signed-off-by: Jordan White <jordan.page.white@gmail.com>
Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
Co-authored-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
This commit is contained in:
Jordan WhiteandCesar Berrospi Ramis authored and GitHub committed 2026-08-19 10:00:09 +02:00
1 parent f756342105
commit ec0069c6ec
3 files changed
+996 -734

No files matched your search

+25 -6
View File
@@ -20,6 +20,21 @@ class BaseMeta(BaseModel):
excluded_embed: ClassVar[list[str]] = []
excluded_llm: ClassVar[list[str]] = []
def _excluded_embed_field_names(self) -> set[str]:
"""Return the Python attribute names for all fields listed in `excluded_embed`.
`excluded_embed` conventionally stores alias keys (the serialised field
names). `model_dump(exclude=...)` requires Python attribute names, not
aliases. This helper resolves each entry to its canonical attribute name
so that exclusion is correct even when a subclass assigns an alias that
differs from the Python attribute name.
Returns:
Python attribute names to exclude from embedding serialization.
"""
alias_to_name = {(field_info.alias or name): name for name, field_info in type(self).model_fields.items()}
return {alias_to_name.get(key, key) for key in self.excluded_embed}
def export_json_dict(self) -> dict[str, Any]:
"""Helper method for exporting non-None keys to JSON mode.
@@ -73,15 +88,19 @@ class BaseChunker(BaseModel, ABC):
Returns:
str: the serialized form of the chunk
"""
meta = chunk.meta.export_json_dict()
meta = chunk.meta.model_dump(
mode="json",
by_alias=True,
exclude_none=True,
exclude=chunk.meta._excluded_embed_field_names(),
)
items = []
for k in meta:
if k not in chunk.meta.excluded_embed:
if isinstance(meta[k], list):
items.append(self.delim.join([d if isinstance(d, str) else json.dumps(d) for d in meta[k]]))
else:
items.append(json.dumps(meta[k]))
if isinstance(meta[k], list):
items.append(self.delim.join([d if isinstance(d, str) else json.dumps(d) for d in meta[k]]))
else:
items.append(json.dumps(meta[k]))
items.append(chunk.text)
return self.delim.join(items)
+33
View File
@@ -1,6 +1,10 @@
from pathlib import Path
from typing import ClassVar
from pydantic import Field
from docling_core.transforms.chunker import HierarchicalChunker
from docling_core.transforms.chunker.base import BaseChunk, BaseChunker, BaseMeta
from docling_core.transforms.chunker.hierarchical_chunker import (
ChunkingDocSerializer,
ChunkingSerializerProvider,
@@ -293,3 +297,32 @@ def test_chunk_rich_table_custom_serializer(rich_table_doc: DoclingDocument):
act_data = dict(root=[DocChunk.model_validate(n).export_json_dict() for n in chunks])
assert_or_generate_json_ground_truth(act_data, "test/data/chunker/0c_out_chunks.json")
def test_contextualize_excludes_fields_when_alias_differs_from_attribute_name():
"""contextualize() must not leak excluded fields even when alias != attribute name.
This guards against the assumption that excluded_embed entries are Python
attribute names. model_dump(exclude=...) requires attribute names; if a
subclass uses an alias that differs, the raw alias would be silently ignored
and the field would appear in the embedding context.
"""
class AliasedMeta(BaseMeta):
visible: str = Field(default="keep me", alias="visibleAlias")
hidden: str = Field(default="drop me", alias="hiddenAlias")
excluded_embed: ClassVar[list[str]] = ["hiddenAlias"]
class AliasedChunk(BaseChunk):
meta: AliasedMeta
class _ConcreteChunker(BaseChunker):
def chunk(self, dl_doc, **kwargs):
return iter([])
chunk = AliasedChunk(text="body", meta=AliasedMeta())
result = _ConcreteChunker().contextualize(chunk)
assert "drop me" not in result
assert "keep me" in result
assert "body" in result
Generated
+938 -728
View File
File diff suppressed because it is too large. Load diff