Files
docling-core/test/test_code_chunker.py
Peter W. J. Staar 8951e6f202 chore: removing all legacy / deepsearch code (#644)
* What was removed

  The legacy_doc document model and the rec (Record) types, plus all code/resources/tests that existed only to serve them.

  Deleted packages/modules
  - docling_core/types/legacy_doc/ (whole folder)
  - docling_core/types/rec/ (whole folder)
  - docling_core/utils/legacy.py (legacy conversion functions)
  - docling_core/utils/validate.py (RAW/ANN/OCR CLI)

  Edited production code
  - types/__init__.py — dropped the Record export
  - utils/validators.py — removed validate_schema/validate_raw_schema/validate_ann_schema/validate_ocr_schema and their now-unused imports; kept
  validate_unique_list/validate_datetime
  - utils/generate_docs.py — dropped Record from MODELS

  Removed resources/artifacts
  - resources/schemas/doc/ (legacy RAW/ANN/OCR/DOC), resources/schemas/generated/ccs_document_schema.json, docs/Record.json

  Tests — deleted the 5 legacy/rec-only suites + 2 stale gold files; trimmed test_collection.py, test_base.py, test_doc_base.py to keep their non-legacy parts;
  replaced the _create_hash import in test_code_chunker.py with a local helper. Removed test/data/legacy_doc/ and test/data/rec/.

  Verification

  - No dangling code references remain (only string literals and the vendored _reference/chunker_repo scratch data, intentionally untouched).
  - from docling_core.types import DoclingDocument, Generic works; Record now correctly raises ImportError.
  - Full project suite: 496 passed.
  - generate_docs.py now emits only DoclingDocument.json and Generic.json.

  The full plan is saved at _plans/remove-legacy-doc-and-rec.md.

  Two notes worth flagging:
  - Breaking public API: docling_core.types.Record and the docling_core.utils.legacy conversion functions were public — downstream docling uses
  legacy_to_docling_document/docling_document_to_legacy. The branch name (dev/remove-legacy-document) suggests this is intended, but it'll need a coordinated
  bump downstream.
  - I left _reference/ and _plans/ alone (untracked scratch dirs). Nothing was committed — all changes are staged/working-tree only, per your edit-by-edit
  approval flow.

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* updated the checks.yaml

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* removed docling_core/types/gen/, docling_core/types/nlp, docling_core/types/legacy_doc, docling_core/types/rec

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* removed the minimal_document_schema_flat.json

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* removed docling_core/resources

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* removed the files and updated the imports according to Cesar's suggestion

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

* fixed the ruff linter

Signed-off-by: Peter Staar <taa@zurich.ibm.com>

---------

Signed-off-by: Peter Staar <taa@zurich.ibm.com>
2026-06-18 17:00:52 +02:00

174 lines
5.7 KiB
Python

import glob
import hashlib
import json
import os
import pathlib
import sys
from typing import Optional
import git
import pytest
from docling_core.transforms.chunker.code_chunking._utils import _get_file_extensions
from docling_core.transforms.chunker.code_chunking.code_chunk import CodeChunk
from docling_core.transforms.chunker.code_chunking.standard_code_chunking_strategy import (
StandardCodeChunkingStrategy,
)
from docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker
from docling_core.types.doc import DoclingDocument, DocumentOrigin
from docling_core.types.doc.labels import CodeLanguageLabel, DocItemLabel
from .test_data_gen_flag import GEN_TEST_DATA
def _create_hash(string: str) -> str:
"""Return a SHA-256 hex digest of the given string."""
hasher = hashlib.sha256(usedforsecurity=False)
hasher.update(string.encode("utf-8"))
return hasher.hexdigest()
def get_latest_commit_id(file_dir: str) -> str:
"""Returns the latest commit ID in the given Git repository directory."""
try:
repo = git.Repo(file_dir, search_parent_directories=True)
return repo.head.commit.hexsha
except Exception:
return ""
def create_documents_from_repository(
file_dir: str,
repo_url: str,
language: CodeLanguageLabel,
commit_id: Optional[str] = None,
) -> list[DoclingDocument]:
"""Build DoclingDocument objects from a local checkout, one per code file."""
documents: list[DoclingDocument] = []
if commit_id is None:
commit_id = get_latest_commit_id(file_dir)
all_extensions = set()
for lang in [
CodeLanguageLabel.PYTHON,
CodeLanguageLabel.TYPESCRIPT,
CodeLanguageLabel.JAVASCRIPT,
CodeLanguageLabel.C,
CodeLanguageLabel.JAVA,
]:
all_extensions.update(_get_file_extensions(lang))
all_files = []
for extension in all_extensions:
all_files.extend(sorted(glob.glob(f"{file_dir}/**/*{extension}", recursive=True)))
all_files = sorted(set(all_files))
for file_path in all_files:
with open(file_path, encoding="utf-8") as f:
file_content = f.read()
file_relative = os.path.relpath(file_path, start=file_dir).replace("\\", "/")
origin = DocumentOrigin(
filename=file_relative,
uri=(f"{repo_url}/blob/{commit_id}/{file_relative}" if commit_id else f"{repo_url}/{file_relative}"),
mimetype="text/plain",
binary_hash=_create_hash(file_content),
)
doc = DoclingDocument(name=file_relative, origin=origin)
doc.add_code(text=file_content, code_language=language)
documents.append(doc)
return documents
HERE = pathlib.Path(__file__).parent
DATA = HERE / "data" / "chunker_repo"
DATA.mkdir(parents=True, exist_ok=True)
REPO_SPECS = [
(
"Java",
"/test/data/chunker_repo/repos/acmeair",
"https://github.com/acmeair/acmeair",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"TypeScript",
"/test/data/chunker_repo/repos/outline",
"https://github.com/outline/outline",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"JavaScript",
"/test/data/chunker_repo/repos/jquery",
"https://github.com/jquery/jquery",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"Python",
"/test/data/chunker_repo/repos/docling",
"https://github.com/docling-project/docling",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"C",
"/test/data/chunker_repo/repos/json-c",
"https://github.com/json-c/json-c",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
]
def _dump_or_assert(act_data: dict, out_path: pathlib.Path):
out_path.parent.mkdir(parents=True, exist_ok=True)
if GEN_TEST_DATA:
with out_path.open("w", encoding="utf-8") as f:
json.dump(act_data, fp=f, indent=4)
f.write("\n")
else:
with out_path.open(encoding="utf-8") as f:
exp_data = json.load(fp=f)
assert exp_data == act_data
@pytest.mark.parametrize("name,local_path,repo_url,chunker_factory", REPO_SPECS)
def test_function_chunkers_repo(name, local_path, repo_url, chunker_factory):
if name == "Java" and sys.version_info < (3, 10):
pytest.skip("Skipping Java tests on python < 3.10.")
local_path_full = os.getcwd() + local_path
if not os.path.isdir(local_path_full):
pytest.skip(f"Missing repo at {local_path_full}; skipping {name} test.")
docs = create_documents_from_repository(
local_path_full,
repo_url,
language=CodeLanguageLabel(name),
commit_id="abc123def456",
)
docs = [doc for doc in docs if any(text.label == DocItemLabel.CODE and text.text for text in doc.texts)]
if not docs:
pytest.skip(f"No documents found in {local_path_full} for {name}.")
sample = docs[:3]
chunker = chunker_factory()
all_chunks = []
for doc in sample:
chunks_iter = chunker.chunk(dl_doc=doc)
chs = list(chunks_iter)
chunks = [CodeChunk.model_validate(n) for n in chs]
all_chunks.extend(chunks)
assert chunks, f"Expected chunks for {doc.name}"
for c in chunks:
assert c.text and isinstance(c.text, str)
act_data = {"root": [c.export_json_dict() for c in all_chunks]}
out_path = DATA / name / "repo_out_chunks.json"
_dump_or_assert(act_data, out_path)