mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* What was removed The legacy_doc document model and the rec (Record) types, plus all code/resources/tests that existed only to serve them. Deleted packages/modules - docling_core/types/legacy_doc/ (whole folder) - docling_core/types/rec/ (whole folder) - docling_core/utils/legacy.py (legacy conversion functions) - docling_core/utils/validate.py (RAW/ANN/OCR CLI) Edited production code - types/__init__.py — dropped the Record export - utils/validators.py — removed validate_schema/validate_raw_schema/validate_ann_schema/validate_ocr_schema and their now-unused imports; kept validate_unique_list/validate_datetime - utils/generate_docs.py — dropped Record from MODELS Removed resources/artifacts - resources/schemas/doc/ (legacy RAW/ANN/OCR/DOC), resources/schemas/generated/ccs_document_schema.json, docs/Record.json Tests — deleted the 5 legacy/rec-only suites + 2 stale gold files; trimmed test_collection.py, test_base.py, test_doc_base.py to keep their non-legacy parts; replaced the _create_hash import in test_code_chunker.py with a local helper. Removed test/data/legacy_doc/ and test/data/rec/. Verification - No dangling code references remain (only string literals and the vendored _reference/chunker_repo scratch data, intentionally untouched). - from docling_core.types import DoclingDocument, Generic works; Record now correctly raises ImportError. - Full project suite: 496 passed. - generate_docs.py now emits only DoclingDocument.json and Generic.json. The full plan is saved at _plans/remove-legacy-doc-and-rec.md. Two notes worth flagging: - Breaking public API: docling_core.types.Record and the docling_core.utils.legacy conversion functions were public — downstream docling uses legacy_to_docling_document/docling_document_to_legacy. The branch name (dev/remove-legacy-document) suggests this is intended, but it'll need a coordinated bump downstream. - I left _reference/ and _plans/ alone (untracked scratch dirs). Nothing was committed — all changes are staged/working-tree only, per your edit-by-edit approval flow. Signed-off-by: Peter Staar <taa@zurich.ibm.com> * updated the checks.yaml Signed-off-by: Peter Staar <taa@zurich.ibm.com> * removed docling_core/types/gen/, docling_core/types/nlp, docling_core/types/legacy_doc, docling_core/types/rec Signed-off-by: Peter Staar <taa@zurich.ibm.com> * removed the minimal_document_schema_flat.json Signed-off-by: Peter Staar <taa@zurich.ibm.com> * removed docling_core/resources Signed-off-by: Peter Staar <taa@zurich.ibm.com> * removed the files and updated the imports according to Cesar's suggestion Signed-off-by: Peter Staar <taa@zurich.ibm.com> * fixed the ruff linter Signed-off-by: Peter Staar <taa@zurich.ibm.com> --------- Signed-off-by: Peter Staar <taa@zurich.ibm.com>
174 lines
5.7 KiB
Python
174 lines
5.7 KiB
Python
import glob
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
from typing import Optional
|
|
|
|
import git
|
|
import pytest
|
|
|
|
from docling_core.transforms.chunker.code_chunking._utils import _get_file_extensions
|
|
from docling_core.transforms.chunker.code_chunking.code_chunk import CodeChunk
|
|
from docling_core.transforms.chunker.code_chunking.standard_code_chunking_strategy import (
|
|
StandardCodeChunkingStrategy,
|
|
)
|
|
from docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker
|
|
from docling_core.types.doc import DoclingDocument, DocumentOrigin
|
|
from docling_core.types.doc.labels import CodeLanguageLabel, DocItemLabel
|
|
|
|
from .test_data_gen_flag import GEN_TEST_DATA
|
|
|
|
|
|
def _create_hash(string: str) -> str:
|
|
"""Return a SHA-256 hex digest of the given string."""
|
|
hasher = hashlib.sha256(usedforsecurity=False)
|
|
hasher.update(string.encode("utf-8"))
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def get_latest_commit_id(file_dir: str) -> str:
|
|
"""Returns the latest commit ID in the given Git repository directory."""
|
|
try:
|
|
repo = git.Repo(file_dir, search_parent_directories=True)
|
|
return repo.head.commit.hexsha
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def create_documents_from_repository(
|
|
file_dir: str,
|
|
repo_url: str,
|
|
language: CodeLanguageLabel,
|
|
commit_id: Optional[str] = None,
|
|
) -> list[DoclingDocument]:
|
|
"""Build DoclingDocument objects from a local checkout, one per code file."""
|
|
documents: list[DoclingDocument] = []
|
|
if commit_id is None:
|
|
commit_id = get_latest_commit_id(file_dir)
|
|
|
|
all_extensions = set()
|
|
for lang in [
|
|
CodeLanguageLabel.PYTHON,
|
|
CodeLanguageLabel.TYPESCRIPT,
|
|
CodeLanguageLabel.JAVASCRIPT,
|
|
CodeLanguageLabel.C,
|
|
CodeLanguageLabel.JAVA,
|
|
]:
|
|
all_extensions.update(_get_file_extensions(lang))
|
|
|
|
all_files = []
|
|
for extension in all_extensions:
|
|
all_files.extend(sorted(glob.glob(f"{file_dir}/**/*{extension}", recursive=True)))
|
|
|
|
all_files = sorted(set(all_files))
|
|
|
|
for file_path in all_files:
|
|
with open(file_path, encoding="utf-8") as f:
|
|
file_content = f.read()
|
|
|
|
file_relative = os.path.relpath(file_path, start=file_dir).replace("\\", "/")
|
|
|
|
origin = DocumentOrigin(
|
|
filename=file_relative,
|
|
uri=(f"{repo_url}/blob/{commit_id}/{file_relative}" if commit_id else f"{repo_url}/{file_relative}"),
|
|
mimetype="text/plain",
|
|
binary_hash=_create_hash(file_content),
|
|
)
|
|
|
|
doc = DoclingDocument(name=file_relative, origin=origin)
|
|
doc.add_code(text=file_content, code_language=language)
|
|
documents.append(doc)
|
|
|
|
return documents
|
|
|
|
|
|
HERE = pathlib.Path(__file__).parent
|
|
DATA = HERE / "data" / "chunker_repo"
|
|
DATA.mkdir(parents=True, exist_ok=True)
|
|
|
|
REPO_SPECS = [
|
|
(
|
|
"Java",
|
|
"/test/data/chunker_repo/repos/acmeair",
|
|
"https://github.com/acmeair/acmeair",
|
|
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
|
|
),
|
|
(
|
|
"TypeScript",
|
|
"/test/data/chunker_repo/repos/outline",
|
|
"https://github.com/outline/outline",
|
|
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
|
|
),
|
|
(
|
|
"JavaScript",
|
|
"/test/data/chunker_repo/repos/jquery",
|
|
"https://github.com/jquery/jquery",
|
|
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
|
|
),
|
|
(
|
|
"Python",
|
|
"/test/data/chunker_repo/repos/docling",
|
|
"https://github.com/docling-project/docling",
|
|
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
|
|
),
|
|
(
|
|
"C",
|
|
"/test/data/chunker_repo/repos/json-c",
|
|
"https://github.com/json-c/json-c",
|
|
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
|
|
),
|
|
]
|
|
|
|
|
|
def _dump_or_assert(act_data: dict, out_path: pathlib.Path):
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if GEN_TEST_DATA:
|
|
with out_path.open("w", encoding="utf-8") as f:
|
|
json.dump(act_data, fp=f, indent=4)
|
|
f.write("\n")
|
|
else:
|
|
with out_path.open(encoding="utf-8") as f:
|
|
exp_data = json.load(fp=f)
|
|
assert exp_data == act_data
|
|
|
|
|
|
@pytest.mark.parametrize("name,local_path,repo_url,chunker_factory", REPO_SPECS)
|
|
def test_function_chunkers_repo(name, local_path, repo_url, chunker_factory):
|
|
if name == "Java" and sys.version_info < (3, 10):
|
|
pytest.skip("Skipping Java tests on python < 3.10.")
|
|
|
|
local_path_full = os.getcwd() + local_path
|
|
|
|
if not os.path.isdir(local_path_full):
|
|
pytest.skip(f"Missing repo at {local_path_full}; skipping {name} test.")
|
|
|
|
docs = create_documents_from_repository(
|
|
local_path_full,
|
|
repo_url,
|
|
language=CodeLanguageLabel(name),
|
|
commit_id="abc123def456",
|
|
)
|
|
docs = [doc for doc in docs if any(text.label == DocItemLabel.CODE and text.text for text in doc.texts)]
|
|
if not docs:
|
|
pytest.skip(f"No documents found in {local_path_full} for {name}.")
|
|
|
|
sample = docs[:3]
|
|
|
|
chunker = chunker_factory()
|
|
all_chunks = []
|
|
for doc in sample:
|
|
chunks_iter = chunker.chunk(dl_doc=doc)
|
|
chs = list(chunks_iter)
|
|
|
|
chunks = [CodeChunk.model_validate(n) for n in chs]
|
|
all_chunks.extend(chunks)
|
|
assert chunks, f"Expected chunks for {doc.name}"
|
|
for c in chunks:
|
|
assert c.text and isinstance(c.text, str)
|
|
|
|
act_data = {"root": [c.export_json_dict() for c in all_chunks]}
|
|
out_path = DATA / name / "repo_out_chunks.json"
|
|
_dump_or_assert(act_data, out_path)
|