Files
docling-core/tests/test_code_chunker.py
Cesar Berrospi Ramis a59c38c1da chore: rename test folder to tests (#739)
test: rename 'test' folder to 'tests'

To align with other Docling repositories, use 'tests' for test modules.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
2026-09-04 10:49:52 +02:00

174 lines
5.7 KiB
Python

import glob
import hashlib
import json
import os
import pathlib
import sys
from typing import Optional
import git
import pytest
from docling_core.transforms.chunker.code_chunking._utils import _get_file_extensions
from docling_core.transforms.chunker.code_chunking.code_chunk import CodeChunk
from docling_core.transforms.chunker.code_chunking.standard_code_chunking_strategy import (
StandardCodeChunkingStrategy,
)
from docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker
from docling_core.types.doc import DoclingDocument, DocumentOrigin
from docling_core.types.doc.labels import CodeLanguageLabel, DocItemLabel
from .test_data_gen_flag import GEN_TEST_DATA
def _create_hash(string: str) -> str:
"""Return a SHA-256 hex digest of the given string."""
hasher = hashlib.sha256(usedforsecurity=False)
hasher.update(string.encode("utf-8"))
return hasher.hexdigest()
def get_latest_commit_id(file_dir: str) -> str:
"""Returns the latest commit ID in the given Git repository directory."""
try:
repo = git.Repo(file_dir, search_parent_directories=True)
return repo.head.commit.hexsha
except Exception:
return ""
def create_documents_from_repository(
file_dir: str,
repo_url: str,
language: CodeLanguageLabel,
commit_id: str | None = None,
) -> list[DoclingDocument]:
"""Build DoclingDocument objects from a local checkout, one per code file."""
documents: list[DoclingDocument] = []
if commit_id is None:
commit_id = get_latest_commit_id(file_dir)
all_extensions = set()
for lang in [
CodeLanguageLabel.PYTHON,
CodeLanguageLabel.TYPESCRIPT,
CodeLanguageLabel.JAVASCRIPT,
CodeLanguageLabel.C,
CodeLanguageLabel.JAVA,
]:
all_extensions.update(_get_file_extensions(lang))
all_files = []
for extension in all_extensions:
all_files.extend(sorted(glob.glob(f"{file_dir}/**/*{extension}", recursive=True)))
all_files = sorted(set(all_files))
for file_path in all_files:
with open(file_path, encoding="utf-8") as f:
file_content = f.read()
file_relative = os.path.relpath(file_path, start=file_dir).replace("\\", "/")
origin = DocumentOrigin(
filename=file_relative,
uri=(f"{repo_url}/blob/{commit_id}/{file_relative}" if commit_id else f"{repo_url}/{file_relative}"),
mimetype="text/plain",
binary_hash=_create_hash(file_content),
)
doc = DoclingDocument(name=file_relative, origin=origin)
doc.add_code(text=file_content, code_language=language)
documents.append(doc)
return documents
HERE = pathlib.Path(__file__).parent
DATA = HERE / "data" / "chunker_repo"
DATA.mkdir(parents=True, exist_ok=True)
REPO_SPECS = [
(
"Java",
"/tests/data/chunker_repo/repos/acmeair",
"https://github.com/acmeair/acmeair",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"TypeScript",
"/tests/data/chunker_repo/repos/outline",
"https://github.com/outline/outline",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"JavaScript",
"/tests/data/chunker_repo/repos/jquery",
"https://github.com/jquery/jquery",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"Python",
"/tests/data/chunker_repo/repos/docling",
"https://github.com/docling-project/docling",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
(
"C",
"/tests/data/chunker_repo/repos/json-c",
"https://github.com/json-c/json-c",
lambda: HierarchicalChunker(code_chunking_strategy=StandardCodeChunkingStrategy(max_tokens=5000)),
),
]
def _dump_or_assert(act_data: dict, out_path: pathlib.Path):
out_path.parent.mkdir(parents=True, exist_ok=True)
if GEN_TEST_DATA:
with out_path.open("w", encoding="utf-8") as f:
json.dump(act_data, fp=f, indent=4)
f.write("\n")
else:
with out_path.open(encoding="utf-8") as f:
exp_data = json.load(fp=f)
assert exp_data == act_data
@pytest.mark.parametrize("name,local_path,repo_url,chunker_factory", REPO_SPECS)
def test_function_chunkers_repo(name, local_path, repo_url, chunker_factory):
if name == "Java" and sys.version_info < (3, 10):
pytest.skip("Skipping Java tests on python < 3.10.")
local_path_full = os.getcwd() + local_path
if not os.path.isdir(local_path_full):
pytest.skip(f"Missing repo at {local_path_full}; skipping {name} test.")
docs = create_documents_from_repository(
local_path_full,
repo_url,
language=CodeLanguageLabel(name),
commit_id="abc123def456",
)
docs = [doc for doc in docs if any(text.label == DocItemLabel.CODE and text.text for text in doc.texts)]
if not docs:
pytest.skip(f"No documents found in {local_path_full} for {name}.")
sample = docs[:3]
chunker = chunker_factory()
all_chunks = []
for doc in sample:
chunks_iter = chunker.chunk(dl_doc=doc)
chs = list(chunks_iter)
chunks = [CodeChunk.model_validate(n) for n in chs]
all_chunks.extend(chunks)
assert chunks, f"Expected chunks for {doc.name}"
for c in chunks:
assert c.text and isinstance(c.text, str)
act_data = {"root": [c.export_json_dict() for c in all_chunks]}
out_path = DATA / name / "repo_out_chunks.json"
_dump_or_assert(act_data, out_path)