Files
docling-core/tests/test_code_chunking_strategy.py
Cesar Berrospi Ramis a59c38c1da chore: rename test folder to tests (#739)
test: rename 'test' folder to 'tests'

To align with other Docling repositories, use 'tests' for test modules.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
2026-09-04 10:49:52 +02:00

197 lines
6.4 KiB
Python

from pathlib import Path
import pytest
from docling_core.transforms.chunker import (
CodeLanguageLabel,
HierarchicalChunker,
HybridChunker,
StandardCodeChunkingStrategy,
)
from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
from docling_core.transforms.serializer.markdown import (
MarkdownDocSerializer,
MarkdownParams,
)
from docling_core.types.doc.base import Size
from docling_core.types.doc.document import DoclingDocument, DocumentOrigin
from docling_core.types.doc.labels import DocItemLabel
@pytest.fixture
def test_data_dir():
"""Path to test data directory."""
return Path(__file__).parent / "data" / "repo_chunking"
def test_code_chunking_strategies():
"""Test different code chunking strategies."""
python_code = '''
def fibonacci(n):
"""Calculate the nth Fibonacci number."""
if n <= 1:
return n
return fibonacci(n-1) + fibonacci(n-2)
def factorial(n):
"""Calculate factorial of n."""
if n <= 1:
return 1
return n * factorial(n-1)
'''
doc = DoclingDocument(name="test")
code_item = doc.add_code(text=python_code, code_language=CodeLanguageLabel.PYTHON)
strategy = StandardCodeChunkingStrategy(min_chunk_size=10, max_tokens=100)
doc_ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(
format_code_blocks=False,
),
)
chunks = list(strategy.chunk_code_item(item=code_item, doc=doc, doc_serializer=doc_ser))
assert len(chunks) > 0
for chunk in chunks:
assert hasattr(chunk, "text")
assert hasattr(chunk, "meta")
assert hasattr(chunk.meta, "chunk_type")
def test_hierarchical_chunker_integration():
"""Test HierarchicalChunker with and without code chunking strategy."""
python_code = '''
def fibonacci(n):
"""Calculate the nth Fibonacci number."""
if n <= 1:
return n
return fibonacci(n-1) + fibonacci(n-2)
'''
doc = DoclingDocument(name="test")
doc.add_page(page_no=0, size=Size(width=612.0, height=792.0))
doc.add_text(
label=DocItemLabel.TEXT,
text="Here's some Python code:",
orig="Here's some Python code:",
)
doc.add_code(text=python_code, code_language=CodeLanguageLabel.PYTHON, orig=python_code)
doc.origin = DocumentOrigin(filename="test.py", mimetype="text/x-python", binary_hash=12345)
strategy = StandardCodeChunkingStrategy(min_chunk_size=50, max_tokens=1000)
chunker_with_strategy = HierarchicalChunker(code_chunking_strategy=strategy)
chunks_with_strategy = list(chunker_with_strategy.chunk(doc))
assert len(chunks_with_strategy) > 0
for chunk in chunks_with_strategy:
assert hasattr(chunk, "text")
assert hasattr(chunk, "meta")
chunker_without_strategy = HierarchicalChunker()
chunks_without_strategy = list(chunker_without_strategy.chunk(doc))
assert len(chunks_without_strategy) > 0
for chunk in chunks_without_strategy:
assert hasattr(chunk, "text")
assert hasattr(chunk, "meta")
def test_hybrid_chunker_with_code_files(test_data_dir):
"""Test that HybridChunker can process code files."""
tokenizer = HuggingFaceTokenizer.from_pretrained(
model_name="sentence-transformers/all-MiniLM-L6-v2", max_tokens=512
)
chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True)
python_file = test_data_dir / "sample.py"
if not python_file.exists():
pytest.skip("Python test file not found")
doc = DoclingDocument(name="sample.py")
doc.origin = DocumentOrigin(filename="sample.py", mimetype="text/x-python", binary_hash=12345)
with open(python_file, encoding="utf-8") as f:
content = f.read()
doc.add_code(text=content)
chunks = list(chunker.chunk(dl_doc=doc))
assert len(chunks) > 0
for chunk in chunks:
assert hasattr(chunk, "text")
assert hasattr(chunk, "meta")
assert len(chunk.text) > 0
def test_unsupported_language_fallback(test_data_dir):
"""Test that unsupported languages fall back to regular text chunking."""
tokenizer = HuggingFaceTokenizer.from_pretrained(
model_name="sentence-transformers/all-MiniLM-L6-v2", max_tokens=512
)
chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True)
go_file = test_data_dir / "sample.go"
if go_file.exists():
doc = DoclingDocument(name="sample.go")
doc.origin = DocumentOrigin(filename="sample.go", mimetype="text/plain", binary_hash=12345)
with open(go_file, encoding="utf-8") as f:
content = f.read()
doc.add_code(text=content)
assert len(doc.texts) == 1
assert doc.texts[0].code_language == CodeLanguageLabel.UNKNOWN
chunks = list(chunker.chunk(dl_doc=doc))
assert len(chunks) > 0
all_text = " ".join(chunk.text for chunk in chunks)
assert "package main" in all_text
assert "func fibonacci" in all_text
md_file = test_data_dir / "sample.md"
if md_file.exists():
doc = DoclingDocument(name="sample.md")
doc.origin = DocumentOrigin(filename="sample.md", mimetype="text/plain", binary_hash=12345)
with open(md_file, encoding="utf-8") as f:
content = f.read()
doc.add_code(text=content)
assert len(doc.texts) == 1
assert doc.texts[0].code_language == CodeLanguageLabel.UNKNOWN
chunks = list(chunker.chunk(dl_doc=doc))
assert len(chunks) > 0
all_text = " ".join(chunk.text for chunk in chunks)
assert "Sample Markdown File" in all_text
assert "def hello()" in all_text
def test_repository_processing(test_data_dir):
"""Test processing multiple files from a repository."""
tokenizer = HuggingFaceTokenizer.from_pretrained(
model_name="sentence-transformers/all-MiniLM-L6-v2", max_tokens=512
)
chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True)
all_chunks = []
for file_path in test_data_dir.glob("sample.*"):
doc = DoclingDocument(name=file_path.name)
doc.origin = DocumentOrigin(filename=file_path.name, mimetype="text/plain", binary_hash=12345)
with open(file_path, encoding="utf-8") as f:
content = f.read()
doc.add_code(text=content)
chunks = list(chunker.chunk(dl_doc=doc))
all_chunks.extend(chunks)
assert len(all_chunks) > 0
for chunk in all_chunks:
assert hasattr(chunk, "text")
assert hasattr(chunk, "meta")
assert len(chunk.text) > 0