mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* test: catch expected UserWarnings in unit tests Catch intentional UserWarnings across chunker, expander, and document metadata tests using to assert that warnings are correctly triggered and suppress warning noise during test runs Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * test(serialization): fix the wrong way to build lists Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(doctags): suppress deprecation warning when populating legacy annotations Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * feat(latex): add LaTeXMetaSerializer and migrate metadata handling Introduce 'LaTeXMetaSerializer' to serialize 'item.meta' fields as LaTeX comments. Update 'LaTeXPictureSerializer' and 'LaTeXTableSerializer' to prioritize 'item.meta' and only fall back to legacy annotations when 'meta' is absent. Update tabular data serialization on pictures to use 'item.meta.tabular_chart'. Add unit tests covering metadata serialization, legacy annotations fallback, and tabular chart export in LaTeX. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(serializer): repair _should_use_legacy_annotations warning handling Two bugs prevented correct warning scoping: - warnings.simplefilter('ignore', DeprecationWarning) passed DeprecationWarning as lineno rather than category, making the ignore filter a no-op. In Python 3.14, nested catch_warnings no longer isolates the outer filter stack, so the deprecation from accessing item.annotations leaked to callers unconditionally. - The re-emit loop was inside the catch_warnings block, causing infinite recursing when the outer context had simplefilter('always') active. Replace the 'catch_warnings' approach with '__dict__' access for both deprecated fields ('include_annotations' and 'annotations') to bypass their Pydantic property descriptors. Emit the deprecation explicitly only when 'use_legacy' is True. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * fix(serializer): correct include_annotations deprecation message The previous message ("Use include_meta instead.") pointed to a field that does not exist. Since the PR narrowed include_annotations to only affect legacy annotations, update the message to accurately describe the current behaviour and point to blocked_meta_names as the modern alternative for suppressing specific meta fields. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * docs(latex): add documentation to option fields Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
399 lines
14 KiB
Python
399 lines
14 KiB
Python
"""Tests for document profiler."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from docling_core.transforms.profiler import DocumentProfiler
|
|
from docling_core.types.doc import BoundingBox, DoclingDocument, ProvenanceItem
|
|
from docling_core.types.doc.document import DocumentOrigin, PageItem, Size, TableData
|
|
from docling_core.types.doc.labels import DocItemLabel
|
|
|
|
|
|
def test_profile_empty_document():
|
|
"""Test profiling an empty document."""
|
|
doc = DoclingDocument(name="Empty Document")
|
|
|
|
stats = DocumentProfiler.profile_document(doc)
|
|
|
|
assert stats.name == "Empty Document"
|
|
assert stats.num_pages == 0
|
|
assert stats.num_tables == 0
|
|
assert stats.num_pictures == 0
|
|
assert stats.num_texts == 0
|
|
assert stats.num_key_value_items == 0
|
|
assert stats.num_form_items == 0
|
|
assert stats.total_items == 0
|
|
assert stats.avg_items_per_page == 0.0
|
|
assert stats.origin_mimetype is None
|
|
|
|
|
|
def test_profile_simple_document():
|
|
"""Test profiling a simple document with basic content."""
|
|
doc = DoclingDocument(
|
|
name="Simple Document",
|
|
origin=DocumentOrigin(
|
|
mimetype="application/pdf",
|
|
binary_hash=12345,
|
|
filename="test.pdf",
|
|
),
|
|
)
|
|
|
|
# Add some pages
|
|
doc.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
doc.pages[2] = PageItem(page_no=2, size=Size(width=612, height=792))
|
|
|
|
# Add some text items
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Text 1", orig="Text 1")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Text 2", orig="Text 2")
|
|
doc.add_text(label=DocItemLabel.SECTION_HEADER, text="Section", orig="Section")
|
|
|
|
# Add a table
|
|
doc.add_table(data=TableData(num_rows=2, num_cols=2))
|
|
|
|
# Add a picture
|
|
doc.add_picture()
|
|
|
|
stats = DocumentProfiler.profile_document(doc)
|
|
|
|
assert stats.name == "Simple Document"
|
|
assert stats.num_pages == 2
|
|
assert stats.num_tables == 1
|
|
assert stats.num_pictures == 1
|
|
assert stats.num_texts == 3
|
|
assert stats.num_section_headers == 1
|
|
assert stats.total_items == 5
|
|
assert stats.avg_items_per_page == 2.5
|
|
assert stats.origin_mimetype == "application/pdf"
|
|
|
|
|
|
def test_profile_document_with_pictures_for_ocr():
|
|
"""Test profiling pictures that would trigger OCR based on area coverage."""
|
|
doc = DoclingDocument(name="Document with Pictures for OCR")
|
|
|
|
# Add a page
|
|
doc.pages[1] = PageItem(page_no=1, size=Size(width=1000, height=1000))
|
|
|
|
# Add a large picture (10% of page area, above default 5% threshold)
|
|
doc.add_picture(
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox(l=0, t=0, r=316.2, b=316.2), # ~10% of page area
|
|
charspan=(0, 0),
|
|
)
|
|
)
|
|
|
|
# Add a small picture (2% of page area, below default 5% threshold)
|
|
doc.add_picture(
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox(l=0, t=0, r=141.4, b=141.4), # ~2% of page area
|
|
charspan=(0, 0),
|
|
)
|
|
)
|
|
|
|
# Add a medium picture (exactly 5% of page area, at threshold)
|
|
doc.add_picture(
|
|
prov=ProvenanceItem(
|
|
page_no=1,
|
|
bbox=BoundingBox(l=0, t=0, r=223.607, b=223.607), # exactly 5% of page area
|
|
charspan=(0, 0),
|
|
)
|
|
)
|
|
|
|
stats = DocumentProfiler.profile_document(doc)
|
|
|
|
assert stats.num_pictures == 3
|
|
# 2 out of 3 pictures meet the threshold (large and medium)
|
|
assert stats.num_pictures_for_ocr == 2
|
|
|
|
# Test with custom threshold of 10%
|
|
stats_custom = DocumentProfiler.profile_document(doc, bitmap_coverage_threshold=0.10)
|
|
# Only large picture (9.99%) is below 10%, so 0 pictures
|
|
assert stats_custom.num_pictures_for_ocr == 0
|
|
|
|
# Test with custom threshold of 2%
|
|
stats_custom2 = DocumentProfiler.profile_document(doc, bitmap_coverage_threshold=0.02)
|
|
# 2 pictures are above 2% threshold (large and medium, small is 1.99%)
|
|
assert stats_custom2.num_pictures_for_ocr == 2
|
|
|
|
|
|
def test_profile_collection_empty():
|
|
"""Test profiling an empty collection."""
|
|
stats = DocumentProfiler.profile_collection([])
|
|
|
|
assert stats.num_documents == 0
|
|
assert stats.total_pages == 0
|
|
assert stats.total_tables == 0
|
|
assert stats.total_pictures == 0
|
|
assert stats.avg_items_per_document == 0.0
|
|
assert stats.avg_items_per_page == 0.0
|
|
assert stats.deciles_pages == [0.0] * 9
|
|
assert stats.deciles_tables == [0.0] * 9
|
|
assert stats.histogram_pages.bins == []
|
|
assert stats.histogram_pages.frequencies == []
|
|
assert stats.histogram_pages.bin_width == 0.0
|
|
|
|
|
|
def test_profile_collection_single_document():
|
|
"""Test profiling a collection with a single document."""
|
|
doc = DoclingDocument(name="Single Doc")
|
|
doc.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text", orig="Text")
|
|
doc.add_table(data=TableData(num_rows=1, num_cols=1))
|
|
doc.add_picture()
|
|
|
|
stats = DocumentProfiler.profile_collection(doc)
|
|
|
|
assert stats.num_documents == 1
|
|
assert stats.total_pages == 1
|
|
assert stats.total_tables == 1
|
|
assert stats.total_pictures == 1
|
|
assert stats.total_texts == 1
|
|
assert stats.min_pages == 1
|
|
assert stats.max_pages == 1
|
|
assert stats.deciles_pages[4] == 1.0 # median is d5 (5th decile, index 4)
|
|
assert stats.mean_pages == 1.0
|
|
assert stats.std_pages == 0.0
|
|
# Check histogram exists
|
|
assert len(stats.histogram_pages.bins) > 0
|
|
assert len(stats.histogram_pages.frequencies) > 0
|
|
|
|
|
|
def test_profile_collection_multiple_documents():
|
|
"""Test profiling a collection with multiple documents."""
|
|
docs = []
|
|
|
|
# Document 1: 2 pages, 1 table, 2 pictures, 2 texts
|
|
doc1 = DoclingDocument(
|
|
name="Doc1",
|
|
origin=DocumentOrigin(mimetype="application/pdf", binary_hash=1, filename="doc1.pdf"),
|
|
)
|
|
doc1.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
doc1.pages[2] = PageItem(page_no=2, size=Size(width=612, height=792))
|
|
doc1.add_table(data=TableData(num_rows=1, num_cols=1))
|
|
doc1.add_picture()
|
|
doc1.add_picture()
|
|
doc1.add_text(label=DocItemLabel.TEXT, text="Text 1", orig="Text 1")
|
|
doc1.add_text(label=DocItemLabel.TEXT, text="Text 2", orig="Text 2")
|
|
docs.append(doc1)
|
|
|
|
# Document 2: 5 pages, 3 tables, 1 picture, 10 texts
|
|
doc2 = DoclingDocument(
|
|
name="Doc2",
|
|
origin=DocumentOrigin(mimetype="application/pdf", binary_hash=2, filename="doc2.pdf"),
|
|
)
|
|
for i in range(1, 6):
|
|
doc2.pages[i] = PageItem(page_no=i, size=Size(width=612, height=792))
|
|
for _ in range(3):
|
|
doc2.add_table(data=TableData(num_rows=1, num_cols=1))
|
|
doc2.add_picture()
|
|
for i in range(10):
|
|
doc2.add_text(label=DocItemLabel.TEXT, text=f"Text {i}", orig=f"Text {i}")
|
|
docs.append(doc2)
|
|
|
|
# Document 3: 1 page, 0 tables, 5 pictures, 2 texts
|
|
doc3 = DoclingDocument(
|
|
name="Doc3",
|
|
origin=DocumentOrigin(mimetype="text/html", binary_hash=3, filename="doc3.html"),
|
|
)
|
|
doc3.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
for _ in range(5):
|
|
doc3.add_picture()
|
|
doc3.add_text(label=DocItemLabel.TEXT, text="T1", orig="T1")
|
|
doc3.add_text(label=DocItemLabel.TEXT, text="T2", orig="T2")
|
|
docs.append(doc3)
|
|
|
|
stats = DocumentProfiler.profile_collection(docs, include_individual_stats=True)
|
|
|
|
# Basic counts
|
|
assert stats.num_documents == 3
|
|
assert stats.total_pages == 8 # 2 + 5 + 1
|
|
assert stats.total_tables == 4 # 1 + 3 + 0
|
|
assert stats.total_pictures == 8 # 2 + 1 + 5
|
|
assert stats.total_texts == 14 # 2 + 10 + 2
|
|
|
|
# Page statistics
|
|
assert stats.min_pages == 1
|
|
assert stats.max_pages == 5
|
|
assert stats.deciles_pages[4] == 2.0 # median is d5 (5th decile, index 4)
|
|
assert stats.mean_pages == pytest.approx(8 / 3)
|
|
assert stats.std_pages > 0
|
|
# Check deciles are in order: [d1, d2, d3, d4, d5, d6, d7, d8, d9]
|
|
assert stats.deciles_pages[0] <= stats.deciles_pages[4] <= stats.deciles_pages[8]
|
|
# Check histogram exists
|
|
assert len(stats.histogram_pages.bins) > 0
|
|
assert len(stats.histogram_pages.frequencies) > 0
|
|
|
|
# Table statistics
|
|
assert stats.min_tables == 0
|
|
assert stats.max_tables == 3
|
|
assert stats.deciles_tables[4] == 1.0 # median is d5 (5th decile, index 4)
|
|
assert stats.mean_tables == pytest.approx(4 / 3)
|
|
# Check histogram exists
|
|
assert len(stats.histogram_tables.bins) > 0
|
|
|
|
# Picture statistics
|
|
assert stats.min_pictures == 1
|
|
assert stats.max_pictures == 5
|
|
assert stats.deciles_pictures[4] == 2.0 # median is d5 (5th decile, index 4)
|
|
assert stats.mean_pictures == pytest.approx(8 / 3)
|
|
# Check histogram exists
|
|
assert len(stats.histogram_pictures.bins) > 0
|
|
|
|
# Text statistics
|
|
assert stats.min_texts == 2
|
|
assert stats.max_texts == 10
|
|
assert stats.deciles_texts[4] == 2.0 # median is d5 (5th decile, index 4)
|
|
assert stats.mean_texts == pytest.approx(14 / 3)
|
|
# Check histogram exists
|
|
assert len(stats.histogram_texts.bins) > 0
|
|
|
|
# Document characteristics
|
|
assert len(stats.document_stats) == 3
|
|
|
|
# MIME type distribution
|
|
assert stats.mimetype_distribution["application/pdf"] == 2
|
|
assert stats.mimetype_distribution["text/html"] == 1
|
|
|
|
# Computed fields
|
|
assert stats.total_items == 26 # 14 texts + 4 tables + 8 pictures
|
|
assert stats.avg_items_per_document == pytest.approx(26 / 3)
|
|
assert stats.avg_items_per_page == pytest.approx(26 / 8)
|
|
|
|
|
|
def test_profile_collection_with_iterator():
|
|
"""Test profiling a collection using an iterator (generator)."""
|
|
|
|
def doc_generator():
|
|
for i in range(3):
|
|
doc = DoclingDocument(name=f"Doc{i}")
|
|
doc.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
doc.add_text(label=DocItemLabel.TEXT, text=f"Text {i}", orig=f"Text {i}")
|
|
yield doc
|
|
|
|
stats = DocumentProfiler.profile_collection(doc_generator())
|
|
|
|
assert stats.num_documents == 3
|
|
assert stats.total_pages == 3
|
|
assert stats.total_texts == 3
|
|
|
|
|
|
def test_profile_collection_without_individual_stats():
|
|
"""Test that individual stats are not included by default."""
|
|
docs = [DoclingDocument(name=f"Doc{i}") for i in range(3)]
|
|
|
|
stats = DocumentProfiler.profile_collection(docs, include_individual_stats=False)
|
|
|
|
assert len(stats.document_stats) == 0
|
|
|
|
|
|
def test_statistics_serialization():
|
|
"""Test that statistics can be serialized to JSON."""
|
|
doc = DoclingDocument(name="Test Doc")
|
|
doc.pages[1] = PageItem(page_no=1, size=Size(width=612, height=792))
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Text", orig="Text")
|
|
|
|
doc_stats = DocumentProfiler.profile_document(doc)
|
|
|
|
# Test DocumentStatistics serialization
|
|
json_str = doc_stats.model_dump_json()
|
|
data = json.loads(json_str)
|
|
assert data["name"] == "Test Doc"
|
|
assert data["num_pages"] == 1
|
|
assert data["total_items"] == 1
|
|
|
|
# Test CollectionStatistics serialization
|
|
coll_stats = DocumentProfiler.profile_collection([doc])
|
|
json_str = coll_stats.model_dump_json()
|
|
data = json.loads(json_str)
|
|
assert data["num_documents"] == 1
|
|
assert data["total_pages"] == 1
|
|
|
|
|
|
def test_profile_real_document():
|
|
"""Test profiling a real document from test data."""
|
|
test_file = Path("./tests/data/doc/2408.09869v3_enriched.json")
|
|
if not test_file.exists():
|
|
pytest.skip("Test file not found")
|
|
|
|
doc = DoclingDocument.load_from_json(test_file)
|
|
stats = DocumentProfiler.profile_document(doc)
|
|
|
|
# Basic sanity checks
|
|
assert stats.name == doc.name
|
|
assert stats.num_pages == len(doc.pages)
|
|
assert stats.num_tables == len(doc.tables)
|
|
assert stats.num_pictures == len(doc.pictures)
|
|
assert stats.num_texts == len(doc.texts)
|
|
assert stats.total_items > 0
|
|
|
|
|
|
def test_label_specific_counts():
|
|
"""Test that label-specific counts are accurate."""
|
|
doc = DoclingDocument(name="Label Test")
|
|
|
|
# Add various types of text items
|
|
doc.add_text(label=DocItemLabel.SECTION_HEADER, text="Section", orig="Section")
|
|
list_group = doc.add_list_group()
|
|
doc.add_list_item(text="Item 1", orig="Item 1", parent=list_group)
|
|
doc.add_list_item(text="Item 2", orig="Item 2", parent=list_group)
|
|
doc.add_list_item(text="Item 3", orig="Item 3", parent=list_group)
|
|
doc.add_text(label=DocItemLabel.CODE, text="code", orig="code")
|
|
doc.add_text(label=DocItemLabel.FORMULA, text="x=y", orig="x=y")
|
|
doc.add_text(label=DocItemLabel.TEXT, text="Text", orig="Text")
|
|
|
|
stats = DocumentProfiler.profile_document(doc)
|
|
|
|
assert stats.num_section_headers == 1
|
|
assert stats.num_list_items == 3
|
|
assert stats.num_code_items == 1
|
|
assert stats.num_formulas == 1
|
|
assert stats.num_texts == 7
|
|
|
|
|
|
def test_profile_sample_document(sample_doc):
|
|
"""Test profiling the sample document from conftest.py fixture."""
|
|
stats = DocumentProfiler.profile_document(sample_doc)
|
|
|
|
# Verify basic document properties
|
|
assert stats.name == "Untitled 1"
|
|
assert stats.num_pages == 0 # sample_doc doesn't add pages explicitly
|
|
|
|
# Verify item counts based on the sample_doc construction
|
|
assert stats.num_tables == len(sample_doc.tables)
|
|
assert stats.num_pictures == len(sample_doc.pictures)
|
|
assert stats.num_texts == len(sample_doc.texts)
|
|
assert stats.num_key_value_items == len(sample_doc.key_value_items)
|
|
assert stats.num_form_items == len(sample_doc.form_items)
|
|
|
|
# Verify label-specific counts
|
|
assert stats.num_section_headers > 0 # sample_doc has section headers
|
|
assert stats.num_list_items > 0 # sample_doc has many list items
|
|
assert stats.num_code_items > 0 # sample_doc has code items
|
|
assert stats.num_formulas > 0 # sample_doc has formulas
|
|
|
|
# Verify computed fields
|
|
assert stats.total_items > 0
|
|
assert stats.total_items == (
|
|
stats.num_texts + stats.num_tables + stats.num_pictures + stats.num_key_value_items + stats.num_form_items
|
|
)
|
|
|
|
# sample_doc has no pages, so avg_items_per_page should be 0
|
|
assert stats.avg_items_per_page == 0.0
|
|
|
|
|
|
def test_calculate_deciles_empty():
|
|
"""Test _calculate_deciles with empty data (line 191)."""
|
|
result = DocumentProfiler._calculate_deciles([])
|
|
assert result == [0.0] * 9
|
|
|
|
|
|
def test_calculate_histogram_empty():
|
|
"""Test _calculate_histogram with empty data (line 208)."""
|
|
result = DocumentProfiler._calculate_histogram([])
|
|
assert result.bins == []
|
|
assert result.frequencies == []
|
|
assert result.bin_width == 0.0
|