"""Test serialization.""" import threading from pathlib import Path from unittest.mock import MagicMock, patch from xml.etree import ElementTree as ET import pytest from docling_core.transforms.serializer.common import _DEFAULT_LABELS from docling_core.transforms.serializer.html import ( HTMLDocSerializer, HTMLMetaSerializer, HTMLOutputStyle, HTMLParams, HTMLTableSerializer, ) from docling_core.transforms.serializer.markdown import ( MarkdownDocSerializer, MarkdownParams, MarkdownTableSerializer, OrigListItemMarkerMode, _cell_content_has_table, ) from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer from docling_core.types.doc import DoclingDocument from docling_core.types.doc.base import ImageRefMode from docling_core.types.doc.document import ( BaseMeta, CharSpan, DescriptionAnnotation, EntitiesMetaField, EntityMention, LanguageMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, RefItem, RichTableCell, SummaryMetaField, TableCell, TableData, TextItem, ) from docling_core.types.doc.labels import DocItemLabel from .test_data_gen_flag import GEN_TEST_DATA def verify(exp_file: Path, actual: str): if GEN_TEST_DATA: with open(exp_file, "w", encoding="utf-8") as f: f.write(f"{actual}\n") else: with open(exp_file, encoding="utf-8") as f: expected = f.read().rstrip() # Normalize platform-dependent quote escaping for DocTags outputs name = exp_file.name if name.endswith((".dt", ".idt", ".idt.xml")): def _normalize_quotes(s: str) -> str: return s.replace(""", '"').replace(""", '"') expected = _normalize_quotes(expected) actual = _normalize_quotes(actual) assert actual == expected # =============================== # Markdown tests # =============================== def test_md_cross_page_list_page_break(): src = Path("./test/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_checkboxes(): src = Path("./test/data/doc/checkboxes.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}.gt.md", actual=actual) def test_md_cross_page_list_page_break_none(): src = Path("./test/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder=None, labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_none.gt.md", actual=actual) def test_md_cross_page_list_page_break_empty(): src = Path("./test/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_empty.gt.md", actual=actual) def test_md_cross_page_list_page_break_non_empty(): src = Path("./test/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_non_empty.gt.md", actual=actual) def test_md_cross_page_list_page_break_p2(): src = Path("./test/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder=None, pages={2}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_p2.gt.md", actual=actual) def test_md_charts(): src = Path("./test/data/doc/barchart.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_inline_and_formatting(): src = Path("./test/data/doc/inline_and_formatting.yaml") doc = DoclingDocument.load_from_yaml(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_pb_placeholder_and_page_filter(): src = Path("./test/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) # NOTE ambiguous case ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( page_break_placeholder="", pages={3, 4, 6}, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_list_item_markers(sample_doc): root_dir = Path("./test/data/doc") for mode in OrigListItemMarkerMode: for valid in [False, True]: ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( orig_list_item_marker_mode=mode, ensure_valid_list_item_marker=valid, ), ) actual = ser.serialize().text verify( root_dir / f"constructed_mode_{str(mode.value).lower()}_valid_{str(valid).lower()}.gt.md", actual=actual, ) def test_md_mark_meta_true(): src = Path("./test/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( mark_meta=True, pages={1, 5}, ), ) actual = ser.serialize().text verify( exp_file=src.parent / f"{src.stem}_p1_mark_meta_true.gt.md", actual=actual, ) def test_md_mark_meta_false(): src = Path("./test/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( mark_meta=False, pages={1, 5}, ), ) actual = ser.serialize().text verify( exp_file=src.parent / f"{src.stem}_p1_mark_meta_false.gt.md", actual=actual, ) def test_md_legacy_annotations_mark_true(sample_doc): exp_file = Path("./test/data/doc/constructed_legacy_annot_mark_true.gt.md") with pytest.warns(DeprecationWarning): sample_doc.tables[0].annotations.append( DescriptionAnnotation(text="This is a description of table 1.", provenance="foo") ) ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( mark_annotations=True, ), ) actual = ser.serialize().text verify( exp_file=exp_file, actual=actual, ) def test_md_legacy_annotations_mark_false(sample_doc): exp_file = Path("./test/data/doc/constructed_legacy_annot_mark_false.gt.md") with pytest.warns(DeprecationWarning): sample_doc.tables[0].annotations.append( DescriptionAnnotation(text="This is a description of table 1.", provenance="foo") ) ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( mark_annotations=False, ), ) actual = ser.serialize().text verify( exp_file=exp_file, actual=actual, ) def test_md_nested_lists(): src = Path("./test/data/doc/polymers.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer(doc=doc) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_rich_table(rich_table_doc): exp_file = Path("./test/data/doc/rich_table.gt.md") ser = MarkdownDocSerializer(doc=rich_table_doc) actual = ser.serialize().text verify(exp_file=exp_file, actual=actual) def test_md_single_row_table(): exp_file = Path("./test/data/doc/single_row_table.gt.md") words = ["foo", "bar"] doc = DoclingDocument(name="") row_idx = 0 table = doc.add_table(data=TableData(num_rows=1, num_cols=len(words))) for col_idx, word in enumerate(words): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=col_idx, end_col_offset_idx=col_idx + 1, text=word, ), ) ser = MarkdownDocSerializer(doc=doc) actual = ser.serialize().text verify(exp_file=exp_file, actual=actual) def test_md_pipe_in_table(): doc = DoclingDocument(name="Pipe in Table") table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) # TODO: properly handle nested tables, for now just escape the pipe doc.add_table_cell( table, TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, text="Fruits | Veggies", ), ) ser = doc.export_to_markdown() assert ser == "| Fruits | Veggies |\n|-------------------------|" def test_cell_content_has_table_detects_descendant_table(): """Ensure nested tables are detected through non-table parent nodes.""" doc = DoclingDocument(name="descendant_table") wrapper = doc.add_group() nested_table = doc.add_table(data=TableData(num_rows=1, num_cols=1), parent=wrapper) doc.add_table_cell( nested_table, TableCell( text="inner", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ), ) assert _cell_content_has_table(wrapper, doc) def _build_nested_rich_table_doc(depth: int) -> DoclingDocument: """Build a document with `depth` levels of nested RichTableCell tables. Each level is a 1x2 table whose first cell is a RichTableCell referencing the next-level table, and whose second cell is a plain TableCell. This is the structure produced by the HTML backend for Wikipedia clade tables. """ doc = DoclingDocument(name="nested_tables") def _add_level(parent, remaining: int): table = doc.add_table(data=TableData(num_rows=1, num_cols=2), parent=parent) if remaining > 0: nested = _add_level(table, remaining - 1) rich_cell: TableCell = RichTableCell( ref=nested.get_ref(), text="rich", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ) else: rich_cell = TableCell( text="leaf", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ) doc.add_table_cell(table, rich_cell) doc.add_table_cell( table, TableCell( text="plain", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, ), ) return table _add_level(doc.body, depth) return doc def test_md_nested_rich_table_no_hang(): """Regression: export_to_markdown() must not hang on nested RichTableCells. When a RichTableCell's content contains a nested table, the ``_nested_in_table`` flag passed through kwargs causes MarkdownTableSerializer to flatten the inner table instead of re-entering the full table serializer recursively. Without this guard every level of nesting re-enters the table serializer, causing exponential string growth and an indefinite hang. """ doc = _build_nested_rich_table_doc(depth=5) result: list[str] = [] def _run() -> None: result.append(doc.export_to_markdown()) t = threading.Thread(target=_run, daemon=True) t.start() t.join(timeout=5.0) assert not t.is_alive(), "export_to_markdown() hung on a document with nested RichTableCells." assert result, "export_to_markdown() produced no output" # The outer table must be a valid 2-column markdown table. # Without the pipe-escaping fix, inner-table pipes would leak into the outer # table and produce dozens of phantom columns. table_rows = [line for line in result[0].splitlines() if line.startswith("|")] assert table_rows, "Expected at least one markdown table row in output" col_counts = {line.count("|") - 1 for line in table_rows} assert col_counts == {2}, f"Outer table must have exactly 2 columns throughout; got column counts: {col_counts}" def test_md_compact_table(): """Test compact table format removes padding and uses minimal separators.""" # Test the _compact_table method directly padded_table = """| item | qty | description | | ------ | ----: | :-------------------: | | spam | 42 | A canned meat product | | eggs | 451 | Fresh farm eggs | | bacon | 0 | Out of stock |""" expected_compact = """| item | qty | description | | - | -: | :-: | | spam | 42 | A canned meat product | | eggs | 451 | Fresh farm eggs | | bacon | 0 | Out of stock |""" compact_result = MarkdownTableSerializer._compact_table(padded_table) assert compact_result == expected_compact # Verify space savings assert len(compact_result) < len(padded_table) def test_md_numeric_precision_preserved(): """Test that numeric values in tables preserve their full precision. Regression test for issue where tabulate's numparse would silently truncate numeric strings to ~6 significant figures. """ doc = DoclingDocument(name="Numeric Precision Test") precise_values = [ "225.8183", "24797.34", "20896.7184", "17358.138", "123.456789", ] table = doc.add_table(data=TableData(num_rows=len(precise_values) + 1, num_cols=2)) # Add header row doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, text="Description", ), ) doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, text="Value", ), ) # Add data rows with precise numeric values for row_idx, value in enumerate(precise_values, start=1): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=0, end_col_offset_idx=1, text=f"Item {row_idx}", ), ) doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=1, end_col_offset_idx=2, text=value, ), ) markdown_output = doc.export_to_markdown() for value in precise_values: assert value in markdown_output, ( f"Numeric value '{value}' was not preserved in markdown output. " "This indicates precision loss during table serialization." ) def test_md_traverse_pictures(): """Test traverse_pictures parameter to include text inside PictureItems.""" doc = DoclingDocument(name="Test Document") doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text before picture") picture = doc.add_picture() # Manually add a text item as child of picture text_in_picture = TextItem( self_ref=f"#/texts/{len(doc.texts)}", parent=RefItem(cref=picture.self_ref), label=DocItemLabel.PARAGRAPH, text="Text inside picture", orig="Text inside picture", ) doc.texts.append(text_in_picture) picture.children.append(RefItem(cref=text_in_picture.self_ref)) doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text after picture") # Test with traverse_pictures=False (default) ser_no_traverse = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", traverse_pictures=False, ), ) result_no_traverse = ser_no_traverse.serialize().text # Should NOT contain text inside picture assert "Text before picture" in result_no_traverse assert "Text after picture" in result_no_traverse assert "Text inside picture" not in result_no_traverse assert "" in result_no_traverse # Test with traverse_pictures=True ser_with_traverse = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", traverse_pictures=True, ), ) result_with_traverse = ser_with_traverse.serialize().text # Should contain text inside picture assert "Text before picture" in result_with_traverse assert "Text after picture" in result_with_traverse assert "Text inside picture" in result_with_traverse assert "" in result_with_traverse # =============================== # HTML tests # =============================== def test_html_table_serializer_get_header_and_body_lines(): """Test HTMLTableSerializer.get_header_and_body_lines() method.""" serializer = HTMLTableSerializer() # Test 1: Valid HTML with headers and data valid_html = "
| Header1 | Header2 |
|---|---|
| Data1 | Data2 |