"""Test serialization.""" import threading from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from typing import TypeAlias, Union from unittest.mock import MagicMock, patch from xml.etree import ElementTree as ET import pytest from pydantic import AnyUrl from docling_core.transforms.serializer.common import _DEFAULT_LABELS from docling_core.transforms.serializer.html import ( HTMLDocSerializer, HTMLMetaSerializer, HTMLOutputStyle, HTMLParams, HTMLTableSerializer, ) from docling_core.transforms.serializer.markdown import ( MarkdownDocSerializer, MarkdownParams, MarkdownPictureSerializer, MarkdownTableSerializer, OrigListItemMarkerMode, _cell_content_has_table, ) from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer from docling_core.types.doc import DoclingDocument, ImageRef from docling_core.types.doc.base import BoundingBox, CoordOrigin, ImageRefMode, Size from docling_core.types.doc.document import ( BaseMeta, CharSpan, DescriptionAnnotation, EntitiesMetaField, EntityMention, LanguageMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, ProvenanceItem, RefItem, RichTableCell, SummaryMetaField, TableCell, TableData, TextItem, ) from docling_core.types.doc.labels import DocItemLabel from .test_data_gen_flag import GEN_TEST_DATA def verify(exp_file: Path, actual: str): if GEN_TEST_DATA: with open(exp_file, "w", encoding="utf-8") as f: f.write(f"{actual}\n") else: with open(exp_file, encoding="utf-8") as f: expected = f.read().rstrip() # Normalize platform-dependent quote escaping for DocTags outputs name = exp_file.name if name.endswith((".dt", ".idt", ".idt.xml")): def _normalize_quotes(s: str) -> str: return s.replace(""", '"').replace(""", '"') expected = _normalize_quotes(expected) actual = _normalize_quotes(actual) assert actual == expected # =============================== # Markdown tests # =============================== def test_md_cross_page_list_page_break(): src = Path("./tests/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_checkboxes(): src = Path("./tests/data/doc/checkboxes.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}.gt.md", actual=actual) def test_md_cross_page_list_page_break_none(): src = Path("./tests/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder=None, labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_none.gt.md", actual=actual) def test_md_cross_page_list_page_break_empty(): src = Path("./tests/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_empty.gt.md", actual=actual) def test_md_cross_page_list_page_break_non_empty(): src = Path("./tests/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder="", labels=_DEFAULT_LABELS - {DocItemLabel.PICTURE}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_pb_non_empty.gt.md", actual=actual) def test_md_cross_page_list_page_break_p2(): src = Path("./tests/data/doc/activities.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", page_break_placeholder=None, pages={2}, ), ) actual = ser.serialize().text verify(exp_file=src.parent / f"{src.stem}_p2.gt.md", actual=actual) _PAGE_BREAK = "" # builds a document whose items carry the given (text, page number) provenance def _make_paged_doc(entries: list[tuple[str, int]], *, as_list: bool, num_pages: int) -> DoclingDocument: doc = DoclingDocument(name="paged") for page_no in range(1, num_pages + 1): doc.add_page(page_no=page_no, size=Size(width=595, height=842), image=None) group = doc.add_list_group(name="list") if as_list else None for text, page_no in entries: prov = ProvenanceItem( page_no=page_no, bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT), charspan=(0, len(text)), ) if group is not None: doc.add_list_item(text=text, parent=group, prov=prov) else: doc.add_text(label=DocItemLabel.TEXT, text=text, prov=prov) return doc def test_md_cross_page_list_page_break_matches_per_page_export(): doc = _make_paged_doc( [("item one", 1), ("item two", 1), ("item three", 2), ("item four", 2)], as_list=True, num_pages=2, ) globally = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK) per_page = f"\n\n{_PAGE_BREAK}\n\n".join(doc.export_to_markdown(page_no=n) for n in (1, 2)) assert globally == per_page def test_md_cross_page_text_page_break_matches_per_page_export(): # the control: outside a list scope this already held, and must keep holding doc = _make_paged_doc( [("item one", 1), ("item two", 1), ("item three", 2), ("item four", 2)], as_list=False, num_pages=2, ) globally = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK) per_page = f"\n\n{_PAGE_BREAK}\n\n".join(doc.export_to_markdown(page_no=n) for n in (1, 2)) assert globally == per_page def test_md_page_break_spacing_independent_of_scope(): entries = [("item one", 1), ("item two", 1), ("item three", 2)] in_list = _make_paged_doc(entries, as_list=True, num_pages=2).export_to_markdown(page_break_placeholder=_PAGE_BREAK) at_doc_scope = _make_paged_doc(entries, as_list=False, num_pages=2).export_to_markdown( page_break_placeholder=_PAGE_BREAK ) for actual in (in_list, at_doc_scope): assert f"\n\n{_PAGE_BREAK}\n\n" in actual def test_md_cross_page_list_page_break_empty_placeholder(): doc = _make_paged_doc([("item one", 1), ("item two", 2)], as_list=True, num_pages=2) # an empty placeholder still occupies the blank lines the marker would have assert doc.export_to_markdown(page_break_placeholder="") == "- item one\n\n\n\n- item two" def test_md_page_break_at_document_bounds(): doc = _make_paged_doc([("item 1", 1), ("item 2", 2), ("item 3", 3)], as_list=True, num_pages=3) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams(page_break_placeholder=_PAGE_BREAK, pages={2}), ) # a break opening or closing the export gains no blank line on the outer side assert ser.serialize().text == f"{_PAGE_BREAK}\n\n- item 2\n\n{_PAGE_BREAK}" def test_md_consecutive_page_breaks(): doc = _make_paged_doc([(f"item {n}", n) for n in range(1, 7)], as_list=True, num_pages=6) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams(page_break_placeholder=_PAGE_BREAK, pages={3, 4, 6}), ) actual = ser.serialize().text assert f"{_PAGE_BREAK}\n\n{_PAGE_BREAK}" in actual assert "\n\n\n" not in actual def test_md_nested_list_page_break_keeps_indent(): doc = DoclingDocument(name="nested") for page_no in (1, 2): doc.add_page(page_no=page_no, size=Size(width=595, height=842), image=None) def _prov(page_no: int, text: str) -> ProvenanceItem: return ProvenanceItem( page_no=page_no, bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT), charspan=(0, len(text)), ) outer = doc.add_list_group(name="outer") doc.add_list_item(text="outer one", parent=outer, prov=_prov(1, "outer one")) inner = doc.add_list_group(name="inner", parent=outer) doc.add_list_item(text="inner a", parent=inner, prov=_prov(1, "inner a")) doc.add_list_item(text="inner b", parent=inner, prov=_prov(2, "inner b")) doc.add_list_item(text="outer two", parent=outer, prov=_prov(2, "outer two")) # the marker keeps the indentation that holds it inside the outer item assert f"\n\n {_PAGE_BREAK}\n\n" in doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK) def test_md_page_break_between_list_and_table(): doc = _make_paged_doc([("item one", 1), ("item two", 1)], as_list=True, num_pages=2) doc.add_table( data=TableData( num_rows=1, num_cols=1, table_cells=[ TableCell( text="cell", row_span=1, col_span=1, start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ) ], ), prov=ProvenanceItem( page_no=2, bbox=BoundingBox(l=50, t=700, r=500, b=680, coord_origin=CoordOrigin.BOTTOMLEFT), charspan=(0, 4), ), ) actual = doc.export_to_markdown(page_break_placeholder=_PAGE_BREAK) assert f"- item two\n\n{_PAGE_BREAK}\n\n|" in actual def test_md_charts(): src = Path("./tests/data/doc/barchart.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_inline_and_formatting(): src = Path("./tests/data/doc/inline_and_formatting.yaml") doc = DoclingDocument.load_from_yaml(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_pb_placeholder_and_page_filter(): src = Path("./tests/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) # NOTE ambiguous case ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( page_break_placeholder="", pages={3, 4, 6}, ), ) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_list_item_markers(sample_doc): root_dir = Path("./tests/data/doc") for mode in OrigListItemMarkerMode: for valid in [False, True]: ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( orig_list_item_marker_mode=mode, ensure_valid_list_item_marker=valid, ), ) actual = ser.serialize().text verify( root_dir / f"constructed_mode_{str(mode.value).lower()}_valid_{str(valid).lower()}.gt.md", actual=actual, ) def test_md_mark_meta_true(): src = Path("./tests/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( mark_meta=True, pages={1, 5}, ), ) actual = ser.serialize().text verify( exp_file=src.parent / f"{src.stem}_p1_mark_meta_true.gt.md", actual=actual, ) def test_md_mark_meta_false(): src = Path("./tests/data/doc/2408.09869v3_enriched.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer( doc=doc, params=MarkdownParams( mark_meta=False, pages={1, 5}, ), ) actual = ser.serialize().text verify( exp_file=src.parent / f"{src.stem}_p1_mark_meta_false.gt.md", actual=actual, ) def test_md_legacy_annotations_mark_true(sample_doc): exp_file = Path("./tests/data/doc/constructed_legacy_annot_mark_true.gt.md") with pytest.warns(DeprecationWarning): sample_doc.tables[0].annotations.append( DescriptionAnnotation(text="This is a description of table 1.", provenance="foo") ) ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( mark_annotations=True, ), ) actual = ser.serialize().text verify( exp_file=exp_file, actual=actual, ) def test_md_legacy_annotations_mark_false(sample_doc): exp_file = Path("./tests/data/doc/constructed_legacy_annot_mark_false.gt.md") with pytest.warns(DeprecationWarning): sample_doc.tables[0].annotations.append( DescriptionAnnotation(text="This is a description of table 1.", provenance="foo") ) ser = MarkdownDocSerializer( doc=sample_doc, params=MarkdownParams( mark_annotations=False, ), ) actual = ser.serialize().text verify( exp_file=exp_file, actual=actual, ) def test_md_nested_lists(): src = Path("./tests/data/doc/polymers.json") doc = DoclingDocument.load_from_json(src) ser = MarkdownDocSerializer(doc=doc) actual = ser.serialize().text verify(exp_file=src.with_suffix(".gt.md"), actual=actual) def test_md_rich_table(rich_table_doc): exp_file = Path("./tests/data/doc/rich_table.gt.md") ser = MarkdownDocSerializer(doc=rich_table_doc) actual = ser.serialize().text verify(exp_file=exp_file, actual=actual) def test_md_single_row_table(): exp_file = Path("./tests/data/doc/single_row_table.gt.md") words = ["foo", "bar"] doc = DoclingDocument(name="") row_idx = 0 table = doc.add_table(data=TableData(num_rows=1, num_cols=len(words))) for col_idx, word in enumerate(words): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=col_idx, end_col_offset_idx=col_idx + 1, text=word, ), ) ser = MarkdownDocSerializer(doc=doc) actual = ser.serialize().text verify(exp_file=exp_file, actual=actual) def test_md_field_region(): exp_file = Path("./tests/data/doc/field_region.gt.md") doc = DoclingDocument(name="") field_region = doc.add_field_region() field_item = doc.add_field_item(parent=field_region) doc.add_field_key(text="Name:", parent=field_item) doc.add_field_value(text="John Doe", parent=field_item) ser = MarkdownDocSerializer(doc=doc) actual = ser.serialize().text verify(exp_file=exp_file, actual=actual) def _table_with_rows(rows: list[list[str]], num_header_rows: int) -> DoclingDocument: doc = DoclingDocument(name="") table = doc.add_table(data=TableData(num_rows=len(rows), num_cols=len(rows[0]))) for row_idx, row in enumerate(rows): for col_idx, text in enumerate(row): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=col_idx, end_col_offset_idx=col_idx + 1, text=text, column_header=row_idx < num_header_rows, ), ) return doc def test_md_table_stacked_header_is_flattened(): doc = _table_with_rows( [["Product", "Tier 1", "Tier 2"], ["", "0-99", "100+"], ["CAT-001", "10.00", "9.00"]], num_header_rows=2, ) actual = MarkdownDocSerializer(doc=doc).serialize().text assert actual == ( "| Product | Tier 1 - 0-99 | Tier 2 - 100+ |\n" "|-----------|-----------------|-----------------|\n" "| CAT-001 | 10.00 | 9.00 |" ) def test_md_table_stacked_header_drops_repeated_span_text(): doc = _table_with_rows( [["human", "MRCNN"], ["human", "R50"], ["84-89", "68.4"]], num_header_rows=2, ) actual = MarkdownDocSerializer(doc=doc).serialize().text assert actual.splitlines()[0] == "| human | MRCNN - R50 |" def test_md_table_keeps_rows_under_a_vertically_spanning_header(): """A row-spanning header is repeated into each row it covers by `grid`. Only rows a header starts on are header rows, otherwise the data rows under the span get flattened into the header and disappear from the body. """ doc = DoclingDocument(name="") table = doc.add_table(data=TableData(num_rows=2, num_cols=2)) for cell in ( TableCell( text="Category", start_row_offset_idx=0, end_row_offset_idx=2, start_col_offset_idx=0, end_col_offset_idx=1, row_span=2, column_header=True, ), TableCell( text="Price", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, column_header=True, ), TableCell( text="10.00", start_row_offset_idx=1, end_row_offset_idx=2, start_col_offset_idx=1, end_col_offset_idx=2, ), ): doc.add_table_cell(table_item=table, cell=cell) actual = MarkdownDocSerializer(doc=doc).serialize().text assert actual == ("| Category | Price |\n|------------|---------|\n| Category | 10.00 |") def test_md_table_with_header_flags_below_row_zero_keeps_every_row_as_data(): """No leading header block, so nothing is promoted and no row is dropped.""" doc = DoclingDocument(name="") rows = [["title", ""], ["Product", "Price"], ["CAT-001", "10.00"]] table = doc.add_table(data=TableData(num_rows=len(rows), num_cols=2)) for row_idx, row in enumerate(rows): for col_idx, text in enumerate(row): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=col_idx, end_col_offset_idx=col_idx + 1, text=text, column_header=row_idx == 1, ), ) actual = MarkdownDocSerializer(doc=doc).serialize().text assert actual.splitlines()[0] == "| | |" assert "| title | |" in actual assert "| CAT-001 | 10.00 |" in actual def test_md_table_without_header_flags_keeps_first_row_as_header(): """Backends that never set column_header keep the pre-existing behavior.""" doc = _table_with_rows([["foo", "bar"], ["baz", "qux"]], num_header_rows=0) actual = MarkdownDocSerializer(doc=doc).serialize().text assert actual.splitlines()[0] == "| foo | bar |" def test_md_table_single_header_row_with_flags(): """A single flagged header row is used as-is without joining or deduplication.""" doc = _table_with_rows( [["Product", "Price"], ["CAT-001", "10.00"], ["CAT-002", "9.00"]], num_header_rows=1, ) actual = MarkdownDocSerializer(doc=doc).serialize().text lines = actual.splitlines() assert lines[0] == "| Product | Price |" assert lines[2] == "| CAT-001 | 10.00 |" assert lines[3] == "| CAT-002 | 9.00 |" def test_md_pipe_in_table(): doc = DoclingDocument(name="Pipe in Table") table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) # TODO: properly handle nested tables, for now just escape the pipe doc.add_table_cell( table, TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, text="Fruits | Veggies", ), ) ser = doc.export_to_markdown() assert ser == "| Fruits | Veggies |\n|-------------------------|" def test_md_heading_in_rich_table_cell_renders_as_plain_text(): """Test headings inside RichTableCell must not emit `#` markers. According to the Markdown spec, heading syntax is invalid inside tables. A SectionHeaderItem or TitleItem referenced by a RichTableCell must be serialized as plain text, not as `## Heading`. """ doc = DoclingDocument(name="heading_in_table") table = doc.add_table(data=TableData(num_rows=2, num_cols=2)) section_header = doc.add_heading(text="Section Heading", level=1, parent=table) title = doc.add_title(text="Document Title", parent=table) doc.add_table_cell( table, RichTableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ref=section_header.get_ref(), ), ) doc.add_table_cell( table, RichTableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, ref=title.get_ref(), ), ) doc.add_table_cell( table, TableCell( start_row_offset_idx=1, end_row_offset_idx=2, start_col_offset_idx=0, end_col_offset_idx=1, text="value A", ), ) doc.add_table_cell( table, TableCell( start_row_offset_idx=1, end_row_offset_idx=2, start_col_offset_idx=1, end_col_offset_idx=2, text="value B", ), ) md = doc.export_to_markdown() assert "Section Heading" in md assert "Document Title" in md table_lines = [line for line in md.splitlines() if line.startswith("|")] for line in table_lines: assert "#" not in line, f"Heading marker leaked into table cell: {line!r}\nFull output:\n{md}" def test_cell_content_has_table_detects_descendant_table(): """Ensure nested tables are detected through non-table parent nodes.""" doc = DoclingDocument(name="descendant_table") wrapper = doc.add_group() nested_table = doc.add_table(data=TableData(num_rows=1, num_cols=1), parent=wrapper) doc.add_table_cell( nested_table, TableCell( text="inner", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ), ) assert _cell_content_has_table(wrapper, doc) def _build_nested_rich_table_doc(depth: int) -> DoclingDocument: """Build a document with `depth` levels of nested RichTableCell tables. Each level is a 1x2 table whose first cell is a RichTableCell referencing the next-level table, and whose second cell is a plain TableCell. This is the structure produced by the HTML backend for Wikipedia clade tables. """ doc = DoclingDocument(name="nested_tables") def _add_level(parent, remaining: int): table = doc.add_table(data=TableData(num_rows=1, num_cols=2), parent=parent) if remaining > 0: nested = _add_level(table, remaining - 1) rich_cell: TableCell = RichTableCell( ref=nested.get_ref(), text="rich", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ) else: rich_cell = TableCell( text="leaf", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, ) doc.add_table_cell(table, rich_cell) doc.add_table_cell( table, TableCell( text="plain", start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, ), ) return table _add_level(doc.body, depth) return doc def test_md_nested_rich_table_no_hang(): """Regression: export_to_markdown() must not hang on nested RichTableCells. When a RichTableCell's content contains a nested table, the ``_nested_in_table`` flag passed through kwargs causes MarkdownTableSerializer to flatten the inner table instead of re-entering the full table serializer recursively. Without this guard every level of nesting re-enters the table serializer, causing exponential string growth and an indefinite hang. """ doc = _build_nested_rich_table_doc(depth=5) result: list[str] = [] def _run() -> None: result.append(doc.export_to_markdown()) t = threading.Thread(target=_run, daemon=True) t.start() t.join(timeout=5.0) assert not t.is_alive(), "export_to_markdown() hung on a document with nested RichTableCells." assert result, "export_to_markdown() produced no output" # The outer table must be a valid 2-column markdown table. # Without the pipe-escaping fix, inner-table pipes would leak into the outer # table and produce dozens of phantom columns. table_rows = [line for line in result[0].splitlines() if line.startswith("|")] assert table_rows, "Expected at least one markdown table row in output" col_counts = {line.count("|") - 1 for line in table_rows} assert col_counts == {2}, f"Outer table must have exactly 2 columns throughout; got column counts: {col_counts}" def test_md_compact_table(): """Test compact table format removes padding and uses minimal separators.""" # Test the _compact_table method directly padded_table = """| item | qty | description | | ------ | ----: | :-------------------: | | spam | 42 | A canned meat product | | eggs | 451 | Fresh farm eggs | | bacon | 0 | Out of stock |""" expected_compact = """| item | qty | description | | - | -: | :-: | | spam | 42 | A canned meat product | | eggs | 451 | Fresh farm eggs | | bacon | 0 | Out of stock |""" compact_result = MarkdownTableSerializer._compact_table(padded_table) assert compact_result == expected_compact # Verify space savings assert len(compact_result) < len(padded_table) def test_md_numeric_precision_preserved(): """Test that numeric values in tables preserve their full precision. Regression test for issue where tabulate's numparse would silently truncate numeric strings to ~6 significant figures. """ doc = DoclingDocument(name="Numeric Precision Test") precise_values = [ "225.8183", "24797.34", "20896.7184", "17358.138", "123.456789", ] table = doc.add_table(data=TableData(num_rows=len(precise_values) + 1, num_cols=2)) # Add header row doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, text="Description", ), ) doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=1, end_col_offset_idx=2, text="Value", ), ) # Add data rows with precise numeric values for row_idx, value in enumerate(precise_values, start=1): doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=0, end_col_offset_idx=1, text=f"Item {row_idx}", ), ) doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=row_idx, end_row_offset_idx=row_idx + 1, start_col_offset_idx=1, end_col_offset_idx=2, text=value, ), ) markdown_output = doc.export_to_markdown() for value in precise_values: assert value in markdown_output, ( f"Numeric value '{value}' was not preserved in markdown output. " "This indicates precision loss during table serialization." ) def test_md_traverse_pictures(): """Test traverse_pictures parameter to include text inside PictureItems.""" doc = DoclingDocument(name="Test Document") doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text before picture") picture = doc.add_picture() # Manually add a text item as child of picture text_in_picture = TextItem( self_ref=f"#/texts/{len(doc.texts)}", parent=RefItem(cref=picture.self_ref), label=DocItemLabel.PARAGRAPH, text="Text inside picture", orig="Text inside picture", ) doc.texts.append(text_in_picture) picture.children.append(RefItem(cref=text_in_picture.self_ref)) doc.add_text(label=DocItemLabel.PARAGRAPH, text="Text after picture") # Test with traverse_pictures=False (default) ser_no_traverse = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", traverse_pictures=False, ), ) result_no_traverse = ser_no_traverse.serialize().text # Should NOT contain text inside picture assert "Text before picture" in result_no_traverse assert "Text after picture" in result_no_traverse assert "Text inside picture" not in result_no_traverse assert "" in result_no_traverse # Test with traverse_pictures=True ser_with_traverse = MarkdownDocSerializer( doc=doc, params=MarkdownParams( image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="", traverse_pictures=True, ), ) result_with_traverse = ser_with_traverse.serialize().text # Should contain text inside picture assert "Text before picture" in result_with_traverse assert "Text after picture" in result_with_traverse assert "Text inside picture" in result_with_traverse assert "" in result_with_traverse def test_md_line_breaks(): """Newlines inside TextItem and its subclasses are serialized per GFM spec. Single `\\n` in body text and list items becomes a hard line break (` \\n`). Double `\\n\\n` is preserved as a paragraph break. Heading newlines are collapsed to a space (headings cannot span multiple lines). Table cell newlines are replaced with a space (table syntax requires single lines). """ # Single \n in a TextItem -> GFM hard line break (two trailing spaces). doc = DoclingDocument(name="lb") doc.add_text(label=DocItemLabel.TEXT, text="Hello\nWorld") assert MarkdownDocSerializer(doc=doc).serialize().text == "Hello \nWorld" # \n\n in a TextItem -> paragraph break, no trailing spaces added. doc = DoclingDocument(name="lb") doc.add_text(label=DocItemLabel.TEXT, text="Para one\n\nPara two") assert MarkdownDocSerializer(doc=doc).serialize().text == "Para one\n\nPara two" # Mix of single and double newlines in one TextItem. doc = DoclingDocument(name="lb") doc.add_text(label=DocItemLabel.TEXT, text="A\nB\n\nC\nD") assert MarkdownDocSerializer(doc=doc).serialize().text == "A \nB\n\nC \nD" # \n in a SectionHeaderItem -> space (headings cannot span multiple lines; # "Hello\nWorld" must become "### Hello World", not "### HelloWorld"). doc = DoclingDocument(name="lb") doc.add_heading(text="Hello\nWorld", level=2) assert MarkdownDocSerializer(doc=doc).serialize().text == "### Hello World" # \n in a ListItem -> GFM hard line break. doc = DoclingDocument(name="lb") lg = doc.add_group() doc.add_list_item(text="Line one\nLine two", parent=lg) assert MarkdownDocSerializer(doc=doc).serialize().text == "- Line one \nLine two" # \n in a plain table cell -> space (keeps the table row on a single line). doc = DoclingDocument(name="lb") table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) doc.add_table_cell( table_item=table, cell=TableCell( start_row_offset_idx=0, end_row_offset_idx=1, start_col_offset_idx=0, end_col_offset_idx=1, text="Line one\nLine two", ), ) result = MarkdownDocSerializer(doc=doc).serialize().text assert "\n" not in result.split("|")[1].strip() # =============================== # HTML tests # =============================== def test_html_table_serializer_get_header_and_body_lines(): """Test HTMLTableSerializer.get_header_and_body_lines() method.""" serializer = HTMLTableSerializer() # Test 1: Valid HTML with headers and data valid_html = "
| Header1 | Header2 |
|---|---|
| Data1 | Data2 |