mirror of
https://github.com/docling-project/docling-core.git
synced 2026-11-18 20:27:00 +00:00
* chore: update pre-commit hooks and development dependencies Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * docs: update CONTRIBUTING after pre-commit hook changes Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
745 lines
26 KiB
Python
745 lines
26 KiB
Python
import pytest
|
|
from transformers import AutoTokenizer
|
|
|
|
from docling_core.transforms.chunker.line_chunker import LineBasedTokenChunker
|
|
from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
|
|
from docling_core.types.doc import DoclingDocument as DLDocument
|
|
from docling_core.types.doc.labels import DocItemLabel
|
|
|
|
from .test_utils import assert_or_generate_json_ground_truth
|
|
|
|
EMBED_MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2"
|
|
MAX_TOKENS = 25
|
|
INNER_TOKENIZER = AutoTokenizer.from_pretrained(EMBED_MODEL_ID)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def default_tokenizer():
|
|
"""Fixture providing a default HuggingFaceTokenizer for tests."""
|
|
return HuggingFaceTokenizer(
|
|
tokenizer=INNER_TOKENIZER,
|
|
max_tokens=MAX_TOKENS,
|
|
)
|
|
|
|
|
|
def test_chunk_text_with_prefix(default_tokenizer):
|
|
"""Test text chunking with a prefix."""
|
|
prefix = "Context: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
)
|
|
|
|
lines = ["Line 1\n", "Line 2\n", "Line 3"]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
assert isinstance(chunks, list)
|
|
assert len(chunks) > 0
|
|
# Each chunk should start with the prefix
|
|
for chunk in chunks:
|
|
assert isinstance(chunk, str)
|
|
assert chunk.startswith(prefix)
|
|
|
|
|
|
def test_chunk_text_long_prefix_warning(default_tokenizer):
|
|
"""Test that a warning is issued when prefix is too long."""
|
|
# Create a very long prefix that exceeds max_tokens
|
|
long_prefix = "This is a very long prefix " * 50
|
|
|
|
with pytest.warns(UserWarning, match="too long.*will be split into multiple chunks"):
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=long_prefix,
|
|
)
|
|
|
|
# Prefix should be kept but split into chunks
|
|
assert chunker.prefix == long_prefix
|
|
assert chunker.prefix_len == 0 # Returns 0 when prefix is too large
|
|
assert len(chunker.prefix_chunks) > 1 # Should be split into multiple chunks
|
|
|
|
# Test that chunking works with large prefix
|
|
lines = ["Line 1", "Line 2", "Line 3"]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# First chunk(s) should contain the prefix chunks
|
|
assert len(chunks) > 0
|
|
# The prefix chunks should be at the beginning
|
|
for i, prefix_chunk in enumerate(chunker.prefix_chunks):
|
|
if i < len(chunks):
|
|
assert prefix_chunk in chunks[i]
|
|
|
|
|
|
def test_chunk_text_single_long_line(default_tokenizer):
|
|
"""Test chunking when a single line exceeds max_tokens."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# Create a very long line
|
|
long_line = "word " * MAX_TOKENS * 5
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
assert len(chunks) > 1
|
|
# Verify each chunk respects token limit
|
|
for chunk in chunks:
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS
|
|
|
|
|
|
def test_chunk_text_empty_string(default_tokenizer):
|
|
"""Test chunking an empty list."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
chunks = chunker.chunk_text([])
|
|
assert len(chunks) == 0
|
|
|
|
|
|
def test_chunk_text_single_line(default_tokenizer):
|
|
"""Test chunking a single line that fits in one chunk."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
text = "This is a single short line.\n"
|
|
lines = [text]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
assert len(chunks) == 1
|
|
assert chunks[0] == text
|
|
# newline should be preserved
|
|
assert "\n" in chunks[0]
|
|
|
|
|
|
def test_split_by_token_limit(default_tokenizer):
|
|
"""Test the split_by_token_limit method."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
available = 10
|
|
text = "This is a test sentence with multiple words that should be split."
|
|
head, tail = chunker.split_by_token_limit(text, token_limit=available)
|
|
|
|
assert len(head) > 0
|
|
assert len(tail) > 0
|
|
assert chunker.tokenizer.count_tokens(head) <= available
|
|
assert head + tail == text
|
|
|
|
|
|
def test_split_by_token_limit_zero_limit(default_tokenizer):
|
|
"""Test split_by_token_limit with zero token limit."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
text = "Some text"
|
|
head, tail = chunker.split_by_token_limit(text, token_limit=0)
|
|
|
|
assert head == ""
|
|
assert tail == text
|
|
|
|
|
|
def test_split_by_token_limit_fits_entirely(default_tokenizer):
|
|
"""Test split_by_token_limit when text fits within limit."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
text = "Short text"
|
|
head, tail = chunker.split_by_token_limit(text, token_limit=100)
|
|
|
|
assert head == text
|
|
assert tail == ""
|
|
|
|
|
|
def test_split_by_token_limit_word_boundary(default_tokenizer):
|
|
"""Test that split_by_token_limit prefers word boundaries."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
text = "word1 word2 word3 word4 word5"
|
|
head, tail = chunker.split_by_token_limit(text, token_limit=5, prefer_word_boundary=True)
|
|
|
|
# Head should end at a word boundary (space)
|
|
if len(head) > 0 and len(tail) > 0:
|
|
# Either head ends with a space or tail starts with a space
|
|
assert head[-1].isspace() or tail[0].isspace() or not head[-1].isalnum()
|
|
|
|
|
|
def test_chunk_text_with_prefix_and_long_lines(default_tokenizer):
|
|
"""Test chunking with prefix when lines are long."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
)
|
|
|
|
long_line = "This is a long line that will need to be split " * 3
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
assert len(chunks) > 0
|
|
for chunk in chunks:
|
|
assert chunk.startswith(prefix)
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS
|
|
|
|
|
|
def test_chunk_document(default_tokenizer):
|
|
"""Test the chunk() method with a DoclingDocument."""
|
|
# Create a simple DoclingDocument
|
|
doc = DLDocument(name="test_doc")
|
|
paragraphs = [
|
|
"This is the first paragraph with some content.",
|
|
"This is the second paragraph with more content",
|
|
"This is the third paragraph with even more content.",
|
|
]
|
|
|
|
# Add some text items to the document
|
|
for t in paragraphs:
|
|
doc.add_text(label=DocItemLabel.PARAGRAPH, text=t)
|
|
|
|
# Create chunker
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# Chunk the document
|
|
chunks = list(chunker.chunk(doc))
|
|
|
|
# Verify chunks were created
|
|
assert len(chunks) > 0
|
|
|
|
# Verify each chunk is a DocChunk with proper structure
|
|
for chunk in chunks:
|
|
assert hasattr(chunk, "text")
|
|
assert hasattr(chunk, "meta")
|
|
assert isinstance(chunk.text, str)
|
|
assert len(chunk.text) > 0
|
|
|
|
# Verify token count is within limit
|
|
token_count = chunker.tokenizer.count_tokens(chunk.text)
|
|
assert token_count <= MAX_TOKENS
|
|
|
|
# Verify each paragraph resides fully in a chunk
|
|
for t in paragraphs:
|
|
assert any(t in c.text for c in chunks)
|
|
|
|
|
|
def test_chunk_empty_document(default_tokenizer):
|
|
"""Test the chunk() method with an empty document."""
|
|
# Create an empty DoclingDocument
|
|
doc = DLDocument(name="empty_doc")
|
|
|
|
# Create chunker
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# Chunk the document
|
|
chunks = list(chunker.chunk(doc))
|
|
|
|
# Should return no chunks for empty document
|
|
assert len(chunks) == 0
|
|
|
|
|
|
def test_chunk_document_with_long_content(default_tokenizer):
|
|
"""Test the chunk() method with long content that requires multiple chunks."""
|
|
# Create a DoclingDocument with long content
|
|
doc = DLDocument(name="long_doc")
|
|
prefix = "Document: "
|
|
|
|
# Add a very long paragraph
|
|
long_text = "This is a sentence with multiple words. " * 50
|
|
doc.add_text(label=DocItemLabel.PARAGRAPH, text=long_text)
|
|
|
|
chunker = LineBasedTokenChunker(tokenizer=default_tokenizer, prefix=prefix)
|
|
|
|
# Chunk the document
|
|
chunks = list(chunker.chunk(doc))
|
|
|
|
# Should create multiple chunks
|
|
assert len(chunks) > 1
|
|
|
|
# Verify each chunk respects token limit
|
|
for chunk in chunks:
|
|
assert chunk.text.startswith(prefix)
|
|
token_count = chunker.tokenizer.count_tokens(chunk.text)
|
|
assert token_count <= MAX_TOKENS
|
|
|
|
|
|
def test_infinite_loop_regression_long_unbreakable_token(default_tokenizer):
|
|
"""Regression test for infinite loop bug when processing text with a long
|
|
unbreakable token sequence preceded by a space.
|
|
|
|
This test reproduces the issue where LineBasedTokenChunker.chunk_text()
|
|
would enter an infinite loop when the prefer_word_boundary logic in
|
|
split_by_token_limit() snapped best_idx back to 0, producing an empty
|
|
head and returning the tail unchanged.
|
|
|
|
The fix ensures:
|
|
1. split_by_token_limit only snaps to word boundary if it produces non-empty head
|
|
2. chunk_text detects zero-progress and forces character-level splitting as fallback
|
|
"""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# Create text with leading space followed by long unbreakable token
|
|
long_word = "a" * 200
|
|
text = "Header " + long_word + " Footer\n"
|
|
token_count = chunker.tokenizer.count_tokens(text)
|
|
assert token_count <= MAX_TOKENS
|
|
|
|
# This should complete without hanging
|
|
result = chunker.chunk_text(lines=[text])
|
|
|
|
assert len(result) == 1
|
|
assert result[0] == text
|
|
|
|
|
|
def test_split_by_token_limit_leading_space_regression(default_tokenizer):
|
|
"""Test that split_by_token_limit handles text with leading space correctly.
|
|
|
|
Previously, when text started with a space followed by unbreakable content,
|
|
the word boundary snap-back could produce an empty head, causing infinite loops.
|
|
"""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# Text with leading space then long unbreakable sequence
|
|
text = " " + "a" * 200
|
|
|
|
# Split with small token limit
|
|
head, tail = chunker.split_by_token_limit(text, token_limit=10, prefer_word_boundary=True)
|
|
|
|
# Head should not be empty (must make progress)
|
|
assert len(head) > 0, "split_by_token_limit must make progress and return non-empty head"
|
|
|
|
# Tail should be shorter than original
|
|
assert len(tail) < len(text), "split_by_token_limit must consume some input"
|
|
|
|
# Verify head + tail reconstructs original
|
|
assert head + tail == text
|
|
|
|
# Verify head respects token limit
|
|
head_tokens = chunker.tokenizer.count_tokens(head)
|
|
assert head_tokens <= 10
|
|
|
|
|
|
def test_character_level_fallback_on_zero_available(default_tokenizer):
|
|
"""Test that chunk_text uses character-level fallback when available space is 0.
|
|
|
|
This test demonstrates a real scenario where the fallback is needed:
|
|
1. Current chunk is exactly at max_tokens (available = 0)
|
|
2. Remaining text is too long to fit in a fresh chunk (exceeds max_tokens)
|
|
3. split_by_token_limit is called with token_limit=0
|
|
4. It returns ("", text) in split_by_token_limit of line_chunker.py
|
|
5. The fallback takes 1 character to ensure progress
|
|
"""
|
|
# Use a very small max_tokens to make it easier to create the scenario
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
)
|
|
|
|
# First line: 8 tokens (leaves room for 2 more)
|
|
first_line = "word " * (MAX_TOKENS - 2)
|
|
|
|
# Second line: more than 10 tokens (so it can't fit in a fresh chunk)
|
|
second_line = "x" * 100
|
|
|
|
# Verify second line is indeed > max_tokens
|
|
second_line_tokens = chunker.tokenizer.count_tokens(second_line)
|
|
assert second_line_tokens > MAX_TOKENS, (
|
|
f"Second line must exceed max_tokens for test to work: {second_line_tokens} <= {MAX_TOKENS}"
|
|
)
|
|
|
|
lines = [first_line, second_line]
|
|
result = chunker.chunk_text(lines=lines)
|
|
|
|
# Verify we got multiple chunks (the long second line should be split)
|
|
assert len(result) > 1, f"Should have multiple chunks, got {len(result)}"
|
|
|
|
# Verify each chunk respects token limit (allow small overflow due to newline addition)
|
|
for i, chunk in enumerate(result):
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS, f"Chunk {i} exceeds token limit: {token_count} > {MAX_TOKENS}"
|
|
|
|
# Verify all content is preserved
|
|
combined = "".join(result)
|
|
# First line should be in first chunk
|
|
assert first_line in result[0], "First line should be in first chunk"
|
|
# Second line should be split across remaining chunks
|
|
combined_without_newlines = combined.replace("\n", "")
|
|
assert second_line in combined_without_newlines, "Second line should be preserved (possibly split)"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_false(default_tokenizer):
|
|
"""Test default behavior when omit_prefix_on_overflow is False (default)."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=False,
|
|
)
|
|
|
|
# Create a line that fits without prefix but not with prefix
|
|
# prefix_len is around 2 tokens, create a line that's just 1 token short
|
|
line = "word " * (MAX_TOKENS - 1)
|
|
line_tokens = chunker.tokenizer.count_tokens(line)
|
|
|
|
# Verify test setup: line should fit alone but not with prefix
|
|
assert line_tokens <= MAX_TOKENS, f"Line should fit in chunk: {line_tokens} <= {MAX_TOKENS}"
|
|
assert line_tokens + chunker.prefix_len > MAX_TOKENS, (
|
|
f"Line with prefix should NOT fit: {line_tokens} + {chunker.prefix_len} > {MAX_TOKENS}"
|
|
)
|
|
|
|
lines = [line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# With omit_prefix_on_overflow=False, the line should be split
|
|
# and each chunk should have the prefix
|
|
assert len(chunks) > 1, "Line should be split into multiple chunks"
|
|
for chunk in chunks:
|
|
assert chunk.startswith(prefix), "Each chunk should start with prefix"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_true(default_tokenizer):
|
|
"""Test that prefix is added as standalone chunk when first line would overflow with it."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create a line that fits without prefix but not with prefix
|
|
line = "word " * (MAX_TOKENS - 1)
|
|
line_tokens = chunker.tokenizer.count_tokens(line)
|
|
|
|
# Verify our test setup
|
|
assert line_tokens <= MAX_TOKENS, f"Line should fit in chunk: {line_tokens} <= {MAX_TOKENS}"
|
|
assert line_tokens + chunker.prefix_len > MAX_TOKENS, (
|
|
f"Line with prefix should NOT fit: {line_tokens} + {chunker.prefix_len} > {MAX_TOKENS}"
|
|
)
|
|
|
|
lines = [line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# With omit_prefix_on_overflow=True and first line overflowing:
|
|
# - First chunk should be the prefix alone (to ensure it's visible)
|
|
# - Second chunk should be the line content without prefix
|
|
assert len(chunks) == 2, "Should have 2 chunks: prefix chunk + content chunk"
|
|
assert chunks[0] == prefix, "First chunk should be the prefix alone"
|
|
assert not chunks[1].startswith(prefix), "Second chunk should NOT start with prefix"
|
|
assert line.strip() in chunks[1], "Line content should be in second chunk"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_line_too_large(default_tokenizer):
|
|
"""Test that line is still split when it exceeds max_tokens, regardless of omit_prefix_on_overflow."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create a line that's larger than max_tokens even without prefix
|
|
long_line = "word " * (MAX_TOKENS * 2)
|
|
line_tokens = chunker.tokenizer.count_tokens(long_line)
|
|
|
|
# Verify line is too large
|
|
assert line_tokens > MAX_TOKENS, f"Line should exceed max_tokens: {line_tokens} > {MAX_TOKENS}"
|
|
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Line should be split regardless of omit_prefix_on_overflow
|
|
assert len(chunks) > 1, "Line should be split when it exceeds max_tokens"
|
|
|
|
# Verify each chunk respects token limit
|
|
for chunk in chunks:
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS, f"Chunk should respect token limit: {token_count} <= {MAX_TOKENS}"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_multiple_lines(default_tokenizer):
|
|
"""Test omit_prefix_on_overflow with multiple lines of varying sizes."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Line 1: Small, fits with prefix
|
|
line1 = "Small line\n"
|
|
# Line 2: Medium, fits without prefix but not with prefix
|
|
line2 = "word " * (MAX_TOKENS - 1) + "\n"
|
|
# Line 3: Small again, fits with prefix
|
|
line3 = "Another small line\n"
|
|
|
|
lines = [line1, line2, line3]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Verify we have chunks
|
|
assert len(chunks) > 0
|
|
|
|
# First chunk should have prefix and line1
|
|
assert chunks[0].startswith(prefix)
|
|
assert line1.strip() in chunks[0]
|
|
|
|
# line2 should be in a chunk without prefix (due to overflow)
|
|
line2_found = False
|
|
for chunk in chunks:
|
|
if line2.strip() in chunk:
|
|
line2_found = True
|
|
# This chunk should NOT start with prefix
|
|
assert not chunk.startswith(prefix), "Line2 chunk should not have prefix due to overflow"
|
|
break
|
|
assert line2_found, "Line2 should be found in chunks"
|
|
|
|
# line3 should be in a chunk with prefix (it's small enough)
|
|
line3_found = False
|
|
for chunk in chunks:
|
|
if line3.strip() in chunk:
|
|
line3_found = True
|
|
# This chunk should start with prefix
|
|
assert chunk.startswith(prefix), "Line3 chunk should have prefix"
|
|
break
|
|
assert line3_found, "Line3 should be found in chunks"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_no_prefix(default_tokenizer):
|
|
"""Test that omit_prefix_on_overflow has no effect when there's no prefix."""
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix="",
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
line = "word " * (MAX_TOKENS - 1)
|
|
lines = [line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Should work normally without prefix
|
|
assert len(chunks) == 1
|
|
assert line in chunks[0]
|
|
|
|
|
|
def test_omit_prefix_on_overflow_with_line_splitting(default_tokenizer):
|
|
"""Test that overflow chunks from split lines don't have prefix when omit_prefix_on_overflow=True."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create a line that's too long even without prefix (will be split)
|
|
long_line = "word " * (MAX_TOKENS * 2)
|
|
line_tokens = chunker.tokenizer.count_tokens(long_line)
|
|
|
|
# Verify line is too large even without prefix
|
|
assert line_tokens > MAX_TOKENS, f"Line should exceed max_tokens: {line_tokens} > {MAX_TOKENS}"
|
|
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Line should be split into multiple chunks
|
|
assert len(chunks) > 1, "Line should be split when it exceeds max_tokens"
|
|
|
|
# With omit_prefix_on_overflow=True:
|
|
# - First chunk may have prefix (if it was already in current when line started)
|
|
# - Subsequent overflow chunks should NOT have the prefix
|
|
for i, chunk in enumerate(chunks):
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS, f"Chunk {i} should respect token limit: {token_count} <= {MAX_TOKENS}"
|
|
|
|
# At least the overflow chunks (after the first) should not have prefix
|
|
if len(chunks) > 1:
|
|
for i in range(1, len(chunks)):
|
|
assert not chunks[i].startswith(prefix), (
|
|
f"Overflow chunk {i} should NOT have prefix with omit_prefix_on_overflow=True"
|
|
)
|
|
|
|
|
|
def test_omit_prefix_on_overflow_false_with_line_splitting(default_tokenizer):
|
|
"""Test that overflow chunks from split lines DO have prefix when omit_prefix_on_overflow=False."""
|
|
prefix = "PREFIX: "
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=False, # Default behavior
|
|
)
|
|
|
|
# Create a line that's too long even without prefix (will be split)
|
|
long_line = "word " * (MAX_TOKENS * 2)
|
|
line_tokens = chunker.tokenizer.count_tokens(long_line)
|
|
|
|
# Verify line is too large even without prefix
|
|
assert line_tokens > MAX_TOKENS, f"Line should exceed max_tokens: {line_tokens} > {MAX_TOKENS}"
|
|
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Line should be split into multiple chunks
|
|
assert len(chunks) > 1, "Line should be split when it exceeds max_tokens"
|
|
|
|
# With omit_prefix_on_overflow=False, all chunks should have the prefix
|
|
for i, chunk in enumerate(chunks):
|
|
token_count = chunker.tokenizer.count_tokens(chunk)
|
|
assert token_count <= MAX_TOKENS, f"Chunk {i} should respect token limit: {token_count} <= {MAX_TOKENS}"
|
|
|
|
# All chunks should have the prefix when omit_prefix_on_overflow=False
|
|
assert chunk.startswith(prefix), f"Chunk {i} should have prefix with omit_prefix_on_overflow=False"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_warning(default_tokenizer):
|
|
"""Test that a warning is issued once when prefix is actually omitted."""
|
|
prefix = "PREFIX: "
|
|
|
|
# Create chunker with omit_prefix_on_overflow=True
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create a line that fits without prefix but not with prefix
|
|
line = "word " * (MAX_TOKENS - 1)
|
|
line_tokens = chunker.tokenizer.count_tokens(line)
|
|
|
|
# Verify test setup
|
|
assert line_tokens <= MAX_TOKENS
|
|
assert line_tokens + chunker.prefix_len > MAX_TOKENS
|
|
|
|
# Should warn once when prefix is omitted
|
|
with pytest.warns(UserWarning, match="Prefix omitted for at least one line"):
|
|
lines = [line, line, line] # Multiple lines that would cause omission
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Verify chunks were created
|
|
assert len(chunks) > 0
|
|
|
|
|
|
def test_omit_prefix_on_overflow_no_warning_when_not_omitted(default_tokenizer):
|
|
"""Test that no warning is issued when prefix is never omitted."""
|
|
prefix = "PREFIX: "
|
|
|
|
# Create chunker with omit_prefix_on_overflow=True
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create lines that fit with prefix (no omission needed)
|
|
small_line = "Small line\n"
|
|
|
|
# Should NOT warn since prefix is never omitted
|
|
import warnings as warnings_module
|
|
|
|
with warnings_module.catch_warnings(record=True) as warning_list:
|
|
warnings_module.simplefilter("always")
|
|
lines = [small_line, small_line, small_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Filter for UserWarnings about prefix omission
|
|
prefix_warnings = [w for w in warning_list if "Prefix omitted" in str(w.message)]
|
|
|
|
# Verify no prefix omission warnings were issued
|
|
assert len(prefix_warnings) == 0
|
|
|
|
# Verify chunks were created with prefix
|
|
assert len(chunks) > 0
|
|
for chunk in chunks:
|
|
assert chunk.startswith(prefix)
|
|
|
|
|
|
def test_omit_prefix_on_overflow_warning_on_split_line(default_tokenizer):
|
|
"""Test that warning is issued when prefix is omitted for overflow chunks from split lines."""
|
|
prefix = "PREFIX: "
|
|
|
|
# Create chunker with omit_prefix_on_overflow=True
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
# Create a line that's too long even without prefix (will be split)
|
|
long_line = "word " * (MAX_TOKENS * 2)
|
|
line_tokens = chunker.tokenizer.count_tokens(long_line)
|
|
|
|
# Verify line is too large even without prefix
|
|
assert line_tokens > MAX_TOKENS
|
|
|
|
# Should warn once when prefix is omitted for overflow chunks
|
|
with pytest.warns(UserWarning, match="Prefix omitted for at least one line"):
|
|
lines = [long_line]
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
# Verify line was split into multiple chunks
|
|
assert len(chunks) > 1
|
|
|
|
# With omit_prefix_on_overflow=True, overflow chunks should NOT have prefix
|
|
# (first chunk may or may not have prefix depending on initial state)
|
|
for i in range(1, len(chunks)):
|
|
assert not chunks[i].startswith(prefix), f"Overflow chunk {i} should not have prefix"
|
|
|
|
|
|
def test_omit_prefix_on_overflow_all_lines_overflow(default_tokenizer):
|
|
"""Test edge case where omit_prefix_on_overflow=True and ALL lines overflow with prefix."""
|
|
prefix = "HEADER: Column1 | Column2 | Column3\n"
|
|
|
|
chunker = LineBasedTokenChunker(
|
|
tokenizer=default_tokenizer,
|
|
prefix=prefix,
|
|
omit_prefix_on_overflow=True,
|
|
)
|
|
|
|
prefix_tokens = chunker.tokenizer.count_tokens(prefix)
|
|
assert prefix_tokens < MAX_TOKENS, f"Prefix should fit in chunk: {prefix_tokens} < {MAX_TOKENS}"
|
|
assert chunker.prefix_len > 0, "prefix_len should be > 0 when prefix fits"
|
|
|
|
# Each line should be close to max_tokens so that prefix + line > max_tokens
|
|
line1 = "word " * (MAX_TOKENS - chunker.prefix_len + 1) + "\n"
|
|
line2 = "data " * (MAX_TOKENS - chunker.prefix_len + 1) + "\n"
|
|
line3 = "text " * (MAX_TOKENS - chunker.prefix_len + 1) + "\n"
|
|
|
|
for i, line in enumerate([line1, line2, line3], 1):
|
|
line_tokens = chunker.tokenizer.count_tokens(line)
|
|
assert line_tokens <= MAX_TOKENS, f"Line {i} should fit alone: {line_tokens} <= {MAX_TOKENS}"
|
|
assert line_tokens + chunker.prefix_len > MAX_TOKENS, (
|
|
f"Line {i} with prefix should overflow: {line_tokens} + {chunker.prefix_len} > {MAX_TOKENS}"
|
|
)
|
|
|
|
lines = [line1, line2, line3]
|
|
|
|
# Chunk the text
|
|
with pytest.warns(UserWarning, match="Prefix omitted for at least one line"):
|
|
chunks = chunker.chunk_text(lines)
|
|
|
|
assert len(chunks) > 0, "Should create at least one chunk"
|
|
|
|
# Check if prefix appears in any chunk
|
|
prefix_found = any(prefix.strip() in chunk for chunk in chunks)
|
|
|
|
# This assertion will FAIL with the current implementation, demonstrating the bug
|
|
assert prefix_found, (
|
|
"Prefix should appear in at least one chunk even when all lines omit it. "
|
|
"This ensures headers/context are visible. "
|
|
f"Chunks: {chunks}"
|
|
)
|
|
|
|
# Additionally verify that content lines are present (without prefix)
|
|
for line in lines:
|
|
line_found = any(line.strip() in chunk for chunk in chunks)
|
|
assert line_found, f"Line content should be preserved: {line.strip()}"
|