Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/semchunk/semchunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ def chunk(
ilgs_doc = None

if is_first_call := not _recursion_depth:
# A non-positive chunk_size fits no token, so the recursion never terminates.
if chunk_size < 1:
raise ValueError(f"chunk_size must be a positive integer, not {chunk_size!r}.")

# Memoize the token counter if desired.
if memoize:
token_counter = _memoized_token_counters.setdefault(token_counter, lru_cache(cache_maxsize)(token_counter))
Expand Down
24 changes: 23 additions & 1 deletion tests/test_semchunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,5 +199,27 @@ def test_semchunk() -> None:
# Test chunking whitespace to ensure no errors are raised.
semchunk.chunk('\n\n', 512, lambda *args: 0)

def test_non_positive_chunk_size() -> None:
"""Test that a non-positive chunk size raises a clear ``ValueError`` rather than recursing until a ``RecursionError``."""
token_counter = len

for bad_size in (0, -1, -100):
# Directly via `chunk`.
try:
semchunk.chunk('hello world', bad_size, token_counter)
except ValueError:
pass
else:
raise AssertionError(f'chunk() with chunk_size={bad_size!r} did not raise ValueError')

# And via a `chunkerify`-built chunker, which defers to `chunk`.
try:
semchunk.chunkerify(token_counter, bad_size)('hello world')
except ValueError:
pass
else:
raise AssertionError(f'chunkerify() with chunk_size={bad_size!r} did not raise ValueError')

if __name__ == '__main__':
test_semchunk()
test_semchunk()
test_non_positive_chunk_size()