diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 182ff655..b54c000b 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -21,8 +21,11 @@ code_only() { -e 's,^([0-9]*:?\+[[:space:]]*)(\*|/\*).*$,\1,' } -# Decorative comment separators - runs of three or more `-` or `=` used purely as visual dividers inside -# C/C++ comments. Proper sentences communicate structure better than attention-grabbing rules. +# Decorative separators - runs of three or more `-` or `=` used purely as visual dividers, whether in a +# C/C++ comment or printed at runtime as `printf("\n=== Section ===\n")`. Proper sentences communicate +# structure better than attention-grabbing rules, and a banner a test prints is the same habit as a banner +# a comment draws. The literal arm requires the run to sit against whitespace or an escaped newline, so a +# rule used as a divider is caught while `"---"` or `"a---b"` as test DATA is not. mapfile -t staged < <(git diff --cached --name-only --diff-filter=ACM \ -- '*.c' '*.h' '*.hpp' '*.cpp' '*.cc' '*.cxx' '*.cu' '*.cuh' '*.inl') separators=0 @@ -30,9 +33,9 @@ for file in "${staged[@]}"; do [ -f "$file" ] || continue while IFS= read -r line; do added="${line#+}" - if printf '%s\n' "$added" | grep -qE '(//|/\*|^\s*\*).*[-=]{3,}'; then + if printf '%s\n' "$added" | grep -qE '(//|/\*|^\s*\*).*[-=]{3,}|"[^"]*([[:space:]]|\\n)[-=]{3,}|"[^"]*[-=]{3,}([[:space:]]|\\n)'; then if [ "$separators" -eq 0 ]; then - printf 'Decorative comment separators are not allowed (--- / ===):\n\n' >&2 + printf 'Decorative separators are not allowed in comments or printed banners (--- / ===):\n\n' >&2 fi printf ' %s: %s\n' "$file" "$(printf '%s' "$added" | sed 's/^[[:space:]]*//')" >&2 separators=$((separators + 1)) @@ -44,6 +47,27 @@ if [ "$separators" -ne 0 ]; then exit 1 fi +# Doxygen section anchors. `@section ` binds the FIRST token as the link target and the rest as +# the visible title, so `@section Device Memory` silently anchors "Device" and titles the section "Memory" - +# the heading reads wrong and nothing can link to it. House convention is a lowercase snake_case id carrying +# its domain, since ids share one namespace project-wide and two headers both document "Buffer Sizing": +# `utf8_norm_buffer_sizing`, not `buffer_sizing`. A Title-Case first token is the exact signature of a +# missing id, and an id with no title renders an empty heading, so both arms are required. Reads raw lines +# rather than `code_only` ones, since these live inside comments by construction. +doxygen_sections=$(git diff --cached --no-color -U0 -- '*.c' '*.h' '*.hpp' '*.cpp' '*.cc' '*.cxx' '*.cu' '*.cuh' '*.inl' \ + | grep -nE '^\+' | grep -vE '^\+\+\+' \ + | grep -E '@(section|subsection|subsubsection|page)[[:space:]]' \ + | grep -vE '@(section|subsection|subsubsection|page)[[:space:]]+[a-z][a-z0-9_]*[[:space:]]+[^[:space:]]' || true) +if [ -n "$doxygen_sections" ]; then + cat >&2 <<EOF +pre-commit: Doxygen section missing its lowercase snake_case id, or carrying an id with no title. Doxygen +anchors the first token, so '@section Device Memory' anchors "Device". Write '@section domain_topic Title': +$doxygen_sections +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + # The `&*iterator` antipattern (iterator -> reference -> pointer laundering). Use the container's # `.data()`, `std::to_address(it)`, or a raw pointer directly instead. laundering=$(git diff --cached --no-color -U0 -- '*.c' '*.h' '*.cc' '*.hpp' '*.cuh' '*.cu' \ @@ -105,6 +129,42 @@ EOF exit 1 fi +# clang-format's `BreakStringLiterals` splits an overlong string literal in place - two adjacent +# fragments left on one line, continuations re-indented under them. A hand-broken literal never leaves +# two fragments on the same line, so same-line adjacency is the mangle's exact signature. The escape +# hatch is a trailing `//` comment pinning the split as deliberate, as when stopping `"\x9F" "e"` from +# parsing as one escape - clang-format never re-joins a line that ends in a comment. This guard reads +# raw lines, not `code_only` ones, precisely so it can see that pin. +mangled_literals=$(git diff --cached --no-color -U0 -- '*.c' '*.h' '*.cc' '*.hpp' '*.cuh' '*.cu' \ + | grep -nE '^\+' | grep -vE '^\+\+\+' \ + | grep -E '"([^"\\]|\\.)*"[[:space:]]+"' \ + | grep -v '//' || true) +if [ -n "$mangled_literals" ]; then + cat >&2 <<EOF +pre-commit: two string-literal fragments on one line - clang-format's BreakStringLiterals mangle +signature. Hand-break the literal onto per-fragment lines, or pin a deliberate same-line split with a +trailing // comment naming why: +$mangled_literals +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + +# Indexing straight through a call - `owner.view()[index]`, `data()[index]`, a cast-then-index chain. +# The owning containers carry their own `operator[]`, and anything else deserves a named local: the call +# names WHAT is indexed, the local names what it MEANS. The `)[` adjacency is the whole signature. +call_indexing=$(git diff --cached --no-color -U0 -- '*.c' '*.h' '*.cc' '*.hpp' '*.cuh' '*.cu' \ + | grep -nE '^\+' | code_only | grep -E '\)\[' || true) +if [ -n "$call_indexing" ]; then + cat >&2 <<EOF +pre-commit: ')[' chain in staged changes - indexing through a call or cast. Index the owning container +directly, or bind the callee's result to a named local first: +$call_indexing +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + # LibC string/memory symbols under include/stringzilla/**: StringZilla supersedes LibC and cannot share a # translation unit with the compiler `__builtin_*` it emits. Use the project wrappers (`sz_copy`, `sz_fill`, # `sz_move`, `sz_equal`) or a `sz_u(128|256|512)_vec_t` union `.u8s[]` buffer for staging. The default allocator @@ -124,6 +184,22 @@ EOF exit 1 fi +# Iostreams are banned house-wide: <iostream>/<fstream>/<sstream> and their std::cout/cerr/ifstream/ +# ofstream/stringstream/endl symbols drag in the virtual-dispatch streambuf machinery. Use LibC <cstdio>: +# std::fopen/std::fread/std::fwrite/std::fprintf/std::snprintf, as read_file and log_failure already do. +iostreams=$(git diff --cached --no-color -U0 -- '*.c' '*.h' '*.hpp' '*.cpp' '*.cc' '*.cxx' '*.cu' '*.cuh' '*.inl' \ + | grep -nE '^\+' | grep -vE '^\+\+\+' | code_only \ + | grep -E '#include[[:space:]]*<(iostream|fstream|sstream)>|std::(cout|cerr|cin|ifstream|ofstream|fstream|stringstream|ostringstream|istringstream|endl)\b' || true) +if [ -n "$iostreams" ]; then + cat >&2 <<EOF +pre-commit: iostream/fstream/sstream in staged changes - the streambuf machinery is banned house-wide. +Use LibC <cstdio>: std::fopen / std::fread / std::fwrite / std::fprintf / std::snprintf (see read_file): +$iostreams +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + # GitHub Actions job & step names: no parentheses, and the visible name starts with an uppercase letter # (or a `${{ â€Ļ }}` expression). Checks job-level `name:` keys (4-space indent in our 2-space YAML style) # and step-level `- name:` items on ADDED lines only, so `with:` parameters like artifact names are @@ -236,21 +312,61 @@ fi # Raw libc assert() in test/ is silently stripped by the -DNDEBUG that CMake's Release and # RelWithDebInfo configs add by default - exactly the configs CI builds the test binaries with. -# The test suite's oracle must never be a no-op: use verify() / let_verify() / scope_verify() / -# throws_verify() from test/stringzilla.hpp instead, which are never gated by NDEBUG or SZ_DEBUG. +# `sz_assert_` is the same trap wearing the project's own name: `CMakeLists.txt` defines SZ_DEBUG=0 for +# every config but Debug, and types.h then expands it to `((void)(condition))`, so the oracle evaluates +# its argument and throws the verdict away. The test suite's oracle must never be a no-op: use verify() / +# let_verify() / scope_verify() / throws_verify() from test/stringzilla.hpp, which no config gates. raw_assert=$(git diff --cached --no-color -U0 -- 'test/*.c' 'test/*.h' 'test/*.cpp' 'test/*.hpp' 'test/*.cuh' 'test/*.cu' \ | grep -E '^\+' | grep -vE '^\+\+\+' | code_only \ - | grep -E '(^|[^A-Za-z0-9_])assert[[:space:]]*\(' || true) + | grep -E '(^|[^A-Za-z0-9_])(assert|sz_assert_)[[:space:]]*\(' || true) if [ -n "$raw_assert" ]; then cat >&2 <<EOF -pre-commit: raw assert() in staged test/ changes (stripped by -DNDEBUG in CI's Release/RelWithDebInfo -test builds). Use verify() / let_verify() / scope_verify() / throws_verify() instead: +pre-commit: assert() or sz_assert_() in staged test/ changes - both are compiled out of CI's +Release/RelWithDebInfo test builds. Use verify() / let_verify() / scope_verify() / throws_verify(): $raw_assert Bypass with --no-verify only if intentional. EOF exit 1 fi +# A coverage assertion over a randomized sweep: `saw_x |= ...` inside a loop, then a `verify(saw_x)` that +# the loop reached that case at all. `SZ_TESTS_MULTIPLIER` scales fuzz loops down - the QEMU jobs run at +# 0.05, turning a scale_iterations(80) sweep into 4 draws - so the assertion stops being a property of the +# code and becomes a property of the seed. Sweep the fixed pool deterministically instead, or, when the +# trip count really is fixed and the coverage claim really is exact, pin it with `// sz-nolint: coverage`. +# Reads raw lines rather than `code_only` ones, precisely so it can see that pin. +coverage_assert=$(git diff --cached --no-color -U0 -- 'test/*.cpp' 'test/*.hpp' 'test/*.cuh' 'test/*.cu' \ + | grep -E '^\+' | grep -vE '^\+\+\+' \ + | grep -E 'verify\([[:space:]]*(saw|seen|observed)_' \ + | grep -v 'sz-nolint: coverage' || true) +if [ -n "$coverage_assert" ]; then + cat >&2 <<EOF +pre-commit: coverage assertion over a randomized sweep - a low SZ_TESTS_MULTIPLIER makes this a coin +flip rather than a test. Walk the pool deterministically, or pin with '// sz-nolint: coverage': +$coverage_assert +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + +# Export macros on binding-internal definitions. Everything under python/ is one CPython extension module: +# its cross-TU helpers are declared plain `extern` in the module's own header, so defining them with +# SZ_API_RUNTIME / SZ_PUBLIC / SZ_DYNAMIC contradicts that declaration. GCC and Clang shrug; MSVC, where +# SZ_API_RUNTIME is __declspec(dllexport), rejects the pair with "error C2375: redefinition; different +# linkage" and takes both Windows jobs down. Nothing in a binding needs to be exported from the module. +binding_exports=$(git diff --cached --no-color -U0 -- 'python/*.c' 'python/*/*.c' 'javascript/*.c' \ + | grep -E '^\+' | grep -vE '^\+\+\+' | code_only \ + | grep -E '^\+[[:space:]]*(SZ_API_RUNTIME|SZ_PUBLIC|SZ_DYNAMIC)[[:space:]]' || true) +if [ -n "$binding_exports" ]; then + cat >&2 <<EOF +pre-commit: export macro on a binding-internal definition (MSVC C2375 against the plain 'extern' +declaration in the binding's own header). Drop the macro - the module exports only its init function: +$binding_exports +Bypass with --no-verify only if intentional. +EOF + exit 1 +fi + # SIMD register naming across every ISA backend (cipher/hash/find/compare/intersect/memory/sort/utf8_*) # plus types.h, where the union wrapper types themselves live: every raw fixed-width register (x86 # __m128i/__m256i/__m512i, NEON uint8x16_t etc, WASM v128_t, Power __vector unsigned char/int/long long, diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index f3c0dcba..0a76974a 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -183,7 +183,7 @@ jobs: print("stringzilla.__capabilities__:", sz.__capabilities__) PY - name: Test Python - run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/substrings.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace # JavaScript - name: Set up Node.js @@ -313,7 +313,7 @@ jobs: print("stringzilla.__capabilities__:", sz.__capabilities__) PY - name: Test Python - run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/substrings.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace # Rust - name: Set up Rust @@ -387,6 +387,10 @@ jobs: } - name: Test C++ run: build_artifacts/stringzilla_test_cpp20 + - name: Test C++11 language floor + run: build_artifacts/stringzilla_test_cpp11 + - name: Test C++ parallel engines + run: build_artifacts/stringzillas_test_cpp20 - name: Check stringzilla_bare is built correctly run: test -z "$(ldd build_artifacts/libstringzilla_bare.so | grep '.so')" @@ -401,6 +405,8 @@ jobs: python -m pip install --upgrade pip pip install "setuptools>=64" wheel pip install pytest pytest-repeat numpy pyarrow affine-gaps + # The third-party oracles, from the one place they are declared rather than a second list here. + pip install --group tests-oracles SZ_TARGET=stringzilla pip install -e . --force-reinstall --no-build-isolation SZ_TARGET=stringzillas-cpus pip install -e . --force-reinstall --no-build-isolation --no-deps - name: Log Python capabilities @@ -415,7 +421,7 @@ jobs: print("stringzillas.__capabilities__:", szs.__capabilities__) PY - name: Test Python StringZillas-CPUs - run: python -X faulthandler -m pytest test/stringzillas.py test/similarities.py test/fingerprints.py test/doctests.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/stringzillas.py test/similarities.py test/fingerprints.py test/substrings.py test/doctests.py -s -vv --maxfail=1 --full-trace test_ubuntu_cuda: name: Ubuntu StringZillas-CUDA @@ -497,6 +503,8 @@ jobs: python -m pip install --upgrade pip pip install "setuptools>=64" wheel pip install pytest pytest-repeat numpy pyarrow wheel affine-gaps + # The third-party oracles, from the one place they are declared rather than a second list here. + pip install --group tests-oracles SZ_TARGET=stringzilla pip install -e . --force-reinstall --no-build-isolation SZ_TARGET=stringzillas-cuda pip install -e . --force-reinstall --no-build-isolation --no-deps - name: Log Python capabilities @@ -511,7 +519,7 @@ jobs: print("stringzillas.__capabilities__:", szs.__capabilities__) PY - name: Test Python StringZillas-CUDA - run: python -X faulthandler -m pytest test/stringzillas.py test/similarities.py test/fingerprints.py test/doctests.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/stringzillas.py test/similarities.py test/fingerprints.py test/substrings.py test/doctests.py -s -vv --maxfail=1 --full-trace # Temporary workaround to run Swift tests on Linux # Based on: https://github.com/swift-actions/setup-swift/issues/591#issuecomment-1685710678 @@ -958,7 +966,7 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "11.0" - name: Test Python - run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/substrings.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace # Swift - name: Set up Swift ${{ env.SWIFT_VERSION }} @@ -1054,7 +1062,7 @@ jobs: pip install pytest pytest-repeat numpy python -m pip install . - name: Test Python - run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace + run: python -X faulthandler -m pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/substrings.py --ignore=test/szs_helpers.py -s -vv --maxfail=1 --full-trace # Rust - name: Set up Rust @@ -1120,7 +1128,7 @@ jobs: pip install --break-system-packages pytest pytest-repeat pip install --break-system-packages . - name: Test Python - run: pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/szs_helpers.py -s -x + run: pytest test/ --ignore=test/stringzillas.py --ignore=test/similarities.py --ignore=test/fingerprints.py --ignore=test/substrings.py --ignore=test/szs_helpers.py -s -x # Rust - name: Set up Rust diff --git a/.gitignore b/.gitignore index 63216dbf..c3bf692c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ /build*/ /target*/ +# Per-developer preset overrides: toolchain paths that are true of one machine, not of the project +CMakeUserPresets.json + # Yes, everyone loves keeping this file in the history. # But with a very minimalistic binding and just a couple of dependencies # it brings 7000 lines of text polluting the entire repo. diff --git a/CMakeLists.txt b/CMakeLists.txt index 94597899..b610bcbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,8 +55,10 @@ # # * stringzillas_bench_similarities_cpp20: A benchmark for similarity operations. # * stringzillas_bench_similarities_cu20: A benchmark for similarity operations on GPU. -# * stringzillas_bench_fingerprints_cpp20: A benchmark for finding many substrings. -# * stringzillas_bench_fingerprints_cu20: A benchmark for finding many substrings on GPU. +# * stringzillas_bench_fingerprints_cpp20: A benchmark for sketching many strings. +# * stringzillas_bench_fingerprints_cu20: A benchmark for sketching many strings on GPU. +# * stringzillas_bench_substrings_cpp20: A benchmark for multi-pattern search. +# * stringzillas_bench_substrings_cu20: A benchmark for multi-pattern search on GPU. # # For higher-level language bindings separate build scripts are provided, native to each toolchain. cmake_minimum_required(VERSION 3.21 FATAL_ERROR) @@ -82,14 +84,22 @@ message(STATUS "C++ Compiler Version: ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS "C++ Compiler: ${CMAKE_CXX_COMPILER}") # Detect CUDA Support +# `check_language` caches the compiler it finds together with the host compiler it happened to pair with it, +# and that pairing then outranks a caller's `CMAKE_CUDA_HOST_COMPILER` - which is why the probe runs only to +# pick the default below. A caller who asks for CUDA outright gets the real detection from `enable_language`, +# whose own failure message says everything this probe would have. set(STRINGZILLA_CAN_BUILD_CUDA OFF) -include(CheckLanguage) -check_language(CUDA) -if (CMAKE_CUDA_COMPILER) - set(STRINGZILLA_CAN_BUILD_CUDA ON) - message(STATUS "CUDA compiler available") +if (DEFINED STRINGZILLA_BUILD_CUDA) + set(STRINGZILLA_CAN_BUILD_CUDA ${STRINGZILLA_BUILD_CUDA}) else () - message(STATUS "CUDA compiler not available") + include(CheckLanguage) + check_language(CUDA) + if (CMAKE_CUDA_COMPILER) + set(STRINGZILLA_CAN_BUILD_CUDA ON) + message(STATUS "CUDA compiler available") + else () + message(STATUS "CUDA compiler not available") + endif () endif () if (CMAKE_SIZEOF_VOID_P EQUAL 8) @@ -275,6 +285,14 @@ if (STRINGZILLA_BUILD_TEST OR STRINGZILLA_BUILD_BENCHMARK) set(STRINGZILLA_NEEDS_FORKUNION ON) endif () +# A dynamic StringZillas externs the `sz_*` entry points through `SZ_DYNAMIC_DISPATCH=1` and links the core +# library to resolve them, which a Mach-O build demands at link time rather than at load time. There is no core +# to link when the dynamic one was never built. +if (STRINGZILLAS_BUILD_SHARED AND NOT STRINGZILLA_BUILD_SHARED) + message(FATAL_ERROR "STRINGZILLAS_BUILD_SHARED needs STRINGZILLA_BUILD_SHARED - the parallel libraries " + "resolve their `sz_*` calls against it. Enable it, or set STRINGZILLAS_BUILD_SHARED=0.") +endif () + if (STRINGZILLA_NEEDS_FORKUNION AND NOT TARGET forkunion_header) if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/forkunion/CMakeLists.txt") message(FATAL_ERROR "The `forkunion` submodule is not populated - run: git submodule update --init --recursive") @@ -326,6 +344,9 @@ if (STRINGZILLA_BUILD_CUDA) PATH_SUFFIXES cccl NO_DEFAULT_PATH ) + include(cmake/sz_cuda_probes.cmake) + sz_cuda_probe_native_arch_() + message(STATUS "CUDA support enabled") message(STATUS "CUDA Compiler: ${CMAKE_CUDA_COMPILER}") message(STATUS "CUDA Compiler ID: ${CMAKE_CUDA_COMPILER_ID}") @@ -429,9 +450,9 @@ set(STRINGZILLA_SHIM_SOURCES c/stringzilla/utf8_uncased.c ) -# Helper function used for `stringzilla_shared` and `stringzilla_bare` targets -function (define_stringzilla_shared target public_name) - add_library(${target} SHARED ${STRINGZILLA_SHIM_SOURCES}) +# Helper function used for the `stringzilla_shared`, `stringzilla_bare`, and `stringzilla_static` targets +function (define_stringzilla_library target public_name library_type) + add_library(${target} ${library_type} ${STRINGZILLA_SHIM_SOURCES}) # `stringzilla_shared` is published as `stringzilla::shared`. Both spellings resolve, in-tree and installed. add_library(${PROJECT_NAME}::${public_name} ALIAS ${target}) @@ -457,9 +478,12 @@ endfunction () if (STRINGZILLA_BUILD_SHARED) - define_stringzilla_shared(stringzilla_shared shared) + # The shim units default `SZ_OVERRIDE_LIBC` to `SZ_AVOID_LIBC`, so this library exports no `memcpy` of its + # own and interposes nothing. That is what lets anything link it - StringZillas, the tests, a consumer's + # program - without ELF redirecting `libstdc++`'s internal calls into a dispatch table that is not up yet. + # `stringzilla_bare` is the artifact that replaces LibC, and it still does. + define_stringzilla_library(stringzilla_shared shared SHARED) target_compile_definitions(stringzilla_shared PRIVATE "SZ_AVOID_LIBC=0") - target_compile_definitions(stringzilla_shared PRIVATE "SZ_OVERRIDE_LIBC=1") target_include_directories( stringzilla_shared PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> @@ -479,7 +503,7 @@ if (STRINGZILLA_BUILD_SHARED) # Try compiling a version without linking the LibC ! This is only for Linux/MSVC, as on modern Arm-based MacOS # machines ! We can't legally access Arm's "feature registers" without `sysctl` or `sysctlbyname`. if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") - define_stringzilla_shared(stringzilla_bare bare) + define_stringzilla_library(stringzilla_bare bare SHARED) target_compile_definitions(stringzilla_bare PRIVATE "SZ_AVOID_LIBC=1") target_compile_definitions(stringzilla_bare PRIVATE "SZ_OVERRIDE_LIBC=1") target_include_directories( @@ -502,6 +526,15 @@ if (STRINGZILLA_BUILD_SHARED) endif () endif () +# The dispatch core as an archive. Tests and benchmarks link this rather than the shared library: the +# cross-compiled runs are `-static` executables, which cannot take a `.so` at all, and the archive fixes the +# capability set at build time instead of deferring it to a shared library's own runtime detection. +if (STRINGZILLA_BUILD_TEST OR STRINGZILLA_BUILD_BENCHMARK) + define_stringzilla_library(stringzilla_static static STATIC) + target_compile_definitions(stringzilla_static PRIVATE "SZ_AVOID_LIBC=0") + target_include_directories(stringzilla_static PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>) +endif () + # Definitions every StringZillas translation unit shares. set(STRINGZILLAS_COMMON_DEFINITIONS "SZ_DYNAMIC_DISPATCH=1" "SZ_AVOID_LIBC=0" "$<IF:$<CONFIG:Debug>,SZ_DEBUG=1,SZ_DEBUG=0>" @@ -563,6 +596,13 @@ function (define_stringzillas_shared base public_base) target_compile_definitions(${variant} INTERFACE "SZ_DYNAMIC_DISPATCH=1") endforeach () + # `SZ_DYNAMIC_DISPATCH=1` externs the `sz_*` entry points these engines call - `sz_copy` from the rewriting + # paths, among others - and the core library is what resolves them for the dynamic variant. A Mach-O dynamic + # library must have every symbol resolved at link time, where ELF defers to load time, so without this the + # macOS build fails on a dangling reference Linux silently accepts. The archive needs no such line: it + # resolves nothing itself, and whoever links it brings its own core. + target_link_libraries(${target} PUBLIC stringzilla_shared) + find_package(Threads REQUIRED) target_link_libraries(${target} PUBLIC Threads::Threads) target_link_libraries(${static} PUBLIC Threads::Threads) @@ -604,11 +644,11 @@ endfunction () # StringZillas C-API entry units, compiled once per library - as C++ into the CPU one, as CUDA into the GPU one. set(STRINGZILLAS_API_CPP_SOURCES c/stringzillas/runtime.cpp c/stringzillas/levenshtein.cpp c/stringzillas/needleman_wunsch.cpp - c/stringzillas/smith_waterman.cpp c/stringzillas/fingerprints.cpp + c/stringzillas/smith_waterman.cpp c/stringzillas/fingerprints.cpp c/stringzillas/substrings.cpp ) set(STRINGZILLAS_API_CU_SOURCES c/stringzillas/runtime.cu c/stringzillas/levenshtein.cu c/stringzillas/needleman_wunsch.cu - c/stringzillas/smith_waterman.cu c/stringzillas/fingerprints.cu + c/stringzillas/smith_waterman.cu c/stringzillas/fingerprints.cu c/stringzillas/substrings.cu ) # Per-ISA CPU instantiation units, host C++ in every library; off-platform files compile to empty objects. set(STRINGZILLAS_CPUS_SOURCES @@ -627,11 +667,14 @@ set(STRINGZILLAS_CPUS_SOURCES c/stringzillas/smith_waterman_haswell.cpp c/stringzillas/smith_waterman_neon.cpp c/stringzillas/smith_waterman_rvv.cpp + # Multi-pattern search has one CPU unit, not a per-ISA set: the walk is a serial dependency chain, so + # parallelism comes from many independent haystacks rather than from vector instructions. + c/stringzillas/substrings_serial.cpp ) # Per-tier GPU instantiation units, grouped by architecture floor: Hopper DPX needs sm_90, the rest run # from the base set. set(STRINGZILLAS_CUDA_SOURCES c/stringzillas/levenshtein_cuda.cu c/stringzillas/needleman_wunsch_cuda.cu - c/stringzillas/smith_waterman_cuda.cu + c/stringzillas/smith_waterman_cuda.cu c/stringzillas/substrings_cuda.cu ) set(STRINGZILLAS_KEPLER_SOURCES c/stringzillas/levenshtein_kepler.cu) set(STRINGZILLAS_HOPPER_SOURCES c/stringzillas/levenshtein_hopper.cu c/stringzillas/needleman_wunsch_hopper.cu @@ -693,7 +736,8 @@ function ( $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> ) target_compile_definitions(${target} INTERFACE "SZ_DYNAMIC_DISPATCH=1") - target_link_libraries(${target} PUBLIC forkunion::static CUDA::cudart Threads::Threads) + # `cuGetErrorName` and `cuInit` resolve out of `cuda`, not `cudart`, and only a Windows DLL reports them. + target_link_libraries(${target} PUBLIC forkunion::static CUDA::cudart CUDA::cuda_driver Threads::Threads) endfunction () if (STRINGZILLAS_BUILD_SHARED) @@ -737,17 +781,29 @@ if (STRINGZILLA_BUILD_BENCHMARK) # Parallel benchmarks link the precompiled static libraries when those are built. define_launcher(stringzillas_bench_similarities_cpp20 20 "${STRINGZILLA_TARGET_ARCH}" bench/similarities.cpp) define_launcher(stringzillas_bench_fingerprints_cpp20 20 "${STRINGZILLA_TARGET_ARCH}" bench/fingerprints.cpp) + define_launcher(stringzillas_bench_substrings_cpp20 20 "${STRINGZILLA_TARGET_ARCH}" bench/substrings.cpp) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_bench_similarities_cpp20 PRIVATE stringzillas_cpus_static) target_link_libraries(stringzillas_bench_fingerprints_cpp20 PRIVATE stringzillas_cpus_static) + target_link_libraries(stringzillas_bench_substrings_cpp20 PRIVATE stringzillas_cpus_static) endif () + # Resolves the `sz_*` entry points the inherited `SZ_DYNAMIC_DISPATCH=1` externs. + target_link_libraries(stringzillas_bench_similarities_cpp20 PRIVATE stringzilla_static) + target_link_libraries(stringzillas_bench_fingerprints_cpp20 PRIVATE stringzilla_static) + target_link_libraries(stringzillas_bench_substrings_cpp20 PRIVATE stringzilla_static) if (STRINGZILLA_BUILD_CUDA) define_gpu_launcher(stringzillas_bench_similarities_cu20 20 "${STRINGZILLA_TARGET_ARCH}" bench/similarities.cu) define_gpu_launcher(stringzillas_bench_fingerprints_cu20 20 "${STRINGZILLA_TARGET_ARCH}" bench/fingerprints.cu) + define_gpu_launcher(stringzillas_bench_substrings_cu20 20 "${STRINGZILLA_TARGET_ARCH}" bench/substrings.cu) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_bench_similarities_cu20 PRIVATE stringzillas_cuda_static) target_link_libraries(stringzillas_bench_fingerprints_cu20 PRIVATE stringzillas_cuda_static) + target_link_libraries(stringzillas_bench_substrings_cu20 PRIVATE stringzillas_cuda_static) endif () + # Resolves the `sz_*` entry points the inherited `SZ_DYNAMIC_DISPATCH=1` externs. + target_link_libraries(stringzillas_bench_similarities_cu20 PRIVATE stringzilla_static) + target_link_libraries(stringzillas_bench_fingerprints_cu20 PRIVATE stringzilla_static) + target_link_libraries(stringzillas_bench_substrings_cu20 PRIVATE stringzilla_static) # The host object pulls in `std::log` etc.; unlike the host-compiler driver used for the `cpp20` benches, nvcc's # final link does not implicitly add libm. Link it only where it exists as a separate library (Linux/glibc); on # macOS/MSVC the math symbols live in the C runtime. @@ -755,6 +811,7 @@ if (STRINGZILLA_BUILD_BENCHMARK) if (STRINGZILLA_LIBM) target_link_libraries(stringzillas_bench_similarities_cu20 PRIVATE ${STRINGZILLA_LIBM}) target_link_libraries(stringzillas_bench_fingerprints_cu20 PRIVATE ${STRINGZILLA_LIBM}) + target_link_libraries(stringzillas_bench_substrings_cu20 PRIVATE ${STRINGZILLA_LIBM}) endif () endif () endif () @@ -795,11 +852,14 @@ if (STRINGZILLA_BUILD_TEST) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_test_cpp20 PRIVATE stringzillas_cpus_static) endif () + # Resolves the `sz_*` entry points the inherited `SZ_DYNAMIC_DISPATCH=1` externs. + target_link_libraries(stringzillas_test_cpp20 PRIVATE stringzilla_static) if ("cxx_std_23" IN_LIST CMAKE_CXX_COMPILE_FEATURES) define_launcher(stringzillas_test_cpp23 23 "${STRINGZILLA_TARGET_ARCH}" test/stringzillas.cpp) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_test_cpp23 PRIVATE stringzillas_cpus_static) endif () + target_link_libraries(stringzillas_test_cpp23 PRIVATE stringzilla_static) endif () if (STRINGZILLA_BUILD_CUDA) @@ -807,11 +867,13 @@ if (STRINGZILLA_BUILD_TEST) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_test_cu20 PRIVATE stringzillas_cuda_static) endif () + target_link_libraries(stringzillas_test_cu20 PRIVATE stringzilla_static) if ("cuda_std_23" IN_LIST CMAKE_CUDA_COMPILE_FEATURES) define_gpu_launcher(stringzillas_test_cu23 23 "${STRINGZILLA_TARGET_ARCH}" test/stringzillas.cu) if (STRINGZILLAS_BUILD_SHARED) target_link_libraries(stringzillas_test_cu23 PRIVATE stringzillas_cuda_static) endif () + target_link_libraries(stringzillas_test_cu23 PRIVATE stringzilla_static) endif () endif () endif () diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..74d8dcbb --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,108 @@ +{ + "version": 6, + "cmakeMinimumRequired": { + "major": 3, + "minor": 25, + "patch": 0 + }, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "binaryDir": "${sourceDir}/build_${presetName}", + "cacheVariables": { + "STRINGZILLA_BUILD_TEST": "ON", + "STRINGZILLA_BUILD_BENCHMARK": "ON" + } + }, + { + "name": "debug", + "inherits": "base", + "displayName": "Debug, no CUDA", + "description": "Sanitized host build for iterating on kernels and tests.", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "STRINGZILLA_BUILD_CUDA": "OFF", + "STRINGZILLA_USE_SANITIZERS": "ON" + } + }, + { + "name": "release", + "inherits": "base", + "displayName": "Release, no CUDA", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "STRINGZILLA_BUILD_CUDA": "OFF" + } + }, + { + "name": "cuda", + "inherits": "base", + "displayName": "Release with CUDA", + "description": "NVCC over whichever host compiler it defaults to. A toolkit accepts only a range of host versions, so a machine whose default sits outside its own toolkit's range pins one through CMAKE_CUDA_HOST_COMPILER in CMakeUserPresets.json - the version that fits is a fact about the machine, not about this project.", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "STRINGZILLA_BUILD_CUDA": "ON" + } + }, + { + "name": "cuda_clang", + "inherits": "cuda", + "displayName": "Release with CUDA, Clang host", + "description": "NVCC over Clang, and Clang for the C and C++ sources too - the three must agree. Clang mangles a C++20 requires-clause into the symbol name and GCC does not, so a build that compiled .cpp with GCC and .cu with NVCC-over-Clang would give the tree's constrained engine entry points two different symbols and fail to link. GCC also miscompiles the engines' return-by-value in the large benchmark translation units, handing the caller the kernel timing where the status belongs, which is why measured numbers come from here. Each toolkit supports a range of Clang versions and parses that Clang's own headers, so the version that fits belongs in CMakeUserPresets.json; Clang additionally reads its standard library from the newest GCC installed, which -Xcompiler=--gcc-install-dir=<path> redirects when that one is ahead of the toolkit too.", + "cacheVariables": { + "CMAKE_C_COMPILER": "clang", + "CMAKE_CXX_COMPILER": "clang++", + "CMAKE_CUDA_HOST_COMPILER": "clang++" + } + } + ], + "buildPresets": [ + { + "name": "debug", + "configurePreset": "debug" + }, + { + "name": "release", + "configurePreset": "release" + }, + { + "name": "cuda", + "configurePreset": "cuda" + }, + { + "name": "cuda_clang", + "configurePreset": "cuda_clang" + } + ], + "testPresets": [ + { + "name": "debug", + "configurePreset": "debug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "release", + "configurePreset": "release", + "output": { + "outputOnFailure": true + } + }, + { + "name": "cuda", + "configurePreset": "cuda", + "output": { + "outputOnFailure": true + } + }, + { + "name": "cuda_clang", + "configurePreset": "cuda_clang", + "output": { + "outputOnFailure": true + } + } + ] +} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ce2d2ca..b07b3de0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,16 +23,20 @@ The project is split into the following parts: - `include/stringzillas/*` - parallel CPU/GPU header-only backends. - `c/*` - [C, C++, and CUDA](#c-and-c) sources for dynamic dispatch and parallel backends. - `rust/*` - [Rust](#rust) crate sources; `rust/stringzilla/*` and `rust/stringzillas/*` hold one module per kernel domain, re-exported through `rust/stringzilla.rs` and `rust/stringzillas.rs`. -- `python/*` - [Python](#python) bindings; one translation unit per kernel domain, with `python/stringzilla.h` and `python/stringzillas.h` as the two extensions' private headers. +- `python/*` - [Python](#python) bindings; one translation unit per kernel domain, with `python/stringzilla/stringzilla.h` and `python/stringzillas/stringzillas.h` as the two extensions' private headers. - `swift/*` - [Swift](#swift) package sources and tests. - `javascript/*` - [JavaScript](#javascript) bindings. - `golang/*` - [Go](#golang) bindings. +- `java/*` - Java bindings. +- `csharp/*` - C# bindings. +- `probes/*` - single-TU ISA probes used by CMake to detect compiler and platform support. +- `cmake/*` - CMake toolchain files and package config templates. - `test/*` and `bench/*` - per-kernel test and benchmark sources. For minimal test coverage, check the following scripts: - `test/stringzilla.cpp` - drives every per-ISA C kernel directly, spot-checking the C++ wrappers in the `_unit` tier. -- `test/*.py` - tests the Python API against native strings, split per kernel family (`string.py`, `find.py`, `sort.py`, `hash.py`, `cipher.py`, `uncased.py`, `similarities.py`, `fingerprints.py`, `utf8_*.py`), with shared helpers in `sz_helpers.py`, `szs_helpers.py`, and `utf8_helpers.py`. +- `test/*.py` - tests the Python API against native strings, split per kernel family (`string.py`, `find.py`, `sort.py`, `hash.py`, `cipher.py`, `uncased.py`, `similarities.py`, `fingerprints.py`, `substrings.py`, `utf8_*.py`), with shared helpers in `sz_helpers.py`, `szs_helpers.py`, and `utf8_helpers.py`. - `test/stringzilla.js`. At the C++ level all benchmarks also validate the results against the STL baseline, serving as tests on real-world data. @@ -171,10 +175,11 @@ Using modern syntax, this is how you build and run the test suite: cmake -D STRINGZILLA_BUILD_TEST=1 -D STRINGZILLA_USE_SANITIZERS=0 -D CMAKE_BUILD_TYPE=Debug -B build_debug cmake --build build_debug --config Debug --parallel # Which will produce the following targets: build_debug/stringzilla_test_cpp20 # Unit test for the entire library compiled for current hardware -build_debug/stringzilla_test_cpp20_serial # x86 variant compiled for IvyBridge - last arch. before AVX2 -build_debug/stringzilla_test_cpp20_serial # Arm variant compiled without Neon ``` +There is no separate SIMD-disabled target. +To get a build with SIMD dispatch narrowed, pass `-D STRINGZILLA_TARGET_ARCH=<name>` at configure time, for example `ivybridge` for x86 without AVX2, or set the Arm equivalent through the same variable, then rebuild `stringzilla_test_cpp20` against that configuration. + Note, that Address Sanitizers have a hard time with masked load and store instructions in AVX-512 and SVE. #### Test Tiers @@ -294,20 +299,28 @@ For benchmarks, you can use the following commands: ```bash cmake -D STRINGZILLA_BUILD_BENCHMARK=1 -B build_release cmake --build build_release --config Release --parallel # Produces the following targets: -build_release/stringzilla_bench_memory_cpp20 # - for string copies and fills -build_release/stringzilla_bench_find_cpp20 # - for substring search -build_release/stringzilla_bench_token_cpp20 # - for hashing, equality comparisons, etc. -build_release/stringzilla_bench_sequence_cpp20 # - for sorting arrays of strings -build_release/stringzilla_bench_container_cpp20 # - for STL containers with string keys +build_release/stringzilla_bench_find_cpp20 # - for substring search +build_release/stringzilla_bench_sequence_cpp20 # - for sorting arrays of strings +build_release/stringzilla_bench_token_cpp20 # - for hashing, equality comparisons, etc. +build_release/stringzilla_bench_utf8_uncased_cpp20 # - for UTF-8 case-folding and case-insensitive comparisons +build_release/stringzilla_bench_utf8_traverse_cpp20 # - for UTF-8 codepoint iteration +build_release/stringzilla_bench_utf8_scan_cpp20 # - for UTF-8 validation and scanning +build_release/stringzilla_bench_utf8_segment_cpp20 # - for UTF-8 grapheme, word, sentence, and linewrap segmentation +build_release/stringzilla_bench_utf8_norm_cpp20 # - for UTF-8 normalization +build_release/stringzilla_bench_container_cpp20 # - for STL containers with string keys +build_release/stringzilla_bench_memory_cpp20 # - for string copies and fills +build_release/stringzilla_bench_cipher_cpp20 # - for AES encryption and decryption ``` There are also parallel algorithms that need a very different benchmarking setup: ```sh -build_release/stringzillas_bench_fingerprints_cpp20 # - for parallel multi-pattern search on CPU -build_release/stringzillas_bench_fingerprints_cu20 # - for parallel multi-pattern search on GPU build_release/stringzillas_bench_similarities_cpp20 # - for parallel edit distances and alignment scores on CPU build_release/stringzillas_bench_similarities_cu20 # - for parallel edit distances and alignment scores on GPU +build_release/stringzillas_bench_fingerprints_cpp20 # - for parallel Min-Hash sketching on CPU +build_release/stringzillas_bench_fingerprints_cu20 # - for parallel Min-Hash sketching on GPU +build_release/stringzillas_bench_substrings_cpp20 # - for parallel multi-pattern search on CPU +build_release/stringzillas_bench_substrings_cu20 # - for parallel multi-pattern search on GPU ``` All of them support customization via environment variables. @@ -326,21 +339,27 @@ STRINGWARS_STRESS=0 STRINGWARS_FILTER="(cuda|kepler|hopper).*:batch1" STRINGWARS The benchmark harness reads these environment variables: -| Variable | Description | Default | -| :---------------------- | :------------------------------------------------------------ | -------------------: | -| `STRINGWARS_DATASET` | Path to the input corpus | required | -| `STRINGWARS_FILTER` | Regex over benchmark names; only matching backends run | (all) | -| `STRINGWARS_DURATION` | Seconds per benchmark (longer = steadier numbers) | 1 debug / 10 release | -| `STRINGWARS_MAX_TOKENS` | Cap on tokens kept, for faster, smaller runs | unlimited | -| `STRINGWARS_BATCH` | Comma-separated batch-size override (skips the largest sweep) | backend default | -| `STRINGWARS_STRESS` | Run the correctness stress phase (`0` to skip while timing) | on | -| `STRINGWARS_SEED` | Non-zero shuffles tokens; `0` keeps deterministic order | 0 | - -For a fast inner loop, scope to one backend on a small dataset, cap tokens, skip the stress phase, and use short runs: +| Variable | Description | Default | +| :----------------------------- | :------------------------------------------------------------------------------------ | -------------------: | +| `STRINGWARS_DATASET` | Path to the input corpus | required | +| `STRINGWARS_DATASET_LIMIT` | Byte cap on the dataset read, e.g. `64mb`; `0` reads the whole file | 0 (whole file) | +| `STRINGWARS_TOKENS` | Tokenization mode: `file`, `lines`, `words`, or a positive integer for N-grams | per-benchmark | +| `STRINGWARS_UNIQUE` | `1` sorts the tokenized set and drops duplicates before benchmarking | off | +| `STRINGWARS_FILTER` | Regex over benchmark names; only matching backends run | (all) | +| `STRINGWARS_DURATION` | Seconds per benchmark (longer = steadier numbers) | 1 debug / 10 release | +| `STRINGWARS_SEED` | Non-zero shuffles tokens; `0` keeps deterministic order | 0 | +| `STRINGWARS_BATCH` | Comma-separated batch-size override (skips the largest sweep) | backend default | +| `STRINGWARS_BATCH_PER_CORE` | Pairs scored per core for the parallel similarity benchmarks; scales with device parallelism | 16 debug / 256 release | +| `STRINGWARS_STRESS` | Run the correctness stress phase (`0` to skip while timing) | on | +| `STRINGWARS_STRESS_DURATION` | Seconds per stress-test | 1 debug / 10 release | +| `STRINGWARS_STRESS_DIR` | Directory for stress-test failure logs | .tmp | +| `STRINGWARS_STRESS_LIMIT` | Number of stress-test failures tolerated before aborting | 1 | + +For a fast inner loop, scope to one backend on a small dataset, cap the dataset size, skip the stress phase, and use short runs: ```bash STRINGWARS_FILTER='sz_find' STRINGWARS_DATASET=leipzig1M.txt \ - STRINGWARS_MAX_TOKENS=65536 STRINGWARS_BATCH=1024 \ + STRINGWARS_DATASET_LIMIT=65536 STRINGWARS_BATCH=1024 \ STRINGWARS_STRESS=0 STRINGWARS_DURATION=1 \ build_release/stringzilla_bench_find_cpp20 ``` @@ -408,7 +427,7 @@ readelf --sections build_profile/stringzilla_bench_token_cpp20 | grep debug objdump -h build_profile/stringzilla_bench_token_cpp20 | grep debug # Profile -sudo perf record -g build_profile/stringzilla_bench_token_cpp20 ./leipzig1M.txt +sudo env STRINGWARS_DATASET=./leipzig1M.txt perf record -g build_profile/stringzilla_bench_token_cpp20 sudo perf report ``` diff --git a/Cargo.lock b/Cargo.lock index c34bcb76..9ab17305 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.4.0" @@ -31,6 +40,12 @@ dependencies = [ "cc", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "shlex" version = "1.3.0" @@ -50,6 +65,7 @@ dependencies = [ name = "stringzilla" version = "5.1.2" dependencies = [ + "aho-corasick", "allocator-api2", "cc", "forkunion", diff --git a/Cargo.toml b/Cargo.toml index ad9d1caf..1fbde504 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,13 @@ stringtape = {version = "2.4.2", optional = true} [build-dependencies] cc = "1.2.47" +# Third-party oracles the `#[cfg(test)]` suites differential against. Dev-dependencies rather than an +# optional feature, so they never reach a consumer's build graph, never appear on docs.rs, and never +# become public surface this crate has to keep. `aho-corasick` is the reference multi-pattern +# automaton, whose three `MatchKind`s are our three overlap policies under different names. +[dev-dependencies] +aho-corasick = "1.1" + [lints.clippy] # Catch platform-specific type issues like `c_char` differences cast-sign-loss = "warn" diff --git a/MANIFEST.in b/MANIFEST.in index fd20ef5c..c7406cd1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,6 +10,4 @@ graft forkunion/include graft c graft python -include test/stringzilla.py include test/stringzillas.py -include bench/*.py diff --git a/README.md b/README.md index ba3d5ec6..28331d65 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Reach for it from your language of choice: - 🐂 __[C](#c-and-c):__ Upgrade LibC's `<string.h>` to `<stringzilla/stringzilla.h>` in C 99 - 🐉 __[C++](#c-and-c):__ Upgrade STL's `<string>` to `<stringzilla/stringzilla.hpp>` in C++ 11 -- 🧮 __[CUDA](include/stringzillas/README.md):__ Process in-bulk with `<stringzillas/stringzillas.cuh>` in CUDA C++ 17 +- 🧮 __[CUDA](include/stringzillas/README.md):__ Process in-bulk with `<stringzillas/stringzillas.h>` in CUDA C++ 20 - 🐍 __[Python](#python):__ Upgrade your `str` to faster `Str` - đŸĻ€ __[Rust](#rust):__ Use the `StringZilla` traits crate - đŸĻĢ __[Go](#go):__ Use the `StringZilla` cGo module @@ -94,11 +94,11 @@ Needleman-Wunsch scores, ≅ 1 KB DNA, one core (MCUPS) > Treat these as a first impression, not a benchmark suite. > The Unicode numbers were obtained on a 128 MB slice of multilingual XLSum; the similarity rows on synthetic DNA strings. > `Xeon4` is an Intel Sapphire Rapids with GCC and `glibc`, `M5 Pro` an 18-core Apple Silicon with Apple clang and `libc++`, `H100` an Nvidia Hopper GPU. -> The two CPUs therefore differ in standard library as much as in ISA, which is most of the gap in the `strstr`, `std::string::rfind`, and `bytes.translate` rows; the StringZilla rows build from the same source on both. +> The two CPUs therefore differ in standard library as much as in ISA, which is most of the gap in the `strstr`, `strcspn`, and `std::string::find_first_of` rows; the StringZilla rows build from the same source on both. > These will not reproduce exactly; the links below carry the methodology and the per-library breakdowns. Most StringZilla modules ship ready-to-run benchmarks for C, C++, Python, and more. -Grab them from `./scripts`, and see [`CONTRIBUTING.md`](CONTRIBUTING.md), [`test/README.md`](test/README.md), and [`bench/README.md`](bench/README.md) for instructions. +Grab them from `./bench`, and see [`CONTRIBUTING.md`](CONTRIBUTING.md), [`test/README.md`](test/README.md), and [`bench/README.md`](bench/README.md) for instructions. For wider head-to-heads against Rust and Python favorites, browse the __[StringWars][stringwars]__ repository. To inspect collision resistance and distribution shapes for our hashers, see __[HashEvals][hashevals]__. @@ -174,6 +174,7 @@ Consider contributing if you need a feature that's not yet implemented. | | | | | | | | | | | | | Parallel Similarity Scoring | đŸŒŗ | ✅ | ✅ | ✅ | ✅ | âšĒ | âšĒ | âšĒ | âšĒ | âšĒ | | Parallel Rolling Fingerprints | đŸŒŗ | ✅ | ✅ | ✅ | ✅ | âšĒ | âšĒ | âšĒ | âšĒ | âšĒ | +| Parallel Multi-Pattern Search | 🚧 | ✅ | ✅ | ✅ | ✅ | âšĒ | âšĒ | âšĒ | âšĒ | âšĒ | > đŸŒŗ parts are used in production. > 🧐 parts are in beta. @@ -390,8 +391,10 @@ The naive implementation, however: There are several ways to improve the original algorithm. One is to use sparse DFA representation, which is more cache-friendly, but would require extra processing to navigate state transitions. -StringZilla does not ship an Aho-Corasick automaton today. -For multi-pattern workloads, the rolling-fingerprint machinery described below covers the near-duplicate and candidate-filtering cases, and [hyperscan](https://github.com/intel/hyperscan) or [pyahocorasick](https://github.com/WojciechMula/pyahocorasick) remain the better fit for large literal dictionaries. +StringZilla takes that route in [`stringzillas`](include/stringzillas/substrings/README.md), compiling a needle dictionary once into a goto-completed Aho-Corasick automaton and reusing it across every later call. +The automaton is split into two tiers - a dense 256-wide row per frequently-visited state and a double array for the rest - so a step is one load with no failure-following at runtime. +It matches raw bytes or applies full Unicode case folding to both sides, runs across a slice of CPU cores or a CUDA GPU, and is reachable as `szs::Substrings` in Rust and `szs.Substrings` in Python. +Where only near-duplicate detection or candidate filtering is needed, the rolling-fingerprint machinery described below is the cheaper tool. ### Levenshtein Edit Distance diff --git a/bench/README.md b/bench/README.md index 587f70e4..973eb2dc 100644 --- a/bench/README.md +++ b/bench/README.md @@ -13,13 +13,19 @@ This is the internal, cross-backend counterpart to [StringWars](https://github.c - `cipher.cpp` — AES-256 counter mode and Galois/counter mode throughput. - `container.cpp` — STL associative containers with string keys. - `similarities.cpp` — Levenshtein, Needleman-Wunsch, and Smith-Waterman scoring. -- `fingerprints.cpp` — MinHash rolling fingerprints and multi-pattern search. -- `utf8_iterate.cpp` and `utf8_uncased.cpp` — UTF-8 iteration, segmentation, and case-folding throughput. +- `fingerprints.cpp` — MinHash rolling fingerprints. +- `substrings.cpp` — multi-pattern Aho-Corasick counting, locating, rewriting, and BM25 scoring. +- `utf8_traverse.cpp` — codepoint counting, Nth-codepoint seeking, and codepoint iteration. +- `utf8_scan.cpp` — codepoint-class enumeration: newlines, whitespace, and delimiter runs. +- `utf8_segment.cpp` — UAX-29 and UAX-14 boundary segmentation: words, graphemes, sentences, and linebreaks. +- `utf8_norm.cpp` — Unicode normalization and quick-check scanning. +- `utf8_uncased.cpp` — case folding and uncased search. ## CUDA - `similarities.cu` — similarity scoring on CUDA GPUs. - `fingerprints.cu` — fingerprinting on CUDA GPUs. +- `substrings.cu` — multi-pattern search on CUDA GPUs. ## Other Bindings diff --git a/bench/cipher.cpp b/bench/cipher.cpp index a7ccdf78..cd08c2ed 100644 --- a/bench/cipher.cpp +++ b/bench/cipher.cpp @@ -138,6 +138,28 @@ void bench_cipher_ctr(environment_t const &env) { bench_unary(env, "sz_aes256_ctr_xor_sve2aes" + suffix, validator, ctr_from_sz<sz_aes256_key_init_sve2aes, sz_aes256_ctr_xor_sve2aes> {message_bytes, pool, target}) .log(base); +#endif +#if SZ_USE_V128 + bench_unary(env, "sz_aes256_ctr_xor_v128" + suffix, validator, + ctr_from_sz<sz_aes256_key_init_v128, sz_aes256_ctr_xor_v128> {message_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_V128RELAXED + bench_unary( + env, "sz_aes256_ctr_xor_v128relaxed" + suffix, validator, + ctr_from_sz<sz_aes256_key_init_v128relaxed, sz_aes256_ctr_xor_v128relaxed> {message_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_RVVCRYPTO + bench_unary( + env, "sz_aes256_ctr_xor_rvvcrypto" + suffix, validator, + ctr_from_sz<sz_aes256_key_init_rvvcrypto, sz_aes256_ctr_xor_rvvcrypto> {message_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_POWERVSX + bench_unary(env, "sz_aes256_ctr_xor_powervsx" + suffix, validator, + ctr_from_sz<sz_aes256_key_init_powervsx, sz_aes256_ctr_xor_powervsx> {message_bytes, pool, target}) + .log(base); #endif } } @@ -181,6 +203,29 @@ void bench_cipher_gcm(environment_t const &env) { env, "sz_aes256_gcm_encrypt_sve2aes" + suffix, validator, gcm_from_sz<sz_aes256_gcm_key_init_sve2aes, sz_aes256_gcm_encrypt_sve2aes> {message_bytes, pool, target}) .log(base); +#endif +#if SZ_USE_V128 + bench_unary(env, "sz_aes256_gcm_encrypt_v128" + suffix, validator, + gcm_from_sz<sz_aes256_gcm_key_init_v128, sz_aes256_gcm_encrypt_v128> {message_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_V128RELAXED + bench_unary(env, "sz_aes256_gcm_encrypt_v128relaxed" + suffix, validator, + gcm_from_sz<sz_aes256_gcm_key_init_v128relaxed, sz_aes256_gcm_encrypt_v128relaxed> {message_bytes, + pool, target}) + .log(base); +#endif +#if SZ_USE_RVVCRYPTO + bench_unary(env, "sz_aes256_gcm_encrypt_rvvcrypto" + suffix, validator, + gcm_from_sz<sz_aes256_gcm_key_init_rvvcrypto, sz_aes256_gcm_encrypt_rvvcrypto> {message_bytes, pool, + target}) + .log(base); +#endif +#if SZ_USE_POWERVSX + bench_unary( + env, "sz_aes256_gcm_encrypt_powervsx" + suffix, validator, + gcm_from_sz<sz_aes256_gcm_key_init_powervsx, sz_aes256_gcm_encrypt_powervsx> {message_bytes, pool, target}) + .log(base); #endif } } @@ -285,6 +330,37 @@ void bench_cipher_stream(environment_t const &env) { sz_aes256_gcm_encryptor_update_sve2aes, sz_aes256_gcm_encryptor_digest_sve2aes> { message_bytes, chunk_bytes, pool, target}) .log(base); +#endif +#if SZ_USE_V128 + bench_unary(env, "sz_aes256_gcm_stream_v128" + suffix, validator, + gcm_stream_from_sz<sz_aes256_gcm_key_init_v128, sz_aes256_gcm_encryptor_init_v128, + sz_aes256_gcm_encryptor_update_v128, sz_aes256_gcm_encryptor_digest_v128> { + message_bytes, chunk_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_V128RELAXED + bench_unary( + env, "sz_aes256_gcm_stream_v128relaxed" + suffix, validator, + gcm_stream_from_sz<sz_aes256_gcm_key_init_v128relaxed, sz_aes256_gcm_encryptor_init_v128relaxed, + sz_aes256_gcm_encryptor_update_v128relaxed, sz_aes256_gcm_encryptor_digest_v128relaxed> { + message_bytes, chunk_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_RVVCRYPTO + bench_unary( + env, "sz_aes256_gcm_stream_rvvcrypto" + suffix, validator, + gcm_stream_from_sz<sz_aes256_gcm_key_init_rvvcrypto, sz_aes256_gcm_encryptor_init_rvvcrypto, + sz_aes256_gcm_encryptor_update_rvvcrypto, sz_aes256_gcm_encryptor_digest_rvvcrypto> { + message_bytes, chunk_bytes, pool, target}) + .log(base); +#endif +#if SZ_USE_POWERVSX + bench_unary( + env, "sz_aes256_gcm_stream_powervsx" + suffix, validator, + gcm_stream_from_sz<sz_aes256_gcm_key_init_powervsx, sz_aes256_gcm_encryptor_init_powervsx, + sz_aes256_gcm_encryptor_update_powervsx, sz_aes256_gcm_encryptor_digest_powervsx> { + message_bytes, chunk_bytes, pool, target}) + .log(base); #endif } } diff --git a/bench/container.cpp b/bench/container.cpp index ff138838..89499071 100644 --- a/bench/container.cpp +++ b/bench/container.cpp @@ -1,15 +1,13 @@ /** - * @file scripts/bench_container.cpp + * @file bench/container.cpp * @brief Benchmarks STL associative containers with @b `std::string_view`-compatible keys. * The program accepts a file path to a dataset, tokenizes it, and benchmarks the lookup operations. * - * This file is the sibling of `bench_sequence.cpp`, `bench_find.cpp` and `bench_token.cpp`. - * It accepts a file with a list of words, constructs associative containers with string keys, - * using `std::string`, `std::string_view`, `sz::string_view`, and `sz::string`, and then - * evaluates the latency of lookups. + * Memory-bound: associative build and probe are latency-limited over the whole key set, so it reads the whole file by default. * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=0` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -35,7 +33,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is the sibling of `bench_sequence.cpp`, `bench_token.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `sequence.cpp`, `find.cpp`, `token.cpp`, and `memory.cpp`. */ #include <map> // `std::map` #include <unordered_map> // `std::unordered_map` @@ -142,6 +140,12 @@ void bench_associative_lookups_with_different_simd_backends(environment_t const callable_for_associative_lookups<std::map<std::string_view, unsigned, less_from_sz<sz_order_haswell>>>(env); bench_unary(env, "map<sz_order_haswell>::find", callable_no_op_t(), callable_map, callable_map.preprocessor()) .log(base_map); + } +#endif + // There is no AVX2 hasher, so the fastest x86 pairing mixes a Westmere hash with a Haswell comparator - + // and needs both guards, as either family can be compiled out on its own. +#if SZ_USE_WESTMERE && SZ_USE_HASWELL + { auto callable_umap = callable_for_associative_lookups<std::unordered_map< std::string_view, unsigned, hash_from_sz<sz_hash_westmere>, equal_to_from_sz<sz_equal_haswell>>>(env); bench_unary(env, "unordered_map<sz_hash_westmere, sz_equal_haswell>::find", callable_no_op_t(), callable_umap, @@ -149,12 +153,18 @@ void bench_associative_lookups_with_different_simd_backends(environment_t const .log(base_umap); } #endif -#if SZ_USE_NEONAES + // The comparator and the hasher come from different families, so they carry different guards - one block + // under `SZ_USE_NEONAES` would drop the ordered map on a NEON target that ships no crypto extension. +#if SZ_USE_NEON { auto callable_map = callable_for_associative_lookups<std::map<std::string_view, unsigned, less_from_sz<sz_order_neon>>>(env); bench_unary(env, "map<sz_order_neon>::find", callable_no_op_t(), callable_map, callable_map.preprocessor()) .log(base_map); + } +#endif +#if SZ_USE_NEONAES + { auto callable_umap = callable_for_associative_lookups<std::unordered_map< std::string_view, unsigned, hash_from_sz<sz_hash_neonaes>, equal_to_from_sz<sz_equal_neon>>>(env); bench_unary(env, "unordered_map<sz_hash_neonaes, sz_equal_neon>::find", callable_no_op_t(), callable_umap, diff --git a/bench/find.cpp b/bench/find.cpp index 9f6147eb..5b2fb8a1 100644 --- a/bench/find.cpp +++ b/bench/find.cpp @@ -1,9 +1,11 @@ /** - * @file scripts/bench_find.cpp + * @file bench/find.cpp * @brief Benchmarks for bidirectional string search operations. * The program accepts a file path to a dataset, tokenizes it, and benchmarks the search operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Memory-bound: substring search is bandwidth-limited, so it reads the whole file by default and a larger haystack measures throughput truer; shrink the read with `STRINGWARS_DATASET_LIMIT` only when needed. + * * Benchmarks include: * - Substring search: find all inclusions of a token in the dataset - @b find & @b rfind. * - Byte search: find a specific byte value in each token (word, line, or file) - @b find_byte & @b rfind_byte. @@ -20,6 +22,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=0` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -50,7 +53,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is the sibling of `bench_sequence.cpp`, `bench_token.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `sequence.cpp`, `token.cpp`, and `memory.cpp`. */ #include <functional> // `std::boyer_moore_searcher` diff --git a/bench/fingerprints.cpp b/bench/fingerprints.cpp index 58da2bae..6cb460c3 100644 --- a/bench/fingerprints.cpp +++ b/bench/fingerprints.cpp @@ -1,11 +1,14 @@ /** - * @file scripts/bench_fingerprints.cpp - * @brief Benchmarks for exact multi-pattern substring search algorithms. - * The program accepts a file path to a dataset, tokenizes it, and benchmarks the search operations, - * validating the SIMD-accelerated backends against the serial baselines. + * @file bench/fingerprints.cpp + * @brief Benchmarks for rolling min-hash fingerprinting / sketching algorithms. + * The program accepts a file path to a dataset, tokenizes it, and benchmarks the fingerprinting + * engines, validating the SIMD-accelerated backends against the serial baselines. + * + * Compute-bound: min-hash sketching does many hashes per window, so a 64 MiB slice exercises every path. * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -36,7 +39,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is a sibling of `bench_similarities.cpp`. + * This file is a sibling of `similarities.cpp`. */ #include "fingerprints.cuh" #include "stringzilla.hpp" // `log_environment` @@ -54,7 +57,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "leipzig1M.txt", // - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting string fingerprinting benchmarks...\n"); bench_fingerprints(env); diff --git a/bench/fingerprints.cu b/bench/fingerprints.cu index e91a4f04..93e204a1 100644 --- a/bench/fingerprints.cu +++ b/bench/fingerprints.cu @@ -1,11 +1,14 @@ /** - * @file scripts/bench_fingerprints.cu + * @file bench/fingerprints.cu * @brief Benchmarks for exact multi-pattern substring search algorithms on the GPU. * The program accepts a file path to a dataset, tokenizes it, and benchmarks the search operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Compute-bound: min-hash sketching does many hashes per window, so a 64 MiB slice exercises every path. + * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -36,7 +39,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is a sibling of `bench_similarities.cpp`. + * This file is a sibling of `similarities.cpp`. */ #include "fingerprints.cuh" #include "stringzilla.hpp" // `log_environment` @@ -54,7 +57,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "leipzig1M.txt", // - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting string fingerprinting search benchmarks...\n"); bench_fingerprints(env); diff --git a/bench/fingerprints.cuh b/bench/fingerprints.cuh index 69733187..ba810cf3 100644 --- a/bench/fingerprints.cuh +++ b/bench/fingerprints.cuh @@ -1,6 +1,6 @@ /** - * @file scripts/bench_fingerprints.cuh - * @brief Shared code for CPU and GPU batched parallel exact substring search. + * @file bench/fingerprints.cuh + * @brief Shared code for CPU and GPU batched rolling min-hash fingerprinting / sketching. */ #include <tuple> // `std::tuple` #include <span> // `std::span` @@ -43,21 +43,7 @@ using fingerprint_min_counts_t = std::array<u32_t, default_embedding_dims_k>; using fingerprints_min_hashes_t = unified_vector<fingerprint_min_hashes_t>; using fingerprints_min_counts_t = unified_vector<fingerprint_min_counts_t>; -/** - * @brief Reads the device-measured `elapsed_milliseconds` out of a fingerprinting engine's return value. - * - * The CUDA fingerprinting backends return a `cuda_status_t` carrying CUDA-event timing of the launched - * kernels plus the drain; the CPU backends return a plain `sz::status_t` with no such field. The overloads - * below pick the right path at compile time and report a negative sentinel for engines that never measured - * the GPU, so the harness can contrast wall-clock against the kernel time only where it exists. - */ -template <typename status_type_> -float engine_gpu_milliseconds_(status_type_ const &engine_result) noexcept { - if constexpr (requires { engine_result.elapsed_milliseconds; }) return engine_result.elapsed_milliseconds; - else return -1.0f; -} - -#pragma region Multi-Pattern Search +#pragma region Rolling Fingerprinting /** @brief Wraps a hardware-specific fingerprinting backend into something @b `bench_nullary`-compatible. */ template <typename engine_type_, typename... extra_args_> @@ -125,6 +111,7 @@ void bench_fingerprints(environment_t const &env) { forkunion_executor_t pool; if (pool.try_spawn(std::thread::hardware_concurrency()) != status_t::success_k) throw std::runtime_error("Failed to spawn thread pool."); + cpu_specs_t const cpu_specs = pool.specs(); auto scramble_accelerated_results = [&]() { std::shuffle(min_hashes_accelerated.begin(), min_hashes_accelerated.end(), global_random_generator()); @@ -204,8 +191,9 @@ void bench_fingerprints(environment_t const &env) { #endif // SZ_USE_CUDA // Perform the benchmarks, passing the dictionary to the engines - auto basic_rolling_f64_serial_call = fingerprint_callable<basic_rolling_f64_serial_t, forkunion_executor_t &>( - tape, min_hashes_baseline, min_counts_baseline, *basic_rolling_f64_serial, pool); + auto basic_rolling_f64_serial_call = + fingerprint_callable<basic_rolling_f64_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_baseline, min_counts_baseline, *basic_rolling_f64_serial, pool, cpu_specs); bench_result_t basic_rolling_f64_serial_result = bench_nullary(env, "basic_rolling_f64_serial", basic_rolling_f64_serial_call).log(); @@ -213,32 +201,34 @@ void bench_fingerprints(environment_t const &env) { // Only the CUDA backend consumes this as its equality reference, so guard it to keep CPU-only builds free // of an unused-but-set variable under `-Werror`. #if SZ_USE_CUDA - auto basic_rabin_u64_serial_call = fingerprint_callable<basic_rabin_u64_serial_t, forkunion_executor_t &>( - tape, min_hashes_baseline, min_counts_baseline, *basic_rabin_u64_serial, pool); + auto basic_rabin_u64_serial_call = + fingerprint_callable<basic_rabin_u64_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_baseline, min_counts_baseline, *basic_rabin_u64_serial, pool, cpu_specs); #endif // SZ_USE_CUDA // Semi-serial variants bench_nullary(env, "basic_rolling_f32_serial", - fingerprint_callable<basic_rolling_f32_serial_t, forkunion_executor_t &>( - tape, min_hashes_accelerated, min_counts_accelerated, *basic_rolling_f32_serial, pool)) + fingerprint_callable<basic_rolling_f32_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_accelerated, min_counts_accelerated, *basic_rolling_f32_serial, pool, cpu_specs)) .log(basic_rolling_f64_serial_result); scramble_accelerated_results(); bench_nullary(env, "basic_rabin_u64_serial", - fingerprint_callable<basic_rabin_u64_serial_t, forkunion_executor_t &>( - tape, min_hashes_accelerated, min_counts_accelerated, *basic_rabin_u64_serial, pool)) + fingerprint_callable<basic_rabin_u64_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_accelerated, min_counts_accelerated, *basic_rabin_u64_serial, pool, cpu_specs)) .log(basic_rolling_f64_serial_result); scramble_accelerated_results(); bench_nullary(env, "basic_buz_u32_serial", - fingerprint_callable<basic_buz_u32_serial_t, forkunion_executor_t &>( - tape, min_hashes_accelerated, min_counts_accelerated, *basic_buz_u32_serial, pool)) // + fingerprint_callable<basic_buz_u32_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_accelerated, min_counts_accelerated, *basic_buz_u32_serial, pool, cpu_specs)) // .log(basic_rolling_f64_serial_result); scramble_accelerated_results(); - bench_nullary(env, "basic_multiply_u32_serial", - fingerprint_callable<basic_multiply_u32_serial_t, forkunion_executor_t &>( - tape, min_hashes_accelerated, min_counts_accelerated, *basic_multiply_u32_serial, pool)) + bench_nullary( + env, "basic_multiply_u32_serial", + fingerprint_callable<basic_multiply_u32_serial_t, forkunion_executor_t &, cpu_specs_t>( + tape, min_hashes_accelerated, min_counts_accelerated, *basic_multiply_u32_serial, pool, cpu_specs)) .log(basic_rolling_f64_serial_result); scramble_accelerated_results(); @@ -264,45 +254,45 @@ void bench_fingerprints(environment_t const &env) { #endif // SZ_USE_CUDA // Actually unrolled hard-coded variants, including SIMD ports - bench_result_t floating_serial_result = // - bench_nullary( // - env, "floating_serial", basic_rolling_f64_serial_call, // - fingerprint_callable<floating_serial_t, forkunion_executor_t &>( // - tape, min_hashes_accelerated, min_counts_accelerated, *floating_serial, pool), // - callable_no_op_t {}, // preprocessing - fingerprints_equality_t {}) // equality check + bench_result_t floating_serial_result = // + bench_nullary( // + env, "floating_serial", basic_rolling_f64_serial_call, // + fingerprint_callable<floating_serial_t, forkunion_executor_t &, cpu_specs_t>( // + tape, min_hashes_accelerated, min_counts_accelerated, *floating_serial, pool, cpu_specs), // + callable_no_op_t {}, // preprocessing + fingerprints_equality_t {}) // equality check .log(basic_rolling_f64_serial_result); scramble_accelerated_results(); #if SZ_USE_HASWELL - bench_nullary( // - env, "floating_haswell", basic_rolling_f64_serial_call, // - fingerprint_callable<floating_haswell_t, forkunion_executor_t &>( // - tape, min_hashes_accelerated, min_counts_accelerated, *floating_haswell, pool), // - callable_no_op_t {}, // preprocessing - fingerprints_equality_t {}) // equality check + bench_nullary( // + env, "floating_haswell", basic_rolling_f64_serial_call, // + fingerprint_callable<floating_haswell_t, forkunion_executor_t &, cpu_specs_t>( // + tape, min_hashes_accelerated, min_counts_accelerated, *floating_haswell, pool, cpu_specs), // + callable_no_op_t {}, // preprocessing + fingerprints_equality_t {}) // equality check .log(basic_rolling_f64_serial_result, floating_serial_result); scramble_accelerated_results(); #endif // SZ_USE_HASWELL #if SZ_USE_SKYLAKE - bench_nullary( // - env, "floating_skylake", basic_rolling_f64_serial_call, // - fingerprint_callable<floating_skylake_t, forkunion_executor_t &>( // - tape, min_hashes_accelerated, min_counts_accelerated, *floating_skylake, pool), // - callable_no_op_t {}, // preprocessing - fingerprints_equality_t {}) // equality check + bench_nullary( // + env, "floating_skylake", basic_rolling_f64_serial_call, // + fingerprint_callable<floating_skylake_t, forkunion_executor_t &, cpu_specs_t>( // + tape, min_hashes_accelerated, min_counts_accelerated, *floating_skylake, pool, cpu_specs), // + callable_no_op_t {}, // preprocessing + fingerprints_equality_t {}) // equality check .log(basic_rolling_f64_serial_result, floating_serial_result); scramble_accelerated_results(); #endif // SZ_USE_SKYLAKE #if SZ_USE_NEON - bench_nullary( // - env, "floating_neon", basic_rolling_f64_serial_call, // - fingerprint_callable<floating_neon_t, forkunion_executor_t &>( // - tape, min_hashes_accelerated, min_counts_accelerated, *floating_neon, pool), // - callable_no_op_t {}, // preprocessing - fingerprints_equality_t {}) // equality check + bench_nullary( // + env, "floating_neon", basic_rolling_f64_serial_call, // + fingerprint_callable<floating_neon_t, forkunion_executor_t &, cpu_specs_t>( // + tape, min_hashes_accelerated, min_counts_accelerated, *floating_neon, pool, cpu_specs), // + callable_no_op_t {}, // preprocessing + fingerprints_equality_t {}) // equality check .log(basic_rolling_f64_serial_result, floating_serial_result); scramble_accelerated_results(); #endif // SZ_USE_NEON @@ -319,7 +309,7 @@ void bench_fingerprints(environment_t const &env) { #endif // SZ_USE_CUDA } -#pragma endregion +#pragma endregion // Rolling Fingerprinting } // namespace scripts } // namespace stringzilla diff --git a/bench/memory.cpp b/bench/memory.cpp index 404ae4bc..7c671480 100644 --- a/bench/memory.cpp +++ b/bench/memory.cpp @@ -1,12 +1,15 @@ /** - * @file scripts/bench_memory.cpp + * @file bench/memory.cpp * @brief Benchmarks for memory operations like copying, moving, resetting, and converting with lookup tables. * The program accepts a file path to a dataset, tokenizes it, and uses those tokens only for size * references to mimic real-world scenarios dealing with individual strings of different lengths. * + * Memory-bound: the copy, move, and fill primitives are pure bandwidth, so it reads the whole file by default and a larger buffer measures throughput truer. + * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. - * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams + * - `STRINGWARS_DATASET_LIMIT=0` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * * Unlike StringWars, the following additional environment variables are supported: @@ -36,7 +39,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is the sibling of `bench_find.cpp`, `bench_token.cpp`, and `bench_sequence.cpp`. + * This file is the sibling of `find.cpp`, `token.cpp`, and `sequence.cpp`. */ #include <memory> // `std::unique_ptr` #include <numeric> // `std::iota` @@ -299,10 +302,14 @@ struct fill_random_from_sz { void memset_like_sz(sz_ptr_t output, sz_size_t length, sz_u8_t value) { std::memset(output, value, length); } -void generate_like_sz(sz_ptr_t output, sz_size_t length, sz_u64_t nonce) { +/** + * @brief The `std::` baseline for `sz_generate`, measuring generator throughput alone. + * Reseeding per call would put a Mersenne-Twister state fill inside the measured loop, so the nonce + * is dropped and this baseline is not byte-reproducible the way the `sz_generate` kernels are. + */ +void generate_like_sz(sz_ptr_t output, sz_size_t length, [[maybe_unused]] sz_u64_t nonce) { uniform_u8_distribution_t distribution; std::generate(output, output + length, [&]() -> char { return distribution(global_random_generator()); }); - sz_unused_(nonce); } /** @@ -355,6 +362,10 @@ void bench_fill(environment_t const &env) { #if SZ_USE_SVE bench_unary(env, "sz_fill_sve", fill_from_sz<sz_fill_sve> {env, o}).log(zeros); #endif +#if SZ_USE_SVE2AES + bench_unary(env, "sz_fill_random_sve2aes", random_call, fill_random_from_sz<sz_fill_random_sve2aes> {env, o}) + .log(zeros, random); +#endif #if SZ_USE_V128 bench_unary(env, "sz_fill_v128", fill_from_sz<sz_fill_v128> {env, o}).log(zeros); bench_unary(env, "sz_fill_random_v128", random_call, fill_random_from_sz<sz_fill_random_v128> {env, o}) @@ -389,7 +400,7 @@ void bench_fill(environment_t const &env) { #pragma region Lookup Transformations -/** @brief Wraps a hardware-specific @b `memset`-like backend into something compatible with @b `bench_unary`. */ +/** @brief Wraps a hardware-specific lookup-table backend into something similar to @b `std::transform`. */ template <sz_lookup_t lookup_func_> struct lookup_from_sz { diff --git a/bench/sequence.cpp b/bench/sequence.cpp index 24f36efc..cd4cb317 100644 --- a/bench/sequence.cpp +++ b/bench/sequence.cpp @@ -1,10 +1,12 @@ /** - * @file scripts/bench_sequence.cpp + * @file bench/sequence.cpp * @brief Benchmarks sorting, partitioning, and merging operations on string sequences. * The program accepts a file path to a dataset, tokenizes it, and benchmarks the search operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Memory-bound: sort cost is dominated by cache-missing permutation over the whole collection, so it reads the whole file by default. + * * Benchmarks include: * - String sequence sorting algorithms - @b argsort and @b pgrams_sort. * - String sequences intersections - @b intersect. @@ -15,6 +17,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=0` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -45,7 +48,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is the sibling of `bench_find.cpp`, `bench_token.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `find.cpp`, `token.cpp`, and `memory.cpp`. */ #include <memory> // `std::memcpy` #include <numeric> // `std::iota` diff --git a/bench/shared.hpp b/bench/shared.hpp index 04e9f456..e08d641c 100644 --- a/bench/shared.hpp +++ b/bench/shared.hpp @@ -1,5 +1,5 @@ /** - * @file scripts/bench.hpp + * @file bench/shared.hpp * @brief Helper structures and functions for C++ benchmarks. * * The StringZilla benchmarking suite doesn't use any external frameworks like Criterion or Google Benchmark. @@ -37,17 +37,18 @@ #include <cstring> // `std::memcpy` #include <algorithm> -#include <chrono> // `std::chrono::high_resolution_clock` -#include <exception> // `std::invalid_argument` -#include <functional> // `std::equal_to` -#include <limits> // `std::numeric_limits` -#include <numeric> // `std::accumulate` -#include <random> // `std::random_device`, `std::mt19937` -#include <string> // `std::hash` -#include <vector> // `std::vector` -#include <regex> // `std::regex`, `std::regex_search` -#include <thread> // `std::this_thread::sleep_for` -#include <optional> // `std::optional` +#include <chrono> // `std::chrono::high_resolution_clock` +#include <exception> // `std::invalid_argument` +#include <functional> // `std::equal_to` +#include <limits> // `std::numeric_limits` +#include <numeric> // `std::accumulate` +#include <optional> // `std::optional` +#include <random> // `std::random_device`, `std::mt19937` +#include <regex> // `std::regex`, `std::regex_search` +#include <string> // `std::hash` +#include <thread> // `std::this_thread::sleep_for`, `std::thread::hardware_concurrency` +#include <type_traits> // `std::invoke_result_t` +#include <vector> // `std::vector` #include <string_view> // Requires C++17 #include <span> // Requires C++20, used to pass info to batch-capable parallel backends @@ -137,14 +138,6 @@ inline std::uint64_t cpu_cycle_counter() { #endif } -/** @brief Measures the approximate number of CPU cycles per second. */ -inline std::uint64_t cpu_cycles_per_second() { - std::uint64_t start = cpu_cycle_counter(); - std::this_thread::sleep_for(stdc::seconds(1)); - std::uint64_t end = cpu_cycle_counter(); - return end - start; -} - /** @brief Measures the duration of a single call to the given function. */ template <typename function_type_> double seconds_per_call(function_type_ &&function) { @@ -212,6 +205,49 @@ static void do_not_optimize(argument_type_ &&value) noexcept { #endif } +/** @brief The device-measured kernel time an engine reports, in milliseconds, or @b 0 for CPU engines, whose + * plain `status_t` carries no such timer. */ +template <typename status_type_, typename = void> +struct engine_elapsed_milliseconds_trait { + static float read(status_type_ const &) noexcept { return 0.0f; } +}; +template <typename status_type_> +struct engine_elapsed_milliseconds_trait<status_type_, + decltype((void)std::declval<status_type_ const &>().elapsed_milliseconds)> { + static float read(status_type_ const &materialized) noexcept { return materialized.elapsed_milliseconds; } +}; + +/** @brief A status paired with the device-measured kernel time, both read out of one materialized copy. */ +struct engine_timing_t { + sz::status_t status = sz::status_t::success_k; + float kernel_milliseconds = 0.0f; +}; + +/** + * @brief Calls @p invocable and decomposes its returned status through opaque memory, in one non-inlined + * frame. Every timed engine call in the suite goes through here. + * + * Both halves must stay in this frame. Compilers miscompile the engines' return-by-value inside these + * large translation units at @b -O2 : NVCC 12.x corrupts a `cuda_status_t`'s two leading enum fields + * while its `float elapsed_milliseconds` survives, and g++ trunk corrupts the status when it folds the + * return into an inlined caller. The libraries are correct - a separate-TU call returns `success_k`. + * Sharing one `[[gnu::noinline]]` frame reproduces that separate-TU code path. + */ +template <typename invocable_type_> +SZ_NOINLINE engine_timing_t invoke_engine_(invocable_type_ &&invocable) noexcept { + using status_t = std::invoke_result_t<invocable_type_>; + status_t engine_result = invocable(); + do_not_optimize(engine_result); + + status_t materialized; + std::memcpy((void *)&materialized, (void const *)&engine_result, sizeof(status_t)); + + engine_timing_t timing; + timing.status = static_cast<sz::status_t>(materialized); + timing.kernel_milliseconds = engine_elapsed_milliseconds_trait<status_t>::read(materialized); + return timing; +} + /** * @brief Rounds the number @b down to the preceding power of two. * @see Equivalent to `std::bit_floor`: https://en.cppreference.com/w/cpp/numeric/bit_floor @@ -223,6 +259,32 @@ inline std::size_t bit_floor(std::size_t n) { return static_cast<std::size_t>(1) << most_significant_bit_position; } +/** + * @brief Parses a human byte size like `64mb` or `1gb` into bytes; `kb`/`mb`/`gb` are powers of 1024, + * matching StringWars' `parse_size`. A bare number is bytes, and an empty or zero value is the whole file. + */ +inline std::size_t parse_size(std::string const &text) { + std::size_t cursor = 0; + while (cursor < text.size() && (std::isdigit((unsigned char)text[cursor]) || text[cursor] == '.')) ++cursor; + double const number = cursor == 0 ? 0.0 : std::stod(text.substr(0, cursor)); + std::string unit; + for (; cursor < text.size(); ++cursor) + if (!std::isspace((unsigned char)text[cursor])) unit.push_back((char)std::tolower((unsigned char)text[cursor])); + std::size_t multiplier = 1; + if (unit.empty() || unit == "b") multiplier = 1; + else if (unit == "kb") multiplier = 1024ull; + else if (unit == "mb") multiplier = 1024ull * 1024ull; + else if (unit == "gb") multiplier = 1024ull * 1024ull * 1024ull; + else throw std::invalid_argument("Invalid STRINGWARS_DATASET_LIMIT unit: " + unit); + return static_cast<std::size_t>(number * (double)multiplier); +} + +/** + * @brief The smallest read that still exercises a compute-bound bench's control-flow paths on the + * multilingual corpus. Memory-bound benches ignore it and read the whole file. + */ +static constexpr std::size_t compute_bound_slice_bytes_k = 64ull * 1024ull * 1024ull; + #if !SZ_USE_CUDA using dataset_t = std::string; using token_view_t = std::string_view; @@ -259,11 +321,24 @@ tokens_t tokenize(std::string_view str, is_separator_callback_type_ &&is_separat return tokens; } -/** @brief Splits a string into words, using newlines, tabs, and whitespaces as delimiters using @b `std::isspace`. */ -inline tokens_t tokenize(std::string_view str) { - return tokenize(str, [](char c) { return std::isspace(c); }); +/** + * @brief Tokenizes a string around the given separator @p byteset in one lazy SIMD pass. + * + * Each step of the underlying `split` issues one `sz_find_byteset` scan. The whole corpus is already + * bounded by the dataset read, so the walk runs to the end rather than carrying its own cap. + */ +inline tokens_t tokenize(std::string_view str, sz::byteset separators) { + tokens_t tokens; + for (auto token : sz::string_view {str.data(), str.size()}.split(separators)) { + if (token.size() == 0) continue; // ? Runs of separators yield empty segments + tokens.push_back({token.data(), token.size()}); + } + return tokens; } +/** @brief Splits a string into words around newlines, tabs, and other ASCII whitespaces. */ +inline tokens_t tokenize(std::string_view str) { return tokenize(str, sz::whitespaces_set()); } + template <typename result_string_type_ = std::string_view, typename from_string_type_ = result_string_type_, typename comparator_type_ = std::equal_to<std::size_t>, typename allocator_type_ = std::allocator<char>> std::vector<result_string_type_, allocator_type_> filter_by_length( @@ -315,11 +390,10 @@ struct environment_t { std::size_t stress_limit = 1; /** @brief Whether to deduplicate tokens before benchmarking. */ bool unique = false; - /** @brief Optional cap on the number of tokens kept, 0 means unlimited; `STRINGWARS_MAX_TOKENS`. */ - std::size_t max_tokens = 0; + /** @brief Read at most this many dataset bytes, 0 means the whole file; `STRINGWARS_DATASET_LIMIT`. */ + std::size_t dataset_limit_bytes = 0; /** @brief Optional override for per-benchmark batch sizes, empty means the backend default; `STRINGWARS_BATCH`. */ std::vector<std::size_t> batch_sizes_override; - /** @brief Textual content of the dataset file, fully loaded into memory. */ dataset_t dataset; /** @brief Array of tokens extracted from the @p dataset. */ @@ -335,6 +409,9 @@ struct environment_t { } }; +/** @brief Whether `build_environment` stress-tests the backends absent `STRINGWARS_STRESS`. */ +enum class stress_default_t : bool { quick_k, stress_k }; + /** * @brief Prepares the environment for benchmarking based on environment variables and default settings. * It's expected that different workloads may use different default datasets and tokenization modes, @@ -358,8 +435,9 @@ struct environment_t { inline environment_t build_environment( // int argc, char const *argv[], //< Ignored std::string default_dataset, environment_t::tokenization_t default_tokens, //< Mandatory + std::size_t default_dataset_limit_bytes = 0, //< Optional, 0 = whole file std::size_t default_duration = SZ_DEBUG ? 1 : 10, //< Optional - bool default_stress = true, // + stress_default_t default_stress = stress_default_t::stress_k, // std::string default_stress_dir = ".tmp", // std::size_t default_stress_limit = 1, // std::size_t default_stress_duration = SZ_DEBUG ? 1 : 10, // @@ -369,6 +447,7 @@ inline environment_t build_environment( / sz_unused_(argc && argv); // Unused in this context environment_t env; + env.dataset_limit_bytes = default_dataset_limit_bytes; // Use `STRINGWARS_DATASET` if set, otherwise `default_dataset` if (char const *env_var = std::getenv("STRINGWARS_DATASET")) { env.path = env_var; } @@ -403,7 +482,7 @@ inline environment_t build_environment( / env.tokenization = static_cast<environment_t::tokenization_t>(std::stoul(token_arg)); if (env.tokenization == 0) throw std::invalid_argument( - "The tokenization mode must be 'file', 'line', 'word', or a positive integer."); + "The tokenization mode must be 'file', 'lines', 'words', or a positive integer."); } } else { env.tokenization = default_tokens; } @@ -415,7 +494,7 @@ inline environment_t build_environment( / env.stress = is_one; if (!is_zero && !is_one) throw std::invalid_argument("The stress-testing flag must be '0' or '1'."); } - else { env.stress = default_stress; } + else { env.stress = default_stress == stress_default_t::stress_k; } if (char const *env_var = std::getenv("STRINGWARS_STRESS_DURATION")) { env.stress_seconds = std::stoul(env_var); if (env.stress_seconds == 0) @@ -430,17 +509,21 @@ inline environment_t build_environment( / } else { env.stress_limit = default_stress_limit; } - // Use `STRINGWARS_UNIQUE` to deduplicate tokens + // Use `STRINGWARS_UNIQUE` to deduplicate tokens. + // @sa `STRINGWARS_UNIQUE=1` sorts the tokenized set and drops duplicates before benchmarking. if (char const *env_var = std::getenv("STRINGWARS_UNIQUE")) { bool is_one = std::strcmp(env_var, "1") == 0 || std::strcmp(env_var, "true") == 0; env.unique = is_one; } - // Use `STRINGWARS_MAX_TOKENS` to cap the number of tokens kept, for faster and more targeted runs. - if (char const *env_var = std::getenv("STRINGWARS_MAX_TOKENS")) { env.max_tokens = std::stoull(env_var); } + // Use `STRINGWARS_DATASET_LIMIT` to bound the dataset read, so the file tail is never touched. + if (char const *env_var = std::getenv("STRINGWARS_DATASET_LIMIT")) { + env.dataset_limit_bytes = parse_size(env_var); + } // Use `STRINGWARS_BATCH` to override the per-benchmark batch sizes with a comma-separated list, // e.g. `STRINGWARS_BATCH=1024` to run a single batch and skip the slow/largest default sweep entries. + // @sa `STRINGWARS_BATCH=1024,4096` replaces the default batch-size sweep with exactly these sizes. if (char const *env_var = std::getenv("STRINGWARS_BATCH")) { std::string const batch_argument = env_var; for (std::size_t start = 0; start < batch_argument.size();) { @@ -451,14 +534,13 @@ inline environment_t build_environment( / } } - env.dataset = read_file(env.path); - env.dataset.resize(bit_floor(env.dataset.size())); // Shrink to the nearest power of two + env.dataset = read_file(env.path, env.dataset_limit_bytes); // A non-zero limit stops the read early + env.dataset.resize(bit_floor(env.dataset.size())); // Shrink to the nearest power of two - // Tokenize the dataset according to the tokenization mode + // Tokenize the dataset according to the tokenization mode. The corpus is already bounded by the read, + // so each mode walks it to the end. if (env.tokenization == environment_t::file_k) { env.tokens.push_back({env.dataset.data(), env.dataset.size()}); } - else if (env.tokenization == environment_t::lines_k) { - env.tokens = tokenize(env.dataset, [](char c) { return c == '\n'; }); - } + else if (env.tokenization == environment_t::lines_k) { env.tokens = tokenize(env.dataset, sz::byteset {'\n'}); } else if (env.tokenization == environment_t::words_k) { env.tokens = tokenize(env.dataset); } else { std::size_t n = static_cast<std::size_t>(env.tokenization); @@ -472,8 +554,6 @@ inline environment_t build_environment( / env.tokens.erase(last, env.tokens.end()); } - // Optionally cap the token count before the power-of-two shrink, for faster and more targeted runs. - if (env.max_tokens != 0 && env.tokens.size() > env.max_tokens) env.tokens.resize(env.max_tokens); env.tokens.resize(bit_floor(env.tokens.size())); // Shrink to the nearest power of two // In "RELEASE" mode, shuffle tokens to avoid bias. @@ -509,6 +589,8 @@ inline environment_t build_environment( / std::printf(" - Stress-testing: %s\n", env.stress ? "yes" : "no"); std::printf(" - Unique tokens: %s\n", env.unique ? "yes" : "no"); std::printf(" - Loaded dataset size: %zu bytes\n", env.dataset.size()); + if (env.dataset_limit_bytes == 0) std::printf(" - Dataset limit: whole file\n"); + else std::printf(" - Dataset limit: %zu bytes\n", env.dataset_limit_bytes); std::printf(" - Number of tokens: %zu\n", env.tokens.size()); std::printf(" - Mean token length: %.2f bytes\n", mean_token_length); @@ -617,6 +699,9 @@ struct bench_result_t { duration_histogram_t cpu_cycles_histogram; + /** @brief Cheapest single call observed, in CPU cycles; a less noisy cost estimator than the mean. */ + std::uint64_t profiled_cpu_cycles_min = std::numeric_limits<std::uint64_t>::max(); + std::size_t bytes_passed = 0; //< Pulled from the `call_result_t` std::size_t operations = 0; //< Pulled from the `call_result_t` std::size_t errors = 0; //< Pulled from the `call_result_t` @@ -630,6 +715,7 @@ struct bench_result_t { * @code{.unparsed} * Benchmarking `sz_find_skylake`: * > Throughput: 0.00 TB/s @ 0.00 ns/call + * > Latency: min 0.00 ns/call, p99 0.00 ns/call * > Efficiency: 0.00 TOps/s @ 0.00 ops/cycle * > Errors: 0 in 10 calls * > + 3.5 x against `sz_find_serial` @@ -682,6 +768,29 @@ struct bench_result_t { bytes_printable, bytes_printable_unit, // seconds_printable, seconds_printable_unit); + // Scheduler interference only slows a call down, so the minimum is a less noisy estimator than the + // mean above, and the 99th percentile shows whether outliers drag that mean up. Both come from the + // CPU-cycle histogram, rescaled by this run's average seconds-per-cycle ratio. + if (profiled_cpu_cycles > 0 && profiled_cpu_cycles_min != std::numeric_limits<std::uint64_t>::max()) { + double const seconds_per_cycle = profiled_seconds / profiled_cpu_cycles; + + auto minimum_printable = profiled_cpu_cycles_min * seconds_per_cycle * 1e9; + char const *minimum_printable_unit = "ns"; + if (minimum_printable > 1e3) minimum_printable /= 1e3, minimum_printable_unit = "us"; + if (minimum_printable > 1e3) minimum_printable /= 1e3, minimum_printable_unit = "ms"; + if (minimum_printable > 1e3) minimum_printable /= 1e3, minimum_printable_unit = "s"; + + auto p99_printable = cpu_cycles_histogram.percentile(0.99) * seconds_per_cycle * 1e9; + char const *p99_printable_unit = "ns"; + if (p99_printable > 1e3) p99_printable /= 1e3, p99_printable_unit = "us"; + if (p99_printable > 1e3) p99_printable /= 1e3, p99_printable_unit = "ms"; + if (p99_printable > 1e3) p99_printable /= 1e3, p99_printable_unit = "s"; + + std::printf("> Latency: min %.2f %s/call, p99 %.2f %s/call\n", // + minimum_printable, minimum_printable_unit, // + p99_printable, p99_printable_unit); + } + // Print the number of operations, if there was a separate tracking mechanism for those. if (operations) { auto ops_printable = operations * 1.0 / profiled_seconds; @@ -714,7 +823,7 @@ struct bench_result_t { }; // Expand over all provided baselines. - (void)std::initializer_list<int> {(log_relative(bases), 0)...}; + [[maybe_unused]] std::initializer_list<int> const expanded {(log_relative(bases), 0)...}; sz_unused_(log_relative); // In case no `bases` were provided return *this; @@ -784,13 +893,15 @@ bench_result_t bench_nullary( // std::uint64_t cpu_cycles_at_start = cpu_cycle_counter(); call_result_t call_result = callable(); std::uint64_t cpu_cycles_at_end = cpu_cycle_counter(); + std::uint64_t const cpu_cycles_spent = cpu_cycles_at_end - cpu_cycles_at_start; // Aggregate: result.operations += call_result.operations; result.bytes_passed += call_result.bytes_passed; result.profiled_inputs += call_result.inputs_processed; - result.profiled_cpu_cycles += cpu_cycles_at_end - cpu_cycles_at_start; - result.cpu_cycles_histogram[static_cast<double>(cpu_cycles_at_end - cpu_cycles_at_start)] += 1; + result.profiled_cpu_cycles += cpu_cycles_spent; + result.profiled_cpu_cycles_min = std::min(result.profiled_cpu_cycles_min, cpu_cycles_spent); + result.cpu_cycles_histogram[static_cast<double>(cpu_cycles_spent)] += 1; } result.profiled_calls += repeat.count(); result.profiled_seconds = repeat.seconds(); @@ -854,22 +965,28 @@ bench_result_t bench_unary( // } } - // For profiling, we will first run the benchmark just once to get a rough estimate of the time. - // But then we will repeat it in an unrolled fashion for a more accurate measurement. + // Run once to estimate the per-call duration and size the unrolled loop below. This call pays cold-cache + // and branch-predictor costs a steady-state call does not, so it stays out of the reported statistics. + call_result_t warm_up_result; + std::uint64_t warm_up_cpu_cycles = 0; auto const first_call_duration = seconds_per_call([&] { std::uint64_t cpu_cycles_at_start = cpu_cycle_counter(); - call_result_t const call_result = callable((std::size_t)0); //? Use the first token + warm_up_result = callable((std::size_t)0); //? Use the first token std::uint64_t cpu_cycles_at_end = cpu_cycle_counter(); - - result.operations += call_result.operations; - result.bytes_passed += call_result.bytes_passed; - result.profiled_inputs += call_result.inputs_processed; - result.profiled_calls += 1; - result.profiled_cpu_cycles += cpu_cycles_at_end - cpu_cycles_at_start; - result.cpu_cycles_histogram[static_cast<double>(cpu_cycles_at_end - cpu_cycles_at_start)] += 1; + warm_up_cpu_cycles = cpu_cycles_at_end - cpu_cycles_at_start; }); - result.profiled_seconds = first_call_duration; - if (first_call_duration >= env.benchmark_seconds) return result; + if (first_call_duration >= env.benchmark_seconds) { + // No budget remains for a second, uncontaminated sample, so the cold warm-up call is reported as-is. + result.operations += warm_up_result.operations; + result.bytes_passed += warm_up_result.bytes_passed; + result.profiled_inputs += warm_up_result.inputs_processed; + result.profiled_calls += 1; + result.profiled_cpu_cycles += warm_up_cpu_cycles; + result.profiled_cpu_cycles_min = warm_up_cpu_cycles; + result.cpu_cycles_histogram[static_cast<double>(warm_up_cpu_cycles)] += 1; + result.profiled_seconds = first_call_duration; + return result; + } // Repeat the benchmarks in unrolled batches of `unroll_factor` until the time limit is reached. constexpr std::size_t unroll_factor = 8; @@ -901,11 +1018,13 @@ bench_result_t bench_unary( // result.profiled_inputs += r4.inputs_processed, result.profiled_inputs += r5.inputs_processed, // result.profiled_inputs += r6.inputs_processed, result.profiled_inputs += r7.inputs_processed; // - result.profiled_cpu_cycles += t7 - t0; - result.cpu_cycles_histogram[static_cast<double>(t7 - t0)] += unroll_factor; + std::uint64_t const batch_cpu_cycles = t7 - t0; + result.profiled_cpu_cycles += batch_cpu_cycles; + result.profiled_cpu_cycles_min = std::min(result.profiled_cpu_cycles_min, batch_cpu_cycles / unroll_factor); + result.cpu_cycles_histogram[static_cast<double>(batch_cpu_cycles)] += unroll_factor; } result.profiled_calls += repeat.count() * unroll_factor; - result.profiled_seconds = repeat.seconds() + first_call_duration; + result.profiled_seconds = repeat.seconds(); return result; } diff --git a/bench/similarities.cpp b/bench/similarities.cpp index 656eae14..931490f4 100644 --- a/bench/similarities.cpp +++ b/bench/similarities.cpp @@ -1,11 +1,12 @@ /** - * @file scripts/bench_similarities.cpp + * @file bench/similarities.cpp * @brief Benchmarks string similarity computations. - * It accepts a file with a list of words, and benchmarks the levenshtein edit-distance computations, - * alignment scores, and fingerprinting techniques combined with the Hamming distance. + * It accepts a file with a list of words, and benchmarks the levenshtein edit-distance computations + * and alignment scores. + * + * Compute-bound: the O(N*M) alignment is arithmetic-limited, so a 64 MiB slice exercises every path while the batch samples only what it needs. * * Benchmarks include: - * - Linear-complexity basic & bounded Hamming distance computations. * - Quadratic-complexity basic & bounded Levenshtein edit-distance computations. * - Quadratic-complexity Needleman-Wunsch alignment scores for bioinformatics. * @@ -15,11 +16,13 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. - * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * - `STRINGWARS_BATCH_PER_CORE=256` : Pairs scored per core; a CPU core and a GPU streaming-multiprocessor each * count as one core, so the per-device pair budget is `STRINGWARS_BATCH_PER_CORE * cores` and each * cross-product axis (queries, candidates) is its square root. + * - `STRINGWARS_BATCH` : Explicit per-query pair count that overrides the derived batch sizing above, when set. * * Unlike StringWars, the following additional environment variables are supported: * - `STRINGWARS_DURATION=10` : Time limit (in seconds) per benchmark. @@ -48,7 +51,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is a sibling of `bench_fingerprints.cpp`. + * This file is a sibling of `fingerprints.cpp`. */ #include "similarities.cuh" #include "stringzilla.hpp" // `log_environment` @@ -66,7 +69,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Preferred for UTF-8 content - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting string similarity benchmarks...\n"); bench_levenshtein(env); diff --git a/bench/similarities.cu b/bench/similarities.cu index 01acb3f6..05d4e01a 100644 --- a/bench/similarities.cu +++ b/bench/similarities.cu @@ -1,9 +1,11 @@ /** - * @file scripts/bench_similarities.cu + * @file bench/similarities.cu * @brief Benchmarks string similarity computations. * It accepts a file with a list of words, and benchmarks the levenshtein edit-distance computations, * alignment scores, and fingerprinting techniques combined with the Hamming distance. * + * Compute-bound: the O(N*M) alignment is arithmetic-limited, so a 64 MiB slice exercises every path while the batch samples only what it needs. + * * Benchmarks include: * - Linear-complexity basic & bounded Hamming distance computations. * - Quadratic-complexity basic & bounded Levenshtein edit-distance computations. @@ -15,6 +17,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=words` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -45,7 +48,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is a sibling of `bench_fingerprints.cpp`. + * This file is a sibling of `fingerprints.cpp`. */ #include "similarities.cuh" #include "stringzilla.hpp" // `log_environment` @@ -63,7 +66,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Preferred for UTF-8 content - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting string similarity benchmarks...\n"); bench_levenshtein(env); diff --git a/bench/similarities.cuh b/bench/similarities.cuh index 4f3724c4..ffe41ff4 100644 --- a/bench/similarities.cuh +++ b/bench/similarities.cuh @@ -1,5 +1,5 @@ /** - * @file scripts/bench_similarities.cuh + * @file bench/similarities.cuh * @brief Shared code for CPU and GPU batched string similarity kernels. */ #include <tuple> // `std::tuple` @@ -18,6 +18,7 @@ namespace stringzilla { namespace scripts { // StringZillas library symbols available on every backend: +using ashvardanian::stringzillas::dummy_executor_t; using ashvardanian::stringzillas::forkunion_executor_t; using ashvardanian::stringzillas::affine_gap_costs_t; using ashvardanian::stringzillas::affine_levenshtein_icelake_t; @@ -82,70 +83,6 @@ using namespace ashvardanian::stringzilla::scripts; using similarities_t = unified_vector<sz_ssize_t>; -/** - * @brief The device-measured kernel time that an engine reports, in milliseconds, or @b 0 for CPU engines. - * - * GPU engines return a `cuda_status_t` carrying a `float elapsed_milliseconds` measured on the device with - * CUDA events, excluding host-to-device materialization. CPU engines return a plain `status_t` with no such - * timer, so the trait below resolves to @b 0 for them, keeping a single timed call path for every backend. - */ -template <typename status_type_, typename = void> -struct engine_elapsed_milliseconds_trait { - static float read(status_type_ const &) noexcept { return 0.0f; } -}; -template <typename status_type_> -struct engine_elapsed_milliseconds_trait<status_type_, - decltype((void)std::declval<status_type_ const &>().elapsed_milliseconds)> { - static float read(status_type_ const &materialized) noexcept { return materialized.elapsed_milliseconds; } -}; - -/** @brief A status paired with the device-measured kernel time, both surfaced through one materialized read. */ -struct engine_timing_t { - status_t status = status_t::success_k; - float kernel_milliseconds = 0.0f; -}; - -/** - * @brief Reads the `status_t` @b and the device-measured `elapsed_milliseconds` out of an engine's return - * value through opaque memory, defeating NVCC's host-codegen folding of the read. - * - * @note NVCC 12.x miscompiles the `cuda_status_t` return-by-value of the @b affine alignment engine - * instantiations inside this large translation unit at @b -O2 : the `float elapsed_milliseconds` - * field survives, but the two leading enum fields (`status`, `cuda_error`) come back as garbage, so a - * direct `static_cast<status_t>(result)` reports a bogus `unrecognized` status. The StringZillas - * library itself is correct - a direct call to the same engine returns `success_k` - this only bites - * the affine instantiations folded into this benchmark. Bouncing the struct through `std::memcpy` in a - * `[[gnu::noinline]]` helper forces the compiler to materialize the full object before reading both - * fields, so the kernel time is pulled from the same defended copy as the status. - */ -template <typename status_type_> -SZ_NOINLINE engine_timing_t read_engine_timing_(status_type_ const &engine_result) noexcept { - status_type_ materialized; - std::memcpy((void *)&materialized, (void const *)&engine_result, sizeof(status_type_)); - engine_timing_t timing; - timing.status = static_cast<status_t>(materialized); - timing.kernel_milliseconds = engine_elapsed_milliseconds_trait<status_type_>::read(materialized); - return timing; -} - -/** - * @brief Calls an engine and reads its status @b and kernel time entirely inside one @b `[[gnu::noinline]]` frame. - * - * @note `read_engine_timing_` alone is not enough: when the optimizer folds the engine's return-by-value - * into the caller it can corrupt the status @b before the read runs (seen with g++ trunk on the Ice - * Lake instantiations in this large TU, and with NVCC 12.x on the affine `cuda_status_t`). Performing - * the call and the read together in a non-inlined frame reproduces the same code path as a separate-TU - * call, which the library validates as correct in isolation. - */ -template <typename engine_type_, typename queries_type_, typename candidates_type_, typename... rest_types_> -SZ_NOINLINE engine_timing_t invoke_engine_timed_(engine_type_ &engine, queries_type_ const &queries, - candidates_type_ const &candidates, strided_rows<sz_ssize_t> results, - rest_types_ &...rest) noexcept { - auto status = engine(queries, candidates, results, rest...); - do_not_optimize(status); - return read_engine_timing_(status); -} - #pragma region Levenshtein Distance and Alignment Scores /** @@ -153,7 +90,7 @@ SZ_NOINLINE engine_timing_t invoke_engine_timed_(engine_type_ &engine, queries_t * * @b all_pairs : score every query against every candidate, a full `queries x candidates` cross-product tile. * This is the path that exercises the inter-/intra-sequence tiling and lane packing of the SIMD/GPU backends, - * so the two dimensions are scaled @b independently (the candidate axis is no longer capped at a constant). + * so the two dimensions are scaled @b independently. * * @b pairwise : score only the `min(queries, candidates)` diagonal pairs `(query_i, candidate_i)`, one engine * call per pair. This mirrors how a per-pair library is driven, making the GCUPS directly comparable. @@ -367,12 +304,12 @@ struct similarities_callable { } private: - /** @brief Runs the engine call + status/kernel-time read behind the non-inlined `invoke_engine_timed_` boundary. */ + /** @brief Runs the engine call and its status read behind the non-inlined `invoke_engine_` boundary. */ void run_engine_(std::span<token_view_t const> queries_block, std::span<token_view_t const> candidates_block, strided_rows<sz_ssize_t> results_matrix) noexcept(false) { engine_timing_t const timing = std::apply( [&](auto &&...rest) { - return invoke_engine_timed_(engine, queries_block, candidates_block, results_matrix, rest...); + return invoke_engine_([&] { return engine(queries_block, candidates_block, results_matrix, rest...); }); }, extra_args); if (timing.status != status_t::success_k) @@ -446,6 +383,7 @@ void bench_levenshtein(environment_t const &env) { forkunion_executor_t pool; if (pool.try_spawn(std::thread::hardware_concurrency()) != status_t::success_k) throw std::runtime_error("Failed to spawn thread pool."); + cpu_specs_t const cpu_specs = pool.specs(); auto scramble_accelerated_results = [&](similarities_t &results_accelerated) { std::shuffle(results_accelerated.begin(), results_accelerated.end(), global_random_generator()); @@ -474,18 +412,23 @@ void bench_levenshtein(environment_t const &env) { results_affine_baseline.resize(matrix_size), results_affine_accelerated.resize(matrix_size); results_utf8_baseline.resize(matrix_size), results_utf8_accelerated.resize(matrix_size); - auto call_linear_baseline = similarities_callable<levenshtein_serial_t, forkunion_executor_t &>( - env, results_linear_baseline, shape, levenshtein_serial_t {scheme.uniform, scheme.linear}, pool); + auto call_linear_baseline = + similarities_callable<levenshtein_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_baseline, shape, levenshtein_serial_t {scheme.uniform, scheme.linear}, pool, + cpu_specs); auto name_linear_baseline = "levenshtein_serial_"s + scheme.tag + ":" + shape_label; bench_result_t linear_baseline = bench_unary(env, name_linear_baseline, call_linear_baseline).log(); - auto call_utf8_baseline = similarities_callable<levenshtein_utf8_serial_t>( - env, results_utf8_baseline, shape, levenshtein_utf8_serial_t {scheme.uniform, scheme.linear}); + auto call_utf8_baseline = similarities_callable<levenshtein_utf8_serial_t, dummy_executor_t, cpu_specs_t>( + env, results_utf8_baseline, shape, levenshtein_utf8_serial_t {scheme.uniform, scheme.linear}, + dummy_executor_t {}, cpu_specs); auto name_utf8_baseline = "levenshtein_utf8_serial_"s + scheme.tag + ":" + shape_label; bench_result_t utf8_baseline = bench_unary(env, name_utf8_baseline, call_utf8_baseline).log(); - auto call_affine_baseline = similarities_callable<affine_levenshtein_serial_t, forkunion_executor_t &>( - env, results_affine_baseline, shape, affine_levenshtein_serial_t {scheme.uniform, scheme.affine}, pool); + auto call_affine_baseline = + similarities_callable<affine_levenshtein_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_baseline, shape, affine_levenshtein_serial_t {scheme.uniform, scheme.affine}, + pool, cpu_specs); auto name_affine_baseline = "affine_levenshtein_serial_"s + scheme.tag + ":" + shape_label; bench_result_t affine_baseline = bench_unary(env, name_affine_baseline, call_affine_baseline).log(linear_baseline); @@ -493,87 +436,87 @@ void bench_levenshtein(environment_t const &env) { #if SZ_USE_ICELAKE bench_unary(env, "levenshtein_icelake_"s + scheme.tag + ":" + shape_label, call_linear_baseline, - similarities_callable<levenshtein_icelake_t, forkunion_executor_t &>( + similarities_callable<levenshtein_icelake_t, forkunion_executor_t &, cpu_specs_t>( env, results_linear_accelerated, shape, - levenshtein_icelake_t {scheme.uniform, scheme.linear}, pool), + levenshtein_icelake_t {scheme.uniform, scheme.linear}, pool, cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_baseline); scramble_accelerated_results(results_linear_accelerated); bench_unary(env, "affine_levenshtein_icelake_"s + scheme.tag + ":" + shape_label, call_affine_baseline, - similarities_callable<affine_levenshtein_icelake_t, forkunion_executor_t &>( + similarities_callable<affine_levenshtein_icelake_t, forkunion_executor_t &, cpu_specs_t>( env, results_affine_accelerated, shape, - affine_levenshtein_icelake_t {scheme.uniform, scheme.affine}, pool), + affine_levenshtein_icelake_t {scheme.uniform, scheme.affine}, pool, cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_baseline, affine_baseline); scramble_accelerated_results(results_affine_accelerated); - bench_unary( - env, "levenshtein_utf8_icelake_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, - similarities_callable<levenshtein_utf8_icelake_t>( - env, results_utf8_accelerated, shape, levenshtein_utf8_icelake_t {scheme.uniform, scheme.linear}), - callable_no_op_t {}, // preprocessing - similarities_equality_t {}) // equality check + bench_unary(env, "levenshtein_utf8_icelake_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, + similarities_callable<levenshtein_utf8_icelake_t, dummy_executor_t, cpu_specs_t>( + env, results_utf8_accelerated, shape, + levenshtein_utf8_icelake_t {scheme.uniform, scheme.linear}, dummy_executor_t {}, cpu_specs), + callable_no_op_t {}, // preprocessing + similarities_equality_t {}) // equality check .log(utf8_baseline); scramble_accelerated_results(results_utf8_accelerated); #endif #if SZ_USE_NEON - bench_unary( - env, "levenshtein_neon_"s + scheme.tag + ":" + shape_label, call_linear_baseline, - similarities_callable<levenshtein_neon_t, forkunion_executor_t &>( - env, results_linear_accelerated, shape, levenshtein_neon_t {scheme.uniform, scheme.linear}, pool), - callable_no_op_t {}, // preprocessing - similarities_equality_t {}) // equality check + bench_unary(env, "levenshtein_neon_"s + scheme.tag + ":" + shape_label, call_linear_baseline, + similarities_callable<levenshtein_neon_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_accelerated, shape, levenshtein_neon_t {scheme.uniform, scheme.linear}, + pool, cpu_specs), + callable_no_op_t {}, // preprocessing + similarities_equality_t {}) // equality check .log(linear_baseline); scramble_accelerated_results(results_linear_accelerated); bench_unary(env, "affine_levenshtein_neon_"s + scheme.tag + ":" + shape_label, call_affine_baseline, - similarities_callable<affine_levenshtein_neon_t, forkunion_executor_t &>( + similarities_callable<affine_levenshtein_neon_t, forkunion_executor_t &, cpu_specs_t>( env, results_affine_accelerated, shape, - affine_levenshtein_neon_t {scheme.uniform, scheme.affine}, pool), + affine_levenshtein_neon_t {scheme.uniform, scheme.affine}, pool, cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_baseline, affine_baseline); scramble_accelerated_results(results_affine_accelerated); - bench_unary( - env, "levenshtein_utf8_neon_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, - similarities_callable<levenshtein_utf8_neon_t>(env, results_utf8_accelerated, shape, - levenshtein_utf8_neon_t {scheme.uniform, scheme.linear}), - callable_no_op_t {}, // preprocessing - similarities_equality_t {}) // equality check + bench_unary(env, "levenshtein_utf8_neon_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, + similarities_callable<levenshtein_utf8_neon_t, dummy_executor_t, cpu_specs_t>( + env, results_utf8_accelerated, shape, + levenshtein_utf8_neon_t {scheme.uniform, scheme.linear}, dummy_executor_t {}, cpu_specs), + callable_no_op_t {}, // preprocessing + similarities_equality_t {}) // equality check .log(utf8_baseline); scramble_accelerated_results(results_utf8_accelerated); #endif #if SZ_USE_RVV - bench_unary( - env, "levenshtein_rvv_"s + scheme.tag + ":" + shape_label, call_linear_baseline, - similarities_callable<levenshtein_rvv_t, forkunion_executor_t &>( - env, results_linear_accelerated, shape, levenshtein_rvv_t {scheme.uniform, scheme.linear}, pool), - callable_no_op_t {}, // preprocessing - similarities_equality_t {}) // equality check + bench_unary(env, "levenshtein_rvv_"s + scheme.tag + ":" + shape_label, call_linear_baseline, + similarities_callable<levenshtein_rvv_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_accelerated, shape, levenshtein_rvv_t {scheme.uniform, scheme.linear}, + pool, cpu_specs), + callable_no_op_t {}, // preprocessing + similarities_equality_t {}) // equality check .log(linear_baseline); scramble_accelerated_results(results_linear_accelerated); bench_unary(env, "affine_levenshtein_rvv_"s + scheme.tag + ":" + shape_label, call_affine_baseline, - similarities_callable<affine_levenshtein_rvv_t, forkunion_executor_t &>( + similarities_callable<affine_levenshtein_rvv_t, forkunion_executor_t &, cpu_specs_t>( env, results_affine_accelerated, shape, - affine_levenshtein_rvv_t {scheme.uniform, scheme.affine}, pool), + affine_levenshtein_rvv_t {scheme.uniform, scheme.affine}, pool, cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_baseline, affine_baseline); scramble_accelerated_results(results_affine_accelerated); - bench_unary( - env, "levenshtein_utf8_rvv_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, - similarities_callable<levenshtein_utf8_rvv_t>(env, results_utf8_accelerated, shape, - levenshtein_utf8_rvv_t {scheme.uniform, scheme.linear}), - callable_no_op_t {}, // preprocessing - similarities_equality_t {}) // equality check + bench_unary(env, "levenshtein_utf8_rvv_"s + scheme.tag + ":" + shape_label, call_utf8_baseline, + similarities_callable<levenshtein_utf8_rvv_t, dummy_executor_t, cpu_specs_t>( + env, results_utf8_accelerated, shape, + levenshtein_utf8_rvv_t {scheme.uniform, scheme.linear}, dummy_executor_t {}, cpu_specs), + callable_no_op_t {}, // preprocessing + similarities_equality_t {}) // equality check .log(utf8_baseline); scramble_accelerated_results(results_utf8_accelerated); #endif @@ -669,6 +612,7 @@ void bench_needleman_wunsch_smith_waterman(environment_t const &env) { forkunion_executor_t pool; if (pool.try_spawn(std::thread::hardware_concurrency()) != status_t::success_k) throw std::runtime_error("Failed to spawn thread pool."); + cpu_specs_t const cpu_specs = pool.specs(); auto scramble_accelerated_results = [&](similarities_t &results_accelerated) { std::shuffle(results_accelerated.begin(), results_accelerated.end(), global_random_generator()); @@ -683,59 +627,66 @@ void bench_needleman_wunsch_smith_waterman(environment_t const &env) { results_linear_local_baseline.resize(matrix_size), results_linear_local_accelerated.resize(matrix_size); results_affine_local_baseline.resize(matrix_size), results_affine_local_accelerated.resize(matrix_size); - auto call_linear_global_baseline = similarities_callable<needleman_wunsch_serial_t, forkunion_executor_t &>( - env, results_linear_global_baseline, shape, {blosum62_matrix32, blosum62_linear_cost}, pool); + auto call_linear_global_baseline = + similarities_callable<needleman_wunsch_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_global_baseline, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, cpu_specs); auto name_linear_global_baseline = "needleman_wunsch_serial:"s + shape_label; bench_result_t linear_global_baseline = bench_unary(env, name_linear_global_baseline, call_linear_global_baseline).log(); - auto call_linear_local_baseline = similarities_callable<smith_waterman_serial_t, forkunion_executor_t &>( - env, results_linear_local_baseline, shape, {blosum62_matrix32, blosum62_linear_cost}, pool); + auto call_linear_local_baseline = + similarities_callable<smith_waterman_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_local_baseline, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, cpu_specs); auto name_linear_local_baseline = "smith_waterman_serial:"s + shape_label; bench_result_t linear_local_baseline = bench_unary(env, name_linear_local_baseline, call_linear_local_baseline).log(); auto call_affine_global_baseline = - similarities_callable<affine_needleman_wunsch_serial_t, forkunion_executor_t &>( - env, results_affine_global_baseline, shape, {blosum62_matrix32, blosum62_affine_cost}, pool); + similarities_callable<affine_needleman_wunsch_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_global_baseline, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, cpu_specs); auto name_affine_global_baseline = "affine_needleman_wunsch_serial:"s + shape_label; bench_result_t affine_global_baseline = bench_unary(env, name_affine_global_baseline, call_affine_global_baseline).log(); - auto call_affine_local_baseline = similarities_callable<affine_smith_waterman_serial_t, forkunion_executor_t &>( - env, results_affine_local_baseline, shape, {blosum62_matrix32, blosum62_affine_cost}, pool); + auto call_affine_local_baseline = + similarities_callable<affine_smith_waterman_serial_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_local_baseline, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, cpu_specs); auto name_affine_local_baseline = "affine_smith_waterman_serial:"s + shape_label; bench_result_t affine_local_baseline = bench_unary(env, name_affine_local_baseline, call_affine_local_baseline).log(); #if SZ_USE_HASWELL bench_unary(env, "needleman_wunsch_haswell:"s + shape_label, call_linear_global_baseline, - similarities_callable<needleman_wunsch_haswell_t, forkunion_executor_t &>( - env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<needleman_wunsch_haswell_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_global_baseline); scramble_accelerated_results(results_linear_global_accelerated); bench_unary(env, "smith_waterman_haswell:"s + shape_label, call_linear_local_baseline, - similarities_callable<smith_waterman_haswell_t, forkunion_executor_t &>( - env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<smith_waterman_haswell_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_local_baseline); scramble_accelerated_results(results_linear_local_accelerated); bench_unary(env, "affine_needleman_wunsch_haswell:"s + shape_label, call_affine_global_baseline, - similarities_callable<affine_needleman_wunsch_haswell_t, forkunion_executor_t &>( - env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_needleman_wunsch_haswell_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_global_baseline); scramble_accelerated_results(results_affine_global_accelerated); bench_unary(env, "affine_smith_waterman_haswell:"s + shape_label, call_affine_local_baseline, - similarities_callable<affine_smith_waterman_haswell_t, forkunion_executor_t &>( - env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_smith_waterman_haswell_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_local_baseline); @@ -744,32 +695,36 @@ void bench_needleman_wunsch_smith_waterman(environment_t const &env) { #if SZ_USE_ICELAKE bench_unary(env, "needleman_wunsch_icelake:"s + shape_label, call_linear_global_baseline, - similarities_callable<needleman_wunsch_icelake_t, forkunion_executor_t &>( - env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<needleman_wunsch_icelake_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_global_baseline); scramble_accelerated_results(results_linear_global_accelerated); bench_unary(env, "smith_waterman_icelake:"s + shape_label, call_linear_local_baseline, - similarities_callable<smith_waterman_icelake_t, forkunion_executor_t &>( - env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<smith_waterman_icelake_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_local_baseline); scramble_accelerated_results(results_linear_local_accelerated); bench_unary(env, "affine_needleman_wunsch_icelake:"s + shape_label, call_affine_global_baseline, - similarities_callable<affine_needleman_wunsch_icelake_t, forkunion_executor_t &>( - env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_needleman_wunsch_icelake_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_global_baseline); scramble_accelerated_results(results_affine_global_accelerated); bench_unary(env, "affine_smith_waterman_icelake:"s + shape_label, call_affine_local_baseline, - similarities_callable<affine_smith_waterman_icelake_t, forkunion_executor_t &>( - env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_smith_waterman_icelake_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_local_baseline); @@ -778,32 +733,36 @@ void bench_needleman_wunsch_smith_waterman(environment_t const &env) { #if SZ_USE_NEON bench_unary(env, "needleman_wunsch_neon:"s + shape_label, call_linear_global_baseline, - similarities_callable<needleman_wunsch_neon_t, forkunion_executor_t &>( - env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<needleman_wunsch_neon_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_global_baseline); scramble_accelerated_results(results_linear_global_accelerated); bench_unary(env, "smith_waterman_neon:"s + shape_label, call_linear_local_baseline, - similarities_callable<smith_waterman_neon_t, forkunion_executor_t &>( - env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<smith_waterman_neon_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_local_baseline); scramble_accelerated_results(results_linear_local_accelerated); bench_unary(env, "affine_needleman_wunsch_neon:"s + shape_label, call_affine_global_baseline, - similarities_callable<affine_needleman_wunsch_neon_t, forkunion_executor_t &>( - env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_needleman_wunsch_neon_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_global_baseline); scramble_accelerated_results(results_affine_global_accelerated); bench_unary(env, "affine_smith_waterman_neon:"s + shape_label, call_affine_local_baseline, - similarities_callable<affine_smith_waterman_neon_t, forkunion_executor_t &>( - env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_smith_waterman_neon_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_local_baseline); @@ -812,32 +771,36 @@ void bench_needleman_wunsch_smith_waterman(environment_t const &env) { #if SZ_USE_RVV bench_unary(env, "needleman_wunsch_rvv:"s + shape_label, call_linear_global_baseline, - similarities_callable<needleman_wunsch_rvv_t, forkunion_executor_t &>( - env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<needleman_wunsch_rvv_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_global_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_global_baseline); scramble_accelerated_results(results_linear_global_accelerated); bench_unary(env, "smith_waterman_rvv:"s + shape_label, call_linear_local_baseline, - similarities_callable<smith_waterman_rvv_t, forkunion_executor_t &>( - env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool), + similarities_callable<smith_waterman_rvv_t, forkunion_executor_t &, cpu_specs_t>( + env, results_linear_local_accelerated, shape, {blosum62_matrix32, blosum62_linear_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(linear_local_baseline); scramble_accelerated_results(results_linear_local_accelerated); bench_unary(env, "affine_needleman_wunsch_rvv:"s + shape_label, call_affine_global_baseline, - similarities_callable<affine_needleman_wunsch_rvv_t, forkunion_executor_t &>( - env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_needleman_wunsch_rvv_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_global_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_global_baseline); scramble_accelerated_results(results_affine_global_accelerated); bench_unary(env, "affine_smith_waterman_rvv:"s + shape_label, call_affine_local_baseline, - similarities_callable<affine_smith_waterman_rvv_t, forkunion_executor_t &>( - env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool), + similarities_callable<affine_smith_waterman_rvv_t, forkunion_executor_t &, cpu_specs_t>( + env, results_affine_local_accelerated, shape, {blosum62_matrix32, blosum62_affine_cost}, pool, + cpu_specs), callable_no_op_t {}, // preprocessing similarities_equality_t {}) // equality check .log(affine_local_baseline); diff --git a/bench/substrings.cpp b/bench/substrings.cpp new file mode 100644 index 00000000..f968fdd2 --- /dev/null +++ b/bench/substrings.cpp @@ -0,0 +1,70 @@ +/** + * @file bench/substrings.cpp + * @brief Benchmarks the multi-pattern search engine (Aho-Corasick) on CPU: serial and fork-union-parallel. + * Sweeps dictionary sizes and reports the dictionary properties - state count, hot/cold tier split, + * and the fraction of byte steps that stay on the hot path - alongside the throughput they produce, + * since those properties dominate the number far more than the backend choice. + * + * Compute-bound: the automaton is rebuilt per sweep cell, so a 64 MiB slice exercises every hot- and cold-tier path, while a larger corpus only rebuilds the same states. + * + * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: + * - `STRINGWARS_DATASET` : Path to the haystack dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for + * N-grams) that turns the dataset into the haystacks searched. `file` reproduces a single-haystack scan, + * the shape the reference baselines below were measured with. + * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. + * + * Unlike StringWars, the following additional environment variables are supported: + * - `STRINGWARS_DURATION=10` : Time limit (in seconds) per benchmark. + * - `STRINGWARS_STRESS=1` : Cross-checks the parallel backend's occurrence counts against the serial ones. + * - `STRINGWARS_STRESS_DURATION=10` : Stress-testing time limit (in seconds) per benchmark. + * - `STRINGWARS_FILTER` : Regular Expression pattern to filter benchmark names. + * + * Needles come from the corpus itself, deterministically: whitespace-cut words regardless of how + * `STRINGWARS_TOKENS` shapes the haystacks, with the most frequent one percent and every term occurring + * once dropped. Each sweep cell draws one slice - most or least frequent, one or ten percent, or all of + * it - from the frequency-ordered remainder. + * + * Here are a few build & run commands: + * + * @code{.sh} + * g++ -std=c++17 -O3 -march=native -I include -I forkunion/include bench/substrings.cpp -o substrings_cpp \ + * -lpthread + * STRINGWARS_DATASET=haystack_64mib.txt STRINGWARS_TOKENS=file ./substrings_cpp + * @endcode + * + * The CMake target for this file forces `-O2` after `-O3` for benchmark builds, which understates every + * kernel here; the command above compiles directly with `-O3` for numbers worth trusting. + * + * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. + * This file is a sibling of `similarities.cpp`; its GPU counterpart is `substrings.cu`. + */ +#include "substrings.cuh" +#include "stringzilla.hpp" // `log_environment` + +namespace szs = ashvardanian::stringzillas; +using namespace sz::scripts; + +int main(int argc, char const **argv) { + install_test_signal_handlers(); // Backtrace on SIGSEGV/SIGABRT + line-buffered stdout for crash localization. + std::printf("Welcome to the StringZillas substrings benchmark on CPU!\n"); + if (auto code = log_environment(); code != 0) return code; + + try { + std::printf("Building up the environment...\n"); + environment_t env = build_environment( // + argc, argv, // + "xlsum.csv", // + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); + + bench_substrings(env); + } + catch (std::exception const &e) { + std::fprintf(stderr, "Failed with: %s\n", e.what()); + return 1; + } + + std::printf("All benchmarks finished.\n"); + return 0; +} diff --git a/bench/substrings.cu b/bench/substrings.cu new file mode 100644 index 00000000..6eb9e7b0 --- /dev/null +++ b/bench/substrings.cu @@ -0,0 +1,70 @@ +/** + * @file bench/substrings.cu + * @brief Benchmarks the multi-pattern search engine (Aho-Corasick) on the GPU. + * Builds the dictionary on the host - construction is never a bottleneck, so it's never worth + * accelerating - uploads it once, and reports the device-measured throughput of scanning the whole + * haystack tape, alongside the dictionary properties that dominate it: state count, hot/cold tier + * split, and the fraction of byte steps that stay on the branch-free hot path. + * + * Compute-bound: the automaton is rebuilt per sweep cell, so a 64 MiB slice exercises every hot- and cold-tier path, while a larger corpus only rebuilds the same states. + * + * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: + * - `STRINGWARS_DATASET` : Path to the haystack dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for + * N-grams) that turns the dataset into the haystacks concatenated onto one device tape. `file` reproduces + * a single-haystack scan, the shape the reference baselines below were measured with. + * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. + * + * Unlike StringWars, the following additional environment variables are supported: + * - `STRINGWARS_DURATION=10` : Time limit (in seconds) per benchmark. + * - `STRINGWARS_FILTER` : Regular Expression pattern to filter benchmark names. + * + * Needles come from the corpus itself, deterministically: whitespace-cut words regardless of how + * `STRINGWARS_TOKENS` shapes the haystacks, with the most frequent one percent and every term occurring + * once dropped. Each sweep cell draws one slice - most or least frequent, one or ten percent, or all of + * it - from the frequency-ordered remainder. + * + * Here are a few build & run commands: + * + * @code{.sh} + * /usr/local/cuda-12.9/bin/nvcc -std=c++17 -O3 -arch=sm_90 -ccbin g++-14 -I include -I forkunion/include \ + * --expt-relaxed-constexpr bench/substrings.cu -o substrings_cu + * STRINGWARS_DATASET=haystack_64mib.txt STRINGWARS_TOKENS=file ./substrings_cu + * @endcode + * + * `-ccbin g++-14` is required: the system default `g++` is too new for this CUDA Toolkit to accept. + * The CMake target for this file forces `-O2` after `-O3` for benchmark builds, which understates every + * kernel here; the command above compiles directly with `-O3` for numbers worth trusting. + * + * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. + * This file is a sibling of `substrings.cpp` and of `similarities.cu`. + */ +#include "substrings.cuh" +#include "stringzilla.hpp" // `log_environment` + +namespace szs = ashvardanian::stringzillas; +using namespace sz::scripts; + +int main(int argc, char const **argv) { + install_test_signal_handlers(); // Backtrace on SIGSEGV/SIGABRT + line-buffered stdout for crash localization. + std::printf("Welcome to the StringZillas substrings benchmark on GPU!\n"); + if (auto code = log_environment(); code != 0) return code; + + try { + std::printf("Building up the environment...\n"); + environment_t env = build_environment( // + argc, argv, // + "xlsum.csv", // + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); + + bench_substrings(env); + } + catch (std::exception const &e) { + std::fprintf(stderr, "Failed with: %s\n", e.what()); + return 1; + } + + std::printf("All benchmarks finished.\n"); + return 0; +} diff --git a/bench/substrings.cuh b/bench/substrings.cuh new file mode 100644 index 00000000..03f5f959 --- /dev/null +++ b/bench/substrings.cuh @@ -0,0 +1,597 @@ +/** + * @file bench/substrings.cuh + * @brief Shared code for CPU and GPU multi-pattern search (Aho-Corasick) benchmarks. + */ +#include <cstring> // `std::memcpy`, `std::memcmp` + +#include <algorithm> // `std::sort`, `std::unique` +#include <numeric> // `std::accumulate` +#include <string> // `std::string` +#include <string_view> // `std::string_view` keys for the frequency map +#include <unordered_map> // `std::unordered_map` for the word-frequency count +#include <utility> // `std::declval` +#include <vector> // `std::vector` + +#include "stringzillas/substrings/serial.hpp" + +#if SZ_USE_CUDA +#include "stringzillas/substrings/cuda.cuh" +#endif + +#include "shared.hpp" +#include "stringzilla.hpp" // `status_name` + +namespace ashvardanian { +namespace stringzilla { +namespace scripts { + +// Per-symbol: a using-directive re-exports our `memcpy` and nvcc then finds the call ambiguous. + +// StringZillas library symbols available on every backend: +using ashvardanian::stringzillas::substrings_alphabet_size_k; +using ashvardanian::stringzillas::substrings_cased_k; +using ashvardanian::stringzillas::substrings_match_t; +using ashvardanian::stringzillas::substrings_bm25_t; +using ashvardanian::stringzillas::substrings_leftmost_longest_k; +using ashvardanian::stringzillas::substrings_overlapping_k; +using ashvardanian::stringzillas::substrings_case_sensitivity_t; +using ashvardanian::stringzillas::substrings_parallel_t; +using ashvardanian::stringzillas::substrings_serial_t; +using ashvardanian::stringzillas::substrings_uncased_k; +using ashvardanian::stringzillas::dummy_executor_t; +using ashvardanian::stringzillas::forkunion_executor_t; + +// StringZillas library symbols provided only by the CUDA backend: +#if SZ_USE_CUDA +using ashvardanian::stringzillas::cuda_executor_t; +using ashvardanian::stringzillas::cuda_status_t; +using ashvardanian::stringzillas::substrings_cuda_t; +using ashvardanian::stringzillas::gpu_specs_fetch; +#endif + +#pragma region Needle Dictionaries + +/** + * @brief One contiguous slice of the frequency-ordered vocabulary, plus its byte total. + * + * Borrows twice over: the terms are a subrange of the vocabulary's own array, and each term points into + * `env.dataset`. `total_bytes` is the denominator construction throughput is reported against, since a + * needle count alone compares badly across corpora. + */ +struct needle_slice_t { + span<span<char const> const> terms; + size_t total_bytes = 0; + + size_t size() const noexcept { return terms.size(); } +}; + +/** + * @brief The post-cutoff vocabulary: distinct corpus words with both noisy ends removed, frequency-ordered. + * + * Words are cut from the raw dataset by whitespace regardless of how `STRINGWARS_TOKENS` shapes the + * haystacks, so `lines` and `file` searches draw needles from the same vocabulary a `words` search does. + * Two cutoffs, both applied before any slice is taken. The most frequent one percent are stopwords that + * match on nearly every byte while building no trie depth, so they measure output cost rather than + * automaton behaviour. Terms occurring once are hapax - mostly OCR noise and URLs no haystack reaches twice. + * Every span borrows from `env.dataset`, so building this copies no token bytes. + */ +struct vocabulary_t { + unified_vector<span<char const>> terms; // ? Frequency-descending, ties broken by first appearance. + unified_vector<size_t> counts; // ? Parallel to `terms`. + size_t dropped_frequent = 0; + size_t dropped_hapax = 0; + size_t dropped_short = 0; + size_t dropped_long = 0; + size_t total_occurrences = 0; + + size_t size() const noexcept { return terms.size(); } +}; + +/** @brief Shortest word admitted into the vocabulary: anything under three bytes matches at nearly every + * position, so the benchmark would measure match materialization rather than the automaton walk. */ +constexpr size_t vocabulary_min_word_bytes_k = 3; + +/** @brief Longest word admitted into the vocabulary. Longer whitespace-cut tokens are unsegmented CJK or + * Thai runs and URLs rather than words - and the longest needle sets `max_source_match_bytes`, which every + * GPU chunk re-walks as its warm-up, so one runaway "word" would tax every chunk in the corpus. */ +constexpr size_t vocabulary_max_word_bytes_k = 32; + +/** @brief Orders spans by content, so a sort groups equal terms into runs a single pass can count. */ +bool spans_less(span<char const> left, span<char const> right) noexcept { + size_t const shared = left.size() < right.size() ? left.size() : right.size(); + int const ordering = std::memcmp(left.data(), right.data(), shared); + return ordering != 0 ? ordering < 0 : left.size() < right.size(); +} + +/** + * @brief Counts word frequencies over the whole dataset and drops both noisy ends. + * @param[in] frequent_cutoff Fraction of the frequency-ordered vocabulary discarded from the top. + * @param[in] minimum_occurrences Terms appearing fewer times than this are discarded outright. + * + * One hashed counting pass over the corpus, then a sort over only the distinct survivors - the corpus has + * hundreds of millions of words but only a few million distinct ones, so sorting only the survivors keeps + * the whole suite's start-up cheap, where sorting every occurrence would dominate it. + */ +vocabulary_t build_vocabulary(environment_t const &env, double frequent_cutoff = 0.01, size_t minimum_occurrences = 2) { + // No UTF-8 validation: words split on ASCII whitespace, so a malformed word means a malformed corpus, + // which should fail loudly from `try_build` rather than silently shrink the vocabulary here. + // The library's own primitives do the walking: each `split` step is one `sz_find_byteset` SIMD scan and + // the map hashes through `sz_hash`. No `reserve` - growth rehashes move only the distinct entries, + // never the occurrences, so their total cost is a few multiples of the final table. + std::unordered_map<std::string_view, size_t, sz::hash, sz::equal_to> frequencies; + size_t dropped_short = 0, dropped_long = 0; + for (auto word : sz::string_view {env.dataset.data(), env.dataset.size()}.split()) { + if (word.size() == 0) continue; // ? Runs of separators yield empty segments, not words + if (word.size() < vocabulary_min_word_bytes_k) { + ++dropped_short; + continue; + } + if (word.size() > vocabulary_max_word_bytes_k) { + ++dropped_long; + continue; + } + ++frequencies[std::string_view {word.data(), word.size()}]; + } + + // The hapax filter runs before any sort, so the ranking pass touches only the few percent that survive. + // The top cutoff keeps its base: a fraction of ALL distinct terms, hapax included, so the slice matches + // what the run-length formulation selected. + vocabulary_t vocabulary; + vocabulary.dropped_short = dropped_short; + vocabulary.dropped_long = dropped_long; + vocabulary.terms.reserve(frequencies.size()); + vocabulary.counts.reserve(frequencies.size()); + for (auto const &entry : frequencies) { + vocabulary.total_occurrences += entry.second; + if (entry.second < minimum_occurrences) { + ++vocabulary.dropped_hapax; + continue; + } + vocabulary.terms.push_back({entry.first.data(), entry.first.size()}); + vocabulary.counts.push_back(entry.second); + } + + // Order by frequency first, then by content, so the ranking is deterministic across runs and platforms. + unified_vector<size_t> order(vocabulary.terms.size()); + for (size_t index = 0; index < order.size(); ++index) order[index] = index; + std::sort(order.begin(), order.end(), [&](size_t left, size_t right) noexcept { + if (vocabulary.counts[left] != vocabulary.counts[right]) + return vocabulary.counts[left] > vocabulary.counts[right]; + return spans_less(vocabulary.terms[left], vocabulary.terms[right]); + }); + + size_t const drop_from_top = (size_t)((double)frequencies.size() * frequent_cutoff); + vocabulary.dropped_frequent = drop_from_top < order.size() ? drop_from_top : order.size(); + unified_vector<span<char const>> kept_terms; + unified_vector<size_t> kept_counts; + kept_terms.reserve(order.size()); + kept_counts.reserve(order.size()); + for (size_t rank = vocabulary.dropped_frequent; rank < order.size(); ++rank) { + size_t const index = order[rank]; + kept_terms.push_back(vocabulary.terms[index]); + kept_counts.push_back(vocabulary.counts[index]); + } + vocabulary.terms = std::move(kept_terms); + vocabulary.counts = std::move(kept_counts); + return vocabulary; +} + +/** @brief Which end of the frequency-ordered vocabulary a dictionary is drawn from, and how much of it. */ +enum class vocabulary_slice_t { + most_frequent_1_percent_k, + most_frequent_10_percent_k, + least_frequent_1_percent_k, + least_frequent_10_percent_k, + entire_k, +}; + +/** @brief Human-readable slice tag for every benchmark label. + * @note Reaches `std::printf` as an argument to a `%s`, never as a format, so the `%` stays literal. */ +constexpr char const *vocabulary_slice_name(vocabulary_slice_t slice) noexcept { + switch (slice) { + case vocabulary_slice_t::most_frequent_1_percent_k: return "most_frequent_1%"; + case vocabulary_slice_t::most_frequent_10_percent_k: return "most_frequent_10%"; + case vocabulary_slice_t::least_frequent_1_percent_k: return "least_frequent_1%"; + case vocabulary_slice_t::least_frequent_10_percent_k: return "least_frequent_10%"; + case vocabulary_slice_t::entire_k: return "entire"; + } + return "unknown"; +} + +/** + * @brief Takes one slice of @p vocabulary as a dictionary. + * + * Slices are taken by term count, so the frequent and the rare slice of the same percentage hold the same + * number of needles and differing byte totals - Zipf makes frequent terms short. + */ +needle_slice_t needle_slice_of(vocabulary_t const &vocabulary, vocabulary_slice_t slice) { + size_t const available = vocabulary.size(); + size_t wanted = available; + size_t first = 0; + switch (slice) { + case vocabulary_slice_t::most_frequent_1_percent_k: wanted = available / 100; break; + case vocabulary_slice_t::most_frequent_10_percent_k: wanted = available / 10; break; + case vocabulary_slice_t::least_frequent_1_percent_k: wanted = available / 100, first = available - wanted; break; + case vocabulary_slice_t::least_frequent_10_percent_k: wanted = available / 10, first = available - wanted; break; + case vocabulary_slice_t::entire_k: break; + } + if (wanted == 0) return {}; + + needle_slice_t result; + result.terms = {vocabulary.terms.data() + first, wanted}; + for (span<char const> const &term : result.terms) result.total_bytes += term.size(); + return result; +} + +#pragma endregion Needle Dictionaries + +#pragma region Reporting + +/** @brief Structural facts about one compiled automaton. Construction @b throughput is not printed here - + * it is measured by `bench_nullary` like every other operation, so it carries the same units and + * the same min-of-N latency. */ +template <typename dictionary_type_> +void print_dictionary_properties(dictionary_type_ const &dictionary, needle_slice_t const &needles) { + size_t const state_count = dictionary.count_states(); + size_t const hot_count = dictionary.hot_count(); + // Sized at the width this automaton actually settled on - a narrowed one halves every hot row. + size_t const hot_tier_bytes = hot_count * substrings_alphabet_size_k * + sizeof(typename dictionary_type_::state_id_t); + size_t const cold_tier_bytes = dictionary.transitions_bytes() - hot_tier_bytes; + + std::printf(" - Needles: %zu requested, %zu inserted, %zu bytes\n", needles.size(), dictionary.count_needles(), + needles.total_bytes); + std::printf(" - States: %zu total, %zu hot (%.1f%%), %zu cold\n", state_count, hot_count, + state_count ? 100.0 * (double)hot_count / (double)state_count : 0.0, state_count - hot_count); + std::printf(" - Hot tier: %zu bytes (%.2f MiB)\n", hot_tier_bytes, hot_tier_bytes / (1024.0 * 1024.0)); + std::printf(" - Cold tier: %zu bytes (%.2f MiB)\n", cold_tier_bytes, cold_tier_bytes / (1024.0 * 1024.0)); + std::printf(" - Max match length: %zu bytes, max outputs per state: %zu\n", + (size_t)dictionary.max_source_match_bytes(), (size_t)dictionary.view().max_outputs_per_state); +} + +#pragma endregion Reporting + +#pragma region Timed Callables + +/** + * @brief Wraps one `substrings` engine call into a `bench_nullary`-compatible nullary callable. + * + * @p invocable names the method and its arguments at the call site, so this template never learns which + * operation it is timing. Reports the device-measured kernel GB/s once it goes out of scope, as + * `similarities_callable` does; CPU engines accumulate no device time and are skipped. + */ +template <typename invocable_type_, typename output_type_> +struct substrings_callable { + invocable_type_ invocable; // ? Returns the engine's status; the call site supplies the call. + size_t total_bytes = 0; + output_type_ const *output = nullptr; // ? The container `arrays_equality` compares two runs on. + + double kernel_milliseconds_total = 0.0; + double kernel_bytes_total = 0.0; + + ~substrings_callable() { + if (kernel_milliseconds_total <= 0.0 || kernel_bytes_total <= 0.0) return; + double const kernel_gigabytes_per_second = kernel_bytes_total / (kernel_milliseconds_total * 1e6); + std::printf("> Kernel: %.3f GB/s @ %.3f ms device-measured (excludes host materialization)\n", + kernel_gigabytes_per_second, kernel_milliseconds_total); + } + + call_result_t operator()() noexcept(false) { + engine_timing_t const timing = invoke_engine_(invocable); + if (timing.status != status_t::success_k) + throw std::runtime_error(std::string("substrings operation failed: ") + status_name(timing.status) + " (" + + std::to_string((int)timing.status) + ")"); + kernel_milliseconds_total += timing.kernel_milliseconds; + kernel_bytes_total += (double)total_bytes; + + call_result_t result; + result.bytes_passed = total_bytes; + result.operations = total_bytes; // ? Lets `log()` print bytes/cycle from its shared efficiency line. + result.check_value = reinterpret_cast<check_value_t>(output); + return result; + } +}; + +/** @brief Builds a `substrings_callable` with both template arguments deduced. */ +template <typename invocable_type_, typename output_type_> +substrings_callable<invocable_type_, output_type_> make_substrings_callable( // + size_t total_bytes, output_type_ const &output, invocable_type_ &&invocable) { + return {std::forward<invocable_type_>(invocable), total_bytes, &output}; +} + +#pragma endregion Timed Callables + +#pragma region Sweep + +/** @brief A `try_find` output buffer bigger than this is skipped, so a dictionary of very common short + * needles can't run the box out of memory. */ +static constexpr size_t substrings_matches_safety_cap_k = 200'000'000; + +/** + * @brief One measured configuration of the sweep. + * + * The frequent slices are short terms that keep the walk near the root; the rare slices are long terms that + * drive it deep. Every printed line carries this label, so no number is read positionally. + */ +struct substrings_sweep_cell_t { + vocabulary_slice_t slice; + substrings_case_sensitivity_t sensitivity; + + char const *sensitivity_name() const noexcept { return sensitivity == substrings_uncased_k ? "uncased" : "cased"; } + std::string label() const { return std::string(sensitivity_name()) + ":" + vocabulary_slice_name(slice); } +}; + +/** @brief The sweep both the CPU and the CUDA entry points walk, declared once so they cannot drift apart. */ +static substrings_sweep_cell_t const substrings_sweep_k[] = { + {vocabulary_slice_t::most_frequent_1_percent_k, substrings_cased_k}, + {vocabulary_slice_t::most_frequent_10_percent_k, substrings_cased_k}, + {vocabulary_slice_t::least_frequent_1_percent_k, substrings_cased_k}, + {vocabulary_slice_t::least_frequent_10_percent_k, substrings_cased_k}, + {vocabulary_slice_t::entire_k, substrings_cased_k}, + {vocabulary_slice_t::most_frequent_1_percent_k, substrings_uncased_k}, + {vocabulary_slice_t::most_frequent_10_percent_k, substrings_uncased_k}, + {vocabulary_slice_t::least_frequent_1_percent_k, substrings_uncased_k}, + {vocabulary_slice_t::least_frequent_10_percent_k, substrings_uncased_k}, + {vocabulary_slice_t::entire_k, substrings_uncased_k}, +}; + +/** + * @brief Benchmarks one sweep cell on every backend this build links, so a GPU build reports its own CPU + * baselines beside the device numbers. Device work is gated inline, as in `similarities.cuh`. + */ +void bench_substrings_dictionary( // + environment_t const &env, size_t haystack_bytes, // + vocabulary_t const &vocabulary, substrings_sweep_cell_t const &cell, // + cpu_specs_t const &cpu_specs, forkunion_executor_t &pool) { + + std::string const dictionary_label = cell.label(); + std::printf("\nBenchmarking dictionary %s\n", dictionary_label.c_str()); + + needle_slice_t const needles = needle_slice_of(vocabulary, cell.slice); + if (needles.size() == 0) { + std::printf("Skipping: the post-cutoff vocabulary is too small for this slice\n"); + return; + } + + // Owned rather than passed as temporaries: every timed callable below captures its executor by reference + // and outlives the expression that built it, so a temporary would leave the closure holding a dead object. + dummy_executor_t serial_executor; + + // Construction reports needle-bytes per second through the same callable as the searches below. + substrings_serial_t serial_engine; + auto build_call = make_substrings_callable(needles.total_bytes, serial_engine, [&] { + serial_engine.reset(); // ? Rebuilding from scratch on every call IS the measured operation + return serial_engine.try_index(needles.terms, cell.sensitivity, serial_executor, cpu_specs); + }); + + // The measured row leaves a built automaton behind; only a filtered-out row still owes one. + bench_result_t const build_result = bench_nullary(env, "substrings_build_serial:" + dictionary_label, build_call); + build_result.log(); + if (build_result.skipped) { + status_t const serial_build_status = serial_engine.try_index(needles.terms, cell.sensitivity, serial_executor, + cpu_specs); + if (serial_build_status != status_t::success_k) + throw std::runtime_error(std::string("Failed to build the serial dictionary: ") + + status_name(serial_build_status)); + } + + substrings_parallel_t parallel_engine; + status_t const parallel_build_status = parallel_engine.try_index(needles.terms, cell.sensitivity, pool, cpu_specs); + if (parallel_build_status != status_t::success_k) + throw std::runtime_error(std::string("Failed to build the parallel dictionary: ") + + status_name(parallel_build_status)); + + serial_engine.visit_dictionary([&](auto const &dictionary) { print_dictionary_properties(dictionary, needles); }); + + unified_vector<size_t> serial_counts(env.tokens.size()); + unified_vector<size_t> parallel_counts(env.tokens.size()); + + // One shape per operation, invoked once per backend: the three differ only in which engine runs and in the + // `(executor, specs)` pair every engine entry point already takes. + size_t serial_total = 0, parallel_total = 0; + auto count_with = [&](auto &engine, auto &&executor, auto const &specs, unified_vector<size_t> &counts, + size_t &total) { + return make_substrings_callable(haystack_bytes, counts, [&] { + return engine.try_count(env.tokens, substrings_overlapping_k, span<size_t>(counts.data(), counts.size()), + total, executor, specs); + }); + }; + + auto serial_call = count_with(serial_engine, serial_executor, cpu_specs, serial_counts, serial_total); + std::string const serial_name = "substrings_count_serial:" + dictionary_label; + bench_result_t const serial_result = bench_nullary(env, serial_name, serial_call).log(); + + auto parallel_call = count_with(parallel_engine, pool, cpu_specs, parallel_counts, parallel_total); + std::string const parallel_name = "substrings_count_parallel:" + dictionary_label; + bench_nullary(env, parallel_name, serial_call, parallel_call, callable_no_op_t {}, arrays_equality<size_t> {}) + .log(serial_result); + + // Printed once per cell: the same number for every backend, and a bytes-per-second figure means little + // without it, since a match-saturated dictionary and a near-miss one differ by an order of magnitude. + size_t const total_occurrences = serial_total; + std::printf(" - Occurrences: %zu across %zu haystacks\n", total_occurrences, serial_counts.size()); + + // `try_find` materializes every match, so a dictionary of very common short needles can blow up the + // output buffer; size it from the `try_count` pass above and skip outright past the safety cap. + if (total_occurrences == 0 || total_occurrences > substrings_matches_safety_cap_k) { + std::printf("Skipping try_find: %zu occurrences %s the safety cap of %zu\n", total_occurrences, + total_occurrences == 0 ? "is" : "exceeds", substrings_matches_safety_cap_k); + return; + } + + size_t serial_found = 0, parallel_found = 0; + auto find_with = [&](auto &engine, auto &&executor, auto const &specs, unified_vector<substrings_match_t> &matches, + size_t &found) { + return make_substrings_callable(haystack_bytes, matches, [&] { + return engine.try_find(env.tokens, substrings_overlapping_k, + span<substrings_match_t>(matches.data(), matches.size()), found, executor, specs); + }); + }; + + unified_vector<substrings_match_t> serial_matches(total_occurrences); + auto serial_find_call = find_with(serial_engine, serial_executor, cpu_specs, serial_matches, serial_found); + std::string const serial_find_name = "substrings_find_serial:" + dictionary_label; + bench_result_t const serial_find_result = bench_nullary(env, serial_find_name, serial_find_call).log(); + + unified_vector<substrings_match_t> parallel_matches(total_occurrences); + auto parallel_find_call = find_with(parallel_engine, pool, cpu_specs, parallel_matches, parallel_found); + std::string const parallel_find_name = "substrings_find_parallel:" + dictionary_label; + bench_nullary(env, parallel_find_name, serial_find_call, parallel_find_call, callable_no_op_t {}, + arrays_equality<substrings_match_t> {}) + .log(serial_find_result); + +#if SZ_USE_CUDA + gpu_specs_t gpu_specs; + if (gpu_specs_fetch(gpu_specs) != status_t::success_k) throw std::runtime_error("Failed to fetch GPU specs."); + + // One executor for the whole cell, for the same reason the serial one is owned: the timed callables below + // capture it by reference and outlive the expressions that build them. + cuda_executor_t device_executor; + + // The tier split follows the device's own L2, so the fetched specs have to reach the build - a default + // `gpu_specs_t` describes an A100 and would tier this dictionary against hardware that is not here, + // beside CPU dictionaries tiered against the real machine. + substrings_cuda_t device_engine; + cuda_status_t const device_build_status = device_engine.try_index(needles.terms, cell.sensitivity, device_executor, + gpu_specs); + if (device_build_status.status != status_t::success_k) + throw std::runtime_error(std::string("Failed to build the device dictionary: ") + + status_name(device_build_status.status)); + + // The device engine reports the same per-haystack breakdown, so this compares element-wise against the + // serial pass that already ran. + unified_vector<size_t> device_counts(env.tokens.size(), 0); + size_t device_total = 0; + + auto cuda_call = count_with(device_engine, device_executor, gpu_specs, device_counts, device_total); + std::string const cuda_name = "substrings_count_cuda:" + dictionary_label; + bench_nullary(env, cuda_name, serial_call, cuda_call, callable_no_op_t {}, arrays_equality<size_t> {}) + .log(serial_result); + + // Sized from the serial `try_count` above, like the CPU `try_find` calls, so the scatter never grows + // device memory mid-benchmark. + size_t device_found = 0; + unified_vector<substrings_match_t> device_matches(total_occurrences); + auto cuda_find_call = find_with(device_engine, device_executor, gpu_specs, device_matches, device_found); + std::string const cuda_find_name = "substrings_find_cuda:" + dictionary_label; + bench_nullary(env, cuda_find_name, serial_find_call, cuda_find_call, callable_no_op_t {}, + arrays_equality<substrings_match_t> {}) + .log(serial_find_result); +#endif + + // Rewriting and scoring, the two capabilities that reached the GPU last. Both are sized once against the + // serial engine so no call allocates inside a measured loop. + { + // Every needle collapses to the same two bytes, so the rewrite shrinks and its cost is the splice + // rather than the vocabulary - one shared literal keeps the table itself out of the measurement. + static char const replacement_literal[] = "<>"; + unified_vector<span<char const>> replacement_spans( + needles.terms.size(), span<char const> {replacement_literal, sizeof(replacement_literal) - 1}); + span<span<char const> const> const replacements {replacement_spans.data(), replacement_spans.size()}; + + // Shared by the serial and device rewrites below, so it lives where a kernel can write it. + unified_vector<size_t> rewrite_offsets(env.tokens.size() + 1, 0); + span<size_t> const rewrite_offsets_view(rewrite_offsets.data(), rewrite_offsets.size()); + // A zero-capacity rewrite is the size query, so it refuses and names what it wanted. Letting that + // refusal pass unread would leave `rewrite_bytes` at zero and time every rewrite below against an + // empty tape, which still prints a throughput. + size_t rewrite_bytes = 0; + status_t const rewrite_sizing = serial_engine.try_replace( + env.tokens, substrings_leftmost_longest_k, replacements, span<char>(), rewrite_offsets_view, rewrite_bytes); + if (rewrite_sizing != status_t::unexpected_dimensions_k || rewrite_bytes == 0) + throw std::runtime_error("Rewrite sizing query did not name an output size"); + unified_vector<char> rewritten(rewrite_bytes); + + size_t serial_rewrote = 0, parallel_rewrote = 0; + auto replace_with = [&](auto &engine, auto &&executor, auto const &specs, size_t &written) { + return make_substrings_callable(haystack_bytes, rewritten, [&] { + return engine.try_replace(env.tokens, substrings_leftmost_longest_k, replacements, + span<char>(rewritten.data(), rewritten.size()), rewrite_offsets_view, written, + executor, specs); + }); + }; + + auto serial_replace_call = replace_with(serial_engine, serial_executor, cpu_specs, serial_rewrote); + std::string const serial_replace_name = "substrings_replace_serial:" + dictionary_label; + bench_result_t const serial_replace_result = bench_nullary(env, serial_replace_name, serial_replace_call).log(); + + auto parallel_replace_call = replace_with(parallel_engine, pool, cpu_specs, parallel_rewrote); + bench_nullary(env, "substrings_replace_parallel:" + dictionary_label, serial_replace_call, + parallel_replace_call, callable_no_op_t {}, arrays_equality<char> {}) + .log(serial_replace_result); + + // Read per needle by the scoring kernel, so the device backend needs it reachable from there. + unified_vector<float> const weights(needles.terms.size(), 1.0f); + span<float const> const weights_view(weights.data(), weights.size()); + unified_vector<float> scores(env.tokens.size(), 0.0f); + substrings_bm25_t bm25_parameters; + bm25_parameters.average_document_length = env.tokens.size() ? (float)haystack_bytes / (float)env.tokens.size() + : 0.0f; + + auto score_with = [&](auto &engine, auto &&executor, auto const &specs) { + return make_substrings_callable(haystack_bytes, scores, [&] { + return engine.try_score_bm25(env.tokens, span<float const>(), bm25_parameters, weights_view, + span<float>(scores.data(), scores.size()), executor, specs); + }); + }; + + auto serial_score_call = score_with(serial_engine, serial_executor, cpu_specs); + std::string const serial_score_name = "substrings_score_bm25_serial:" + dictionary_label; + bench_result_t const serial_score_result = bench_nullary(env, serial_score_name, serial_score_call).log(); + + // Scores carry no equality validator where every other accelerated cell does: each backend combines + // the per-needle terms in an order fixed by its own shape, so the sums agree numerically but not to + // the last bit, and an exact comparison would report a mismatch on correct output. + auto parallel_score_call = score_with(parallel_engine, pool, cpu_specs); + bench_nullary(env, "substrings_score_bm25_parallel:" + dictionary_label, parallel_score_call) + .log(serial_score_result); + +#if SZ_USE_CUDA + size_t device_rewrote = 0; + auto cuda_replace_call = replace_with(device_engine, device_executor, gpu_specs, device_rewrote); + bench_nullary(env, "substrings_replace_cuda:" + dictionary_label, serial_replace_call, cuda_replace_call, + callable_no_op_t {}, arrays_equality<char> {}) + .log(serial_replace_result); + + auto cuda_score_call = score_with(device_engine, device_executor, gpu_specs); + bench_nullary(env, "substrings_score_bm25_cuda:" + dictionary_label, cuda_score_call).log(serial_score_result); +#endif + } +} + +/** @brief The whole sweep: builds the vocabulary once, then walks every slice and sensitivity on every + * backend this build links. */ +void bench_substrings(environment_t const &env) { + // `STRINGWARS_UNIQUE` deduplicates `env.tokens` in place, leaving both cutoffs nothing to rank by. + if (env.unique) + std::printf( // + "WARNING: STRINGWARS_UNIQUE is set, so every term counts once and both frequency cutoffs " // + "are meaningless. Unset it for a real vocabulary.\n"); + size_t const haystack_bytes = std::accumulate( + env.tokens.begin(), env.tokens.end(), (size_t)0, + [](size_t total, token_view_t const &token) noexcept { return total + token.size(); }); + std::printf(" - Haystacks: %zu spans, %zu bytes total\n", env.tokens.size(), haystack_bytes); + vocabulary_t const vocabulary = build_vocabulary(env); + std::printf( // + " - Vocabulary: %zu terms after cutoffs, %zu occurrences, dropped %zu most-frequent and " // + "%zu under-2-occurrence\n", // + vocabulary.size(), vocabulary.total_occurrences, vocabulary.dropped_frequent, vocabulary.dropped_hapax); + std::printf( // + " - Length gates: dropped %zu words under %zu bytes and %zu over %zu bytes\n", // + vocabulary.dropped_short, vocabulary_min_word_bytes_k, vocabulary.dropped_long, vocabulary_max_word_bytes_k); + + forkunion_executor_t pool; + if (pool.try_spawn(std::thread::hardware_concurrency()) != status_t::success_k) + throw std::runtime_error("Failed to spawn the thread pool."); + cpu_specs_t const cpu_specs = pool.specs(); + + std::printf("Starting substrings benchmarks...\n"); + for (substrings_sweep_cell_t const &cell : substrings_sweep_k) + bench_substrings_dictionary(env, haystack_bytes, vocabulary, cell, cpu_specs, pool); +} + +#pragma endregion Sweep + +} // namespace scripts +} // namespace stringzilla +} // namespace ashvardanian diff --git a/bench/token.cpp b/bench/token.cpp index da6dad81..5d3e8cf2 100644 --- a/bench/token.cpp +++ b/bench/token.cpp @@ -4,16 +4,22 @@ * The program accepts a file path to a dataset, tokenizes it, and benchmarks the search operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Memory-bound: token hashing, equality, ordering, and copies are bandwidth-limited, so it reads the whole file by default. + * * Benchmarks include: * - Checksum calculation and hashing for each token - @b bytesum and @b hash. - * - Stream hashing of a token (file, lines, or words) - @b hash_init, @b hash_stream, @b hash_fold. + * - Stream hashing of a token (file, lines, or words) - @b sz_hash_state_init, @b sz_hash_state_update, + * @b sz_hash_state_digest. * - Equality check between two tokens and their relative order - @b equal and @b ordering. + * - Multi-seed hashing, and SHA-256 digest computation - @b bench_hashing_multiseed, @b bench_sha256, + * @b bench_sha256_multistate. * * For token operations, the number of operations per second are reported as the number of bytes processed * or comparisons performed, depending on the specific operation being benchmarked. * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=0` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -44,7 +50,7 @@ * @endcode * * Unlike the full-blown StringWars, it doesn't use any external frameworks like Criterion or Google Benchmark. - * This file is the sibling of `bench_find.cpp`, `bench_sequence.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `find.cpp`, `sequence.cpp`, and `memory.cpp`. */ #include <numeric> // `std::accumulate` #include <array> // `std::array` @@ -89,62 +95,6 @@ struct bytesum_from_std_t { } }; -/** @brief Wraps a hardware-specific UTF-8 character counting backend. */ -template <sz_utf8_count_t func_> -struct utf8_count_from_sz { - - environment_t const &env; - inline call_result_t operator()(std::size_t token_index) const noexcept { - return operator()(env.tokens[token_index]); - } - - inline call_result_t operator()(std::string_view buffer) const noexcept { - sz_size_t char_count = func_(buffer.data(), buffer.size()); - do_not_optimize(char_count); - return {buffer.size(), static_cast<check_value_t>(char_count)}; - } -}; - -/** @brief Wraps a hardware-specific UTF-8 to UTF-32 unpacking backend. */ -template <sz_utf8_decode_t func_> -struct utf8_unpack_from_sz { - - environment_t const &env; - mutable sz_rune_t runes[64]; // Reusable buffer to avoid repeated stack allocation - - inline call_result_t operator()(std::size_t token_index) const noexcept { - return operator()(env.tokens[token_index]); - } - - inline call_result_t operator()(std::string_view buffer) const noexcept { - check_value_t checksum = 0; - sz_size_t total_bytes = 0; - sz_cptr_t text = buffer.data(); - sz_size_t remaining = buffer.size(); - - // Process entire token, letting the function decode as much as it can each iteration - while (remaining > 0) { - sz_size_t unpacked_count = 0; - sz_cptr_t next = func_(text, remaining, runes, 64, &unpacked_count); - - // Compute checksum of decoded runes - for (sz_size_t i = 0; i < unpacked_count; i++) checksum += static_cast<check_value_t>(runes[i]); - - sz_size_t bytes_consumed = next - text; - total_bytes += bytes_consumed; - text = next; - remaining -= bytes_consumed; - - // Safety check: if no progress, break to avoid infinite loop - if (bytes_consumed == 0) break; - } - - do_not_optimize(runes); - do_not_optimize(checksum); - return {total_bytes, checksum}; - } -}; - /** @brief Wraps a hardware-specific hashing backend into something similar to @b `std::hash`. */ template <sz_hash_t func_> struct hash_from_sz { @@ -277,68 +227,6 @@ void bench_checksums(environment_t const &env) { #endif } -void bench_utf8_count(environment_t const &env) { - - auto validator = utf8_count_from_sz<sz_utf8_count_serial> {env}; - bench_result_t base = bench_unary(env, "sz_utf8_count_serial", validator).log(); - -#if SZ_USE_HASWELL - bench_unary(env, "sz_utf8_count_haswell", validator, utf8_count_from_sz<sz_utf8_count_haswell> {env}).log(base); -#endif -#if SZ_USE_ICELAKE - bench_unary(env, "sz_utf8_count_icelake", validator, utf8_count_from_sz<sz_utf8_count_icelake> {env}).log(base); -#endif -#if SZ_USE_NEON - bench_unary(env, "sz_utf8_count_neon", validator, utf8_count_from_sz<sz_utf8_count_neon> {env}).log(base); -#endif -#if SZ_USE_SVE2 - bench_unary(env, "sz_utf8_count_sve2", validator, utf8_count_from_sz<sz_utf8_count_sve2> {env}).log(base); -#endif -#if SZ_USE_V128 - bench_unary(env, "sz_utf8_count_v128", validator, utf8_count_from_sz<sz_utf8_count_v128> {env}).log(base); -#endif -#if SZ_USE_V128RELAXED - bench_unary(env, "sz_utf8_count_v128relaxed", validator, utf8_count_from_sz<sz_utf8_count_v128relaxed> {env}) - .log(base); -#endif -#if SZ_USE_RVV - bench_unary(env, "sz_utf8_count_rvv", validator, utf8_count_from_sz<sz_utf8_count_rvv> {env}).log(base); -#endif -#if SZ_USE_LASX - bench_unary(env, "sz_utf8_count_lasx", validator, utf8_count_from_sz<sz_utf8_count_lasx> {env}).log(base); -#endif -#if SZ_USE_POWERVSX - bench_unary(env, "sz_utf8_count_powervsx", validator, utf8_count_from_sz<sz_utf8_count_powervsx> {env}).log(base); -#endif -} - -void bench_utf8_unpack(environment_t const &env) { - - auto validator = utf8_unpack_from_sz<sz_utf8_decode_serial> {env, {}}; - bench_result_t base = bench_unary(env, "sz_utf8_decode_serial", validator).log(); - -#if SZ_USE_ICELAKE - bench_unary(env, "sz_utf8_decode_icelake", validator, utf8_unpack_from_sz<sz_utf8_decode_icelake> {env, {}}) - .log(base); -#endif -#if SZ_USE_NEON - bench_unary(env, "sz_utf8_decode_neon", validator, utf8_unpack_from_sz<sz_utf8_decode_neon> {env, {}}).log(base); -#endif -#if SZ_USE_V128 - bench_unary(env, "sz_utf8_decode_v128", validator, utf8_unpack_from_sz<sz_utf8_decode_v128> {env, {}}).log(base); -#endif -#if SZ_USE_RVV - bench_unary(env, "sz_utf8_decode_rvv", validator, utf8_unpack_from_sz<sz_utf8_decode_rvv> {env, {}}).log(base); -#endif -#if SZ_USE_LASX - bench_unary(env, "sz_utf8_decode_lasx", validator, utf8_unpack_from_sz<sz_utf8_decode_lasx> {env, {}}).log(base); -#endif -#if SZ_USE_POWERVSX - bench_unary(env, "sz_utf8_decode_powervsx", validator, utf8_unpack_from_sz<sz_utf8_decode_powervsx> {env, {}}) - .log(base); -#endif -} - void bench_hashing(environment_t const &env) { auto validator = hash_from_sz<sz_hash_serial> {env}; @@ -398,6 +286,15 @@ void bench_hashing_multiseed(environment_t const &env) { bench_unary(env, "sz_hash_multiseed_neonaes", validator, hash_multiseed_from_sz<sz_hash_multiseed_neonaes> {env}) .log(base); #endif +#if SZ_USE_V128 + bench_unary(env, "sz_hash_multiseed_v128", validator, hash_multiseed_from_sz<sz_hash_multiseed_v128> {env}) + .log(base); +#endif +#if SZ_USE_V128RELAXED + bench_unary(env, "sz_hash_multiseed_v128relaxed", validator, + hash_multiseed_from_sz<sz_hash_multiseed_v128relaxed> {env}) + .log(base); +#endif } void bench_stream_hashing(environment_t const &env) { @@ -430,11 +327,18 @@ void bench_stream_hashing(environment_t const &env) { #endif #if SZ_USE_NEONAES bench_unary( - env, "sz_hash_stream_neon", validator, + env, "sz_hash_stream_neonaes", validator, hash_stream_from_sz<sz_hash_state_init_neonaes, sz_hash_state_update_neonaes, sz_hash_state_digest_neonaes> { env}) .log(base, base_stl); #endif +#if SZ_USE_SVE2AES + bench_unary( + env, "sz_hash_stream_sve2aes", validator, + hash_stream_from_sz<sz_hash_state_init_sve2aes, sz_hash_state_update_sve2aes, sz_hash_state_digest_sve2aes> { + env}) + .log(base, base_stl); +#endif #if SZ_USE_V128 bench_unary( env, "sz_hash_stream_v128", validator, @@ -505,7 +409,7 @@ struct sha256_lanes_uniform_t { static inline std::size_t length(std::size_t, std::size_t token_length) noexcept { return token_length; } }; -/** @brief One lane far shorter than the rest, which used to drop its whole group to the scalar kernel. */ +/** @brief One lane far shorter than the rest, which drops its whole group to the scalar kernel. */ struct sha256_lanes_one_short_t { static constexpr char const *name_k = "_one_short"; static inline std::size_t length(std::size_t lane_index, std::size_t token_length) noexcept { @@ -612,26 +516,29 @@ static void bench_sha256_multistate_shape(environment_t const &env, std::string auto validator = sha256_multistate_loop_from_sz<lanes_> {env}; bench_result_t base = bench_unary(env, "sz_sha256_multistate_loop" + suffix, validator).log(); - bench_unary(env, "sz_sha256_multistate_serial" + suffix, validator, - sha256_multistate_from_sz<sz_sha256_multistate_update_serial, sz_sha256_multistate_digest_serial, - lanes_> {env}) + bench_unary( + env, "sz_sha256_multistate_serial" + suffix, validator, + sha256_multistate_from_sz<sz_sha256_multistate_update_serial, sz_sha256_multistate_digest_serial, lanes_> {env}) .log(base); #if SZ_USE_GOLDMONT - bench_unary(env, "sz_sha256_multistate_goldmont" + suffix, validator, - sha256_multistate_from_sz<sz_sha256_multistate_update_goldmont, sz_sha256_multistate_digest_goldmont, - lanes_> {env}) + bench_unary( + env, "sz_sha256_multistate_goldmont" + suffix, validator, + sha256_multistate_from_sz<sz_sha256_multistate_update_goldmont, sz_sha256_multistate_digest_goldmont, lanes_> { + env}) .log(base); #endif #if SZ_USE_HASWELL - bench_unary(env, "sz_sha256_multistate_haswell" + suffix, validator, - sha256_multistate_from_sz<sz_sha256_multistate_update_haswell, sz_sha256_multistate_digest_haswell, - lanes_> {env}) + bench_unary( + env, "sz_sha256_multistate_haswell" + suffix, validator, + sha256_multistate_from_sz<sz_sha256_multistate_update_haswell, sz_sha256_multistate_digest_haswell, lanes_> { + env}) .log(base); #endif #if SZ_USE_SKYLAKE - bench_unary(env, "sz_sha256_multistate_skylake" + suffix, validator, - sha256_multistate_from_sz<sz_sha256_multistate_update_skylake, sz_sha256_multistate_digest_skylake, - lanes_> {env}) + bench_unary( + env, "sz_sha256_multistate_skylake" + suffix, validator, + sha256_multistate_from_sz<sz_sha256_multistate_update_skylake, sz_sha256_multistate_digest_skylake, lanes_> { + env}) .log(base); #endif } @@ -655,7 +562,7 @@ void bench_sha256(environment_t const &env) { .log(base); #endif #if SZ_USE_NEONSHA - bench_unary(env, "sz_sha256_neon", validator, + bench_unary(env, "sz_sha256_neonsha", validator, sha256_stream_from_sz<sz_sha256_state_init_neonsha, sz_sha256_state_update_neonsha, sz_sha256_state_digest_neonsha> {env}) .log(base); @@ -745,7 +652,7 @@ struct equality_from_memcmp_t { }; /** - * @brief Wraps a hardware-specific order-checking backend into something similar to @b `std::equal_to`. + * @brief Wraps a hardware-specific order-checking backend into something similar to @b `std::less`. * Assuming that almost any random pair of strings would differ in the very first byte, to make benchmarks * more similar to mixed cases, like Hash Table lookups, where during probing we meet both differing * and equivalent strings. @@ -809,6 +716,9 @@ void bench_comparing_equality(environment_t const &env) { bench_result_t base = bench_unary(env, "sz_equal_serial", validator, equality_from_sz<sz_equal_serial> {env}).log(); bench_result_t base_stl = bench_unary(env, "equal<std::memcmp>", validator).log(base); +#if SZ_USE_WESTMERE + bench_unary(env, "sz_equal_westmere", validator, equality_from_sz<sz_equal_westmere> {env}).log(base, base_stl); +#endif #if SZ_USE_HASWELL bench_unary(env, "sz_equal_haswell", validator, equality_from_sz<sz_equal_haswell> {env}).log(base, base_stl); #endif @@ -845,6 +755,9 @@ void bench_comparing_order(environment_t const &env) { bench_result_t base = bench_unary(env, "sz_order_serial", validator, ordering_from_sz<sz_order_serial> {env}).log(); bench_result_t base_stl = bench_unary(env, "order<std::memcmp>", validator).log(base); +#if SZ_USE_WESTMERE + bench_unary(env, "sz_order_westmere", validator, ordering_from_sz<sz_order_westmere> {env}).log(base, base_stl); +#endif #if SZ_USE_HASWELL bench_unary(env, "sz_order_haswell", validator, ordering_from_sz<sz_order_haswell> {env}).log(base, base_stl); #endif @@ -854,6 +767,9 @@ void bench_comparing_order(environment_t const &env) { #if SZ_USE_NEON bench_unary(env, "sz_order_neon", validator, ordering_from_sz<sz_order_neon> {env}).log(base, base_stl); #endif +#if SZ_USE_SVE + bench_unary(env, "sz_order_sve", validator, ordering_from_sz<sz_order_sve> {env}).log(base, base_stl); +#endif #if SZ_USE_V128 bench_unary(env, "sz_order_v128", validator, ordering_from_sz<sz_order_v128> {env}).log(base, base_stl); #endif @@ -889,8 +805,6 @@ int main(int argc, char const **argv) { // Unary operations bench_checksums(env); - bench_utf8_count(env); - bench_utf8_unpack(env); bench_hashing(env); bench_hashing_multiseed(env); bench_stream_hashing(env); diff --git a/bench/utf8_norm.cpp b/bench/utf8_norm.cpp index 8944bb23..982f94c1 100644 --- a/bench/utf8_norm.cpp +++ b/bench/utf8_norm.cpp @@ -1,23 +1,26 @@ /** - * @file scripts/bench_utf8_norm.cpp + * @file bench/utf8_norm.cpp * @brief Benchmarks the @b `sz_utf8_norm_*` family — Unicode normalization and quick-check scanning. * The program accepts a file path to a dataset and benchmarks the normalization operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Compute-bound: Unicode normalization is table- and branch-heavy per codepoint, so a 64 MiB slice exercises every path on the multilingual corpus. + * * Benchmarks include: * - Unicode normalization for UTF-8 text - @b utf8_norm. * - Normalization-form violation scanning (quick-check) - @b utf8_find_denormalized. * * Both sections normalize to @b NFC, the most common interchange form. * - * Its sibling @b `bench_utf8_uncased.cpp` covers the @b `sz_utf8_uncased_*` family (case folding and - * uncased substring search), and @b `bench_utf8_iterate.cpp` covers the @b `sz_utf8_*` iteration/segmentation - * family (codepoint counting, Nth-codepoint, newline/whitespace scanning, UAX-29 word/grapheme/sentence - * boundaries, UAX-14 line breaking, transcoding). + * Its sibling @b `utf8_uncased.cpp` covers the @b `sz_utf8_uncased_*` family (case folding and + * uncased substring search), and @b `utf8_traverse.cpp`, @b `utf8_scan.cpp`, and @b `utf8_segment.cpp` cover + * the @b `sz_utf8_*` iteration/segmentation family (codepoint counting, Nth-codepoint, newline/whitespace + * scanning, UAX-29 word/grapheme/sentence boundaries, UAX-14 line breaking, transcoding). * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. - * - `STRINGWARS_TOKENS=line` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams. * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * * Unlike StringWars, the following additional environment variables are supported: @@ -27,6 +30,7 @@ * - `STRINGWARS_STRESS_LIMIT=1` : Controls the number of failures we're willing to tolerate. * - `STRINGWARS_STRESS_DURATION=10` : Stress-testing time limit (in seconds) per benchmark. * - `STRINGWARS_FILTER` : Regular Expression pattern to filter algorithm/backend names. + * - `STRINGWARS_UNIQUE=1` : Deduplicates tokens, sorting the set and dropping duplicates before benchmarking. * * Here are a few build & run commands: * @@ -37,8 +41,8 @@ * build_release/stringzilla_bench_utf8_norm_cpp20 * @endcode * - * This file is the sibling of `bench_utf8_uncased.cpp`, `bench_utf8_iterate.cpp`, `bench_token.cpp`, - * `bench_find.cpp`, `bench_sequence.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `utf8_uncased.cpp`, `utf8_traverse.cpp`, `utf8_scan.cpp`, `utf8_segment.cpp`, + * `token.cpp`, `find.cpp`, `sequence.cpp`, and `memory.cpp`. */ #include "shared.hpp" #include "stringzilla.hpp" // `log_environment` @@ -221,7 +225,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Default to xlsum for multilingual testing - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting Unicode benchmarks...\n"); diff --git a/bench/utf8_scan.cpp b/bench/utf8_scan.cpp index 90c2ed55..57a72698 100644 --- a/bench/utf8_scan.cpp +++ b/bench/utf8_scan.cpp @@ -5,6 +5,8 @@ * across all available SIMD backends side-by-side, and each backend's result is validated (via a * per-call checksum) against the serial reference — so this file doubles as a differential harness. * + * Compute-bound: per-codepoint class scanning is branch-heavy, so a 64 MiB slice exercises every path. + * * Benchmarks include: * - Newline enumeration - @b utf8_newlines. * - Whitespace enumeration - @b utf8_whitespaces (Unicode White_Space property). @@ -12,6 +14,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or [1:200] for N-grams). * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -29,7 +32,7 @@ * build_release/stringzilla_bench_utf8_scan_cpp20 * @endcode * - * This file is the sibling of `bench_utf8_traverse.cpp`, `bench_utf8_segment.cpp`, and `bench_utf8_uncased.cpp`. + * This file is the sibling of `utf8_traverse.cpp`, `utf8_segment.cpp`, and `utf8_uncased.cpp`. */ #include <vector> @@ -174,7 +177,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Default to xlsum for multilingual coverage - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting UTF-8 class-scan benchmarks...\n"); diff --git a/bench/utf8_segment.cpp b/bench/utf8_segment.cpp index 64540d9f..b4d28396 100644 --- a/bench/utf8_segment.cpp +++ b/bench/utf8_segment.cpp @@ -5,6 +5,8 @@ * each backend's result is validated (via a per-call checksum) against the serial reference — so this * file doubles as a differential correctness harness. * + * Compute-bound: UTF-8 segmentation is branch-heavy per codepoint, so a 64 MiB slice exercises every path. + * * Benchmarks include: * - UAX-29 word-boundary segmentation - @b utf8_wordbreaks. * - UAX-29 grapheme-cluster segmentation - @b utf8_graphemes. @@ -13,6 +15,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or [1:200] for N-grams). * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -30,7 +33,7 @@ * build_release/stringzilla_bench_utf8_segment_cpp20 * @endcode * - * This file is the sibling of `bench_utf8_traverse.cpp`, `bench_utf8_scan.cpp`, and `bench_utf8_uncased.cpp`. + * This file is the sibling of `utf8_traverse.cpp`, `utf8_scan.cpp`, and `utf8_uncased.cpp`. */ #include <vector> @@ -187,7 +190,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Default to xlsum for multilingual coverage - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting UTF-8 segmentation benchmarks...\n"); diff --git a/bench/utf8_traverse.cpp b/bench/utf8_traverse.cpp index f0921305..bb897033 100644 --- a/bench/utf8_traverse.cpp +++ b/bench/utf8_traverse.cpp @@ -5,6 +5,8 @@ * backend's result is validated (via a per-call checksum) against the serial reference — so this file * doubles as a differential correctness harness. * + * Compute-bound: codepoint iteration is branch-heavy, so a 64 MiB slice exercises every path. + * * Benchmarks include: * - Codepoint counting - @b utf8_count. * - Nth-codepoint location - @b utf8_seek (the BMI/PDEP "Nth set bit" kernel on x86). @@ -12,6 +14,7 @@ * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or [1:200] for N-grams). * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * @@ -29,7 +32,7 @@ * build_release/stringzilla_bench_utf8_traverse_cpp20 * @endcode * - * This file is the sibling of `bench_utf8_scan.cpp`, `bench_utf8_segment.cpp`, and `bench_utf8_uncased.cpp`. + * This file is the sibling of `utf8_scan.cpp`, `utf8_segment.cpp`, and `utf8_uncased.cpp`. */ #include <vector> @@ -213,7 +216,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Default to xlsum for multilingual coverage - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting UTF-8 traversal benchmarks...\n"); diff --git a/bench/utf8_uncased.cpp b/bench/utf8_uncased.cpp index 9c4a4dfc..ddce6072 100644 --- a/bench/utf8_uncased.cpp +++ b/bench/utf8_uncased.cpp @@ -1,20 +1,23 @@ /** - * @file scripts/bench_utf8_uncased.cpp + * @file bench/utf8_uncased.cpp * @brief Benchmarks the @b `sz_utf8_uncased_*` family — case folding and uncased search. * The program accepts a file path to a dataset and benchmarks the case folding operations, * validating the SIMD-accelerated backends against the serial baselines. * + * Compute-bound: case-folded search is table- and branch-heavy per codepoint, so a 64 MiB slice exercises every path. + * * Benchmarks include: * - Case folding for Unicode text - @b utf8_uncased_fold. * - Uncased substring search for Unicode text - @b utf8_uncased_search. * - * Its sibling @b `bench_utf8_iterate.cpp` covers the @b `sz_utf8_*` iteration/segmentation family - * (codepoint counting, Nth-codepoint, newline/whitespace scanning, UAX-29 word/grapheme/sentence boundaries, - * UAX-14 line breaking, transcoding). + * Its siblings @b `utf8_traverse.cpp`, @b `utf8_scan.cpp`, and @b `utf8_segment.cpp` cover the + * @b `sz_utf8_*` iteration/segmentation family (codepoint counting, Nth-codepoint, newline/whitespace + * scanning, UAX-29 word/grapheme/sentence boundaries, UAX-14 line breaking, transcoding). * * Instead of CLI arguments, for compatibility with @b StringWars, the following environment variables are used: * - `STRINGWARS_DATASET` : Path to the dataset file. - * - `STRINGWARS_TOKENS=line` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams. + * - `STRINGWARS_DATASET_LIMIT=64mb` : Reads at most this many dataset bytes; `0` reads the whole file. + * - `STRINGWARS_TOKENS=lines` : Tokenization model ("file", "lines", "words", or positive integer [1:200] for N-grams. * - `STRINGWARS_SEED=42` : Optional seed for shuffling reproducibility. * * Unlike StringWars, the following additional environment variables are supported: @@ -24,6 +27,7 @@ * - `STRINGWARS_STRESS_LIMIT=1` : Controls the number of failures we're willing to tolerate. * - `STRINGWARS_STRESS_DURATION=10` : Stress-testing time limit (in seconds) per benchmark. * - `STRINGWARS_FILTER` : Regular Expression pattern to filter algorithm/backend names. + * - `STRINGWARS_UNIQUE=1` : Deduplicates tokens, sorting the set and dropping duplicates before benchmarking. * * Here are a few build & run commands: * @@ -34,8 +38,8 @@ * build_release/stringzilla_bench_utf8_uncased_cpp20 * @endcode * - * This file is the sibling of `bench_utf8_iterate.cpp`, `bench_token.cpp`, `bench_find.cpp`, - * `bench_sequence.cpp`, and `bench_memory.cpp`. + * This file is the sibling of `utf8_traverse.cpp`, `utf8_scan.cpp`, `utf8_segment.cpp`, `token.cpp`, + * `find.cpp`, `sequence.cpp`, and `memory.cpp`. */ #include "shared.hpp" #include "stringzilla.hpp" // `log_environment` @@ -286,7 +290,7 @@ int main(int argc, char const **argv) { environment_t env = build_environment( // argc, argv, // "xlsum.csv", // Default to xlsum for multilingual testing - environment_t::tokenization_t::lines_k); + environment_t::tokenization_t::lines_k, compute_bound_slice_bytes_k); std::printf("Starting Unicode benchmarks...\n"); diff --git a/build.rs b/build.rs index 0911c991..7d8a9b10 100644 --- a/build.rs +++ b/build.rs @@ -657,24 +657,26 @@ fn stringzillas_base_build(serial_flags: &HashMap<String, bool>) -> cc::Build { } /// StringZillas C-API entry units, compiled once per backend - as C++ into the CPU one, as CUDA into the GPU one. -const STRINGZILLAS_API_CPP_SOURCES: [&str; 5] = [ +const STRINGZILLAS_API_CPP_SOURCES: [&str; 6] = [ "c/stringzillas/runtime.cpp", "c/stringzillas/levenshtein.cpp", "c/stringzillas/needleman_wunsch.cpp", "c/stringzillas/smith_waterman.cpp", "c/stringzillas/fingerprints.cpp", + "c/stringzillas/substrings.cpp", ]; -const STRINGZILLAS_API_CU_SOURCES: [&str; 5] = [ +const STRINGZILLAS_API_CU_SOURCES: [&str; 6] = [ "c/stringzillas/runtime.cu", "c/stringzillas/levenshtein.cu", "c/stringzillas/needleman_wunsch.cu", "c/stringzillas/smith_waterman.cu", "c/stringzillas/fingerprints.cu", + "c/stringzillas/substrings.cu", ]; /// Per-ISA CPU instantiation units, host C++ in every backend - NVCC forwards `.cpp` straight to the host /// compiler, keeping CPU SIMD out of its frontend; off-platform files compile to empty objects. -const STRINGZILLAS_CPUS_SOURCES: [&str; 15] = [ +const STRINGZILLAS_CPUS_SOURCES: [&str; 16] = [ "c/stringzillas/levenshtein_serial.cpp", "c/stringzillas/levenshtein_icelake.cpp", "c/stringzillas/levenshtein_haswell.cpp", @@ -690,14 +692,16 @@ const STRINGZILLAS_CPUS_SOURCES: [&str; 15] = [ "c/stringzillas/smith_waterman_haswell.cpp", "c/stringzillas/smith_waterman_neon.cpp", "c/stringzillas/smith_waterman_rvv.cpp", + "c/stringzillas/substrings_serial.cpp", ]; /// Per-tier GPU instantiation units, grouped by architecture floor: Hopper DPX needs sm_90, the rest run /// from the base set. -const STRINGZILLAS_CUDA_SOURCES: [&str; 3] = [ +const STRINGZILLAS_CUDA_SOURCES: [&str; 4] = [ "c/stringzillas/levenshtein_cuda.cu", "c/stringzillas/needleman_wunsch_cuda.cu", "c/stringzillas/smith_waterman_cuda.cu", + "c/stringzillas/substrings_cuda.cu", ]; const STRINGZILLAS_KEPLER_SOURCES: [&str; 1] = ["c/stringzillas/levenshtein_kepler.cu"]; const STRINGZILLAS_HOPPER_SOURCES: [&str; 3] = [ diff --git a/build_backend.py b/build_backend.py index 45dc6469..c60c663e 100644 --- a/build_backend.py +++ b/build_backend.py @@ -180,6 +180,7 @@ def cli_run_tests(project_dir: Optional[str] = None) -> None: "--ignore=" + str(proj / "test" / "stringzillas.py"), "--ignore=" + str(proj / "test" / "similarities.py"), "--ignore=" + str(proj / "test" / "fingerprints.py"), + "--ignore=" + str(proj / "test" / "substrings.py"), "--ignore=" + str(proj / "test" / "szs_helpers.py"), ] else: @@ -189,6 +190,7 @@ def cli_run_tests(project_dir: Optional[str] = None) -> None: str(proj / "test" / "stringzillas.py"), str(proj / "test" / "similarities.py"), str(proj / "test" / "fingerprints.py"), + str(proj / "test" / "substrings.py"), ] # A free-threaded interpreter is the only place a data race can surface, and only if the suite runs # concurrently. Doctests sit out, asserting against a process-wide stdout capture that threads interleave. diff --git a/c/stringzilla/dispatch.h b/c/stringzilla/dispatch.h index 8f9b4889..35286a1d 100644 --- a/c/stringzilla/dispatch.h +++ b/c/stringzilla/dispatch.h @@ -13,8 +13,8 @@ * wrappers that call through the table. The thin `runtime.c` owns the table definition and the * one-time initialization. */ -#ifndef STRINGZILLA_DISPATCH_H_ -#define STRINGZILLA_DISPATCH_H_ +#ifndef SZ_DISPATCH_H_ +#define SZ_DISPATCH_H_ // Overwrite `SZ_DYNAMIC_DISPATCH` before including StringZilla. #ifdef SZ_DYNAMIC_DISPATCH @@ -152,4 +152,4 @@ SZ_MAYBE_UNUSED SZ_C_INLINE sz_bool_t sz_sve_wider_than_neon_(void) { return svc #endif #endif // SZ_IS_64BIT_ARM_ && (SZ_USE_SVE || SZ_USE_SVE2) && !defined(_MSC_VER) -#endif // STRINGZILLA_DISPATCH_H_ +#endif // SZ_DISPATCH_H_ diff --git a/c/stringzilla/runtime.c b/c/stringzilla/runtime.c index f2e0fd46..3a91b6c7 100644 --- a/c/stringzilla/runtime.c +++ b/c/stringzilla/runtime.c @@ -112,5 +112,8 @@ SZ_API_RUNTIME sz_capability_t sz_capabilities(void) { return (sz_capability_t)(sz_capabilities_comptime_implementation_() & sz_capabilities_runtime_implementation_()); } SZ_API_RUNTIME sz_cptr_t sz_capabilities_to_string(sz_capability_t caps) { - return sz_capabilities_to_string_implementation_(caps); + // The one place that must own storage, because the signature returns a string it does not receive. + static char names[256]; + sz_capabilities_to_string_implementation_(caps, names, sizeof(names)); + return names; } diff --git a/c/stringzillas/fingerprints.cuh b/c/stringzillas/fingerprints.cuh index 24c82224..6f0032a3 100644 --- a/c/stringzillas/fingerprints.cuh +++ b/c/stringzillas/fingerprints.cuh @@ -4,8 +4,8 @@ * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_SZS_FINGERPRINTS_CUH_ -#define STRINGZILLAS_SZS_FINGERPRINTS_CUH_ +#ifndef SZS_FINGERPRINTS_CUH_ +#define SZS_FINGERPRINTS_CUH_ #include "stringzillas.cuh" /** @@ -264,4 +264,4 @@ SZ_API_RUNTIME void szs_fingerprints_utf8_free(szs_fingerprints_utf8_t engine_pu #pragma endregion Fingerprints UTF8 } -#endif // STRINGZILLAS_SZS_FINGERPRINTS_CUH_ +#endif // SZS_FINGERPRINTS_CUH_ diff --git a/c/stringzillas/levenshtein.cuh b/c/stringzillas/levenshtein.cuh index b55f9c25..0de9113e 100644 --- a/c/stringzillas/levenshtein.cuh +++ b/c/stringzillas/levenshtein.cuh @@ -4,8 +4,8 @@ * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_SZS_LEVENSHTEIN_CUH_ -#define STRINGZILLAS_SZS_LEVENSHTEIN_CUH_ +#ifndef SZS_LEVENSHTEIN_CUH_ +#define SZS_LEVENSHTEIN_CUH_ #include "stringzillas.cuh" /** @@ -368,4 +368,4 @@ SZ_API_RUNTIME void szs_levenshtein_distances_utf8_free(szs_levenshtein_distance #pragma endregion Levenshtein UTF8 Distances } -#endif // STRINGZILLAS_SZS_LEVENSHTEIN_CUH_ +#endif // SZS_LEVENSHTEIN_CUH_ diff --git a/c/stringzillas/needleman_wunsch.cuh b/c/stringzillas/needleman_wunsch.cuh index 4425f61b..1177598c 100644 --- a/c/stringzillas/needleman_wunsch.cuh +++ b/c/stringzillas/needleman_wunsch.cuh @@ -4,8 +4,8 @@ * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_SZS_NEEDLEMAN_WUNSCH_CUH_ -#define STRINGZILLAS_SZS_NEEDLEMAN_WUNSCH_CUH_ +#ifndef SZS_NEEDLEMAN_WUNSCH_CUH_ +#define SZS_NEEDLEMAN_WUNSCH_CUH_ #include "stringzillas.cuh" /** @@ -236,4 +236,4 @@ SZ_API_RUNTIME void szs_needleman_wunsch_scores_free(szs_needleman_wunsch_scores #pragma endregion Needleman Wunsch } -#endif // STRINGZILLAS_SZS_NEEDLEMAN_WUNSCH_CUH_ +#endif // SZS_NEEDLEMAN_WUNSCH_CUH_ diff --git a/c/stringzillas/runtime.cuh b/c/stringzillas/runtime.cuh index a9ca49ae..b73b6fe3 100644 --- a/c/stringzillas/runtime.cuh +++ b/c/stringzillas/runtime.cuh @@ -4,8 +4,8 @@ * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_RUNTIME_CUH_ -#define STRINGZILLAS_RUNTIME_CUH_ +#ifndef SZS_RUNTIME_CUH_ +#define SZS_RUNTIME_CUH_ #include "stringzillas.cuh" extern "C" { @@ -91,13 +91,12 @@ SZ_API_RUNTIME sz_status_t szs_device_scope_init_cpu_cores(sz_size_t cpu_cores, // If `cpu_cores` is 1, redirect to default scope if (cpu_cores == 1) return szs_device_scope_init_default(scope_punned, error_message); - sz::cpu_specs_t specs; auto executor = std::make_unique<szs::forkunion_executor_t>(); if (executor->try_spawn(cpu_cores) != sz::status_t::success_k) return propagate_error(sz::status_t::bad_alloc_k, error_message, "Failed to spawn thread pool"); + sz::cpu_specs_t const specs = executor->specs(); - auto *scope = new (std::nothrow) - device_scope_t(std::in_place_type_t<cpu_scope_t> {}, std::move(executor), std::move(specs)); + auto *scope = new (std::nothrow) device_scope_t(std::in_place_type_t<cpu_scope_t> {}, std::move(executor), specs); if (!scope) return propagate_error(sz::status_t::bad_alloc_k, error_message, "Failed to allocate CPU device scope"); *scope_punned = reinterpret_cast<szs_device_scope_t>(scope); @@ -223,4 +222,4 @@ SZ_API_RUNTIME void szs_unified_free(void *ptr, sz_size_t size_bytes) { #pragma endregion Unified Allocator } -#endif // STRINGZILLAS_RUNTIME_CUH_ +#endif // SZS_RUNTIME_CUH_ diff --git a/c/stringzillas/smith_waterman.cuh b/c/stringzillas/smith_waterman.cuh index 6c71e1c1..de209c73 100644 --- a/c/stringzillas/smith_waterman.cuh +++ b/c/stringzillas/smith_waterman.cuh @@ -4,8 +4,8 @@ * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_SZS_SMITH_WATERMAN_CUH_ -#define STRINGZILLAS_SZS_SMITH_WATERMAN_CUH_ +#ifndef SZS_SMITH_WATERMAN_CUH_ +#define SZS_SMITH_WATERMAN_CUH_ #include "stringzillas.cuh" /** @@ -235,4 +235,4 @@ SZ_API_RUNTIME void szs_smith_waterman_scores_free(szs_smith_waterman_scores_t e #pragma endregion Smith Waterman } -#endif // STRINGZILLAS_SZS_SMITH_WATERMAN_CUH_ +#endif // SZS_SMITH_WATERMAN_CUH_ diff --git a/c/stringzillas/stringzillas.cuh b/c/stringzillas/stringzillas.cuh index b449abd8..5d4061a2 100644 --- a/c/stringzillas/stringzillas.cuh +++ b/c/stringzillas/stringzillas.cuh @@ -1,24 +1,25 @@ /** - * @file c/stringzillas.cuh + * @file c/stringzillas/stringzillas.cuh * @brief StringZillas shared scaffolding (scopes, backend variant lists, dispatch) included by * the per-algorithm CPU & CUDA shims. * @author Ash Vardanian * @date March 23, 2025 */ -#ifndef STRINGZILLAS_SCAFFOLDING_CUH_ -#define STRINGZILLAS_SCAFFOLDING_CUH_ - -#include <stringzillas/stringzillas.h> // StringZillas library header +#ifndef SZS_STRINGZILLAS_CUH_ +#define SZS_STRINGZILLAS_CUH_ #include <cstring> // For `std::memcpy` -#include <variant> // For `std::variant` #include <string_view> // For `std::string_view` +#include <variant> // For `std::variant` +#include <stringzillas/stringzillas.h> // StringZillas library header +#include <stringzillas/substrings.hpp> // C++ templates for multi-pattern search #include <stringzillas/fingerprints.hpp> // C++ templates for string processing #include <stringzillas/similarities.hpp> // C++ templates for string similarity #if SZ_USE_CUDA +#include <stringzillas/substrings.cuh> // Parallel multi-pattern search in CUDA #include <stringzillas/fingerprints.cuh> // Parallel string processing in CUDA #include <stringzillas/similarities.cuh> // Parallel string similarity in CUDA #endif @@ -41,9 +42,15 @@ overloaded(callable_types_...) -> overloaded<callable_types_...>; /** Wraps a `sz_sequence_t` to feel like `std::vector<std::string_view>>` in the implementation layer. */ struct sz_sequence_as_cpp_container_t { + using self_t = sz_sequence_as_cpp_container_t; using value_type = std::string_view; + using iterator = sz::indexed_container_iterator<self_t>; + sz_sequence_t const *sequence_ = nullptr; + iterator begin() const noexcept { return iterator(*this, 0); } + iterator end() const noexcept { return iterator(*this, size()); } + std::size_t size() const noexcept { sz_assert_(sequence_ != nullptr && "Sequence must not be null"); return sequence_->count; @@ -59,9 +66,16 @@ struct sz_sequence_as_cpp_container_t { /** Wraps a `sz_sequence_u64tape_t` to feel like `std::vector<std::string_view>>` in the implementation layer. */ struct sz_sequence_u64tape_as_cpp_container_t { + using self_t = sz_sequence_u64tape_as_cpp_container_t; using value_type = std::string_view; + using offset_t = sz_u64_t; + using iterator = sz::indexed_container_iterator<self_t>; + sz_sequence_u64tape_t const *tape_ = nullptr; + iterator begin() const noexcept { return iterator(*this, 0); } + iterator end() const noexcept { return iterator(*this, size()); } + std::size_t size() const noexcept { sz_assert_(tape_ != nullptr && "Tape must not be null"); return tape_->count; @@ -71,13 +85,34 @@ struct sz_sequence_u64tape_as_cpp_container_t { sz_assert_(index < tape_->count && "Index out of bounds"); return {tape_->data + tape_->offsets[index], tape_->offsets[index + 1] - tape_->offsets[index]}; } + + /** @brief The contiguous block the elements slice; starts wherever `offsets[0]` points. */ + sz::span<char const> tape_bytes() const noexcept { + if (size() == 0) return {}; + return {tape_->data + tape_->offsets[0], (std::size_t)(tape_->offsets[size()] - tape_->offsets[0])}; + } + /** @brief Every element's length summed, in the tape's own offset width, without walking the elements. */ + offset_t tape_total_bytes() const noexcept { + return size() == 0 ? offset_t {} : (offset_t)(tape_->offsets[size()] - tape_->offsets[0]); + } + /** @brief One element's length, in the tape's own offset width. */ + offset_t tape_length_at(std::size_t index) const noexcept { + return (offset_t)(tape_->offsets[index + 1] - tape_->offsets[index]); + } }; /** Wraps a `sz_sequence_u32tape_t` to feel like `std::vector<std::string_view>>` in the implementation layer. */ struct sz_sequence_u32tape_as_cpp_container_t { + using self_t = sz_sequence_u32tape_as_cpp_container_t; using value_type = std::string_view; + using offset_t = sz_u32_t; + using iterator = sz::indexed_container_iterator<self_t>; + sz_sequence_u32tape_t const *tape_ = nullptr; + iterator begin() const noexcept { return iterator(*this, 0); } + iterator end() const noexcept { return iterator(*this, size()); } + std::size_t size() const noexcept { sz_assert_(tape_ != nullptr && "Tape must not be null"); return tape_->count; @@ -87,6 +122,20 @@ struct sz_sequence_u32tape_as_cpp_container_t { sz_assert_(index < tape_->count && "Index out of bounds"); return {tape_->data + tape_->offsets[index], tape_->offsets[index + 1] - tape_->offsets[index]}; } + + /** @brief The contiguous block the elements slice; starts wherever `offsets[0]` points. */ + sz::span<char const> tape_bytes() const noexcept { + if (size() == 0) return {}; + return {tape_->data + tape_->offsets[0], (std::size_t)(tape_->offsets[size()] - tape_->offsets[0])}; + } + /** @brief Every element's length summed, in the tape's own offset width, without walking the elements. */ + offset_t tape_total_bytes() const noexcept { + return size() == 0 ? offset_t {} : (offset_t)(tape_->offsets[size()] - tape_->offsets[0]); + } + /** @brief One element's length, in the tape's own offset width. */ + offset_t tape_length_at(std::size_t index) const noexcept { + return (offset_t)(tape_->offsets[index + 1] - tape_->offsets[index]); + } }; /** Convenience class for slicing a strided fingerprints output. */ @@ -224,7 +273,9 @@ inline sz_status_t propagate_error(sz::status_t status, char const **reporter_me case sz::status_t::unexpected_dimensions_k: *reporter_message = "Input/output size mismatch"; break; case sz::status_t::missing_gpu_k: *reporter_message = "GPU device not available or CUDA not initialized"; break; case sz::status_t::device_code_mismatch_k: *reporter_message = "Backend and executor mismatch"; break; - case sz::status_t::device_memory_mismatch_k: *reporter_message = "Use device-reachable or unified memory"; break; + case sz::status_t::device_memory_mismatch_k: + *reporter_message = "Use device-reachable or unified memory; page-locked host memory is not either"; + break; case sz::status_t::unknown_k: *reporter_message = "Unknown error"; break; default: *reporter_message = "Unrecognized error code"; break; } @@ -540,6 +591,25 @@ struct fingerprints_backends_t { : variants(std::forward<variants_arguments_>(args)...) {} }; +struct substrings_backends_t { + + /** + * Multi-pattern search has no per-ISA CPU kernels - a transition is one data-dependent load - so the + * alternatives are one per capability, exactly as the similarity engines list theirs. The state-id width + * is not an axis here: each engine settles it from its own needle set and stores whichever automaton won. + */ + std::variant< +#if SZ_USE_CUDA + szs::substrings_cuda_t, +#endif + szs::substrings_parallel_t, szs::substrings_serial_t> + variants; + + template <typename... variants_arguments_> + substrings_backends_t(variants_arguments_ &&...args) noexcept + : variants(std::forward<variants_arguments_>(args)...) {} +}; + template <typename texts_type_> sz_status_t szs_fingerprints_for_( // szs_fingerprints_t engine_punned, szs_device_scope_t device_punned, // @@ -665,4 +735,4 @@ sz_status_t szs_fingerprints_for_( // return result; } -#endif // STRINGZILLAS_SCAFFOLDING_CUH_ +#endif // SZS_STRINGZILLAS_CUH_ diff --git a/c/stringzillas/substrings.cpp b/c/stringzillas/substrings.cpp new file mode 100644 index 00000000..9d5ac9c1 --- /dev/null +++ b/c/stringzillas/substrings.cpp @@ -0,0 +1,7 @@ +/** + * @file c/stringzillas/substrings.cpp + * @brief StringZillas substrings shim, CPU backend. + * @author Ash Vardanian + * @date August 6, 2026 + */ +#include "substrings.cuh" diff --git a/c/stringzillas/substrings.cu b/c/stringzillas/substrings.cu new file mode 100644 index 00000000..94aba4c2 --- /dev/null +++ b/c/stringzillas/substrings.cu @@ -0,0 +1,7 @@ +/** + * @file c/stringzillas/substrings.cu + * @brief StringZillas substrings shim, CUDA backend. + * @author Ash Vardanian + * @date August 6, 2026 + */ +#include "substrings.cuh" diff --git a/c/stringzillas/substrings.cuh b/c/stringzillas/substrings.cuh new file mode 100644 index 00000000..2bf6a94f --- /dev/null +++ b/c/stringzillas/substrings.cuh @@ -0,0 +1,507 @@ +/** + * @file c/stringzillas/substrings.cuh + * @brief Multi-pattern Aho-Corasick search shim (CPU + CUDA backends). + * @author Ash Vardanian + * @date August 6, 2026 + */ +#ifndef SZS_SUBSTRINGS_CUH_ +#define SZS_SUBSTRINGS_CUH_ +#include "stringzillas.cuh" + +#include <cstddef> // `offsetof` + +#pragma region Dispatch + +/** + * @brief Allocates a `substrings_backends_t` holding an empty @p engine_type_ and publishes the opaque handle. + * + * The engine arrives with no automaton: indexing needs a device, which only `szs_substrings_index` names, and + * a default-constructed engine touches neither a driver nor a device allocation. + */ +template <typename engine_type_> +inline sz_status_t emplace_empty_substrings_engine(szs_substrings_t *engine_punned, + char const **error_message) noexcept { + auto engine = new (std::nothrow) substrings_backends_t(std::in_place_type_t<engine_type_>()); + if (!engine) + return propagate_error(sz::status_t::bad_alloc_k, error_message, "Failed to allocate Substrings engine"); + *engine_punned = reinterpret_cast<szs_substrings_t>(engine); + return propagate_error(sz::status_t::success_k, error_message); +} + +/** @brief Narrows the C policy to the engine's own enumeration, rather than assuming the two orders match. */ +inline szs::substrings_overlap_policy_t szs_substrings_policy_( + szs_substrings_overlap_policy_t overlap_policy) noexcept { + switch (overlap_policy) { + case szs_substrings_leftmost_longest_k: return szs::substrings_leftmost_longest_k; + case szs_substrings_leftmost_first_k: return szs::substrings_leftmost_first_k; + default: return szs::substrings_overlapping_k; + } +} + +/** + * @brief The compiled dictionary's needle count, which every per-needle array is validated against. + * + * Templated on the backends type so the engines' members instantiate where it is called rather than here: + * reaching into a CUDA engine at namespace scope pulls its kernel table's addresses ahead of the device + * stubs NVCC emits for them, which a Clang host then rejects as a specialization after instantiation. + */ +template <typename backends_type_> +inline size_t szs_substrings_needles_count_(backends_type_ const &engine) noexcept { + return std::visit([](auto const &engine_variant) { return (size_t)engine_variant.count_needles(); }, + engine.variants); +} + +/** + * @brief Runs @p operation on whichever engine and scope the two handles carry: the engine arm chosen by its + * own visit, the executor and specs by the scope's, and an engine-to-scope mismatch refused before + * either runs. + * + * Every substrings operation reaches its engine through here, so the four cannot drift apart in how they + * pair a backend with a device. @sa `szs_levenshtein_cross_` for the same skeleton over one operation. + */ +/** + * @brief Refuses an engine whose needles were never indexed, before any operation walks an empty automaton. + * + * Zero needles cannot mean anything else: `aho_corasick_dictionary::try_insert` rejects an empty needle, so an + * indexed engine always holds at least one. Without this a search would report zeros and call it success. + */ +inline sz_status_t szs_substrings_require_indexed_(szs_substrings_t engine_punned, + char const **error_message) noexcept { + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + auto const &engine = *reinterpret_cast<substrings_backends_t const *>(engine_punned); + if (szs_substrings_needles_count_(engine) != 0) return sz_success_k; + return propagate_error(sz::status_t::unexpected_dimensions_k, error_message, + "No needles indexed; call `szs_substrings_index` first"); +} + +template <typename operation_type_> +inline sz_status_t szs_substrings_dispatch_(szs_substrings_t engine_punned, szs_device_scope_t device_punned, + char const **error_message, operation_type_ &&operation) noexcept { + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + sz_assert_(device_punned != nullptr && "Device must be initialized"); + + auto &engine = *reinterpret_cast<substrings_backends_t *>(engine_punned); + auto &device = *reinterpret_cast<device_scope_t *>(device_punned); + sz::status_t const status = std::visit( + [&](auto &engine_variant) -> sz::status_t { + using engine_variant_t = std::decay_t<decltype(engine_variant)>; + if constexpr (is_gpu_capability(engine_variant_t::capability_k)) { +#if SZ_USE_CUDA + auto [gpu_scope, scope_status] = gpu_scope_for(device); + if (scope_status.status != sz::status_t::success_k) return scope_status.status; + return operation(engine_variant, get_executor(gpu_scope), get_specs(gpu_scope)); +#else + return sz::status_t::missing_gpu_k; +#endif // SZ_USE_CUDA + } + // CPU scopes differ only in the executor they hand out, so one visitor covers both. + else + return std::visit( + [&](auto &scope_variant) -> sz::status_t { + using scope_t = std::decay_t<decltype(scope_variant)>; + if constexpr (!is_cpu_scope<scope_t>()) return sz::status_t::device_code_mismatch_k; + else return operation(engine_variant, get_executor(scope_variant), get_specs(scope_variant)); + }, + device.variants); + }, + engine.variants); + return propagate_error(status, error_message); +} + +/** + * @brief Counts every needle's occurrences in every haystack, on whichever scope @p device_punned names. + * @sa `szs_levenshtein_cross_` for the same dispatch skeleton. + */ +template <typename haystacks_type_> +inline sz_status_t szs_substrings_count_(szs_substrings_t engine_punned, szs_device_scope_t device_punned, + haystacks_type_ const &haystacks, + szs_substrings_overlap_policy_t overlap_policy, sz_size_t *counts, + sz_size_t *matches_total_out, char const **error_message) noexcept { + sz_assert_(counts != nullptr && "Counts output cannot be null"); + sz_assert_(matches_total_out != nullptr && "Match-total output cannot be null"); + + *matches_total_out = 0; + if (sz_status_t const indexed = szs_substrings_require_indexed_(engine_punned, error_message); + indexed != sz_success_k) + return indexed; + sz::span<size_t> const counts_span {counts, haystacks.size()}; + szs::substrings_overlap_policy_t const policy = szs_substrings_policy_(overlap_policy); + size_t matches_total = 0; + + sz_status_t const result = szs_substrings_dispatch_( // + engine_punned, device_punned, error_message, + [&](auto &engine_variant, auto &&executor, auto const &specs) -> sz::status_t { + return engine_variant.try_count(haystacks, policy, counts_span, matches_total, executor, specs); + }); + *matches_total_out = (sz_size_t)matches_total; + return result; +} + +/** + * @brief Locates every needle's occurrences in every haystack. @sa `szs_substrings_count_`. + * + * The caller's array IS the output on every arm, viewed as the C++ match. Capacity checks and host-memory + * staging live in the engines, not here. + * + * Capacity is the engine's call alone. It counts before it can check anything anyway, and it reports the + * size it wanted rather than zero, which makes `matches_capacity == 0` a single-call size query. + */ +template <typename haystacks_type_> +inline sz_status_t szs_substrings_find_(szs_substrings_t engine_punned, szs_device_scope_t device_punned, + haystacks_type_ const &haystacks, + szs_substrings_overlap_policy_t overlap_policy, szs_substrings_match_t *matches, + sz_size_t matches_capacity, sz_size_t *matches_found_out, + char const **error_message) noexcept { + sz_assert_(matches_found_out != nullptr && "Matches-found output cannot be null"); + + *matches_found_out = 0; + if (sz_status_t const indexed = szs_substrings_require_indexed_(engine_punned, error_message); + indexed != sz_success_k) + return indexed; + + static_assert( + sizeof(szs::substrings_match_t) == sizeof(szs_substrings_match_t) && + offsetof(szs::substrings_match_t, haystack_index) == offsetof(szs_substrings_match_t, haystack_index) && + offsetof(szs::substrings_match_t, needle_index) == offsetof(szs_substrings_match_t, needle_index) && + offsetof(szs::substrings_match_t, byte_offset) == offsetof(szs_substrings_match_t, byte_offset) && + offsetof(szs::substrings_match_t, byte_length) == offsetof(szs_substrings_match_t, byte_length), + "The C++ match must mirror the C ABI field for field, or this view would misreport matches"); + + sz::span<szs::substrings_match_t> const matches_out {reinterpret_cast<szs::substrings_match_t *>(matches), + matches_capacity}; + szs::substrings_overlap_policy_t const policy = szs_substrings_policy_(overlap_policy); + size_t matches_found = 0; + + sz_status_t const result = szs_substrings_dispatch_( // + engine_punned, device_punned, error_message, + [&](auto &engine_variant, auto &&executor, auto const &specs) -> sz::status_t { + return engine_variant.try_find(haystacks, policy, matches_out, matches_found, executor, specs); + }); + + // The engines report the capacity they wanted rather than zeroing it, so the refused-buffer arm needs no + // second walk here - `matches_found` already names the need, which is what makes a zero capacity a size query. + *matches_found_out = (sz_size_t)matches_found; + return result; +} + +/** + * @brief Scores every haystack against the compiled dictionary. @sa `szs_substrings_count_`. + * + * Weights are one per needle rather than a matrix: the dictionary @b is the query, so a second weighting + * is a second call. No overlap policy either - term frequencies are raw counts, which is classic BM25. + */ +template <typename haystacks_type_> +inline sz_status_t szs_substrings_score_bm25_(szs_substrings_t engine_punned, szs_device_scope_t device_punned, + haystacks_type_ const &haystacks, sz_f32_t const *document_lengths, + szs_substrings_bm25_t parameters, sz_f32_t const *needle_weights, + sz_f32_t *scores, char const **error_message) noexcept { + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + sz_assert_(scores != nullptr && "Scores output cannot be null"); + if (sz_status_t const indexed = szs_substrings_require_indexed_(engine_punned, error_message); + indexed != sz_success_k) + return indexed; + + // The engines only assert their per-needle array sizes, and asserts compile out of release builds. + if (needle_weights == nullptr) + return propagate_error(sz::status_t::unexpected_dimensions_k, error_message, + "One weight per needle, no more and no fewer"); + + // Normalizing divides by the corpus mean, so without one there is nothing to divide by. Letting that + // through would drop `length_normalization` and every entry of `document_lengths` without a word. + if (parameters.length_normalization > 0.0f && !(parameters.average_document_length > 0.0f)) + return propagate_error(sz::status_t::unexpected_dimensions_k, error_message, + "BM25 length normalization needs a positive `average_document_length`: " // + "pass the corpus mean, or set `length_normalization` to zero"); + + auto const needles_count = szs_substrings_needles_count_(*reinterpret_cast<substrings_backends_t *>(engine_punned)); + szs::substrings_bm25_t const engine_parameters { + parameters.term_frequency_saturation, parameters.length_normalization, parameters.average_document_length}; + sz::span<sz::f32_t> const scores_span {scores, haystacks.size()}; + sz::span<sz::f32_t const> const lengths_span {document_lengths, document_lengths ? haystacks.size() : 0}; + sz::span<sz::f32_t const> const weights_span {needle_weights, needles_count}; + + return szs_substrings_dispatch_( // + engine_punned, device_punned, error_message, + [&](auto &engine_variant, auto &&executor, auto const &specs) -> sz::status_t { + return engine_variant.try_score_bm25(haystacks, lengths_span, engine_parameters, weights_span, scores_span, + executor, specs); + }); +} + +/** + * @brief Rewrites every haystack into one output tape. @sa `szs_substrings_count_`. + * + * Tape in, tape out, because a rewrite's product is itself a tape. The offsets array is always filled, + * even when the byte capacity is refused, so a refused call still names the size it wanted. + */ +template <typename haystacks_type_> +inline sz_status_t szs_substrings_replace_(szs_substrings_t engine_punned, szs_device_scope_t device_punned, + haystacks_type_ const &haystacks, + szs_substrings_overlap_policy_t overlap_policy, + sz_sequence_t const *replacements, sz_ptr_t output_data, + sz_size_t output_data_capacity, sz_size_t *output_offsets, + sz_size_t *output_bytes_written, char const **error_message) noexcept { + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + sz_assert_(replacements != nullptr && "Replacement collection cannot be null"); + sz_assert_(output_offsets != nullptr && "Offsets output cannot be null"); + sz_assert_(output_bytes_written != nullptr && "Bytes-written output cannot be null"); + + *output_bytes_written = 0; + auto const replacements_container = sz_sequence_as_cpp_container_t {replacements}; + auto const needles_count = szs_substrings_needles_count_(*reinterpret_cast<substrings_backends_t *>(engine_punned)); + if (replacements_container.size() != needles_count) + return propagate_error(sz::status_t::unexpected_dimensions_k, error_message, + "One replacement per needle, no more and no fewer"); + + szs::substrings_overlap_policy_t const policy = szs_substrings_policy_(overlap_policy); + // The engines refuse this too, but only with a bare status - and a rewrite under overlapping matches is + // the one refusal a caller is likely to hit by misreading the API rather than by mis-sizing a buffer. + if (sz::status_t const rewritable = szs::substrings_check_rewritable(policy); rewritable != sz::status_t::success_k) + return propagate_error(rewritable, error_message, + "A rewrite needs a cover whose matches share no bytes, so overlapping is refused"); + + sz::span<char> const output_span {output_data, output_data_capacity}; + sz::span<size_t> const offsets_span {output_offsets, haystacks.size() + 1}; + size_t bytes_written = 0; + + sz_status_t const result = szs_substrings_dispatch_( // + engine_punned, device_punned, error_message, + [&](auto &engine_variant, auto &&executor, auto const &specs) -> sz::status_t { + return engine_variant.try_replace(haystacks, policy, replacements_container, output_span, offsets_span, + bytes_written, executor, specs); + }); + + // The engines report the capacity they wanted rather than zeroing it, which is what makes a zero + // capacity a size query rather than a wasted call. + *output_bytes_written = (sz_size_t)bytes_written; + return result; +} + +#pragma endregion Dispatch + +extern "C" { + +#pragma region Substrings + +SZ_API_RUNTIME sz_status_t szs_substrings_init( // + sz_memory_allocator_t const *alloc, sz_capability_t capabilities, // + szs_substrings_t *engine_punned, char const **error_message) { + + sz_unused_(alloc); // Custom allocator not yet implemented, using default + sz_assert_(engine_punned != nullptr && *engine_punned == nullptr && "Engine must be uninitialized"); + +#if SZ_USE_CUDA + if ((capabilities & sz_cap_cuda_k) == sz_cap_cuda_k) + return emplace_empty_substrings_engine<szs::substrings_cuda_t>(engine_punned, error_message); +#endif // SZ_USE_CUDA + if ((capabilities & sz_caps_sp_k) == sz_caps_sp_k) + return emplace_empty_substrings_engine<szs::substrings_parallel_t>(engine_punned, error_message); + return emplace_empty_substrings_engine<szs::substrings_serial_t>(engine_punned, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_index( // + szs_substrings_t engine_punned, sz_sequence_t const *needles, // + szs_substrings_case_sensitivity_t case_sensitivity, szs_device_scope_t device_punned, // + char const **error_message) { + + sz_assert_(needles != nullptr && "Needle collection cannot be null"); + + szs::substrings_case_sensitivity_t const sensitivity = case_sensitivity == szs_substrings_uncased_k + ? szs::substrings_uncased_k + : szs::substrings_cased_k; + auto const needles_container = sz_sequence_as_cpp_container_t {needles}; + + // The same dispatch every operation uses, so the hot tier is sized from the scope that will walk it and an + // engine/scope mismatch is refused here rather than at the first search. + return szs_substrings_dispatch_( // + engine_punned, device_punned, error_message, + [&](auto &engine_variant, auto &&executor, auto const &specs) -> sz::status_t { + return engine_variant.try_index(needles_container, sensitivity, executor, specs); + }); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_count( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_count_(engine_punned, device_punned, sz_sequence_as_cpp_container_t {haystacks}, + overlap_policy, counts, matches_total, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_find( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_find_(engine_punned, device_punned, sz_sequence_as_cpp_container_t {haystacks}, + overlap_policy, matches, matches_capacity, matches_found, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_count_u32tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u32tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_count_(engine_punned, device_punned, sz_sequence_u32tape_as_cpp_container_t {haystacks}, + overlap_policy, counts, matches_total, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_count_u64tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u64tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_count_(engine_punned, device_punned, sz_sequence_u64tape_as_cpp_container_t {haystacks}, + overlap_policy, counts, matches_total, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_find_u32tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u32tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_find_(engine_punned, device_punned, sz_sequence_u32tape_as_cpp_container_t {haystacks}, + overlap_policy, matches, matches_capacity, matches_found, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_find_u64tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u64tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_find_(engine_punned, device_punned, sz_sequence_u64tape_as_cpp_container_t {haystacks}, + overlap_policy, matches, matches_capacity, matches_found, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_score_bm25_(engine_punned, device_punned, sz_sequence_as_cpp_container_t {haystacks}, + document_lengths, parameters, needle_weights, scores, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25_u32tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u32tape_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_score_bm25_(engine_punned, device_punned, sz_sequence_u32tape_as_cpp_container_t {haystacks}, + document_lengths, parameters, needle_weights, scores, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25_u64tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u64tape_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_score_bm25_(engine_punned, device_punned, sz_sequence_u64tape_as_cpp_container_t {haystacks}, + document_lengths, parameters, needle_weights, scores, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_replace_bound( // + szs_substrings_t engine_punned, // + sz_sequence_t const *replacements, // + sz_size_t input_bytes, sz_size_t *output_bytes_bound, // + char const **error_message) { + + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + sz_assert_(replacements != nullptr && "Replacement collection cannot be null"); + sz_assert_(output_bytes_bound != nullptr && "Bound output cannot be null"); + + auto &engine = *reinterpret_cast<substrings_backends_t *>(engine_punned); + auto const replacements_container = sz_sequence_as_cpp_container_t {replacements}; + sz_status_t result = sz_success_k; + std::visit( + [&](auto &engine_variant) { + if (replacements_container.size() != engine_variant.count_needles()) { + result = propagate_error(sz::status_t::unexpected_dimensions_k, error_message, + "One replacement per needle, no more and no fewer"); + return; + } + + // The densest rewrite tiles the input with the shortest match there is and swaps each one for + // the widest replacement there is. The bytes past the last whole match survive verbatim, so they + // are added rather than dropped - integer division alone under-counts, and a caller sizing an + // output tape to an under-count would have it overflowed by a rewrite that cannot be refused. + size_t widest_replacement = 0; + for (size_t needle_index = 0; needle_index < replacements_container.size(); ++needle_index) + widest_replacement = sz_max_of_two(widest_replacement, replacements_container[needle_index].size()); + size_t const shortest_match = sz_max_of_two(engine_variant.min_source_match_bytes(), (size_t)1); + size_t const whole_matches = input_bytes / shortest_match; + + // A dictionary that only ever shrinks still bounds at the input length, never below it. + *output_bytes_bound = (sz_size_t)sz_max_of_two( + input_bytes, whole_matches * widest_replacement + input_bytes % shortest_match); + result = propagate_error(sz::status_t::success_k, error_message); + }, + engine.variants); + return result; +} + +SZ_API_RUNTIME sz_status_t szs_substrings_replace_u32tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u32tape_t const *haystacks, // + szs_substrings_overlap_policy_t overlap_policy, // + sz_sequence_t const *replacements, // + sz_ptr_t output_data, sz_size_t output_data_capacity, sz_size_t *output_offsets, // + sz_size_t *output_bytes_written, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_replace_(engine_punned, device_punned, sz_sequence_u32tape_as_cpp_container_t {haystacks}, + overlap_policy, replacements, output_data, output_data_capacity, output_offsets, + output_bytes_written, error_message); +} + +SZ_API_RUNTIME sz_status_t szs_substrings_replace_u64tape( // + szs_substrings_t engine_punned, szs_device_scope_t device_punned, // + sz_sequence_u64tape_t const *haystacks, // + szs_substrings_overlap_policy_t overlap_policy, // + sz_sequence_t const *replacements, // + sz_ptr_t output_data, sz_size_t output_data_capacity, sz_size_t *output_offsets, // + sz_size_t *output_bytes_written, // + char const **error_message) { + + sz_assert_(haystacks != nullptr && "Haystack collection cannot be null"); + return szs_substrings_replace_(engine_punned, device_punned, sz_sequence_u64tape_as_cpp_container_t {haystacks}, + overlap_policy, replacements, output_data, output_data_capacity, output_offsets, + output_bytes_written, error_message); +} + +SZ_API_RUNTIME void szs_substrings_free(szs_substrings_t engine_punned) { + sz_assert_(engine_punned != nullptr && "Engine must be initialized"); + auto *engine = reinterpret_cast<substrings_backends_t *>(engine_punned); + delete engine; +} + +#pragma endregion Substrings +} + +#endif // SZS_SUBSTRINGS_CUH_ diff --git a/c/stringzillas/substrings_cuda.cu b/c/stringzillas/substrings_cuda.cu new file mode 100644 index 00000000..f25628d1 --- /dev/null +++ b/c/stringzillas/substrings_cuda.cu @@ -0,0 +1,45 @@ +/** + * @file c/stringzillas/substrings_cuda.cu + * @brief Base-CUDA-tier instantiations for multi-pattern Aho-Corasick search. + * @author Ash Vardanian + */ +#include "stringzillas/substrings.cuh" + +namespace ashvardanian { +namespace stringzillas { + +/* The engine's `kernels()` table takes these kernels' addresses from host code, which implicitly instantiates + * NVCC's device-stub wrappers mid-file; instantiating the kernels first makes the stubs precede that use, + * keeping the generated host code well-formed for host compilers that enforce [temp.expl.spec] ordering + * (Clang) rather than tolerating the inversion (GCC). */ +template __global__ void substrings_walk_per_cuda_chunk_<u16_t, substrings_pass_t::sizing_k>( + aho_corasick_view<u16_t>, u16_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<size_t const>, + size_t, size_t, span<size_t>, span<substrings_match_t>); +template __global__ void substrings_walk_per_cuda_chunk_<u32_t, substrings_pass_t::sizing_k>( + aho_corasick_view<u32_t>, u32_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<size_t const>, + size_t, size_t, span<size_t>, span<substrings_match_t>); +template __global__ void substrings_walk_per_cuda_chunk_<u16_t, substrings_pass_t::writing_k>( + aho_corasick_view<u16_t>, u16_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<size_t const>, + size_t, size_t, span<size_t>, span<substrings_match_t>); +template __global__ void substrings_walk_per_cuda_chunk_<u32_t, substrings_pass_t::writing_k>( + aho_corasick_view<u32_t>, u32_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<size_t const>, + size_t, size_t, span<size_t>, span<substrings_match_t>); + +template __global__ void exclusive_sum_across_cuda_device_<size_t>( // + size_t const *, size_t, size_t *); +template __global__ void exclusive_sum_reduce_tiles_across_cuda_device_<size_t>( // + size_t const *, size_t, size_t, size_t *); +template __global__ void exclusive_sum_apply_tiles_across_cuda_device_<size_t>( // + size_t const *, size_t, size_t, size_t const *, size_t *); + +template __global__ void substrings_score_bm25_per_haystack_<u16_t>( // + aho_corasick_view<u16_t>, u16_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<f32_t const>, + substrings_bm25_t, span<f32_t const>, span<u32_t>, span<f32_t>); +template __global__ void substrings_score_bm25_per_haystack_<u32_t>( // + aho_corasick_view<u32_t>, u32_t, span<u32_t const>, u32_t, span<span<byte_t const> const>, span<f32_t const>, + substrings_bm25_t, span<f32_t const>, span<u32_t>, span<f32_t>); + +template struct substrings_cuda<unified_alloc_t, sz_cap_cuda_k>; + +} // namespace stringzillas +} // namespace ashvardanian diff --git a/c/stringzillas/substrings_serial.cpp b/c/stringzillas/substrings_serial.cpp new file mode 100644 index 00000000..91a9d9da --- /dev/null +++ b/c/stringzillas/substrings_serial.cpp @@ -0,0 +1,28 @@ +/** + * @file c/stringzillas/substrings_serial.cpp + * @brief Single CPU-variant instantiation unit for the multi-pattern Aho-Corasick engine. + * @author Ash Vardanian + * @date August 6, 2026 + * + * Multi-pattern search has no per-ISA kernels - a transition is one data-dependent load on a serial + * dependency chain, so parallelism comes from many independent haystacks rather than from vector + * instructions. Unlike the similarity siblings, which get one instantiation unit per ISA, this is the only + * one `substrings` needs. + */ +#include "stringzillas/substrings.hpp" // Dictionary + engine + +namespace ashvardanian { +namespace stringzillas { + +// Both state-id widths, because the engine holds whichever one its needle set fits: a `u16` automaton halves +// the hot-row footprint, so twice as much of it stays cache-resident. +template struct aho_corasick_dictionary<u16_t, std::allocator<char>>; +template struct aho_corasick_dictionary<u32_t, std::allocator<char>>; + +// Both capabilities are listed: the serial engine is what the C shim builds for a single-threaded scope, so +// omitting it only moves its instantiation into every consumer translation unit. +template struct substrings<std::allocator<char>, sz_cap_serial_k>; +template struct substrings<std::allocator<char>, sz_caps_sp_k>; + +} // namespace stringzillas +} // namespace ashvardanian diff --git a/cmake/sz_compiler_flags.cmake b/cmake/sz_compiler_flags.cmake index 8340e98c..116aea99 100644 --- a/cmake/sz_compiler_flags.cmake +++ b/cmake/sz_compiler_flags.cmake @@ -70,32 +70,49 @@ function (set_optimization_flags target compiler_id target_type) target_compile_options(${target} PRIVATE "$<$<CONFIG:Release>:-O2>") elseif (compiler_id STREQUAL "NVIDIA") if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - set(sz_nvcc_debug_ "-G" # Device debug symbols - "-no-compress" # No compression of debug info - "-Xcompiler=/Zi" # Host debugging symbols - "-Xcompiler=/Oy-" # Frame pointers for stack traces - "-Xcompiler=/Ob0" # Prevent host inlining - "-maxrregcount=0" # No register count limits + set(sz_nvcc_debug_ + "-G" # Device debug symbols + "-no-compress" # No compression of debug info + "-Xcompiler=/Zi" # Host debugging symbols + "-Xcompiler=/Oy-" # Frame pointers for stack traces + "-Xcompiler=/Ob0" # Prevent host inlining + "-maxrregcount=0" # No register count limits ) - set(sz_nvcc_release_ "-O2" # NVCC optimizations - "-Xptxas=-O2" # PTX assembler optimizations - "-Xcompiler=/O2" # Host optimizations + set(sz_nvcc_lineinfo_ + "-lineinfo" # Source correlation through optimized device code + "-Xcompiler=/Zi" # Host debugging symbols + "-Xcompiler=/Oy-" # Frame pointers for stack traces + ) + set(sz_nvcc_release_ + "-O2" # NVCC optimizations + "-Xptxas=-O2" # PTX assembler optimizations + "-Xcompiler=/O2" # Host optimizations ) else () - set(sz_nvcc_debug_ "-G" # Device debug symbols - "-no-compress" # No compression of debug info - "-Xcompiler=-g" # Host debugging symbols explicitly - "-Xcompiler=-fno-omit-frame-pointer" # Stack trace clarity - "-Xcompiler=-fno-inline" # Prevent host inlining - "-maxrregcount=0" # No register count limits + set(sz_nvcc_debug_ + "-G" # Device debug symbols + "-no-compress" # No compression of debug info + "-Xcompiler=-g" # Host debugging symbols explicitly + "-Xcompiler=-fno-omit-frame-pointer" # Stack trace clarity + "-Xcompiler=-fno-inline" # Prevent host inlining + "-maxrregcount=0" # No register count limits + ) + set(sz_nvcc_lineinfo_ + "-lineinfo" # Source correlation through optimized device code + "-Xcompiler=-g" # Host debugging symbols explicitly + "-Xcompiler=-fno-omit-frame-pointer" # Stack trace clarity ) - set(sz_nvcc_release_ "-O2" # NVCC optimizations - "-Xptxas=-O2" # PTX assembler optimizations - "-Xcompiler=-O2" # Host optimizations + set(sz_nvcc_release_ + "-O2" # NVCC optimizations + "-Xptxas=-O2" # PTX assembler optimizations + "-Xcompiler=-O2" # Host optimizations ) endif () + # `RelWithDebInfo` matched both lists and asked `ptxas` for `-G` beside `-O2`, which it refuses. + # `-G` stays with `Debug`; `-lineinfo` carries the same source correlation through optimized code. target_compile_options( - ${target} PRIVATE "$<$<CONFIG:Debug,RelWithDebInfo>:${sz_nvcc_debug_}>" + ${target} PRIVATE "$<$<CONFIG:Debug>:${sz_nvcc_debug_}>" + "$<$<CONFIG:RelWithDebInfo>:${sz_nvcc_lineinfo_}>" "$<$<CONFIG:Release,RelWithDebInfo>:${sz_nvcc_release_}>" ) endif () @@ -212,11 +229,8 @@ function (set_compiler_flags target cpp_standard target_arch compiler_id) else () target_compile_options(${target} PRIVATE "-Xcompiler=/arch:AVX2") endif () - else () - check_cxx_compiler_flag("-march=native" supports_march_native) - if (supports_march_native) - target_compile_options(${target} PRIVATE "-Xcompiler=-march=native") - endif () + elseif (STRINGZILLA_CUDA_ACCEPTS_NATIVE_ARCH) + target_compile_options(${target} PRIVATE "-Xcompiler=-march=native") endif () elseif (NOT (compiler_id MATCHES "MSVC")) check_cxx_compiler_flag("-march=native" supports_march_native) diff --git a/cmake/sz_cuda_probes.cmake b/cmake/sz_cuda_probes.cmake new file mode 100644 index 00000000..c7151894 --- /dev/null +++ b/cmake/sz_cuda_probes.cmake @@ -0,0 +1,31 @@ +# cmake/sz_cuda_probes.cmake — CUDA toolchain probes over the checked-in `probes/cuda_*.cu` sources. +# +# Separate from the `sz_*_isa_probes.cmake` family, which asks the C compiler what it can emit and answers in +# `SZ_ISA_CAPABILITIES`. These ask what NVCC and its host compiler will accept together: NVCC delegates host +# compilation but parses the host's headers itself on the device pass, so the pair decides, and putting the +# question to the C++ compiler gets an answer about the wrong toolchain - it accepts flags NVCC then chokes on. + +# Whether `.cu` sources may be built for this machine's own instruction set, cached in +# `STRINGZILLA_CUDA_ACCEPTS_NATIVE_ARCH`. A `Failed` verdict leaves them on the baseline architecture and says +# nothing about the CPU tiers, which are compiled by the C and C++ compilers and probed separately. +function (sz_cuda_probe_native_arch_) + if (DEFINED STRINGZILLA_CUDA_ACCEPTS_NATIVE_ARCH) + return() + endif () + set(CMAKE_TRY_COMPILE_CONFIGURATION "Release") + try_compile( + sz_cuda_native_arch_ ${CMAKE_BINARY_DIR}/sz_probes + ${CMAKE_CURRENT_SOURCE_DIR}/probes/cuda_native_arch.cu + CMAKE_FLAGS "-DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS} -Xcompiler=-march=native" + OUTPUT_VARIABLE sz_cuda_native_arch_output_ + ) + set(STRINGZILLA_CUDA_ACCEPTS_NATIVE_ARCH + "${sz_cuda_native_arch_}" + CACHE INTERNAL "Whether NVCC and its host compiler build a `.cu` for this machine's own architecture" + ) + if (sz_cuda_native_arch_) + message(STATUS "Performing CUDA probe native_arch - Success") + else () + message(STATUS "Performing CUDA probe native_arch - Failed") + endif () +endfunction () diff --git a/golang/lib.go b/golang/lib.go index 459537b7..da3544ed 100644 --- a/golang/lib.go +++ b/golang/lib.go @@ -1,4 +1,4 @@ -// StringZilla is a SIMD-accelerated string library modern CPUs, written in C 99, +// StringZilla is a SIMD-accelerated string library for modern CPUs, written in C 99, // and using AVX2, AVX512, Arm NEON, and SVE intrinsics to accelerate processing. // // The GoLang binding is intended to provide a simple interface to a precompiled @@ -97,7 +97,7 @@ func Index(str string, substr string) int64 { return int64(uintptr(matchPtr) - uintptr(unsafe.Pointer(strPtr))) } -// Index returns the index of the last instance of `substr` in `str`, or -1 if `substr` is not present. +// LastIndex returns the index of the last instance of `substr` in `str`, or -1 if `substr` is not present. // https://pkg.go.dev/strings#LastIndex func LastIndex(str string, substr string) int64 { substrLen := len(substr) @@ -114,7 +114,7 @@ func LastIndex(str string, substr string) int64 { return int64(uintptr(matchPtr) - uintptr(unsafe.Pointer(strPtr))) } -// Index returns the index of the first instance of a byte in `str`, or -1 if a byte is not present. +// IndexByte returns the index of the first instance of a byte in `str`, or -1 if a byte is not present. // https://pkg.go.dev/strings#IndexByte func IndexByte(str string, c byte) int64 { strPtr := (*C.char)(unsafe.Pointer(unsafe.StringData(str))) @@ -127,7 +127,7 @@ func IndexByte(str string, c byte) int64 { return int64(uintptr(matchPtr) - uintptr(unsafe.Pointer(strPtr))) } -// Index returns the index of the last instance of a byte in `str`, or -1 if a byte is not present. +// LastIndexByte returns the index of the last instance of a byte in `str`, or -1 if a byte is not present. // https://pkg.go.dev/strings#LastIndexByte func LastIndexByte(str string, c byte) int64 { strPtr := (*C.char)(unsafe.Pointer(unsafe.StringData(str))) @@ -140,7 +140,7 @@ func LastIndexByte(str string, c byte) int64 { return int64(uintptr(matchPtr) - uintptr(unsafe.Pointer(strPtr))) } -// Index returns the index of the first instance of any byte from `substr` in `str`, or -1 if none are present. +// IndexAny returns the index of the first instance of any byte from `substr` in `str`, or -1 if none are present. // Note: This is byte-set based (ASCII/bytes), not Unicode rune semantics like strings.IndexAny. // https://pkg.go.dev/strings#IndexAny func IndexAny(str string, substr string) int64 { @@ -155,7 +155,7 @@ func IndexAny(str string, substr string) int64 { return int64(uintptr(matchPtr) - uintptr(unsafe.Pointer(strPtr))) } -// Index returns the index of the last instance of any byte from `substr` in `str`, or -1 if none are present. +// LastIndexAny returns the index of the last instance of any byte from `substr` in `str`, or -1 if none are present. // Note: This is byte-set based (ASCII/bytes), not Unicode rune semantics like strings.LastIndexAny. // https://pkg.go.dev/strings#LastIndexAny func LastIndexAny(str string, substr string) int64 { @@ -211,7 +211,9 @@ func Utf8CaseFold(str string, validate bool) (string, error) { } // Utf8Count returns the number of Unicode codepoints in a UTF-8 string, SIMD-accelerated. -// Malformed bytes are each counted as a single codepoint, matching utf8.RuneCount semantics. +// It counts non-continuation bytes (bytes not matching the 10xxxxxx pattern), which agrees with +// utf8.RuneCount on well-formed UTF-8 but can differ on malformed input: for example, a run of +// lone continuation bytes counts as zero here, while utf8.RuneCount counts one per byte. func Utf8Count(str string) int { if len(str) == 0 { return 0 diff --git a/include/stringzilla/README.md b/include/stringzilla/README.md index 51cf7b23..38cecc2b 100644 --- a/include/stringzilla/README.md +++ b/include/stringzilla/README.md @@ -392,7 +392,7 @@ assert(sentences == 2); // the dot inside $9.99 is not a sentence break ``` The full family of ranges, each borrowing from the source and yielding `sz::string_view` segments, or `sz_rune_t` for runes. -Naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the **separators** the kernel finds, while `split_*` yields the content **between** them: +Naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the __separators__ the kernel finds, while `split_*` yields the content __between__ them: - `utf8_runes()` — every codepoint as a decoded `sz_rune_t` UTF-32 scalar. - `utf8_graphemes()` — UAX-29 grapheme clusters, the user-perceived characters. diff --git a/include/stringzilla/cipher/icelake.h b/include/stringzilla/cipher/icelake.h index 4cd8db57..a087847b 100644 --- a/include/stringzilla/cipher/icelake.h +++ b/include/stringzilla/cipher/icelake.h @@ -556,7 +556,7 @@ SZ_HELPER_INLINE void sz_aes256_gcm_begin_icelake_(sz_aes256_gcm_state_t *state, } /** @brief Absorbs associated data into the payload both directions share. */ -SZ_HELPER_AUTO void sz_aes256_gcm_associate_icelake_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { +SZ_HELPER_INLINE void sz_aes256_gcm_associate_icelake_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { sz_u8_t const *input_bytes = (sz_u8_t const *)text; __m128i const reverse_u8x16 = _mm512_castsi512_si128(_mm512_load_si512(sz_ghash_byte_reverse_icelake_())); __m512i powers_high_u8x64, powers_low_u8x64; @@ -601,7 +601,7 @@ SZ_HELPER_AUTO void sz_aes256_gcm_associate_icelake_(sz_aes256_gcm_state_t *stat } /** @brief Absorbs whatever `partial` holds, zero padded to a full block, and empties it. */ -SZ_HELPER_AUTO void sz_aes256_gcm_flush_partial_icelake_(sz_aes256_gcm_state_t *state) { +SZ_HELPER_INLINE void sz_aes256_gcm_flush_partial_icelake_(sz_aes256_gcm_state_t *state) { __m128i reverse_u8x16, subkey_u8x16, padded_u8x16, accumulator_u8x16; if (state->buffered == 0) return; reverse_u8x16 = _mm512_castsi512_si128(_mm512_load_si512(sz_ghash_byte_reverse_icelake_())); @@ -849,7 +849,7 @@ SZ_HELPER_INLINE void sz_aes256_gcm_transform_icelake_(sz_aes256_gcm_state_t *st * @param state The state, left untouched. * @param tag Receives the sixteen tag bytes. */ -SZ_HELPER_AUTO void sz_aes256_gcm_digest_icelake_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { +SZ_HELPER_INLINE void sz_aes256_gcm_digest_icelake_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { __m128i const reverse_u8x16 = _mm512_castsi512_si128(_mm512_load_si512(sz_ghash_byte_reverse_icelake_())); __m128i const subkey_u8x16 = _mm_shuffle_epi8(_mm_maskz_loadu_epi8((__mmask16)0xFFFFu, state->key.powers), reverse_u8x16); diff --git a/include/stringzilla/cipher/neonaes.h b/include/stringzilla/cipher/neonaes.h index 2a140dd0..eb954ec4 100644 --- a/include/stringzilla/cipher/neonaes.h +++ b/include/stringzilla/cipher/neonaes.h @@ -415,8 +415,7 @@ SZ_HELPER_INLINE uint8x16_t sz_ghash_product_halves_neonaes_(poly64x1_t multipli #if defined(_MSC_VER) && !defined(__clang__) return vreinterpretq_u8_p128(vmull_p64(multiplicand_p64x1, multiplier_p64x1)); #else - return vreinterpretq_u8_p128( - vmull_p64(vget_lane_p64(multiplicand_p64x1, 0), vget_lane_p64(multiplier_p64x1, 0))); + return vreinterpretq_u8_p128(vmull_p64(vget_lane_p64(multiplicand_p64x1, 0), vget_lane_p64(multiplier_p64x1, 0))); #endif } @@ -650,7 +649,7 @@ SZ_HELPER_INLINE void sz_aes256_gcm_begin_neonaes_(sz_aes256_gcm_state_t *state, } /** @brief Absorbs associated data into the payload both directions share. */ -SZ_HELPER_AUTO void sz_aes256_gcm_associate_neonaes_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { +SZ_HELPER_INLINE void sz_aes256_gcm_associate_neonaes_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { sz_u8_t const *input_bytes = (sz_u8_t const *)text; uint8x16_t const subkey_u8x16 = sz_ghash_load_neonaes_(state->key.powers); uint8x16_t powers_u8x16[8]; diff --git a/include/stringzilla/cipher/powervsx.h b/include/stringzilla/cipher/powervsx.h index c481f874..a662cea9 100644 --- a/include/stringzilla/cipher/powervsx.h +++ b/include/stringzilla/cipher/powervsx.h @@ -681,7 +681,8 @@ SZ_HELPER_INLINE void sz_aes256_gcm_begin_powervsx_(sz_aes256_gcm_state_t *state } /** @brief Absorbs associated data into the payload both directions share. */ -SZ_HELPER_AUTO void sz_aes256_gcm_associate_powervsx_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { +SZ_HELPER_INLINE void sz_aes256_gcm_associate_powervsx_(sz_aes256_gcm_state_t *state, sz_cptr_t text, + sz_size_t length) { sz_u8_t const *input_bytes = (sz_u8_t const *)text; __vector unsigned char const subkey_u8x16 = sz_aes256_block_load_powervsx_(state->key.powers); __vector unsigned char powers_u8x16[8]; @@ -949,7 +950,8 @@ SZ_HELPER_INLINE void sz_aes256_gcm_transform_powervsx_(sz_aes256_gcm_state_t *s * @param state The state, left unmodified so a caller may keep appending. * @param tag Receives the sixteen tag bytes. */ -SZ_HELPER_AUTO void sz_aes256_gcm_digest_powervsx_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { +SZ_HELPER_INLINE void sz_aes256_gcm_digest_powervsx_(sz_aes256_gcm_state_t const *state, + sz_u8_t tag[sz_at_least_(16)]) { __vector unsigned char const subkey_u8x16 = sz_aes256_block_load_powervsx_(state->key.powers); __vector unsigned char accumulator_u8x16 = sz_aes256_block_load_powervsx_(state->accumulator); sz_u128_vec_t lengths_vec; diff --git a/include/stringzilla/cipher/rvvcrypto.h b/include/stringzilla/cipher/rvvcrypto.h index 47309a2d..4f0a27ec 100644 --- a/include/stringzilla/cipher/rvvcrypto.h +++ b/include/stringzilla/cipher/rvvcrypto.h @@ -485,7 +485,8 @@ SZ_HELPER_INLINE void sz_aes256_gcm_begin_rvvcrypto_(sz_aes256_gcm_state_t *stat } /** @brief Absorbs associated data into the payload both directions share. */ -SZ_HELPER_AUTO void sz_aes256_gcm_associate_rvvcrypto_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { +SZ_HELPER_INLINE void sz_aes256_gcm_associate_rvvcrypto_(sz_aes256_gcm_state_t *state, sz_cptr_t text, + sz_size_t length) { sz_u8_t const *input_bytes = (sz_u8_t const *)text; vuint32m1_t const subkey_u32m1 = sz_aes256_block_load_rvvcrypto_(state->key.powers); vuint32m1_t accumulator_u32m1 = sz_aes256_block_load_rvvcrypto_(state->accumulator); @@ -526,9 +527,9 @@ SZ_HELPER_AUTO void sz_aes256_gcm_associate_rvvcrypto_(sz_aes256_gcm_state_t *st * @param subkey_u32m1 The hash subkey `H`. * @return The updated hash, unchanged when nothing was pending. */ -SZ_HELPER_AUTO vuint32m1_t sz_aes256_gcm_flush_partial_rvvcrypto_(sz_aes256_gcm_state_t *state, - vuint32m1_t accumulator_u32m1, - vuint32m1_t subkey_u32m1) { +SZ_HELPER_INLINE vuint32m1_t sz_aes256_gcm_flush_partial_rvvcrypto_(sz_aes256_gcm_state_t *state, + vuint32m1_t accumulator_u32m1, + vuint32m1_t subkey_u32m1) { vuint32m1_t padded_u32m1; if (state->buffered == 0) return accumulator_u32m1; padded_u32m1 = sz_aes256_block_load_padded_rvvcrypto_(state->partial, (sz_size_t)state->buffered); @@ -643,7 +644,8 @@ SZ_HELPER_INLINE void sz_aes256_gcm_transform_rvvcrypto_(sz_aes256_gcm_state_t * * @param state The finished state, left untouched. * @param tag Receives the 16 authentication bytes. */ -SZ_HELPER_AUTO void sz_aes256_gcm_digest_rvvcrypto_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { +SZ_HELPER_INLINE void sz_aes256_gcm_digest_rvvcrypto_(sz_aes256_gcm_state_t const *state, + sz_u8_t tag[sz_at_least_(16)]) { vuint32m1_t const subkey_u32m1 = sz_aes256_block_load_rvvcrypto_(state->key.powers); vuint32m1_t accumulator_u32m1 = sz_aes256_block_load_rvvcrypto_(state->accumulator); sz_u128_vec_t staged_block_vec; diff --git a/include/stringzilla/cipher/serial.h b/include/stringzilla/cipher/serial.h index fe7b43fb..6b28bb7b 100644 --- a/include/stringzilla/cipher/serial.h +++ b/include/stringzilla/cipher/serial.h @@ -124,7 +124,8 @@ SZ_HELPER_INLINE sz_u8_t sz_aes256_gf_double_serial_(sz_u8_t value) { * @param block The 16 plaintext bytes. * @param output Receives the 16 ciphertext bytes; may alias @p block. */ -SZ_HELPER_AUTO void sz_aes256_block_encrypt_serial_(sz_aes256_key_t const *key, sz_u8_t const *block, sz_u8_t *output) { +SZ_HELPER_INLINE void sz_aes256_block_encrypt_serial_(sz_aes256_key_t const *key, sz_u8_t const *block, + sz_u8_t *output) { sz_u8_t state[16], shifted[16]; sz_size_t round_index, byte_index, column_index; diff --git a/include/stringzilla/cipher/sve2aes.h b/include/stringzilla/cipher/sve2aes.h index 48123729..dcd2ac1a 100644 --- a/include/stringzilla/cipher/sve2aes.h +++ b/include/stringzilla/cipher/sve2aes.h @@ -481,7 +481,7 @@ SZ_HELPER_INLINE void sz_aes256_gcm_begin_sve2aes_(sz_aes256_gcm_state_t *state, } /** @brief Absorbs associated data into the payload both directions share. */ -SZ_HELPER_AUTO void sz_aes256_gcm_associate_sve2aes_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { +SZ_HELPER_INLINE void sz_aes256_gcm_associate_sve2aes_(sz_aes256_gcm_state_t *state, sz_cptr_t text, sz_size_t length) { sz_u8_t const *input_bytes = (sz_u8_t const *)text; sz_size_t const lane_count = svcntb() / 8; sz_size_t const group_blocks = sz_ghash_group_blocks_sve2aes_(); @@ -691,7 +691,7 @@ SZ_HELPER_INLINE void sz_aes256_gcm_transform_sve2aes_(sz_aes256_gcm_state_t *st * @param state The state, left untouched, so a caller may digest and keep streaming. * @param tag Receives the sixteen tag bytes. */ -SZ_HELPER_AUTO void sz_aes256_gcm_digest_sve2aes_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { +SZ_HELPER_INLINE void sz_aes256_gcm_digest_sve2aes_(sz_aes256_gcm_state_t const *state, sz_u8_t tag[sz_at_least_(16)]) { svbool_t const all_b8x = svptrue_b8(); svbool_t const first_b8x = svptrue_pat_b8(SV_VL16); svuint8_t const subkey_u8x = sz_ghash_load_sve2aes_(state->key.powers); diff --git a/include/stringzilla/cipher/v128.h b/include/stringzilla/cipher/v128.h index 34390cbf..1bf86241 100644 --- a/include/stringzilla/cipher/v128.h +++ b/include/stringzilla/cipher/v128.h @@ -144,7 +144,7 @@ SZ_HELPER_INLINE v128_t sz_aes256_nibble_map_v128_(v128_t low_table_u8x16, v128_ * swizzles cover between them: the low one answers exponents below sixteen and returns zero elsewhere, and * the high one is fed the sum less sixteen so its own out-of-range indices fall away the same way. */ -SZ_HELPER_AUTO v128_t sz_aes256_nibble_multiply_v128_(v128_t first_u8x16, v128_t second_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_nibble_multiply_v128_(v128_t first_u8x16, v128_t second_u8x16) { v128_t const logarithm_table_u8x16 = wasm_v128_load(sz_aes256_nibble_logarithm_v128_()); v128_t const exponent_low_table_u8x16 = wasm_v128_load(sz_aes256_nibble_exponent_low_v128_()); v128_t const exponent_high_table_u8x16 = wasm_v128_load(sz_aes256_nibble_exponent_high_v128_()); @@ -163,7 +163,7 @@ SZ_HELPER_AUTO v128_t sz_aes256_nibble_multiply_v128_(v128_t first_u8x16, v128_t * The byte is carried into `GF(2^4)^2`, where its inverse costs three four-bit multiplies, two squarings and * one four-bit inversion, all of them sixteen-entry swizzles. */ -SZ_HELPER_AUTO v128_t sz_aes256_substitute_v128_(v128_t bytes_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_substitute_v128_(v128_t bytes_u8x16) { v128_t const mapped_u8x16 = sz_aes256_nibble_map_v128_(wasm_v128_load(sz_aes256_tower_forward_low_v128_()), wasm_v128_load(sz_aes256_tower_forward_high_v128_()), bytes_u8x16); @@ -223,7 +223,7 @@ SZ_HELPER_INLINE v128_t sz_aes256_key_fold_v128_(v128_t previous_u8x16, v128_t s * @param round_constant The round constant for this step. * @return The finished word, broadcast across all four lanes. */ -SZ_HELPER_AUTO v128_t sz_aes256_key_turn_v128_(v128_t previous_u8x16, sz_u8_t round_constant) { +SZ_HELPER_INLINE v128_t sz_aes256_key_turn_v128_(v128_t previous_u8x16, sz_u8_t round_constant) { v128_t const last_word_u8x16 = wasm_i32x4_shuffle(previous_u8x16, previous_u8x16, 3, 3, 3, 3); v128_t const rotated_u8x16 = wasm_i8x16_shuffle(last_word_u8x16, last_word_u8x16, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12); @@ -235,7 +235,7 @@ SZ_HELPER_AUTO v128_t sz_aes256_key_turn_v128_(v128_t previous_u8x16, sz_u8_t ro * @param previous_u8x16 The four schedule words immediately before the new quadruple. * @return The finished word, broadcast across all four lanes. */ -SZ_HELPER_AUTO v128_t sz_aes256_key_half_turn_v128_(v128_t previous_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_key_half_turn_v128_(v128_t previous_u8x16) { return sz_aes256_substitute_v128_(wasm_i32x4_shuffle(previous_u8x16, previous_u8x16, 3, 3, 3, 3)); } @@ -342,7 +342,7 @@ SZ_HELPER_INLINE v128_t sz_aes256_mix_columns_v128_(v128_t bytes_u8x16) { * @param block_u8x16 The plaintext block. * @return The ciphertext block. */ -SZ_HELPER_AUTO v128_t sz_aes256_block_encrypt_v128_(sz_aes256_key_t const *key, v128_t block_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_block_encrypt_v128_(sz_aes256_key_t const *key, v128_t block_u8x16) { sz_size_t round_index; block_u8x16 = wasm_v128_xor(block_u8x16, sz_aes256_round_key_v128_(key, 0)); for (round_index = 1; round_index != 14; ++round_index) @@ -458,7 +458,7 @@ SZ_HELPER_INLINE v128_t sz_ghash_double_v128_(v128_t value_u8x16) { * Eight places is a whole byte, so the shift itself is one immediate shuffle and only the byte that leaves * the block needs work. */ -SZ_HELPER_AUTO v128_t sz_ghash_double_byte_v128_(v128_t value_u8x16) { +SZ_HELPER_INLINE v128_t sz_ghash_double_byte_v128_(v128_t value_u8x16) { v128_t const zeros_u8x16 = wasm_u64x2_splat(0); v128_t const shifted_u8x16 = wasm_i8x16_shuffle(value_u8x16, zeros_u8x16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14); @@ -478,7 +478,7 @@ SZ_HELPER_AUTO v128_t sz_ghash_double_byte_v128_(v128_t value_u8x16) { * table is derived from the subkey, and its cache footprint would leak the key on exactly the platforms with * no cipher instructions to fall back on. */ -SZ_HELPER_AUTO v128_t sz_ghash_multiply_v128_(v128_t accumulator_u8x16, v128_t subkey_u8x16) { +SZ_HELPER_INLINE v128_t sz_ghash_multiply_v128_(v128_t accumulator_u8x16, v128_t subkey_u8x16) { v128_t shifted_subkeys_u8x16[8]; v128_t product_u8x16 = wasm_u64x2_splat(0); sz_size_t byte_index, bit_index; diff --git a/include/stringzilla/cipher/v128relaxed.h b/include/stringzilla/cipher/v128relaxed.h index 7207269b..8457dac0 100644 --- a/include/stringzilla/cipher/v128relaxed.h +++ b/include/stringzilla/cipher/v128relaxed.h @@ -58,7 +58,7 @@ SZ_HELPER_INLINE v128_t sz_aes256_nibble_map_v128relaxed_(v128_t low_table_u8x16 * * The two logarithm lookups take nibbles and may be relaxed. */ -SZ_HELPER_AUTO v128_t sz_aes256_nibble_multiply_v128relaxed_(v128_t first_u8x16, v128_t second_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_nibble_multiply_v128relaxed_(v128_t first_u8x16, v128_t second_u8x16) { v128_t const logarithm_table_u8x16 = wasm_v128_load(sz_aes256_nibble_logarithm_v128_()); v128_t const exponent_low_table_u8x16 = wasm_v128_load(sz_aes256_nibble_exponent_low_v128_()); v128_t const exponent_high_table_u8x16 = wasm_v128_load(sz_aes256_nibble_exponent_high_v128_()); @@ -77,7 +77,7 @@ SZ_HELPER_AUTO v128_t sz_aes256_nibble_multiply_v128relaxed_(v128_t first_u8x16, * The same tower-field construction the `_v128` kernel uses, with every provably in-range swizzle taken in * its relaxed form. */ -SZ_HELPER_AUTO v128_t sz_aes256_substitute_v128relaxed_(v128_t bytes_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_substitute_v128relaxed_(v128_t bytes_u8x16) { v128_t const mapped_u8x16 = sz_aes256_nibble_map_v128relaxed_(wasm_v128_load(sz_aes256_tower_forward_low_v128_()), wasm_v128_load(sz_aes256_tower_forward_high_v128_()), bytes_u8x16); @@ -113,7 +113,7 @@ SZ_HELPER_AUTO v128_t sz_aes256_substitute_v128relaxed_(v128_t bytes_u8x16) { * @param round_constant The round constant for this step. * @return The finished word, broadcast across all four lanes. */ -SZ_HELPER_AUTO v128_t sz_aes256_key_turn_v128relaxed_(v128_t previous_u8x16, sz_u8_t round_constant) { +SZ_HELPER_INLINE v128_t sz_aes256_key_turn_v128relaxed_(v128_t previous_u8x16, sz_u8_t round_constant) { v128_t const last_word_u8x16 = wasm_i32x4_shuffle(previous_u8x16, previous_u8x16, 3, 3, 3, 3); v128_t const rotated_u8x16 = wasm_i8x16_shuffle(last_word_u8x16, last_word_u8x16, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12); @@ -125,7 +125,7 @@ SZ_HELPER_AUTO v128_t sz_aes256_key_turn_v128relaxed_(v128_t previous_u8x16, sz_ * @param previous_u8x16 The four schedule words immediately before the new quadruple. * @return The finished word, broadcast across all four lanes. */ -SZ_HELPER_AUTO v128_t sz_aes256_key_half_turn_v128relaxed_(v128_t previous_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_key_half_turn_v128relaxed_(v128_t previous_u8x16) { return sz_aes256_substitute_v128relaxed_(wasm_i32x4_shuffle(previous_u8x16, previous_u8x16, 3, 3, 3, 3)); } @@ -189,7 +189,7 @@ SZ_API_COMPTIME void sz_aes256_key_init_v128relaxed(sz_aes256_key_t *key, sz_u8_ * @param block_u8x16 The plaintext block. * @return The ciphertext block. */ -SZ_HELPER_AUTO v128_t sz_aes256_block_encrypt_v128relaxed_(sz_aes256_key_t const *key, v128_t block_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes256_block_encrypt_v128relaxed_(sz_aes256_key_t const *key, v128_t block_u8x16) { sz_size_t round_index; block_u8x16 = wasm_v128_xor(block_u8x16, sz_aes256_round_key_v128_(key, 0)); for (round_index = 1; round_index != 14; ++round_index) diff --git a/include/stringzilla/compare/README.md b/include/stringzilla/compare/README.md index 44b7f94c..f495c24a 100644 --- a/include/stringzilla/compare/README.md +++ b/include/stringzilla/compare/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are throughput in GB/s, measured with `bench/token.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Numbers are throughput in GB/s, measured with `bench/token.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Each row is the library compiled with that single backend forced on one fixed chip, and each column is one operation, so coverage and cross-chip comparison read down a single column. The Standard row is the platform's best stock equivalent per column, `std::memcmp` for both Equal and Order. Comparison is decided in the first differing bytes, so a Short Words table (tokens averaging 5 bytes) and a Long Lines table (tokens averaging 130 bytes) are enough to show how token length shifts the balance. diff --git a/include/stringzilla/find/README.md b/include/stringzilla/find/README.md index db157d5b..94d5973e 100644 --- a/include/stringzilla/find/README.md +++ b/include/stringzilla/find/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are throughput, shown in GB/s in each cell, measured with `bench/find.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Numbers are throughput, shown in GB/s in each cell, measured with `bench/find.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Each row is the library compiled with that single backend forced on one fixed chip, and each column is one operation, so coverage and cross-chip comparison read down a single column. The Standard row is the platform's best stock equivalent per column — `strstr` and `std::find_end` for substring search, `memchr` for the byte variants, and `strpbrk`/`strcspn` for the byte-set variants. Substring search depends sharply on needle length, so results are split into a Short Words table with tokens averaging 5 bytes and a Long Lines table with tokens averaging 130 bytes. diff --git a/include/stringzilla/find/icelake.h b/include/stringzilla/find/icelake.h index ee6953af..91e92078 100644 --- a/include/stringzilla/find/icelake.h +++ b/include/stringzilla/find/icelake.h @@ -35,7 +35,7 @@ extern "C" { #elif defined(__GNUC__) #pragma GCC push_options #pragma GCC target("avx", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512vbmi", "avx512vbmi2", "bmi", "bmi2", \ - "lzcnt") + "lzcnt") #endif SZ_API_COMPTIME sz_cptr_t sz_find_byteset_icelake(sz_cptr_t text, sz_size_t length, sz_byteset_t const *filter) { diff --git a/include/stringzilla/find/neon.h b/include/stringzilla/find/neon.h index d20d86d2..97b259f9 100644 --- a/include/stringzilla/find/neon.h +++ b/include/stringzilla/find/neon.h @@ -110,7 +110,7 @@ SZ_API_COMPTIME sz_u64_t sz_find_byteset_neon_register_( // * to avoid the per-candidate call + length re-dispatch. Loops over 16-byte `vceqq_u8` chunks with * a `vminvq_u8` all-match reduction and closes with one overlapping tail window. */ -SZ_HELPER_AUTO sz_bool_t sz_find_verify_neon_(sz_cptr_t a, sz_cptr_t b, sz_size_t length) { +SZ_HELPER_INLINE sz_bool_t sz_find_verify_neon_(sz_cptr_t a, sz_cptr_t b, sz_size_t length) { if (length < 16) return sz_equal_serial(a, b, length); sz_size_t offset = 0; diff --git a/include/stringzilla/find/rvv.h b/include/stringzilla/find/rvv.h index 3da0a4a7..5f1872b9 100644 --- a/include/stringzilla/find/rvv.h +++ b/include/stringzilla/find/rvv.h @@ -77,8 +77,8 @@ SZ_API_COMPTIME sz_cptr_t sz_rfind_byte_rvv(sz_cptr_t haystack, sz_size_t haysta * @param vector_length Vector length for this strip. * @return Predicate mask where lane `i` is set if `haystack_u8m8[i]` is in the set. */ -SZ_HELPER_AUTO vbool1_t sz_find_byteset_rvv_mask_m8_(vuint8m8_t haystack_u8m8, sz_u8_t const *set_u8s, - sz_size_t vector_length) { +SZ_HELPER_INLINE vbool1_t sz_find_byteset_rvv_mask_m8_(vuint8m8_t haystack_u8m8, sz_u8_t const *set_u8s, + sz_size_t vector_length) { vuint8m8_t byte_index_u8m8 = __riscv_vsrl_vx_u8m8(haystack_u8m8, 3, vector_length); // c >> 3, in [0, 31] vuint8m8_t bit_position_u8m8 = __riscv_vand_vx_u8m8(haystack_u8m8, 7, vector_length); // c & 7 vuint8m8_t one_u8m8 = __riscv_vmv_v_x_u8m8(1, vector_length); @@ -91,8 +91,8 @@ SZ_HELPER_AUTO vbool1_t sz_find_byteset_rvv_mask_m8_(vuint8m8_t haystack_u8m8, s /** * @brief `m4` sibling of @ref sz_find_byteset_rvv_mask_m8_, used on the reversed backward strip. */ -SZ_HELPER_AUTO vbool2_t sz_find_byteset_rvv_mask_m4_(vuint8m4_t haystack_u8m4, sz_u8_t const *set_u8s, - sz_size_t vector_length) { +SZ_HELPER_INLINE vbool2_t sz_find_byteset_rvv_mask_m4_(vuint8m4_t haystack_u8m4, sz_u8_t const *set_u8s, + sz_size_t vector_length) { vuint8m4_t byte_index_u8m4 = __riscv_vsrl_vx_u8m4(haystack_u8m4, 3, vector_length); // c >> 3, in [0, 31] vuint8m4_t bit_position_u8m4 = __riscv_vand_vx_u8m4(haystack_u8m4, 7, vector_length); // c & 7 vuint8m4_t one_u8m4 = __riscv_vmv_v_x_u8m4(1, vector_length); diff --git a/include/stringzilla/find/v128.h b/include/stringzilla/find/v128.h index 81ef7d0b..dbc6552d 100644 --- a/include/stringzilla/find/v128.h +++ b/include/stringzilla/find/v128.h @@ -120,8 +120,8 @@ SZ_API_COMPTIME sz_cptr_t sz_rfind_byte_v128(sz_cptr_t haystack, sz_size_t hayst * @param set_bottom_u8x16 Bottom half of the byteset (byte indices 16..31). * @return 0xFF per lane where the byte belongs to the set, 0x00 otherwise. */ -SZ_HELPER_AUTO v128_t sz_find_byteset_match_v128_(v128_t haystack_u8x16, v128_t set_top_u8x16, - v128_t set_bottom_u8x16) { +SZ_HELPER_INLINE v128_t sz_find_byteset_match_v128_(v128_t haystack_u8x16, v128_t set_top_u8x16, + v128_t set_bottom_u8x16) { // Serial equivalent per byte `c`: `(set->_u8s[c >> 3] & (1u << (c & 7u))) != 0`. v128_t byte_index_u8x16 = wasm_u8x16_shr(haystack_u8x16, 3); // c >> 3, in [0, 31] // The bit mask `1 << (c & 7)` is produced via a swizzle into a tiny power-of-two table. @@ -237,7 +237,7 @@ SZ_HELPER_INLINE v128_t sz_find_substr_match_v128_( * @param needle_length Length of `needle` in bytes. * @return Pointer to the first verified match, or SZ_NULL_CHAR if none. */ -SZ_HELPER_AUTO sz_cptr_t sz_locate_substr_first_v128_( // +SZ_HELPER_INLINE sz_cptr_t sz_locate_substr_first_v128_( // v128_t match_u8x16, sz_cptr_t window_start, sz_cptr_t needle, sz_size_t needle_length) { sz_u32_t matches = (sz_u32_t)wasm_i8x16_bitmask(match_u8x16); while (matches) { @@ -257,7 +257,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_locate_substr_first_v128_( // * @param needle_length Length of `needle` in bytes. * @return Pointer to the last verified match, or SZ_NULL_CHAR if none. */ -SZ_HELPER_AUTO sz_cptr_t sz_locate_substr_last_v128_( // +SZ_HELPER_INLINE sz_cptr_t sz_locate_substr_last_v128_( // v128_t match_u8x16, sz_cptr_t window_start, sz_cptr_t needle, sz_size_t needle_length) { sz_u32_t matches = (sz_u32_t)wasm_i8x16_bitmask(match_u8x16); while (matches) { diff --git a/include/stringzilla/find/v128relaxed.h b/include/stringzilla/find/v128relaxed.h index 9cdd0fd9..ab67d11e 100644 --- a/include/stringzilla/find/v128relaxed.h +++ b/include/stringzilla/find/v128relaxed.h @@ -62,8 +62,8 @@ SZ_API_COMPTIME sz_cptr_t sz_rfind_v128relaxed(sz_cptr_t haystack, sz_size_t hay * @param set_bottom_u8x16 Bottom half of the byteset (byte indices 16..31). * @return 0xFF per lane where the byte belongs to the set, 0x00 otherwise. */ -SZ_HELPER_AUTO v128_t sz_find_byteset_match_v128relaxed_(v128_t haystack_u8x16, v128_t set_top_u8x16, - v128_t set_bottom_u8x16) { +SZ_HELPER_INLINE v128_t sz_find_byteset_match_v128relaxed_(v128_t haystack_u8x16, v128_t set_top_u8x16, + v128_t set_bottom_u8x16) { v128_t byte_index_u8x16 = wasm_u8x16_shr(haystack_u8x16, 3); // c >> 3, in [0, 31] v128_t bit_table_u8x16 = wasm_i8x16_make(1, 2, 4, 8, 16, 32, 64, (sz_i8_t)128, 0, 0, 0, 0, 0, 0, 0, 0); // Index `c & 7` is in [0, 7] -> always in range -> relaxed swizzle is exact. diff --git a/include/stringzilla/hash/README.md b/include/stringzilla/hash/README.md index dbc109a9..b460f73d 100644 --- a/include/stringzilla/hash/README.md +++ b/include/stringzilla/hash/README.md @@ -1,6 +1,6 @@ # Hash: Byte Sum, AES Hash, and SHA-256 -This directory holds the digest kernels behind `sz_bytesum`, `sz_hash`, the multi-seed `sz_hash` fan-out, `sz_sha256`, and the batched `sz_sha256_multistate`. +This directory holds the digest kernels behind `sz_bytesum`, `sz_hash`, the multi-seed `sz_hash` fan-out, the streaming `sz_sha256_state_init` / `sz_sha256_state_update` / `sz_sha256_state_digest` trio, and the batched `sz_sha256_multistate_update` / `sz_sha256_multistate_digest` pair. Each operation has a serial baseline plus per-ISA SIMD backends — `westmere`, `haswell`, `skylake`, `icelake` on x86, with a dedicated `goldmont` SHA-NI path for SHA-256. SHA-256 has no Ice Lake form: SHA-NI is 128-bit only and has no VEX or EVEX encoding, so `goldmont` is the widest single-message kernel there will be. The dispatcher picks the fastest one available on the running CPU. @@ -23,7 +23,7 @@ At line length the sixteen-wide kernel is about half again as fast as the single ## Methodology -Cells are dual: throughput in GB/s and hash rate in millions of hashes per second, measured with `bench/token.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Cells are dual: throughput in GB/s and hash rate in millions of hashes per second, measured with `bench/token.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Every backend registers its own row in a single binary, so a row is one kernel rather than one build, and each column is one operation — coverage and cross-chip comparison read down a single column. The Standard row is the platform's best stock equivalent per column — `std::accumulate` for Byte Sum and `std::hash` for the hashing columns. Token length matters, so results are split into a Short Words table (tokens averaging 5 bytes) and a Long Lines table (tokens averaging 130 bytes). diff --git a/include/stringzilla/hash/goldmont.h b/include/stringzilla/hash/goldmont.h index 68441482..8ade12fb 100644 --- a/include/stringzilla/hash/goldmont.h +++ b/include/stringzilla/hash/goldmont.h @@ -28,8 +28,8 @@ extern "C" { * @param hash Pointer to 8x 32-bit hash values, modified in place. * @param block Pointer to 64-byte message block. */ -SZ_HELPER_AUTO void sz_sha256_process_block_goldmont_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_goldmont_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); // Load and byte-swap the first 16 words (big-endian) using SSE diff --git a/include/stringzilla/hash/haswell.h b/include/stringzilla/hash/haswell.h index 95186c6f..02771a4e 100644 --- a/include/stringzilla/hash/haswell.h +++ b/include/stringzilla/hash/haswell.h @@ -107,12 +107,15 @@ SZ_API_COMPTIME sz_u64_t sz_bytesum_haswell(sz_cptr_t text, sz_size_t length) { * ports on this class of core, so the round mix stays throughput-bound rather than shift-bound. */ /** @brief Evaluates `(state_e & state_f) ^ (~state_e & state_g)` across 8 lanes. */ -SZ_HELPER_INLINE __m256i sz_sha256_choice_haswell_(__m256i state_e_u32x8, __m256i state_f_u32x8, __m256i state_g_u32x8) { - return _mm256_xor_si256(state_g_u32x8, _mm256_and_si256(state_e_u32x8, _mm256_xor_si256(state_f_u32x8, state_g_u32x8))); +SZ_HELPER_INLINE __m256i sz_sha256_choice_haswell_(__m256i state_e_u32x8, __m256i state_f_u32x8, + __m256i state_g_u32x8) { + return _mm256_xor_si256(state_g_u32x8, + _mm256_and_si256(state_e_u32x8, _mm256_xor_si256(state_f_u32x8, state_g_u32x8))); } /** @brief Evaluates `(state_a & state_b) ^ (state_a & state_c) ^ (state_b & state_c)` across 8 lanes. */ -SZ_HELPER_INLINE __m256i sz_sha256_majority_haswell_(__m256i state_a_u32x8, __m256i state_b_u32x8, __m256i state_c_u32x8) { +SZ_HELPER_INLINE __m256i sz_sha256_majority_haswell_(__m256i state_a_u32x8, __m256i state_b_u32x8, + __m256i state_c_u32x8) { return _mm256_xor_si256(_mm256_and_si256(_mm256_xor_si256(state_a_u32x8, state_b_u32x8), state_c_u32x8), _mm256_and_si256(state_a_u32x8, state_b_u32x8)); } @@ -120,31 +123,31 @@ SZ_HELPER_INLINE __m256i sz_sha256_majority_haswell_(__m256i state_a_u32x8, __m2 /** @brief Evaluates `ror(state_a_u32x8, 2) ^ ror(state_a_u32x8, 13) ^ ror(state_a_u32x8, 22)` across 8 lanes. */ SZ_HELPER_INLINE __m256i sz_sha256_big_sigma0_haswell_(__m256i state_a_u32x8) { __m256i const rotated_by_2_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_a_u32x8, 2), - _mm256_slli_epi32(state_a_u32x8, 30)); + _mm256_slli_epi32(state_a_u32x8, 30)); __m256i const rotated_by_13_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_a_u32x8, 13), - _mm256_slli_epi32(state_a_u32x8, 19)); + _mm256_slli_epi32(state_a_u32x8, 19)); __m256i const rotated_by_22_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_a_u32x8, 22), - _mm256_slli_epi32(state_a_u32x8, 10)); + _mm256_slli_epi32(state_a_u32x8, 10)); return _mm256_xor_si256(_mm256_xor_si256(rotated_by_2_u32x8, rotated_by_13_u32x8), rotated_by_22_u32x8); } /** @brief Evaluates `ror(state_e_u32x8, 6) ^ ror(state_e_u32x8, 11) ^ ror(state_e_u32x8, 25)` across 8 lanes. */ SZ_HELPER_INLINE __m256i sz_sha256_big_sigma1_haswell_(__m256i state_e_u32x8) { __m256i const rotated_by_6_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_e_u32x8, 6), - _mm256_slli_epi32(state_e_u32x8, 26)); + _mm256_slli_epi32(state_e_u32x8, 26)); __m256i const rotated_by_11_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_e_u32x8, 11), - _mm256_slli_epi32(state_e_u32x8, 21)); + _mm256_slli_epi32(state_e_u32x8, 21)); __m256i const rotated_by_25_u32x8 = _mm256_or_si256(_mm256_srli_epi32(state_e_u32x8, 25), - _mm256_slli_epi32(state_e_u32x8, 7)); + _mm256_slli_epi32(state_e_u32x8, 7)); return _mm256_xor_si256(_mm256_xor_si256(rotated_by_6_u32x8, rotated_by_11_u32x8), rotated_by_25_u32x8); } /** @brief Evaluates `ror(word, 7) ^ ror(word, 18) ^ (word >> 3)` across 8 lanes. */ SZ_HELPER_INLINE __m256i sz_sha256_small_sigma0_haswell_(__m256i message_word_u32x8) { __m256i const rotated_by_7_u32x8 = _mm256_or_si256(_mm256_srli_epi32(message_word_u32x8, 7), - _mm256_slli_epi32(message_word_u32x8, 25)); + _mm256_slli_epi32(message_word_u32x8, 25)); __m256i const rotated_by_18_u32x8 = _mm256_or_si256(_mm256_srli_epi32(message_word_u32x8, 18), - _mm256_slli_epi32(message_word_u32x8, 14)); + _mm256_slli_epi32(message_word_u32x8, 14)); return _mm256_xor_si256(_mm256_xor_si256(rotated_by_7_u32x8, rotated_by_18_u32x8), _mm256_srli_epi32(message_word_u32x8, 3)); } @@ -152,9 +155,9 @@ SZ_HELPER_INLINE __m256i sz_sha256_small_sigma0_haswell_(__m256i message_word_u3 /** @brief Evaluates `ror(word, 17) ^ ror(word, 19) ^ (word >> 10)` across 8 lanes. */ SZ_HELPER_INLINE __m256i sz_sha256_small_sigma1_haswell_(__m256i message_word_u32x8) { __m256i const rotated_by_17_u32x8 = _mm256_or_si256(_mm256_srli_epi32(message_word_u32x8, 17), - _mm256_slli_epi32(message_word_u32x8, 15)); + _mm256_slli_epi32(message_word_u32x8, 15)); __m256i const rotated_by_19_u32x8 = _mm256_or_si256(_mm256_srli_epi32(message_word_u32x8, 19), - _mm256_slli_epi32(message_word_u32x8, 13)); + _mm256_slli_epi32(message_word_u32x8, 13)); return _mm256_xor_si256(_mm256_xor_si256(rotated_by_17_u32x8, rotated_by_19_u32x8), _mm256_srli_epi32(message_word_u32x8, 10)); } @@ -218,17 +221,18 @@ SZ_HELPER_INLINE __m256i sz_sha256_extend_haswell_(__m256i oldest_word_u32x8, __ * `state_d` becomes the next round's `state_e`, and `state_h` is dead on entry so it receives the next * round's `state_a`. */ -SZ_HELPER_INLINE void sz_sha256_round_haswell_( // +SZ_HELPER_INLINE void sz_sha256_round_haswell_( // __m256i state_a_u32x8, __m256i state_b_u32x8, __m256i state_c_u32x8, __m256i *state_d_u32x8, // __m256i state_e_u32x8, __m256i state_f_u32x8, __m256i state_g_u32x8, __m256i *state_h_u32x8, // __m256i message_word_u32x8, sz_u32_t round_constant) { __m256i const temporary_first_u32x8 = _mm256_add_epi32( // - _mm256_add_epi32( - _mm256_add_epi32(*state_h_u32x8, sz_sha256_big_sigma1_haswell_(state_e_u32x8)), - _mm256_add_epi32(sz_sha256_choice_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8), message_word_u32x8)), + _mm256_add_epi32(_mm256_add_epi32(*state_h_u32x8, sz_sha256_big_sigma1_haswell_(state_e_u32x8)), + _mm256_add_epi32(sz_sha256_choice_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8), + message_word_u32x8)), _mm256_set1_epi32((int)round_constant)); __m256i const temporary_second_u32x8 = _mm256_add_epi32( - sz_sha256_big_sigma0_haswell_(state_a_u32x8), sz_sha256_majority_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8)); + sz_sha256_big_sigma0_haswell_(state_a_u32x8), + sz_sha256_majority_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8)); *state_d_u32x8 = _mm256_add_epi32(*state_d_u32x8, temporary_first_u32x8); *state_h_u32x8 = _mm256_add_epi32(temporary_first_u32x8, temporary_second_u32x8); } @@ -255,7 +259,7 @@ SZ_HELPER_INLINE void sz_sha256_compress_haswell_(__m256i hashes_u32x8[8], sz_u8 sz_u32_t const *round_constants = (sz_u32_t const *)sz_x86_hide_pointer_origin_(sz_sha256_round_constants_()); __m256i const byte_swap_u8x32 = _mm256_setr_epi8(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12, // - 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); __m256i low_lanes_u32x8[8], high_lanes_u32x8[8], schedule_u32x8[16]; for (sz_size_t lane_index = 0; lane_index != 8; ++lane_index) { low_lanes_u32x8[lane_index] = _mm256_shuffle_epi8( @@ -271,103 +275,105 @@ SZ_HELPER_INLINE void sz_sha256_compress_haswell_(__m256i hashes_u32x8[8], sz_u8 __m256i state_e_u32x8 = hashes_u32x8[4], state_f_u32x8 = hashes_u32x8[5]; __m256i state_g_u32x8 = hashes_u32x8[6], state_h_u32x8 = hashes_u32x8[7]; - sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, state_g_u32x8, - &state_h_u32x8, schedule_u32x8[0], round_constants[0]); - sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, state_f_u32x8, - &state_g_u32x8, schedule_u32x8[1], round_constants[1]); - sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, state_e_u32x8, - &state_f_u32x8, schedule_u32x8[2], round_constants[2]); - sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, state_d_u32x8, - &state_e_u32x8, schedule_u32x8[3], round_constants[3]); - sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, state_c_u32x8, - &state_d_u32x8, schedule_u32x8[4], round_constants[4]); - sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, state_b_u32x8, - &state_c_u32x8, schedule_u32x8[5], round_constants[5]); - sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, state_a_u32x8, - &state_b_u32x8, schedule_u32x8[6], round_constants[6]); - sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, state_h_u32x8, - &state_a_u32x8, schedule_u32x8[7], round_constants[7]); - sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, state_g_u32x8, - &state_h_u32x8, schedule_u32x8[8], round_constants[8]); - sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, state_f_u32x8, - &state_g_u32x8, schedule_u32x8[9], round_constants[9]); - sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, state_e_u32x8, - &state_f_u32x8, schedule_u32x8[10], round_constants[10]); - sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, state_d_u32x8, - &state_e_u32x8, schedule_u32x8[11], round_constants[11]); - sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, state_c_u32x8, - &state_d_u32x8, schedule_u32x8[12], round_constants[12]); - sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, state_b_u32x8, - &state_c_u32x8, schedule_u32x8[13], round_constants[13]); - sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, state_a_u32x8, - &state_b_u32x8, schedule_u32x8[14], round_constants[14]); - sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, state_h_u32x8, - &state_a_u32x8, schedule_u32x8[15], round_constants[15]); + sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, + state_g_u32x8, &state_h_u32x8, schedule_u32x8[0], round_constants[0]); + sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, + state_f_u32x8, &state_g_u32x8, schedule_u32x8[1], round_constants[1]); + sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, + state_e_u32x8, &state_f_u32x8, schedule_u32x8[2], round_constants[2]); + sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, + state_d_u32x8, &state_e_u32x8, schedule_u32x8[3], round_constants[3]); + sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, + state_c_u32x8, &state_d_u32x8, schedule_u32x8[4], round_constants[4]); + sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, + state_b_u32x8, &state_c_u32x8, schedule_u32x8[5], round_constants[5]); + sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, + state_a_u32x8, &state_b_u32x8, schedule_u32x8[6], round_constants[6]); + sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, + state_h_u32x8, &state_a_u32x8, schedule_u32x8[7], round_constants[7]); + sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, + state_g_u32x8, &state_h_u32x8, schedule_u32x8[8], round_constants[8]); + sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, + state_f_u32x8, &state_g_u32x8, schedule_u32x8[9], round_constants[9]); + sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, + state_e_u32x8, &state_f_u32x8, schedule_u32x8[10], round_constants[10]); + sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, + state_d_u32x8, &state_e_u32x8, schedule_u32x8[11], round_constants[11]); + sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, + state_c_u32x8, &state_d_u32x8, schedule_u32x8[12], round_constants[12]); + sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, + state_b_u32x8, &state_c_u32x8, schedule_u32x8[13], round_constants[13]); + sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, + state_a_u32x8, &state_b_u32x8, schedule_u32x8[14], round_constants[14]); + sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, + state_h_u32x8, &state_a_u32x8, schedule_u32x8[15], round_constants[15]); for (sz_size_t turn_index = 1; turn_index != 4; ++turn_index) { sz_u32_t const *turn_constants = round_constants + turn_index * 16; schedule_u32x8[0] = sz_sha256_extend_haswell_(schedule_u32x8[0], schedule_u32x8[1], schedule_u32x8[9], - schedule_u32x8[14]); - sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, - state_g_u32x8, &state_h_u32x8, schedule_u32x8[0], turn_constants[0]); + schedule_u32x8[14]); + sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, + state_f_u32x8, state_g_u32x8, &state_h_u32x8, schedule_u32x8[0], turn_constants[0]); schedule_u32x8[1] = sz_sha256_extend_haswell_(schedule_u32x8[1], schedule_u32x8[2], schedule_u32x8[10], - schedule_u32x8[15]); - sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, - state_f_u32x8, &state_g_u32x8, schedule_u32x8[1], turn_constants[1]); + schedule_u32x8[15]); + sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, + state_e_u32x8, state_f_u32x8, &state_g_u32x8, schedule_u32x8[1], turn_constants[1]); schedule_u32x8[2] = sz_sha256_extend_haswell_(schedule_u32x8[2], schedule_u32x8[3], schedule_u32x8[11], - schedule_u32x8[0]); - sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, - state_e_u32x8, &state_f_u32x8, schedule_u32x8[2], turn_constants[2]); + schedule_u32x8[0]); + sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, + state_d_u32x8, state_e_u32x8, &state_f_u32x8, schedule_u32x8[2], turn_constants[2]); schedule_u32x8[3] = sz_sha256_extend_haswell_(schedule_u32x8[3], schedule_u32x8[4], schedule_u32x8[12], - schedule_u32x8[1]); - sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, - state_d_u32x8, &state_e_u32x8, schedule_u32x8[3], turn_constants[3]); + schedule_u32x8[1]); + sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, + state_c_u32x8, state_d_u32x8, &state_e_u32x8, schedule_u32x8[3], turn_constants[3]); schedule_u32x8[4] = sz_sha256_extend_haswell_(schedule_u32x8[4], schedule_u32x8[5], schedule_u32x8[13], - schedule_u32x8[2]); - sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, - state_c_u32x8, &state_d_u32x8, schedule_u32x8[4], turn_constants[4]); + schedule_u32x8[2]); + sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, + state_b_u32x8, state_c_u32x8, &state_d_u32x8, schedule_u32x8[4], turn_constants[4]); schedule_u32x8[5] = sz_sha256_extend_haswell_(schedule_u32x8[5], schedule_u32x8[6], schedule_u32x8[14], - schedule_u32x8[3]); - sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, - state_b_u32x8, &state_c_u32x8, schedule_u32x8[5], turn_constants[5]); + schedule_u32x8[3]); + sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, + state_a_u32x8, state_b_u32x8, &state_c_u32x8, schedule_u32x8[5], turn_constants[5]); schedule_u32x8[6] = sz_sha256_extend_haswell_(schedule_u32x8[6], schedule_u32x8[7], schedule_u32x8[15], - schedule_u32x8[4]); - sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, - state_a_u32x8, &state_b_u32x8, schedule_u32x8[6], turn_constants[6]); - schedule_u32x8[7] = sz_sha256_extend_haswell_(schedule_u32x8[7], schedule_u32x8[8], schedule_u32x8[0], schedule_u32x8[5]); - sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, - state_h_u32x8, &state_a_u32x8, schedule_u32x8[7], turn_constants[7]); - schedule_u32x8[8] = sz_sha256_extend_haswell_(schedule_u32x8[8], schedule_u32x8[9], schedule_u32x8[1], schedule_u32x8[6]); - sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, state_f_u32x8, - state_g_u32x8, &state_h_u32x8, schedule_u32x8[8], turn_constants[8]); + schedule_u32x8[4]); + sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, + state_h_u32x8, state_a_u32x8, &state_b_u32x8, schedule_u32x8[6], turn_constants[6]); + schedule_u32x8[7] = sz_sha256_extend_haswell_(schedule_u32x8[7], schedule_u32x8[8], schedule_u32x8[0], + schedule_u32x8[5]); + sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, + state_g_u32x8, state_h_u32x8, &state_a_u32x8, schedule_u32x8[7], turn_constants[7]); + schedule_u32x8[8] = sz_sha256_extend_haswell_(schedule_u32x8[8], schedule_u32x8[9], schedule_u32x8[1], + schedule_u32x8[6]); + sz_sha256_round_haswell_(state_a_u32x8, state_b_u32x8, state_c_u32x8, &state_d_u32x8, state_e_u32x8, + state_f_u32x8, state_g_u32x8, &state_h_u32x8, schedule_u32x8[8], turn_constants[8]); schedule_u32x8[9] = sz_sha256_extend_haswell_(schedule_u32x8[9], schedule_u32x8[10], schedule_u32x8[2], - schedule_u32x8[7]); - sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, state_e_u32x8, - state_f_u32x8, &state_g_u32x8, schedule_u32x8[9], turn_constants[9]); + schedule_u32x8[7]); + sz_sha256_round_haswell_(state_h_u32x8, state_a_u32x8, state_b_u32x8, &state_c_u32x8, state_d_u32x8, + state_e_u32x8, state_f_u32x8, &state_g_u32x8, schedule_u32x8[9], turn_constants[9]); schedule_u32x8[10] = sz_sha256_extend_haswell_(schedule_u32x8[10], schedule_u32x8[11], schedule_u32x8[3], - schedule_u32x8[8]); - sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, state_d_u32x8, - state_e_u32x8, &state_f_u32x8, schedule_u32x8[10], turn_constants[10]); + schedule_u32x8[8]); + sz_sha256_round_haswell_(state_g_u32x8, state_h_u32x8, state_a_u32x8, &state_b_u32x8, state_c_u32x8, + state_d_u32x8, state_e_u32x8, &state_f_u32x8, schedule_u32x8[10], turn_constants[10]); schedule_u32x8[11] = sz_sha256_extend_haswell_(schedule_u32x8[11], schedule_u32x8[12], schedule_u32x8[4], - schedule_u32x8[9]); - sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, state_c_u32x8, - state_d_u32x8, &state_e_u32x8, schedule_u32x8[11], turn_constants[11]); + schedule_u32x8[9]); + sz_sha256_round_haswell_(state_f_u32x8, state_g_u32x8, state_h_u32x8, &state_a_u32x8, state_b_u32x8, + state_c_u32x8, state_d_u32x8, &state_e_u32x8, schedule_u32x8[11], turn_constants[11]); schedule_u32x8[12] = sz_sha256_extend_haswell_(schedule_u32x8[12], schedule_u32x8[13], schedule_u32x8[5], - schedule_u32x8[10]); - sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, state_b_u32x8, - state_c_u32x8, &state_d_u32x8, schedule_u32x8[12], turn_constants[12]); + schedule_u32x8[10]); + sz_sha256_round_haswell_(state_e_u32x8, state_f_u32x8, state_g_u32x8, &state_h_u32x8, state_a_u32x8, + state_b_u32x8, state_c_u32x8, &state_d_u32x8, schedule_u32x8[12], turn_constants[12]); schedule_u32x8[13] = sz_sha256_extend_haswell_(schedule_u32x8[13], schedule_u32x8[14], schedule_u32x8[6], - schedule_u32x8[11]); - sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, state_a_u32x8, - state_b_u32x8, &state_c_u32x8, schedule_u32x8[13], turn_constants[13]); + schedule_u32x8[11]); + sz_sha256_round_haswell_(state_d_u32x8, state_e_u32x8, state_f_u32x8, &state_g_u32x8, state_h_u32x8, + state_a_u32x8, state_b_u32x8, &state_c_u32x8, schedule_u32x8[13], turn_constants[13]); schedule_u32x8[14] = sz_sha256_extend_haswell_(schedule_u32x8[14], schedule_u32x8[15], schedule_u32x8[7], - schedule_u32x8[12]); - sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, state_h_u32x8, - state_a_u32x8, &state_b_u32x8, schedule_u32x8[14], turn_constants[14]); + schedule_u32x8[12]); + sz_sha256_round_haswell_(state_c_u32x8, state_d_u32x8, state_e_u32x8, &state_f_u32x8, state_g_u32x8, + state_h_u32x8, state_a_u32x8, &state_b_u32x8, schedule_u32x8[14], turn_constants[14]); schedule_u32x8[15] = sz_sha256_extend_haswell_(schedule_u32x8[15], schedule_u32x8[0], schedule_u32x8[8], - schedule_u32x8[13]); - sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, state_g_u32x8, - state_h_u32x8, &state_a_u32x8, schedule_u32x8[15], turn_constants[15]); + schedule_u32x8[13]); + sz_sha256_round_haswell_(state_b_u32x8, state_c_u32x8, state_d_u32x8, &state_e_u32x8, state_f_u32x8, + state_g_u32x8, state_h_u32x8, &state_a_u32x8, schedule_u32x8[15], turn_constants[15]); } hashes_u32x8[0] = _mm256_add_epi32(hashes_u32x8[0], _mm256_and_si256(active_u32x8, state_a_u32x8)); @@ -398,9 +404,9 @@ SZ_HELPER_INLINE void sz_sha256_compress_haswell_(__m256i hashes_u32x8[8], sz_u8 * live across the loop between them. * */ -SZ_HELPER_AUTO void sz_sha256_multistate_blocks_haswell_(sz_sha256_state_t *states, sz_size_t active_lanes_count, - sz_u32_t buffered_bitmask, sz_u8_t const **cursors, - sz_size_t const *blocks_per_lane) { +SZ_HELPER_INLINE void sz_sha256_multistate_blocks_haswell_(sz_sha256_state_t *states, sz_size_t active_lanes_count, + sz_u32_t buffered_bitmask, sz_u8_t const **cursors, + sz_size_t const *blocks_per_lane) { __m256i hashes_u32x8[8]; sz_u256_vec_t counts_vec, buffered_vec; sz_u8_t const *sources[8]; @@ -518,8 +524,8 @@ SZ_API_COMPTIME void sz_sha256_multistate_update_haswell(sz_sha256_state_t *stat * Same two-pass shape as the Skylake path, except the lane select is a blend vector rather than a k-mask, * since AVX2 has no mask registers. */ -SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_haswell_(sz_sha256_state_t const *states, - sz_size_t active_lanes_count, sz_u8_t *digests) { +SZ_HELPER_INLINE void sz_sha256_multistate_digest_lanes_haswell_(sz_sha256_state_t const *states, + sz_size_t active_lanes_count, sz_u8_t *digests) { // A SHA256 block is 64 bytes whatever the vector width, so the staged blocks are 512-bit unions even // though the lanes themselves are 256-bit wide. sz_u512_vec_t staged_vec[8]; @@ -548,7 +554,7 @@ SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_haswell_(sz_sha256_state_t sz_size_t const buffered = states[source_lane].block_length; for (sz_size_t byte_index = 0; byte_index != SZ_SHA256_BLOCK_LENGTH; ++byte_index) staged_vec[lane_index].u8s[byte_index] = byte_index < buffered ? states[source_lane].block[byte_index] - : (sz_u8_t)0; + : (sz_u8_t)0; staged_vec[lane_index].u8s[buffered] = 0x80; } sz_sha256_compress_haswell_(hashes_u32x8, staged_blocks, overflow_vec.ymm); @@ -563,7 +569,7 @@ SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_haswell_(sz_sha256_state_t sz_size_t const kept = carried ? 0 : buffered; for (sz_size_t byte_index = 0; byte_index != SZ_SHA256_BLOCK_LENGTH; ++byte_index) staged_vec[lane_index].u8s[byte_index] = byte_index < kept ? states[source_lane].block[byte_index] - : (sz_u8_t)0; + : (sz_u8_t)0; if (!carried) staged_vec[lane_index].u8s[buffered] = 0x80; staged_vec[lane_index].u64s[7] = sz_u64_bytes_reverse(states[source_lane].total_length * 8); } @@ -571,8 +577,8 @@ SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_haswell_(sz_sha256_state_t // Big-endian output is a byte reverse inside each word, and the same transpose that gathered the state // returns each lane's digest as one 32-byte register. - __m256i const byte_swap_u8x32 = _mm256_setr_epi8( // - 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12, // + __m256i const byte_swap_u8x32 = _mm256_setr_epi8( // + 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12, // 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); for (sz_size_t word_index = 0; word_index != 8; ++word_index) hashes_u32x8[word_index] = _mm256_shuffle_epi8(hashes_u32x8[word_index], byte_swap_u8x32); diff --git a/include/stringzilla/hash/icelake.h b/include/stringzilla/hash/icelake.h index 4746ad5d..36e9663b 100644 --- a/include/stringzilla/hash/icelake.h +++ b/include/stringzilla/hash/icelake.h @@ -17,9 +17,8 @@ extern "C" { #if SZ_USE_ICELAKE #if defined(__clang__) && SZ_CLANG_HAS_EVEX512_ -#pragma clang attribute push( \ - __attribute__(( \ - target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vnni,bmi,bmi2,aes,vaes,evex512"))), \ +#pragma clang attribute push( \ + __attribute__((target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vnni,bmi,bmi2,aes,vaes,evex512"))), \ apply_to = function) #elif defined(__clang__) #pragma clang attribute push( \ @@ -255,7 +254,7 @@ SZ_API_COMPTIME void sz_hash_state_init_icelake(sz_hash_state_t *state, sz_u64_t } /** @brief Loads the packed public state into the aligned twin (one `_mm512_loadu_si512` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_icelake_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_icelake_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; state.aes.zmm = _mm512_loadu_si512((__m512i const *)packed->aes); state.sum.zmm = _mm512_loadu_si512((__m512i const *)packed->sum); @@ -266,7 +265,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_icelake_(sz_hash_state } /** @brief Stores the aligned twin back into the packed public state (one `_mm512_storeu_si512` per field). */ -SZ_HELPER_AUTO void sz_hash_state_store_icelake_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_icelake_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { _mm512_storeu_si512((__m512i *)packed->aes, state->aes.zmm); _mm512_storeu_si512((__m512i *)packed->sum, state->sum.zmm); _mm512_storeu_si512((__m512i *)packed->ins, state->ins.zmm); @@ -275,7 +274,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_icelake_(sz_hash_state_t *packed, sz_has } /** @brief Absorbs the buffered 64-byte block into the aligned state with a single VAES `VAESENC` over four lanes. */ -SZ_HELPER_AUTO void sz_hash_state_update_icelake_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_icelake_(sz_hash_state_aligned_t *state) { __m512i const order_u8x64 = _mm512_load_si512((__m512i const *)sz_hash_u8x16x4_shuffle_()); state->aes.zmm = _mm512_aesenc_epi128(state->aes.zmm, state->ins.zmm); state->sum.zmm = _mm512_add_epi64(_mm512_shuffle_epi8(state->sum.zmm, order_u8x64), state->ins.zmm); @@ -395,7 +394,7 @@ typedef struct sz_hash_state_aligned_for_short_x4_t { * @param state Pointer to the 4-wide minimal hash state to initialize. * @param seed 64-bit seed XOR-ed with Pi constants replicated across all four 128-bit lanes. */ -SZ_HELPER_AUTO void sz_hash_state_short_x4_init_icelake_(sz_hash_state_aligned_for_short_x4_t *state, sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_x4_init_icelake_(sz_hash_state_aligned_for_short_x4_t *state, sz_u64_t seed) { // The key is made from the seed and half of it will be mixed with the length in the end __m512i seed_u64x8 = _mm512_set1_epi64(seed); @@ -427,9 +426,9 @@ SZ_HELPER_AUTO void sz_hash_state_short_x4_init_icelake_(sz_hash_state_aligned_f * @param length3 Total byte count for the fourth 128-bit lane. * @return 256-bit vector containing four 64-bit hash values (one per lane). */ -SZ_HELPER_AUTO __m256i sz_hash_state_short_x4_finalize_icelake_(sz_hash_state_aligned_for_short_x4_t const *state, // - sz_size_t length0, sz_size_t length1, sz_size_t length2, - sz_size_t length3) { +SZ_HELPER_INLINE __m256i sz_hash_state_short_x4_finalize_icelake_(sz_hash_state_aligned_for_short_x4_t const *state, // + sz_size_t length0, sz_size_t length1, + sz_size_t length2, sz_size_t length3) { __m512i const padded_lengths_u64x8 = _mm512_set_epi64(0, length3, 0, length2, 0, length1, 0, length0); // Mix the length into the key __m512i key_with_length_u64x8 = _mm512_add_epi64(state->key_vec.zmm, padded_lengths_u64x8); @@ -450,8 +449,8 @@ SZ_HELPER_AUTO __m256i sz_hash_state_short_x4_finalize_icelake_(sz_hash_state_al * @param state Pointer to the 4-wide minimal hash state. * @param blocks_u8x64 512-bit register containing four 128-bit data blocks, one per lane. */ -SZ_HELPER_AUTO void sz_hash_state_short_x4_update_icelake_(sz_hash_state_aligned_for_short_x4_t *state, - __m512i blocks_u8x64) { +SZ_HELPER_INLINE void sz_hash_state_short_x4_update_icelake_(sz_hash_state_aligned_for_short_x4_t *state, + __m512i blocks_u8x64) { __m512i const order_u8x64 = _mm512_load_si512((__m512i const *)sz_hash_u8x16x4_shuffle_()); state->aes_vec.zmm = _mm512_aesenc_epi128(state->aes_vec.zmm, blocks_u8x64); state->sum_vec.zmm = _mm512_add_epi64(_mm512_shuffle_epi8(state->sum_vec.zmm, order_u8x64), blocks_u8x64); @@ -464,8 +463,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_x4_update_icelake_(sz_hash_state_aligned * @param state Pointer to the 4-wide minimal hash state to initialize. * @param seeds_u64x8 Four seeds spread as `[s0,s0,s1,s1,s2,s2,s3,s3]` across the 512-bit register. */ -SZ_HELPER_AUTO void sz_hash_multiseed_x4_init_icelake_(sz_hash_state_aligned_for_short_x4_t *state, - __m512i seeds_u64x8) { +SZ_HELPER_INLINE void sz_hash_multiseed_x4_init_icelake_(sz_hash_state_aligned_for_short_x4_t *state, + __m512i seeds_u64x8) { state->key_vec.zmm = seeds_u64x8; // Replicate the first 128 bits of each Pi half across all four lanes, then XOR the per-lane seeds. sz_u64_t const *pi = sz_hash_pi_constants_(); @@ -485,8 +484,8 @@ SZ_HELPER_AUTO void sz_hash_multiseed_x4_init_icelake_(sz_hash_state_aligned_for * builds it once and reuses it across all seed groups. * @return 256-bit vector with four 64-bit hashes, one per lane. */ -SZ_HELPER_AUTO __m256i sz_hash_multiseed_x4_finalize_icelake_(sz_hash_state_aligned_for_short_x4_t const *state, - __m512i lengths_u64x8) { +SZ_HELPER_INLINE __m256i sz_hash_multiseed_x4_finalize_icelake_(sz_hash_state_aligned_for_short_x4_t const *state, + __m512i lengths_u64x8) { __m512i key_with_length_u64x8 = _mm512_add_epi64(state->key_vec.zmm, lengths_u64x8); __m512i mixed_u8x64 = _mm512_aesenc_epi128(state->sum_vec.zmm, state->aes_vec.zmm); __m512i mixed_in_register_u8x64 = _mm512_aesenc_epi128(_mm512_aesenc_epi128(mixed_u8x64, key_with_length_u64x8), diff --git a/include/stringzilla/hash/lasx.h b/include/stringzilla/hash/lasx.h index 9a47e79c..1628e7cc 100644 --- a/include/stringzilla/hash/lasx.h +++ b/include/stringzilla/hash/lasx.h @@ -243,8 +243,8 @@ SZ_HELPER_INLINE __m128i sz_lsx_pshufb_(__m128i table_u8x16, __m128i indices_u8x } /** @brief Lane-wise GF(16) multiply (poly 0x13) of two nibble vectors via log/antilog tables. */ -SZ_HELPER_AUTO __m128i sz_lsx_gf16_mul_(__m128i factor_a_u8x16, __m128i factor_b_u8x16, __m128i gf16_log_u8x16, - __m128i gf16_exp_u8x16, __m128i zero_u8x16) { +SZ_HELPER_INLINE __m128i sz_lsx_gf16_mul_(__m128i factor_a_u8x16, __m128i factor_b_u8x16, __m128i gf16_log_u8x16, + __m128i gf16_exp_u8x16, __m128i zero_u8x16) { __m128i log_sum_u8x16 = __lsx_vadd_b(sz_lsx_pshufb_(gf16_log_u8x16, factor_a_u8x16), sz_lsx_pshufb_(gf16_log_u8x16, factor_b_u8x16)); // 0..28 __m128i fifteen_u8x16 = __lsx_vreplgr2vr_b(15); @@ -261,7 +261,7 @@ SZ_HELPER_AUTO __m128i sz_lsx_gf16_mul_(__m128i factor_a_u8x16, __m128i factor_b * @brief AES SubBytes on the ShiftRows-permuted state via tower-field GF((2^4)^2) inversion + affine. * @return The S-box output (with the `^ 0x63` AES affine constant already applied). */ -SZ_HELPER_AUTO __m128i sz_emulate_aes_subbytes_lasx_( // +SZ_HELPER_INLINE __m128i sz_emulate_aes_subbytes_lasx_( // __m128i shifted_state_u8x16, __m128i zero_u8x16, __m128i low_nibble_mask_u8x16, __m128i input_transform_low_u8x16, __m128i input_transform_high_u8x16, __m128i sbox_output_low_u8x16, __m128i sbox_output_high_u8x16, __m128i gf16_log_u8x16, __m128i gf16_exp_u8x16, __m128i gf16_inverse_u8x16) { @@ -295,7 +295,7 @@ SZ_HELPER_AUTO __m128i sz_emulate_aes_subbytes_lasx_( // * @brief AES MixColumns over the four 4-byte columns of the S-box output. * @return `c[j] ^ (col_base^col_rot1^col_rot2^col_rot3) ^ xtime(c[j] ^ c[j+1])` per column. */ -SZ_HELPER_AUTO __m128i sz_emulate_aes_mixcolumns_lasx_(__m128i sbox_output_u8x16) { +SZ_HELPER_INLINE __m128i sz_emulate_aes_mixcolumns_lasx_(__m128i sbox_output_u8x16) { // Build rotate masks on the fly: add j and (j+1 within group) shuffle indices. static sz_align_(16) sz_u8_t const rot1_bytes[16] = {1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12}; static sz_align_(16) sz_u8_t const rot2_bytes[16] = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}; @@ -319,7 +319,7 @@ SZ_HELPER_AUTO __m128i sz_emulate_aes_mixcolumns_lasx_(__m128i sbox_output_u8x16 * @return Result of `MixColumns(SubBytes(ShiftRows(state))) ^ round_key`, bit-identical to * `sz_emulate_aesenc_si128_serial_`. */ -SZ_HELPER_AUTO __m128i sz_emulate_aesenc_lasx_(__m128i state_u8x16, __m128i round_key_u8x16) { +SZ_HELPER_INLINE __m128i sz_emulate_aesenc_lasx_(__m128i state_u8x16, __m128i round_key_u8x16) { sz_u8_t const *tables = sz_aes_lasx_tables_(); __m128i zero_u8x16 = __lsx_vreplgr2vr_b(0); __m128i low_nibble_mask_u8x16 = __lsx_vreplgr2vr_b(0x0F); @@ -349,15 +349,15 @@ SZ_HELPER_INLINE __m128i sz_lsx_load128_(void const *pointer) { return __lsx_vld /** @brief Store an LSX register into a 16-byte buffer. */ SZ_HELPER_INLINE void sz_lsx_store128_(void *pointer, __m128i value_u8x16) { __lsx_vst(value_u8x16, pointer, 0); } -SZ_HELPER_AUTO void sz_hash_state_short_init_lasx_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_init_lasx_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { state->key.u64s[0] = seed, state->key.u64s[1] = seed; sz_u64_t const *pi = sz_hash_pi_constants_(); state->aes.u64s[0] = seed ^ pi[0], state->aes.u64s[1] = seed ^ pi[1]; state->sum.u64s[0] = seed ^ pi[8], state->sum.u64s[1] = seed ^ pi[9]; } -SZ_HELPER_AUTO void sz_hash_state_short_update_lasx_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_lasx_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); __m128i aes_u8x16 = sz_emulate_aesenc_lasx_(sz_lsx_load128_(&state->aes), sz_lsx_load128_(&block_vec)); sz_lsx_store128_(&state->aes, aes_u8x16); @@ -365,8 +365,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_update_lasx_(sz_hash_state_aligned_for_s state->sum.u64s[0] += block_vec.u64s[0], state->sum.u64s[1] += block_vec.u64s[1]; } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_lasx_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_lasx_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; __m128i mixed_u8x16 = sz_emulate_aesenc_lasx_(sz_lsx_load128_(&state->sum), sz_lsx_load128_(&state->aes)); @@ -378,10 +378,10 @@ SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_lasx_(sz_hash_state_aligned } SZ_API_COMPTIME void sz_hash_state_init_lasx(sz_hash_state_t *state, sz_u64_t seed); -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_lasx_(sz_hash_state_t const *packed); -SZ_HELPER_AUTO void sz_hash_state_store_lasx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state); -SZ_HELPER_AUTO void sz_hash_state_update_lasx_(sz_hash_state_aligned_t *state); -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_lasx_(sz_hash_state_aligned_t state); +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_lasx_(sz_hash_state_t const *packed); +SZ_HELPER_INLINE void sz_hash_state_store_lasx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state); +SZ_HELPER_INLINE void sz_hash_state_update_lasx_(sz_hash_state_aligned_t *state); +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_lasx_(sz_hash_state_aligned_t state); SZ_API_COMPTIME SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_lasx(sz_cptr_t start, sz_size_t length, sz_u64_t seed) { if (length <= 16) { @@ -463,7 +463,7 @@ SZ_API_COMPTIME void sz_hash_state_init_lasx(sz_hash_state_t *state, sz_u64_t se /** * @brief Loads the packed public state into the aligned internal twin (LASX: 2x `__lasx_xvld` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_lasx_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_lasx_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; __lasx_xvst(__lasx_xvld(packed->aes, 0), state.aes.u8s, 0); __lasx_xvst(__lasx_xvld(packed->aes + 32, 0), state.aes.u8s + 32, 0); @@ -477,7 +477,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_lasx_(sz_hash_state_t } /** @brief Stores the aligned internal twin back into the packed public state. */ -SZ_HELPER_AUTO void sz_hash_state_store_lasx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_lasx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { __lasx_xvst(__lasx_xvld(state->aes.u8s, 0), packed->aes, 0); __lasx_xvst(__lasx_xvld(state->aes.u8s + 32, 0), packed->aes + 32, 0); __lasx_xvst(__lasx_xvld(state->sum.u8s, 0), packed->sum, 0); @@ -492,7 +492,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_lasx_(sz_hash_state_t *packed, sz_hash_s * @brief Absorbs the buffered 64-byte block into the aligned state (four 128-bit lanes), in place. * @param state Pointer to the aligned hash state whose `ins` lanes are consumed. */ -SZ_HELPER_AUTO void sz_hash_state_update_lasx_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_lasx_(sz_hash_state_aligned_t *state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_u128_vec_t *aes_vec = &state->aes.u128s[lane_index]; @@ -509,7 +509,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_lasx_(sz_hash_state_aligned_t *state) { * @param state The hash state, taken by value. * @return 64-bit hash value derived by folding the four AES lanes together with the key. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_lasx_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_lasx_(sz_hash_state_aligned_t state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; @@ -648,8 +648,8 @@ SZ_HELPER_INLINE __m128i sz_sha256_sigma1_lower_lasx_(__m128i words_u32x4) { __lsx_vsrli_w(words_u32x4, 10)); } -SZ_HELPER_AUTO void sz_sha256_process_block_lasx_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_lasx_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); sz_align_(16) sz_u32_t message_schedule[64]; diff --git a/include/stringzilla/hash/neonaes.h b/include/stringzilla/hash/neonaes.h index fa9ba18b..b29ea117 100644 --- a/include/stringzilla/hash/neonaes.h +++ b/include/stringzilla/hash/neonaes.h @@ -28,7 +28,7 @@ extern "C" { * @see "Emulating x86 AES Intrinsics on ARMv8-A" by Michael Brase: * https://blog.michaelbrase.com/2018/05/08/emulating-x86-aes-intrinsics-on-armv8-a/ */ -SZ_HELPER_AUTO uint8x16_t sz_emulate_aesenc_u8x16_neon_(uint8x16_t state_u8x16, uint8x16_t round_key_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_emulate_aesenc_u8x16_neon_(uint8x16_t state_u8x16, uint8x16_t round_key_u8x16) { return veorq_u8(vaesmcq_u8(vaeseq_u8(state_u8x16, vdupq_n_u8(0))), round_key_u8x16); } @@ -52,7 +52,7 @@ SZ_HELPER_INLINE uint64x2_t sz_emulate_aesenc_u64x2_neon_(uint64x2_t state_u64x2 * @param state Pointer to the minimal hash state to initialize. * @param seed 64-bit seed value for the hash. */ -SZ_HELPER_AUTO void sz_hash_state_short_init_neon_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_init_neon_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { // The key is made from the seed and half of it will be mixed with the length in the end uint64x2_t seed_u64x2 = vdupq_n_u64(seed); @@ -77,8 +77,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_init_neon_(sz_hash_state_aligned_for_sho * @param length Total number of bytes that were hashed. * @return 64-bit hash digest. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_neon_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_neon_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { // Mix the length into the key uint64x2_t key_with_length_u64x2 = vaddq_u64(state->key.u64x2, vsetq_lane_u64(length, vdupq_n_u64(0), 0)); // Combine the "sum" and the "AES" blocks @@ -97,7 +97,8 @@ SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_neon_(sz_hash_state_aligned * @param state Pointer to the minimal hash state to update. * @param block_u8x16 16-byte input block as a NEON register. */ -SZ_HELPER_AUTO void sz_hash_state_short_update_neon_(sz_hash_state_aligned_for_short_t *state, uint8x16_t block_u8x16) { +SZ_HELPER_INLINE void sz_hash_state_short_update_neon_(sz_hash_state_aligned_for_short_t *state, + uint8x16_t block_u8x16) { uint8x16_t const order_u8x16 = vld1q_u8(sz_hash_u8x16x4_shuffle_()); state->aes.u8x16 = sz_emulate_aesenc_u8x16_neon_(state->aes.u8x16, block_u8x16); uint8x16_t sum_shuffled_u8x16 = vqtbl1q_u8(vreinterpretq_u8_u64(state->sum.u64x2), order_u8x16); @@ -125,7 +126,7 @@ SZ_API_COMPTIME void sz_hash_state_init_neonaes(sz_hash_state_t *state, sz_u64_t } /** @brief Loads the packed public state into the aligned internal twin (NEON: 4x `vld1q_u8` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_neonaes_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_neonaes_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (int lane_index = 0; lane_index < 4; ++lane_index) { state.aes.u8x16s[lane_index] = vld1q_u8(packed->aes + lane_index * 16); @@ -138,7 +139,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_neonaes_(sz_hash_state } /** @brief Stores the aligned internal twin back into the packed public state (NEON: 4x `vst1q_u8` per field). */ -SZ_HELPER_AUTO void sz_hash_state_store_neonaes_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_neonaes_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (int lane_index = 0; lane_index < 4; ++lane_index) { vst1q_u8(packed->aes + lane_index * 16, state->aes.u8x16s[lane_index]); vst1q_u8(packed->sum + lane_index * 16, state->sum.u8x16s[lane_index]); @@ -152,7 +153,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_neonaes_(sz_hash_state_t *packed, sz_has * @brief Absorbs the buffered 64-byte block into the aligned state (four 128-bit lanes), in place. * @param state Pointer to the aligned hash state whose `ins` lanes are consumed. */ -SZ_HELPER_AUTO void sz_hash_state_update_neonaes_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_neonaes_(sz_hash_state_aligned_t *state) { uint8x16_t const order_u8x16 = vld1q_u8(sz_hash_u8x16x4_shuffle_()); state->aes.u8x16s[0] = sz_emulate_aesenc_u8x16_neon_(state->aes.u8x16s[0], state->ins.u8x16s[0]); uint8x16_t sum_shuffled_0_u8x16 = vqtbl1q_u8(vreinterpretq_u8_u64(state->sum.u64x2s[0]), order_u8x16); @@ -174,7 +175,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_neonaes_(sz_hash_state_aligned_t *state * @param state The internal hash state to finalize. * @return 64-bit hash digest. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_neonaes_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_neonaes_(sz_hash_state_aligned_t state) { // Mix the length into the key uint64x2_t key_with_length_u64x2 = vaddq_u64(state.key.u64x2, vsetq_lane_u64(state.ins_length, vdupq_n_u64(0), 0)); @@ -367,8 +368,8 @@ SZ_API_COMPTIME SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_neonaes(sz_cptr_t text, s * the `< 16` case, where a 16-byte NEON load could read past the input. * @return The number of populated text-lanes (1..4). */ -SZ_HELPER_AUTO sz_size_t sz_hash_multiseed_prepare_neon_(sz_cptr_t text, sz_size_t length, - sz_u512_vec_t *text_lanes_vec) { +SZ_HELPER_INLINE sz_size_t sz_hash_multiseed_prepare_neon_(sz_cptr_t text, sz_size_t length, + sz_u512_vec_t *text_lanes_vec) { if (length <= 16) { sz_u128_vec_t lane_vec; if (length == 16) { lane_vec.u8x16 = vld1q_u8((sz_u8_t const *)text); } diff --git a/include/stringzilla/hash/neonsha.h b/include/stringzilla/hash/neonsha.h index eb11ebd8..62c674ea 100644 --- a/include/stringzilla/hash/neonsha.h +++ b/include/stringzilla/hash/neonsha.h @@ -28,8 +28,8 @@ extern "C" { * @param hash Pointer to 8x 32-bit hash values, modified in place. * @param block Pointer to 64-byte message block. */ -SZ_HELPER_AUTO void sz_sha256_process_block_neon_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_neon_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); // Pre-load all round constants using multi-vector loads (4x16 = 64 bytes per load) diff --git a/include/stringzilla/hash/powervsx.h b/include/stringzilla/hash/powervsx.h index 4026bd80..9a2b552b 100644 --- a/include/stringzilla/hash/powervsx.h +++ b/include/stringzilla/hash/powervsx.h @@ -89,7 +89,7 @@ SZ_HELPER_INLINE __vector unsigned char sz_aes_byte_reverse_mask_powervsx_(void) * @brief Bit-exact VSX equivalent of `sz_emulate_aesenc_si128_serial_` using hardware AES. * @return `MixColumns(SubBytes(ShiftRows(state))) ^ round_key`, identical to the serial reference. */ -SZ_HELPER_AUTO sz_u128_vec_t sz_aesenc_powervsx_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { +SZ_HELPER_INLINE sz_u128_vec_t sz_aesenc_powervsx_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { __vector unsigned char const rev_u8x16 = sz_aes_byte_reverse_mask_powervsx_(); __vector unsigned char state_u8x16 = state_vec.vsx_u8; __vector unsigned char reversed_u8x16 = vec_perm(state_u8x16, state_u8x16, rev_u8x16); @@ -123,16 +123,16 @@ SZ_HELPER_INLINE sz_u128_vec_t sz_shuffle_epi8_powervsx_(sz_u128_vec_t state_vec #pragma region Minimal state for short inputs -SZ_HELPER_AUTO void sz_hash_state_short_update_powervsx_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_powervsx_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); state->aes = sz_aesenc_powervsx_(state->aes, block_vec); state->sum = sz_shuffle_epi8_powervsx_(state->sum, shuffle); state->sum.u64s[0] += block_vec.u64s[0], state->sum.u64s[1] += block_vec.u64s[1]; } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_powervsx_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_powervsx_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; sz_u128_vec_t mixed_vec = sz_aesenc_powervsx_(state->sum, state->aes); @@ -164,7 +164,7 @@ SZ_API_COMPTIME void sz_hash_state_init_powervsx(sz_hash_state_t *state, sz_u64_ /** * @brief Loads the packed public state into the aligned internal twin (4x `vec_xl` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_powervsx_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_powervsx_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; @@ -178,7 +178,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_powervsx_(sz_hash_stat } /** @brief Stores the aligned internal twin back into the packed public state (4x `vec_xst` per 64-byte field). */ -SZ_HELPER_AUTO void sz_hash_state_store_powervsx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_powervsx_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; vec_xst(state->aes.u128s[lane_index].vsx_u8, 0, (unsigned char *)(packed->aes + offset)); @@ -189,7 +189,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_powervsx_(sz_hash_state_t *packed, sz_ha packed->ins_length = state->ins_length; } -SZ_HELPER_AUTO void sz_hash_state_update_powervsx_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_powervsx_(sz_hash_state_aligned_t *state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { state->aes.u128s[lane_index] = sz_aesenc_powervsx_(state->aes.u128s[lane_index], state->ins.u128s[lane_index]); @@ -199,7 +199,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_powervsx_(sz_hash_state_aligned_t *stat } } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_powervsx_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_powervsx_(sz_hash_state_aligned_t state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; diff --git a/include/stringzilla/hash/rvv.h b/include/stringzilla/hash/rvv.h index 76aab698..613cd865 100644 --- a/include/stringzilla/hash/rvv.h +++ b/include/stringzilla/hash/rvv.h @@ -118,8 +118,8 @@ SZ_HELPER_INLINE sz_u8_t const *sz_aes_rot1_rvv_(void) { /* General GF(2^4) multiply of two vector operands via log/antilog, with the discrete-log sum * reduced modulo 15 so the antilog gather index stays inside the 16 active lanes, and a zero-select * for the `0 * x` and `x * 0` cases. */ -SZ_HELPER_AUTO vuint8m1_t sz_gf16_mul_rvv_(vuint8m1_t a_u8m1, vuint8m1_t b_u8m1, vuint8m1_t log_table_u8m1, - vuint8m1_t antilog_table_u8m1, sz_size_t vector_length) { +SZ_HELPER_INLINE vuint8m1_t sz_gf16_mul_rvv_(vuint8m1_t a_u8m1, vuint8m1_t b_u8m1, vuint8m1_t log_table_u8m1, + vuint8m1_t antilog_table_u8m1, sz_size_t vector_length) { vuint8m1_t log_a_u8m1 = __riscv_vrgather_vv_u8m1(log_table_u8m1, a_u8m1, vector_length); vuint8m1_t log_b_u8m1 = __riscv_vrgather_vv_u8m1(log_table_u8m1, b_u8m1, vector_length); vuint8m1_t log_sum_u8m1 = __riscv_vadd_vv_u8m1(log_a_u8m1, log_b_u8m1, vector_length); @@ -138,7 +138,7 @@ SZ_HELPER_AUTO vuint8m1_t sz_gf16_mul_rvv_(vuint8m1_t a_u8m1, vuint8m1_t b_u8m1, * `sz_emulate_aesenc_si128_serial_`. * @see Mike Hamburg, "Accelerating AES with Vector Permute Instructions" (CHES 2009). */ -SZ_HELPER_AUTO sz_u128_vec_t sz_emulate_aesenc_rvv_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { +SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_aesenc_rvv_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { sz_size_t vector_length = __riscv_vsetvl_e8m1(sizeof(sz_u128_vec_t)); // the AES state is exactly one 128-bit block sz_u8_t const *tables = sz_aes_tables_rvv_(); @@ -236,15 +236,16 @@ SZ_HELPER_AUTO sz_u128_vec_t sz_emulate_aesenc_rvv_(sz_u128_vec_t state_vec, sz_ * serial AES round. Every non-AES step (the additive `sum` shuffle, length folding, block layout) * reuses the shared serial helpers, so the digests are guaranteed value-identical. */ -SZ_HELPER_AUTO void sz_hash_state_short_update_rvv_(sz_hash_state_aligned_for_short_t *state, sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_rvv_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); state->aes = sz_emulate_aesenc_rvv_(state->aes, block_vec); state->sum = sz_emulate_shuffle_epi8_serial_(state->sum, shuffle); state->sum.u64s[0] += block_vec.u64s[0], state->sum.u64s[1] += block_vec.u64s[1]; } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_rvv_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_rvv_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; sz_u128_vec_t mixed_vec = sz_emulate_aesenc_rvv_(state->sum, state->aes); @@ -270,7 +271,7 @@ SZ_HELPER_INLINE void sz_hash_store_block_rvv_(sz_ptr_t target, sz_u128_vec_t so /** * @brief Loads the packed public state into the aligned internal twin (one `vle8` block per 16-byte lane). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_rvv_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_rvv_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; @@ -284,7 +285,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_rvv_(sz_hash_state_t c } /** @brief Stores the aligned internal twin back into the packed public state (one `vse8` block per 16-byte lane). */ -SZ_HELPER_AUTO void sz_hash_state_store_rvv_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_rvv_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; sz_hash_store_block_rvv_((sz_ptr_t)(packed->aes + offset), state->aes.u128s[lane_index]); @@ -295,7 +296,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_rvv_(sz_hash_state_t *packed, sz_hash_st packed->ins_length = state->ins_length; } -SZ_HELPER_AUTO void sz_hash_state_update_rvv_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_rvv_(sz_hash_state_aligned_t *state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { state->aes.u128s[lane_index] = sz_emulate_aesenc_rvv_(state->aes.u128s[lane_index], @@ -306,7 +307,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_rvv_(sz_hash_state_aligned_t *state) { } } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_rvv_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_rvv_(sz_hash_state_aligned_t state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; diff --git a/include/stringzilla/hash/rvvcrypto.h b/include/stringzilla/hash/rvvcrypto.h index d61a9b5a..6aec54c9 100644 --- a/include/stringzilla/hash/rvvcrypto.h +++ b/include/stringzilla/hash/rvvcrypto.h @@ -61,16 +61,16 @@ SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_aesenc_rvvcrypto_(sz_u128_vec_t state_ * for the AES round. Every non-AES step reuses the shared serial helpers, so the digests are * guaranteed value-identical to `sz_hash_serial`. */ -SZ_HELPER_AUTO void sz_hash_state_short_update_rvvcrypto_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_rvvcrypto_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); state->aes = sz_emulate_aesenc_rvvcrypto_(state->aes, block_vec); state->sum = sz_emulate_shuffle_epi8_serial_(state->sum, shuffle); state->sum.u64s[0] += block_vec.u64s[0], state->sum.u64s[1] += block_vec.u64s[1]; } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_rvvcrypto_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_rvvcrypto_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; sz_u128_vec_t mixed_vec = sz_emulate_aesenc_rvvcrypto_(state->sum, state->aes); @@ -79,7 +79,7 @@ SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_rvvcrypto_(sz_hash_state_al return mixed_in_register_vec.u64s[0]; } -SZ_HELPER_AUTO void sz_hash_state_update_rvvcrypto_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_rvvcrypto_(sz_hash_state_aligned_t *state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { state->aes.u128s[lane_index] = sz_emulate_aesenc_rvvcrypto_(state->aes.u128s[lane_index], @@ -90,7 +90,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_rvvcrypto_(sz_hash_state_aligned_t *sta } } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_rvvcrypto_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_rvvcrypto_(sz_hash_state_aligned_t state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; @@ -143,7 +143,7 @@ SZ_HELPER_INLINE void sz_hash_store_block_rvvcrypto_(sz_ptr_t target, sz_u128_ve /** * @brief Loads the packed public state into the aligned internal twin (one `vle8` block per 16-byte lane). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_rvvcrypto_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_rvvcrypto_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; @@ -157,7 +157,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_rvvcrypto_(sz_hash_sta } /** @brief Stores the aligned internal twin back into the packed public state (one `vse8` block per 16-byte lane). */ -SZ_HELPER_AUTO void sz_hash_state_store_rvvcrypto_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_rvvcrypto_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; sz_hash_store_block_rvvcrypto_((sz_ptr_t)(packed->aes + offset), state->aes.u128s[lane_index]); @@ -352,8 +352,8 @@ SZ_API_COMPTIME void sz_fill_random_rvvcrypto(sz_ptr_t text, sz_size_t length, s * SHA-256 words are big-endian; we load them with a scalar byte-swap so this path needs only the * `Zvknhb` extension (no `Zvbb`/`Zvkb` `vrev8`). */ -SZ_HELPER_AUTO void sz_sha256_process_block_rvvcrypto_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_rvvcrypto_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); sz_size_t const vector_length = __riscv_vsetvl_e32m1(4); diff --git a/include/stringzilla/hash/serial.h b/include/stringzilla/hash/serial.h index ce529505..6538f020 100644 --- a/include/stringzilla/hash/serial.h +++ b/include/stringzilla/hash/serial.h @@ -73,7 +73,7 @@ SZ_API_COMPTIME sz_u64_t sz_bytesum_serial(sz_cptr_t text, sz_size_t length) { * @return Result of `MixColumns(SubBytes(ShiftRows(state))) ^ round_key`. * @see Based on Jean-Philippe Aumasson's reference implementation: https://github.com/veorq/aesenc-noNI */ -SZ_HELPER_AUTO sz_u128_vec_t sz_emulate_aesenc_si128_serial_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { +SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_aesenc_si128_serial_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { sz_u8_t const *sbox = sz_aes_sbox_(); // Combine `ShiftRows` and `SubBytes` @@ -272,7 +272,7 @@ SZ_HELPER_INLINE sz_u32_t const *sz_sha256_round_constants_(void) { * @param state Pointer to the minimal hash state to initialize. * @param seed 64-bit seed value mixed with Pi constants to form the initial state. */ -SZ_HELPER_AUTO void sz_hash_state_short_init_serial_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_init_serial_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { // The key is made from the seed and half of it will be mixed with the length in the end state->key.u64s[1] = seed; @@ -291,8 +291,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_init_serial_(sz_hash_state_aligned_for_s * @param state Pointer to the minimal hash state. * @param block_vec 128-bit data block to absorb. */ -SZ_HELPER_AUTO void sz_hash_state_short_update_serial_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_serial_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); state->aes = sz_emulate_aesenc_si128_serial_(state->aes, block_vec); state->sum = sz_emulate_shuffle_epi8_serial_(state->sum, shuffle); @@ -305,8 +305,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_update_serial_(sz_hash_state_aligned_for * @param length Total number of bytes hashed, mixed into the key for length sensitivity. * @return 64-bit hash value. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_serial_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_serial_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { // Mix the length into the key sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; @@ -375,7 +375,7 @@ SZ_API_COMPTIME void sz_hash_state_init_serial(sz_hash_state_t *state, sz_u64_t /** * @brief Loads the packed public state into the aligned internal twin (serial: 8x `sz_u64_load` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_serial_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_serial_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; sz_cptr_t const aes = (sz_cptr_t)packed->aes, sum = (sz_cptr_t)packed->sum, ins = (sz_cptr_t)packed->ins; for (int word = 0; word < 8; ++word) { @@ -390,7 +390,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_serial_(sz_hash_state_ } /** @brief Stores the aligned internal twin back into the packed public state. */ -SZ_HELPER_AUTO void sz_hash_state_store_serial_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_serial_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { sz_ptr_t const aes = (sz_ptr_t)packed->aes, sum = (sz_ptr_t)packed->sum, ins = (sz_ptr_t)packed->ins; for (int word = 0; word < 8; ++word) { sz_u64_store(aes + word * 8, state->aes.u64s[word]); @@ -406,7 +406,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_serial_(sz_hash_state_t *packed, sz_hash * @brief Absorbs the buffered 64-byte block into the aligned state (four 128-bit lanes), in place. * @param state Pointer to the aligned hash state whose `ins` lanes are consumed. */ -SZ_HELPER_AUTO void sz_hash_state_update_serial_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_serial_(sz_hash_state_aligned_t *state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); // First 128-bit block @@ -439,7 +439,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_serial_(sz_hash_state_aligned_t *state) * @param state Pointer to the (const) hash state. * @return 64-bit hash value derived by folding the four AES lanes together with the key. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_serial_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_serial_(sz_hash_state_aligned_t state) { sz_u8_t const *shuffle = sz_hash_u8x16x4_shuffle_(); // Mix the length into the key @@ -699,8 +699,9 @@ SZ_HELPER_AUTO sz_size_t sz_hash_multiseed_prepare_serial_(sz_cptr_t text, sz_si * @param seed 64-bit seed for this output. * @return 64-bit hash, identical to `sz_hash_serial(text, length, seed)`. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_multiseed_replay_serial_(sz_u512_vec_t const *text_lanes_vec, - sz_size_t text_lanes_count, sz_size_t length, sz_u64_t seed) { +SZ_HELPER_INLINE sz_u64_t sz_hash_multiseed_replay_serial_(sz_u512_vec_t const *text_lanes_vec, + sz_size_t text_lanes_count, sz_size_t length, + sz_u64_t seed) { sz_hash_state_aligned_for_short_t state; sz_hash_state_short_init_serial_(&state, seed); for (sz_size_t lane_index = 0; lane_index < text_lanes_count; ++lane_index) @@ -771,8 +772,8 @@ SZ_HELPER_INLINE sz_u32_t sz_sha256_sigma1_lower_(sz_u32_t x) { * @param hash Pointer to 8x 32-bit hash values, modified in place. * @param block Pointer to 64-byte message block. */ -SZ_HELPER_AUTO void sz_sha256_process_block_serial_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_serial_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); sz_u32_t message_schedule[16]; sz_u32_t a, b, c, d, e, f, g, h, temp1, temp2; diff --git a/include/stringzilla/hash/skylake.h b/include/stringzilla/hash/skylake.h index a4533db4..ec08ef8b 100644 --- a/include/stringzilla/hash/skylake.h +++ b/include/stringzilla/hash/skylake.h @@ -262,12 +262,14 @@ SZ_API_COMPTIME void sz_fill_random_skylake(sz_ptr_t text, sz_size_t length, sz_ * families need is the parity truth table. */ /** @brief Evaluates `(state_e & state_f) ^ (~state_e & state_g)` across 16 lanes. */ -SZ_HELPER_INLINE __m512i sz_sha256_choice_skylake_(__m512i state_e_u32x16, __m512i state_f_u32x16, __m512i state_g_u32x16) { +SZ_HELPER_INLINE __m512i sz_sha256_choice_skylake_(__m512i state_e_u32x16, __m512i state_f_u32x16, + __m512i state_g_u32x16) { return _mm512_ternarylogic_epi32(state_e_u32x16, state_f_u32x16, state_g_u32x16, 0xCA); } /** @brief Evaluates `(state_a & state_b) ^ (state_a & state_c) ^ (state_b & state_c)` across 16 lanes. */ -SZ_HELPER_INLINE __m512i sz_sha256_majority_skylake_(__m512i state_a_u32x16, __m512i state_b_u32x16, __m512i state_c_u32x16) { +SZ_HELPER_INLINE __m512i sz_sha256_majority_skylake_(__m512i state_a_u32x16, __m512i state_b_u32x16, + __m512i state_c_u32x16) { return _mm512_ternarylogic_epi32(state_a_u32x16, state_b_u32x16, state_c_u32x16, 0xE8); } @@ -285,13 +287,15 @@ SZ_HELPER_INLINE __m512i sz_sha256_big_sigma1_skylake_(__m512i state_e_u32x16) { /** @brief Evaluates `ror(word, 7) ^ ror(word, 18) ^ (word >> 3)` across 16 lanes. */ SZ_HELPER_INLINE __m512i sz_sha256_small_sigma0_skylake_(__m512i message_word_u32x16) { - return _mm512_ternarylogic_epi32(_mm512_ror_epi32(message_word_u32x16, 7), _mm512_ror_epi32(message_word_u32x16, 18), + return _mm512_ternarylogic_epi32(_mm512_ror_epi32(message_word_u32x16, 7), + _mm512_ror_epi32(message_word_u32x16, 18), _mm512_srli_epi32(message_word_u32x16, 3), 0x96); } /** @brief Evaluates `ror(word, 17) ^ ror(word, 19) ^ (word >> 10)` across 16 lanes. */ SZ_HELPER_INLINE __m512i sz_sha256_small_sigma1_skylake_(__m512i message_word_u32x16) { - return _mm512_ternarylogic_epi32(_mm512_ror_epi32(message_word_u32x16, 17), _mm512_ror_epi32(message_word_u32x16, 19), + return _mm512_ternarylogic_epi32(_mm512_ror_epi32(message_word_u32x16, 17), + _mm512_ror_epi32(message_word_u32x16, 19), _mm512_srli_epi32(message_word_u32x16, 10), 0x96); } @@ -357,7 +361,8 @@ SZ_HELPER_INLINE void sz_sha256_transpose_8x16_skylake_(__m512i words_u32x16[8]) * with intermediate buffers instead, the four live arrays push this kernel's frame past 4 KB, and MSVC then * reaches for the CRT's `__chkstk` to probe it, which the `SZ_AVOID_LIBC` build has no way to resolve. */ -SZ_HELPER_INLINE void sz_sha256_transpose_16x16_skylake_(sz_u8_t const *const *lane_blocks, __m512i schedule_u32x16[16]) { +SZ_HELPER_INLINE void sz_sha256_transpose_16x16_skylake_(sz_u8_t const *const *lane_blocks, + __m512i schedule_u32x16[16]) { __m512i const byte_swap_u8x64 = _mm512_set_epi8( // 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, // 44, 45, 46, 47, 40, 41, 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, // @@ -365,8 +370,7 @@ SZ_HELPER_INLINE void sz_sha256_transpose_16x16_skylake_(sz_u8_t const *const *l 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); for (sz_size_t lane_index = 0; lane_index != 16; ++lane_index) - schedule_u32x16[lane_index] = - _mm512_shuffle_epi8(_mm512_loadu_si512(lane_blocks[lane_index]), byte_swap_u8x64); + schedule_u32x16[lane_index] = _mm512_shuffle_epi8(_mm512_loadu_si512(lane_blocks[lane_index]), byte_swap_u8x64); for (sz_size_t butterfly_index = 0; butterfly_index != 8; ++butterfly_index) { __m512i const even_u32x16 = schedule_u32x16[butterfly_index * 2 + 0]; @@ -411,8 +415,9 @@ SZ_HELPER_INLINE void sz_sha256_transpose_16x16_skylake_(sz_u8_t const *const *l */ SZ_HELPER_INLINE __m512i sz_sha256_extend_skylake_(__m512i oldest_word_u32x16, __m512i next_word_u32x16, __m512i ninth_word_u32x16, __m512i fourteenth_word_u32x16) { - return _mm512_add_epi32(_mm512_add_epi32(oldest_word_u32x16, sz_sha256_small_sigma0_skylake_(next_word_u32x16)), - _mm512_add_epi32(ninth_word_u32x16, sz_sha256_small_sigma1_skylake_(fourteenth_word_u32x16))); + return _mm512_add_epi32( + _mm512_add_epi32(oldest_word_u32x16, sz_sha256_small_sigma0_skylake_(next_word_u32x16)), + _mm512_add_epi32(ninth_word_u32x16, sz_sha256_small_sigma1_skylake_(fourteenth_word_u32x16))); } /** @@ -423,17 +428,18 @@ SZ_HELPER_INLINE __m512i sz_sha256_extend_skylake_(__m512i oldest_word_u32x16, _ * `state_d` becomes the next round's `state_e`, and `state_h` is dead on entry so it receives the next * round's `state_a`. */ -SZ_HELPER_INLINE void sz_sha256_round_skylake_( // +SZ_HELPER_INLINE void sz_sha256_round_skylake_( // __m512i state_a_u32x16, __m512i state_b_u32x16, __m512i state_c_u32x16, __m512i *state_d_u32x16, // __m512i state_e_u32x16, __m512i state_f_u32x16, __m512i state_g_u32x16, __m512i *state_h_u32x16, // __m512i message_word_u32x16, sz_u32_t round_constant) { __m512i const temporary_first_u32x16 = _mm512_add_epi32( // - _mm512_add_epi32( - _mm512_add_epi32(*state_h_u32x16, sz_sha256_big_sigma1_skylake_(state_e_u32x16)), - _mm512_add_epi32(sz_sha256_choice_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16), message_word_u32x16)), + _mm512_add_epi32(_mm512_add_epi32(*state_h_u32x16, sz_sha256_big_sigma1_skylake_(state_e_u32x16)), + _mm512_add_epi32(sz_sha256_choice_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16), + message_word_u32x16)), _mm512_set1_epi32((int)round_constant)); __m512i const temporary_second_u32x16 = _mm512_add_epi32( - sz_sha256_big_sigma0_skylake_(state_a_u32x16), sz_sha256_majority_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16)); + sz_sha256_big_sigma0_skylake_(state_a_u32x16), + sz_sha256_majority_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16)); *state_d_u32x16 = _mm512_add_epi32(*state_d_u32x16, temporary_first_u32x16); *state_h_u32x16 = _mm512_add_epi32(temporary_first_u32x16, temporary_second_u32x16); } @@ -457,9 +463,15 @@ SZ_HELPER_INLINE void sz_sha256_round_skylake_( * Written as one straight-line turn of sixteen rounds, then three more turns of the same shape with the * window extension folded in. Sixteen rounds return the eight working variables to their original names and * advance the window exactly one full turn, so every index below is a compile-time constant. + * + * Out-of-line rather than fused into its callers, for the same 4 KB frame budget the transpose above keeps. + * The digest and update kernels each call this twice, and each already hold a 1 KB staging window and 512 + * bytes of hash state; forced inline, MSVC gives every call site its own copy of the schedule below and the + * frame reaches 4640 bytes, past the page that makes it reach for the CRT's `__chkstk`. One turn of sixteen + * rounds is far too much work for a call to show up against, and the window stays local either way. */ -SZ_HELPER_INLINE void sz_sha256_compress_skylake_(__m512i hashes_u32x16[8], sz_u8_t const *const *lane_blocks, - __mmask16 active_m16) { +SZ_HELPER_NOINLINE void sz_sha256_compress_skylake_(__m512i hashes_u32x16[8], sz_u8_t const *const *lane_blocks, + __mmask16 active_m16) { sz_u32_t const *round_constants = (sz_u32_t const *)sz_x86_hide_pointer_origin_(sz_sha256_round_constants_()); __m512i schedule_u32x16[16]; sz_sha256_transpose_16x16_skylake_(lane_blocks, schedule_u32x16); @@ -469,103 +481,121 @@ SZ_HELPER_INLINE void sz_sha256_compress_skylake_(__m512i hashes_u32x16[8], sz_u __m512i state_e_u32x16 = hashes_u32x16[4], state_f_u32x16 = hashes_u32x16[5]; __m512i state_g_u32x16 = hashes_u32x16[6], state_h_u32x16 = hashes_u32x16[7]; - sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, state_f_u32x16, state_g_u32x16, - &state_h_u32x16, schedule_u32x16[0], round_constants[0]); - sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, state_e_u32x16, state_f_u32x16, - &state_g_u32x16, schedule_u32x16[1], round_constants[1]); - sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, state_d_u32x16, state_e_u32x16, - &state_f_u32x16, schedule_u32x16[2], round_constants[2]); - sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, state_c_u32x16, state_d_u32x16, - &state_e_u32x16, schedule_u32x16[3], round_constants[3]); - sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, state_b_u32x16, state_c_u32x16, - &state_d_u32x16, schedule_u32x16[4], round_constants[4]); - sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, state_a_u32x16, state_b_u32x16, - &state_c_u32x16, schedule_u32x16[5], round_constants[5]); - sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, state_h_u32x16, state_a_u32x16, - &state_b_u32x16, schedule_u32x16[6], round_constants[6]); - sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, state_g_u32x16, state_h_u32x16, - &state_a_u32x16, schedule_u32x16[7], round_constants[7]); - sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, state_f_u32x16, state_g_u32x16, - &state_h_u32x16, schedule_u32x16[8], round_constants[8]); - sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, state_e_u32x16, state_f_u32x16, - &state_g_u32x16, schedule_u32x16[9], round_constants[9]); - sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, state_d_u32x16, state_e_u32x16, - &state_f_u32x16, schedule_u32x16[10], round_constants[10]); - sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, state_c_u32x16, state_d_u32x16, - &state_e_u32x16, schedule_u32x16[11], round_constants[11]); - sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, state_b_u32x16, state_c_u32x16, - &state_d_u32x16, schedule_u32x16[12], round_constants[12]); - sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, state_a_u32x16, state_b_u32x16, - &state_c_u32x16, schedule_u32x16[13], round_constants[13]); - sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, state_h_u32x16, state_a_u32x16, - &state_b_u32x16, schedule_u32x16[14], round_constants[14]); - sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, state_g_u32x16, state_h_u32x16, - &state_a_u32x16, schedule_u32x16[15], round_constants[15]); + sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, + state_f_u32x16, state_g_u32x16, &state_h_u32x16, schedule_u32x16[0], round_constants[0]); + sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, + state_e_u32x16, state_f_u32x16, &state_g_u32x16, schedule_u32x16[1], round_constants[1]); + sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, + state_d_u32x16, state_e_u32x16, &state_f_u32x16, schedule_u32x16[2], round_constants[2]); + sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, + state_c_u32x16, state_d_u32x16, &state_e_u32x16, schedule_u32x16[3], round_constants[3]); + sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, + state_b_u32x16, state_c_u32x16, &state_d_u32x16, schedule_u32x16[4], round_constants[4]); + sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, + state_a_u32x16, state_b_u32x16, &state_c_u32x16, schedule_u32x16[5], round_constants[5]); + sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, + state_h_u32x16, state_a_u32x16, &state_b_u32x16, schedule_u32x16[6], round_constants[6]); + sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, + state_g_u32x16, state_h_u32x16, &state_a_u32x16, schedule_u32x16[7], round_constants[7]); + sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, + state_f_u32x16, state_g_u32x16, &state_h_u32x16, schedule_u32x16[8], round_constants[8]); + sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, + state_e_u32x16, state_f_u32x16, &state_g_u32x16, schedule_u32x16[9], round_constants[9]); + sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, + state_d_u32x16, state_e_u32x16, &state_f_u32x16, schedule_u32x16[10], round_constants[10]); + sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, + state_c_u32x16, state_d_u32x16, &state_e_u32x16, schedule_u32x16[11], round_constants[11]); + sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, + state_b_u32x16, state_c_u32x16, &state_d_u32x16, schedule_u32x16[12], round_constants[12]); + sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, + state_a_u32x16, state_b_u32x16, &state_c_u32x16, schedule_u32x16[13], round_constants[13]); + sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, + state_h_u32x16, state_a_u32x16, &state_b_u32x16, schedule_u32x16[14], round_constants[14]); + sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, + state_g_u32x16, state_h_u32x16, &state_a_u32x16, schedule_u32x16[15], round_constants[15]); for (sz_size_t turn_index = 1; turn_index != 4; ++turn_index) { sz_u32_t const *turn_constants = round_constants + turn_index * 16; schedule_u32x16[0] = sz_sha256_extend_skylake_(schedule_u32x16[0], schedule_u32x16[1], schedule_u32x16[9], - schedule_u32x16[14]); - sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, state_f_u32x16, - state_g_u32x16, &state_h_u32x16, schedule_u32x16[0], turn_constants[0]); + schedule_u32x16[14]); + sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, + state_f_u32x16, state_g_u32x16, &state_h_u32x16, schedule_u32x16[0], + turn_constants[0]); schedule_u32x16[1] = sz_sha256_extend_skylake_(schedule_u32x16[1], schedule_u32x16[2], schedule_u32x16[10], - schedule_u32x16[15]); - sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, state_e_u32x16, - state_f_u32x16, &state_g_u32x16, schedule_u32x16[1], turn_constants[1]); + schedule_u32x16[15]); + sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, + state_e_u32x16, state_f_u32x16, &state_g_u32x16, schedule_u32x16[1], + turn_constants[1]); schedule_u32x16[2] = sz_sha256_extend_skylake_(schedule_u32x16[2], schedule_u32x16[3], schedule_u32x16[11], - schedule_u32x16[0]); - sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, state_d_u32x16, - state_e_u32x16, &state_f_u32x16, schedule_u32x16[2], turn_constants[2]); + schedule_u32x16[0]); + sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, + state_d_u32x16, state_e_u32x16, &state_f_u32x16, schedule_u32x16[2], + turn_constants[2]); schedule_u32x16[3] = sz_sha256_extend_skylake_(schedule_u32x16[3], schedule_u32x16[4], schedule_u32x16[12], - schedule_u32x16[1]); - sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, state_c_u32x16, - state_d_u32x16, &state_e_u32x16, schedule_u32x16[3], turn_constants[3]); + schedule_u32x16[1]); + sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, + state_c_u32x16, state_d_u32x16, &state_e_u32x16, schedule_u32x16[3], + turn_constants[3]); schedule_u32x16[4] = sz_sha256_extend_skylake_(schedule_u32x16[4], schedule_u32x16[5], schedule_u32x16[13], - schedule_u32x16[2]); - sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, state_b_u32x16, - state_c_u32x16, &state_d_u32x16, schedule_u32x16[4], turn_constants[4]); + schedule_u32x16[2]); + sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, + state_b_u32x16, state_c_u32x16, &state_d_u32x16, schedule_u32x16[4], + turn_constants[4]); schedule_u32x16[5] = sz_sha256_extend_skylake_(schedule_u32x16[5], schedule_u32x16[6], schedule_u32x16[14], - schedule_u32x16[3]); - sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, state_a_u32x16, - state_b_u32x16, &state_c_u32x16, schedule_u32x16[5], turn_constants[5]); + schedule_u32x16[3]); + sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, + state_a_u32x16, state_b_u32x16, &state_c_u32x16, schedule_u32x16[5], + turn_constants[5]); schedule_u32x16[6] = sz_sha256_extend_skylake_(schedule_u32x16[6], schedule_u32x16[7], schedule_u32x16[15], - schedule_u32x16[4]); - sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, state_h_u32x16, - state_a_u32x16, &state_b_u32x16, schedule_u32x16[6], turn_constants[6]); - schedule_u32x16[7] = sz_sha256_extend_skylake_(schedule_u32x16[7], schedule_u32x16[8], schedule_u32x16[0], schedule_u32x16[5]); - sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, state_g_u32x16, - state_h_u32x16, &state_a_u32x16, schedule_u32x16[7], turn_constants[7]); - schedule_u32x16[8] = sz_sha256_extend_skylake_(schedule_u32x16[8], schedule_u32x16[9], schedule_u32x16[1], schedule_u32x16[6]); - sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, state_f_u32x16, - state_g_u32x16, &state_h_u32x16, schedule_u32x16[8], turn_constants[8]); + schedule_u32x16[4]); + sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, + state_h_u32x16, state_a_u32x16, &state_b_u32x16, schedule_u32x16[6], + turn_constants[6]); + schedule_u32x16[7] = sz_sha256_extend_skylake_(schedule_u32x16[7], schedule_u32x16[8], schedule_u32x16[0], + schedule_u32x16[5]); + sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, + state_g_u32x16, state_h_u32x16, &state_a_u32x16, schedule_u32x16[7], + turn_constants[7]); + schedule_u32x16[8] = sz_sha256_extend_skylake_(schedule_u32x16[8], schedule_u32x16[9], schedule_u32x16[1], + schedule_u32x16[6]); + sz_sha256_round_skylake_(state_a_u32x16, state_b_u32x16, state_c_u32x16, &state_d_u32x16, state_e_u32x16, + state_f_u32x16, state_g_u32x16, &state_h_u32x16, schedule_u32x16[8], + turn_constants[8]); schedule_u32x16[9] = sz_sha256_extend_skylake_(schedule_u32x16[9], schedule_u32x16[10], schedule_u32x16[2], - schedule_u32x16[7]); - sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, state_e_u32x16, - state_f_u32x16, &state_g_u32x16, schedule_u32x16[9], turn_constants[9]); + schedule_u32x16[7]); + sz_sha256_round_skylake_(state_h_u32x16, state_a_u32x16, state_b_u32x16, &state_c_u32x16, state_d_u32x16, + state_e_u32x16, state_f_u32x16, &state_g_u32x16, schedule_u32x16[9], + turn_constants[9]); schedule_u32x16[10] = sz_sha256_extend_skylake_(schedule_u32x16[10], schedule_u32x16[11], schedule_u32x16[3], - schedule_u32x16[8]); - sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, state_d_u32x16, - state_e_u32x16, &state_f_u32x16, schedule_u32x16[10], turn_constants[10]); + schedule_u32x16[8]); + sz_sha256_round_skylake_(state_g_u32x16, state_h_u32x16, state_a_u32x16, &state_b_u32x16, state_c_u32x16, + state_d_u32x16, state_e_u32x16, &state_f_u32x16, schedule_u32x16[10], + turn_constants[10]); schedule_u32x16[11] = sz_sha256_extend_skylake_(schedule_u32x16[11], schedule_u32x16[12], schedule_u32x16[4], - schedule_u32x16[9]); - sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, state_c_u32x16, - state_d_u32x16, &state_e_u32x16, schedule_u32x16[11], turn_constants[11]); + schedule_u32x16[9]); + sz_sha256_round_skylake_(state_f_u32x16, state_g_u32x16, state_h_u32x16, &state_a_u32x16, state_b_u32x16, + state_c_u32x16, state_d_u32x16, &state_e_u32x16, schedule_u32x16[11], + turn_constants[11]); schedule_u32x16[12] = sz_sha256_extend_skylake_(schedule_u32x16[12], schedule_u32x16[13], schedule_u32x16[5], - schedule_u32x16[10]); - sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, state_b_u32x16, - state_c_u32x16, &state_d_u32x16, schedule_u32x16[12], turn_constants[12]); + schedule_u32x16[10]); + sz_sha256_round_skylake_(state_e_u32x16, state_f_u32x16, state_g_u32x16, &state_h_u32x16, state_a_u32x16, + state_b_u32x16, state_c_u32x16, &state_d_u32x16, schedule_u32x16[12], + turn_constants[12]); schedule_u32x16[13] = sz_sha256_extend_skylake_(schedule_u32x16[13], schedule_u32x16[14], schedule_u32x16[6], - schedule_u32x16[11]); - sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, state_a_u32x16, - state_b_u32x16, &state_c_u32x16, schedule_u32x16[13], turn_constants[13]); + schedule_u32x16[11]); + sz_sha256_round_skylake_(state_d_u32x16, state_e_u32x16, state_f_u32x16, &state_g_u32x16, state_h_u32x16, + state_a_u32x16, state_b_u32x16, &state_c_u32x16, schedule_u32x16[13], + turn_constants[13]); schedule_u32x16[14] = sz_sha256_extend_skylake_(schedule_u32x16[14], schedule_u32x16[15], schedule_u32x16[7], - schedule_u32x16[12]); - sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, state_h_u32x16, - state_a_u32x16, &state_b_u32x16, schedule_u32x16[14], turn_constants[14]); + schedule_u32x16[12]); + sz_sha256_round_skylake_(state_c_u32x16, state_d_u32x16, state_e_u32x16, &state_f_u32x16, state_g_u32x16, + state_h_u32x16, state_a_u32x16, &state_b_u32x16, schedule_u32x16[14], + turn_constants[14]); schedule_u32x16[15] = sz_sha256_extend_skylake_(schedule_u32x16[15], schedule_u32x16[0], schedule_u32x16[8], - schedule_u32x16[13]); - sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, state_g_u32x16, - state_h_u32x16, &state_a_u32x16, schedule_u32x16[15], turn_constants[15]); + schedule_u32x16[13]); + sz_sha256_round_skylake_(state_b_u32x16, state_c_u32x16, state_d_u32x16, &state_e_u32x16, state_f_u32x16, + state_g_u32x16, state_h_u32x16, &state_a_u32x16, schedule_u32x16[15], + turn_constants[15]); } hashes_u32x16[0] = _mm512_mask_add_epi32(hashes_u32x16[0], active_m16, hashes_u32x16[0], state_a_u32x16); @@ -596,15 +626,14 @@ SZ_HELPER_INLINE void sz_sha256_compress_skylake_(__m512i hashes_u32x16[8], sz_u * words move by real gather and scatter rather than by 128 scalar loads through a stack union, whose * store-to-load forwarding was the largest fixed cost of a call and the one short messages cannot amortize. */ -SZ_HELPER_AUTO void sz_sha256_multistate_blocks_skylake_(sz_sha256_state_t *states, sz_size_t active_lanes_count, - sz_u32_t buffered_bitmask, sz_u8_t const **cursors, - sz_size_t const *blocks_per_lane) { +SZ_HELPER_INLINE void sz_sha256_multistate_blocks_skylake_(sz_sha256_state_t *states, sz_size_t active_lanes_count, + sz_u32_t buffered_bitmask, sz_u8_t const **cursors, + sz_size_t const *blocks_per_lane) { __m512i hashes_u32x16[8]; sz_u512_vec_t counts_vec; sz_u8_t const *sources[16]; sz_size_t largest_blocks_count = 0; - // The topped-up block and the fallback for a lane with nothing whole left are the same buffer, so one // source array serves both phases and the head costs no extra stack. An absent lane, or one with // nothing whole left, keeps reading that buffer and never advances, so the transpose stays in bounds @@ -650,8 +679,7 @@ SZ_HELPER_AUTO void sz_sha256_multistate_blocks_skylake_(sz_sha256_state_t *stat sz_sha256_transpose_8x16_skylake_(hashes_u32x16); for (sz_size_t pair_index = 0; pair_index != 8; ++pair_index) { if (pair_index < active_lanes_count) - _mm256_storeu_si256((__m256i *)states[pair_index].hash, - _mm512_castsi512_si256(hashes_u32x16[pair_index])); + _mm256_storeu_si256((__m256i *)states[pair_index].hash, _mm512_castsi512_si256(hashes_u32x16[pair_index])); if (pair_index + 8 < active_lanes_count) _mm256_storeu_si256((__m256i *)states[pair_index + 8].hash, _mm512_extracti64x4_epi64(hashes_u32x16[pair_index], 1)); @@ -724,8 +752,8 @@ SZ_API_COMPTIME void sz_sha256_multistate_update_skylake(sz_sha256_state_t *stat * decides which lanes actually consumed the carrier block, so a lane that needed only one block keeps its * state bit-for-bit. Inactive lanes borrow lane zero's hash so the gather stays in bounds. */ -SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_skylake_(sz_sha256_state_t const *states, - sz_size_t active_lanes_count, sz_u8_t *digests) { +SZ_HELPER_INLINE void sz_sha256_multistate_digest_lanes_skylake_(sz_sha256_state_t const *states, + sz_size_t active_lanes_count, sz_u8_t *digests) { sz_u512_vec_t staged_vec[16]; sz_u8_t const *staged_blocks[16]; __m512i hashes_u32x16[8]; @@ -757,8 +785,7 @@ SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_skylake_(sz_sha256_state_t sz_size_t const source_lane = lane_index < active_lanes_count ? lane_index : 0; sz_size_t const buffered = states[source_lane].block_length; __mmask64 const buffered_m64 = sz_u64_clamp_mask_until_(buffered); - staged_vec[lane_index].zmm = - _mm512_maskz_loadu_epi8(buffered_m64, (void const *)states[source_lane].block); + staged_vec[lane_index].zmm = _mm512_maskz_loadu_epi8(buffered_m64, (void const *)states[source_lane].block); staged_vec[lane_index].u8s[buffered] = 0x80; } sz_sha256_compress_skylake_(hashes_u32x16, staged_blocks, overflow_m16); @@ -780,10 +807,10 @@ SZ_HELPER_AUTO void sz_sha256_multistate_digest_lanes_skylake_(sz_sha256_state_t // Big-endian output is a byte reverse inside each word, and the same butterfly that gathered the state // scatters it back to one 32-byte digest per lane. - __m512i const byte_swap_u8x64 = _mm512_set_epi8( // - 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, // - 44, 45, 46, 47, 40, 41, 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, // - 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, // + __m512i const byte_swap_u8x64 = _mm512_set_epi8( // + 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, // + 44, 45, 46, 47, 40, 41, 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, // + 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, // 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); for (sz_size_t word_index = 0; word_index != 8; ++word_index) hashes_u32x16[word_index] = _mm512_shuffle_epi8(hashes_u32x16[word_index], byte_swap_u8x64); diff --git a/include/stringzilla/hash/sve2aes.h b/include/stringzilla/hash/sve2aes.h index 88c8b8e3..0f9f6404 100644 --- a/include/stringzilla/hash/sve2aes.h +++ b/include/stringzilla/hash/sve2aes.h @@ -33,7 +33,7 @@ SZ_HELPER_INLINE svuint8_t sz_emulate_aesenc_u8x16_sve2_(svuint8_t state_u8x, sv } /** @brief A variant of `sz_hash_sve2aes` for strings up to 16 bytes long - smallest SVE register size. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_sve2_upto16_(sz_cptr_t text, sz_size_t length, sz_u64_t seed) { +SZ_HELPER_INLINE sz_u64_t sz_hash_sve2_upto16_(sz_cptr_t text, sz_size_t length, sz_u64_t seed) { svuint8_t state_aes_u8x, state_sum_u8x, state_key_u8x; // To load and store the seed, we don't even need a `svwhilelt_b64(0, 2)`. diff --git a/include/stringzilla/hash/v128.h b/include/stringzilla/hash/v128.h index 5948e062..ba8d44f5 100644 --- a/include/stringzilla/hash/v128.h +++ b/include/stringzilla/hash/v128.h @@ -64,7 +64,7 @@ SZ_HELPER_INLINE v128_t sz_aes_linear_v128_(v128_t low_table_u8x16, v128_t high_ } /** @brief GF(2^4) (modulus `x^4+x+1`) lane-wise multiply via log/antilog swizzles with zero masking. */ -SZ_HELPER_AUTO v128_t sz_aes_gf4_mul_v128_(v128_t a_u8x16, v128_t b_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes_gf4_mul_v128_(v128_t a_u8x16, v128_t b_u8x16) { static sz_align_(16) sz_u8_t const log_table[16] = {0x00, 0x00, 0x01, 0x04, 0x02, 0x08, 0x05, 0x0a, 0x03, 0x0e, 0x09, 0x07, 0x06, 0x0d, 0x0b, 0x0c}; static sz_align_(16) sz_u8_t const exp_lo_table[16] = {0x01, 0x02, 0x04, 0x08, 0x03, 0x06, 0x0c, 0x0b, @@ -91,7 +91,7 @@ SZ_HELPER_AUTO v128_t sz_aes_gf4_mul_v128_(v128_t a_u8x16, v128_t b_u8x16) { * @brief Bit-exact `_mm_aesenc_si128` for one round, identical to `sz_emulate_aesenc_si128_serial_`. * @return `MixColumns(SubBytes(ShiftRows(state))) ^ round_key`, computed with the vpaes tower field. */ -SZ_HELPER_AUTO sz_u128_vec_t sz_emulate_aesenc_v128_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { +SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_aesenc_v128_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { // GF(2^8) <-> GF(2^4)^2 change-of-basis (forward `M` and inverse `M^-1`), as linear nibble tables. static sz_align_(16) sz_u8_t const fwd_lo[16] = {0x00, 0x01, 0x20, 0x21, 0x46, 0x47, 0x66, 0x67, 0x4c, 0x4d, 0x6c, 0x6d, 0x0a, 0x0b, 0x2a, 0x2b}; @@ -175,20 +175,20 @@ SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_shuffle_epi8_v128_(sz_u128_vec_t state #pragma region Hash with SIMD128 AES -SZ_HELPER_AUTO void sz_hash_state_short_init_v128_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_init_v128_(sz_hash_state_aligned_for_short_t *state, sz_u64_t seed) { sz_hash_state_short_init_serial_(state, seed); } -SZ_HELPER_AUTO void sz_hash_state_short_update_v128_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_v128_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); state->aes = sz_emulate_aesenc_v128_(state->aes, block_vec); state->sum = sz_emulate_shuffle_epi8_v128_(state->sum, shuffle_u8x16); state->sum.v128 = wasm_i64x2_add(state->sum.v128, block_vec.v128); } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_v128_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_v128_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; sz_u128_vec_t mixed_vec = sz_emulate_aesenc_v128_(state->sum, state->aes); @@ -198,7 +198,7 @@ SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_v128_(sz_hash_state_aligned } /** @brief Loads the packed public state into the aligned internal twin (4x `wasm_v128_load` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_v128_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_v128_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; @@ -212,7 +212,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_v128_(sz_hash_state_t } /** @brief Stores the aligned internal twin back into the packed public state. */ -SZ_HELPER_AUTO void sz_hash_state_store_v128_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_v128_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; wasm_v128_store(packed->aes + offset, state->aes.u128s[lane_index].v128); @@ -223,7 +223,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_v128_(sz_hash_state_t *packed, sz_hash_s packed->ins_length = state->ins_length; } -SZ_HELPER_AUTO void sz_hash_state_update_v128_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_v128_(sz_hash_state_aligned_t *state) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_u128_vec_t ins_vec = state->ins.u128s[lane_index]; @@ -233,7 +233,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_v128_(sz_hash_state_aligned_t *state) { } } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_v128_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_v128_(sz_hash_state_aligned_t state) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; @@ -432,8 +432,8 @@ SZ_API_COMPTIME void sz_fill_random_v128(sz_ptr_t text, sz_size_t length, sz_u64 * @return 64-bit hash, bit-identical to `sz_hash_v128(text, length, seed)` (hence to serial). * @sa sz_hash_multiseed_replay_serial_, sz_hash_multiseed_prepare_serial_ */ -SZ_HELPER_AUTO sz_u64_t sz_hash_multiseed_replay_v128_(sz_u512_vec_t const *text_lanes_vec, sz_size_t text_lanes_count, - sz_size_t length, sz_u64_t seed) { +SZ_HELPER_INLINE sz_u64_t sz_hash_multiseed_replay_v128_(sz_u512_vec_t const *text_lanes_vec, + sz_size_t text_lanes_count, sz_size_t length, sz_u64_t seed) { sz_align_(16) sz_hash_state_aligned_for_short_t state; sz_hash_state_short_init_v128_(&state, seed); for (sz_size_t lane_index = 0; lane_index < text_lanes_count; ++lane_index) @@ -498,8 +498,8 @@ SZ_HELPER_INLINE v128_t sz_sha256_sigma1_lower_v128_(v128_t x_u32x4) { * so within a group of four the upper two lanes depend on the lower two just computed — handled with a * two-phase `sigma1` and a final lane blend. The input words are loaded big-endian via one shuffle. */ -SZ_HELPER_AUTO void sz_sha256_process_block_v128_(sz_u32_t hash[sz_at_least_(8)], - sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { +SZ_HELPER_INLINE void sz_sha256_process_block_v128_(sz_u32_t hash[sz_at_least_(8)], + sz_u8_t const block[sz_at_least_(SZ_SHA256_BLOCK_LENGTH)]) { sz_u32_t const *round_constants = sz_sha256_round_constants_(); sz_align_(16) sz_u32_t w[64]; diff --git a/include/stringzilla/hash/v128relaxed.h b/include/stringzilla/hash/v128relaxed.h index 059ccc56..a32b22f5 100644 --- a/include/stringzilla/hash/v128relaxed.h +++ b/include/stringzilla/hash/v128relaxed.h @@ -95,7 +95,7 @@ SZ_HELPER_INLINE v128_t sz_aes_linear_v128relaxed_(v128_t low_table_u8x16, v128_ } /** @brief `relaxed_swizzle` counterpart of `sz_aes_gf4_mul_v128_`. */ -SZ_HELPER_AUTO v128_t sz_aes_gf4_mul_v128relaxed_(v128_t a_u8x16, v128_t b_u8x16) { +SZ_HELPER_INLINE v128_t sz_aes_gf4_mul_v128relaxed_(v128_t a_u8x16, v128_t b_u8x16) { static sz_align_(16) sz_u8_t const log_table[16] = {0x00, 0x00, 0x01, 0x04, 0x02, 0x08, 0x05, 0x0a, 0x03, 0x0e, 0x09, 0x07, 0x06, 0x0d, 0x0b, 0x0c}; static sz_align_(16) sz_u8_t const exp_lo_table[16] = {0x01, 0x02, 0x04, 0x08, 0x03, 0x06, 0x0c, 0x0b, @@ -120,7 +120,7 @@ SZ_HELPER_AUTO v128_t sz_aes_gf4_mul_v128relaxed_(v128_t a_u8x16, v128_t b_u8x16 } /** @brief `relaxed_swizzle` counterpart of `sz_emulate_aesenc_v128_` (bit-exact with the serial round). */ -SZ_HELPER_AUTO sz_u128_vec_t sz_emulate_aesenc_v128relaxed_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { +SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_aesenc_v128relaxed_(sz_u128_vec_t state_vec, sz_u128_vec_t round_key_vec) { static sz_align_(16) sz_u8_t const fwd_lo[16] = {0x00, 0x01, 0x20, 0x21, 0x46, 0x47, 0x66, 0x67, 0x4c, 0x4d, 0x6c, 0x6d, 0x0a, 0x0b, 0x2a, 0x2b}; static sz_align_(16) sz_u8_t const fwd_hi[16] = {0x00, 0x3c, 0xd5, 0xe9, 0x34, 0x08, 0xe1, 0xdd, @@ -193,16 +193,16 @@ SZ_HELPER_INLINE sz_u128_vec_t sz_emulate_shuffle_epi8_v128relaxed_(sz_u128_vec_ return result_vec; } -SZ_HELPER_AUTO void sz_hash_state_short_update_v128relaxed_(sz_hash_state_aligned_for_short_t *state, - sz_u128_vec_t block_vec) { +SZ_HELPER_INLINE void sz_hash_state_short_update_v128relaxed_(sz_hash_state_aligned_for_short_t *state, + sz_u128_vec_t block_vec) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); state->aes = sz_emulate_aesenc_v128relaxed_(state->aes, block_vec); state->sum = sz_emulate_shuffle_epi8_v128relaxed_(state->sum, shuffle_u8x16); state->sum.v128 = wasm_i64x2_add(state->sum.v128, block_vec.v128); } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_v128relaxed_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_v128relaxed_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { sz_u128_vec_t key_with_length_vec = state->key; key_with_length_vec.u64s[0] += length; sz_u128_vec_t mixed_vec = sz_emulate_aesenc_v128relaxed_(state->sum, state->aes); @@ -212,7 +212,7 @@ SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_v128relaxed_(sz_hash_state_ } /** @brief Loads the packed public state into the aligned internal twin (4x `wasm_v128_load` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_v128relaxed_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_v128relaxed_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; @@ -226,7 +226,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_v128relaxed_(sz_hash_s } /** @brief Stores the aligned internal twin back into the packed public state. */ -SZ_HELPER_AUTO void sz_hash_state_store_v128relaxed_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_v128relaxed_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_size_t const offset = lane_index * 16; wasm_v128_store(packed->aes + offset, state->aes.u128s[lane_index].v128); @@ -237,7 +237,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_v128relaxed_(sz_hash_state_t *packed, sz packed->ins_length = state->ins_length; } -SZ_HELPER_AUTO void sz_hash_state_update_v128relaxed_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_v128relaxed_(sz_hash_state_aligned_t *state) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); for (sz_size_t lane_index = 0; lane_index < 4; ++lane_index) { sz_u128_vec_t ins_vec = state->ins.u128s[lane_index]; @@ -248,7 +248,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_v128relaxed_(sz_hash_state_aligned_t *s } } -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_v128relaxed_(sz_hash_state_aligned_t state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_v128relaxed_(sz_hash_state_aligned_t state) { v128_t shuffle_u8x16 = wasm_v128_load(sz_hash_u8x16x4_shuffle_()); sz_u128_vec_t key_with_length_vec; key_with_length_vec.u64s[0] = state.key.u64s[0] + state.ins_length; @@ -442,9 +442,9 @@ SZ_API_COMPTIME void sz_fill_random_v128relaxed(sz_ptr_t text, sz_size_t length, * @brief Replays prepared text-lanes through the relaxed-SIMD minimal AES state for a single seed. * @return 64-bit hash, bit-identical to `sz_hash_v128relaxed(text, length, seed)` (hence to serial). */ -SZ_HELPER_AUTO sz_u64_t sz_hash_multiseed_replay_v128relaxed_(sz_u512_vec_t const *text_lanes_vec, - sz_size_t text_lanes_count, sz_size_t length, - sz_u64_t seed) { +SZ_HELPER_INLINE sz_u64_t sz_hash_multiseed_replay_v128relaxed_(sz_u512_vec_t const *text_lanes_vec, + sz_size_t text_lanes_count, sz_size_t length, + sz_u64_t seed) { sz_align_(16) sz_hash_state_aligned_for_short_t state; sz_hash_state_short_init_serial_(&state, seed); for (sz_size_t lane_index = 0; lane_index < text_lanes_count; ++lane_index) diff --git a/include/stringzilla/hash/westmere.h b/include/stringzilla/hash/westmere.h index 0fcdd0d6..9af779a4 100644 --- a/include/stringzilla/hash/westmere.h +++ b/include/stringzilla/hash/westmere.h @@ -29,8 +29,8 @@ extern "C" { * @param state Pointer to the aligned minimal hash state to initialize. * @param seed 64-bit seed value XOR-ed with Pi constants to form the initial state. */ -SZ_HELPER_AUTO void sz_hash_state_short_init_westmere_aligned_(sz_hash_state_aligned_for_short_t *state, - sz_u64_t seed) { +SZ_HELPER_INLINE void sz_hash_state_short_init_westmere_aligned_(sz_hash_state_aligned_for_short_t *state, + sz_u64_t seed) { // The key is made from the seed and half of it will be mixed with the length in the end __m128i seed_u8x16 = _mm_set1_epi64x(seed); @@ -54,8 +54,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_init_westmere_aligned_(sz_hash_state_ali * @param block_u8x16 128-bit data block to absorb. * @param order_u8x16 Shuffle permutation for the additive accumulator lane (loaded from `sz_hash_u8x16x4_shuffle_`). */ -SZ_HELPER_AUTO void sz_hash_state_short_update_westmere_aligned_(sz_hash_state_aligned_for_short_t *state_ptr, - __m128i block_u8x16, __m128i order_u8x16) { +SZ_HELPER_INLINE void sz_hash_state_short_update_westmere_aligned_(sz_hash_state_aligned_for_short_t *state_ptr, + __m128i block_u8x16, __m128i order_u8x16) { state_ptr->aes.xmm = _mm_aesenc_si128(state_ptr->aes.xmm, block_u8x16); state_ptr->sum.xmm = _mm_add_epi64(_mm_shuffle_epi8(state_ptr->sum.xmm, order_u8x16), block_u8x16); } @@ -66,8 +66,8 @@ SZ_HELPER_AUTO void sz_hash_state_short_update_westmere_aligned_(sz_hash_state_a * @param length Total number of bytes hashed, mixed into the key for length sensitivity. * @return 64-bit hash value. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_short_finalize_westmere_aligned_(sz_hash_state_aligned_for_short_t const *state, - sz_size_t length) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_short_finalize_westmere_aligned_(sz_hash_state_aligned_for_short_t const *state, + sz_size_t length) { // Mix the length into the key __m128i key_with_length_u64x2 = _mm_add_epi64(state->key.xmm, _mm_set_epi64x(0, length)); // Combine the "sum" and the "AES" blocks @@ -109,7 +109,7 @@ SZ_API_COMPTIME void sz_hash_state_init_westmere(sz_hash_state_t *state, sz_u64_ /** * @brief Loads the packed public state into the aligned internal twin (4x `_mm_lddqu_si128` per 64-byte field). */ -SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_westmere_(sz_hash_state_t const *packed) { +SZ_HELPER_INLINE sz_hash_state_aligned_t sz_hash_state_load_westmere_(sz_hash_state_t const *packed) { sz_hash_state_aligned_t state; for (int lane_index = 0; lane_index < 4; ++lane_index) { state.aes.xmms[lane_index] = _mm_lddqu_si128((__m128i const *)&packed->aes[lane_index * 16]); @@ -122,7 +122,7 @@ SZ_HELPER_AUTO sz_hash_state_aligned_t sz_hash_state_load_westmere_(sz_hash_stat } /** @brief Stores the aligned internal twin back into the packed public state (4x `_mm_storeu_si128` per field). */ -SZ_HELPER_AUTO void sz_hash_state_store_westmere_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE void sz_hash_state_store_westmere_(sz_hash_state_t *packed, sz_hash_state_aligned_t const *state) { for (int lane_index = 0; lane_index < 4; ++lane_index) { _mm_storeu_si128((__m128i *)&packed->aes[lane_index * 16], state->aes.xmms[lane_index]); _mm_storeu_si128((__m128i *)&packed->sum[lane_index * 16], state->sum.xmms[lane_index]); @@ -136,7 +136,7 @@ SZ_HELPER_AUTO void sz_hash_state_store_westmere_(sz_hash_state_t *packed, sz_ha * @brief Absorbs the buffered 64-byte block into the aligned state (four 128-bit lanes), in place. * @param state Pointer to the aligned hash state whose `ins` lanes are consumed. */ -SZ_HELPER_AUTO void sz_hash_state_update_westmere_(sz_hash_state_aligned_t *state) { +SZ_HELPER_INLINE void sz_hash_state_update_westmere_(sz_hash_state_aligned_t *state) { __m128i const order_u8x16 = _mm_load_si128((__m128i const *)sz_hash_u8x16x4_shuffle_()); state->aes.xmms[0] = _mm_aesenc_si128(state->aes.xmms[0], state->ins.xmms[0]); state->aes.xmms[1] = _mm_aesenc_si128(state->aes.xmms[1], state->ins.xmms[1]); @@ -153,7 +153,7 @@ SZ_HELPER_AUTO void sz_hash_state_update_westmere_(sz_hash_state_aligned_t *stat * @param state Pointer to the (const) aligned hash state; lanes are read directly. * @return 64-bit hash value derived by folding the four AES lanes together with the key. */ -SZ_HELPER_AUTO sz_u64_t sz_hash_state_finalize_westmere_(sz_hash_state_aligned_t const *state) { +SZ_HELPER_INLINE sz_u64_t sz_hash_state_finalize_westmere_(sz_hash_state_aligned_t const *state) { // Mix the length into the key __m128i key_with_length_u64x2 = _mm_add_epi64(state->key.xmm, _mm_set_epi64x(0, state->ins_length)); @@ -296,8 +296,8 @@ SZ_API_COMPTIME SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_westmere(sz_cptr_t start, * the `< 16` case, where a 16-byte SSE load could read past the input. * @return The number of populated text-lanes (1..4). */ -SZ_HELPER_AUTO sz_size_t sz_hash_multiseed_prepare_westmere_(sz_cptr_t text, sz_size_t length, - sz_u512_vec_t *text_lanes_vec) { +SZ_HELPER_INLINE sz_size_t sz_hash_multiseed_prepare_westmere_(sz_cptr_t text, sz_size_t length, + sz_u512_vec_t *text_lanes_vec) { if (length <= 16) { sz_u128_vec_t lane_vec; if (length == 16) { lane_vec.xmm = _mm_lddqu_si128((__m128i const *)text); } diff --git a/include/stringzilla/intersect/README.md b/include/stringzilla/intersect/README.md index 54f33988..5fa41caf 100644 --- a/include/stringzilla/intersect/README.md +++ b/include/stringzilla/intersect/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are throughput in comparisons/s, rendered as Mcmp/s, measured with `bench/sequence.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Numbers are throughput in comparisons/s, rendered as Mcmp/s, measured with `bench/sequence.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Each row is the library compiled with that single backend forced on one fixed chip, and the single column is the operation, so coverage and cross-chip comparison read down the column. The Standard row is the platform's best stock equivalent, `std::unordered_map`. Token length affects per-element cost, so results are split into a Short Words table (tokens averaging 5 bytes) and a Long Lines table (tokens averaging 130 bytes). diff --git a/include/stringzilla/intersect/icelake.h b/include/stringzilla/intersect/icelake.h index 4c62682c..58cf8e59 100644 --- a/include/stringzilla/intersect/icelake.h +++ b/include/stringzilla/intersect/icelake.h @@ -48,7 +48,7 @@ extern "C" { * @param values_u64x4 A 256-bit vector holding four 64-bit values [a, b, c, d]. * @return Non-zero if at least two of the four values are identical, zero otherwise. */ -SZ_HELPER_AUTO int sz_u64x4_contains_collisions_haswell_(__m256i values_u64x4) { +SZ_HELPER_INLINE int sz_u64x4_contains_collisions_haswell_(__m256i values_u64x4) { // Assume `values_u64x4` stores: [a, b, c, d]. // 0xB1 produces [b, a, d, c], 0x4E produces [c, d, a, b], 0x1B produces [d, c, b, a]. __m256i cmp1_u64x4 = _mm256_cmpeq_epi64(values_u64x4, _mm256_permute4x64_epi64(values_u64x4, 0xB1)); diff --git a/include/stringzilla/memory/README.md b/include/stringzilla/memory/README.md index d9da9b06..ed7d929a 100644 --- a/include/stringzilla/memory/README.md +++ b/include/stringzilla/memory/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are throughput in GB/s, measured with `bench/memory.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Numbers are throughput in GB/s, measured with `bench/memory.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Memory operations are bandwidth-bound and measured solo, so they are not tokenized and appear in a single table. Each row is the library compiled with that single backend forced on one fixed chip, and each column is one operation, so coverage and cross-chip comparison read down a single column. The Standard row is the platform's best stock equivalent per column — `std::memcpy`, `std::memmove`, `std::memset`, and `std::transform`. diff --git a/include/stringzilla/memory/lasx.h b/include/stringzilla/memory/lasx.h index cf49cfcb..7fd48b8f 100644 --- a/include/stringzilla/memory/lasx.h +++ b/include/stringzilla/memory/lasx.h @@ -23,7 +23,7 @@ extern "C" { * @param offset Byte offset into `lut` at which to start the 16-byte slice. * @return A 256-bit LASX register with the 16-byte slice duplicated into both 128-bit lanes. */ -SZ_HELPER_AUTO __m256i sz_lookup_load_lut_lasx_(char const lut[sz_at_least_(256)], sz_size_t offset) { +SZ_HELPER_INLINE __m256i sz_lookup_load_lut_lasx_(char const lut[sz_at_least_(256)], sz_size_t offset) { sz_u8_t lut_pairs[32]; for (sz_size_t lane_index = 0; lane_index < 16; ++lane_index) lut_pairs[lane_index] = lut_pairs[lane_index + 16] = (sz_u8_t)lut[offset + lane_index]; diff --git a/include/stringzilla/small_string.h b/include/stringzilla/small_string.h index 22a8cee7..9b021ed9 100644 --- a/include/stringzilla/small_string.h +++ b/include/stringzilla/small_string.h @@ -50,7 +50,7 @@ extern "C" { * It's designed to avoid any branches on read-only operations, and can store up * to 22 characters on stack on 64-bit machines, followed by the SZ_NULL-termination character. * - * @section Changing Length + * @section small_string_changing_length Changing Length * * One nice thing about this design, is that you can, in many cases, change the length of the string * without any branches, invoking a `+=` or `-=` on the 64-bit `length` field. If the string is on heap, diff --git a/include/stringzilla/sort/README.md b/include/stringzilla/sort/README.md index e0efdd55..40a587a4 100644 --- a/include/stringzilla/sort/README.md +++ b/include/stringzilla/sort/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are sorting throughput in comparisons/s, rendered as Mcmp/s, measured with `bench/sequence.cpp` over the `leipzig1M_en.txt` corpus, reporting the median of repeated runs. +Numbers are sorting throughput in comparisons/s, rendered as Mcmp/s, measured with `bench/sequence.cpp` over the `leipzig1M.txt` corpus, reporting the median of repeated runs. Sorting throughput is reported as comparisons/s, with the operation count modeled as N¡log2 N, matching StringWars. Each row is the library compiled with that single backend forced on one fixed chip, and each column is one operation, so coverage and cross-chip comparison read down a single column. The Standard row is the platform's best stock equivalent per column — `std::sort` for Argsort and Pgram Sort, `std::stable_sort` for Uncased Argsort. diff --git a/include/stringzilla/sort/haswell.h b/include/stringzilla/sort/haswell.h index 9746f25c..ab74cc05 100644 --- a/include/stringzilla/sort/haswell.h +++ b/include/stringzilla/sort/haswell.h @@ -155,7 +155,7 @@ SZ_HELPER_INLINE void sz_sort_haswell_compact_block_into_( // * left-packs all three comparison kinds together, so each block is loaded once and the equal mask * is derived for free. */ -SZ_HELPER_AUTO void sz_sequence_argsort_haswell_3way_partition_( // +SZ_HELPER_INLINE void sz_sequence_argsort_haswell_3way_partition_( // sz_pgram_t *const initial_pgrams, sz_sorted_idx_t *const initial_order, // sz_pgram_t *const partitioned_pgrams, sz_sorted_idx_t *const partitioned_order, // sz_size_t const start_in_sequence, sz_size_t const end_in_sequence, // diff --git a/include/stringzilla/sort/neon.h b/include/stringzilla/sort/neon.h index fd0450e3..d02a569b 100644 --- a/include/stringzilla/sort/neon.h +++ b/include/stringzilla/sort/neon.h @@ -98,7 +98,7 @@ SZ_HELPER_INLINE sz_size_t sz_sort_neon_compact4_( // * the three regions are then copied back contiguously. A single block-major pass left-packs all three * comparison kinds together, so each block is loaded once and the equal mask is derived for free. */ -SZ_HELPER_AUTO void sz_sequence_argsort_neon_3way_partition_( // +SZ_HELPER_INLINE void sz_sequence_argsort_neon_3way_partition_( // sz_pgram_t *const initial_pgrams, sz_sorted_idx_t *const initial_order, // sz_pgram_t *const partitioned_pgrams, sz_sorted_idx_t *const partitioned_order, // sz_size_t const start_in_sequence, sz_size_t const end_in_sequence, // diff --git a/include/stringzilla/sort/rvv.h b/include/stringzilla/sort/rvv.h index aa5765cf..e069b866 100644 --- a/include/stringzilla/sort/rvv.h +++ b/include/stringzilla/sort/rvv.h @@ -51,7 +51,7 @@ extern "C" { * @param first_pivot_offset Receives the index of the first element equal to the pivot. * @param last_pivot_offset Receives the index of the last element equal to the pivot. */ -SZ_HELPER_AUTO void sz_sequence_argsort_rvv_3way_partition_( +SZ_HELPER_INLINE void sz_sequence_argsort_rvv_3way_partition_( sz_pgram_t *const initial_pgrams, sz_sorted_idx_t *const initial_order, sz_pgram_t *const partitioned_pgrams, sz_sorted_idx_t *const partitioned_order, sz_size_t const start_in_sequence, sz_size_t const end_in_sequence, sz_size_t *const first_pivot_offset, sz_size_t *const last_pivot_offset) { diff --git a/include/stringzilla/sort/skylake.h b/include/stringzilla/sort/skylake.h index cd023cbd..f2827e6a 100644 --- a/include/stringzilla/sort/skylake.h +++ b/include/stringzilla/sort/skylake.h @@ -50,7 +50,7 @@ extern "C" { * @param first_pivot_offset Receives the index of the first element equal to the pivot. * @param last_pivot_offset Receives the index of the last element equal to the pivot. */ -SZ_HELPER_AUTO void sz_sequence_argsort_skylake_3way_partition_( // +SZ_HELPER_INLINE void sz_sequence_argsort_skylake_3way_partition_( // sz_pgram_t *const initial_pgrams, sz_sorted_idx_t *const initial_order, // sz_pgram_t *const partitioned_pgrams, sz_sorted_idx_t *const partitioned_order, // sz_size_t const start_in_sequence, sz_size_t const end_in_sequence, // diff --git a/include/stringzilla/sort/sve.h b/include/stringzilla/sort/sve.h index e433c2ef..f3a16c7f 100644 --- a/include/stringzilla/sort/sve.h +++ b/include/stringzilla/sort/sve.h @@ -39,7 +39,7 @@ extern "C" { * @param first_pivot_offset Receives the index of the first element equal to the pivot. * @param last_pivot_offset Receives the index of the last element equal to the pivot. */ -SZ_HELPER_AUTO void sz_sequence_argsort_sve_3way_partition_( +SZ_HELPER_INLINE void sz_sequence_argsort_sve_3way_partition_( sz_pgram_t *const initial_pgrams, sz_sorted_idx_t *const initial_order, sz_pgram_t *const partitioned_pgrams, sz_sorted_idx_t *const partitioned_order, sz_size_t const start_in_sequence, sz_size_t const end_in_sequence, sz_size_t *const first_pivot_offset, sz_size_t *const last_pivot_offset) { diff --git a/include/stringzilla/stringzilla.h b/include/stringzilla/stringzilla.h index 995095cb..babf8260 100644 --- a/include/stringzilla/stringzilla.h +++ b/include/stringzilla/stringzilla.h @@ -11,7 +11,7 @@ * @see StringZilla docs: https://github.com/ashvardanian/StringZilla/blob/main/README.md * @see LibC string docs: https://pubs.opengroup.org/onlinepubs/009695399/basedefs/string.h.html * - * @section Introduction + * @section sz_introduction Introduction * * StringZilla is multi-language project designed for high-throughput string processing, differentiating * the low-level "embeddable" mostly-C core implementation, containing: @@ -36,7 +36,7 @@ * The core implementations of those algorithms are mostly structured as callable structure templates, as opposed to * template functions to simplify specialized overloads and reusing the state between invocations. * - * @section Compilation Settings + * @section sz_compilation_settings Compilation Settings * * Consider overriding the following macros to customize the library: * @@ -268,35 +268,28 @@ SZ_HELPER_AUTO sz_capability_t sz_capability_from_string_implementation_(char co } /** - * @brief Internal helper function to convert SIMD capabilities to a string. + * @brief Writes the comma-separated capability names into @p buffer, always null-terminating. + * @return Bytes written, excluding the terminator; the text truncates rather than overflowing @p capacity. * @sa sz_capabilities_to_string, sz_capabilities */ -SZ_HELPER_AUTO sz_cptr_t sz_capabilities_to_string_implementation_(sz_capability_t caps) { +SZ_HELPER_AUTO sz_size_t sz_capabilities_to_string_implementation_(sz_capability_t caps, char *buffer, + sz_size_t capacity) { - static char buffer[256]; + if (capacity == 0) return 0; char *p = buffer; - char *const end = buffer + sizeof(buffer); + char *const end = buffer + capacity; - // Use the new function to get capability strings char const *cap_strings[SZ_CAPABILITIES_COUNT]; sz_size_t cap_count = sz_capabilities_to_strings_implementation_(caps, cap_strings, SZ_CAPABILITIES_COUNT); - // Build the comma-separated string for (sz_size_t capability_index = 0; capability_index < cap_count; capability_index++) { - if (capability_index > 0) { - // Add separator if this is not the first capability. - char const sep[2] = {',', '\0'}; - char const *s = sep; - while (*s && p < end - 1) *p++ = *s++; - } - // Append the capability name character by character. + if (capability_index > 0 && p < end - 1) *p++ = ','; char const *s = cap_strings[capability_index]; while (*s && p < end - 1) *p++ = *s++; } - // Null-terminate the string. *p = '\0'; - return buffer; + return (sz_size_t)(p - buffer); } /* The runtime detectors below report the FULL hardware capability set, independent of which `SZ_USE_*` @@ -576,7 +569,7 @@ SZ_API_COMPTIME sz_capability_t sz_capabilities_implementation_x86_(void) { * @brief Function to determine the SIMD capabilities of the current 64-bit RISC-V machine at @b runtime. * @return A bitmask of the SIMD capabilities represented as a `sz_capability_t` enum value. */ -SZ_HELPER_AUTO sz_capability_t sz_capabilities_implementation_riscv_(void) { +SZ_HELPER_INLINE sz_capability_t sz_capabilities_implementation_riscv_(void) { #if defined(SZ_IS_LINUX_) && !SZ_AVOID_LIBC // The base "V" extension is reported through the auxiliary vector, but the individual @@ -634,7 +627,7 @@ SZ_HELPER_AUTO sz_capability_t sz_capabilities_implementation_riscv_(void) { * @brief Function to determine the SIMD capabilities of the current LoongArch machine at @b runtime. * @return A bitmask of the SIMD capabilities represented as a `sz_capability_t` enum value. */ -SZ_HELPER_AUTO sz_capability_t sz_capabilities_implementation_loongarch_(void) { +SZ_HELPER_INLINE sz_capability_t sz_capabilities_implementation_loongarch_(void) { #if defined(SZ_IS_LINUX_) && !SZ_AVOID_LIBC // The SIMD extensions are reported through the auxiliary vector, matching `asm/hwcap.h`: @@ -657,7 +650,7 @@ SZ_HELPER_AUTO sz_capability_t sz_capabilities_implementation_loongarch_(void) { * @brief Function to determine the SIMD capabilities of the current IBM POWER machine at @b runtime. * @return A bitmask of the SIMD capabilities represented as a `sz_capability_t` enum value. */ -SZ_HELPER_AUTO sz_capability_t sz_capabilities_implementation_power_(void) { +SZ_HELPER_INLINE sz_capability_t sz_capabilities_implementation_power_(void) { #if (defined(SZ_IS_LINUX_) || defined(SZ_IS_FREEBSD_)) && !SZ_AVOID_LIBC // The `powervsx` kernels target POWER9 (`-mcpu=power9 -mvsx`), so both facts are required, @@ -760,7 +753,10 @@ SZ_API_RUNTIME sz_capability_t sz_capabilities(void) { return (sz_capability_t)(sz_capabilities_comptime_implementation_() & sz_capabilities_runtime_implementation_()); } SZ_API_RUNTIME sz_cptr_t sz_capabilities_to_string(sz_capability_t caps) { - return sz_capabilities_to_string_implementation_(caps); + // The one place that must own storage, because the signature returns a string it does not receive. + static char names[256]; + sz_capabilities_to_string_implementation_(caps, names, sizeof(names)); + return names; } SZ_API_RUNTIME void sz_dispatch_table_init(void) {} SZ_API_RUNTIME void sz_dispatch_table_update(sz_capability_t caps) { sz_unused_(caps); } // No-op in non-dynamic builds diff --git a/include/stringzilla/stringzilla.hpp b/include/stringzilla/stringzilla.hpp index 8b746f9e..7c2b5691 100644 --- a/include/stringzilla/stringzilla.hpp +++ b/include/stringzilla/stringzilla.hpp @@ -3216,7 +3216,7 @@ class basic_string_slice { /** * @brief Memory-owning string class with a Small String Optimization. * - * @section API + * @section sz_cpp_api API * * Some APIs are different from `basic_string_slice`: * * `lstrip`, `rstrip`, `strip` modify the string in-place, instead of returning a new view. @@ -3233,7 +3233,7 @@ class basic_string_slice { * * `[r]partition`, `[r]split`, `[r]find_all` missing to enforce lifetime on long operations. * * `remove_prefix`, `remove_suffix` for now. * - * @section Exceptions + * @section sz_cpp_exceptions Exceptions * * Default constructor is `constexpr`. Move constructor and move assignment operator are `noexcept`. * Copy constructor and copy assignment operator are not! They may throw `std::bad_alloc` if the memory diff --git a/include/stringzilla/types.h b/include/stringzilla/types.h index aad9073b..40bba961 100644 --- a/include/stringzilla/types.h +++ b/include/stringzilla/types.h @@ -175,7 +175,31 @@ #endif #define SZ_API_COMPTIME SZ_MAYBE_UNUSED SZ_C_INLINE + +// A portable scalar helper, inline in whichever translation unit uses it, and `constexpr` from C++20 +// onwards. That qualifier is what lets a caller fold the helper at compile time, and what lets a CUDA +// kernel call it at all - `--expt-relaxed-constexpr` reaches a host `constexpr` function from device code, +// so this layer never has to name an execution space of its own. +// +// A helper reaching an intrinsic can never be constant-evaluated, and a `constexpr` function with no +// constant-evaluated path is ill-formed: Clang and MSVC reject the definition, GCC 12 too, and only +// GCC 13+ softens it to `-Winvalid-constexpr`. Every such helper - the whole of each ISA backend, plus the +// few portable ones wrapping a builtin or a type-punned load - carries `SZ_HELPER_INLINE` instead, which +// is why no translation unit needs a `-Wno-` flag to compile this header. +// +// C++20 is the floor rather than C++11 because these helpers declare their locals before filling them, and +// only C++20 permits an uninitialized local in a `constexpr` function. An older dialect - the Python +// extensions build at C++17 - gets the same plain inline function it had before the qualifier existed. +// +// MSVC is the one front end that gets the plain helper: its bit-scan and byte-swap intrinsics are not +// constant-evaluable, so `sz_u64_ctz` and every helper that reaches one - `sz_size_bit_ceil`, the folded +// rune iterators, the uncased search - is rejected with C3615, an error no `/wd` can silence. The qualifier +// buys compile-time folding and `nvcc` reach, and MSVC hosts neither, so nothing is lost by dropping it. +#if defined(__cplusplus) && __cplusplus >= 202002L && !(defined(_MSC_VER) && !defined(__clang__)) +#define SZ_HELPER_AUTO SZ_MAYBE_UNUSED SZ_C_INLINE constexpr +#else #define SZ_HELPER_AUTO SZ_MAYBE_UNUSED SZ_C_INLINE +#endif // Exported symbol under dynamic dispatch or `SZ_EXPORT` (emitted from one amalgamation TU, links like a // normal C library — the Rust binding without `dynamic-dispatch`); otherwise a header-inline tier. @@ -1560,21 +1584,21 @@ SZ_HELPER_AUTO int sz_u32_popcount(sz_u32_t x) { return (((x + (x >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24; } #else -SZ_HELPER_AUTO int sz_u64_ctz(sz_u64_t x) { return (int)_tzcnt_u64(x); } -SZ_HELPER_AUTO int sz_u64_clz(sz_u64_t x) { return (int)_lzcnt_u64(x); } -SZ_HELPER_AUTO int sz_u64_popcount(sz_u64_t x) { return (int)__popcnt64(x); } -SZ_HELPER_AUTO int sz_u32_ctz(sz_u32_t x) { return (int)_tzcnt_u32(x); } -SZ_HELPER_AUTO int sz_u32_clz(sz_u32_t x) { return (int)_lzcnt_u32(x); } -SZ_HELPER_AUTO int sz_u32_popcount(sz_u32_t x) { return (int)__popcnt(x); } +SZ_HELPER_INLINE int sz_u64_ctz(sz_u64_t x) { return (int)_tzcnt_u64(x); } +SZ_HELPER_INLINE int sz_u64_clz(sz_u64_t x) { return (int)_lzcnt_u64(x); } +SZ_HELPER_INLINE int sz_u64_popcount(sz_u64_t x) { return (int)__popcnt64(x); } +SZ_HELPER_INLINE int sz_u32_ctz(sz_u32_t x) { return (int)_tzcnt_u32(x); } +SZ_HELPER_INLINE int sz_u32_clz(sz_u32_t x) { return (int)_lzcnt_u32(x); } +SZ_HELPER_INLINE int sz_u32_popcount(sz_u32_t x) { return (int)__popcnt(x); } #endif /* * Force the byteswap functions to be intrinsics, because when `/Oi-` is given, * these will turn into CRT function calls, which breaks when `SZ_AVOID_LIBC` is given. */ #pragma intrinsic(_byteswap_uint64) -SZ_HELPER_AUTO sz_u64_t sz_u64_bytes_reverse(sz_u64_t val) { return _byteswap_uint64(val); } +SZ_HELPER_INLINE sz_u64_t sz_u64_bytes_reverse(sz_u64_t val) { return _byteswap_uint64(val); } #pragma intrinsic(_byteswap_ulong) -SZ_HELPER_AUTO sz_u32_t sz_u32_bytes_reverse(sz_u32_t val) { return _byteswap_ulong(val); } +SZ_HELPER_INLINE sz_u32_t sz_u32_bytes_reverse(sz_u32_t val) { return _byteswap_ulong(val); } #else SZ_HELPER_AUTO int sz_u64_popcount(sz_u64_t x) { return __builtin_popcountll(x); } SZ_HELPER_AUTO int sz_u32_popcount(sz_u32_t x) { return __builtin_popcount(x); } @@ -1727,10 +1751,12 @@ SZ_HELPER_AUTO sz_i32_t sz_i32_max_of_two(sz_i32_t x, sz_i32_t y) { return x - ( #pragma GCC push_options #pragma GCC target("bmi", "bmi2") #endif -SZ_HELPER_AUTO __mmask8 sz_u8_mask_until_(sz_size_t n) { return (__mmask8)_bzhi_u32(0xFFu, (unsigned char)n); } -SZ_HELPER_AUTO __mmask16 sz_u16_mask_until_(sz_size_t n) { return (__mmask16)_bzhi_u32(0xFFFFu, (unsigned char)n); } -SZ_HELPER_AUTO __mmask32 sz_u32_mask_until_(sz_size_t n) { return (__mmask32)_bzhi_u64(0xFFFFFFFFu, (unsigned char)n); } -SZ_HELPER_AUTO __mmask64 sz_u64_mask_until_(sz_size_t n) { +SZ_HELPER_INLINE __mmask8 sz_u8_mask_until_(sz_size_t n) { return (__mmask8)_bzhi_u32(0xFFu, (unsigned char)n); } +SZ_HELPER_INLINE __mmask16 sz_u16_mask_until_(sz_size_t n) { return (__mmask16)_bzhi_u32(0xFFFFu, (unsigned char)n); } +SZ_HELPER_INLINE __mmask32 sz_u32_mask_until_(sz_size_t n) { + return (__mmask32)_bzhi_u64(0xFFFFFFFFu, (unsigned char)n); +} +SZ_HELPER_INLINE __mmask64 sz_u64_mask_until_(sz_size_t n) { return (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, (unsigned char)n); } SZ_HELPER_AUTO __mmask8 sz_u8_clamp_mask_until_(sz_size_t n) { return n < 8 ? sz_u8_mask_until_(n) : 0xFFu; } @@ -1859,7 +1885,7 @@ SZ_HELPER_AUTO sz_u64_t sz_u64_transpose(sz_u64_t x) { /** @brief Load a 16-bit unsigned integer from a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO sz_u16_vec_t sz_u16_load(sz_cptr_t ptr) { +SZ_HELPER_INLINE sz_u16_vec_t sz_u16_load(sz_cptr_t ptr) { #if !SZ_USE_MISALIGNED_LOADS sz_u16_vec_t result_vec; result_vec.u8s[0] = ptr[0]; @@ -1879,7 +1905,7 @@ SZ_HELPER_AUTO sz_u16_vec_t sz_u16_load(sz_cptr_t ptr) { /** @brief Load a 32-bit unsigned integer from a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO sz_u32_vec_t sz_u32_load(sz_cptr_t ptr) { +SZ_HELPER_INLINE sz_u32_vec_t sz_u32_load(sz_cptr_t ptr) { #if !SZ_USE_MISALIGNED_LOADS sz_u32_vec_t result_vec; result_vec.u8s[0] = ptr[0]; @@ -1901,7 +1927,7 @@ SZ_HELPER_AUTO sz_u32_vec_t sz_u32_load(sz_cptr_t ptr) { /** @brief Load a 64-bit unsigned integer from a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO sz_u64_vec_t sz_u64_load(sz_cptr_t ptr) { +SZ_HELPER_INLINE sz_u64_vec_t sz_u64_load(sz_cptr_t ptr) { #if !SZ_USE_MISALIGNED_LOADS sz_u64_vec_t result_vec; result_vec.u8s[0] = ptr[0]; @@ -1926,7 +1952,7 @@ SZ_HELPER_AUTO sz_u64_vec_t sz_u64_load(sz_cptr_t ptr) { } /** @brief Store a 16-bit unsigned integer to a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO void sz_u16_store(sz_ptr_t ptr, sz_u16_t value) { +SZ_HELPER_INLINE void sz_u16_store(sz_ptr_t ptr, sz_u16_t value) { #if !SZ_USE_MISALIGNED_LOADS sz_u16_vec_t vec; vec.u16 = value; @@ -1945,7 +1971,7 @@ SZ_HELPER_AUTO void sz_u16_store(sz_ptr_t ptr, sz_u16_t value) { } /** @brief Store a 32-bit unsigned integer to a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO void sz_u32_store(sz_ptr_t ptr, sz_u32_t value) { +SZ_HELPER_INLINE void sz_u32_store(sz_ptr_t ptr, sz_u32_t value) { #if !SZ_USE_MISALIGNED_LOADS sz_u32_vec_t vec; vec.u32 = value; @@ -1966,7 +1992,7 @@ SZ_HELPER_AUTO void sz_u32_store(sz_ptr_t ptr, sz_u32_t value) { } /** @brief Store a 64-bit unsigned integer to a potentially unaligned pointer. Can be expensive on some platforms. */ -SZ_HELPER_AUTO void sz_u64_store(sz_ptr_t ptr, sz_u64_t value) { +SZ_HELPER_INLINE void sz_u64_store(sz_ptr_t ptr, sz_u64_t value) { #if !SZ_USE_MISALIGNED_LOADS sz_u64_vec_t vec; vec.u64 = value; diff --git a/include/stringzilla/types.hpp b/include/stringzilla/types.hpp index f7378e8c..a9967c81 100644 --- a/include/stringzilla/types.hpp +++ b/include/stringzilla/types.hpp @@ -179,6 +179,12 @@ using rune_t = sz_rune_t; using size_t = sz_size_t; using ssize_t = sz_ssize_t; +/** + * @brief A size or offset deliberately held in 32 bits, where the narrower arithmetic is cheaper - GPU + * address math above all. Every use pairs with a range check at the site that establishes the bound. + */ +using small_size_t = sz_u32_t; + using f32_t = float; using f64_t = double; @@ -568,6 +574,31 @@ struct arrow_strings_view { return {&buffer_[offsets_[i]], static_cast<size_t>(offsets_[i + 1] - offsets_[i]) - terminator_width_k}; } + /** + * @brief The contiguous block the elements slice, from the first element's start to the last one's end. + * @note Starts at `offsets_[0]`, which a tape that is a slice of a wider one leaves non-zero. + */ + constexpr span<char_t const> tape_bytes() const noexcept { + return size() == 0 + ? span<char_t const> {} + : span<char_t const> {&buffer_[offsets_[0]], static_cast<size_t>(offsets_[size()] - offsets_[0])}; + } + + /** + * @brief Every element's length summed, terminators excluded, without walking the elements. + * @note Returns the tape's own offset width, so a 32-bit tape stays 32-bit until a caller needs more. + */ + constexpr offset_t tape_total_bytes() const noexcept { + return size() == 0 ? offset_t {} + : static_cast<offset_t>(offsets_[size()] - offsets_[0] - + static_cast<offset_t>(size()) * terminator_width_k); + } + + /** @brief One element's length, terminator excluded, in the tape's own offset width. */ + constexpr offset_t tape_length_at(size_t i) const noexcept { + return static_cast<offset_t>(offsets_[i + 1] - offsets_[i] - terminator_width_k); + } + constexpr iterator_t begin() const noexcept { return iterator_t(*this, 0); } constexpr iterator_t end() const noexcept { return iterator_t(*this, size()); } constexpr iterator_t cbegin() const noexcept { return begin(); } @@ -651,10 +682,9 @@ struct arrow_strings_tape { status_t try_assign(strings_iterator_type_ first, strings_iterator_type_ last) noexcept { // The range is walked twice - once to measure, once to copy - so single-pass "input" // iterators, like `std::istream_iterator`, would compile but silently copy nothing. - static_assert( - std::is_base_of<std::forward_iterator_tag, - typename std::iterator_traits<strings_iterator_type_>::iterator_category>::value, - "arrow_strings_tape::try_assign needs multi-pass (forward) iterators"); + static_assert(std::is_base_of<std::forward_iterator_tag, + typename std::iterator_traits<strings_iterator_type_>::iterator_category>::value, + "arrow_strings_tape::try_assign needs multi-pass (forward) iterators"); reset(); // ? Drops the old contents, so every failure below leaves an empty tape rather than a stale one @@ -898,6 +928,7 @@ struct cpu_specs_t { */ struct gpu_specs_t { size_t vram_bytes = 40ul * 1024 * 1024 * 1024; // ? On A100 it's 40 GB + size_t l2_bytes = 40ul * 1024 * 1024; // ? On A100 it's 40 MB, shared by every multiprocessor size_t constant_memory_bytes = 64 * 1024; // ? On A100 it's 64 KB size_t shared_memory_bytes = 192 * 1024 * 108; // ? On A100 it's 192 KB per SM size_t streaming_multiprocessors = 108; // ? On A100 diff --git a/include/stringzilla/utf8_graphemes.h b/include/stringzilla/utf8_graphemes.h index 389a18ba..e3336d2e 100644 --- a/include/stringzilla/utf8_graphemes.h +++ b/include/stringzilla/utf8_graphemes.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UAX-29 grapheme cluster segmentation. - * @file utf8_graphemes.h + * @file include/stringzilla/utf8_graphemes.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_GRAPHEMES_H_ diff --git a/include/stringzilla/utf8_graphemes/README.md b/include/stringzilla/utf8_graphemes/README.md index e7fd7adb..deb83592 100644 --- a/include/stringzilla/utf8_graphemes/README.md +++ b/include/stringzilla/utf8_graphemes/README.md @@ -5,7 +5,7 @@ Each operation has a serial baseline plus `haswell` and `icelake` SIMD backends ## Methodology -Numbers are throughput in MB/s, measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput in MB/s, measured with `bench/utf8_segment.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its single column is the `sz_utf8_graphemes` operation and its rows are a backend on a chip, so reading down the column compares the backend ladder on one fixed input shape. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes) and a Long Lines workload (full text lines) to expose how each kernel scales with token length. A `↑` cell means there is no dedicated kernel at that backend, so the dispatcher reuses the tier above it. diff --git a/include/stringzilla/utf8_graphemes/haswell.h b/include/stringzilla/utf8_graphemes/haswell.h index 32a30190..346e9d72 100644 --- a/include/stringzilla/utf8_graphemes/haswell.h +++ b/include/stringzilla/utf8_graphemes/haswell.h @@ -36,7 +36,7 @@ extern "C" { * `bmp_page_lut_` page LUT selects one of the 54 distinct 256-byte pages, then `flat_bmp_` is fetched by * `vpgatherdd`. The leaf carries the descriptor directly (the serial `id_to_desc` permute is folded in). * Bit-exact with `sz_rune_grapheme_break_property` over the whole BMP (Hangul included, no separate formula). */ -SZ_HELPER_AUTO __m256i sz_grapheme_bmp_descriptor_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { +SZ_HELPER_INLINE __m256i sz_grapheme_bmp_descriptor_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { return sz_utf8_rune_flat_lookup_haswell_(sz_utf8_grapheme_break_bmp_page_lut_, sz_utf8_grapheme_break_flat_bmp_, high_bytes_u8x32, low_bytes_u8x32); } @@ -44,8 +44,8 @@ SZ_HELPER_AUTO __m256i sz_grapheme_bmp_descriptor_haswell_(__m256i high_bytes_u8 /** @brief Packed descriptor byte for thirty-two ASTRAL codepoints over offset = cp - 0x10000 (5-nibble cascade), the * AVX2 twin of the icelake astral trie. Per-lane bytes: @p plane = (offset>>16)&0xFF (low nibble meaningful), * @p high = (offset>>8)&0xFF, @p low = offset&0xFF. Gather-free; bit-exact. */ -SZ_HELPER_AUTO __m256i sz_grapheme_astral_descriptor_haswell_(__m256i plane_u8x32, __m256i high_u8x32, - __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_grapheme_astral_descriptor_haswell_(__m256i plane_u8x32, __m256i high_u8x32, + __m256i low_u8x32) { __m256i const low_nibble_mask_u8x32 = _mm256_set1_epi8(0x0F); __m256i const n4_u8x32 = _mm256_and_si256(plane_u8x32, low_nibble_mask_u8x32); __m256i const n3_u8x32 = _mm256_and_si256(_mm256_srli_epi16(high_u8x32, 4), low_nibble_mask_u8x32); @@ -89,9 +89,9 @@ SZ_HELPER_INLINE __m256i sz_grapheme_cmpge_epu8_haswell_(__m256i value_u8x32, __ /** @brief 64-bit unsigned `low <= cp <= high` mask over reconstructed BMP codepoints carried in @p high_byte / * @p low_byte halves. cp = (high<<8)|low, so the inclusive 16-bit range test is: high in (lo_hi,hi_hi) * unconditionally, or on the boundary high bytes the low byte within bound. Two halves, branchless. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_cp_in_range_haswell_(__m256i high_lo_u8x32, __m256i low_lo_u8x32, - __m256i high_hi_u8x32, __m256i low_hi_u8x32, sz_u16_t lo, - sz_u16_t hi) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_cp_in_range_haswell_(__m256i high_lo_u8x32, __m256i low_lo_u8x32, + __m256i high_hi_u8x32, __m256i low_hi_u8x32, sz_u16_t lo, + sz_u16_t hi) { sz_u8_t const lo_h = (sz_u8_t)(lo >> 8), lo_l = (sz_u8_t)(lo & 0xFF); sz_u8_t const hi_h = (sz_u8_t)(hi >> 8), hi_l = (sz_u8_t)(hi & 0xFF); __m256i const lo_h_v_u8x32 = _mm256_set1_epi8((char)lo_h), lo_l_v_u8x32 = _mm256_set1_epi8((char)lo_l); @@ -124,8 +124,8 @@ SZ_HELPER_AUTO sz_u64_t sz_grapheme_cp_in_range_haswell_(__m256i high_lo_u8x32, /** @brief Lanes whose BMP codepoint resolves uniformly to GCB=Other via the CJK / Kana arithmetic ranges (the AVX2 * twin of `sz_grapheme_cjk_other_icelake_`): `[0x3000,0xA66E] | [0xD7FC,0xFB1D]` minus the interior Extend / * enclosed exceptions. Such lanes need no cold cascade (their descriptor is 0). */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_cjk_other_haswell_(__m256i high_lo_u8x32, __m256i low_lo_u8x32, - __m256i high_hi_u8x32, __m256i low_hi_u8x32) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_cjk_other_haswell_(__m256i high_lo_u8x32, __m256i low_lo_u8x32, + __m256i high_hi_u8x32, __m256i low_hi_u8x32) { sz_u64_t const run_a = sz_grapheme_cp_in_range_haswell_(high_lo_u8x32, low_lo_u8x32, high_hi_u8x32, low_hi_u8x32, 0x3000, 0xA66E); sz_u64_t const run_b = sz_grapheme_cp_in_range_haswell_(high_lo_u8x32, low_lo_u8x32, high_hi_u8x32, low_hi_u8x32, @@ -156,11 +156,11 @@ SZ_HELPER_INLINE void sz_grapheme_next3_haswell_(__m256i window_lo_u8x32, __m256 /** @brief Per-window per-lane descriptors as two `__m256i` halves, plus the codepoint-start lane mask and geometry. * The AVX2 twin of the icelake `sz_grapheme_classify_window_full_icelake_` outputs. */ typedef struct sz_grapheme_classified_haswell_t { - __m256i descriptors_lo; /**< Packed descriptor per byte-lane, lanes [0,32) (valid only on start lanes). */ - __m256i descriptors_hi; /**< Packed descriptor per byte-lane, lanes [32,64). */ - sz_u64_t start_lanes; /**< Codepoint-start lanes within the effective span (trimmed to `byte_span`). */ - sz_size_t codepoint_count; /**< Number of codepoint starts resolved (<= 64). */ - sz_size_t byte_span; /**< Bytes consumed by the resolved codepoints (offset of the next start). */ + __m256i descriptors_low_u8x32; /**< Packed descriptor per byte-lane, lanes [0,32) (valid only on start lanes). */ + __m256i descriptors_high_u8x32; /**< Packed descriptor per byte-lane, lanes [32,64). */ + sz_u64_t start_lanes; /**< Codepoint-start lanes within the effective span (trimmed to `byte_span`). */ + sz_size_t codepoint_count; /**< Number of codepoint starts resolved (<= 64). */ + sz_size_t byte_span; /**< Bytes consumed by the resolved codepoints (offset of the next start). */ } sz_grapheme_classified_haswell_t; /** @@ -168,7 +168,7 @@ typedef struct sz_grapheme_classified_haswell_t { * descriptor halves with the trimmed codepoint-start geometry. Mirrors the icelake decode/classify path * (value-based blind reconstruction so malformed input agrees byte-for-byte) without `vpermb`/`vpgather`. */ -SZ_HELPER_AUTO sz_grapheme_classified_haswell_t sz_grapheme_classify_window_haswell_( // +SZ_HELPER_INLINE sz_grapheme_classified_haswell_t sz_grapheme_classify_window_haswell_( // sz_u8_t const *text, sz_size_t length, sz_size_t base) { sz_utf8_rune_window_haswell_t const decoded = sz_utf8_rune_decode_window_haswell_(text + base, length - base); @@ -190,7 +190,7 @@ SZ_HELPER_AUTO sz_grapheme_classified_haswell_t sz_grapheme_classify_window_hasw start_lanes &= sz_u64_mask_until_serial_(byte_span); } - __m256i const raw_lo_u8x32 = decoded.window_lo, raw_hi_u8x32 = decoded.window_hi; + __m256i const raw_lo_u8x32 = decoded.window_low_u8x32, raw_hi_u8x32 = decoded.window_high_u8x32; __m256i next1_lo_u8x32, next1_hi_u8x32, next2_lo_u8x32, next2_hi_u8x32, next3_lo_u8x32, next3_hi_u8x32; sz_utf8_forward_neighbours_haswell_(raw_lo_u8x32, raw_hi_u8x32, &next1_lo_u8x32, &next1_hi_u8x32, &next2_lo_u8x32, &next2_hi_u8x32); @@ -376,8 +376,8 @@ SZ_HELPER_AUTO sz_grapheme_classified_haswell_t sz_grapheme_classify_window_hasw desc_hi_u8x32 = _mm256_and_si256(desc_hi_u8x32, valid_sel_hi_u8x32); sz_grapheme_classified_haswell_t result; - result.descriptors_lo = desc_lo_u8x32; - result.descriptors_hi = desc_hi_u8x32; + result.descriptors_low_u8x32 = desc_lo_u8x32; + result.descriptors_high_u8x32 = desc_hi_u8x32; result.start_lanes = start_lanes; result.codepoint_count = (sz_size_t)_mm_popcnt_u64(start_lanes); result.byte_span = byte_span; @@ -398,7 +398,7 @@ SZ_HELPER_AUTO sz_grapheme_classified_haswell_t sz_grapheme_classify_window_hasw SZ_HELPER_INLINE sz_grapheme_window_masks_t sz_grapheme_build_masks_haswell_( sz_grapheme_classified_haswell_t classified, sz_u64_t valid) { sz_u64_t const starts = classified.start_lanes; - __m256i const desc_lo_u8x32 = classified.descriptors_lo, desc_hi_u8x32 = classified.descriptors_hi; + __m256i const desc_lo_u8x32 = classified.descriptors_low_u8x32, desc_hi_u8x32 = classified.descriptors_high_u8x32; __m256i const class_lo_u8x32 = _mm256_and_si256(desc_lo_u8x32, _mm256_set1_epi8(0x0F)); __m256i const class_hi_u8x32 = _mm256_and_si256(desc_hi_u8x32, _mm256_set1_epi8(0x0F)); diff --git a/include/stringzilla/utf8_graphemes/icelake.h b/include/stringzilla/utf8_graphemes/icelake.h index e8cec113..bdfc014c 100644 --- a/include/stringzilla/utf8_graphemes/icelake.h +++ b/include/stringzilla/utf8_graphemes/icelake.h @@ -62,7 +62,7 @@ enum { }; /** @brief Descriptor of a codepoint < 0x80 by a `vpermb` over the two aligned `ascii_desc` `.rodata` tiles. */ -SZ_HELPER_AUTO __m512i sz_grapheme_ascii_descriptor_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_grapheme_ascii_descriptor_icelake_(__m512i codepoints_u32x16) { __m512i const low_six_u32x16 = _mm512_and_si512(codepoints_u32x16, _mm512_set1_epi32(0x3F)); __mmask16 const high_half_m16 = _mm512_test_epi32_mask(codepoints_u32x16, _mm512_set1_epi32(0x40)); __m512i const tile_low_u8x64 = _mm512_load_si512((void const *)(sz_utf8_grapheme_break_ascii_desc_ + 0)); @@ -83,7 +83,7 @@ SZ_HELPER_INLINE __m512i sz_grapheme_small_page_icelake_(__m512i codepoints_u32x /** @brief Descriptor of a BMP codepoint: the `bmp_page_lut_` page LUT (one `vpermb`) selects one of the 54 distinct * 256-byte pages, then `flat_bmp_` is fetched by one `vpgatherdd` for all sixteen lanes. The leaf carries * the packed descriptor directly, so no `id_to_desc` permute follows. */ -SZ_HELPER_AUTO __m512i sz_grapheme_classify_bmp_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_grapheme_classify_bmp_icelake_(__m512i codepoints_u32x16) { return _mm512_and_si512(sz_utf8_rune_flat_lookup_icelake_(sz_utf8_grapheme_break_bmp_page_lut_, sz_utf8_grapheme_break_flat_bmp_, codepoints_u32x16), _mm512_set1_epi32(0xFF)); @@ -92,7 +92,7 @@ SZ_HELPER_AUTO __m512i sz_grapheme_classify_bmp_icelake_(__m512i codepoints_u32x /** @brief Descriptor of an astral codepoint (>= 0x10000) via the 4-stage trie over offset = codepoint - 0x10000 * (an 8/4/4/4 split), every tile read straight from aligned `.rodata`. Byte-identical to the serial * sorted-range scan, replacing the per-window linear fold. */ -SZ_HELPER_AUTO __m512i sz_grapheme_classify_astral16_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_grapheme_classify_astral16_icelake_(__m512i codepoints_u32x16) { __m512i const offset_u32x16 = _mm512_sub_epi32(codepoints_u32x16, _mm512_set1_epi32(0x10000)); __m512i const stage1_u32x16 = sz_utf8_rune_permute256_icelake_( sz_utf8_grapheme_break_astral_s0_, @@ -119,7 +119,7 @@ SZ_HELPER_AUTO __m512i sz_grapheme_classify_astral16_icelake_(__m512i codepoints * (Extend voicing marks `302A-3030` / `3099-309A`, `303D`, enclosed `3297` / `3299`). Mirrors the word * kernel's `cjk_combined` carve. Six `vpcmp`; called only when a cold lane is present. */ -SZ_HELPER_AUTO __mmask16 sz_grapheme_cjk_other_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __mmask16 sz_grapheme_cjk_other_icelake_(__m512i codepoints_u32x16) { __mmask16 const run_a_m16 = _kand_mask16(_mm512_cmpge_epu32_mask(codepoints_u32x16, _mm512_set1_epi32(0x3000)), _mm512_cmple_epu32_mask(codepoints_u32x16, _mm512_set1_epi32(0xA66E))); __mmask16 const run_b_m16 = _kand_mask16(_mm512_cmpge_epu32_mask(codepoints_u32x16, _mm512_set1_epi32(0xD7FC)), @@ -145,7 +145,7 @@ SZ_HELPER_AUTO __mmask16 sz_grapheme_cjk_other_icelake_(__m512i codepoints_u32x1 * table (one `vpgatherdd`); and codepoint >= 0x10000 by the 4-stage astral trie. No linear range scan and no * scalar loop. */ -SZ_HELPER_AUTO __m512i sz_grapheme_classify16_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_grapheme_classify16_icelake_(__m512i codepoints_u32x16) { __m512i const hangul_base_u32x16 = _mm512_set1_epi32(0xAC00); __mmask16 const is_hangul_m16 = _kand_mask16(_mm512_cmpge_epu32_mask(codepoints_u32x16, hangul_base_u32x16), _mm512_cmple_epu32_mask(codepoints_u32x16, _mm512_set1_epi32(0xD7A3))); @@ -203,7 +203,7 @@ SZ_HELPER_AUTO __m512i sz_grapheme_classify16_icelake_(__m512i codepoints_u32x16 * (4-byte) lanes in the quarter take the reconstructed plane/mid/low codepoint; all others take the BMP * `(high << 8) | low`. Returns 16 descriptors in the low byte of each 32-bit lane. */ -SZ_HELPER_AUTO __m512i sz_grapheme_classify_quarter_icelake_( // +SZ_HELPER_INLINE __m512i sz_grapheme_classify_quarter_icelake_( // __m128i high_slice_u8x16, __m128i low_slice_u8x16, __m128i plane_slice_u8x16, __m128i mid_slice_u8x16, __m128i lo_slice_u8x16, __mmask16 astral_quarter_m16) { __m512i const codepoint_bmp_u32x16 = _mm512_or_si512(_mm512_slli_epi32(_mm512_cvtepu8_epi32(high_slice_u8x16), 8), @@ -225,13 +225,13 @@ SZ_HELPER_AUTO __m512i sz_grapheme_classify_quarter_icelake_( // * reassemble plane/mid/low from the four UTF-8 bytes) and classified; the four descriptor quarters are written back * as one byte per lane. No scalar per-lane loop and no spill round-trip. */ -SZ_HELPER_AUTO __m512i sz_grapheme_classify_window_icelake_( // +SZ_HELPER_INLINE __m512i sz_grapheme_classify_window_icelake_( // sz_utf8_rune_window_t const *decoded, __m512i next1_u8x64, __m512i next2_u8x64, __m512i next3_u8x64) { // Astral (4-byte) lead reconstruction: plane = ((b0 & 7) << 2) | ((b1 >> 4) & 3); mid = ((b1 & F) << 4) | // ((b2 >> 2) & F); low = ((b2 & 3) << 6) | (b3 & 3F); codepoint = (plane << 16) | (mid << 8) | low. - __m512i const window_u8x64 = decoded->window; + __m512i const window_u8x64 = decoded->window_u8x64; // The BMP `high`/`low` are reconstructed HERE from the raw lead and the (already edge-zeroed) neighbour bytes - // `next1`/`next2`, not read from `decoded->high`/`decoded->low`. The substrate's neighbour fetch is an in-register + // `next1`/`next2`, not read from `decoded->high_byte_u8x64`/`decoded->low_byte_u8x64`. The substrate's neighbour fetch is an in-register // rotate that wraps the window head into a truncated trailing lead's missing continuation byte; recomputing from // the zeroed neighbours pads out-of-window bytes with 0, matching the serial blind decode byte-for-byte so a // 2-/3-byte lead at the loaded edge classifies neighbour-independently. ASCII (1-byte) lanes take the identity @@ -340,8 +340,8 @@ SZ_HELPER_INLINE sz_grapheme_window_masks_t sz_grapheme_build_masks_icelake_(__m * i. Builds the per-class masks in-register (the only `__m512i`->`sz_u64_t` contact), then delegates every * GB1-GB13 decision to the shared portable @ref sz_grapheme_window_boundaries_ engine. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_window_boundaries_icelake_(__m512i descriptors_u8x64, int codepoint_count, - sz_grapheme_carry_t *carry) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_window_boundaries_icelake_(__m512i descriptors_u8x64, int codepoint_count, + sz_grapheme_carry_t *carry) { sz_u64_t const valid = (codepoint_count >= 64) ? ~0ull : ((1ull << codepoint_count) - 1); sz_grapheme_window_masks_t const window = sz_grapheme_build_masks_icelake_(descriptors_u8x64, valid); return sz_grapheme_window_boundaries_(&window, codepoint_count, valid, carry); @@ -370,7 +370,7 @@ typedef struct sz_grapheme_window_t { * trailing partial codepoint (whose continuation bytes fall outside the window) is left for the next * window so cross-window runs stay exact. Pure register dataflow: one decode, one classify, one compress. */ -SZ_HELPER_AUTO sz_grapheme_window_t sz_grapheme_classify_window_full_icelake_( // +SZ_HELPER_INLINE sz_grapheme_window_t sz_grapheme_classify_window_full_icelake_( // sz_u8_t const *text, sz_size_t length, sz_size_t base, __m512i lane_identity_u8x64, sz_grapheme_carry_t *carry) { sz_utf8_rune_window_t const decoded = sz_utf8_rune_decode_window_icelake_(text + base, length - base, @@ -395,7 +395,7 @@ SZ_HELPER_AUTO sz_grapheme_window_t sz_grapheme_classify_window_full_icelake_( / start_lanes &= sz_u64_mask_until_(byte_span); } - // The neighbour fetches `next{1,2,3}` are an in-register byte rotate of `decoded.window`, so a lane near the loaded + // The neighbour fetches `next{1,2,3}` are an in-register byte rotate of `decoded.window_u8x64`, so a lane near the loaded // end reads a WRAPPED lane-0 byte for any continuation that falls at or beyond `loaded`. On a short final window // (`loaded < 64`) those continuation bytes are genuinely absent (the input ended mid-sequence), so a truncated // trailing lead would otherwise decode against the wrapped window head and classify neighbour-dependently. Zero @@ -408,11 +408,11 @@ SZ_HELPER_AUTO sz_grapheme_window_t sz_grapheme_classify_window_full_icelake_( / __mmask64 const next2_present_m64 = _cvtu64_mask64(in_window >> 2); __mmask64 const next3_present_m64 = _cvtu64_mask64(in_window >> 3); __m512i const next1_u8x64 = _mm512_maskz_permutexvar_epi8( - next1_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), decoded.window); + next1_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), decoded.window_u8x64); __m512i const next2_u8x64 = _mm512_maskz_permutexvar_epi8( - next2_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(2)), decoded.window); + next2_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(2)), decoded.window_u8x64); __m512i const next3_u8x64 = _mm512_maskz_permutexvar_epi8( - next3_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(3)), decoded.window); + next3_present_m64, _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(3)), decoded.window_u8x64); __m512i const descriptors_per_lane_u8x64 = sz_grapheme_classify_window_icelake_(&decoded, next1_u8x64, next2_u8x64, next3_u8x64); diff --git a/include/stringzilla/utf8_graphemes/neon.h b/include/stringzilla/utf8_graphemes/neon.h index 41f3e571..f46c911a 100644 --- a/include/stringzilla/utf8_graphemes/neon.h +++ b/include/stringzilla/utf8_graphemes/neon.h @@ -36,13 +36,13 @@ extern "C" { /** @brief Drop-in branchless `_pext_u64` (builds a one-shot route). Prefer `sz_grapheme_bit_gather_` when one * selector serves several values, as inside `sz_grapheme_build_masks_neon_`. Bit-exact with BMI2. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_pext_neon_(sz_u64_t value, sz_u64_t selector) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_pext_neon_(sz_u64_t value, sz_u64_t selector) { sz_grapheme_bit_route_t const route = sz_grapheme_bit_route_build_(selector); return sz_grapheme_bit_gather_(value, &route); } /** @brief Drop-in branchless `_pdep_u64`. Bit-exact with BMI2. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_pdep_neon_(sz_u64_t value, sz_u64_t selector) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_pdep_neon_(sz_u64_t value, sz_u64_t selector) { sz_grapheme_bit_route_t const route = sz_grapheme_bit_route_build_(selector); return sz_grapheme_bit_scatter_(value, &route); } @@ -55,7 +55,7 @@ SZ_HELPER_AUTO sz_u64_t sz_grapheme_pdep_neon_(sz_u64_t value, sz_u64_t selector * page LUT selects one of the 54 distinct 256-byte pages, then `flat_bmp_` is read per lane. The leaf * carries the descriptor directly (the serial `id_to_desc` permute is folded in). Bit-exact with * `sz_rune_grapheme_break_property` over the whole BMP (Hangul included, no separate formula). */ -SZ_HELPER_AUTO uint8x16_t sz_grapheme_bmp_descriptor_neon_(uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_grapheme_bmp_descriptor_neon_(uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_neon_(sz_utf8_grapheme_break_bmp_page_lut_, sz_utf8_grapheme_break_flat_bmp_, (int)sz_utf8_grapheme_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -64,8 +64,8 @@ SZ_HELPER_AUTO uint8x16_t sz_grapheme_bmp_descriptor_neon_(uint8x16_t high_bytes * NEON twin of the AVX2 astral trie. Per-lane bytes: @p plane_u8x16 = (offset>>16)&0xFF (low nibble * meaningful), @p high_u8x16 = (offset>>8)&0xFF, @p low_u8x16 = offset&0xFF. Gather-free; bit-exact. * Addresses ONE quarter. */ -SZ_HELPER_AUTO uint8x16_t sz_grapheme_astral_descriptor_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, - uint8x16_t low_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_grapheme_astral_descriptor_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, + uint8x16_t low_u8x16) { uint8x16_t const low_nibble_mask_u8x16 = vdupq_n_u8(0x0F); uint8x16_t const n4_u8x16 = vandq_u8(plane_u8x16, low_nibble_mask_u8x16); uint8x16_t const n3_u8x16 = vandq_u8(vshrq_n_u8(high_u8x16, 4), low_nibble_mask_u8x16); @@ -114,8 +114,8 @@ SZ_HELPER_INLINE uint8x16_t sz_grapheme_cmpge_epu8_neon_(uint8x16_t value_u8x16, /** @brief 64-bit unsigned `low <= cp <= high` mask over reconstructed BMP codepoints carried in @p high_byte / * @p low_byte quarters. cp = (high<<8)|low, so the inclusive 16-bit range test is: high in (lo_hi,hi_hi) * unconditionally, or on the boundary high bytes the low byte within bound. Four quarters, branchless. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_cp_in_range_neon_(uint8x16_t const *high_u8x16, uint8x16_t const *low_u8x16, - sz_u16_t lo, sz_u16_t hi) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_cp_in_range_neon_(uint8x16_t const *high_u8x16, uint8x16_t const *low_u8x16, + sz_u16_t lo, sz_u16_t hi) { sz_u8_t const lo_h = (sz_u8_t)(lo >> 8), lo_l = (sz_u8_t)(lo & 0xFF); sz_u8_t const hi_h = (sz_u8_t)(hi >> 8), hi_l = (sz_u8_t)(hi & 0xFF); uint8x16_t const lo_h_u8x16 = vdupq_n_u8(lo_h), lo_l_u8x16 = vdupq_n_u8(lo_l); @@ -139,7 +139,7 @@ SZ_HELPER_AUTO sz_u64_t sz_grapheme_cp_in_range_neon_(uint8x16_t const *high_u8x /** @brief Lanes whose BMP codepoint resolves uniformly to GCB=Other via the CJK / Kana arithmetic ranges (the NEON * twin of `sz_grapheme_cjk_other_haswell_`): `[0x3000,0xA66E] | [0xD7FC,0xFB1D]` minus the interior Extend / * enclosed exceptions. Such lanes need no cold cascade (their descriptor is 0). */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_cjk_other_neon_(uint8x16_t const *high_u8x16, uint8x16_t const *low_u8x16) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_cjk_other_neon_(uint8x16_t const *high_u8x16, uint8x16_t const *low_u8x16) { sz_u64_t const run_a = sz_grapheme_cp_in_range_neon_(high_u8x16, low_u8x16, 0x3000, 0xA66E); sz_u64_t const run_b = sz_grapheme_cp_in_range_neon_(high_u8x16, low_u8x16, 0xD7FC, 0xFB1D); sz_u64_t const exc_a = sz_grapheme_cp_in_range_neon_(high_u8x16, low_u8x16, 0x302A, 0x3030); @@ -165,10 +165,10 @@ SZ_HELPER_INLINE uint8x16_t sz_grapheme_byte_mask_from_bits_neon_(sz_u64_t bits, /** @brief Per-window per-lane descriptors as four `uint8x16_t` quarters, plus the codepoint-start lane mask and * geometry. The NEON twin of the AVX2 `sz_grapheme_classified_haswell_t` outputs. */ typedef struct sz_grapheme_classified_neon_t { - uint8x16_t descriptors[4]; /**< Packed descriptor per byte-lane (valid only on start lanes). */ - sz_u64_t start_lanes; /**< Codepoint-start lanes within the effective span (trimmed to `byte_span`). */ - sz_size_t codepoint_count; /**< Number of codepoint starts resolved (<= 64). */ - sz_size_t byte_span; /**< Bytes consumed by the resolved codepoints (offset of the next start). */ + uint8x16_t descriptors_u8x16s[4]; /**< Packed descriptor per byte-lane (valid only on start lanes). */ + sz_u64_t start_lanes; /**< Codepoint-start lanes within the effective span (trimmed to `byte_span`). */ + sz_size_t codepoint_count; /**< Number of codepoint starts resolved (<= 64). */ + sz_size_t byte_span; /**< Bytes consumed by the resolved codepoints (offset of the next start). */ } sz_grapheme_classified_neon_t; /** @@ -176,7 +176,7 @@ typedef struct sz_grapheme_classified_neon_t { * descriptor quarters with the trimmed codepoint-start geometry. Mirrors the haswell decode/classify path * (value-based blind reconstruction so malformed input agrees byte-for-byte) without table gather. */ -SZ_HELPER_AUTO sz_grapheme_classified_neon_t sz_grapheme_classify_window_neon_( // +SZ_HELPER_INLINE sz_grapheme_classified_neon_t sz_grapheme_classify_window_neon_( // sz_u8_t const *text, sz_size_t length, sz_size_t base) { sz_utf8_rune_window_neon_t const decoded = sz_utf8_rune_decode_window_neon_(text + base, length - base); @@ -200,7 +200,7 @@ SZ_HELPER_AUTO sz_grapheme_classified_neon_t sz_grapheme_classify_window_neon_( int const quarters_used = (int)((loaded + 15) >> 4); // classify work proportional to the loaded span uint8x16_t raw_u8x16[4]; - for (int quarter = 0; quarter < 4; ++quarter) raw_u8x16[quarter] = decoded.window[quarter]; + for (int quarter = 0; quarter < 4; ++quarter) raw_u8x16[quarter] = decoded.window_u8x16s[quarter]; uint8x16_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; sz_utf8_forward_neighbours_neon_(raw_u8x16, next1_u8x16, next2_u8x16, next3_u8x16); @@ -351,7 +351,7 @@ SZ_HELPER_AUTO sz_grapheme_classified_neon_t sz_grapheme_classify_window_neon_( sz_grapheme_byte_mask_from_bits_neon_(invalid_lead, quarter * 16)); sz_grapheme_classified_neon_t result; - for (int quarter = 0; quarter < 4; ++quarter) result.descriptors[quarter] = desc_u8x16[quarter]; + for (int quarter = 0; quarter < 4; ++quarter) result.descriptors_u8x16s[quarter] = desc_u8x16[quarter]; result.start_lanes = start_lanes; result.codepoint_count = (sz_size_t)sz_u64_popcount_neon_(start_lanes); result.byte_span = byte_span; @@ -378,7 +378,7 @@ SZ_HELPER_INLINE sz_grapheme_window_masks_t sz_grapheme_build_masks_neon_(sz_gra uint8x16_t const nibble_mask_u8x16 = vdupq_n_u8(0x0F); uint8x16_t class_q_u8x16[4], desc_q_u8x16[4]; for (int quarter = 0; quarter < 4; ++quarter) { - desc_q_u8x16[quarter] = classified.descriptors[quarter]; + desc_q_u8x16[quarter] = classified.descriptors_u8x16s[quarter]; class_q_u8x16[quarter] = vandq_u8(desc_q_u8x16[quarter], nibble_mask_u8x16); } diff --git a/include/stringzilla/utf8_graphemes/serial.h b/include/stringzilla/utf8_graphemes/serial.h index b5104559..791e0487 100644 --- a/include/stringzilla/utf8_graphemes/serial.h +++ b/include/stringzilla/utf8_graphemes/serial.h @@ -58,7 +58,7 @@ SZ_HELPER_AUTO sz_size_t sz_grapheme_break_next_start_(sz_cptr_t text, sz_size_t * ill-formed input (UAX-29 leaves such bytes undefined). Valid UTF-8 decodes identically to the checked * path; only malformed input differs, by design. */ -SZ_HELPER_AUTO sz_u8_t sz_grapheme_break_property_at_(sz_cptr_t text, sz_size_t length, sz_size_t start) { +SZ_HELPER_INLINE sz_u8_t sz_grapheme_break_property_at_(sz_cptr_t text, sz_size_t length, sz_size_t start) { sz_u8_t const lead = (sz_u8_t)text[start]; sz_u8_t const byte1 = (start + 1 < length) ? (sz_u8_t)text[start + 1] : 0; sz_u8_t const byte2 = (start + 2 < length) ? (sz_u8_t)text[start + 2] : 0; @@ -103,6 +103,9 @@ SZ_HELPER_INLINE sz_bool_t sz_grapheme_break_descriptor_extpict_(sz_u8_t descrip /** * @brief Check if `position` is a grapheme cluster boundary per Unicode TR29 (GB1-GB999, incl. GB9c and GB11). + * + * Nothing in the library calls this: the segmenters run a streaming state machine instead. It is the + * independent second opinion the tests measure that machine against. */ SZ_API_COMPTIME sz_bool_t sz_utf8_is_grapheme_boundary_serial(sz_cptr_t text, sz_size_t length, sz_size_t position) { if (position == 0) return sz_true_k; // GB1 @@ -220,7 +223,7 @@ typedef struct sz_grapheme_serial_state_t { } sz_grapheme_serial_state_t; /** @brief Boundary decision between @p state's previous codepoint and the @p after codepoint, GB3..GB13 in O(1). */ -SZ_HELPER_AUTO sz_bool_t sz_grapheme_serial_boundary_(sz_grapheme_serial_state_t const *state, sz_u8_t after) { +SZ_HELPER_INLINE sz_bool_t sz_grapheme_serial_boundary_(sz_grapheme_serial_state_t const *state, sz_u8_t after) { sz_u8_t const before_class = sz_grapheme_break_descriptor_gcb_(state->previous_descriptor); sz_u8_t const after_class = sz_grapheme_break_descriptor_gcb_(after); if (before_class == sz_grapheme_break_cr_k && after_class == sz_grapheme_break_lf_k) return sz_false_k; // GB3 @@ -257,7 +260,7 @@ SZ_HELPER_AUTO sz_bool_t sz_grapheme_serial_boundary_(sz_grapheme_serial_state_t } /** @brief Advance @p state by the @p after codepoint: toggle/close the RI, ExtPict-ZWJ and InCB runs. */ -SZ_HELPER_AUTO void sz_grapheme_serial_advance_(sz_grapheme_serial_state_t *state, sz_u8_t after) { +SZ_HELPER_INLINE void sz_grapheme_serial_advance_(sz_grapheme_serial_state_t *state, sz_u8_t after) { sz_u8_t const after_class = sz_grapheme_break_descriptor_gcb_(after); state->regional_indicator_run_odd = (after_class == sz_grapheme_break_regional_indicator_k) ? (sz_bool_t)(!state->regional_indicator_run_odd) @@ -545,8 +548,8 @@ SZ_HELPER_INLINE sz_u64_t sz_grapheme_previous_(sz_u64_t current, sz_grapheme_ca * the carry forward. The ISA-independent twin of the icelake fused window-boundaries; a per-ISA extractor * builds @p window from descriptors, then calls this. Pure `sz_u64_t` algebra, no intrinsics. */ -SZ_HELPER_AUTO sz_u64_t sz_grapheme_window_boundaries_(sz_grapheme_window_masks_t const *window, int codepoint_count, - sz_u64_t valid, sz_grapheme_carry_t *carry) { +SZ_HELPER_INLINE sz_u64_t sz_grapheme_window_boundaries_(sz_grapheme_window_masks_t const *window, int codepoint_count, + sz_u64_t valid, sz_grapheme_carry_t *carry) { sz_grapheme_carry_t const previous = *carry; sz_u64_t const carriage_return = window->class_bit[sz_grapheme_break_cr_k]; diff --git a/include/stringzilla/utf8_graphemes/sve2.h b/include/stringzilla/utf8_graphemes/sve2.h index e3440a99..f2fa52e2 100644 --- a/include/stringzilla/utf8_graphemes/sve2.h +++ b/include/stringzilla/utf8_graphemes/sve2.h @@ -28,8 +28,8 @@ extern "C" { /** @brief Packed descriptor byte for one chunk of ASTRAL codepoints over offset = cp - 0x10000 (5-nibble cascade), * the SVE2 twin of @ref sz_grapheme_astral_descriptor_neon_. Per-lane bytes: @p plane = (offset>>16)&0xFF * (low nibble meaningful), @p high = (offset>>8)&0xFF, @p low = offset&0xFF. Bit-exact. */ -SZ_HELPER_AUTO svuint8_t sz_grapheme_astral_descriptor_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, - svuint8_t low_u8x) { +SZ_HELPER_INLINE svuint8_t sz_grapheme_astral_descriptor_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, + svuint8_t low_u8x) { svbool_t const all_b8x = svptrue_b8(); svuint8_t const n4_u8x = svand_n_u8_x(all_b8x, plane_u8x, 0x0F); svuint8_t const n3_u8x = svand_n_u8_x(all_b8x, svlsr_n_u8_x(all_b8x, high_u8x, 4), 0x0F); @@ -82,7 +82,7 @@ SZ_HELPER_INLINE svbool_t sz_grapheme_cp_in_range_sve2_(svuint8_t high_u8x, svui /** @brief Lanes whose BMP codepoint resolves uniformly to GCB=Other via the CJK / Kana arithmetic ranges - the * SVE2 twin of @ref sz_grapheme_cjk_other_neon_. Such lanes need no cold cascade (descriptor 0). */ -SZ_HELPER_AUTO svbool_t sz_grapheme_cjk_other_sve2_(svuint8_t high_u8x, svuint8_t low_u8x) { +SZ_HELPER_INLINE svbool_t sz_grapheme_cjk_other_sve2_(svuint8_t high_u8x, svuint8_t low_u8x) { svbool_t const all_b8x = svptrue_b8(); svbool_t const run_a_b8x = sz_grapheme_cp_in_range_sve2_(high_u8x, low_u8x, 0x3000, 0xA66E); svbool_t const run_b_b8x = sz_grapheme_cp_in_range_sve2_(high_u8x, low_u8x, 0xD7FC, 0xFB1D); diff --git a/include/stringzilla/utf8_linebreaks.h b/include/stringzilla/utf8_linebreaks.h index 42114ef2..fae855a6 100644 --- a/include/stringzilla/utf8_linebreaks.h +++ b/include/stringzilla/utf8_linebreaks.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UAX-14 line break segmentation. - * @file utf8_linebreaks.h + * @file include/stringzilla/utf8_linebreaks.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_LINEBREAKS_H_ diff --git a/include/stringzilla/utf8_linebreaks/README.md b/include/stringzilla/utf8_linebreaks/README.md index 2f77fdcf..454480a0 100644 --- a/include/stringzilla/utf8_linebreaks/README.md +++ b/include/stringzilla/utf8_linebreaks/README.md @@ -6,7 +6,7 @@ The dispatcher picks the fastest one available on the running CPU. ## Methodology -Numbers are throughput in MB/s, measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput in MB/s, measured with `bench/utf8_segment.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its single column is the `sz_utf8_linebreaks` operation and its rows are a backend on a chip, so reading down the column compares the backend ladder on one fixed input shape. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes) and a Long Lines workload (full text lines) to expose how each kernel scales with token length. diff --git a/include/stringzilla/utf8_linebreaks/haswell.h b/include/stringzilla/utf8_linebreaks/haswell.h index 94e3f220..fa5ae711 100644 --- a/include/stringzilla/utf8_linebreaks/haswell.h +++ b/include/stringzilla/utf8_linebreaks/haswell.h @@ -46,7 +46,7 @@ extern "C" { * `flat_bmp_[page * 256 + low]` is fetched by four `vpgatherdd`. The leaf byte indexes * `sz_utf8_line_break_flat_palette_`, NOT the 62-entry cascade palette. * Bit-exact with `sz_rune_line_break_property` over the whole BMP. */ -SZ_HELPER_AUTO __m256i sz_line_break_bmp_index_haswell_(__m256i high_u8x32, __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_line_break_bmp_index_haswell_(__m256i high_u8x32, __m256i low_u8x32) { return sz_utf8_rune_flat_lookup_haswell_(sz_utf8_line_break_bmp_page_lut_, sz_utf8_line_break_flat_bmp_, high_u8x32, low_u8x32); } @@ -62,9 +62,9 @@ SZ_HELPER_INLINE __m256i sz_line_break_bit_mask_haswell_(__m256i bytes_u8x32, sz * icelake's `vpermi2w` over the two palette tiles, AVX2 having no cross-lane word permute -- and the shared * `pack4_u32_to_u8_haswell_` narrows each descriptor byte back into lane order. Every index originates in the * flat leaf, so it is always < 56 and the scale-2 gather stays inside the 64-word padded palette. */ -SZ_HELPER_AUTO void sz_line_break_flat_palette_descriptors_haswell_(__m256i palette_indices_u8x32, - __m256i *descriptor_low_bytes_u8x32, - __m256i *descriptor_high_bytes_u8x32) { +SZ_HELPER_INLINE void sz_line_break_flat_palette_descriptors_haswell_(__m256i palette_indices_u8x32, + __m256i *descriptor_low_bytes_u8x32, + __m256i *descriptor_high_bytes_u8x32) { __m128i const indices_low_u8x16 = _mm256_castsi256_si128(palette_indices_u8x32); __m128i const indices_high_u8x16 = _mm256_extracti128_si256(palette_indices_u8x32, 1); __m128i const index_quarters_u8x16[4] = {indices_low_u8x16, _mm_srli_si128(indices_low_u8x16, 8), @@ -91,9 +91,9 @@ SZ_HELPER_AUTO void sz_line_break_flat_palette_descriptors_haswell_(__m256i pale * unpack stays in the BYTE domain over the descriptor's low and high byte, with no 16-bit lane widening. * Applies the serial resolution aliasing (SA → AL/CM, AI/SG/XX → AL, CJ → NS); RI/ZWJ side bits come from the * RAW class, the mark side bit from the resolved class. */ -SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_haswell_(__m256i palette_indices_u8x32, - __m256i *classes_u8x32_out, __m256i *side_u8x32_out, - __m256i *dotted_select_u8x32_out) { +SZ_HELPER_INLINE void sz_line_break_flat_palette_unpack_haswell_(__m256i palette_indices_u8x32, + __m256i *classes_out_u8x32, __m256i *side_out_u8x32, + __m256i *dotted_select_out_u8x32) { __m256i descriptor_low_bytes_u8x32, descriptor_high_bytes_u8x32; sz_line_break_flat_palette_descriptors_haswell_(palette_indices_u8x32, &descriptor_low_bytes_u8x32, &descriptor_high_bytes_u8x32); @@ -138,17 +138,17 @@ SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_haswell_(__m256i palette_i side_u8x32 = _mm256_or_si256( side_u8x32, _mm256_and_si256(class_is_mark_u8x32, _mm256_set1_epi8((char)sz_line_break_side_mark_k))); - *classes_u8x32_out = classes_u8x32; - *side_u8x32_out = side_u8x32; - *dotted_select_u8x32_out = sz_line_break_bit_mask_haswell_(descriptor_high_bytes_u8x32, 1 << 5); + *classes_out_u8x32 = classes_u8x32; + *side_out_u8x32 = side_u8x32; + *dotted_select_out_u8x32 = sz_line_break_bit_mask_haswell_(descriptor_high_bytes_u8x32, 1 << 5); } /** @brief Palette index for thirty-two ASTRAL codepoints over the 20-bit offset = cp - 0x10000 (5-nibble cascade), * the AVX2 twin of `sz_line_break_classify_astral16_icelake_`. Per-lane bytes: @p plane_u8x32 = * (offset>>16)&0xFF (low nibble meaningful), @p high_u8x32 = (offset>>8)&0xFF, @p low_u8x32 = offset&0xFF. * Bit-exact. */ -SZ_HELPER_AUTO __m256i sz_line_break_classify_astral_haswell_(__m256i plane_u8x32, __m256i high_u8x32, - __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_line_break_classify_astral_haswell_(__m256i plane_u8x32, __m256i high_u8x32, + __m256i low_u8x32) { __m256i const low_nibble_mask_u8x32 = _mm256_set1_epi8(0x0F); __m256i const n4_u8x32 = _mm256_and_si256(plane_u8x32, low_nibble_mask_u8x32); __m256i const n3_u8x32 = _mm256_and_si256(_mm256_srli_epi16(high_u8x32, 4), low_nibble_mask_u8x32); @@ -244,15 +244,15 @@ SZ_HELPER_INLINE sz_u8_t sz_line_break_byte_at_haswell_(__m256i lanes_lo_u8x32, /** @brief Per-window byte-lane classification (AVX2): class/side per lane as two `__m256i` halves plus the * effective-start and U+FFFD masks. The Haswell twin of @ref sz_line_break_classified_t. */ typedef struct sz_line_break_classified_haswell_t { - __m256i classes_lo; /**< Per-byte-lane Line_Break class, lanes [0,32) (valid only on `starts` lanes). */ - __m256i classes_hi; /**< Per-byte-lane Line_Break class, lanes [32,64). */ - __m256i side_lo; /**< Per-byte-lane engine side byte, lanes [0,32). */ - __m256i side_hi; /**< Per-byte-lane engine side byte, lanes [32,64). */ - sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ - sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ - sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ - sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ - sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ + __m256i classes_low_u8x32; /**< Per-byte-lane Line_Break class, lanes [0,32) (valid only on `starts` lanes). */ + __m256i classes_high_u8x32; /**< Per-byte-lane Line_Break class, lanes [32,64). */ + __m256i side_low_u8x32; /**< Per-byte-lane engine side byte, lanes [0,32). */ + __m256i side_high_u8x32; /**< Per-byte-lane engine side byte, lanes [32,64). */ + sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ + sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ + sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ + sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ + sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ } sz_line_break_classified_haswell_t; /** @brief Compute the third forward neighbour `next3[i] = window[i+3]` over all 64 lanes with mod-64 wrap, the @@ -282,14 +282,14 @@ SZ_HELPER_INLINE void sz_line_break_palette_unpack_haswell_(__m256i index_u8x32, * "consume-1 U+FFFD" malformed policy: an invalid lead / short or stray continuation / overlong / * surrogate / out-of-range lead each become one single-byte U+FFFD unit (class AL). */ -SZ_HELPER_AUTO sz_line_break_classified_haswell_t sz_line_break_classify_window_haswell_( +SZ_HELPER_INLINE sz_line_break_classified_haswell_t sz_line_break_classify_window_haswell_( sz_utf8_rune_window_haswell_t window) { sz_u64_t const loaded_mask = sz_u64_mask_until_serial_(window.loaded); sz_u64_t const continuation = window.continuation & loaded_mask; sz_u64_t const two_byte = window.two_byte_starts; sz_u64_t const three_byte = window.three_byte_starts; sz_u64_t const four_byte = window.four_byte_starts; - __m256i const raw_lo_u8x32 = window.window_lo, raw_hi_u8x32 = window.window_hi; + __m256i const raw_lo_u8x32 = window.window_low_u8x32, raw_hi_u8x32 = window.window_high_u8x32; // Forward neighbours (mod-64 wrap, matching icelake's `_mm512_permutexvar_epi8`). __m256i next1_lo_u8x32, next1_hi_u8x32, next2_lo_u8x32, next2_hi_u8x32, next3_lo_u8x32, next3_hi_u8x32; @@ -365,13 +365,13 @@ SZ_HELPER_AUTO sz_line_break_classified_haswell_t sz_line_break_classify_window_ __m256i const four_select_lo_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)four_byte); __m256i const four_select_hi_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)(four_byte >> 32)); - __m256i low_lo_u8x32 = _mm256_blendv_epi8(window.low_lo, raw_lo_u8x32, ascii_select_lo_u8x32); - __m256i low_hi_u8x32 = _mm256_blendv_epi8(window.low_hi, raw_hi_u8x32, ascii_select_hi_u8x32); + __m256i low_lo_u8x32 = _mm256_blendv_epi8(window.low_byte_low_u8x32, raw_lo_u8x32, ascii_select_lo_u8x32); + __m256i low_hi_u8x32 = _mm256_blendv_epi8(window.low_byte_high_u8x32, raw_hi_u8x32, ascii_select_hi_u8x32); low_lo_u8x32 = _mm256_blendv_epi8(low_lo_u8x32, low_four_lo_u8x32, four_select_lo_u8x32); low_hi_u8x32 = _mm256_blendv_epi8(low_hi_u8x32, low_four_hi_u8x32, four_select_hi_u8x32); // high is zeroed on ASCII lanes (cp == raw byte, high == 0), then 4-byte high reconstructed. - __m256i high_lo_u8x32 = _mm256_andnot_si256(ascii_select_lo_u8x32, window.high_lo); - __m256i high_hi_u8x32 = _mm256_andnot_si256(ascii_select_hi_u8x32, window.high_hi); + __m256i high_lo_u8x32 = _mm256_andnot_si256(ascii_select_lo_u8x32, window.high_byte_low_u8x32); + __m256i high_hi_u8x32 = _mm256_andnot_si256(ascii_select_hi_u8x32, window.high_byte_high_u8x32); high_lo_u8x32 = _mm256_blendv_epi8(high_lo_u8x32, high_four_lo_u8x32, four_select_lo_u8x32); high_hi_u8x32 = _mm256_blendv_epi8(high_hi_u8x32, high_four_hi_u8x32, four_select_hi_u8x32); @@ -416,10 +416,10 @@ SZ_HELPER_AUTO sz_line_break_classified_haswell_t sz_line_break_classify_window_ sz_line_break_classified_haswell_t result; __m256i dotted_select_low_u8x32, dotted_select_high_u8x32; - sz_line_break_flat_palette_unpack_haswell_(palette_indices_low_u8x32, &result.classes_lo, &result.side_lo, - &dotted_select_low_u8x32); - sz_line_break_flat_palette_unpack_haswell_(palette_indices_high_u8x32, &result.classes_hi, &result.side_hi, - &dotted_select_high_u8x32); + sz_line_break_flat_palette_unpack_haswell_(palette_indices_low_u8x32, &result.classes_low_u8x32, + &result.side_low_u8x32, &dotted_select_low_u8x32); + sz_line_break_flat_palette_unpack_haswell_(palette_indices_high_u8x32, &result.classes_high_u8x32, + &result.side_high_u8x32, &dotted_select_high_u8x32); if (is_astral) { // The astral cascade is addressed by offset = codepoint - 0x10000; the codepoint's plane byte is // `(cp>>16)` (>=1 for astral), so the offset plane nibble is `plane - 1`. The low 16 bits are unchanged @@ -437,15 +437,17 @@ SZ_HELPER_AUTO sz_line_break_classified_haswell_t sz_line_break_classify_window_ __m256i astral_classes_u8x32, astral_side_u8x32, astral_dotted_bytes_u8x32; sz_line_break_palette_unpack_haswell_(astral_indices_low_u8x32, &astral_classes_u8x32, &astral_side_u8x32, &astral_dotted_bytes_u8x32); - result.classes_lo = _mm256_blendv_epi8(result.classes_lo, astral_classes_u8x32, astral_select_lo_u8x32); - result.side_lo = _mm256_blendv_epi8(result.side_lo, astral_side_u8x32, astral_select_lo_u8x32); + result.classes_low_u8x32 = _mm256_blendv_epi8(result.classes_low_u8x32, astral_classes_u8x32, + astral_select_lo_u8x32); + result.side_low_u8x32 = _mm256_blendv_epi8(result.side_low_u8x32, astral_side_u8x32, astral_select_lo_u8x32); dotted_select_low_u8x32 = _mm256_blendv_epi8( dotted_select_low_u8x32, _mm256_cmpgt_epi8(astral_dotted_bytes_u8x32, _mm256_setzero_si256()), astral_select_lo_u8x32); sz_line_break_palette_unpack_haswell_(astral_indices_high_u8x32, &astral_classes_u8x32, &astral_side_u8x32, &astral_dotted_bytes_u8x32); - result.classes_hi = _mm256_blendv_epi8(result.classes_hi, astral_classes_u8x32, astral_select_hi_u8x32); - result.side_hi = _mm256_blendv_epi8(result.side_hi, astral_side_u8x32, astral_select_hi_u8x32); + result.classes_high_u8x32 = _mm256_blendv_epi8(result.classes_high_u8x32, astral_classes_u8x32, + astral_select_hi_u8x32); + result.side_high_u8x32 = _mm256_blendv_epi8(result.side_high_u8x32, astral_side_u8x32, astral_select_hi_u8x32); dotted_select_high_u8x32 = _mm256_blendv_epi8( dotted_select_high_u8x32, _mm256_cmpgt_epi8(astral_dotted_bytes_u8x32, _mm256_setzero_si256()), astral_select_hi_u8x32); @@ -482,18 +484,18 @@ SZ_HELPER_INLINE sz_u64_t sz_line_break_side_mask_haswell_(__m256i side_lo_u8x32 /** @brief Byte-lane gate/base derivation (LB9/LB10) — the AVX2 twin of @ref sz_line_break_byte_frame_icelake_. */ typedef struct sz_line_break_byte_frame_haswell_t { - __m256i classes_lo; /**< Class per lane [0,32) with lone marks reclassified to AL (LB10). */ - __m256i classes_hi; /**< Class per lane [32,64). */ - sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ - sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ - sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ - sz_u64_t lone_mark; /**< LB10 lone marks reclassified to AL; their side bits must be cleared. */ + __m256i classes_low_u8x32; /**< Class per lane [0,32) with lone marks reclassified to AL (LB10). */ + __m256i classes_high_u8x32; /**< Class per lane [32,64). */ + sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ + sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ + sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ + sz_u64_t lone_mark; /**< LB10 lone marks reclassified to AL; their side bits must be cleared. */ } sz_line_break_byte_frame_haswell_t; SZ_HELPER_INLINE sz_line_break_byte_frame_haswell_t sz_line_break_byte_frame_haswell_( sz_line_break_classified_haswell_t classified) { sz_u64_t const starts = classified.starts, non_start = classified.non_start; - __m256i const classes_lo_u8x32 = classified.classes_lo, classes_hi_u8x32 = classified.classes_hi; + __m256i const classes_lo_u8x32 = classified.classes_low_u8x32, classes_hi_u8x32 = classified.classes_high_u8x32; sz_u64_t const mark_start = (sz_line_break_class_mask_haswell_(classes_lo_u8x32, classes_hi_u8x32, sz_line_break_cm_k) | sz_line_break_class_mask_haswell_(classes_lo_u8x32, classes_hi_u8x32, sz_line_break_zwj_k)) & @@ -517,8 +519,8 @@ SZ_HELPER_INLINE sz_line_break_byte_frame_haswell_t sz_line_break_byte_frame_has __m256i const al_u8x32 = _mm256_set1_epi8((char)sz_line_break_al_k); __m256i const lone_select_lo_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)lone_mark); __m256i const lone_select_hi_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)(lone_mark >> 32)); - frame.classes_lo = _mm256_blendv_epi8(classes_lo_u8x32, al_u8x32, lone_select_lo_u8x32); - frame.classes_hi = _mm256_blendv_epi8(classes_hi_u8x32, al_u8x32, lone_select_hi_u8x32); + frame.classes_low_u8x32 = _mm256_blendv_epi8(classes_lo_u8x32, al_u8x32, lone_select_lo_u8x32); + frame.classes_high_u8x32 = _mm256_blendv_epi8(classes_hi_u8x32, al_u8x32, lone_select_hi_u8x32); frame.base = starts & ~attached; frame.gate = non_start | attached; frame.attached = attached; @@ -536,13 +538,13 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_haswell_(sz_lin sz_u8_t *effective_class_byte_out, sz_u8_t *side_byte_out) { sz_line_break_byte_frame_haswell_t const byte_frame = sz_line_break_byte_frame_haswell_(classified); - __m256i const classes_lo_u8x32 = byte_frame.classes_lo, classes_hi_u8x32 = byte_frame.classes_hi; + __m256i const classes_lo_u8x32 = byte_frame.classes_low_u8x32, classes_hi_u8x32 = byte_frame.classes_high_u8x32; // LB10 reclassify carries the side bits with it: zero the side byte on lone-mark lanes (serial zeros the // descriptor). `andnot(lone_select, side)` clears those lanes. __m256i const lone_select_lo_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)byte_frame.lone_mark); __m256i const lone_select_hi_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)(byte_frame.lone_mark >> 32)); - __m256i const side_lo_u8x32 = _mm256_andnot_si256(lone_select_lo_u8x32, classified.side_lo); - __m256i const side_hi_u8x32 = _mm256_andnot_si256(lone_select_hi_u8x32, classified.side_hi); + __m256i const side_lo_u8x32 = _mm256_andnot_si256(lone_select_lo_u8x32, classified.side_low_u8x32); + __m256i const side_hi_u8x32 = _mm256_andnot_si256(lone_select_hi_u8x32, classified.side_high_u8x32); sz_line_break_frame_t frame; frame.base = byte_frame.base; @@ -563,7 +565,7 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_haswell_(sz_lin for (sz_size_t cls = 0; cls < sz_line_break_class_count_k; ++cls) frame.effective_class[cls] = sz_line_break_class_mask_haswell_(classes_lo_u8x32, classes_hi_u8x32, (sz_u8_t)cls); - frame.raw_zwj = sz_line_break_class_mask_haswell_(classified.classes_lo, classified.classes_hi, + frame.raw_zwj = sz_line_break_class_mask_haswell_(classified.classes_low_u8x32, classified.classes_high_u8x32, sz_line_break_zwj_k); frame.side_pi = sz_line_break_side_mask_haswell_(side_lo_u8x32, side_hi_u8x32, sz_line_break_side_pi_k); frame.side_pf = sz_line_break_side_mask_haswell_(side_lo_u8x32, side_hi_u8x32, sz_line_break_side_pf_k); @@ -598,8 +600,8 @@ SZ_HELPER_INLINE sz_line_break_window_t sz_line_break_decide_window_haswell_( * @brief Largest byte prefix of the window whose codepoints are all fully loaded — the AVX2 twin of * @ref sz_line_break_complete_limit_ over the Haswell window struct. Never below 1. */ -SZ_HELPER_AUTO sz_size_t sz_line_break_complete_limit_haswell_(sz_utf8_rune_window_haswell_t window, - sz_bool_t more_text) { +SZ_HELPER_INLINE sz_size_t sz_line_break_complete_limit_haswell_(sz_utf8_rune_window_haswell_t window, + sz_bool_t more_text) { sz_size_t const loaded = window.loaded; if (!more_text) return loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); diff --git a/include/stringzilla/utf8_linebreaks/icelake.h b/include/stringzilla/utf8_linebreaks/icelake.h index 7ea9d0e8..e58f6fe1 100644 --- a/include/stringzilla/utf8_linebreaks/icelake.h +++ b/include/stringzilla/utf8_linebreaks/icelake.h @@ -55,7 +55,7 @@ extern "C" { * offset = codepoint - 0x10000 (s0 -> s1 -> s2 -> leaf). Re-init-free: every tile is read straight from * aligned .rodata through the substrate permute256_/lut_cascade_ helpers. Bit-exact with * `sz_rune_line_break_property` over the astral planes. */ -SZ_HELPER_AUTO __m512i sz_line_break_classify_astral16_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_line_break_classify_astral16_icelake_(__m512i codepoints_u32x16) { __m512i const offset_u32x16 = _mm512_sub_epi32(codepoints_u32x16, _mm512_set1_epi32(0x10000)); __m512i const stage1_u32x16 = sz_utf8_rune_permute256_icelake_( sz_utf8_line_break_astral_s0_, _mm512_and_si512(_mm512_srli_epi32(offset_u32x16, 12), _mm512_set1_epi32(0xFF))); @@ -80,7 +80,7 @@ SZ_HELPER_AUTO __m512i sz_line_break_classify_astral16_icelake_(__m512i codepoin * `flat_bmp_[page * 256 + (cp & 0xFF)]`. The leaf byte is an index into * `sz_utf8_line_break_flat_palette_`, NOT the 62-entry cascade palette. Only the low byte of each u32 lane * is the index; the caller truncates with `vpmovdb`. */ -SZ_HELPER_AUTO __m512i sz_line_break_bmp_flat_index16_icelake_(__m512i codepoints_u32x16) { +SZ_HELPER_INLINE __m512i sz_line_break_bmp_flat_index16_icelake_(__m512i codepoints_u32x16) { return sz_utf8_rune_flat_lookup_icelake_(sz_utf8_line_break_bmp_page_lut_, sz_utf8_line_break_flat_bmp_, codepoints_u32x16); } @@ -90,7 +90,7 @@ SZ_HELPER_AUTO __m512i sz_line_break_bmp_flat_index16_icelake_(__m512i codepoint * the decode. Lanes whose codepoint is >= 0x10000 are * undefined (the caller blends the astral path over them). The sixteen-lane groups are unrolled because * `vextracti32x4` / `vinserti32x4` take an immediate lane selector. */ -SZ_HELPER_AUTO __m512i sz_line_break_bmp_flat_index_icelake_(__m512i high_bytes_u8x64, __m512i low_bytes_u8x64) { +SZ_HELPER_INLINE __m512i sz_line_break_bmp_flat_index_icelake_(__m512i high_bytes_u8x64, __m512i low_bytes_u8x64) { __m512i palette_indices_u8x64 = _mm512_setzero_si512(); __m512i high_u32x16, low_u32x16, codepoints_u32x16, group_indices_u32x16; #define SZ_LINE_BREAK_FLAT_GROUP_ICELAKE_(group) \ @@ -112,9 +112,9 @@ SZ_HELPER_AUTO __m512i sz_line_break_bmp_flat_index_icelake_(__m512i high_bytes_ * resolution aliasing (SA → AL/CM, AI/SG/XX → AL, CJ → NS); Pi/Pf/EAW/Cn|Ext side bits come from descriptor * bits 6/7/8/9; RI/ZWJ side from the raw class; CM|ZWJ -> mark side bit; DottedCircle from bit 13. */ SZ_HELPER_INLINE void sz_line_break_descriptor_unpack_half_icelake_(__m512i descriptors_u16x32, - __m512i *classes_u16x32_out, - __m512i *side_u16x32_out, - __mmask32 *dotted_m32_out) { + __m512i *classes_out_u16x32, + __m512i *side_out_u16x32, + __mmask32 *dotted_out_m32) { __m512i classes_u16x32 = _mm512_and_si512(descriptors_u16x32, _mm512_set1_epi16(0x3F)); __mmask32 const is_sa_m32 = _mm512_cmpeq_epi16_mask(classes_u16x32, _mm512_set1_epi16(sz_line_break_sa_k)); __mmask32 const sa_is_mark_m32 = _mm512_test_epi16_mask(descriptors_u16x32, _mm512_set1_epi16(1 << 12)); @@ -155,9 +155,9 @@ SZ_HELPER_INLINE void sz_line_break_descriptor_unpack_half_icelake_(__m512i desc side_u16x32 = _mm512_or_si512( side_u16x32, _mm512_maskz_mov_epi16(class_is_mark_m32, _mm512_set1_epi16(sz_line_break_side_mark_k))); - *classes_u16x32_out = classes_u16x32; - *side_u16x32_out = side_u16x32; - *dotted_m32_out = _mm512_test_epi16_mask(descriptors_u16x32, _mm512_set1_epi16(1 << 13)); + *classes_out_u16x32 = classes_u16x32; + *side_out_u16x32 = side_u16x32; + *dotted_out_m32 = _mm512_test_epi16_mask(descriptors_u16x32, _mm512_set1_epi16(1 << 13)); } /** @brief Expand sixty-four flat-palette indices to the LB1-resolved class byte, the engine side byte and the @@ -167,9 +167,9 @@ SZ_HELPER_INLINE void sz_line_break_descriptor_unpack_half_icelake_(__m512i desc * @ref sz_line_break_descriptor_unpack_half_icelake_ and narrow back to bytes with `vpmovwb`. Bit-identical * to the cascade's `palette_class_` / `_side_` / `_dotted_` byte-table permutes, which carry the same * resolution baked in. */ -SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_icelake_(__m512i palette_indices_u8x64, - __m512i *classes_u8x64_out, __m512i *side_u8x64_out, - sz_u64_t *dotted_out) { +SZ_HELPER_INLINE void sz_line_break_flat_palette_unpack_icelake_(__m512i palette_indices_u8x64, + __m512i *classes_out_u8x64, __m512i *side_out_u8x64, + sz_u64_t *dotted_out) { __m512i const palette_low_tile_u16x32 = _mm512_load_si512((void const *)sz_utf8_line_break_flat_palette_); __m512i const palette_high_tile_u16x32 = _mm512_load_si512((void const *)(sz_utf8_line_break_flat_palette_ + 32)); __m512i const indices_low_u16x32 = _mm512_cvtepu8_epi16(_mm512_castsi512_si256(palette_indices_u8x64)); @@ -185,22 +185,22 @@ SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_icelake_(__m512i palette_i &dotted_low_m32); sz_line_break_descriptor_unpack_half_icelake_(descriptors_high_u16x32, &classes_high_u16x32, &side_high_u16x32, &dotted_high_m32); - *classes_u8x64_out = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtepi16_epi8(classes_low_u16x32)), + *classes_out_u8x64 = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtepi16_epi8(classes_low_u16x32)), _mm512_cvtepi16_epi8(classes_high_u16x32), 1); - *side_u8x64_out = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtepi16_epi8(side_low_u16x32)), + *side_out_u8x64 = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtepi16_epi8(side_low_u16x32)), _mm512_cvtepi16_epi8(side_high_u16x32), 1); *dotted_out = (sz_u64_t)_cvtmask32_u32(dotted_low_m32) | ((sz_u64_t)_cvtmask32_u32(dotted_high_m32) << 32); } /** @brief Per-window byte-lane classification: class/side per lane, plus the effective-start and U+FFFD masks. */ typedef struct sz_line_break_classified_t { - __m512i classes; /**< Per-byte-lane Line_Break class (valid only on `starts` lanes). */ - __m512i side; /**< Per-byte-lane engine side byte. */ - sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ - sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ - sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ - sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ - sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ + __m512i classes_u8x64; /**< Per-byte-lane Line_Break class (valid only on `starts` lanes). */ + __m512i side_u8x64; /**< Per-byte-lane engine side byte. */ + sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ + sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ + sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ + sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ + sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ } sz_line_break_classified_t; /** @@ -210,14 +210,14 @@ typedef struct sz_line_break_classified_t { * icelake agree on malformed input. Valid leads classify by decoded VALUE (page / trie / big / astral), * matching the serial resolution precedence. The BMP trie uses the shared substrate `trie_walk_icelake_`. */ -SZ_HELPER_AUTO sz_line_break_classified_t sz_line_break_classify_window_icelake_(sz_utf8_rune_window_t window, - __m512i lane_identity_u8x64) { +SZ_HELPER_INLINE sz_line_break_classified_t sz_line_break_classify_window_icelake_(sz_utf8_rune_window_t window, + __m512i lane_identity_u8x64) { sz_u64_t const loaded_mask = sz_u64_mask_until_(window.loaded); sz_u64_t const continuation = _cvtmask64_u64(window.continuation) & loaded_mask; sz_u64_t const two_byte = _cvtmask64_u64(window.two_byte_starts); sz_u64_t const three_byte = _cvtmask64_u64(window.three_byte_starts); sz_u64_t const four_byte = _cvtmask64_u64(window.four_byte_starts); - __m512i const raw_u8x64 = window.window; + __m512i const raw_u8x64 = window.window_u8x64; __m512i const next1_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), raw_u8x64); @@ -290,9 +290,9 @@ SZ_HELPER_AUTO sz_line_break_classified_t sz_line_break_classify_window_icelake_ _mm512_and_si512(_mm512_slli_epi16(_mm512_and_si512(next1_u8x64, _mm512_set1_epi8(0x0F)), 4), _mm512_set1_epi8((char)0xF0)), sz_utf8_srl8_icelake_(next2_u8x64, 2, 0x0F)); - __m512i low_fixed_u8x64 = _mm512_mask_mov_epi8(window.low, _cvtu64_mask64(true_ascii), raw_u8x64); + __m512i low_fixed_u8x64 = _mm512_mask_mov_epi8(window.low_byte_u8x64, _cvtu64_mask64(true_ascii), raw_u8x64); low_fixed_u8x64 = _mm512_mask_mov_epi8(low_fixed_u8x64, _cvtu64_mask64(four_byte), low_four_u8x64); - __m512i high_fixed_u8x64 = _mm512_maskz_mov_epi8(_cvtu64_mask64(~true_ascii), window.high); + __m512i high_fixed_u8x64 = _mm512_maskz_mov_epi8(_cvtu64_mask64(~true_ascii), window.high_byte_u8x64); high_fixed_u8x64 = _mm512_mask_mov_epi8(high_fixed_u8x64, _cvtu64_mask64(four_byte), high_four_u8x64); sz_u64_t const valid_start = true_ascii | valid2 | valid3 | valid4; @@ -383,8 +383,8 @@ SZ_HELPER_AUTO sz_line_break_classified_t sz_line_break_classify_window_icelake_ } sz_line_break_classified_t result; - result.classes = classes_u8x64; - result.side = side_u8x64; + result.classes_u8x64 = classes_u8x64; + result.side_u8x64 = side_u8x64; result.dotted = dotted & starts; result.starts = starts; result.replacement = replacement; @@ -426,25 +426,25 @@ SZ_HELPER_INLINE sz_u64_t sz_line_break_class_range_mask_icelake_(__m512i classe /** @brief Byte-lane gate/base derivation for the byte-level rule engine: identifies cluster bases, the transparent * gate (continuations + attached combining marks), and reclassifies lone marks (LB10) to AL in @p classes. */ typedef struct sz_line_break_byte_frame_t { - __m512i classes; /**< Class per lane with lone marks reclassified to AL (LB10). */ - sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ - sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ - sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ + __m512i classes_u8x64; /**< Class per lane with lone marks reclassified to AL (LB10). */ + sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ + sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ + sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ sz_u64_t lone_mark; /**< LB10 lone marks reclassified to AL; their side bits must be cleared (serial zeros the descriptor). */ } sz_line_break_byte_frame_t; SZ_HELPER_INLINE sz_line_break_byte_frame_t sz_line_break_byte_frame_icelake_(sz_line_break_classified_t classified) { sz_u64_t const starts = classified.starts, non_start = classified.non_start; - sz_u64_t const mark_start = (sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_cm_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_zwj_k)) & + sz_u64_t const mark_start = (sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_cm_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_zwj_k)) & starts; - sz_u64_t const excluded = (sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_bk_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_cr_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_lf_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_nl_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_sp_k) | - sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_zw_k)) & + sz_u64_t const excluded = (sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_bk_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_cr_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_lf_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_nl_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_sp_k) | + sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_zw_k)) & starts; sz_u64_t const good_base = starts & ~excluded & ~mark_start; // A mark attaches (LB9) when reachable from a good base across only continuations and other marks. Flood each @@ -455,8 +455,8 @@ SZ_HELPER_INLINE sz_line_break_byte_frame_t sz_line_break_byte_frame_icelake_(sz sz_u64_t const lone_mark = mark_start & ~attached; // LB10: a mark with no attachable base acts as AL sz_line_break_byte_frame_t frame; - frame.classes = _mm512_mask_mov_epi8(classified.classes, _cvtu64_mask64(lone_mark), - _mm512_set1_epi8((char)sz_line_break_al_k)); + frame.classes_u8x64 = _mm512_mask_mov_epi8(classified.classes_u8x64, _cvtu64_mask64(lone_mark), + _mm512_set1_epi8((char)sz_line_break_al_k)); frame.base = starts & ~attached; frame.gate = non_start | attached; frame.attached = attached; @@ -475,10 +475,10 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_icelake_(sz_lin sz_u8_t *effective_class_byte_out, sz_u8_t *side_byte_out) { sz_line_break_byte_frame_t const byte_frame = sz_line_break_byte_frame_icelake_(classified); - __m512i const classes_u8x64 = byte_frame.classes; + __m512i const classes_u8x64 = byte_frame.classes_u8x64; // LB10 reclassifies a lone CM/ZWJ to AL; its descriptor side bits (EAW/Pi/Pf/...) must go with it, else LB19/LB15 // see a phantom East-Asian / quote cluster. Mirrors the serial path zeroing `codepoint_descriptors` on LB10. - __m512i const side_u8x64 = _mm512_maskz_mov_epi8(_cvtu64_mask64(~byte_frame.lone_mark), classified.side); + __m512i const side_u8x64 = _mm512_maskz_mov_epi8(_cvtu64_mask64(~byte_frame.lone_mark), classified.side_u8x64); sz_line_break_frame_t frame; frame.base = byte_frame.base; @@ -499,7 +499,7 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_icelake_(sz_lin #endif for (sz_size_t cls = 0; cls < sz_line_break_class_count_k; ++cls) frame.effective_class[cls] = sz_line_break_class_mask_icelake_(classes_u8x64, (sz_u8_t)cls); - frame.raw_zwj = sz_line_break_class_mask_icelake_(classified.classes, sz_line_break_zwj_k); + frame.raw_zwj = sz_line_break_class_mask_icelake_(classified.classes_u8x64, sz_line_break_zwj_k); frame.side_pi = sz_line_break_side_mask_icelake_(side_u8x64, sz_line_break_side_pi_k); frame.side_pf = sz_line_break_side_mask_icelake_(side_u8x64, sz_line_break_side_pf_k); frame.side_eaw = sz_line_break_side_mask_icelake_(side_u8x64, sz_line_break_side_eaw_k); @@ -535,7 +535,7 @@ SZ_HELPER_INLINE sz_line_break_window_t sz_line_break_decide_window_icelake_(sz_ * 64-byte edge). Mirrors the word kernel's complete-limit: a declared-length lead whose span exceeds `loaded` * ends the trusted region just before it; with no more text the whole window is complete. Never below 1. */ -SZ_HELPER_AUTO sz_size_t sz_line_break_complete_limit_(sz_utf8_rune_window_t window, sz_bool_t more_text) { +SZ_HELPER_INLINE sz_size_t sz_line_break_complete_limit_(sz_utf8_rune_window_t window, sz_bool_t more_text) { sz_size_t const loaded = window.loaded; if (!more_text) return loaded; sz_u64_t const valid = sz_u64_mask_until_(loaded); diff --git a/include/stringzilla/utf8_linebreaks/neon.h b/include/stringzilla/utf8_linebreaks/neon.h index 59677f14..7899967b 100644 --- a/include/stringzilla/utf8_linebreaks/neon.h +++ b/include/stringzilla/utf8_linebreaks/neon.h @@ -56,7 +56,7 @@ SZ_HELPER_INLINE uint8x16_t sz_line_break_byte_mask_from_bits_neon_(sz_u64_t bit * page-compressed flat leaf via @ref sz_utf8_rune_flat_lookup_neon_, the NEON twin of * @ref sz_line_break_bmp_index_haswell_. Bit-exact with `sz_rune_line_break_property` over the whole BMP * once `flat_palette_` expands the index. Operates on one quarter; the caller iterates the four. */ -SZ_HELPER_AUTO uint8x16_t sz_line_break_bmp_index_neon_(uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_line_break_bmp_index_neon_(uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_neon_(sz_utf8_line_break_bmp_page_lut_, sz_utf8_line_break_flat_bmp_, (int)sz_utf8_line_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -65,8 +65,8 @@ SZ_HELPER_AUTO uint8x16_t sz_line_break_bmp_index_neon_(uint8x16_t high_bytes_u8 * the NEON twin of @ref sz_line_break_classify_astral_haswell_. Per-lane bytes: @p plane_u8x16 = * (offset>>16)&0xFF (low nibble meaningful), @p high_u8x16 = (offset>>8)&0xFF, @p low_u8x16 = offset&0xFF. * Bit-exact. */ -SZ_HELPER_AUTO uint8x16_t sz_line_break_classify_astral_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, - uint8x16_t low_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_line_break_classify_astral_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, + uint8x16_t low_u8x16) { uint8x16_t const low_nibble_mask_u8x16 = vdupq_n_u8(0x0F); uint8x16_t const n4_u8x16 = vandq_u8(plane_u8x16, low_nibble_mask_u8x16); uint8x16_t const n3_u8x16 = vandq_u8(vshrq_n_u8(high_u8x16, 4), low_nibble_mask_u8x16); @@ -127,9 +127,9 @@ SZ_HELPER_INLINE void sz_line_break_flat_palette_descriptors_neon_(uint8x16_t pa * in 8/9, SA-is-mark in 12, DottedCircle in 13 -- so the whole unpack stays in the byte domain. Applies the * serial resolution aliasing (SA → AL/CM, AI/SG/XX → AL, CJ → NS); RI/ZWJ side bits come from the RAW * class, the mark side bit from the resolved class. */ -SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_neon_(uint8x16_t palette_indices_u8x16, - uint8x16_t *classes_u8x16_out, uint8x16_t *side_u8x16_out, - uint8x16_t *dotted_select_u8x16_out) { +SZ_HELPER_INLINE void sz_line_break_flat_palette_unpack_neon_(uint8x16_t palette_indices_u8x16, + uint8x16_t *classes_out_u8x16, uint8x16_t *side_out_u8x16, + uint8x16_t *dotted_select_out_u8x16) { uint8x16_t descriptor_low_bytes_u8x16, descriptor_high_bytes_u8x16; sz_line_break_flat_palette_descriptors_neon_(palette_indices_u8x16, &descriptor_low_bytes_u8x16, &descriptor_high_bytes_u8x16); @@ -165,9 +165,9 @@ SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_neon_(uint8x16_t palette_i vceqq_u8(classes_u8x16, vdupq_n_u8((sz_u8_t)sz_line_break_zwj_k))); side_u8x16 = vorrq_u8(side_u8x16, vandq_u8(class_is_mark_u8x16, vdupq_n_u8((sz_u8_t)sz_line_break_side_mark_k))); - *classes_u8x16_out = classes_u8x16; - *side_u8x16_out = side_u8x16; - *dotted_select_u8x16_out = vtstq_u8(descriptor_high_bytes_u8x16, vdupq_n_u8(1 << 5)); + *classes_out_u8x16 = classes_u8x16; + *side_out_u8x16 = side_u8x16; + *dotted_select_out_u8x16 = vtstq_u8(descriptor_high_bytes_u8x16, vdupq_n_u8(1 << 5)); } /** @brief A 64-bit "(byte & mask) == pattern" lane mask over the four window quarters. */ @@ -206,13 +206,13 @@ SZ_HELPER_INLINE sz_u64_t sz_line_break_byte_lt_neon_(uint8x16_t const *quarters /** @brief Per-window byte-lane classification (NEON): class/side per lane as four `uint8x16_t` quarters plus the * effective-start and U+FFFD masks. The NEON twin of @ref sz_line_break_classified_haswell_t. */ typedef struct sz_line_break_classified_neon_t { - uint8x16_t classes[4]; /**< Per-byte-lane Line_Break class (valid only on `starts` lanes), per quarter. */ - uint8x16_t side[4]; /**< Per-byte-lane engine side byte, per quarter. */ - sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ - sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ - sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ - sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ - sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ + uint8x16_t classes_u8x16s[4]; /**< Per-byte-lane Line_Break class (valid only on `starts` lanes), per quarter. */ + uint8x16_t side_u8x16s[4]; /**< Per-byte-lane engine side byte, per quarter. */ + sz_u64_t dotted; /**< Bit i set => lane i is DottedCircle U+25CC. */ + sz_u64_t starts; /**< Effective codepoint starts: valid leads (at their lane) + 1-byte U+FFFD units. */ + sz_u64_t replacement; /**< Effective-start lanes that are ill-formed (decoded as U+FFFD, class AL). */ + sz_u64_t non_start; /**< Bytes that are NOT effective starts (consumed continuations) within `loaded`. */ + sz_size_t loaded; /**< Bytes loaded into this window (<= 64). */ } sz_line_break_classified_neon_t; /** @brief Resolve the per-lane 62-entry-palette index (one quarter) to class / side / dotted bytes through the @@ -232,13 +232,14 @@ SZ_HELPER_INLINE void sz_line_break_palette_unpack_neon_(uint8x16_t index_u8x16, * "consume-1 U+FFFD" malformed policy: an invalid lead / short or stray continuation / overlong / * surrogate / out-of-range lead each become one single-byte U+FFFD unit (class AL). */ -SZ_HELPER_AUTO sz_line_break_classified_neon_t sz_line_break_classify_window_neon_(sz_utf8_rune_window_neon_t window) { +SZ_HELPER_INLINE sz_line_break_classified_neon_t sz_line_break_classify_window_neon_( + sz_utf8_rune_window_neon_t window) { sz_u64_t const loaded_mask = sz_u64_mask_until_serial_(window.loaded); sz_u64_t const continuation = window.continuation & loaded_mask; sz_u64_t const two_byte = window.two_byte_starts; sz_u64_t const three_byte = window.three_byte_starts; sz_u64_t const four_byte = window.four_byte_starts; - uint8x16_t const *raw_u8x16 = window.window; + uint8x16_t const *raw_u8x16 = window.window_u8x16s; // Forward neighbours (mod-64 wrap, matching icelake's `_mm512_permutexvar_epi8`). uint8x16_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; @@ -328,9 +329,10 @@ SZ_HELPER_AUTO sz_line_break_classified_neon_t sz_line_break_classify_window_neo sz_utf8_srl8_neon_(n2_u8x16, 2, 0x0F)); // Blend ASCII (cp == raw byte, high == 0) then 4-byte reconstruction over the decode-window halves. - uint8x16_t low_q_u8x16 = vbslq_u8(ascii_select_u8x16, raw_q_u8x16, window.low[quarter]); + uint8x16_t low_q_u8x16 = vbslq_u8(ascii_select_u8x16, raw_q_u8x16, window.low_byte_u8x16s[quarter]); low_q_u8x16 = vbslq_u8(four_select_u8x16, low_four_u8x16, low_q_u8x16); - uint8x16_t high_q_u8x16 = vbicq_u8(window.high[quarter], ascii_select_u8x16); // zero high on ASCII lanes + uint8x16_t high_q_u8x16 = vbicq_u8(window.high_byte_u8x16s[quarter], + ascii_select_u8x16); // zero high on ASCII lanes high_q_u8x16 = vbslq_u8(four_select_u8x16, high_four_u8x16, high_q_u8x16); // 4-byte plane bits (bits 16..20 of the codepoint); zero on every non-4-byte lane. @@ -348,8 +350,8 @@ SZ_HELPER_AUTO sz_line_break_classified_neon_t sz_line_break_classify_window_neo uint8x16_t const rep_select_u8x16 = sz_line_break_byte_mask_from_bits_neon_(replacement >> lane_base); bmp_index_u8x16 = vbslq_u8(rep_select_u8x16, fffd_index_u8x16, bmp_index_u8x16); } - sz_line_break_flat_palette_unpack_neon_(bmp_index_u8x16, &result.classes[quarter], &result.side[quarter], - &dotted_q_u8x16[quarter]); + sz_line_break_flat_palette_unpack_neon_(bmp_index_u8x16, &result.classes_u8x16s[quarter], + &result.side_u8x16s[quarter], &dotted_q_u8x16[quarter]); if (is_astral) { // The astral cascade is addressed by offset = codepoint - 0x10000; the offset plane nibble is // `plane - 1`. The low 16 bits are unchanged by subtracting 0x10000, so `high`/`low` feed directly. @@ -362,8 +364,9 @@ SZ_HELPER_AUTO sz_line_break_classified_neon_t sz_line_break_classify_window_neo uint8x16_t astral_classes_u8x16, astral_side_u8x16, astral_dotted_bytes_u8x16; sz_line_break_palette_unpack_neon_(astral_index_u8x16, &astral_classes_u8x16, &astral_side_u8x16, &astral_dotted_bytes_u8x16); - result.classes[quarter] = vbslq_u8(astral_select_u8x16, astral_classes_u8x16, result.classes[quarter]); - result.side[quarter] = vbslq_u8(astral_select_u8x16, astral_side_u8x16, result.side[quarter]); + result.classes_u8x16s[quarter] = vbslq_u8(astral_select_u8x16, astral_classes_u8x16, + result.classes_u8x16s[quarter]); + result.side_u8x16s[quarter] = vbslq_u8(astral_select_u8x16, astral_side_u8x16, result.side_u8x16s[quarter]); dotted_q_u8x16[quarter] = vbslq_u8(astral_select_u8x16, vtstq_u8(astral_dotted_bytes_u8x16, astral_dotted_bytes_u8x16), dotted_q_u8x16[quarter]); @@ -399,17 +402,17 @@ SZ_HELPER_INLINE sz_u64_t sz_line_break_side_mask_neon_(uint8x16_t const *side_u /** @brief Byte-lane gate/base derivation (LB9/LB10) — the NEON twin of @ref sz_line_break_byte_frame_haswell_t. */ typedef struct sz_line_break_byte_frame_neon_t { - uint8x16_t classes[4]; /**< Class per lane with lone marks reclassified to AL (LB10), per quarter. */ - sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ - sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ - sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ - sz_u64_t lone_mark; /**< LB10 lone marks reclassified to AL; their side bits must be cleared. */ + uint8x16_t classes_u8x16s[4]; /**< Class per lane with lone marks reclassified to AL (LB10), per quarter. */ + sz_u64_t base; /**< Cluster-base lanes (every effective start except an attached CM/ZWJ). */ + sz_u64_t gate; /**< Transparent lanes for neighbour fills: continuations + attached-mark starts. */ + sz_u64_t attached; /**< Attached CM/ZWJ start lanes (LB9). */ + sz_u64_t lone_mark; /**< LB10 lone marks reclassified to AL; their side bits must be cleared. */ } sz_line_break_byte_frame_neon_t; SZ_HELPER_INLINE sz_line_break_byte_frame_neon_t sz_line_break_byte_frame_neon_( sz_line_break_classified_neon_t classified) { sz_u64_t const starts = classified.starts, non_start = classified.non_start; - uint8x16_t const *classes_u8x16 = classified.classes; + uint8x16_t const *classes_u8x16 = classified.classes_u8x16s; sz_u64_t const mark_start = (sz_line_break_class_mask_neon_(classes_u8x16, sz_line_break_cm_k) | sz_line_break_class_mask_neon_(classes_u8x16, sz_line_break_zwj_k)) & starts; @@ -431,7 +434,7 @@ SZ_HELPER_INLINE sz_line_break_byte_frame_neon_t sz_line_break_byte_frame_neon_( uint8x16_t const al_u8x16 = vdupq_n_u8(sz_line_break_al_k); for (int quarter = 0; quarter < 4; ++quarter) { uint8x16_t const lone_select_u8x16 = sz_line_break_byte_mask_from_bits_neon_(lone_mark >> (quarter * 16)); - frame.classes[quarter] = vbslq_u8(lone_select_u8x16, al_u8x16, classes_u8x16[quarter]); + frame.classes_u8x16s[quarter] = vbslq_u8(lone_select_u8x16, al_u8x16, classes_u8x16[quarter]); } frame.base = starts & ~attached; frame.gate = non_start | attached; @@ -450,14 +453,14 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_neon_(sz_line_b sz_u8_t *effective_class_byte_out, sz_u8_t *side_byte_out) { sz_line_break_byte_frame_neon_t const byte_frame = sz_line_break_byte_frame_neon_(classified); - uint8x16_t const *classes_u8x16 = byte_frame.classes; + uint8x16_t const *classes_u8x16 = byte_frame.classes_u8x16s; // LB10 reclassify carries the side bits with it: zero the side byte on lone-mark lanes (serial zeros the // descriptor). `vbicq_u8(side, lone_select)` clears those lanes. uint8x16_t side_u8x16[4]; for (int quarter = 0; quarter < 4; ++quarter) { uint8x16_t const lone_select_u8x16 = sz_line_break_byte_mask_from_bits_neon_(byte_frame.lone_mark >> (quarter * 16)); - side_u8x16[quarter] = vbicq_u8(classified.side[quarter], lone_select_u8x16); + side_u8x16[quarter] = vbicq_u8(classified.side_u8x16s[quarter], lone_select_u8x16); } sz_line_break_frame_t frame; @@ -478,7 +481,7 @@ SZ_HELPER_INLINE sz_line_break_frame_t sz_line_break_build_frame_neon_(sz_line_b #endif for (sz_size_t cls = 0; cls < sz_line_break_class_count_k; ++cls) frame.effective_class[cls] = sz_line_break_class_mask_neon_(classes_u8x16, (sz_u8_t)cls); - frame.raw_zwj = sz_line_break_class_mask_neon_(classified.classes, sz_line_break_zwj_k); + frame.raw_zwj = sz_line_break_class_mask_neon_(classified.classes_u8x16s, sz_line_break_zwj_k); frame.side_pi = sz_line_break_side_mask_neon_(side_u8x16, sz_line_break_side_pi_k); frame.side_pf = sz_line_break_side_mask_neon_(side_u8x16, sz_line_break_side_pf_k); frame.side_eaw = sz_line_break_side_mask_neon_(side_u8x16, sz_line_break_side_eaw_k); @@ -514,7 +517,7 @@ SZ_HELPER_INLINE sz_line_break_window_t sz_line_break_decide_window_neon_(sz_lin * @brief Largest byte prefix of the window whose codepoints are all fully loaded — the NEON twin of * @ref sz_line_break_complete_limit_haswell_ over the NEON window struct. Never below 1. */ -SZ_HELPER_AUTO sz_size_t sz_line_break_complete_limit_neon_(sz_utf8_rune_window_neon_t window, sz_bool_t more_text) { +SZ_HELPER_INLINE sz_size_t sz_line_break_complete_limit_neon_(sz_utf8_rune_window_neon_t window, sz_bool_t more_text) { sz_size_t const loaded = window.loaded; if (!more_text) return loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); diff --git a/include/stringzilla/utf8_linebreaks/serial.h b/include/stringzilla/utf8_linebreaks/serial.h index e1659bc7..892a984d 100644 --- a/include/stringzilla/utf8_linebreaks/serial.h +++ b/include/stringzilla/utf8_linebreaks/serial.h @@ -67,8 +67,8 @@ SZ_HELPER_INLINE sz_bool_t sz_line_break_is_cm_or_zwj_(sz_u8_t line_break_class) } /** @brief One decoded codepoint's LB1-resolved Line_Break class; advances @p position and returns the descriptor. */ -SZ_HELPER_AUTO sz_u8_t sz_line_break_decode_one_(sz_cptr_t text, sz_size_t length, sz_size_t *position, - sz_u16_t *descriptor_out) { +SZ_HELPER_INLINE sz_u8_t sz_line_break_decode_one_(sz_cptr_t text, sz_size_t length, sz_size_t *position, + sz_u16_t *descriptor_out) { sz_size_t decode = *position; sz_rune_t const rune = sz_utf8_next_rune_(text, length, &decode); sz_u16_t const descriptor = sz_rune_line_break_property(rune); @@ -111,9 +111,9 @@ SZ_HELPER_INLINE sz_line_break_cluster_t sz_line_break_cluster_invalid_(void) { * A combining mark with no attachable base (start of text, or after BK/CR/LF/NL/SP/ZW) is LB10: kept as a * lone AL cluster. @p last_codepoint_was_zwj carries the LB8a "preceded by ZWJ" bit across calls. */ -SZ_HELPER_AUTO sz_line_break_cluster_t sz_line_break_next_cluster_(sz_cptr_t text, sz_size_t length, - sz_size_t *position, - sz_bool_t *last_codepoint_was_zwj) { +SZ_HELPER_INLINE sz_line_break_cluster_t sz_line_break_next_cluster_(sz_cptr_t text, sz_size_t length, + sz_size_t *position, + sz_bool_t *last_codepoint_was_zwj) { sz_line_break_cluster_t cluster; if (*position >= length) return sz_line_break_cluster_invalid_(); cluster.valid = sz_true_k; @@ -647,10 +647,10 @@ SZ_HELPER_AUTO sz_size_t sz_line_break_complete_limit_masks_(sz_size_t loaded, s /** @brief Per-lane class/side membership of one decoded 64-byte window, precomputed by a per-ISA extractor so the * portable rule engine sources every mask from `sz_u64_t` words without touching the codepoint vectors. */ typedef struct sz_line_break_frame_t { - sz_u64_t base, gate, attached, lone_mark; /**< from the byte-level cluster frame */ - sz_u64_t non_start, dotted, starts, replacement; /**< from the classifier */ - sz_u64_t effective_class[sz_line_break_class_count_k]; /**< membership per class, AFTER LB10 lone->AL; NOT &base */ - sz_u64_t raw_zwj; /**< class_mask(classified.classes, zwj_k), pre-effective */ + sz_u64_t base, gate, attached, lone_mark; /**< from the byte-level cluster frame */ + sz_u64_t non_start, dotted, starts, replacement; /**< from the classifier */ + sz_u64_t effective_class[sz_line_break_class_count_k]; /**< membership per class, AFTER LB10 lone->AL; NOT &base */ + sz_u64_t raw_zwj; /**< class_mask(classified.classes_*, zwj_k), pre-effective */ sz_u64_t side_pi, side_pf, side_eaw, side_cn, side_ext; /**< side-bit membership masks (NOT yet &base) */ } sz_line_break_frame_t; diff --git a/include/stringzilla/utf8_linebreaks/sve2.h b/include/stringzilla/utf8_linebreaks/sve2.h index b8ee9ce7..8a2a5c6e 100644 --- a/include/stringzilla/utf8_linebreaks/sve2.h +++ b/include/stringzilla/utf8_linebreaks/sve2.h @@ -27,8 +27,8 @@ extern "C" { /** @brief Flat-palette index for one chunk of ASTRAL codepoints over the 20-bit offset = cp - 0x10000 (5-nibble * cascade), the SVE2 twin of @ref sz_line_break_classify_astral_neon_. Returns 62-entry palette indices. */ -SZ_HELPER_AUTO svuint8_t sz_line_break_classify_astral_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, - svuint8_t low_u8x) { +SZ_HELPER_INLINE svuint8_t sz_line_break_classify_astral_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, + svuint8_t low_u8x) { svbool_t const all_b8x = svptrue_b8(); svuint8_t const n4_u8x = svand_n_u8_x(all_b8x, plane_u8x, 0x0F); svuint8_t const n3_u8x = svand_n_u8_x(all_b8x, svlsr_n_u8_x(all_b8x, high_u8x, 4), 0x0F); @@ -63,9 +63,9 @@ SZ_HELPER_AUTO svuint8_t sz_line_break_classify_astral_sve2_(svuint8_t plane_u8x /** @brief Split one chunk of flat-palette indices into the low and high bytes of their 16-bit Line_Break * descriptors, gathered straight from the 64-word palette by one `svld1uh_gather` per 32-bit quarter - * the SVE2 stand-in for the NEON resident `vqtbl4q` pair and the AVX2 `vpgatherdd`. */ -SZ_HELPER_AUTO void sz_line_break_flat_descriptors_sve2_(svuint8_t palette_indices_u8x, - svuint8_t *descriptor_low_u8x_out, - svuint8_t *descriptor_high_u8x_out) { +SZ_HELPER_INLINE void sz_line_break_flat_descriptors_sve2_(svuint8_t palette_indices_u8x, + svuint8_t *descriptor_low_out_u8x, + svuint8_t *descriptor_high_out_u8x) { svbool_t const all_b32x = svptrue_b32(); sz_u16_t const *palette = sz_utf8_line_break_flat_palette_; svuint16_t const indices_lo_u16x = svunpklo_u16(palette_indices_u8x), @@ -85,20 +85,20 @@ SZ_HELPER_AUTO void sz_line_break_flat_descriptors_sve2_(svuint8_t palette_indic svreinterpret_u16_u32(svand_n_u32_x(all_b32x, second_u32x, 0xFF))); svuint16_t const low_second_u16x = svuzp1_u16(svreinterpret_u16_u32(svand_n_u32_x(all_b32x, third_u32x, 0xFF)), svreinterpret_u16_u32(svand_n_u32_x(all_b32x, fourth_u32x, 0xFF))); - *descriptor_low_u8x_out = svuzp1_u8(svreinterpret_u8_u16(low_first_u16x), svreinterpret_u8_u16(low_second_u16x)); + *descriptor_low_out_u8x = svuzp1_u8(svreinterpret_u8_u16(low_first_u16x), svreinterpret_u8_u16(low_second_u16x)); svuint16_t const high_first_u16x = svuzp1_u16(svreinterpret_u16_u32(svlsr_n_u32_x(all_b32x, first_u32x, 8)), svreinterpret_u16_u32(svlsr_n_u32_x(all_b32x, second_u32x, 8))); svuint16_t const high_second_u16x = svuzp1_u16(svreinterpret_u16_u32(svlsr_n_u32_x(all_b32x, third_u32x, 8)), svreinterpret_u16_u32(svlsr_n_u32_x(all_b32x, fourth_u32x, 8))); - *descriptor_high_u8x_out = svuzp1_u8(svreinterpret_u8_u16(high_first_u16x), svreinterpret_u8_u16(high_second_u16x)); + *descriptor_high_out_u8x = svuzp1_u8(svreinterpret_u8_u16(high_first_u16x), svreinterpret_u8_u16(high_second_u16x)); } /** @brief Expand one chunk of flat-palette indices to the LB1-resolved class byte, the engine side byte and the * DottedCircle predicate - the SVE2 twin of @ref sz_line_break_flat_palette_unpack_neon_. Applies the * serial resolution aliasing (SA -> AL/CM, AI/SG/XX -> AL, CJ -> NS); RI/ZWJ side bits come from the RAW * class, the mark side bit from the resolved class. */ -SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_sve2_(svuint8_t palette_indices_u8x, svuint8_t *classes_u8x_out, - svuint8_t *side_u8x_out, svbool_t *dotted_b8x_out) { +SZ_HELPER_INLINE void sz_line_break_flat_palette_unpack_sve2_(svuint8_t palette_indices_u8x, svuint8_t *classes_out_u8x, + svuint8_t *side_out_u8x, svbool_t *dotted_out_b8x) { svbool_t const all_b8x = svptrue_b8(); svuint8_t descriptor_low_u8x, descriptor_high_u8x; sz_line_break_flat_descriptors_sve2_(palette_indices_u8x, &descriptor_low_u8x, &descriptor_high_u8x); @@ -135,9 +135,9 @@ SZ_HELPER_AUTO void sz_line_break_flat_palette_unpack_sve2_(svuint8_t palette_in svcmpeq_n_u8(all_b8x, classes_u8x, (sz_u8_t)sz_line_break_zwj_k)); side_u8x = svorr_u8_m(class_is_mark_b8x, side_u8x, svdup_n_u8((sz_u8_t)sz_line_break_side_mark_k)); - *classes_u8x_out = classes_u8x; - *side_u8x_out = side_u8x; - *dotted_b8x_out = svcmpne_n_u8(all_b8x, svand_n_u8_x(all_b8x, descriptor_high_u8x, 1 << 5), 0); + *classes_out_u8x = classes_u8x; + *side_out_u8x = side_u8x; + *dotted_out_b8x = svcmpne_n_u8(all_b8x, svand_n_u8_x(all_b8x, descriptor_high_u8x, 1 << 5), 0); } /** @brief Membership mask of class @p cls over the six class bit-planes (class ids are < 64). */ diff --git a/include/stringzilla/utf8_norm.h b/include/stringzilla/utf8_norm.h index 9d0fbea4..5c921157 100644 --- a/include/stringzilla/utf8_norm.h +++ b/include/stringzilla/utf8_norm.h @@ -26,7 +26,7 @@ extern "C" { /** * @brief Transform a UTF-8 string into a Unicode normalization form. * - * @section Buffer Sizing + * @section utf8_norm_buffer_sizing Buffer Sizing * * Decomposition forms (NFD, NFKD) can expand the input; the destination must hold up to * `source_length * 18` bytes for the worst single-codepoint compatibility decomposition. The diff --git a/include/stringzilla/utf8_norm/serial.h b/include/stringzilla/utf8_norm/serial.h index 373be0f5..3645d3a5 100644 --- a/include/stringzilla/utf8_norm/serial.h +++ b/include/stringzilla/utf8_norm/serial.h @@ -91,7 +91,7 @@ enum sz_utf8_norm_quick_check_t { }; /** @brief 3-stage trie index for @p codepoint (0 for out-of-range / default). Shared by the props and scan lookups. */ -SZ_HELPER_INLINE sz_u16_t sz_utf8_norm_index_(sz_rune_t codepoint) { +SZ_HELPER_AUTO sz_u16_t sz_utf8_norm_index_(sz_rune_t codepoint) { if (codepoint >= sz_utf8_norm_table_max_k) return 0; sz_size_t leaf = codepoint >> sz_utf8_norm_low_bits_k; sz_u16_t mid = sz_utf8_norm_stage1_[leaf >> sz_utf8_norm_mid_bits_k]; @@ -112,7 +112,7 @@ SZ_HELPER_INLINE sz_utf8_norm_props_t sz_utf8_norm_lookup_(sz_rune_t codepoint) * (~20% slower). Hangul's decomposition bits are baked into the generated values, so no runtime Hangul * test is needed here. */ -SZ_HELPER_INLINE sz_u16_t sz_utf8_norm_value_(sz_rune_t codepoint) { +SZ_HELPER_AUTO sz_u16_t sz_utf8_norm_value_(sz_rune_t codepoint) { if (codepoint >= sz_utf8_norm_table_max_k) return 0; sz_size_t leaf = codepoint >> sz_utf8_norm_scan_low_bits_k; sz_u16_t mid = sz_utf8_norm_scan_stage1_[leaf >> sz_utf8_norm_scan_mid_bits_k]; @@ -259,7 +259,7 @@ SZ_HELPER_AUTO void sz_utf8_norm_emit_(sz_utf8_norm_out_t *out, sz_rune_t rune) * output verbatim, never round-tripped through `sz_rune_encode`. It is an opaque barrier: it does * not decompose, compose, or participate in canonical ordering. */ -SZ_HELPER_INLINE void sz_utf8_norm_emit_byte_(sz_utf8_norm_out_t *out, sz_u8_t byte) { +SZ_HELPER_AUTO void sz_utf8_norm_emit_byte_(sz_utf8_norm_out_t *out, sz_u8_t byte) { if (out->dst) { *out->dst++ = byte, ++out->written; } else if (out->matches) { if (out->cmp == out->cmp_end || *out->cmp++ != byte) out->matches = sz_false_k; @@ -301,8 +301,8 @@ SZ_HELPER_AUTO void sz_utf8_norm_flush_(sz_rune_t *runes, sz_u8_t *canonical_com } /** @brief Core normalization engine, shared by the write and compare entry points. */ -SZ_HELPER_AUTO void sz_utf8_norm_run_(sz_cptr_t source, sz_size_t source_length, sz_normal_form_t form, - sz_utf8_norm_out_t *out) { +SZ_HELPER_INLINE void sz_utf8_norm_run_(sz_cptr_t source, sz_size_t source_length, sz_normal_form_t form, + sz_utf8_norm_out_t *out) { sz_bool_t compat = (form == sz_normal_form_nfkd_k || form == sz_normal_form_nfkc_k) ? sz_true_k : sz_false_k; sz_bool_t compose = (form == sz_normal_form_nfc_k || form == sz_normal_form_nfkc_k) ? sz_true_k : sz_false_k; @@ -427,7 +427,7 @@ SZ_HELPER_NOINLINE sz_cptr_t sz_utf8_norm_classify_serial_(sz_cptr_t text, sz_si } /** @brief Map a normalization form to its hot-path `sz_utf8_norm_quick_check_k*` flag bit. */ -SZ_HELPER_INLINE sz_u8_t sz_utf8_norm_form_flag_(sz_normal_form_t form) { +SZ_HELPER_AUTO sz_u8_t sz_utf8_norm_form_flag_(sz_normal_form_t form) { switch (form) { case sz_normal_form_nfc_k: return sz_utf8_norm_quick_check_nfc_k; case sz_normal_form_nfkc_k: return sz_utf8_norm_quick_check_nfkc_k; @@ -526,7 +526,7 @@ SZ_HELPER_INLINE sz_bool_t sz_utf8_norm_boundary_at_(sz_u8_t const *position, sz * well-formed lead (or that would cross @p begin) is treated as single literal bytes, so the cursor * retreats exactly one byte rather than over-reading. */ -SZ_HELPER_AUTO sz_u8_t const *sz_utf8_norm_step_back_(sz_u8_t const *position, sz_u8_t const *begin) { +SZ_HELPER_INLINE sz_u8_t const *sz_utf8_norm_step_back_(sz_u8_t const *position, sz_u8_t const *begin) { sz_u8_t const *probe = position - 1; while (probe > begin && (*probe & 0xC0u) == 0x80u && (position - probe) < 4) --probe; sz_rune_t rune; @@ -541,8 +541,8 @@ SZ_HELPER_AUTO sz_u8_t const *sz_utf8_norm_step_back_(sz_u8_t const *position, s * @p scan primitive) and run the decompose/reorder/compose engine only on the short dirty regions, * each delimited by safe boundaries so composition never crosses a split. Shared across ISAs. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_norm_engine_(sz_cptr_t source, sz_size_t source_length, sz_normal_form_t form, - sz_ptr_t destination, sz_utf8_norm_scan_t scan) { +SZ_HELPER_INLINE sz_size_t sz_utf8_norm_engine_(sz_cptr_t source, sz_size_t source_length, sz_normal_form_t form, + sz_ptr_t destination, sz_utf8_norm_scan_t scan) { sz_u8_t const *const begin = (sz_u8_t const *)source; sz_u8_t const *const end = begin + source_length; sz_u8_t *out = (sz_u8_t *)destination; @@ -607,8 +607,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_norm_engine_(sz_cptr_t source, sz_size_t source * benign segments and back up to the same boundary), and it carries the clean guarantee that every * byte before the returned pointer is provably in @p form. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_find_denormalized_engine_(sz_cptr_t source, sz_size_t source_length, - sz_normal_form_t form, sz_utf8_norm_scan_t scan) { +SZ_HELPER_INLINE sz_cptr_t sz_utf8_find_denormalized_engine_(sz_cptr_t source, sz_size_t source_length, + sz_normal_form_t form, sz_utf8_norm_scan_t scan) { sz_u8_t const *const end = (sz_u8_t const *)source + source_length; sz_u8_t const *cur = (sz_u8_t const *)source; diff --git a/include/stringzilla/utf8_norm/skylake.h b/include/stringzilla/utf8_norm/skylake.h index 84f14ce6..e7129fe4 100644 --- a/include/stringzilla/utf8_norm/skylake.h +++ b/include/stringzilla/utf8_norm/skylake.h @@ -69,8 +69,8 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_norm_lead_classify_shuffle_skylake_(__m512i * @brief Shared AVX-512 scan skeleton: 64-byte all-ASCII gate, lead-classify via @p classify, then the * shared scalar verify on any block that survives the gate. Ice Lake reuses this verbatim. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_norm_classify_avx512_(sz_cptr_t text, sz_size_t length, sz_normal_form_t form, - sz_utf8_norm_lead_classify_avx512_t classify) { +SZ_HELPER_INLINE sz_cptr_t sz_utf8_norm_classify_avx512_(sz_cptr_t text, sz_size_t length, sz_normal_form_t form, + sz_utf8_norm_lead_classify_avx512_t classify) { sz_u8_t const *position = (sz_u8_t const *)text; sz_u8_t const *const end = position + length; sz_u8_t const form_flag = sz_utf8_norm_form_flag_(form); diff --git a/include/stringzilla/utf8_runes.h b/include/stringzilla/utf8_runes.h index 1b035869..fa9fd0a9 100644 --- a/include/stringzilla/utf8_runes.h +++ b/include/stringzilla/utf8_runes.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UTF-8 codepoint mechanics: count, find-nth, and chunk unpacking. - * @file utf8_runes.h + * @file include/stringzilla/utf8_runes.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_RUNES_H_ diff --git a/include/stringzilla/utf8_runes/README.md b/include/stringzilla/utf8_runes/README.md index 49311490..9f49cdfa 100644 --- a/include/stringzilla/utf8_runes/README.md +++ b/include/stringzilla/utf8_runes/README.md @@ -5,7 +5,7 @@ Each operation has a serial baseline plus `haswell` and `icelake` SIMD backends ## Methodology -Numbers are throughput measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput measured with `bench/utf8_traverse.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its columns are the rune operations and its rows are a backend on a chip, so reading down a column compares the same operation across the backend ladder while reading across a row compares operations on one backend. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes, reported in MB/s) and a Long Lines workload (full text lines, reported in GB/s) to expose how each kernel scales with token length. A `↑` cell means there is no dedicated kernel for that operation at that backend, so the dispatcher reuses the tier above it. diff --git a/include/stringzilla/utf8_runes/haswell.h b/include/stringzilla/utf8_runes/haswell.h index 43a4e22c..cd345797 100644 --- a/include/stringzilla/utf8_runes/haswell.h +++ b/include/stringzilla/utf8_runes/haswell.h @@ -93,18 +93,18 @@ SZ_API_COMPTIME sz_cptr_t sz_utf8_seek_haswell(sz_cptr_t text, sz_size_t length, * shape. Masks are `sz_u64_t` (`vpmovmskb` per half, OR-combined) rather than the Ice Lake `__mmask64`. Field * names and semantics match @ref sz_utf8_rune_window_t so the portable rule algebra is unchanged. */ typedef struct sz_utf8_rune_window_haswell_t { - __m256i window_lo; /**< Raw input bytes for lanes [0, 32). */ - __m256i window_hi; /**< Raw input bytes for lanes [32, 64). */ - __m256i high_lo; /**< Per-lane `codepoint >> 8` for lanes [0, 32). */ - __m256i high_hi; /**< Per-lane `codepoint >> 8` for lanes [32, 64). */ - __m256i low_lo; /**< Per-lane `codepoint & 0xFF` for lanes [0, 32). */ - __m256i low_hi; /**< Per-lane `codepoint & 0xFF` for lanes [32, 64). */ - sz_u64_t continuation; /**< Bit `i` => lane `i` is a continuation byte `10xxxxxx`. */ - sz_u64_t codepoint_starts; /**< Bit `i` => lane `i` begins a codepoint (loaded, non-continuation). */ - sz_u64_t two_byte_starts; /**< Bit `i` => lane `i` is a 2-byte lead `110xxxxx`. */ - sz_u64_t three_byte_starts; /**< Bit `i` => lane `i` is a 3-byte lead `1110xxxx`. */ - sz_u64_t four_byte_starts; /**< Bit `i` => lane `i` is a 4-byte lead `11110xxx`. */ - sz_size_t loaded; /**< Number of bytes actually loaded (<= 64). */ + __m256i window_low_u8x32; /**< Raw input bytes for lanes [0, 32). */ + __m256i window_high_u8x32; /**< Raw input bytes for lanes [32, 64). */ + __m256i high_byte_low_u8x32; /**< Per-lane `codepoint >> 8` for lanes [0, 32). */ + __m256i high_byte_high_u8x32; /**< Per-lane `codepoint >> 8` for lanes [32, 64). */ + __m256i low_byte_low_u8x32; /**< Per-lane `codepoint & 0xFF` for lanes [0, 32). */ + __m256i low_byte_high_u8x32; /**< Per-lane `codepoint & 0xFF` for lanes [32, 64). */ + sz_u64_t continuation; /**< Bit `i` => lane `i` is a continuation byte `10xxxxxx`. */ + sz_u64_t codepoint_starts; /**< Bit `i` => lane `i` begins a codepoint (loaded, non-continuation). */ + sz_u64_t two_byte_starts; /**< Bit `i` => lane `i` is a 2-byte lead `110xxxxx`. */ + sz_u64_t three_byte_starts; /**< Bit `i` => lane `i` is a 3-byte lead `1110xxxx`. */ + sz_u64_t four_byte_starts; /**< Bit `i` => lane `i` is a 4-byte lead `11110xxx`. */ + sz_size_t loaded; /**< Number of bytes actually loaded (<= 64). */ } sz_utf8_rune_window_haswell_t; /** @brief Per-byte logical right shift by @p shift keeping the low @p keep bits — the AVX2 twin of `srl8_`. */ @@ -122,7 +122,7 @@ SZ_HELPER_INLINE sz_u64_t sz_utf8_mask_combine_haswell_(__m256i low_half_u8x32, /** @brief Masked 64-byte load into two halves; bytes [loaded, 64) read as zero (the AVX2 stand-in for * `_mm512_maskz_loadu_epi8`). A small stack staging union covers the partial tail so we never read past * `text + loaded`. */ -SZ_HELPER_AUTO void sz_utf8_load_window_haswell_( // +SZ_HELPER_INLINE void sz_utf8_load_window_haswell_( // sz_u8_t const *text, sz_size_t loaded, __m256i *out_low_u8x32, __m256i *out_high_u8x32) { if (loaded >= 64) { *out_low_u8x32 = _mm256_loadu_si256((__m256i const *)(text + 0)); @@ -176,14 +176,14 @@ SZ_HELPER_INLINE __m256i sz_utf8_byte_mask_from_bits_haswell_(sz_u32_t bits) { /** @brief Load up to 64 bytes (masked tail) and decode every lane into byte-domain halves — the AVX2 twin of * @ref sz_utf8_rune_decode_window_, bit-identical to it on every lane. */ -SZ_HELPER_AUTO sz_utf8_rune_window_haswell_t sz_utf8_rune_decode_window_haswell_( // +SZ_HELPER_INLINE sz_utf8_rune_window_haswell_t sz_utf8_rune_decode_window_haswell_( // sz_u8_t const *text, sz_size_t available) { sz_utf8_rune_window_haswell_t result; result.loaded = available < 64 ? available : 64; __m256i window_bytes_low_u8x32, window_bytes_high_u8x32; sz_utf8_load_window_haswell_(text, result.loaded, &window_bytes_low_u8x32, &window_bytes_high_u8x32); - result.window_lo = window_bytes_low_u8x32, result.window_hi = window_bytes_high_u8x32; + result.window_low_u8x32 = window_bytes_low_u8x32, result.window_high_u8x32 = window_bytes_high_u8x32; __m256i next_byte_1_low_u8x32, next_byte_1_high_u8x32, next_byte_2_low_u8x32, next_byte_2_high_u8x32; sz_utf8_forward_neighbours_haswell_(window_bytes_low_u8x32, window_bytes_high_u8x32, &next_byte_1_low_u8x32, @@ -261,14 +261,14 @@ SZ_HELPER_AUTO sz_utf8_rune_window_haswell_t sz_utf8_rune_decode_window_haswell_ (sz_u32_t)(result.three_byte_starts & 0xFFFFFFFFu)); __m256i const is_three_byte_high_select_u8x32 = sz_utf8_byte_mask_from_bits_haswell_( (sz_u32_t)((result.three_byte_starts >> 32) & 0xFFFFFFFFu)); - result.high_lo = _mm256_blendv_epi8(high_two_byte_low_u8x32, high_three_byte_low_u8x32, - is_three_byte_low_select_u8x32); - result.high_hi = _mm256_blendv_epi8(high_two_byte_high_u8x32, high_three_byte_high_u8x32, - is_three_byte_high_select_u8x32); - result.low_lo = _mm256_blendv_epi8(low_two_byte_low_u8x32, low_three_byte_low_u8x32, - is_three_byte_low_select_u8x32); - result.low_hi = _mm256_blendv_epi8(low_two_byte_high_u8x32, low_three_byte_high_u8x32, - is_three_byte_high_select_u8x32); + result.high_byte_low_u8x32 = _mm256_blendv_epi8(high_two_byte_low_u8x32, high_three_byte_low_u8x32, + is_three_byte_low_select_u8x32); + result.high_byte_high_u8x32 = _mm256_blendv_epi8(high_two_byte_high_u8x32, high_three_byte_high_u8x32, + is_three_byte_high_select_u8x32); + result.low_byte_low_u8x32 = _mm256_blendv_epi8(low_two_byte_low_u8x32, low_three_byte_low_u8x32, + is_three_byte_low_select_u8x32); + result.low_byte_high_u8x32 = _mm256_blendv_epi8(low_two_byte_high_u8x32, low_three_byte_high_u8x32, + is_three_byte_high_select_u8x32); return result; } @@ -280,7 +280,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_haswell_t sz_utf8_rune_decode_window_haswell_ * * ! Cost scales with @p tile_count, not with the window: every resident row is shuffled and blended on the shuffle * ! port. The BMP classifiers use @ref sz_utf8_rune_flat_lookup_haswell_ instead for that reason. */ -SZ_HELPER_AUTO __m256i sz_utf8_rune_cascade_stage_haswell_( // +SZ_HELPER_INLINE __m256i sz_utf8_rune_cascade_stage_haswell_( // sz_u8_t const *table, int tile_count, __m256i selector_u8x32, __m256i within_u8x32) { __m256i result_u8x32 = _mm256_setzero_si256(); for (int tile = 0; tile < tile_count; ++tile) { @@ -340,7 +340,7 @@ SZ_HELPER_INLINE __m256i sz_utf8_rune_pack4_u32_to_u8_haswell_( // * window, not the table: the cascade scanned every 16-byte tile of every stage on the shuffle port, while the * gather issues on the load ports and leaves the shuffle port to the decode. @p flat must extend four bytes * past its last index, since the dword gather over-reads three. */ -SZ_HELPER_AUTO __m256i sz_utf8_rune_flat_lookup_haswell_( // +SZ_HELPER_INLINE __m256i sz_utf8_rune_flat_lookup_haswell_( // sz_u8_t const *page_lut, sz_u8_t const *flat, __m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { __m256i const page_indices_u8x32 = sz_utf8_rune_lut256_haswell_(page_lut, high_bytes_u8x32); __m128i const page_indices_low_u8x16 = _mm256_castsi256_si128(page_indices_u8x32); @@ -371,7 +371,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_rune_flat_lookup_haswell_( // * BMI2 path: `tzcnt` pulls the lowest set bit's index, `blsr` clears it. On Intel Haswell `tzcnt`/`blsr` * are single-uop and beat a `vpshufb` left-pack LUT; the LUT only wins where `pext`/`blsr` is microcoded * (AMD pre-Zen3). */ -SZ_HELPER_AUTO void sz_utf8_unpack_indices_haswell_(sz_u64_t mask, sz_u8_t *out) { +SZ_HELPER_INLINE void sz_utf8_unpack_indices_haswell_(sz_u64_t mask, sz_u8_t *out) { while (mask) { *out++ = (sz_u8_t)(int)_tzcnt_u64(mask); mask = _blsr_u64(mask); // clear the lowest set bit @@ -384,7 +384,7 @@ SZ_HELPER_AUTO void sz_utf8_unpack_indices_haswell_(sz_u64_t mask, sz_u8_t *out) * streamed in waves of four u64 positions (`vpmovzxbq` widen + `base`, segment starts via `vpermq` shift + * `vpblendd` carry-seat, lengths via `vpsubq`), with a scalar tail for the final partial wave (no AVX2 * masked store). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_haswell_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_haswell_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { sz_size_t const boundary_count = (sz_size_t)_mm_popcnt_u64(boundary); @@ -459,274 +459,279 @@ SZ_HELPER_INLINE __m256i sz_utf8_rune_gather8_window_haswell_( // return _mm256_blendv_epi8(window_low_picked_u8x32, window_high_picked_u8x32, offset_high_bit_select_u32x8); } +/** + * @brief Ascending set-bit positions of every 8-bit sub-mask, eight bytes per row, keyed by the sub-mask. + * Unused slots are 0x80, whose high bit makes `vpshufb` read zero and never store. + */ +static sz_u8_t const sz_utf8_leftpack8_haswell_[256 * 8] = { + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 + 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x01 + 0x01, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x02 + 0x00, 0x01, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x03 + 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x04 + 0x00, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x05 + 0x01, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x06 + 0x00, 0x01, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x07 + 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 + 0x00, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x09 + 0x01, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0A + 0x00, 0x01, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0B + 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0C + 0x00, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0D + 0x01, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0E + 0x00, 0x01, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, // 0x0F + 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 + 0x00, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x11 + 0x01, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x12 + 0x00, 0x01, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x13 + 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x14 + 0x00, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x15 + 0x01, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x16 + 0x00, 0x01, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x17 + 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 + 0x00, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x19 + 0x01, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x1A + 0x00, 0x01, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1B + 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x1C + 0x00, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1D + 0x01, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1E + 0x00, 0x01, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, // 0x1F + 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x20 + 0x00, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x21 + 0x01, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x22 + 0x00, 0x01, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x23 + 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x24 + 0x00, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x25 + 0x01, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x26 + 0x00, 0x01, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x27 + 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x28 + 0x00, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x29 + 0x01, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x2A + 0x00, 0x01, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2B + 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x2C + 0x00, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2D + 0x01, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2E + 0x00, 0x01, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, // 0x2F + 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x30 + 0x00, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x31 + 0x01, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x32 + 0x00, 0x01, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x33 + 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x34 + 0x00, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x35 + 0x01, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x36 + 0x00, 0x01, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x37 + 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x38 + 0x00, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x39 + 0x01, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x3A + 0x00, 0x01, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3B + 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x3C + 0x00, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3D + 0x01, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3E + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, // 0x3F + 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x40 + 0x00, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x41 + 0x01, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x42 + 0x00, 0x01, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x43 + 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x44 + 0x00, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x45 + 0x01, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x46 + 0x00, 0x01, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x47 + 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x48 + 0x00, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x49 + 0x01, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x4A + 0x00, 0x01, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4B + 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x4C + 0x00, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4D + 0x01, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4E + 0x00, 0x01, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, // 0x4F + 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x50 + 0x00, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x51 + 0x01, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x52 + 0x00, 0x01, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x53 + 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x54 + 0x00, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x55 + 0x01, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x56 + 0x00, 0x01, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x57 + 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x58 + 0x00, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x59 + 0x01, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x5A + 0x00, 0x01, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5B + 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x5C + 0x00, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5D + 0x01, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5E + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, // 0x5F + 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x60 + 0x00, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x61 + 0x01, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x62 + 0x00, 0x01, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x63 + 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x64 + 0x00, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x65 + 0x01, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x66 + 0x00, 0x01, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x67 + 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x68 + 0x00, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x69 + 0x01, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x6A + 0x00, 0x01, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6B + 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x6C + 0x00, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6D + 0x01, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6E + 0x00, 0x01, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, // 0x6F + 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x70 + 0x00, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x71 + 0x01, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x72 + 0x00, 0x01, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x73 + 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x74 + 0x00, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x75 + 0x01, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x76 + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x77 + 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x78 + 0x00, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x79 + 0x01, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x7A + 0x00, 0x01, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7B + 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x7C + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7D + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7E + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, // 0x7F + 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x80 + 0x00, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x81 + 0x01, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x82 + 0x00, 0x01, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x83 + 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x84 + 0x00, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x85 + 0x01, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x86 + 0x00, 0x01, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x87 + 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x88 + 0x00, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x89 + 0x01, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x8A + 0x00, 0x01, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8B + 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x8C + 0x00, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8D + 0x01, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8E + 0x00, 0x01, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, // 0x8F + 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x90 + 0x00, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x91 + 0x01, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x92 + 0x00, 0x01, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x93 + 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x94 + 0x00, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x95 + 0x01, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x96 + 0x00, 0x01, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x97 + 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x98 + 0x00, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x99 + 0x01, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x9A + 0x00, 0x01, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9B + 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x9C + 0x00, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9D + 0x01, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9E + 0x00, 0x01, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, // 0x9F + 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA0 + 0x00, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA1 + 0x01, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA2 + 0x00, 0x01, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA3 + 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA4 + 0x00, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA5 + 0x01, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA6 + 0x00, 0x01, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xA7 + 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA8 + 0x00, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA9 + 0x01, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xAA + 0x00, 0x01, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAB + 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xAC + 0x00, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAD + 0x01, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAE + 0x00, 0x01, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, // 0xAF + 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xB0 + 0x00, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB1 + 0x01, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB2 + 0x00, 0x01, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB3 + 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB4 + 0x00, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB5 + 0x01, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB6 + 0x00, 0x01, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xB7 + 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB8 + 0x00, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB9 + 0x01, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xBA + 0x00, 0x01, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBB + 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xBC + 0x00, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBD + 0x01, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBE + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, // 0xBF + 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC0 + 0x00, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC1 + 0x01, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC2 + 0x00, 0x01, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC3 + 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC4 + 0x00, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC5 + 0x01, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC6 + 0x00, 0x01, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xC7 + 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC8 + 0x00, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC9 + 0x01, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xCA + 0x00, 0x01, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCB + 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xCC + 0x00, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCD + 0x01, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCE + 0x00, 0x01, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, // 0xCF + 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xD0 + 0x00, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD1 + 0x01, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD2 + 0x00, 0x01, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD3 + 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD4 + 0x00, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD5 + 0x01, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD6 + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xD7 + 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD8 + 0x00, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD9 + 0x01, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xDA + 0x00, 0x01, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDB + 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xDC + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDD + 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDE + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, // 0xDF + 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xE0 + 0x00, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE1 + 0x01, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE2 + 0x00, 0x01, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE3 + 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE4 + 0x00, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE5 + 0x01, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE6 + 0x00, 0x01, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xE7 + 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE8 + 0x00, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE9 + 0x01, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xEA + 0x00, 0x01, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xEB + 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xEC + 0x00, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xED + 0x01, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xEE + 0x00, 0x01, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, // 0xEF + 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xF0 + 0x00, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF1 + 0x01, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF2 + 0x00, 0x01, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF3 + 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF4 + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF5 + 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF6 + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xF7 + 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF8 + 0x00, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF9 + 0x01, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xFA + 0x00, 0x01, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFB + 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xFC + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFD + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFE + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // 0xFF +}; + /** * @brief Left-pack the set-bit positions of a 32-bit lane @p mask into @p out as a dense, ascending array of * byte-offsets in [0, 32), returning the count - the AVX2 `vpcompressb`-free start-compaction shared with the - * NEON / LASX / PowerVSX backends. Replaces a scalar `ctz` walk with a 2 KB shuffle-LUT (`leftpack8`) keyed by - * each 8-bit sub-mask: for every 16-lane half the mask splits into a low and a high byte, each `vpshufb` over - * the LUT row of its set-bit positions, the high half offset by +8, the two stitched at `popcount(low8)` via a - * gap-shift `vpshufb` (no scalar per-lane index walk). The half offset `h*16` is added in vector; one loop over - * the two halves. + * NEON / LASX / PowerVSX backends. Replaces a scalar `ctz` walk with a 2 KB shuffle-LUT keyed by each 8-bit + * sub-mask: for every 16-lane half the mask splits into a low and a high byte, each `vpshufb` over the LUT row + * of its set-bit positions, the high half offset by +8, the two stitched at `popcount(low8)` via a gap-shift + * `vpshufb` (no scalar per-lane index walk). The half offset `h*16` is added in vector; one loop over the two + * halves. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_leftpack_offsets_haswell_(sz_u32_t mask, sz_u8_t *out) { - static sz_u8_t const leftpack8[256 * 8] = { - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 - 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x01 - 0x01, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x02 - 0x00, 0x01, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x03 - 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x04 - 0x00, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x05 - 0x01, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x06 - 0x00, 0x01, 0x02, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x07 - 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 - 0x00, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x09 - 0x01, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0A - 0x00, 0x01, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0B - 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0C - 0x00, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0D - 0x01, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x0E - 0x00, 0x01, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80, // 0x0F - 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 - 0x00, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x11 - 0x01, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x12 - 0x00, 0x01, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x13 - 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x14 - 0x00, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x15 - 0x01, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x16 - 0x00, 0x01, 0x02, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x17 - 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 - 0x00, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x19 - 0x01, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x1A - 0x00, 0x01, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1B - 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x1C - 0x00, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1D - 0x01, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, 0x80, // 0x1E - 0x00, 0x01, 0x02, 0x03, 0x04, 0x80, 0x80, 0x80, // 0x1F - 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x20 - 0x00, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x21 - 0x01, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x22 - 0x00, 0x01, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x23 - 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x24 - 0x00, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x25 - 0x01, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x26 - 0x00, 0x01, 0x02, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x27 - 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x28 - 0x00, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x29 - 0x01, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x2A - 0x00, 0x01, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2B - 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x2C - 0x00, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2D - 0x01, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x2E - 0x00, 0x01, 0x02, 0x03, 0x05, 0x80, 0x80, 0x80, // 0x2F - 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x30 - 0x00, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x31 - 0x01, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x32 - 0x00, 0x01, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x33 - 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x34 - 0x00, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x35 - 0x01, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x36 - 0x00, 0x01, 0x02, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x37 - 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x38 - 0x00, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x39 - 0x01, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x3A - 0x00, 0x01, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3B - 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, 0x80, // 0x3C - 0x00, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3D - 0x01, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, 0x80, // 0x3E - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x80, 0x80, // 0x3F - 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x40 - 0x00, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x41 - 0x01, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x42 - 0x00, 0x01, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x43 - 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x44 - 0x00, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x45 - 0x01, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x46 - 0x00, 0x01, 0x02, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x47 - 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x48 - 0x00, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x49 - 0x01, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x4A - 0x00, 0x01, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4B - 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x4C - 0x00, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4D - 0x01, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x4E - 0x00, 0x01, 0x02, 0x03, 0x06, 0x80, 0x80, 0x80, // 0x4F - 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x50 - 0x00, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x51 - 0x01, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x52 - 0x00, 0x01, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x53 - 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x54 - 0x00, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x55 - 0x01, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x56 - 0x00, 0x01, 0x02, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x57 - 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x58 - 0x00, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x59 - 0x01, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x5A - 0x00, 0x01, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5B - 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x5C - 0x00, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5D - 0x01, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, 0x80, // 0x5E - 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x80, 0x80, // 0x5F - 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x60 - 0x00, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x61 - 0x01, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x62 - 0x00, 0x01, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x63 - 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x64 - 0x00, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x65 - 0x01, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x66 - 0x00, 0x01, 0x02, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x67 - 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x68 - 0x00, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x69 - 0x01, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x6A - 0x00, 0x01, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6B - 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x6C - 0x00, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6D - 0x01, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x6E - 0x00, 0x01, 0x02, 0x03, 0x05, 0x06, 0x80, 0x80, // 0x6F - 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x70 - 0x00, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x71 - 0x01, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x72 - 0x00, 0x01, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x73 - 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x74 - 0x00, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x75 - 0x01, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x76 - 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x77 - 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, 0x80, // 0x78 - 0x00, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x79 - 0x01, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x7A - 0x00, 0x01, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7B - 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, 0x80, // 0x7C - 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7D - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, 0x80, // 0x7E - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x80, // 0x7F - 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x80 - 0x00, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x81 - 0x01, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x82 - 0x00, 0x01, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x83 - 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x84 - 0x00, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x85 - 0x01, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x86 - 0x00, 0x01, 0x02, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x87 - 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x88 - 0x00, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x89 - 0x01, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x8A - 0x00, 0x01, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8B - 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x8C - 0x00, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8D - 0x01, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x8E - 0x00, 0x01, 0x02, 0x03, 0x07, 0x80, 0x80, 0x80, // 0x8F - 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x90 - 0x00, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x91 - 0x01, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x92 - 0x00, 0x01, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x93 - 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x94 - 0x00, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x95 - 0x01, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x96 - 0x00, 0x01, 0x02, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x97 - 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x98 - 0x00, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x99 - 0x01, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x9A - 0x00, 0x01, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9B - 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, 0x80, // 0x9C - 0x00, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9D - 0x01, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, 0x80, // 0x9E - 0x00, 0x01, 0x02, 0x03, 0x04, 0x07, 0x80, 0x80, // 0x9F - 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA0 - 0x00, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA1 - 0x01, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA2 - 0x00, 0x01, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA3 - 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA4 - 0x00, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA5 - 0x01, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA6 - 0x00, 0x01, 0x02, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xA7 - 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xA8 - 0x00, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xA9 - 0x01, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xAA - 0x00, 0x01, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAB - 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xAC - 0x00, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAD - 0x01, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xAE - 0x00, 0x01, 0x02, 0x03, 0x05, 0x07, 0x80, 0x80, // 0xAF - 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xB0 - 0x00, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB1 - 0x01, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB2 - 0x00, 0x01, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB3 - 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB4 - 0x00, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB5 - 0x01, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB6 - 0x00, 0x01, 0x02, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xB7 - 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xB8 - 0x00, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xB9 - 0x01, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xBA - 0x00, 0x01, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBB - 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, 0x80, // 0xBC - 0x00, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBD - 0x01, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, 0x80, // 0xBE - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x07, 0x80, // 0xBF - 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC0 - 0x00, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC1 - 0x01, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC2 - 0x00, 0x01, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC3 - 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC4 - 0x00, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC5 - 0x01, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC6 - 0x00, 0x01, 0x02, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xC7 - 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xC8 - 0x00, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xC9 - 0x01, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xCA - 0x00, 0x01, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCB - 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xCC - 0x00, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCD - 0x01, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xCE - 0x00, 0x01, 0x02, 0x03, 0x06, 0x07, 0x80, 0x80, // 0xCF - 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xD0 - 0x00, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD1 - 0x01, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD2 - 0x00, 0x01, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD3 - 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD4 - 0x00, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD5 - 0x01, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD6 - 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xD7 - 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xD8 - 0x00, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xD9 - 0x01, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xDA - 0x00, 0x01, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDB - 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xDC - 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDD - 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, 0x80, // 0xDE - 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x80, // 0xDF - 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xE0 - 0x00, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE1 - 0x01, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE2 - 0x00, 0x01, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE3 - 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE4 - 0x00, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE5 - 0x01, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE6 - 0x00, 0x01, 0x02, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xE7 - 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xE8 - 0x00, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xE9 - 0x01, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xEA - 0x00, 0x01, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xEB - 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xEC - 0x00, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xED - 0x01, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xEE - 0x00, 0x01, 0x02, 0x03, 0x05, 0x06, 0x07, 0x80, // 0xEF - 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, // 0xF0 - 0x00, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF1 - 0x01, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF2 - 0x00, 0x01, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF3 - 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF4 - 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF5 - 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF6 - 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xF7 - 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, // 0xF8 - 0x00, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xF9 - 0x01, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xFA - 0x00, 0x01, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFB - 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, // 0xFC - 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFD - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, // 0xFE - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // 0xFF - }; +SZ_HELPER_INLINE sz_size_t sz_utf8_leftpack_offsets_haswell_(sz_u32_t mask, sz_u8_t *out) { __m128i const lane_iota_u8x16 = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); __m128i const constant_eight_u8x16 = _mm_set1_epi8(8); @@ -741,8 +746,10 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_leftpack_offsets_haswell_(sz_u32_t mask, sz_u8_ // Ascending set-bit positions for each 8-bit byte; the high byte's positions are +8 (lanes 8..15). Unused slots // are 0x80 (high bit set → `vpshufb` reads 0, never stored). - __m128i const packed_offsets_low_u8x16 = _mm_loadl_epi64((__m128i const *)(leftpack8 + low8 * 8)); - __m128i const packed_offsets_high_raw_u8x16 = _mm_loadl_epi64((__m128i const *)(leftpack8 + high8 * 8)); + __m128i const packed_offsets_low_u8x16 = _mm_loadl_epi64( + (__m128i const *)(sz_utf8_leftpack8_haswell_ + low8 * 8)); + __m128i const packed_offsets_high_raw_u8x16 = _mm_loadl_epi64( + (__m128i const *)(sz_utf8_leftpack8_haswell_ + high8 * 8)); __m128i const packed_offsets_high_adjusted_u8x16 = _mm_add_epi8(packed_offsets_high_raw_u8x16, _mm_set1_epi8(8)); __m128i const packed_halves_u8x16 = _mm_unpacklo_epi64(packed_offsets_low_u8x16, @@ -776,7 +783,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_leftpack_offsets_haswell_(sz_u32_t mask, sz_u8_ * @return Number of runes emitted; sets @p last_off_out to the last emitted start's window byte-offset (the caller * turns it into the resume cursor by adding that lane's maximal-subpart length). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_haswell_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_haswell_( // __m256i window_u8x32, __m256i ill_formed_lanes_u8x32, sz_u32_t emit_starts, // int has_three, int has_four, int has_ill, // sz_size_t emit_count, sz_rune_t *runes, sz_size_t capacity, sz_u8_t *last_off_out) { @@ -879,9 +886,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_haswell_( * first lead's declared sequence crosses the window edge (a boundary truncation), which the public entry * finalizes without a serial re-decode. The decode is TOTAL: no decline-to-serial. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_haswell_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_haswell_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_size_t const chunk = length < 32 ? length : 32; diff --git a/include/stringzilla/utf8_runes/icelake.h b/include/stringzilla/utf8_runes/icelake.h index 5121c199..47171daa 100644 --- a/include/stringzilla/utf8_runes/icelake.h +++ b/include/stringzilla/utf8_runes/icelake.h @@ -62,7 +62,7 @@ SZ_HELPER_INLINE __mmask64 sz_utf8_rune_start_mask_icelake_(__m512i window_u8x64 * widen-stores. `_mm512_alignr_epi64` shifts the compressed registers down between waves, so @p emit may * exceed 8. Per-lane byte length is 1, plus 1 on a 2-byte start, plus 2 on a 3-byte start (disjoint masks). */ -SZ_HELPER_AUTO void sz_utf8_rune_peel_icelake_( // +SZ_HELPER_INLINE void sz_utf8_rune_peel_icelake_( // sz_u64_t start_bits, __mmask64 two_byte_starts_m64, __mmask64 three_byte_starts_m64, // sz_size_t emit, sz_size_t position, __m512i lane_identity_u8x64, // sz_size_t *match_offsets, sz_size_t *match_lengths) { @@ -110,9 +110,9 @@ SZ_HELPER_INLINE __m512i sz_utf8_srl8_icelake_(__m512i value_u8x64, int shift, s * `high`/`low` holding the low 16 bits, which the caller resolves through arithmetic ranges. */ typedef struct sz_utf8_rune_window_t { - __m512i window; /**< The raw 64 input bytes (continuation bytes included). */ - __m512i high; /**< Per-lane `codepoint >> 8` for the codepoint that starts at this lane. */ - __m512i low; /**< Per-lane `codepoint & 0xFF` for the codepoint that starts at this lane. */ + __m512i window_u8x64; /**< The raw 64 input bytes (continuation bytes included). */ + __m512i high_byte_u8x64; /**< Per-lane `codepoint >> 8` for the codepoint that starts at this lane. */ + __m512i low_byte_u8x64; /**< Per-lane `codepoint & 0xFF` for the codepoint that starts at this lane. */ __mmask64 continuation; /**< Bit `i` set => lane `i` is a UTF-8 continuation byte `10xxxxxx`. */ __mmask64 codepoint_starts; /**< Bit `i` set => lane `i` begins a codepoint (loaded, non-continuation). */ __mmask64 two_byte_starts; /**< Bit `i` set => lane `i` is a 2-byte lead `110xxxxx`. */ @@ -122,13 +122,13 @@ typedef struct sz_utf8_rune_window_t { } sz_utf8_rune_window_t; /** @brief Load up to 64 bytes from @p text (masked tail) and decode every lane into byte-domain halves. */ -SZ_HELPER_AUTO sz_utf8_rune_window_t sz_utf8_rune_decode_window_icelake_( // +SZ_HELPER_INLINE sz_utf8_rune_window_t sz_utf8_rune_decode_window_icelake_( // sz_u8_t const *text, sz_size_t available, __m512i lane_identity_u8x64) { sz_utf8_rune_window_t result; result.loaded = available < 64 ? available : 64; __mmask64 const load_mask_m64 = sz_u64_clamp_mask_until_(result.loaded); __m512i const window_u8x64 = _mm512_maskz_loadu_epi8(load_mask_m64, text); - result.window = window_u8x64; + result.window_u8x64 = window_u8x64; // The three forward neighbours of each lane, gathered via in-register permutes (never `vpgather`). __m512i const next1_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), @@ -162,8 +162,10 @@ SZ_HELPER_AUTO sz_utf8_rune_window_t sz_utf8_rune_decode_window_icelake_( // _mm512_slli_epi16(_mm512_and_si512(next1_u8x64, _mm512_set1_epi8(0x03)), 6), _mm512_and_si512(next2_u8x64, _mm512_set1_epi8(0x3F))); - result.high = _mm512_mask_blend_epi8(result.three_byte_starts, high_two_byte_u8x64, high_three_byte_u16x32); - result.low = _mm512_mask_blend_epi8(result.three_byte_starts, low_two_byte_u16x32, low_three_byte_u16x32); + result.high_byte_u8x64 = _mm512_mask_blend_epi8(result.three_byte_starts, high_two_byte_u8x64, + high_three_byte_u16x32); + result.low_byte_u8x64 = _mm512_mask_blend_epi8(result.three_byte_starts, low_two_byte_u16x32, + low_three_byte_u16x32); return result; } @@ -178,7 +180,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_t sz_utf8_rune_decode_window_icelake_( // * high index bits via masked moves. The final partial page is `maskz`-loaded so an unpadded @p table * is never over-read. Out-of-range lanes (none in valid trie use) read as zero. */ -SZ_HELPER_AUTO __m512i sz_utf8_rune_gather_byte_(sz_u8_t const *table, int count, __m512i indices_u16x32) { +SZ_HELPER_INLINE __m512i sz_utf8_rune_gather_byte_(sz_u8_t const *table, int count, __m512i indices_u16x32) { __m512i const within_u16x32 = _mm512_and_si512(indices_u16x32, _mm512_set1_epi16(0x7F)); __m512i const page_u16x32 = _mm512_srli_epi16(indices_u16x32, 7); int const page_count = (count + 127) / 128; @@ -219,7 +221,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_rune_gather_byte_(sz_u8_t const *table, int count * masked blends. Tiles load directly from `.rodata` (no per-call materialization), so the family * classifiers stay re-init-free. @p table must be `sz_align_(64)` and exactly 256 bytes. */ -SZ_HELPER_AUTO __m512i sz_utf8_rune_permute256_icelake_(sz_u8_t const *table, __m512i index_u32x16) { +SZ_HELPER_INLINE __m512i sz_utf8_rune_permute256_icelake_(sz_u8_t const *table, __m512i index_u32x16) { __m512i const quad0_u8x64 = _mm512_load_si512((void const *)(table + 0 * 64)); __m512i const quad1_u8x64 = _mm512_load_si512((void const *)(table + 1 * 64)); __m512i const quad2_u8x64 = _mm512_load_si512((void const *)(table + 2 * 64)); @@ -254,8 +256,8 @@ SZ_HELPER_AUTO __m512i sz_utf8_rune_permute256_icelake_(sz_u8_t const *table, __ * ! Cost scales with @p tile_count, not with the window: every tile is scanned on the single cross-lane shuffle port. * ! The BMP classifiers use @ref sz_utf8_rune_flat_lookup_icelake_ instead for exactly that reason. */ -SZ_HELPER_AUTO __m512i sz_utf8_rune_lut_cascade_icelake_(sz_u8_t const *table, int tile_count, - __m512i index_dwords_u32x16) { +SZ_HELPER_INLINE __m512i sz_utf8_rune_lut_cascade_icelake_(sz_u8_t const *table, int tile_count, + __m512i index_dwords_u32x16) { __m512i const within_u32x16 = _mm512_and_si512(index_dwords_u32x16, _mm512_set1_epi32(0x7F)); __m512i const selector_u32x16 = _mm512_srli_epi32(index_dwords_u32x16, 7); __m512i result_u32x16 = _mm512_setzero_si512(); @@ -281,8 +283,8 @@ SZ_HELPER_AUTO __m512i sz_utf8_rune_lut_cascade_icelake_(sz_u8_t const *table, i * tiles; @p index_dwords_u32x16 is the unpacked cell index per 32-bit lane. Reads straight from aligned * `.rodata`. */ -SZ_HELPER_AUTO __m512i sz_utf8_rune_lut_cascade_nibble_icelake_(sz_u8_t const *packed, int tile_count, - __m512i index_dwords_u32x16) { +SZ_HELPER_INLINE __m512i sz_utf8_rune_lut_cascade_nibble_icelake_(sz_u8_t const *packed, int tile_count, + __m512i index_dwords_u32x16) { __m512i const byte_index_u32x16 = _mm512_srli_epi32(index_dwords_u32x16, 1); __m512i const packed_byte_u8x64 = sz_utf8_rune_lut_cascade_icelake_(packed, tile_count, byte_index_u32x16); __mmask16 const odd_cell_m16 = _mm512_test_epi32_mask(index_dwords_u32x16, _mm512_set1_epi32(1)); @@ -305,7 +307,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_rune_lut_cascade_nibble_icelake_(sz_u8_t const *p * ! Only the low byte of each lane is the class; the upper three carry gather garbage. Most callers truncate with * ! `vpmovdb` anyway; a caller that keeps the u32 lanes must mask with 0xFF itself. */ -SZ_HELPER_AUTO __m512i sz_utf8_rune_flat_lookup_icelake_( // +SZ_HELPER_INLINE __m512i sz_utf8_rune_flat_lookup_icelake_( // sz_u8_t const *page_lut, sz_u8_t const *flat, __m512i codepoints_u32x16) { __m512i const high_bytes_u32x16 = _mm512_and_si512(_mm512_srli_epi32(codepoints_u32x16, 8), _mm512_set1_epi32(0xFF)); @@ -329,7 +331,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_rune_flat_lookup_icelake_( // * start is the previous boundary position and whose length reaches to `base + i`. Output is widened to 64-bit * `starts[]` / `lengths[]` in waves of eight, carrying the open segment across waves and windows via @p previous_io. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_( // sz_u64_t boundary, sz_size_t base, __m512i lane_identity_u8x64, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { __m512i const wave_shift_u8x64 = _mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(8)); @@ -378,7 +380,7 @@ SZ_HELPER_INLINE __m128i sz_utf8_rune_pick16_icelake_(__m512i value_u8x64, __m51 * U+FFFD. * @return Number of runes emitted; sets @p consumed_bytes to the byte span they cover (the resume cursor delta). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_icelake_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_icelake_( // __m512i window_u8x64, sz_u64_t emit_starts, sz_u64_t ill_formed, __m512i consumed_length_u8x64, __m512i lane_identity_u8x64, int has_three, int has_four, sz_size_t emit_count, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { @@ -545,9 +547,9 @@ SZ_API_COMPTIME sz_cptr_t sz_utf8_seek_icelake(sz_cptr_t text, sz_size_t length, * declines (`*runes_unpacked == 0`, cursor unchanged) ONLY when the first lead's declared sequence crosses * the window edge (a boundary truncation), which the public entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_icelake_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_icelake_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { __m512i const lane_identity_u8x64 = sz_utf8_lane_identity_icelake_(); diff --git a/include/stringzilla/utf8_runes/lasx.h b/include/stringzilla/utf8_runes/lasx.h index 64ebc3c3..5dae8fc9 100644 --- a/include/stringzilla/utf8_runes/lasx.h +++ b/include/stringzilla/utf8_runes/lasx.h @@ -320,7 +320,7 @@ SZ_HELPER_INLINE sz_u8_t const *sz_utf8_pack8_lut_lasx_(void) { * lane. The four packed groups are stitched in order by their `popcount` offset (the group base + local * index gives the absolute byte offset). @return the popcount of @p mask. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_pack_indices_lasx_(sz_u32_t mask, sz_u8_t *out) { +SZ_HELPER_INLINE sz_size_t sz_utf8_pack_indices_lasx_(sz_u32_t mask, sz_u8_t *out) { sz_u8_t const *lut = sz_utf8_pack8_lut_lasx_(); // Lane-local byte identity {0..15, 0..15}: each 128-bit lane shuffles its own 0..15 identity by the LUT row, so // the packed values are the within-half bit positions; we add the half base to recover absolute offsets. @@ -355,7 +355,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_pack_indices_lasx_(sz_u32_t mask, sz_u8_t *out) * lane never skips bytes owing their own next U+FFFD). * @return Number of runes emitted; sets @p consumed_bytes to the byte span they cover. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_lasx_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_lasx_( // __m256i window_u8x32, sz_u32_t emit_starts, sz_u32_t ill_formed, __m256i consumed_length_u8x32, int has_three, int has_four, sz_size_t emit_count, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { @@ -488,9 +488,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_lasx_( // * declines (`*runes_unpacked == 0`, cursor unchanged) ONLY when the first lead's declared sequence crosses * the window edge (a boundary truncation), which the public entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_lasx_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_lasx_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_size_t const chunk = length < 32 ? length : 32; @@ -914,8 +914,8 @@ SZ_HELPER_INLINE __m256i sz_utf8_byte_mask_from_bits_lasx_(sz_u32_t bits) { /** @brief Masked 64-byte load into two halves; bytes [loaded, 64) read as zero (the LASX stand-in for * `_mm512_maskz_loadu_epi8`). A stack staging union covers the partial tail so we never read past * `text + loaded`. */ -SZ_HELPER_AUTO void sz_utf8_load_window_lasx_(sz_u8_t const *text, sz_size_t loaded, __m256i *out_low_u8x32, - __m256i *out_high_u8x32) { +SZ_HELPER_INLINE void sz_utf8_load_window_lasx_(sz_u8_t const *text, sz_size_t loaded, __m256i *out_low_u8x32, + __m256i *out_high_u8x32) { if (loaded >= 64) { *out_low_u8x32 = __lasx_xvld(text + 0, 0); *out_high_u8x32 = __lasx_xvld(text + 32, 0); @@ -962,7 +962,7 @@ SZ_HELPER_INLINE void sz_utf8_forward_neighbours_lasx_( // /** @brief Load up to 64 bytes (masked tail) and decode every lane into byte-domain halves - the LASX twin of * @ref sz_utf8_rune_decode_window_, bit-identical on every lane. */ -SZ_HELPER_AUTO sz_utf8_rune_window_lasx_t sz_utf8_rune_decode_window_lasx_(sz_u8_t const *text, sz_size_t available) { +SZ_HELPER_INLINE sz_utf8_rune_window_lasx_t sz_utf8_rune_decode_window_lasx_(sz_u8_t const *text, sz_size_t available) { sz_utf8_rune_window_lasx_t result; result.loaded = available < 64 ? available : 64; @@ -1054,7 +1054,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_lasx_t sz_utf8_rune_decode_window_lasx_(sz_u8 /** @brief 256-entry byte LUT addressed by a per-lane byte index in `[0,256)`: `result[lane] = table[index[lane]]` via * a bounded scalar L1 walk (LASX has no gather). The LASX stand-in for the substrate `lut256` leaf. */ -SZ_HELPER_AUTO __m256i sz_utf8_rune_lut256_scalar_lasx_(sz_u8_t const *table, __m256i index_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_rune_lut256_scalar_lasx_(sz_u8_t const *table, __m256i index_u8x32) { sz_u256_vec_t index_vec, result_vec; index_vec.lasx = index_u8x32; for (int lane = 0; lane < 32; ++lane) result_vec.u8s[lane] = table[index_vec.u8s[lane]]; @@ -1065,7 +1065,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_rune_lut256_scalar_lasx_(sz_u8_t const *table, __ * (0 when @p selector reaches @p tile_count). Each 16-byte row is double-broadcast into both 128-bit lanes with * `xvpermi.q(row, row, 0x00)` and shuffled by @p within (a nibble, so `xvshuf.b` never crosses the seam), then * blended in for the lanes whose @p selector picks that row - the LASX twin of the AVX2 cascade stage. */ -SZ_HELPER_AUTO __m256i sz_utf8_rune_cascade_stage_lasx_( // +SZ_HELPER_INLINE __m256i sz_utf8_rune_cascade_stage_lasx_( // sz_u8_t const *table, int tile_count, __m256i selector_u8x32, __m256i within_u8x32) { __m256i result_u8x32 = __lasx_xvreplgr2vr_b(0); for (int tile = 0; tile < tile_count; ++tile) { @@ -1082,7 +1082,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_rune_cascade_stage_lasx_( // * `flat[page * 256 + low]` per lane. LASX has no gather, so the leaf read is a bounded scalar L1 walk over the * fused 16-bit index `(page << 8) | low` (the NEON strategy). Lanes whose page index reaches @p page_count * return zero. */ -SZ_HELPER_AUTO __m256i sz_utf8_rune_flat_lookup_lasx_( // +SZ_HELPER_INLINE __m256i sz_utf8_rune_flat_lookup_lasx_( // sz_u8_t const *page_lut, sz_u8_t const *flat, int page_count, __m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { sz_u256_vec_t high_vec, low_vec, result_vec; high_vec.lasx = high_bytes_u8x32; @@ -1109,7 +1109,7 @@ SZ_HELPER_INLINE int sz_utf8_rune_byte_popcount_lasx_(sz_u32_t byte) { /** @brief Left-pack the set lane indices (in [0, 64), ascending) of a 64-bit @p mask into @p out[0..popcount) via the * existing 256-row @ref sz_utf8_pack8_lut_lasx_ + `xvshuf.b` (eight 8-bit groups), NOT a scalar `ctz` walk. * @return the popcount of @p mask. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_pack_boundaries_lasx_(sz_u64_t mask, sz_u8_t *out) { +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_pack_boundaries_lasx_(sz_u64_t mask, sz_u8_t *out) { sz_u8_t const *lut = sz_utf8_pack8_lut_lasx_(); static sz_u8_t const identity_bytes[32] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; @@ -1134,7 +1134,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_pack_boundaries_lasx_(sz_u64_t mask, sz_u8 * via @p previous_io; bit-exact with the Ice Lake leaf. The set lanes are left-packed once (pack8 LUT), then * streamed in waves of four u64 positions (`xvperm.w` shift + lane-0 carry seat via `xvinsgr2vr.d`, lengths via * `xvsub.d`), with a scalar tail for the final partial wave. Count is `xvpcnt.d`, never a scalar `popcount`. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_lasx_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_lasx_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { sz_size_t const boundary_count = sz_utf8_rune_popcount64_lasx_(boundary); diff --git a/include/stringzilla/utf8_runes/neon.h b/include/stringzilla/utf8_runes/neon.h index af8942f4..6aaff60f 100644 --- a/include/stringzilla/utf8_runes/neon.h +++ b/include/stringzilla/utf8_runes/neon.h @@ -103,15 +103,15 @@ SZ_API_COMPTIME sz_cptr_t sz_utf8_seek_neon(sz_cptr_t text, sz_size_t length, sz * `__mmask64`. Field names and semantics match @ref sz_utf8_rune_window_t so the portable rule * algebra is unchanged. */ typedef struct sz_utf8_rune_window_neon_t { - uint8x16_t window[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ - uint8x16_t high[4]; /**< Per-lane `codepoint >> 8`. */ - uint8x16_t low[4]; /**< Per-lane `codepoint & 0xFF`. */ - sz_u64_t continuation; /**< Bit `i` => lane `i` is a continuation byte `10xxxxxx`. */ - sz_u64_t codepoint_starts; /**< Bit `i` => lane `i` begins a codepoint (loaded, non-continuation). */ - sz_u64_t two_byte_starts; /**< Bit `i` => lane `i` is a 2-byte lead `110xxxxx`. */ - sz_u64_t three_byte_starts; /**< Bit `i` => lane `i` is a 3-byte lead `1110xxxx`. */ - sz_u64_t four_byte_starts; /**< Bit `i` => lane `i` is a 4-byte lead `11110xxx`. */ - sz_size_t loaded; /**< Number of bytes actually loaded (<= 64). */ + uint8x16_t window_u8x16s[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ + uint8x16_t high_byte_u8x16s[4]; /**< Per-lane `codepoint >> 8`. */ + uint8x16_t low_byte_u8x16s[4]; /**< Per-lane `codepoint & 0xFF`. */ + sz_u64_t continuation; /**< Bit `i` => lane `i` is a continuation byte `10xxxxxx`. */ + sz_u64_t codepoint_starts; /**< Bit `i` => lane `i` begins a codepoint (loaded, non-continuation). */ + sz_u64_t two_byte_starts; /**< Bit `i` => lane `i` is a 2-byte lead `110xxxxx`. */ + sz_u64_t three_byte_starts; /**< Bit `i` => lane `i` is a 3-byte lead `1110xxxx`. */ + sz_u64_t four_byte_starts; /**< Bit `i` => lane `i` is a 4-byte lead `11110xxx`. */ + sz_size_t loaded; /**< Number of bytes actually loaded (<= 64). */ } sz_utf8_rune_window_neon_t; /** @brief Per-byte logical right shift by @p shift keeping the low @p keep bits — the NEON twin of `srl8_`. @@ -159,7 +159,7 @@ SZ_HELPER_INLINE sz_u64_t sz_utf8_mask_combine_neon_( // /** @brief Masked 64-byte load into four quarters; bytes [loaded, 64) read as zero (the NEON stand-in for * `_mm512_maskz_loadu_epi8`). A zero-initialized vector union stages the partial tail so we never read past * `text + loaded`. Mirrors @ref sz_utf8_load_window_haswell_. */ -SZ_HELPER_AUTO void sz_utf8_load_window_neon_(sz_u8_t const *text, sz_size_t loaded, uint8x16_t *out_u8x16) { +SZ_HELPER_INLINE void sz_utf8_load_window_neon_(sz_u8_t const *text, sz_size_t loaded, uint8x16_t *out_u8x16) { if (loaded >= 64) { out_u8x16[0] = vld1q_u8(text + 0); out_u8x16[1] = vld1q_u8(text + 16); @@ -184,7 +184,7 @@ SZ_HELPER_AUTO void sz_utf8_load_window_neon_(sz_u8_t const *text, sz_size_t loa * quarter wraps to quarter 0 (byte 64 aliases byte 0). The three neighbour distances are provided because * the family classifiers need up to `next3_u8x16` (4-byte sequences). */ -SZ_HELPER_AUTO void sz_utf8_forward_neighbours_neon_( // +SZ_HELPER_INLINE void sz_utf8_forward_neighbours_neon_( // uint8x16_t const *window_u8x16, uint8x16_t *next1_u8x16, uint8x16_t *next2_u8x16, uint8x16_t *next3_u8x16) { for (int quarter = 0; quarter < 4; ++quarter) { uint8x16_t const here_u8x16 = window_u8x16[quarter]; @@ -197,14 +197,14 @@ SZ_HELPER_AUTO void sz_utf8_forward_neighbours_neon_( // /** @brief Load up to 64 bytes (masked tail) and decode every lane into byte-domain halves — the NEON twin of * @ref sz_utf8_rune_decode_window_, bit-identical to it (and to `_haswell_`) on every lane. */ -SZ_HELPER_AUTO sz_utf8_rune_window_neon_t sz_utf8_rune_decode_window_neon_( // +SZ_HELPER_INLINE sz_utf8_rune_window_neon_t sz_utf8_rune_decode_window_neon_( // sz_u8_t const *text, sz_size_t available) { sz_utf8_rune_window_neon_t result; result.loaded = available < 64 ? available : 64; uint8x16_t window_u8x16[4]; sz_utf8_load_window_neon_(text, result.loaded, window_u8x16); - for (int quarter = 0; quarter < 4; ++quarter) result.window[quarter] = window_u8x16[quarter]; + for (int quarter = 0; quarter < 4; ++quarter) result.window_u8x16s[quarter] = window_u8x16[quarter]; uint8x16_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; sz_utf8_forward_neighbours_neon_(window_u8x16, next1_u8x16, next2_u8x16, next3_u8x16); @@ -262,8 +262,8 @@ SZ_HELPER_AUTO sz_utf8_rune_window_neon_t sz_utf8_rune_decode_window_neon_( // // Blend 2-byte vs 3-byte per lane: select the 3-byte value where this lane is a 3-byte lead. uint8x16_t const three_select_u8x16 = three_byte_bool_u8x16[quarter]; - result.high[quarter] = vbslq_u8(three_select_u8x16, high_three_u8x16, high_two_u8x16); - result.low[quarter] = vbslq_u8(three_select_u8x16, low_three_u8x16, low_two_u8x16); + result.high_byte_u8x16s[quarter] = vbslq_u8(three_select_u8x16, high_three_u8x16, high_two_u8x16); + result.low_byte_u8x16s[quarter] = vbslq_u8(three_select_u8x16, low_three_u8x16, low_two_u8x16); } return result; } @@ -274,7 +274,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_neon_t sz_utf8_rune_decode_window_neon_( // * loaded into a `uint8x16_t` and shuffled by @p within_u8x16 via `vqtbl1q_u8`, then blended in for the lanes * whose @p selector_u8x16 picks that row. Gather-free — only `vld1q`/`vqtbl1q`/`vceqq`/`vbslq`. * @p within_u8x16 / @p selector_u8x16 address one quarter; the caller iterates the four quarters. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_rune_cascade_stage_neon_( // +SZ_HELPER_INLINE uint8x16_t sz_utf8_rune_cascade_stage_neon_( // sz_u8_t const *table, int tile_count, uint8x16_t selector_u8x16, uint8x16_t within_u8x16) { // Each lane reads table[selector[lane] * 16 + within[lane]] when selector < tile_count, else 0 (no tile matched). // The original linear scan accumulates through a `tile_count`-deep `vbslq` chain over 16-byte rows, which is a @@ -298,7 +298,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_rune_cascade_stage_neon_( // * Four `vqtbl4q_u8` reads over the four resident 64-byte quads; `vqtbl4q_u8` returns zero for indices >= 64, * so subtracting 64/128/192 routes each lane to exactly one quad and the four results OR together. The NEON * twin of the substrate `lut256` leaf. @p index_u8x16 addresses one quarter. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_rune_lut256_neon_(sz_u8_t const *group_base, uint8x16_t index_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_rune_lut256_neon_(sz_u8_t const *group_base, uint8x16_t index_u8x16) { uint8x16x4_t const quad0_u8x16x4 = vld1q_u8_x4(group_base + 0 * 64); uint8x16x4_t const quad1_u8x16x4 = vld1q_u8_x4(group_base + 1 * 64); uint8x16x4_t const quad2_u8x16x4 = vld1q_u8_x4(group_base + 2 * 64); @@ -327,7 +327,7 @@ SZ_HELPER_INLINE uint8x16_t sz_utf8_rune_lut64_neon_(sz_u8_t const *group_base, * so the page LUT resolves in-register while the leaf read is a bounded scalar L1 walk over fused 16-bit * indices `(page << 8) | low`. Lanes whose page index reaches @p page_count return zero. SVE2 does the same * lookup with a real `LD1B` gather; see @ref sz_utf8_rune_flat_lookup_sve2_. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_rune_flat_lookup_neon_( // +SZ_HELPER_INLINE uint8x16_t sz_utf8_rune_flat_lookup_neon_( // sz_u8_t const *page_lut, sz_u8_t const *flat, int page_count, uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { uint8x16_t const page_indices_u8x16 = sz_utf8_rune_lut256_neon_(page_lut, high_bytes_u8x16); @@ -353,7 +353,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_rune_flat_lookup_neon_( // * NEON has no `vpcompressb`/`pext`: pull the lowest set bit's index via `ctz`, clear it via `mask & (mask-1)`. * This matches the AVX2 `sz_utf8_unpack_indices_haswell_` BMI2 path semantically; on AArch64 * `ctz` lowers to `rbit`+`clz`, and the loop trip count is the boundary popcount (sparse for real text). */ -SZ_HELPER_AUTO void sz_utf8_unpack_indices_neon_(sz_u64_t mask, sz_u8_t *out) { +SZ_HELPER_INLINE void sz_utf8_unpack_indices_neon_(sz_u64_t mask, sz_u8_t *out) { while (mask) { *out++ = (sz_u8_t)sz_u64_ctz_neon_(mask); mask &= mask - 1; // clear the lowest set bit @@ -381,7 +381,7 @@ SZ_HELPER_INLINE uint8x16_t sz_utf8_expand16_neon_(sz_u32_t submask) { return vceqq_u8(vandq_u8(per_half_u8x16, bit_position_u8x16), bit_position_u8x16); } -SZ_HELPER_AUTO sz_size_t sz_utf8_leftpack_offsets_neon_(sz_u64_t mask, sz_u8_t *out) { +SZ_HELPER_INLINE sz_size_t sz_utf8_leftpack_offsets_neon_(sz_u64_t mask, sz_u8_t *out) { static sz_u8_t const leftpack8[256 * 8] = { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x01 @@ -679,7 +679,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_leftpack_offsets_neon_(sz_u64_t mask, sz_u8_t * * (start, length) per set boundary lane (ascending), honoring @p capacity and the carried previous-boundary * via @p previous_io; bit-exact with the Ice Lake leaf. Indices are unpacked once, then each segment's * (start, length) is computed from the carried previous boundary. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_neon_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_neon_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { sz_size_t const boundary_count = (sz_size_t)sz_u64_popcount_neon_(boundary); @@ -728,7 +728,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_neon_( // * @param consumed_bytes Set to the byte span the emitted runes cover (the resume cursor delta). * @return Number of runes emitted (<= min(emit_count, capacity)). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_neon_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_neon_( // uint8x16_t const *window_u8x16, sz_u64_t emit_starts, uint8x16_t const *ill_byte_u8x16, // int has_three, int has_four, sz_u8_t const *consumed_length, // sz_size_t emit_count, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { @@ -881,8 +881,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_neon_( * * @return Number of runes emitted (0 => tile declined); sets @p consumed_bytes to `clean * 3` when it emits. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_tile3_neon_( // - sz_u8_t const *text, sz_size_t length, // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_tile3_neon_( // + sz_u8_t const *text, sz_size_t length, // sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { if (length < 48 || capacity == 0 || (text[0] & 0xF0) != 0xE0) return 0; @@ -948,9 +948,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_tile3_neon_( // * surrogate / out-of-range / framing) or truncated-only window declines (`*runes_unpacked == 0`, cursor * unchanged) and the public entry hands the remainder to the serial reference (the U+FFFD oracle). */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_neon_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_neon_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_size_t const chunk = length < 64 ? length : 64; diff --git a/include/stringzilla/utf8_runes/powervsx.h b/include/stringzilla/utf8_runes/powervsx.h index efa6f60d..44cf05df 100644 --- a/include/stringzilla/utf8_runes/powervsx.h +++ b/include/stringzilla/utf8_runes/powervsx.h @@ -418,7 +418,7 @@ SZ_HELPER_INLINE __vector unsigned char sz_utf8_gather16_powervsx_( // * analogue: split @p emit16 into low8 / high8, `vec_perm` the byte-index identity vector by the matching * 2 KB shuffle-LUT rows, add the register base @p base, and stitch the two halves by `popcount(low8)`. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_compress_starts_powervsx_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_compress_starts_powervsx_( // sz_u32_t emit16, int base, sz_u8_t *packed, sz_size_t produced) { sz_u32_t const low8 = emit16 & 0xFFu; sz_u32_t const high8 = (emit16 >> 8) & 0xFFu; @@ -458,7 +458,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_compress_starts_powervsx_( // * length), so an ill-formed trailing lane never skips bytes owed their own next U+FFFD. * @return Number of runes emitted; sets @p consumed_bytes to the byte span they cover (the resume cursor delta). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_powervsx_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_powervsx_( // __vector unsigned char const *regs_u8x16, sz_u64_t emit_starts, sz_u64_t ill_formed, sz_u8_t const *consumed_length, int has_three, int has_four, sz_size_t emit_count, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { @@ -582,9 +582,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_powervsx_( // * unchanged) ONLY when the first lead's declared sequence crosses the window edge (a boundary truncation), * which the public entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_powervsx_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_powervsx_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_size_t const chunk = length < 64 ? length : 64; @@ -891,10 +891,10 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_powervsx_( // * per-lane byte-domain codepoint halves `high`/`low` share that shape. Masks are `sz_u64_t` (one bit per * byte-lane, quarter `q` at bit positions [16*q, 16*q+16)). */ typedef struct sz_utf8_rune_window_powervsx_t { - __vector unsigned char window[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ - __vector unsigned char high[4]; /**< Per-lane `codepoint >> 8`. */ - __vector unsigned char low[4]; /**< Per-lane `codepoint & 0xFF`. */ - sz_u64_t continuation; /**< Bit `i` is set when lane `i` is a continuation byte `10xxxxxx`. */ + __vector unsigned char window_u8x16s[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ + __vector unsigned char high_byte_u8x16s[4]; /**< Per-lane `codepoint >> 8`. */ + __vector unsigned char low_byte_u8x16s[4]; /**< Per-lane `codepoint & 0xFF`. */ + sz_u64_t continuation; /**< Bit `i` is set when lane `i` is a continuation byte `10xxxxxx`. */ sz_u64_t codepoint_starts; /**< Bit `i` is set when lane `i` begins a codepoint (loaded, non-continuation). */ sz_u64_t two_byte_starts; /**< Bit `i` is set when lane `i` is a 2-byte lead `110xxxxx`. */ sz_u64_t three_byte_starts; /**< Bit `i` is set when lane `i` is a 3-byte lead `1110xxxx`. */ @@ -912,8 +912,8 @@ SZ_HELPER_INLINE __vector unsigned char sz_utf8_srl8_powervsx_(__vector unsigned /** @brief Masked 64-byte load into four quarters; bytes [loaded, 64) read as zero, staged through a zeroed * `sz_u512_vec_t` byte buffer so no read runs past `text + loaded` (the sanctioned union tail idiom). * Mirrors @ref sz_utf8_load_window_neon_. */ -SZ_HELPER_AUTO void sz_utf8_load_window_powervsx_(sz_u8_t const *text, sz_size_t loaded, - __vector unsigned char *out_u8x16) { +SZ_HELPER_INLINE void sz_utf8_load_window_powervsx_(sz_u8_t const *text, sz_size_t loaded, + __vector unsigned char *out_u8x16) { if (loaded >= 64) { out_u8x16[0] = vec_xl(0, text + 0); out_u8x16[1] = vec_xl(0, text + 16); @@ -935,7 +935,7 @@ SZ_HELPER_AUTO void sz_utf8_load_window_powervsx_(sz_u8_t const *text, sz_size_t * `r` is stitched to its successor `(r+1) & 3` by one `vec_perm` per distance over the register pair * `{window[r], window[(r+1)&3]}`, indexed by `{k, k+1, ..., k+15}` (index `>= 16` reads the successor). * Requires the zero-padded window so wrapped lanes past `loaded` are deterministic zeros. */ -SZ_HELPER_AUTO void sz_utf8_forward_neighbours_powervsx_( // +SZ_HELPER_INLINE void sz_utf8_forward_neighbours_powervsx_( // __vector unsigned char const *window_u8x16, __vector unsigned char *next1_u8x16, __vector unsigned char *next2_u8x16, __vector unsigned char *next3_u8x16) { __vector unsigned char const index_next1_u8x16 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; @@ -952,14 +952,14 @@ SZ_HELPER_AUTO void sz_utf8_forward_neighbours_powervsx_( // /** @brief Load up to 64 bytes (masked tail) and decode every lane into byte-domain halves, the VSX twin of * @ref sz_utf8_rune_decode_window_neon_, bit-identical on every lane. */ -SZ_HELPER_AUTO sz_utf8_rune_window_powervsx_t sz_utf8_rune_decode_window_powervsx_( // +SZ_HELPER_INLINE sz_utf8_rune_window_powervsx_t sz_utf8_rune_decode_window_powervsx_( // sz_u8_t const *text, sz_size_t available) { sz_utf8_rune_window_powervsx_t result; result.loaded = available < 64 ? available : 64; __vector unsigned char window_u8x16[4]; sz_utf8_load_window_powervsx_(text, result.loaded, window_u8x16); - for (int quarter = 0; quarter < 4; ++quarter) result.window[quarter] = window_u8x16[quarter]; + for (int quarter = 0; quarter < 4; ++quarter) result.window_u8x16s[quarter] = window_u8x16[quarter]; __vector unsigned char next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; sz_utf8_forward_neighbours_powervsx_(window_u8x16, next1_u8x16, next2_u8x16, next3_u8x16); @@ -1022,8 +1022,8 @@ SZ_HELPER_AUTO sz_utf8_rune_window_powervsx_t sz_utf8_rune_decode_window_powervs // Blend 2-byte versus 3-byte per lane: select the 3-byte value where this lane is a 3-byte lead. __vector bool char const three_select_u8x16 = (__vector bool char)three_byte_bool_u8x16[quarter]; - result.high[quarter] = vec_sel(high_two_u8x16, high_three_u8x16, three_select_u8x16); - result.low[quarter] = vec_sel(low_two_u8x16, low_three_u8x16, three_select_u8x16); + result.high_byte_u8x16s[quarter] = vec_sel(high_two_u8x16, high_three_u8x16, three_select_u8x16); + result.low_byte_u8x16s[quarter] = vec_sel(low_two_u8x16, low_three_u8x16, three_select_u8x16); } return result; } @@ -1033,7 +1033,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_powervsx_t sz_utf8_rune_decode_window_powervs * Each 16-byte row is one selector value: it is `vec_perm`-gathered by the nibble @p within over its own * register pair `{row, row}` and blended in for the lanes whose @p selector equals that row. Selectors past * the table match no row, so those lanes stay 0 (the NEON `< tile_count` clamp). Gather-free. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_cascade_stage_powervsx_( // +SZ_HELPER_INLINE __vector unsigned char sz_utf8_rune_cascade_stage_powervsx_( // sz_u8_t const *table, int tile_count, __vector unsigned char selector_u8x16, __vector unsigned char within_u8x16) { __vector unsigned char const within_nibble_u8x16 = vec_and(within_u8x16, vec_splats((unsigned char)0x0F)); __vector unsigned char result_u8x16 = vec_splats((unsigned char)0); @@ -1049,8 +1049,8 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_cascade_stage_powervsx_( // /** @brief 256-entry byte LUT addressed by a per-lane byte index in [0, 256): `result[lane] = group_base[index[lane]]`, * the VSX twin of @ref sz_utf8_rune_lut256_neon_. VSX `vec_perm` reaches only 32 bytes, so the 256-byte * table is read by a bounded scalar L1 walk staged through a `sz_u128_vec_t` union. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_lut256_powervsx_(sz_u8_t const *group_base, - __vector unsigned char index_u8x16) { +SZ_HELPER_INLINE __vector unsigned char sz_utf8_rune_lut256_powervsx_(sz_u8_t const *group_base, + __vector unsigned char index_u8x16) { sz_u128_vec_t index_vec, result_vec; index_vec.vsx_u8 = index_u8x16; for (int lane = 0; lane < 16; ++lane) result_vec.u8s[lane] = group_base[index_vec.u8s[lane]]; @@ -1061,7 +1061,7 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_lut256_powervsx_(sz_u8_t cons * `page_lut[high]` selects one 256-byte page, then `flat[page*256 + low]` is read per lane. VSX has no * gather, so the whole lookup is a bounded scalar L1 walk over the fused index; lanes whose page reaches * @p page_count return 0. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_flat_lookup_powervsx_( // +SZ_HELPER_INLINE __vector unsigned char sz_utf8_rune_flat_lookup_powervsx_( // sz_u8_t const *page_lut, sz_u8_t const *flat, int page_count, __vector unsigned char high_bytes_u8x16, __vector unsigned char low_bytes_u8x16) { sz_u128_vec_t high_vec, low_vec, result_vec; @@ -1079,7 +1079,7 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_rune_flat_lookup_powervsx_( // * carried previous boundary @p previous_io. The set boundary lanes are left-packed to ascending byte * offsets by the shuffle-LUT compaction (no scalar `ctz` walk); that compaction's return value is the * boundary count, so no popcount is needed either. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_powervsx_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_powervsx_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { sz_size_t previous = *previous_io; diff --git a/include/stringzilla/utf8_runes/rvv.h b/include/stringzilla/utf8_runes/rvv.h index 1c3f57f0..c4a4e361 100644 --- a/include/stringzilla/utf8_runes/rvv.h +++ b/include/stringzilla/utf8_runes/rvv.h @@ -28,7 +28,7 @@ extern "C" { * straight into the rune output; any non-ASCII byte takes a single serial `sz_rune_decode` step. */ /** @brief Widen `count` ASCII bytes (u8 -> u16 -> u32) and store them as runes. */ -SZ_HELPER_AUTO void sz_utf8_decode_ascii_run_rvv_(sz_rune_t *runes_out, sz_u8_t const *src, sz_size_t count) { +SZ_HELPER_INLINE void sz_utf8_decode_ascii_run_rvv_(sz_rune_t *runes_out, sz_u8_t const *src, sz_size_t count) { sz_size_t done = 0; while (done < count) { sz_size_t widened_vector_length = __riscv_vsetvl_e8m2(count - done); @@ -58,7 +58,7 @@ SZ_HELPER_AUTO void sz_utf8_decode_ascii_run_rvv_(sz_rune_t *runes_out, sz_u8_t * @param consumed_bytes Set to `2 * runes_emitted` (the byte span of the decoded prefix). * @return Number of runes emitted (0 if the very first pair is not a well-formed 2-byte sequence). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_decode_two_byte_run_rvv_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_decode_two_byte_run_rvv_( // sz_cptr_t text, sz_size_t length, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { sz_u8_t const *bytes = (sz_u8_t const *)text; @@ -114,7 +114,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_decode_two_byte_run_rvv_( // * @param consumed_bytes Set to `3 * runes_emitted` (the byte span of the decoded prefix). * @return Number of runes emitted (0 if the very first triple is not a well-formed 3-byte sequence). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_decode_three_byte_run_rvv_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_decode_three_byte_run_rvv_( // sz_cptr_t text, sz_size_t length, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { sz_u8_t const *bytes = (sz_u8_t const *)text; @@ -199,7 +199,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_decode_three_byte_run_rvv_( // * @param consumed_bytes Set to the byte span the emitted runes cover (the resume-cursor delta). * @return Number of runes emitted. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_rvv_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_rvv_( // vuint8m1_t window_bytes_u8m1, vbool8_t emit_mask_b8, vbool8_t ill_mask_b8, vuint8m1_t consumed_length_u8m1, sz_size_t decodable, sz_size_t vector_length, sz_rune_t *runes, sz_size_t capacity, sz_size_t *consumed_bytes) { @@ -288,9 +288,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_rvv_( // * (`*runes_unpacked == 0`, cursor unchanged) ONLY when the first lead's declared sequence crosses the window * edge (a boundary truncation), which the public entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_rvv_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_rvv_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { // Cap the window at 192 bytes so every lane index, length, and `lane + length` stays exact in the `u8` domain @@ -684,8 +684,8 @@ SZ_HELPER_INLINE vuint8m4_t sz_utf8_rune_forward_neighbour_rvv_(vuint8m4_t windo * @ref sz_utf8_rune_decode_window_neon_. The raw window vector arrives from the driver's single * @ref sz_utf8_rune_load64_rvv_ materialization; the BMP halves are recomputed in-leaf via * @ref sz_utf8_rune_bmp_halves_rvv_, so no byte array is ever staged. */ -SZ_HELPER_AUTO sz_utf8_rune_window_rvv_t sz_utf8_rune_decode_window_rvv_(vuint8m4_t const raw_u8m4, - sz_size_t const loaded) { +SZ_HELPER_INLINE sz_utf8_rune_window_rvv_t sz_utf8_rune_decode_window_rvv_(vuint8m4_t const raw_u8m4, + sz_size_t const loaded) { sz_u64_t const loaded_mask = sz_u64_mask_until_serial_(loaded); sz_utf8_rune_window_rvv_t window; window.loaded = loaded; @@ -709,7 +709,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_rvv_t sz_utf8_rune_decode_window_rvv_(vuint8m * register tuple `{high, low}` — 2-/3-byte reconstruction merged on the 3-byte-lead mask, bit-identical * to the NEON decode. ASCII and 4-byte lanes carry don't-cares, exactly like NEON. Recomputed in each * consuming leaf instead of threaded, so no 8-register liveness spans the leaves. */ -SZ_HELPER_AUTO vuint8m4x2_t sz_utf8_rune_bmp_halves_rvv_(vuint8m4_t const raw_u8m4) { +SZ_HELPER_INLINE vuint8m4x2_t sz_utf8_rune_bmp_halves_rvv_(vuint8m4_t const raw_u8m4) { vuint8m4_t const next1_u8m4 = sz_utf8_rune_forward_neighbour_rvv_(raw_u8m4, 1); vuint8m4_t const next2_u8m4 = sz_utf8_rune_forward_neighbour_rvv_(raw_u8m4, 2); vbool2_t const three_byte_b2 = __riscv_vmseq_vx_u8m4_b2(__riscv_vand_vx_u8m4(raw_u8m4, 0xF0, 64), 0xE0, 64); @@ -737,7 +737,7 @@ SZ_HELPER_AUTO vuint8m4x2_t sz_utf8_rune_bmp_halves_rvv_(vuint8m4_t const raw_u8 * total over the byte domain, so every lane is in-bounds by construction), then `flat[(page << 8) | low]` * by a masked `vluxei16` gather. @p inactive_u8m4 rides through on masked-off lanes, which perform no * memory access. Index safety never depends on @p active_b2. */ -SZ_HELPER_AUTO vuint8m4_t sz_utf8_rune_flat_lookup_rvv_( // +SZ_HELPER_INLINE vuint8m4_t sz_utf8_rune_flat_lookup_rvv_( // sz_u8_t const *page_lut, sz_u8_t const *flat, vuint8m4_t high_u8m4, vuint8m4_t low_u8m4, vbool2_t active_b2, vuint8m4_t inactive_u8m4) { vuint8m4_t const page_u8m4 = __riscv_vluxei8_v_u8m4(page_lut, high_u8m4, 64); @@ -750,7 +750,7 @@ SZ_HELPER_AUTO vuint8m4_t sz_utf8_rune_flat_lookup_rvv_( // * set boundary lanes compress to dense u16 indices (`vid` + `vcompress`), widen to u64 absolute * positions, and emit as a shifted-difference stream: `starts = vslide1up(positions, previous)`, * `lengths = positions - starts`, honoring @p capacity and the carried open-word start @p previous_io. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_rvv_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_rvv_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { if (!boundary || produced >= capacity) return produced; diff --git a/include/stringzilla/utf8_runes/serial.h b/include/stringzilla/utf8_runes/serial.h index 62db909e..826440d7 100644 --- a/include/stringzilla/utf8_runes/serial.h +++ b/include/stringzilla/utf8_runes/serial.h @@ -13,9 +13,14 @@ extern "C" { #endif #pragma region Rune Codec +/** @brief The unsigned value of the byte at @p offset. A `char const *` cannot be reinterpreted as + * `sz_u8_t const *` inside a constant expression, and that qualifier is what carries these + * helpers into device code, so the sign is taken off one byte at a time instead. */ +SZ_HELPER_AUTO sz_u8_t sz_utf8_byte_at_(sz_cptr_t utf8, sz_size_t offset) { return (sz_u8_t)utf8[offset]; } + /** @brief Whether @p byte is a UTF-8 continuation byte (`0x80..0xBF`). The single low-level predicate every decode * path shares, so `sz_rune_decode` and `sz_utf8_maximal_subpart_` can never disagree on validity. */ -SZ_HELPER_INLINE sz_bool_t sz_utf8_is_continuation_(sz_u8_t byte) { return (sz_bool_t)((byte & 0xC0) == 0x80); } +SZ_HELPER_AUTO sz_bool_t sz_utf8_is_continuation_(sz_u8_t byte) { return (sz_bool_t)((byte & 0xC0) == 0x80); } /** @brief Whether @p second is a valid @b first continuation for lead byte @p lead: a continuation byte that also * satisfies the E0/ED/F0/F4 overlong/surrogate/range constraint (Unicode Table 3-7). For C2..DF and the @@ -36,31 +41,33 @@ SZ_HELPER_AUTO sz_bool_t sz_utf8_first_continuation_ok_(sz_u8_t lead, sz_u8_t se * single authority for "is this a foldable/normalizable rune"; the decode-side mirror of `sz_rune_encode`. * On failure use `sz_utf8_maximal_subpart_` for how many bytes the resulting U+FFFD consumes. */ SZ_HELPER_AUTO sz_rune_length_t sz_rune_decode(sz_cptr_t utf8, sz_cptr_t utf8_end, sz_rune_t *rune) { - sz_u8_t const *u = (sz_u8_t const *)utf8; - sz_size_t const available = (sz_size_t)((sz_u8_t const *)utf8_end - u); - sz_u8_t const lead = u[0]; + sz_size_t const available = (sz_size_t)(utf8_end - utf8); + sz_u8_t const lead = sz_utf8_byte_at_(utf8, 0); if (lead < 0x80) { *rune = lead; return sz_rune_1byte_k; } if (lead < 0xC2) return sz_rune_invalid_k; // continuation byte, or C0/C1 (overlong 2-byte) if (lead < 0xE0) { // C2..DF - if (available < 2 || !sz_utf8_first_continuation_ok_(lead, u[1])) return sz_rune_invalid_k; - *rune = (sz_rune_t)(lead & 0x1F) << 6 | (u[1] & 0x3F); + if (available < 2 || !sz_utf8_first_continuation_ok_(lead, sz_utf8_byte_at_(utf8, 1))) return sz_rune_invalid_k; + *rune = (sz_rune_t)(lead & 0x1F) << 6 | (sz_utf8_byte_at_(utf8, 1) & 0x3F); return sz_rune_2bytes_k; } if (lead < 0xF0) { // E0..EF - if (available < 3 || !sz_utf8_first_continuation_ok_(lead, u[1]) || !sz_utf8_is_continuation_(u[2])) + if (available < 3 || !sz_utf8_first_continuation_ok_(lead, sz_utf8_byte_at_(utf8, 1)) || + !sz_utf8_is_continuation_(sz_utf8_byte_at_(utf8, 2))) return sz_rune_invalid_k; - *rune = (sz_rune_t)(lead & 0x0F) << 12 | (sz_rune_t)(u[1] & 0x3F) << 6 | (u[2] & 0x3F); + *rune = (sz_rune_t)(lead & 0x0F) << 12 | (sz_rune_t)(sz_utf8_byte_at_(utf8, 1) & 0x3F) << 6 | + (sz_utf8_byte_at_(utf8, 2) & 0x3F); return sz_rune_3bytes_k; } if (lead <= 0xF4) { // F0..F4 - if (available < 4 || !sz_utf8_first_continuation_ok_(lead, u[1]) || !sz_utf8_is_continuation_(u[2]) || - !sz_utf8_is_continuation_(u[3])) + if (available < 4 || !sz_utf8_first_continuation_ok_(lead, sz_utf8_byte_at_(utf8, 1)) || + !sz_utf8_is_continuation_(sz_utf8_byte_at_(utf8, 2)) || + !sz_utf8_is_continuation_(sz_utf8_byte_at_(utf8, 3))) return sz_rune_invalid_k; - *rune = (sz_rune_t)(lead & 0x07) << 18 | (sz_rune_t)(u[1] & 0x3F) << 12 | (sz_rune_t)(u[2] & 0x3F) << 6 | - (u[3] & 0x3F); + *rune = (sz_rune_t)(lead & 0x07) << 18 | (sz_rune_t)(sz_utf8_byte_at_(utf8, 1) & 0x3F) << 12 | + (sz_rune_t)(sz_utf8_byte_at_(utf8, 2) & 0x3F) << 6 | (sz_utf8_byte_at_(utf8, 3) & 0x3F); return sz_rune_4bytes_k; } return sz_rune_invalid_k; // F5..FF @@ -92,16 +99,16 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_serial_( // } SZ_HELPER_AUTO sz_size_t sz_utf8_maximal_subpart_(sz_cptr_t utf8, sz_cptr_t utf8_end) { - sz_u8_t const *u = (sz_u8_t const *)utf8; - sz_size_t const available = (sz_size_t)((sz_u8_t const *)utf8_end - u); - sz_u8_t const lead = u[0]; + sz_size_t const available = (sz_size_t)(utf8_end - utf8); + sz_u8_t const lead = sz_utf8_byte_at_(utf8, 0); // A bad lead is its own 1-byte subpart: stray continuation (< 0xC2), C0/C1 overlong, F5..FF out of range. if (lead < 0xC2 || lead > 0xF4) return 1; // 2/3/4-byte leads: count leading bytes consistent with a well-formed sequence, stopping at the first break. - if (available < 2 || !sz_utf8_first_continuation_ok_(lead, u[1])) return 1; // byte 1 breaks it (incl. C2..DF) - if (lead < 0xE0) return 1; // C2..DF + good b1 would be well-formed (defensive) - if (available < 3 || !sz_utf8_is_continuation_(u[2])) return 2; // byte 2 breaks it - return 3; // E0..F4: break is at byte 3 (b1, b2 are good) + if (available < 2 || !sz_utf8_first_continuation_ok_(lead, sz_utf8_byte_at_(utf8, 1))) + return 1; // byte 1 breaks it (incl. C2..DF) + if (lead < 0xE0) return 1; // C2..DF + good b1 would be well-formed (defensive) + if (available < 3 || !sz_utf8_is_continuation_(sz_utf8_byte_at_(utf8, 2))) return 2; // byte 2 breaks it + return 3; // E0..F4: break is at byte 3 (b1, b2 are good) } /** @brief Decode the 1-4 byte sequence the lead byte declares, with NO bounds check and NO validation; returns @@ -109,15 +116,17 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_maximal_subpart_(sz_cptr_t utf8, sz_cptr_t utf8 * @warning Assumes valid, complete UTF-8 (a truncated trailing sequence over-reads). Use `sz_rune_decode` for the * bounds-checked + validating variant, or `sz_utf8_find_malformed()` first. */ SZ_HELPER_AUTO sz_rune_length_t sz_rune_decode_unchecked(sz_cptr_t utf8, sz_rune_t *rune) { - sz_u8_t const *u8s = (sz_u8_t const *)utf8; - sz_u8_t lead = *u8s++; + sz_u8_t lead = sz_utf8_byte_at_(utf8, 0); sz_rune_length_t length = (sz_rune_length_t)(1 + (lead >= 0xC0U) + (lead >= 0xE0U) + (lead >= 0xF0U)); switch (length) { case 1: *rune = lead; break; - case 2: *rune = (lead & 0x1FU) << 6 | (u8s[0] & 0x3FU); break; - case 3: *rune = (lead & 0x0FU) << 12 | (u8s[0] & 0x3FU) << 6 | (u8s[1] & 0x3FU); break; + case 2: *rune = (lead & 0x1FU) << 6 | (sz_utf8_byte_at_(utf8, 1) & 0x3FU); break; + case 3: + *rune = (lead & 0x0FU) << 12 | (sz_utf8_byte_at_(utf8, 1) & 0x3FU) << 6 | (sz_utf8_byte_at_(utf8, 2) & 0x3FU); + break; default: - *rune = (sz_rune_t)(lead & 0x07U) << 18 | (u8s[0] & 0x3FU) << 12 | (u8s[1] & 0x3FU) << 6 | (u8s[2] & 0x3FU); + *rune = (sz_rune_t)(lead & 0x07U) << 18 | (sz_utf8_byte_at_(utf8, 1) & 0x3FU) << 12 | + (sz_utf8_byte_at_(utf8, 2) & 0x3FU) << 6 | (sz_utf8_byte_at_(utf8, 3) & 0x3FU); break; } return length; @@ -170,10 +179,9 @@ SZ_API_COMPTIME sz_cptr_t sz_utf8_find_malformed(sz_cptr_t text, sz_size_t lengt * U+FFFD. Genuinely ill-formed bytes (a bad lead, a malformed present continuation, or an overlong/surrogate/ * out-of-range prefix) return false so the caller emits the replacement character. */ SZ_HELPER_AUTO sz_bool_t sz_utf8_incomplete_tail_(sz_cptr_t text, sz_cptr_t end) { - sz_u8_t const *u = (sz_u8_t const *)text; - sz_size_t const available = (sz_size_t)((sz_u8_t const *)end - u); + sz_size_t const available = (sz_size_t)(end - text); if (!available) return sz_false_k; - sz_u8_t const lead = u[0]; + sz_u8_t const lead = sz_utf8_byte_at_(text, 0); sz_rune_length_t declared; if (lead < 0x80) return sz_false_k; else if (lead >= 0xC2 && lead < 0xE0) declared = sz_rune_2bytes_k; @@ -182,12 +190,14 @@ SZ_HELPER_AUTO sz_bool_t sz_utf8_incomplete_tail_(sz_cptr_t text, sz_cptr_t end) else return sz_false_k; // C0/C1, F5..FF, or a lone continuation - ill-formed, not merely truncated if (available >= (sz_size_t)declared) return sz_false_k; // all bytes present; `sz_rune_decode` judges validity for (sz_size_t index = 1; index < available; ++index) - if ((u[index] & 0xC0) != 0x80) return sz_false_k; // a present continuation is malformed - ill-formed now - if (available >= 2) { // first-continuation range constraints, where present - if (lead == 0xE0 && u[1] < 0xA0) return sz_false_k; // overlong - if (lead == 0xED && u[1] >= 0xA0) return sz_false_k; // surrogate - if (lead == 0xF0 && u[1] < 0x90) return sz_false_k; // overlong - if (lead == 0xF4 && u[1] >= 0x90) return sz_false_k; // > U+10FFFF + if ((sz_utf8_byte_at_(text, index) & 0xC0) != 0x80) + return sz_false_k; // a present continuation is malformed - ill-formed now + if (available >= 2) { // first-continuation range constraints, where present + sz_u8_t const second = sz_utf8_byte_at_(text, 1); + if (lead == 0xE0 && second < 0xA0) return sz_false_k; // overlong + if (lead == 0xED && second >= 0xA0) return sz_false_k; // surrogate + if (lead == 0xF0 && second < 0x90) return sz_false_k; // overlong + if (lead == 0xF4 && second >= 0x90) return sz_false_k; // > U+10FFFF } return sz_true_k; } @@ -274,13 +284,12 @@ SZ_HELPER_AUTO sz_rune_t sz_utf8_next_rune_(sz_cptr_t text, sz_size_t length, sz /** * @brief Get the UTF-8 sequence length from a lead byte, branchlessly. * - * The length is fully determined by the lead byte's high nibble: 0x0-0xB map to 1 (ASCII and, for robustness, - * stray continuation bytes treated as single bytes), 0xC-0xD to 2, 0xE to 3, 0xF to 4. A single 16-entry - * table resolves it without a four-way `if`-ladder on the codepoint advance. + * Three comparisons rather than a lookup: 0x00-0xBF is 1 (ASCII and, for robustness, stray continuation + * bytes treated as single bytes), 0xC0-0xDF is 2, 0xE0-0xEF is 3, 0xF0 and above is 4. The same form + * `sz_rune_decode_unchecked` uses, so the two can never disagree on how far a lead byte advances. */ -SZ_HELPER_INLINE sz_size_t sz_utf8_lead_length_(sz_u8_t lead_byte) { - static sz_u8_t const length_by_high_nibble[16] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4}; - return length_by_high_nibble[lead_byte >> 4]; +SZ_HELPER_AUTO sz_size_t sz_utf8_lead_length_(sz_u8_t lead_byte) { + return (sz_size_t)(1 + (lead_byte >= 0xC0U) + (lead_byte >= 0xE0U) + (lead_byte >= 0xF0U)); } /** @brief Returns the start offset of the codepoint preceding `position` (a codepoint start), or `position` if none. */ @@ -291,6 +300,19 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_previous_rune_start_(sz_cptr_t text, sz_size_t return previous; } +/** + * @brief Moves `position`, which may sit mid-codepoint, back to the lead byte of the codepoint holding it. + * Bounded to three steps, the widest continuation run a well-formed codepoint has; malformed input therefore + * bounds the cost rather than guaranteeing a real lead byte. + */ +SZ_HELPER_AUTO sz_size_t sz_utf8_rune_start_at_(sz_cptr_t text, sz_size_t length, sz_size_t position) { + for (sz_size_t step = 0; step < 3 && position > 0 && position < length; ++step) { + if (((sz_u8_t)text[position] & 0xC0) != 0x80) break; + --position; + } + return position; +} + #pragma region Shared bitmask boundary algebra /* ISA-independent `sz_u64_t` boundary-mask algebra shared by every backend (serial / haswell / icelake / neon). @@ -352,7 +374,7 @@ SZ_HELPER_AUTO sz_u64_t sz_u64_segmented_parity_(sz_u64_t seed, sz_u64_t gate) { /** @brief Low @p count bits set (`[0, count)`), 0 for `count==0`, all-ones for `count>=64`. Portable, branch-light * replacement for the BMI2 `sz_u64_mask_until_` so the shared boundary algebra compiles on every backend. */ -SZ_HELPER_INLINE sz_u64_t sz_u64_mask_until_serial_(sz_size_t count) { +SZ_HELPER_AUTO sz_u64_t sz_u64_mask_until_serial_(sz_size_t count) { return count >= 64 ? (sz_u64_t) ~(sz_u64_t)0 : (((sz_u64_t)1 << count) - 1); } diff --git a/include/stringzilla/utf8_runes/sve2.h b/include/stringzilla/utf8_runes/sve2.h index 0c6b90f3..1df5599d 100644 --- a/include/stringzilla/utf8_runes/sve2.h +++ b/include/stringzilla/utf8_runes/sve2.h @@ -133,7 +133,7 @@ SZ_HELPER_INLINE svuint32_t sz_utf8_rune_flat_lookup_quarter_sve2_( // * `svtbl_u8` scan of the 256-entry page LUT would cost sixteen shuffle-pipe lookups at the architectural * minimum vector length, while gathering both stages stays length-agnostic with no VL-dependent branch. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_rune_flat_lookup_sve2_( // +SZ_HELPER_INLINE svuint8_t sz_utf8_rune_flat_lookup_sve2_( // sz_u8_t const *page_lut, sz_u8_t const *flat, svuint8_t high_bytes_u8x, svuint8_t low_bytes_u8x) { // Widen both byte-lane vectors into four 32-bit-lane quarters each, preserving lane order. SVE vector types @@ -164,7 +164,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_rune_flat_lookup_sve2_( // /** @brief Read up to 256 LUT entries by per-lane u8 index via overlapping `svtbl_u8` chunks (gather-free); lanes * outside a chunk's span select zero and OR away. Serves the astral cascade stage-1 tables. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_rune_lut_sve2_(sz_u8_t const *table, int count, svuint8_t index_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_rune_lut_sve2_(sz_u8_t const *table, int count, svuint8_t index_u8x) { svbool_t const all_bytes_b8x = svptrue_b8(); int const vector_length = (int)svcntb(); svuint8_t result_u8x = svdup_n_u8(0); @@ -184,7 +184,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_rune_lut_sve2_(sz_u8_t const *table, int count, * falls to the full gather unchanged (so mixed/multibyte chunks pay only one compare + `ptest`, never an * extra LUT). Requires inactive lanes of @p bytes_u8x to be zero-filled (any `svld1` with a `whilelt` * predicate is), so the whole-chunk ASCII test never trips on tail garbage. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_rune_flat_lookup_ascii_gated_sve2_( // +SZ_HELPER_INLINE svuint8_t sz_utf8_rune_flat_lookup_ascii_gated_sve2_( // sz_u8_t const *page_lut, sz_u8_t const *flat, svuint8_t bytes_u8x, svuint8_t high_u8x, svuint8_t low_u8x) { svbool_t const all_b8x = svptrue_b8(); if (!svptest_any(all_b8x, svcmpge_n_u8(all_b8x, bytes_u8x, 0x80))) @@ -194,8 +194,8 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_rune_flat_lookup_ascii_gated_sve2_( // /** @brief Select one of `tile_count` 16-entry rows by `selector` and index it by `within` (nibble cascade tile), * serving the astral cascade stages on every property. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_rune_cascade_sve2_(sz_u8_t const *table, int tile_count, svuint8_t selector_u8x, - svuint8_t within_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_rune_cascade_sve2_(sz_u8_t const *table, int tile_count, svuint8_t selector_u8x, + svuint8_t within_u8x) { svbool_t const all_bytes_b8x = svptrue_b8(); svbool_t const row_b8x = svwhilelt_b8_u64(0, 16); svuint8_t result_u8x = svdup_n_u8(0); @@ -364,9 +364,9 @@ SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_packed_sve2_( * (`*runes_unpacked == 0`, cursor unchanged) ONLY when that happens on the very first lead (a boundary * truncation), which the public entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_sve2_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_sve2_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_u8_t const *text_u8 = (sz_u8_t const *)text; diff --git a/include/stringzilla/utf8_runes/v128.h b/include/stringzilla/utf8_runes/v128.h index 02aaf77f..a63be8aa 100644 --- a/include/stringzilla/utf8_runes/v128.h +++ b/include/stringzilla/utf8_runes/v128.h @@ -100,9 +100,9 @@ SZ_API_COMPTIME sz_cptr_t sz_utf8_seek_v128(sz_cptr_t text, sz_size_t length, sz * each quarter placed at bit positions [0,16)/[16,32)/[32,48)/[48,64)). Field names and semantics match * @ref sz_utf8_rune_window_neon_t so the portable rule algebra is unchanged. */ typedef struct sz_utf8_rune_window_v128_t { - v128_t window[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ - v128_t high[4]; /**< Per-lane `codepoint >> 8`. */ - v128_t low[4]; /**< Per-lane `codepoint & 0xFF`. */ + v128_t window_u8x16s[4]; /**< Raw input bytes for lanes [16*q, 16*q+16). */ + v128_t high_byte_u8x16s[4]; /**< Per-lane `codepoint >> 8`. */ + v128_t low_byte_u8x16s[4]; /**< Per-lane `codepoint & 0xFF`. */ sz_u64_t continuation; /**< Bit `i` => lane `i` is a continuation byte `10xxxxxx`. */ sz_u64_t codepoint_starts; /**< Bit `i` => lane `i` begins a codepoint (loaded, non-continuation). */ sz_u64_t two_byte_starts; /**< Bit `i` => lane `i` is a 2-byte lead `110xxxxx`. */ @@ -135,7 +135,7 @@ SZ_HELPER_INLINE sz_u64_t sz_utf8_mask_combine_v128_( // /** @brief Masked 64-byte load into four quarters; bytes [loaded, 64) read as zero. A zero-initialized vector union * stages the partial tail so we never read past `text + loaded`. Mirrors @ref sz_utf8_load_window_neon_. */ -SZ_HELPER_AUTO void sz_utf8_rune_load_window_v128_(sz_u8_t const *text, sz_size_t loaded, v128_t *out_u8x16) { +SZ_HELPER_INLINE void sz_utf8_rune_load_window_v128_(sz_u8_t const *text, sz_size_t loaded, v128_t *out_u8x16) { if (loaded >= 64) { out_u8x16[0] = wasm_v128_load(text + 0); out_u8x16[1] = wasm_v128_load(text + 16); @@ -161,7 +161,7 @@ SZ_HELPER_AUTO void sz_utf8_rune_load_window_v128_(sz_u8_t const *text, sz_size_ * span extracted. The three neighbour distances are provided because the family classifiers need up to * `next3` (4-byte sequences). */ -SZ_HELPER_AUTO void sz_utf8_forward_neighbours_v128_( // +SZ_HELPER_INLINE void sz_utf8_forward_neighbours_v128_( // v128_t const *window_u8x16, v128_t *next1_u8x16, v128_t *next2_u8x16, v128_t *next3_u8x16) { for (int quarter = 0; quarter < 4; ++quarter) { v128_t const here_u8x16 = window_u8x16[quarter]; @@ -177,14 +177,14 @@ SZ_HELPER_AUTO void sz_utf8_forward_neighbours_v128_( // /** @brief Load up to 64 bytes (masked tail) and decode every lane into byte-domain halves — the v128 twin of * @ref sz_utf8_rune_decode_window_neon_, bit-identical to it on every lane. */ -SZ_HELPER_AUTO sz_utf8_rune_window_v128_t sz_utf8_rune_decode_window_v128_( // +SZ_HELPER_INLINE sz_utf8_rune_window_v128_t sz_utf8_rune_decode_window_v128_( // sz_u8_t const *text, sz_size_t available) { sz_utf8_rune_window_v128_t result; result.loaded = available < 64 ? available : 64; v128_t window_u8x16[4]; sz_utf8_rune_load_window_v128_(text, result.loaded, window_u8x16); - for (int quarter = 0; quarter < 4; ++quarter) result.window[quarter] = window_u8x16[quarter]; + for (int quarter = 0; quarter < 4; ++quarter) result.window_u8x16s[quarter] = window_u8x16[quarter]; v128_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; sz_utf8_forward_neighbours_v128_(window_u8x16, next1_u8x16, next2_u8x16, next3_u8x16); @@ -247,8 +247,8 @@ SZ_HELPER_AUTO sz_utf8_rune_window_v128_t sz_utf8_rune_decode_window_v128_( // // Blend 2-byte vs 3-byte per lane: select the 3-byte value where this lane is a 3-byte lead. v128_t const three_select_u8x16 = three_byte_bool_u8x16[quarter]; - result.high[quarter] = wasm_v128_bitselect(high_three_u8x16, high_two_u8x16, three_select_u8x16); - result.low[quarter] = wasm_v128_bitselect(low_three_u8x16, low_two_u8x16, three_select_u8x16); + result.high_byte_u8x16s[quarter] = wasm_v128_bitselect(high_three_u8x16, high_two_u8x16, three_select_u8x16); + result.low_byte_u8x16s[quarter] = wasm_v128_bitselect(low_three_u8x16, low_two_u8x16, three_select_u8x16); } return result; } @@ -259,7 +259,7 @@ SZ_HELPER_AUTO sz_utf8_rune_window_v128_t sz_utf8_rune_decode_window_v128_( // * and shuffled by @p within_u8x16, then blended in where @p selector_u8x16 picks it; the final `wasm_u8x16_lt` * clamp reproduces the all-zero result for selectors past the table. @p within_u8x16 is a nibble by * construction. The v128 twin of @ref sz_utf8_rune_cascade_stage_neon_. */ -SZ_HELPER_AUTO v128_t sz_utf8_rune_cascade_stage_v128_( // +SZ_HELPER_INLINE v128_t sz_utf8_rune_cascade_stage_v128_( // sz_u8_t const *table, int tile_count, v128_t selector_u8x16, v128_t within_u8x16) { v128_t result_u8x16 = wasm_i8x16_splat(0); for (int tile = 0; tile < tile_count; ++tile) { @@ -275,7 +275,7 @@ SZ_HELPER_AUTO v128_t sz_utf8_rune_cascade_stage_v128_( // * `result[lane] = group_base[index_u8x16[lane]]`. `wasm_i8x16_swizzle` reaches only 16 B, so the read is a * bounded scalar L1 walk (the v128 twin of the substrate `lut256` leaf); the index byte is total over the * 256-entry table by construction. */ -SZ_HELPER_AUTO v128_t sz_utf8_rune_lut256_v128_(sz_u8_t const *group_base, v128_t index_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_rune_lut256_v128_(sz_u8_t const *group_base, v128_t index_u8x16) { sz_align_(16) sz_u8_t index_lanes[16], out_lanes[16]; wasm_v128_store(index_lanes, index_u8x16); for (int lane = 0; lane < 16; ++lane) out_lanes[lane] = group_base[index_lanes[lane]]; @@ -286,7 +286,7 @@ SZ_HELPER_AUTO v128_t sz_utf8_rune_lut256_v128_(sz_u8_t const *group_base, v128_ * `flat[page * 256 + low]` is read per lane. The page LUT resolves via the in-register scalar `lut256` walk; * the leaf read is a bounded scalar L1 walk over fused 16-bit indices `(page << 8) | low`. Lanes whose page * index reaches @p page_count return zero. The v128 twin of @ref sz_utf8_rune_flat_lookup_neon_. */ -SZ_HELPER_AUTO v128_t sz_utf8_rune_flat_lookup_v128_( // +SZ_HELPER_INLINE v128_t sz_utf8_rune_flat_lookup_v128_( // sz_u8_t const *page_lut, sz_u8_t const *flat, int page_count, v128_t high_bytes_u8x16, v128_t low_bytes_u8x16) { v128_t const page_indices_u8x16 = sz_utf8_rune_lut256_v128_(page_lut, high_bytes_u8x16); v128_t const in_range_u8x16 = wasm_u8x16_lt(page_indices_u8x16, wasm_i8x16_splat((sz_i8_t)page_count)); @@ -304,7 +304,7 @@ SZ_HELPER_AUTO v128_t sz_utf8_rune_flat_lookup_v128_( // * via @p previous_io; bit-exact with the Ice Lake leaf. Consumption is inherently scalar (one output pair * per lane), so each set lane is isolated with the `63 - clz(mask & -mask)` first-set idiom and cleared with * `mask & (mask - 1)` — the cost scales with the boundary count, and no `ctz` / `popcount` builtin is used. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_forward_v128_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_forward_v128_( // sz_u64_t boundary, sz_size_t base, sz_size_t *starts, sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, sz_size_t *previous_io) { sz_size_t previous = *previous_io; @@ -362,7 +362,7 @@ SZ_HELPER_INLINE v128_t sz_utf8_rune_widen4_v128_(v128_t bytes_u8x16, int quarte * bytes owed their own next U+FFFD. * @return Number of runes emitted; sets @p consumed_bytes to the byte span they cover (the resume cursor delta). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_v128_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_rune_drain_v128_( // v128_t const *regs_u8x16, sz_u64_t emit_starts, sz_u64_t ill_formed, sz_u8_t const *consumed_length, int has_three, int has_four, sz_size_t capacity, sz_rune_t *runes, sz_size_t *consumed_bytes) { @@ -475,9 +475,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_rune_drain_v128_( // * first decodable lead's declared sequence crosses the window edge (a boundary truncation), which the public * entry finalizes without a serial re-decode. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_decode_once_v128_( // - sz_cptr_t text, sz_size_t length, // - sz_rune_t *runes, sz_size_t runes_capacity, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_decode_once_v128_( // + sz_cptr_t text, sz_size_t length, // + sz_rune_t *runes, sz_size_t runes_capacity, // sz_size_t *runes_unpacked) { sz_size_t const chunk = length < 64 ? length : 64; diff --git a/include/stringzilla/utf8_sentences.h b/include/stringzilla/utf8_sentences.h index 0310e8aa..da29cff4 100644 --- a/include/stringzilla/utf8_sentences.h +++ b/include/stringzilla/utf8_sentences.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UAX-29 sentence segmentation. - * @file utf8_sentences.h + * @file include/stringzilla/utf8_sentences.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_SENTENCES_H_ diff --git a/include/stringzilla/utf8_sentences/README.md b/include/stringzilla/utf8_sentences/README.md index a1e7b6d4..48f3d778 100644 --- a/include/stringzilla/utf8_sentences/README.md +++ b/include/stringzilla/utf8_sentences/README.md @@ -5,7 +5,7 @@ Each operation has a serial baseline plus `haswell` and `icelake` SIMD backends ## Methodology -Numbers are throughput in MB/s, measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput in MB/s, measured with `bench/utf8_segment.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its single column is the `sz_utf8_sentences` operation and its rows are a backend on a chip, so reading down the column compares the backend ladder on one fixed input shape. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes) and a Long Lines workload (full text lines) to expose how each kernel scales with token length. A `↑` cell means there is no dedicated kernel at that backend, so the dispatcher reuses the tier above it. diff --git a/include/stringzilla/utf8_sentences/haswell.h b/include/stringzilla/utf8_sentences/haswell.h index 36900683..b226d976 100644 --- a/include/stringzilla/utf8_sentences/haswell.h +++ b/include/stringzilla/utf8_sentences/haswell.h @@ -42,7 +42,7 @@ extern "C" { /** @brief Sentence_Break class byte for thirty-two BMP codepoints (per-lane high = cp>>8, low = cp&0xFF): the * `bmp_page_lut_` page LUT selects one of the 57 distinct 256-byte pages, then `flat_bmp_` is fetched by * `vpgatherdd`. Bit-exact with `sz_rune_sentence_break_property` over the whole BMP. */ -SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_bmp_class_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_sentence_break_bmp_class_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { return sz_utf8_rune_flat_lookup_haswell_(sz_utf8_sentence_break_bmp_page_lut_, sz_utf8_sentence_break_flat_bmp_, high_bytes_u8x32, low_bytes_u8x32); } @@ -51,8 +51,8 @@ SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_bmp_class_haswell_(__m256i high_by * cascade). Per-lane bytes: @p plane_u8x32 = (offset>>16)&0xFF (low nibble meaningful), @p high_u8x32 = * (offset>>8)&0xFF, @p low_u8x32 = offset&0xFF. Bit-exact with `sz_rune_sentence_break_property` over all * astral. */ -SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_astral_class_haswell_(__m256i plane_u8x32, __m256i high_u8x32, - __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_sentence_break_astral_class_haswell_(__m256i plane_u8x32, __m256i high_u8x32, + __m256i low_u8x32) { __m256i const low_nibble_mask_u8x32 = _mm256_set1_epi8(0x0F); __m256i const n4_u8x32 = _mm256_and_si256(plane_u8x32, low_nibble_mask_u8x32); __m256i const n3_u8x32 = _mm256_and_si256(_mm256_srli_epi16(high_u8x32, 4), low_nibble_mask_u8x32); @@ -93,7 +93,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_astral_class_haswell_(__m256i plan * the cascade, exactly as the icelake driver reconstructs them. BMP lanes go through the BMP cascade; * 4-byte lanes are routed by reconstructed plane through the astral cascade. The class on non-codepoint-start * lanes is irrelevant (the dense compaction only reads start lanes), so those lanes are never selected. */ -SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_classify_half_haswell_( // +SZ_HELPER_INLINE __m256i sz_utf8_sentence_break_classify_half_haswell_( // __m256i window_high_u8x32, __m256i window_low_u8x32, __m256i raw_u8x32, __m256i next1_u8x32, __m256i next2_u8x32, __m256i next3_u8x32, sz_u32_t four_byte_bits) { __m256i const low_two_bits_u8x32 = _mm256_set1_epi8(0x03); @@ -167,7 +167,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_sentence_break_classify_half_haswell_( // * `high_u8x32 = 0` (the classifier re-seats raw / 4-byte lanes anyway). Used by the sentence driver so a * truncated trailing multi-byte lead reads its missing continuations as zero, exactly like serial / icelake * (no mod-64 wrap aliasing). */ -SZ_HELPER_AUTO void sz_utf8_sentence_break_bmp_highlow_haswell_( // +SZ_HELPER_INLINE void sz_utf8_sentence_break_bmp_highlow_haswell_( // __m256i raw_u8x32, __m256i next1_u8x32, __m256i next2_u8x32, sz_u32_t two_byte_bits, sz_u32_t three_byte_bits, __m256i *out_high_u8x32, __m256i *out_low_u8x32) { __m256i const low_two_bits_u8x32 = _mm256_set1_epi8(0x03); @@ -213,8 +213,8 @@ SZ_HELPER_INLINE void sz_utf8_sentence_break_next3_haswell_(__m256i window_lo_u8 /** @brief Build the per-class membership frame from the dense class byte stream with AVX2 compares: each class is one * `vpcmpeqb` per 32-lane half OR-combined to a u64, the AVX2 twin of the icelake fifteen-`vpcmpeqb` build (no * scalar pass). The dense stream is at most 64 lanes, held as two `__m256i`. */ -SZ_HELPER_AUTO sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_haswell_(sz_u8_t const *dense_classes, - sz_u64_t valid) { +SZ_HELPER_INLINE sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_haswell_(sz_u8_t const *dense_classes, + sz_u64_t valid) { __m256i const dense_lo_u8x32 = _mm256_loadu_si256((__m256i const *)(dense_classes + 0)); __m256i const dense_hi_u8x32 = _mm256_loadu_si256((__m256i const *)(dense_classes + 32)); sz_utf8_sentence_break_frame_t frame; @@ -237,9 +237,9 @@ SZ_HELPER_INLINE sz_utf8_sentence_break_window_t sz_utf8_sentence_break_decide_d /** @brief Largest byte prefix of the window whose codepoints are all fully loaded — the AVX2 twin of the icelake * driver's effective-window<64 trim. Never below 1 when the window is non-empty. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_sentence_break_complete_limit_haswell_(sz_utf8_rune_window_haswell_t window, - sz_u8_t const *bytes_after, - sz_bool_t more_text) { +SZ_HELPER_INLINE sz_size_t sz_utf8_sentence_break_complete_limit_haswell_(sz_utf8_rune_window_haswell_t window, + sz_u8_t const *bytes_after, + sz_bool_t more_text) { sz_size_t const loaded = window.loaded; if (!more_text) return loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); @@ -308,9 +308,10 @@ SZ_API_COMPTIME sz_size_t sz_utf8_sentences_haswell( // // `loaded - k`, matching icelake's `maskz_permutexvar` `keep1`/`keep2`/`keep3`, so the missing continuations // read as zero exactly like serial's blind decode (`text[start+k]` past the input reads 0). __m256i next1_lo_u8x32, next1_hi_u8x32, next2_lo_u8x32, next2_hi_u8x32, next3_lo_u8x32, next3_hi_u8x32; - sz_utf8_forward_neighbours_haswell_(window.window_lo, window.window_hi, &next1_lo_u8x32, &next1_hi_u8x32, - &next2_lo_u8x32, &next2_hi_u8x32); - sz_utf8_sentence_break_next3_haswell_(window.window_lo, window.window_hi, &next3_lo_u8x32, &next3_hi_u8x32); + sz_utf8_forward_neighbours_haswell_(window.window_low_u8x32, window.window_high_u8x32, &next1_lo_u8x32, + &next1_hi_u8x32, &next2_lo_u8x32, &next2_hi_u8x32); + sz_utf8_sentence_break_next3_haswell_(window.window_low_u8x32, window.window_high_u8x32, &next3_lo_u8x32, + &next3_hi_u8x32); sz_u64_t const keep1 = sz_u64_mask_until_serial_(loaded >= 1 ? loaded - 1 : 0); sz_u64_t const keep2 = sz_u64_mask_until_serial_(loaded >= 2 ? loaded - 2 : 0); sz_u64_t const keep3 = sz_u64_mask_until_serial_(loaded >= 3 ? loaded - 3 : 0); @@ -329,20 +330,20 @@ SZ_API_COMPTIME sz_size_t sz_utf8_sentences_haswell( // // the loaded edge would read a wrapped byte as its missing continuation (icelake recomputes high/low from // its `keep*`-masked neighbours for exactly this reason; here we patch the decoded pair to match). __m256i high_lo_u8x32, high_hi_u8x32, low_lo_u8x32, low_hi_u8x32; - sz_utf8_sentence_break_bmp_highlow_haswell_(window.window_lo, next1_lo_u8x32, next2_lo_u8x32, + sz_utf8_sentence_break_bmp_highlow_haswell_(window.window_low_u8x32, next1_lo_u8x32, next2_lo_u8x32, (sz_u32_t)window.two_byte_starts, (sz_u32_t)window.three_byte_starts, &high_lo_u8x32, &low_lo_u8x32); sz_utf8_sentence_break_bmp_highlow_haswell_( - window.window_hi, next1_hi_u8x32, next2_hi_u8x32, (sz_u32_t)(window.two_byte_starts >> 32), + window.window_high_u8x32, next1_hi_u8x32, next2_hi_u8x32, (sz_u32_t)(window.two_byte_starts >> 32), (sz_u32_t)(window.three_byte_starts >> 32), &high_hi_u8x32, &low_hi_u8x32); // The classifier reconstructs the raw-byte (ASCII / continuation / `>= 0xF8`) and 4-byte codepoints from // the raw window bytes itself, so no per-half ASCII mask needs to be threaded in. __m256i const classes_lo_u8x32 = sz_utf8_sentence_break_classify_half_haswell_( - high_lo_u8x32, low_lo_u8x32, window.window_lo, next1_lo_u8x32, next2_lo_u8x32, next3_lo_u8x32, + high_lo_u8x32, low_lo_u8x32, window.window_low_u8x32, next1_lo_u8x32, next2_lo_u8x32, next3_lo_u8x32, (sz_u32_t)window.four_byte_starts); __m256i const classes_hi_u8x32 = sz_utf8_sentence_break_classify_half_haswell_( - high_hi_u8x32, low_hi_u8x32, window.window_hi, next1_hi_u8x32, next2_hi_u8x32, next3_hi_u8x32, + high_hi_u8x32, low_hi_u8x32, window.window_high_u8x32, next1_hi_u8x32, next2_hi_u8x32, next3_hi_u8x32, (sz_u32_t)(window.four_byte_starts >> 32)); sz_size_t const complete_limit = sz_utf8_sentence_break_complete_limit_haswell_( diff --git a/include/stringzilla/utf8_sentences/icelake.h b/include/stringzilla/utf8_sentences/icelake.h index 9fae8f1c..22b58bae 100644 --- a/include/stringzilla/utf8_sentences/icelake.h +++ b/include/stringzilla/utf8_sentences/icelake.h @@ -17,8 +17,9 @@ extern "C" { #if SZ_USE_ICELAKE #if defined(__clang__) && SZ_CLANG_HAS_EVEX512_ -#pragma clang attribute push( \ - __attribute__((target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,evex512,popcnt"))), \ +#pragma clang attribute push( \ + __attribute__(( \ + target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,evex512,popcnt"))), \ apply_to = function) #elif defined(__clang__) #pragma clang attribute push( \ @@ -27,8 +28,7 @@ extern "C" { #elif defined(__GNUC__) #pragma GCC push_options #pragma GCC target("avx", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512vbmi", "avx512vbmi2", "bmi", "bmi2", \ - "lzcnt", \ - "popcnt") + "lzcnt", "popcnt") #endif #pragma region Sentence_Break classifier @@ -40,7 +40,7 @@ extern "C" { * `vpermb` + one `vpgatherdd` each), then `vpexpandb`-scattered back onto @p classes_u8x64 at their original * byte-lane positions. The second half only runs when more than sixteen cold starts are present. Every other * lane keeps its prior value. */ -SZ_HELPER_AUTO __m512i sz_utf8_sentence_break_cold_compact_icelake_( // +SZ_HELPER_INLINE __m512i sz_utf8_sentence_break_cold_compact_icelake_( // __m512i classes_u8x64, __m512i high_bytes_u8x64, __m512i low_bytes_u8x64, sz_u64_t cold_starts) { __mmask64 const cold_start_mask_m64 = _cvtu64_mask64(cold_starts); __m512i const high_packed_u8x64 = _mm512_maskz_compress_epi8(cold_start_mask_m64, high_bytes_u8x64); @@ -84,7 +84,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_sentence_break_cold_compact_icelake_( // * @param four_byte_starts_m64 Lanes that begin a 4-byte UTF-8 sequence (gates the >= 0x10000 astral test). * @param codepoint_starts_m64 Lanes that begin any codepoint (non-continuation, in range). */ -SZ_HELPER_AUTO __m512i sz_utf8_sentence_break_classify_window_icelake_( // +SZ_HELPER_INLINE __m512i sz_utf8_sentence_break_classify_window_icelake_( // __m512i raw_window_u8x64, __m512i raw_next1_u8x64, __m512i raw_next2_u8x64, __m512i raw_next3_u8x64, // __m512i high_u8x64, __m512i low_u8x64, // __mmask64 four_byte_starts_m64, __mmask64 codepoint_starts_m64) { @@ -286,7 +286,7 @@ SZ_API_COMPTIME sz_size_t sz_utf8_sentences_icelake( // __mmask64 const codepoint_starts_m64 = decoded.codepoint_starts | lead_continuation_m64; sz_u64_t const start_bytes = _cvtmask64_u64(codepoint_starts_m64); - __m512i const window_u8x64 = decoded.window; + __m512i const window_u8x64 = decoded.window_u8x64; __mmask64 const keep1_m64 = sz_u64_mask_until_(loaded >= 1 ? loaded - 1 : 0); __mmask64 const keep2_m64 = sz_u64_mask_until_(loaded >= 2 ? loaded - 2 : 0); __mmask64 const keep3_m64 = sz_u64_mask_until_(loaded >= 3 ? loaded - 3 : 0); diff --git a/include/stringzilla/utf8_sentences/neon.h b/include/stringzilla/utf8_sentences/neon.h index 17bf1d7a..0e2e6997 100644 --- a/include/stringzilla/utf8_sentences/neon.h +++ b/include/stringzilla/utf8_sentences/neon.h @@ -30,7 +30,7 @@ extern "C" { /** @brief Software `_pext_u64`: gather the bits of @p value selected by @p selector, packed to the low end (bit `j` * of the result = the `j`-th set bit of @p value within @p selector). NEON has no `pext`; the sparse loop * trips once per set @p selector bit (codepoint-dense compaction over the start lanes). Bit-exact with BMI2. */ -SZ_HELPER_AUTO sz_u64_t sz_sentence_break_pext_neon_(sz_u64_t value, sz_u64_t selector) { +SZ_HELPER_INLINE sz_u64_t sz_sentence_break_pext_neon_(sz_u64_t value, sz_u64_t selector) { sz_u64_t result = 0; sz_u64_t out_bit = 1; while (selector) { @@ -45,7 +45,7 @@ SZ_HELPER_AUTO sz_u64_t sz_sentence_break_pext_neon_(sz_u64_t value, sz_u64_t se /** @brief Software `_pdep_u64`: scatter the low bits of @p value into the positions set in @p selector (the `j`-th * set bit of @p selector receives bit `j` of @p value). NEON has no `pdep`; the sparse loop trips once per * set @p selector bit (the dense-boundary scatter back onto codepoint-start lanes). Bit-exact with BMI2. */ -SZ_HELPER_AUTO sz_u64_t sz_sentence_break_pdep_neon_(sz_u64_t value, sz_u64_t selector) { +SZ_HELPER_INLINE sz_u64_t sz_sentence_break_pdep_neon_(sz_u64_t value, sz_u64_t selector) { sz_u64_t result = 0; while (selector) { sz_u64_t const low = selector & (~selector + 1); // lowest set bit of `selector` @@ -91,7 +91,7 @@ SZ_HELPER_INLINE uint8x16_t sz_sentence_break_byte_mask_from_bits_neon_(sz_u64_t * makes those missing continuations read as zero, exactly like serial's blind decode (`text[start+k]` past the * input reads 0). Lanes that are neither a 2- nor a 3-byte lead keep `low = raw`, `high = 0`; the classifier * re-seats raw / 4-byte lanes anyway. */ -SZ_HELPER_AUTO void sz_utf8_sentence_break_bmp_highlow_neon_( // +SZ_HELPER_INLINE void sz_utf8_sentence_break_bmp_highlow_neon_( // uint8x16_t raw_u8x16, uint8x16_t next1_u8x16, uint8x16_t next2_u8x16, sz_u64_t two_bits, sz_u64_t three_bits, uint8x16_t *out_high_u8x16, uint8x16_t *out_low_u8x16) { uint8x16_t const low_two_bits_u8x16 = vdupq_n_u8(0x03); @@ -122,8 +122,8 @@ SZ_HELPER_AUTO void sz_utf8_sentence_break_bmp_highlow_neon_( // * page-compressed table via @ref sz_utf8_rune_flat_lookup_neon_, the NEON twin of * @ref sz_utf8_sentence_break_bmp_class_haswell_. Bit-exact with `sz_rune_sentence_break_property` over the * whole BMP. Operates on one quarter; the caller iterates the four quarters. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_bmp_class_neon_(uint8x16_t high_bytes_u8x16, - uint8x16_t low_bytes_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_sentence_break_bmp_class_neon_(uint8x16_t high_bytes_u8x16, + uint8x16_t low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_neon_(sz_utf8_sentence_break_bmp_page_lut_, sz_utf8_sentence_break_flat_bmp_, (int)sz_utf8_sentence_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -132,8 +132,8 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_bmp_class_neon_(uint8x16_t high * cascade), the NEON twin of @ref sz_utf8_sentence_break_astral_class_haswell_. Per-lane bytes: * @p plane = (offset>>16)&0xFF (low nibble meaningful), @p high = (offset>>8)&0xFF, @p low = offset&0xFF. * Bit-exact with `sz_rune_sentence_break_property` over all astral. Operates on one quarter. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_astral_class_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, - uint8x16_t low_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_sentence_break_astral_class_neon_(uint8x16_t plane_u8x16, uint8x16_t high_u8x16, + uint8x16_t low_u8x16) { uint8x16_t const low_nibble_mask_u8x16 = vdupq_n_u8(0x0F); uint8x16_t const n4_u8x16 = vandq_u8(plane_u8x16, low_nibble_mask_u8x16); uint8x16_t const n3_u8x16 = vandq_u8(vshrq_n_u8(high_u8x16, 4), low_nibble_mask_u8x16); @@ -173,7 +173,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_astral_class_neon_(uint8x16_t p * reconstructs them. BMP lanes go through the BMP cascade; 4-byte lanes are routed by reconstructed plane * through the astral cascade. The class on non-codepoint-start lanes is irrelevant (the dense compaction only * reads start lanes), so those lanes are never selected. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_classify_quarter_neon_( // +SZ_HELPER_INLINE uint8x16_t sz_utf8_sentence_break_classify_quarter_neon_( // uint8x16_t window_high_u8x16, uint8x16_t window_low_u8x16, uint8x16_t raw_u8x16, uint8x16_t next1_u8x16, uint8x16_t next2_u8x16, uint8x16_t next3_u8x16, sz_u64_t four_byte_bits) { uint8x16_t const low_two_bits_u8x16 = vdupq_n_u8(0x03); @@ -236,8 +236,8 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_sentence_break_classify_quarter_neon_( // /** @brief Build the per-class membership frame from the dense class byte stream with NEON compares: each class is one * `vceqq_u8` per quarter OR-combined to a u64, the NEON twin of @ref sz_utf8_sentence_break_frame_haswell_ (no * scalar pass). The dense stream is at most 64 lanes, held as four `uint8x16_t` quarters. */ -SZ_HELPER_AUTO sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_neon_(sz_u8_t const *dense_classes, - sz_u64_t valid) { +SZ_HELPER_INLINE sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_neon_(sz_u8_t const *dense_classes, + sz_u64_t valid) { uint8x16_t dense_u8x16[4]; dense_u8x16[0] = vld1q_u8(dense_classes + 0); dense_u8x16[1] = vld1q_u8(dense_classes + 16); @@ -265,8 +265,9 @@ SZ_HELPER_INLINE sz_utf8_sentence_break_window_t sz_utf8_sentence_break_decide_d /** @brief Largest byte prefix of the window whose codepoints are all fully loaded — the NEON twin of * @ref sz_utf8_sentence_break_complete_limit_haswell_ over the NEON window struct. Never below 1 when the * window is non-empty. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_sentence_break_complete_limit_neon_(sz_utf8_rune_window_neon_t window, - sz_u8_t const *bytes_after, sz_bool_t more_text) { +SZ_HELPER_INLINE sz_size_t sz_utf8_sentence_break_complete_limit_neon_(sz_utf8_rune_window_neon_t window, + sz_u8_t const *bytes_after, + sz_bool_t more_text) { sz_size_t const loaded = window.loaded; if (!more_text) return loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); @@ -336,7 +337,7 @@ SZ_API_COMPTIME sz_size_t sz_utf8_sentences_neon( // // `loaded - k`, matching haswell's `keep1`/`keep2`/`keep3` and icelake's `maskz_permutexvar`, so the missing // continuations read as zero exactly like serial's blind decode (`text[start+k]` past the input reads 0). uint8x16_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; - sz_utf8_forward_neighbours_neon_(window.window, next1_u8x16, next2_u8x16, next3_u8x16); + sz_utf8_forward_neighbours_neon_(window.window_u8x16s, next1_u8x16, next2_u8x16, next3_u8x16); sz_u64_t const keep1 = sz_u64_mask_until_serial_(loaded >= 1 ? loaded - 1 : 0); sz_u64_t const keep2 = sz_u64_mask_until_serial_(loaded >= 2 ? loaded - 2 : 0); sz_u64_t const keep3 = sz_u64_mask_until_serial_(loaded >= 3 ? loaded - 3 : 0); @@ -359,11 +360,12 @@ SZ_API_COMPTIME sz_size_t sz_utf8_sentences_neon( // for (int quarter = 0; quarter < 4; ++quarter) { int const lane_base = quarter * 16; uint8x16_t high_q_u8x16, low_q_u8x16; - sz_utf8_sentence_break_bmp_highlow_neon_( - window.window[quarter], next1_u8x16[quarter], next2_u8x16[quarter], window.two_byte_starts >> lane_base, - window.three_byte_starts >> lane_base, &high_q_u8x16, &low_q_u8x16); + sz_utf8_sentence_break_bmp_highlow_neon_(window.window_u8x16s[quarter], next1_u8x16[quarter], + next2_u8x16[quarter], window.two_byte_starts >> lane_base, + window.three_byte_starts >> lane_base, &high_q_u8x16, + &low_q_u8x16); uint8x16_t const classes_q_u8x16 = sz_utf8_sentence_break_classify_quarter_neon_( - high_q_u8x16, low_q_u8x16, window.window[quarter], next1_u8x16[quarter], next2_u8x16[quarter], + high_q_u8x16, low_q_u8x16, window.window_u8x16s[quarter], next1_u8x16[quarter], next2_u8x16[quarter], next3_u8x16[quarter], window.four_byte_starts >> lane_base); vst1q_u8(class_bytes + lane_base, classes_q_u8x16); } diff --git a/include/stringzilla/utf8_sentences/serial.h b/include/stringzilla/utf8_sentences/serial.h index e1df1932..04eabd1e 100644 --- a/include/stringzilla/utf8_sentences/serial.h +++ b/include/stringzilla/utf8_sentences/serial.h @@ -45,21 +45,21 @@ SZ_API_COMPTIME sz_u8_t sz_rune_sentence_break_property(sz_rune_t rune) { } /** @brief True for a Sentence_Break ParaSep (Sep, CR, or LF). */ -SZ_HELPER_INLINE sz_bool_t sz_sentence_break_is_parasep_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_sentence_break_is_parasep_(sz_u8_t property) { return (sz_bool_t)(property == sz_sentence_break_sep_k || property == sz_sentence_break_cr_k || property == sz_sentence_break_lf_k); } /** @brief True for a Sentence_Break SATerm (STerm or ATerm). */ -SZ_HELPER_INLINE sz_bool_t sz_sentence_break_is_saterm_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_sentence_break_is_saterm_(sz_u8_t property) { return (sz_bool_t)(property == sz_sentence_break_sterm_k || property == sz_sentence_break_aterm_k); } /** @brief True for an SB5-transparent character (Extend or Format). */ -SZ_HELPER_INLINE sz_bool_t sz_sentence_break_is_transparent_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_sentence_break_is_transparent_(sz_u8_t property) { return (sz_bool_t)(property == sz_sentence_break_extend_k || property == sz_sentence_break_format_k); } /** @brief SB8 stop set excluding Lower: OLetter, Upper, ParaSep, or SATerm — a significant class that ends the * ATerm neutral run and confirms the deferred SB11 break (a Lower in the run suppresses it instead). */ -SZ_HELPER_INLINE sz_bool_t sz_sentence_break_sb8_stops_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_sentence_break_sb8_stops_(sz_u8_t property) { return (sz_bool_t)(property == sz_sentence_break_oletter_k || property == sz_sentence_break_upper_k || sz_sentence_break_is_parasep_(property) || sz_sentence_break_is_saterm_(property)); } @@ -82,7 +82,7 @@ SZ_HELPER_AUTO sz_size_t sz_sentence_break_next_start_(sz_cptr_t text, sz_size_t * folded in with no continuation/overlong/surrogate validation, missing trailing bytes read as zero. * Valid UTF-8 decodes identically to the checked path; only ill-formed input differs, by design. */ -SZ_HELPER_AUTO sz_u8_t sz_sentence_break_property_at_(sz_cptr_t text, sz_size_t length, sz_size_t start) { +SZ_HELPER_INLINE sz_u8_t sz_sentence_break_property_at_(sz_cptr_t text, sz_size_t length, sz_size_t start) { sz_u8_t const lead = (sz_u8_t)text[start]; int const lead_length = ((lead & 0xE0u) == 0xC0u) ? 2 : ((lead & 0xF0u) == 0xE0u) ? 3 diff --git a/include/stringzilla/utf8_sentences/sve2.h b/include/stringzilla/utf8_sentences/sve2.h index c43d8044..c91adf5b 100644 --- a/include/stringzilla/utf8_sentences/sve2.h +++ b/include/stringzilla/utf8_sentences/sve2.h @@ -29,8 +29,8 @@ extern "C" { * (5-nibble cascade), the SVE2 twin of @ref sz_utf8_sentence_break_astral_class_neon_. Per-lane bytes: * @p plane = (offset>>16)&0xFF (low nibble meaningful), @p high = (offset>>8)&0xFF, @p low = offset&0xFF. * Bit-exact with `sz_rune_sentence_break_property` over all astral planes. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_sentence_break_astral_class_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, - svuint8_t low_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_sentence_break_astral_class_sve2_(svuint8_t plane_u8x, svuint8_t high_u8x, + svuint8_t low_u8x) { svbool_t const all_b8x = svptrue_b8(); svuint8_t const n4_u8x = svand_n_u8_x(all_b8x, plane_u8x, 0x0F); svuint8_t const n3_u8x = svand_n_u8_x(all_b8x, svlsr_n_u8_x(all_b8x, high_u8x, 4), 0x0F); @@ -69,7 +69,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_sentence_break_astral_class_sve2_(svuint8_t pla * (high, low) pair from the peeked neighbours; 4-byte leads split by their blind plane between the BMP * cascade (plane 0, overlong encodings), the astral cascade (planes 1..16), and class Other (planes over * 16, e.g. `F5..F7` leads). The class on non-start lanes is irrelevant - only start lanes compact. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_sentence_break_classify_chunk_sve2_( // +SZ_HELPER_INLINE svuint8_t sz_utf8_sentence_break_classify_chunk_sve2_( // svuint8_t bytes_u8x, svuint8_t next1_u8x, svuint8_t next2_u8x, svuint8_t next3_u8x, // svbool_t two_b8x, svbool_t three_b8x, svbool_t four_b8x) { svbool_t const all_b8x = svptrue_b8(); @@ -124,8 +124,8 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_sentence_break_classify_chunk_sve2_( /** @brief Build the per-class membership frame from the dense class byte stream via its four bit-planes: each * plane lowers to a u64 with the shared predicate bridge, and the fifteen class masks assemble from the * planes with scalar mask algebra - 4 predicate compares per dense vector instead of 15. */ -SZ_HELPER_AUTO sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_sve2_(sz_u8_t const *dense_classes, - sz_size_t count) { +SZ_HELPER_INLINE sz_utf8_sentence_break_frame_t sz_utf8_sentence_break_frame_sve2_(sz_u8_t const *dense_classes, + sz_size_t count) { sz_size_t const vector_bytes = svcntb() < 64 ? svcntb() : 64; sz_u64_t planes[4] = {0, 0, 0, 0}; for (sz_size_t base = 0; base < count; base += vector_bytes) { diff --git a/include/stringzilla/utf8_tokens.h b/include/stringzilla/utf8_tokens.h index 7bc4d935..a56c4b33 100644 --- a/include/stringzilla/utf8_tokens.h +++ b/include/stringzilla/utf8_tokens.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UTF-8 newline, whitespace, and general delimiter scanning. - * @file utf8_tokens.h + * @file include/stringzilla/utf8_tokens.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_TOKENS_H_ diff --git a/include/stringzilla/utf8_tokens/README.md b/include/stringzilla/utf8_tokens/README.md index 0fc7cb21..017a6c8c 100644 --- a/include/stringzilla/utf8_tokens/README.md +++ b/include/stringzilla/utf8_tokens/README.md @@ -5,7 +5,7 @@ Each operation has a serial baseline plus `haswell` and `icelake` SIMD backends ## Methodology -Numbers are throughput in MB/s, measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput in MB/s, measured with `bench/utf8_scan.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its columns are the Whitespace Split and Newline Split operations and its rows are a backend on a chip, so reading down a column compares the same operation across the backend ladder while reading across a row compares operations on one backend. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes) and a Long Lines workload (full text lines) to expose how each kernel scales with token length. A `↑` cell means there is no dedicated kernel for that operation at that backend, so the dispatcher reuses the tier above it. diff --git a/include/stringzilla/utf8_tokens/haswell.h b/include/stringzilla/utf8_tokens/haswell.h index 5d03e8ca..1ad3c6df 100644 --- a/include/stringzilla/utf8_tokens/haswell.h +++ b/include/stringzilla/utf8_tokens/haswell.h @@ -36,24 +36,26 @@ SZ_HELPER_INLINE __m256i sz_mm256_cmpge_epu8_haswell_(__m256i a_u8x32, __m256i b * `start_bits` mask plus per-length start masks, then the peel left-packs matches with `vpermd`. Starts in * lanes [0,29] are trusted and the cursor steps 30, so any 2-/3-byte delimiter is fully loaded. */ +/** + * @brief Left-pack table: row `[m]` holds the 8 dword indices that gather the `m`-selected u64 lanes (of 4, + * each a dword pair) to the front for `_mm256_permutevar8x32_epi32`. + */ +static sz_u32_t const sz_utf8_compact_lut_haswell_[16][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, {0, 1, 0, 0, 0, 0, 0, 0}, {2, 3, 0, 0, 0, 0, 0, 0}, {0, 1, 2, 3, 0, 0, 0, 0}, + {4, 5, 0, 0, 0, 0, 0, 0}, {0, 1, 4, 5, 0, 0, 0, 0}, {2, 3, 4, 5, 0, 0, 0, 0}, {0, 1, 2, 3, 4, 5, 0, 0}, + {6, 7, 0, 0, 0, 0, 0, 0}, {0, 1, 6, 7, 0, 0, 0, 0}, {2, 3, 6, 7, 0, 0, 0, 0}, {0, 1, 2, 3, 6, 7, 0, 0}, + {4, 5, 6, 7, 0, 0, 0, 0}, {0, 1, 4, 5, 6, 7, 0, 0}, {2, 3, 4, 5, 6, 7, 0, 0}, {0, 1, 2, 3, 4, 5, 6, 7}, +}; + /** * @brief Peel the window's first `emit_count` matches with a `vpermd` left-pack, 4 lanes per sub-block. * Each sub-block gathers its set lanes to the front and masked-stores them at the advancing cursor. */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_haswell_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_haswell_( // sz_u32_t start_bits, sz_u32_t two_byte_starts, sz_u32_t three_byte_starts, // sz_size_t emit_count, sz_size_t position, // sz_size_t *match_offsets, sz_size_t *match_lengths) { - // Per-file copy of the sorting backend's left-pack table: row `[m]` holds the 8 dword indices that gather the - // `m`-selected u64 lanes (of 4, each a dword pair) to the front for `_mm256_permutevar8x32_epi32`. - static sz_u32_t const compact_lut[16][8] = { - {0, 0, 0, 0, 0, 0, 0, 0}, {0, 1, 0, 0, 0, 0, 0, 0}, {2, 3, 0, 0, 0, 0, 0, 0}, {0, 1, 2, 3, 0, 0, 0, 0}, - {4, 5, 0, 0, 0, 0, 0, 0}, {0, 1, 4, 5, 0, 0, 0, 0}, {2, 3, 4, 5, 0, 0, 0, 0}, {0, 1, 2, 3, 4, 5, 0, 0}, - {6, 7, 0, 0, 0, 0, 0, 0}, {0, 1, 6, 7, 0, 0, 0, 0}, {2, 3, 6, 7, 0, 0, 0, 0}, {0, 1, 2, 3, 6, 7, 0, 0}, - {4, 5, 6, 7, 0, 0, 0, 0}, {0, 1, 4, 5, 6, 7, 0, 0}, {2, 3, 4, 5, 6, 7, 0, 0}, {0, 1, 2, 3, 4, 5, 6, 7}, - }; - __m256i const lane_ramp_u64x4 = _mm256_setr_epi64x(0, 1, 2, 3); sz_size_t emitted = 0; for (sz_size_t sub_block = 0; sub_block < 8 && emitted < emit_count; ++sub_block) { @@ -74,7 +76,7 @@ SZ_HELPER_AUTO void sz_utf8_iterate_peel_haswell_( / _mm256_set1_epi64x(1), _mm256_add_epi64(two_byte_add_u64x4, _mm256_add_epi64(three_byte_add_u64x4, three_byte_add_u64x4))); - __m256i const permutation_u32x8 = _mm256_loadu_si256((__m256i const *)compact_lut[submask]); + __m256i const permutation_u32x8 = _mm256_loadu_si256((__m256i const *)sz_utf8_compact_lut_haswell_[submask]); __m256i const packed_offsets_u64x4 = _mm256_permutevar8x32_epi32(offsets_u64x4, permutation_u32x8); __m256i const packed_lengths_u64x4 = _mm256_permutevar8x32_epi32(lengths_u64x4, permutation_u32x8); @@ -275,8 +277,8 @@ SZ_HELPER_INLINE __m256i sz_delimiter_test_bit_haswell_(__m256i bitmap_byte_u8x3 * column reads (each a 64-entry `cascade_stage` over `block_id`) blended by which column `(low >> 3)` selects. * The transposed `..._columns_` layout (column c holds `bitmaps[id*32+c]`) makes each column lut256-addressable * for `block_id < 64` without a page network. */ -SZ_HELPER_AUTO __m256i sz_delimiter_bitmap_byte_haswell_(sz_u8_t const *columns, __m256i block_id_u8x32, - __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_delimiter_bitmap_byte_haswell_(sz_u8_t const *columns, __m256i block_id_u8x32, + __m256i low_u8x32) { __m256i const selector_u8x32 = _mm256_and_si256(_mm256_srli_epi16(block_id_u8x32, 4), _mm256_set1_epi8(0x0F)); // block_id>>4 (0..3) __m256i const within_u8x32 = _mm256_and_si256(block_id_u8x32, _mm256_set1_epi8(0x0F)); // block_id&15 @@ -294,8 +296,8 @@ SZ_HELPER_AUTO __m256i sz_delimiter_bitmap_byte_haswell_(sz_u8_t const *columns, /** @brief BMP (codepoint < 0x10000) delimiter membership for one 32-lane half, as a 0xFF/0x00 byte mask. ASCII lanes * (top bit clear) carry their codepoint in the raw byte, so are overridden to (high=0, low=byte). */ -SZ_HELPER_AUTO __m256i sz_delimiter_bmp_membership_haswell_(__m256i window_u8x32, __m256i high_in_u8x32, - __m256i low_in_u8x32) { +SZ_HELPER_INLINE __m256i sz_delimiter_bmp_membership_haswell_(__m256i window_u8x32, __m256i high_in_u8x32, + __m256i low_in_u8x32) { __m256i const ascii_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(window_u8x32, _mm256_set1_epi8((char)0x80)), _mm256_setzero_si256()); __m256i const high_u8x32 = _mm256_andnot_si256(ascii_u8x32, high_in_u8x32); @@ -309,7 +311,7 @@ SZ_HELPER_AUTO __m256i sz_delimiter_bmp_membership_haswell_(__m256i window_u8x32 /** @brief Astral (codepoint >= 0x10000) delimiter membership for one 32-lane half, as a 0xFF/0x00 byte mask. The full * 21-bit codepoint is reconstructed in byte-domain from the raw lead/continuation bytes; the small L1/L2 network * and bitmap are then walked exactly as for the BMP path. Only meaningful on 4-byte lead lanes (caller blends). */ -SZ_HELPER_AUTO __m256i sz_delimiter_astral_membership_haswell_( // +SZ_HELPER_INLINE __m256i sz_delimiter_astral_membership_haswell_( // __m256i window_u8x32, __m256i next1_u8x32, __m256i next2_u8x32, __m256i next3_u8x32) { __m256i const b0_u8x32 = _mm256_and_si256(window_u8x32, _mm256_set1_epi8(0x07)); // lead bits cp[20:18] __m256i const b1_u8x32 = _mm256_and_si256(next1_u8x32, _mm256_set1_epi8(0x3F)); // cp[17:12] @@ -348,7 +350,7 @@ SZ_HELPER_AUTO __m256i sz_delimiter_astral_membership_haswell_( // /** @brief Per-lane UTF-8 validity for codepoint-start lanes, mirroring `sz_rune_decode` exactly: a 2/3/4-byte lead is * valid only when its continuation bytes are present (within the loaded span) and well-formed, and it is not * overlong, a surrogate, or beyond U+10FFFF. Returned as a `sz_u64_t` lane mask. */ -SZ_HELPER_AUTO sz_u64_t sz_delimiter_valid_starts_haswell_( // +SZ_HELPER_INLINE sz_u64_t sz_delimiter_valid_starts_haswell_( // sz_utf8_rune_window_haswell_t const *decoded, __m256i next1_lo_u8x32, __m256i next1_hi_u8x32, __m256i next2_lo_u8x32, __m256i next2_hi_u8x32, __m256i next3_lo_u8x32, __m256i next3_hi_u8x32) { sz_size_t const loaded = decoded->loaded; @@ -368,16 +370,16 @@ SZ_HELPER_AUTO sz_u64_t sz_delimiter_valid_starts_haswell_( // // 2-byte: lead >= 0xC2 (reject C0/C1 overlong). sz_u64_t const lead_ge_c2 = sz_utf8_mask_combine_haswell_( - sz_delimiter_cmpge_epu8_haswell_(decoded->window_lo, _mm256_set1_epi8((char)0xC2)), - sz_delimiter_cmpge_epu8_haswell_(decoded->window_hi, _mm256_set1_epi8((char)0xC2))); + sz_delimiter_cmpge_epu8_haswell_(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xC2)), + sz_delimiter_cmpge_epu8_haswell_(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xC2))); // 3-byte: not overlong (E0 with next1 < 0xA0), not surrogate (ED with next1 >= 0xA0). sz_u64_t const lead_e0 = sz_utf8_mask_combine_haswell_( - _mm256_cmpeq_epi8(decoded->window_lo, _mm256_set1_epi8((char)0xE0)), - _mm256_cmpeq_epi8(decoded->window_hi, _mm256_set1_epi8((char)0xE0))); + _mm256_cmpeq_epi8(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xE0)), + _mm256_cmpeq_epi8(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xE0))); sz_u64_t const lead_ed = sz_utf8_mask_combine_haswell_( - _mm256_cmpeq_epi8(decoded->window_lo, _mm256_set1_epi8((char)0xED)), - _mm256_cmpeq_epi8(decoded->window_hi, _mm256_set1_epi8((char)0xED))); + _mm256_cmpeq_epi8(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xED)), + _mm256_cmpeq_epi8(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xED))); __m256i const n1_lo_ge_a0_u8x32 = sz_delimiter_cmpge_epu8_haswell_(next1_lo_u8x32, _mm256_set1_epi8((char)0xA0)); __m256i const n1_hi_ge_a0_u8x32 = sz_delimiter_cmpge_epu8_haswell_(next1_hi_u8x32, _mm256_set1_epi8((char)0xA0)); sz_u64_t const n1_ge_a0 = sz_utf8_mask_combine_haswell_(n1_lo_ge_a0_u8x32, n1_hi_ge_a0_u8x32); @@ -385,27 +387,27 @@ SZ_HELPER_AUTO sz_u64_t sz_delimiter_valid_starts_haswell_( // // 4-byte: lead <= 0xF4, not overlong (F0 with next1 < 0x90), not > U+10FFFF (F4 with next1 >= 0x90). sz_u64_t const lead_le_f4 = sz_utf8_mask_combine_haswell_( - _mm256_andnot_si256(sz_delimiter_cmpge_epu8_haswell_(decoded->window_lo, _mm256_set1_epi8((char)0xF5)), + _mm256_andnot_si256(sz_delimiter_cmpge_epu8_haswell_(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xF5)), _mm256_set1_epi8((char)0xFF)), - _mm256_andnot_si256(sz_delimiter_cmpge_epu8_haswell_(decoded->window_hi, _mm256_set1_epi8((char)0xF5)), + _mm256_andnot_si256(sz_delimiter_cmpge_epu8_haswell_(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xF5)), _mm256_set1_epi8((char)0xFF))); sz_u64_t const lead_f0 = sz_utf8_mask_combine_haswell_( - _mm256_cmpeq_epi8(decoded->window_lo, _mm256_set1_epi8((char)0xF0)), - _mm256_cmpeq_epi8(decoded->window_hi, _mm256_set1_epi8((char)0xF0))); + _mm256_cmpeq_epi8(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xF0)), + _mm256_cmpeq_epi8(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xF0))); sz_u64_t const lead_f4 = sz_utf8_mask_combine_haswell_( - _mm256_cmpeq_epi8(decoded->window_lo, _mm256_set1_epi8((char)0xF4)), - _mm256_cmpeq_epi8(decoded->window_hi, _mm256_set1_epi8((char)0xF4))); + _mm256_cmpeq_epi8(decoded->window_low_u8x32, _mm256_set1_epi8((char)0xF4)), + _mm256_cmpeq_epi8(decoded->window_high_u8x32, _mm256_set1_epi8((char)0xF4))); sz_u64_t const n1_ge_90 = sz_utf8_mask_combine_haswell_( sz_delimiter_cmpge_epu8_haswell_(next1_lo_u8x32, _mm256_set1_epi8((char)0x90)), sz_delimiter_cmpge_epu8_haswell_(next1_hi_u8x32, _mm256_set1_epi8((char)0x90))); sz_u64_t const n1_lt_90 = ~n1_ge_90; - sz_u64_t const ascii = loaded_mask & - ~sz_utf8_mask_combine_haswell_( - _mm256_cmpeq_epi8(_mm256_and_si256(decoded->window_lo, _mm256_set1_epi8((char)0x80)), - _mm256_set1_epi8((char)0x80)), - _mm256_cmpeq_epi8(_mm256_and_si256(decoded->window_hi, _mm256_set1_epi8((char)0x80)), - _mm256_set1_epi8((char)0x80))); + sz_u64_t const ascii = + loaded_mask & ~sz_utf8_mask_combine_haswell_( + _mm256_cmpeq_epi8(_mm256_and_si256(decoded->window_low_u8x32, _mm256_set1_epi8((char)0x80)), + _mm256_set1_epi8((char)0x80)), + _mm256_cmpeq_epi8(_mm256_and_si256(decoded->window_high_u8x32, _mm256_set1_epi8((char)0x80)), + _mm256_set1_epi8((char)0x80))); // Spans must lie within the loaded window (so we never validate against a wrapped neighbour or read past loaded). sz_u64_t const span2 = loaded >= 1 ? sz_u64_mask_until_serial_(loaded - 1) : 0; @@ -438,10 +440,10 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_haswell( // sz_size_t const loaded = decoded.loaded; __m256i next1_lo_u8x32, next1_hi_u8x32, next2_lo_u8x32, next2_hi_u8x32; - sz_utf8_forward_neighbours_haswell_(decoded.window_lo, decoded.window_hi, &next1_lo_u8x32, &next1_hi_u8x32, - &next2_lo_u8x32, &next2_hi_u8x32); + sz_utf8_forward_neighbours_haswell_(decoded.window_low_u8x32, decoded.window_high_u8x32, &next1_lo_u8x32, + &next1_hi_u8x32, &next2_lo_u8x32, &next2_hi_u8x32); __m256i next3_lo_u8x32, next3_hi_u8x32; - sz_delimiter_forward_neighbour3_haswell_(decoded.window_lo, decoded.window_hi, &next3_lo_u8x32, + sz_delimiter_forward_neighbour3_haswell_(decoded.window_low_u8x32, decoded.window_high_u8x32, &next3_lo_u8x32, &next3_hi_u8x32); // Effective-window trim: a multi-byte lead near the 64-byte edge whose span runs past `loaded` would decode @@ -462,14 +464,16 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_haswell( // // BMP membership for every lane; astral membership blended onto the four-byte lanes (gated on presence). sz_u64_t member = sz_utf8_mask_combine_haswell_( - sz_delimiter_bmp_membership_haswell_(decoded.window_lo, decoded.high_lo, decoded.low_lo), - sz_delimiter_bmp_membership_haswell_(decoded.window_hi, decoded.high_hi, decoded.low_hi)); + sz_delimiter_bmp_membership_haswell_(decoded.window_low_u8x32, decoded.high_byte_low_u8x32, + decoded.low_byte_low_u8x32), + sz_delimiter_bmp_membership_haswell_(decoded.window_high_u8x32, decoded.high_byte_high_u8x32, + decoded.low_byte_high_u8x32)); sz_u64_t const four_byte = decoded.four_byte_starts & span_mask; if (four_byte) { sz_u64_t const astral_member = sz_utf8_mask_combine_haswell_( - sz_delimiter_astral_membership_haswell_(decoded.window_lo, next1_lo_u8x32, next2_lo_u8x32, + sz_delimiter_astral_membership_haswell_(decoded.window_low_u8x32, next1_lo_u8x32, next2_lo_u8x32, next3_lo_u8x32), - sz_delimiter_astral_membership_haswell_(decoded.window_hi, next1_hi_u8x32, next2_hi_u8x32, + sz_delimiter_astral_membership_haswell_(decoded.window_high_u8x32, next1_hi_u8x32, next2_hi_u8x32, next3_hi_u8x32)); member = (member & ~four_byte) | (astral_member & four_byte); } diff --git a/include/stringzilla/utf8_tokens/icelake.h b/include/stringzilla/utf8_tokens/icelake.h index ea9d0895..7b1cc63d 100644 --- a/include/stringzilla/utf8_tokens/icelake.h +++ b/include/stringzilla/utf8_tokens/icelake.h @@ -230,8 +230,8 @@ SZ_HELPER_INLINE __m512i sz_delimiter_pack_chunks_epi8_icelake_(__m512i chunk0_u return result_u8x64; } -SZ_HELPER_AUTO __mmask64 sz_delimiter_bmp_membership_icelake_(__m512i window_u8x64, __m512i high_in_u8x64, - __m512i low_in_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_delimiter_bmp_membership_icelake_(__m512i window_u8x64, __m512i high_in_u8x64, + __m512i low_in_u8x64) { // The decode window only reconstructs `high`/`low` for 2-/3-byte leads; ASCII lanes (top bit clear) carry their // codepoint in the raw byte itself, so override them with (high=0, low=byte) before addressing the BMP tables. __mmask64 const ascii_m64 = ~_mm512_movepi8_mask(window_u8x64); @@ -276,8 +276,8 @@ SZ_HELPER_AUTO __mmask64 sz_delimiter_bmp_membership_icelake_(__m512i window_u8x * network: `super = offset>>16` selects an L1 group, `group*256 + ((offset>>8)&0xFF)` selects a bitmap row id, and the * bit `(offset & 7)` is tested. Resolved over all 64 lanes; the caller blends the result onto the four-byte lanes. */ -SZ_HELPER_AUTO __mmask64 sz_delimiter_astral_membership_icelake_(__m512i window_u8x64, __m512i next1_u8x64, - __m512i next2_u8x64, __m512i next3_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_delimiter_astral_membership_icelake_(__m512i window_u8x64, __m512i next1_u8x64, + __m512i next2_u8x64, __m512i next3_u8x64) { __m512i const byte0_u8x64 = _mm512_and_si512(window_u8x64, _mm512_set1_epi8(0x07)); __m512i const byte1_u8x64 = _mm512_and_si512(next1_u8x64, _mm512_set1_epi8(0x3F)); __m512i const byte2_u8x64 = _mm512_and_si512(next2_u8x64, _mm512_set1_epi8(0x3F)); @@ -340,7 +340,7 @@ SZ_HELPER_AUTO __mmask64 sz_delimiter_astral_membership_icelake_(__m512i window_ * overlong, a surrogate, or beyond U+10FFFF. Invalid leads are never reported (serial advances one byte and * re-syncs, which never matches the cleared lane). */ -SZ_HELPER_AUTO __mmask64 sz_delimiter_valid_starts_icelake_( // +SZ_HELPER_INLINE __mmask64 sz_delimiter_valid_starts_icelake_( // __m512i window_u8x64, __m512i next1_u8x64, __m512i next2_u8x64, __m512i next3_u8x64, sz_utf8_rune_window_t const *decoded) { __mmask64 const loaded_m64 = sz_u64_clamp_mask_until_(decoded->loaded); @@ -402,7 +402,7 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_icelake( // sz_utf8_rune_window_t const decoded = sz_utf8_rune_decode_window_icelake_(text_u8 + base, length - base, lane_identity_u8x64); sz_size_t const loaded = decoded.loaded; - __m512i const window_u8x64 = decoded.window; + __m512i const window_u8x64 = decoded.window_u8x64; __m512i const next1_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), window_u8x64); __m512i const next2_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(2)), @@ -427,7 +427,8 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_icelake( // decoded.codepoint_starts & span_mask_m64; // BMP membership for every lane; astral membership blended onto the four-byte lanes (gated on presence). - __mmask64 member_m64 = sz_delimiter_bmp_membership_icelake_(window_u8x64, decoded.high, decoded.low); + __mmask64 member_m64 = sz_delimiter_bmp_membership_icelake_(window_u8x64, decoded.high_byte_u8x64, + decoded.low_byte_u8x64); __mmask64 const four_byte_m64 = decoded.four_byte_starts & span_mask_m64; if (four_byte_m64) { __mmask64 const astral_member_m64 = sz_delimiter_astral_membership_icelake_(window_u8x64, next1_u8x64, diff --git a/include/stringzilla/utf8_tokens/lasx.h b/include/stringzilla/utf8_tokens/lasx.h index 06825018..03733547 100644 --- a/include/stringzilla/utf8_tokens/lasx.h +++ b/include/stringzilla/utf8_tokens/lasx.h @@ -18,7 +18,7 @@ extern "C" { /** @brief Peel the tile's first `emit_count` matches with a `__lasx_xvperm_w` left-pack, 4 lanes per sub-block. * Each sub-block gathers its set `(position+lane, length)` pairs to the front (same dword-index table as * `sz_utf8_iterate_peel_haswell_`) and element-stores `min(popcount, remaining)` at the advancing cursor. */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_lasx_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_lasx_( // sz_u32_t start_bits, sz_u32_t two_byte_starts, sz_u32_t three_byte_starts, // sz_size_t emit_count, sz_size_t position, // sz_size_t *match_offsets, sz_size_t *match_lengths) { diff --git a/include/stringzilla/utf8_tokens/neon.h b/include/stringzilla/utf8_tokens/neon.h index 641013af..4db37a93 100644 --- a/include/stringzilla/utf8_tokens/neon.h +++ b/include/stringzilla/utf8_tokens/neon.h @@ -70,7 +70,7 @@ SZ_HELPER_INLINE sz_size_t sz_utf8_iterate_compact4_neon_( // * @brief Peel the tile's first @p emit_count matches by SIMD left-pack over four 4-lane sub-blocks into a * fixed-width stack scratch, then copy the surviving prefix to the caller (no `ctz`, no per-match branch). */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_neon_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_neon_( // sz_u64_t start_bits, uint8x16_t length_per_lane_u8x16, // sz_size_t emit_count, sz_size_t position, // sz_size_t *match_offsets, sz_size_t *match_lengths) { @@ -325,8 +325,8 @@ SZ_HELPER_INLINE uint8x16_t sz_delimiter_test_bit_neon_(uint8x16_t bitmap_byte_u * three highs per quarter, and each round loads that block's 32-byte row into a `vqtbl2q_u8` pair and settles * every lane carrying it - no per-lane scalar walk. */ -SZ_HELPER_AUTO uint8x16_t sz_delimiter_bmp_membership_neon_(uint8x16_t window_u8x16, uint8x16_t high_in_u8x16, - uint8x16_t low_in_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_delimiter_bmp_membership_neon_(uint8x16_t window_u8x16, uint8x16_t high_in_u8x16, + uint8x16_t low_in_u8x16) { uint8x16_t const is_ascii_u8x16 = vcltq_u8(window_u8x16, vdupq_n_u8(0x80)); uint8x16_t const high_u8x16 = vbicq_u8(high_in_u8x16, is_ascii_u8x16); // high = is_ascii ? 0 : high_in uint8x16_t const low_u8x16 = vbslq_u8(is_ascii_u8x16, window_u8x16, low_in_u8x16); // low = is_ascii ? byte : low_in @@ -365,8 +365,8 @@ SZ_HELPER_AUTO uint8x16_t sz_delimiter_bmp_membership_neon_(uint8x16_t window_u8 * `low8 = cp & 0xFF`. The `super` (0..15) selects an L1 group; `group*256 + sub` selects a bitmap row id (group < 2, * so the two 256-entry halves of the L2 table are read and blended by the group bit); the bit `(low8 & 7)` is tested. */ -SZ_HELPER_AUTO uint8x16_t sz_delimiter_astral_membership_neon_(uint8x16_t window_u8x16, uint8x16_t next1_u8x16, - uint8x16_t next2_u8x16, uint8x16_t next3_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_delimiter_astral_membership_neon_(uint8x16_t window_u8x16, uint8x16_t next1_u8x16, + uint8x16_t next2_u8x16, uint8x16_t next3_u8x16) { uint8x16_t const b0_u8x16 = vandq_u8(window_u8x16, vdupq_n_u8(0x07)); uint8x16_t const b1_u8x16 = vandq_u8(next1_u8x16, vdupq_n_u8(0x3F)); uint8x16_t const b2_u8x16 = vandq_u8(next2_u8x16, vdupq_n_u8(0x3F)); @@ -412,13 +412,13 @@ SZ_HELPER_AUTO uint8x16_t sz_delimiter_astral_membership_neon_(uint8x16_t window * continuation bytes wrap is rejected) is applied by the caller via `byte_span`; here the substrate masks are * already loaded-clamped. An invalid lead is never reported (serial advances one byte and re-syncs). */ -SZ_HELPER_AUTO sz_u64_t sz_delimiter_valid_starts_neon_(sz_utf8_rune_window_neon_t const *decoded, - uint8x16_t const *next1_u8x16, uint8x16_t const *next2_u8x16, - uint8x16_t const *next3_u8x16) { +SZ_HELPER_INLINE sz_u64_t sz_delimiter_valid_starts_neon_(sz_utf8_rune_window_neon_t const *decoded, + uint8x16_t const *next1_u8x16, uint8x16_t const *next2_u8x16, + uint8x16_t const *next3_u8x16) { uint8x16_t const continuation_mask_u8x16 = vdupq_n_u8(0xC0), continuation_pattern_u8x16 = vdupq_n_u8(0x80); uint8x16_t valid_bool_u8x16[4]; for (int quarter = 0; quarter < 4; ++quarter) { - uint8x16_t const here_u8x16 = decoded->window[quarter]; + uint8x16_t const here_u8x16 = decoded->window_u8x16s[quarter]; uint8x16_t const n1_u8x16 = next1_u8x16[quarter]; uint8x16_t const c1_ok_u8x16 = vceqq_u8(vandq_u8(n1_u8x16, continuation_mask_u8x16), continuation_pattern_u8x16); @@ -501,14 +501,15 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_neon( // uint8x16_t member_bool_u8x16[4]; for (int quarter = 0; quarter < 4; ++quarter) member_bool_u8x16[quarter] = sz_delimiter_test_bit_neon_( - vqtbl1q_u8(row0_u8x16, vshrq_n_u8(decoded.window[quarter], 3)), decoded.window[quarter]); + vqtbl1q_u8(row0_u8x16, vshrq_n_u8(decoded.window_u8x16s[quarter], 3)), + decoded.window_u8x16s[quarter]); hits = sz_utf8_mask_combine_neon_(member_bool_u8x16[0], member_bool_u8x16[1], member_bool_u8x16[2], member_bool_u8x16[3]) & loaded_mask; } else { uint8x16_t next1_u8x16[4], next2_u8x16[4], next3_u8x16[4]; - sz_utf8_forward_neighbours_neon_(decoded.window, next1_u8x16, next2_u8x16, next3_u8x16); + sz_utf8_forward_neighbours_neon_(decoded.window_u8x16s, next1_u8x16, next2_u8x16, next3_u8x16); // Effective-window trim: a multi-byte lead near the 64-byte edge whose span runs past `loaded` would // decode against a wrapped neighbour; defer it to the next window. @@ -525,8 +526,9 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_neon( // sz_u64_t const four_byte = decoded.four_byte_starts & span_mask; sz_u64_t member = 0; for (int quarter = 0; quarter < 4; ++quarter) { - uint8x16_t const bmp_u8x16 = sz_delimiter_bmp_membership_neon_( - decoded.window[quarter], decoded.high[quarter], decoded.low[quarter]); + uint8x16_t const bmp_u8x16 = sz_delimiter_bmp_membership_neon_(decoded.window_u8x16s[quarter], + decoded.high_byte_u8x16s[quarter], + decoded.low_byte_u8x16s[quarter]); member |= sz_utf8_movemask16_neon_(bmp_u8x16) << (16 * quarter); } // Blend astral over the four-byte lanes; the per-lane BMP/astral decision stays exact on the bit masks. @@ -534,7 +536,8 @@ SZ_API_COMPTIME sz_size_t sz_utf8_delimiters_neon( // sz_u64_t astral_member = 0; for (int quarter = 0; quarter < 4; ++quarter) { uint8x16_t const astral_u8x16 = sz_delimiter_astral_membership_neon_( - decoded.window[quarter], next1_u8x16[quarter], next2_u8x16[quarter], next3_u8x16[quarter]); + decoded.window_u8x16s[quarter], next1_u8x16[quarter], next2_u8x16[quarter], + next3_u8x16[quarter]); astral_member |= sz_utf8_movemask16_neon_(astral_u8x16) << (16 * quarter); } member = (member & ~four_byte) | (astral_member & four_byte); diff --git a/include/stringzilla/utf8_tokens/powervsx.h b/include/stringzilla/utf8_tokens/powervsx.h index 1472612a..67ff2e49 100644 --- a/include/stringzilla/utf8_tokens/powervsx.h +++ b/include/stringzilla/utf8_tokens/powervsx.h @@ -28,7 +28,7 @@ extern "C" { * `vec_perm` and full-stores to an 18-wide scratch (absorbing the last sub-block's 2-lane spill, since * VSX has no masked store); the low `emit_count` entries copy out in ascending lane order, byte-exact. */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_powervsx_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_powervsx_( // sz_u32_t start_bits, sz_u32_t two_byte_starts, sz_u32_t three_byte_starts, // sz_size_t emit_count, sz_size_t position, // sz_size_t *match_offsets, sz_size_t *match_lengths) { diff --git a/include/stringzilla/utf8_tokens/rvv.h b/include/stringzilla/utf8_tokens/rvv.h index 463b70d9..783e1f5f 100644 --- a/include/stringzilla/utf8_tokens/rvv.h +++ b/include/stringzilla/utf8_tokens/rvv.h @@ -31,7 +31,7 @@ SZ_HELPER_INLINE sz_size_t sz_utf8_iterate_tile_bytes_rvv_(void) { } /** @brief Peel a window's first `emit_count` matches: compress lane offsets + lengths, widen-store absolute pairs. */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_tile_rvv_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_tile_rvv_( // vuint8m4_t length_u8m4, vbool2_t start_mask_b2, sz_size_t tile_position, // sz_size_t vector_length, sz_size_t emit_count, sz_size_t *match_offsets, sz_size_t *match_lengths) { @@ -66,9 +66,9 @@ SZ_HELPER_AUTO void sz_utf8_iterate_peel_tile_rvv_( // /* Classify a tile into a per-lane byte-length vector (0 = no delimiter starts here). The 2nd/3rd bytes are * carried from the buffer via `next`/`after_next`; multi-byte masks are computed unconditionally. */ -SZ_HELPER_AUTO vuint8m4_t sz_utf8_classify_newlines_rvv_(sz_u8_t const *text_u8, sz_size_t position, - vuint8m4_t bytes_u8m4, vuint8m4_t next_u8m4, - vuint8m4_t after_next_u8m4, sz_size_t vector_length) { +SZ_HELPER_INLINE vuint8m4_t sz_utf8_classify_newlines_rvv_(sz_u8_t const *text_u8, sz_size_t position, + vuint8m4_t bytes_u8m4, vuint8m4_t next_u8m4, + vuint8m4_t after_next_u8m4, sz_size_t vector_length) { vuint8m4_t length_u8m4 = __riscv_vmv_v_x_u8m4(0, vector_length); // '\n' '\v' '\f' (0x0A-0x0C) and a lone '\r' (0x0D): length 1. vbool2_t is_lf_vt_ff_b2 = __riscv_vmsltu_vx_u8m4_b2(__riscv_vsub_vx_u8m4(bytes_u8m4, 0x0A, vector_length), 3, @@ -102,8 +102,8 @@ SZ_HELPER_AUTO vuint8m4_t sz_utf8_classify_newlines_rvv_(sz_u8_t const *text_u8, return __riscv_vmerge_vxm_u8m4(length_u8m4, 0, is_lf_of_crlf_b2, vector_length); } -SZ_HELPER_AUTO vuint8m4_t sz_utf8_classify_whitespaces_rvv_(vuint8m4_t bytes_u8m4, vuint8m4_t next_u8m4, - vuint8m4_t after_next_u8m4, sz_size_t vector_length) { +SZ_HELPER_INLINE vuint8m4_t sz_utf8_classify_whitespaces_rvv_(vuint8m4_t bytes_u8m4, vuint8m4_t next_u8m4, + vuint8m4_t after_next_u8m4, sz_size_t vector_length) { vuint8m4_t length_u8m4 = __riscv_vmv_v_x_u8m4(0, vector_length); // ASCII whitespace: '\t'-'\r' (0x09-0x0D) and ' ' (0x20): length 1. vbool2_t is_ascii_ws_b2 = __riscv_vmor_mm_b2( @@ -150,9 +150,9 @@ SZ_HELPER_AUTO vuint8m4_t sz_utf8_classify_whitespaces_rvv_(vuint8m4_t bytes_u8m /* Shared window/carry/trusted-lane/peel scaffolding for both delimiter sets. `classify_newlines` selects the * per-lane classifier and the post-loop CRLF straddle fixup; otherwise the two paths are identical. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_iterate_multistep_rvv_( // - sz_cptr_t text, sz_size_t length, // - sz_size_t *match_offsets, sz_size_t *match_lengths, // +SZ_HELPER_INLINE sz_size_t sz_utf8_iterate_multistep_rvv_( // + sz_cptr_t text, sz_size_t length, // + sz_size_t *match_offsets, sz_size_t *match_lengths, // sz_size_t matches_capacity, sz_size_t *bytes_consumed, int classify_newlines) { sz_u8_t const *text_u8 = (sz_u8_t const *)text; diff --git a/include/stringzilla/utf8_tokens/serial.h b/include/stringzilla/utf8_tokens/serial.h index e53b464d..fb8d5860 100644 --- a/include/stringzilla/utf8_tokens/serial.h +++ b/include/stringzilla/utf8_tokens/serial.h @@ -20,7 +20,7 @@ extern "C" { * A @c "\r\n" CRLF is one match of length 2 (its trailing LF is never emitted alone). `base` is added to every * emitted offset and to `*bytes_consumed`, the resume offset, which is always a true delimiter boundary. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_newlines_serial_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_newlines_serial_( // sz_cptr_t text, sz_size_t length, sz_size_t base, // sz_size_t *match_offsets, sz_size_t *match_lengths, // sz_size_t matches_capacity, sz_size_t *bytes_consumed) { @@ -61,7 +61,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_newlines_serial_( // * Same contract as `sz_utf8_newlines_serial_` but for the Unicode White_Space set. There is no CRLF * merging here - CR and LF are independent length-1 matches. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_whitespaces_serial_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_whitespaces_serial_( // sz_cptr_t text, sz_size_t length, sz_size_t base, // sz_size_t *match_offsets, sz_size_t *match_lengths, // sz_size_t matches_capacity, sz_size_t *bytes_consumed) { diff --git a/include/stringzilla/utf8_tokens/sve2.h b/include/stringzilla/utf8_tokens/sve2.h index e66a960a..6a044e12 100644 --- a/include/stringzilla/utf8_tokens/sve2.h +++ b/include/stringzilla/utf8_tokens/sve2.h @@ -30,7 +30,7 @@ extern "C" { * rides a biased iota) and one `svcompact_u32` packs the start indices while a lockstep `svcompact_u32` packs * their byte lengths; the offsets turn absolute only after the 64-bit widening, so multi-gigabyte inputs never * truncate. The caller's `while` loop resumes past the last emitted match when the capacity cuts the tile. */ -SZ_HELPER_AUTO void sz_utf8_token_drain_sve2_( // +SZ_HELPER_INLINE void sz_utf8_token_drain_sve2_( // svbool_t starts_b8x, svuint8_t lengths_u8x, sz_size_t position, sz_size_t span, // sz_size_t emit_count, sz_size_t *match_offsets, sz_size_t *match_lengths) { diff --git a/include/stringzilla/utf8_tokens/tables.h b/include/stringzilla/utf8_tokens/tables.h index d7ebf616..dac75758 100644 --- a/include/stringzilla/utf8_tokens/tables.h +++ b/include/stringzilla/utf8_tokens/tables.h @@ -1,6 +1,6 @@ /** * @brief UTF-8 delimiter membership tables (General_Category Punctuation/Symbol/Separator union White_Space). - * @file include/stringzilla/utf8_tokens/delimiters_tables.h + * @file include/stringzilla/utf8_tokens/tables.h * @author Ash Vardanian * * A "delimiter" is any codepoint whose Unicode General_Category is one of diff --git a/include/stringzilla/utf8_tokens/v128.h b/include/stringzilla/utf8_tokens/v128.h index f9f510d4..69dad925 100644 --- a/include/stringzilla/utf8_tokens/v128.h +++ b/include/stringzilla/utf8_tokens/v128.h @@ -36,7 +36,7 @@ SZ_HELPER_INLINE v128_t sz_utf8_rotate2_v128_(v128_t bytes_u8x16) { * `(position+lane, length)` pairs to the front of a 16-wide stack scratch via one swizzle from `compact_lut`, * then copies the low @p emit_count entries out - ascending lane order, byte-exact, no per-match `ctz`. */ -SZ_HELPER_AUTO void sz_utf8_iterate_peel_v128_( // +SZ_HELPER_INLINE void sz_utf8_iterate_peel_v128_( // sz_u32_t start_bits, sz_u32_t two_byte_starts, sz_u32_t three_byte_starts, // sz_size_t emit_count, sz_size_t position, // sz_size_t *match_offsets, sz_size_t *match_lengths) { diff --git a/include/stringzilla/utf8_uncased/haswell.h b/include/stringzilla/utf8_uncased/haswell.h index beef3990..dde0643a 100644 --- a/include/stringzilla/utf8_uncased/haswell.h +++ b/include/stringzilla/utf8_uncased/haswell.h @@ -77,7 +77,7 @@ SZ_HELPER_INLINE sz_u32_t sz_utf8_uncased_haswell_mask_until_(sz_size_t n) { * no probe inside a valid window and trip no alarm, so tail chunks reuse the main-loop * logic unchanged instead of branching into a separate epilogue. */ -SZ_HELPER_AUTO __m256i sz_utf8_uncased_haswell_load_padded_ymm_(sz_cptr_t source, sz_size_t length) { +SZ_HELPER_INLINE __m256i sz_utf8_uncased_haswell_load_padded_ymm_(sz_cptr_t source, sz_size_t length) { sz_u8_t buffer[32] = {0}; for (sz_size_t byte_index = 0; byte_index < length; ++byte_index) buffer[byte_index] = (sz_u8_t)source[byte_index]; return _mm256_lddqu_si256((__m256i const *)buffer); @@ -88,7 +88,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_uncased_haswell_load_padded_ymm_(sz_cptr_t source * haystack: the fast full load is taken whenever 16 bytes remain, and only the last few * candidates near the haystack end pay for the zero-padded stack copy. */ -SZ_HELPER_AUTO __m128i sz_utf8_uncased_haswell_load_window_xmm_(sz_cptr_t source, sz_size_t available) { +SZ_HELPER_INLINE __m128i sz_utf8_uncased_haswell_load_window_xmm_(sz_cptr_t source, sz_size_t available) { if (available >= 16) return _mm_lddqu_si128((__m128i const *)source); sz_u8_t buffer[16] = {0}; for (sz_size_t byte_index = 0; byte_index < available; ++byte_index) @@ -104,7 +104,7 @@ SZ_HELPER_AUTO __m128i sz_utf8_uncased_haswell_load_window_xmm_(sz_cptr_t source * @brief Fold a YMM register using ASCII case folding rules. * @sa sz_utf8_uncased_rune_ascii_invariant_k */ -SZ_HELPER_AUTO __m256i sz_utf8_uncased_search_haswell_ascii_fold_ymm_(__m256i text_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_uncased_search_haswell_ascii_fold_ymm_(__m256i text_u8x32) { // Only fold bytes in range A-Z; the masked add avoids `VPBLENDVB` (2 uops on Haswell) __m256i is_ascii_upper_u8x32 = sz_utf8_uncased_haswell_in_byte_range_(text_u8x32, 'A', 26); return _mm256_add_epi8(text_u8x32, _mm256_and_si256(is_ascii_upper_u8x32, _mm256_set1_epi8(0x20))); @@ -119,10 +119,10 @@ SZ_HELPER_AUTO __m256i sz_utf8_uncased_search_haswell_ascii_fold_ymm_(__m256i te * and the probe equality masks are shifted as 32-bit `VPMOVMSKB` integers: with windows ≤ 16 * bytes every chunk still exposes â‰Ĩ 17 valid start positions per iteration. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_ascii_3probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_ascii_3probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { sz_size_t const folded_window_length = needle_metadata->folded_slice_length; @@ -343,10 +343,10 @@ SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_scripted_( // * and no alarm - ASCII never changes byte width when folded, so the danger machinery * compiles away entirely and the step covers every valid start position. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_ascii_4probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_ascii_4probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_ascii_fold_ymm_, @@ -451,10 +451,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_western_europe_alarm_ * @brief Western European uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_western_europe_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_western_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_western_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_western_europe_fold_ymm_, @@ -557,10 +557,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_central_europe_alarm_ * @brief Central European uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_central_europe_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_central_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_central_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_central_europe_fold_ymm_, @@ -635,10 +635,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_cyrillic_alarm_ymm_(_ * @brief Cyrillic uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_cyrillic_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_cyrillic_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_cyrillic_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_cyrillic_fold_ymm_, @@ -720,10 +720,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_armenian_alarm_ymm_(_ * @brief Armenian uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_armenian_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_armenian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_armenian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_armenian_fold_ymm_, @@ -853,10 +853,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_greek_alarm_ymm_(__m2 * @brief Greek uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_greek_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_greek_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_greek_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_greek_fold_ymm_, @@ -993,10 +993,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_vietnamese_alarm_ymm_ * @brief Vietnamese uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_vietnamese_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_vietnamese_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_vietnamese_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_vietnamese_fold_ymm_, @@ -1053,10 +1053,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_haswell_georgian_alarm_ymm_(_ * The fastest non-ASCII kernel: Mkhedruli is caseless, so the fold callback is just the * ASCII fold for mixed Latin text and the alarm only watches for the historical scripts. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_haswell_georgian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_haswell_georgian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_haswell_scripted_( // sz_utf8_uncased_search_haswell_ascii_fold_ymm_, diff --git a/include/stringzilla/utf8_uncased/icelake.h b/include/stringzilla/utf8_uncased/icelake.h index 8e7b0012..de87a85c 100644 --- a/include/stringzilla/utf8_uncased/icelake.h +++ b/include/stringzilla/utf8_uncased/icelake.h @@ -55,10 +55,10 @@ SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_ascii_fold_zmm_(__m512i * VPTERNLOG to combine all 3, and VPTESTNMB to find matches. * No window verification needed since probes cover the entire window. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_ascii_3probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_ascii_3probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { sz_size_t const folded_window_length = needle_metadata->folded_slice_length; @@ -169,10 +169,10 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_ascii_3probe_( // * and VPTESTNMB to find matches. Window verification IS required since probes * don't cover all positions in the folded window. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_ascii_4probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_ascii_4probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { sz_size_t const folded_window_length = needle_metadata->folded_slice_length; @@ -479,7 +479,7 @@ SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_scripted_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_western_europe_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_western_europe_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -559,7 +559,7 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_western_europe_fold_ef * @param text_u8x64 The haystack ZMM register. * @return Bitmask of positions where danger characters are detected. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_western_europe_alarm_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_western_europe_alarm_naively_zmm_(__m512i text_u8x64) { // Lead byte constants __m512i const x_e1_u8x64 = _mm512_set1_epi8((char)0xE1); __m512i const x_e2_u8x64 = _mm512_set1_epi8((char)0xE2); @@ -697,10 +697,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_western_europe_alarm * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_western_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_western_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_western_europe_fold_efficiently_zmm_, @@ -719,7 +719,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_western_europe_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_central_europe_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_central_europe_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -926,7 +926,7 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_central_europe_fold_ef * @param text_u8x64 The haystack ZMM register. * @return Bitmask of positions where danger characters are detected. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_central_europe_alarm_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_central_europe_alarm_naively_zmm_(__m512i text_u8x64) { // Lead byte constants __m512i const x_e2_u8x64 = _mm512_set1_epi8((char)0xE2); // for Kelvin sign __m512i const x_c3_u8x64 = _mm512_set1_epi8((char)0xC3); // for Sharp S @@ -1035,10 +1035,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_central_europe_alarm * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_central_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_central_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_central_europe_fold_efficiently_zmm_, @@ -1059,7 +1059,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_central_europe_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_cyrillic_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_cyrillic_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -1189,7 +1189,7 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_cyrillic_fold_efficien * @param text_u8x64 The haystack ZMM register. * @return Bitmask of positions where danger characters are detected. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_cyrillic_alarm_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_cyrillic_alarm_naively_zmm_(__m512i text_u8x64) { __mmask64 is_e1_m64 = _mm512_cmpeq_epi8_mask(text_u8x64, _mm512_set1_epi8((char)0xE1)); __mmask64 is_b2_m64 = _mm512_cmpeq_epi8_mask(text_u8x64, _mm512_set1_epi8((char)0xB2)); __mmask64 is_folding_third_m64 = _mm512_cmplt_epu8_mask(_mm512_sub_epi8(text_u8x64, _mm512_set1_epi8((char)0x80)), @@ -1236,10 +1236,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_cyrillic_alarm_effic * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_cyrillic_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_cyrillic_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_cyrillic_fold_efficiently_zmm_, @@ -1258,7 +1258,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_cyrillic_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_armenian_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_armenian_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -1402,7 +1402,7 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_armenian_fold_efficien * @param text_u8x64 The haystack ZMM register. * @return Bitmask of positions where danger characters are detected. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_armenian_alarm_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_armenian_alarm_naively_zmm_(__m512i text_u8x64) { // Lead byte constants __m512i const x_d6_u8x64 = _mm512_set1_epi8((char)0xD6); // for Ech-Yiwn __m512i const x_ef_u8x64 = _mm512_set1_epi8((char)0xEF); // for ligatures @@ -1454,10 +1454,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_armenian_alarm_effic * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_armenian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_armenian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_armenian_fold_efficiently_zmm_, @@ -1476,7 +1476,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_armenian_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_greek_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_greek_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -1703,7 +1703,7 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_greek_fold_efficiently * @param text_u8x64 The haystack ZMM register. * @return Bitmask of positions where danger characters are detected. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_greek_alarm_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_greek_alarm_naively_zmm_(__m512i text_u8x64) { // All constants local to function __m512i const x_ce_u8x64 = _mm512_set1_epi8((char)0xCE); __m512i const x_cf_u8x64 = _mm512_set1_epi8((char)0xCF); @@ -1815,10 +1815,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_greek_alarm_efficien * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_greek_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_greek_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_greek_fold_efficiently_zmm_, @@ -1837,7 +1837,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_greek_( // * @param text_u8x64 The text ZMM register. * @return The folded ZMM register. */ -SZ_HELPER_AUTO __m512i sz_utf8_uncased_search_icelake_vietnamese_fold_naively_zmm_(__m512i text_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_uncased_search_icelake_vietnamese_fold_naively_zmm_(__m512i text_u8x64) { // Start with ASCII folded __m512i result_u8x64 = sz_utf8_uncased_search_icelake_ascii_fold_zmm_(text_u8x64); @@ -2076,8 +2076,8 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_vietnamese_fold_effici * @param load_m64 Mask of valid bytes in the ZMM register. * @return Bitmask of positions where danger characters are detected (at sequence start). */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_vietnamese_alarm_naively_zmm_(__m512i text_u8x64, - __mmask64 load_m64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_vietnamese_alarm_naively_zmm_(__m512i text_u8x64, + __mmask64 load_m64) { // Lead byte constants __m512i const x_e1_u8x64 = _mm512_set1_epi8((char)0xE1); __m512i const x_c3_u8x64 = _mm512_set1_epi8((char)0xC3); @@ -2203,10 +2203,10 @@ SZ_HELPER_NOINLINE __mmask64 sz_utf8_uncased_search_icelake_vietnamese_alarm_eff * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_vietnamese_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_vietnamese_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_vietnamese_fold_efficiently_zmm_, @@ -2230,7 +2230,7 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_vietnamese_( // * * All Georgian scripts use 3-byte UTF-8, so no length changes during folding. */ -SZ_HELPER_AUTO __mmask64 sz_utf8_uncased_search_icelake_georgian_alarm_zmm_(__m512i text_u8x64, __mmask64 load_m64) { +SZ_HELPER_INLINE __mmask64 sz_utf8_uncased_search_icelake_georgian_alarm_zmm_(__m512i text_u8x64, __mmask64 load_m64) { sz_unused_(load_m64); // Present for the shared `sz_utf8_uncased_alarm_zmm_t` signature // Lead byte detection @@ -2349,10 +2349,10 @@ SZ_HELPER_NOINLINE __m512i sz_utf8_uncased_search_icelake_georgian_fold_zmm_(__m * @param matched_length Haystack bytes consumed by the match. * @return Pointer to match start or SZ_NULL_CHAR if not found. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_icelake_georgian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_icelake_georgian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_icelake_scripted_( // sz_utf8_uncased_search_icelake_georgian_fold_zmm_, diff --git a/include/stringzilla/utf8_uncased/neon.h b/include/stringzilla/utf8_uncased/neon.h index 8d4cd24b..65b21d48 100644 --- a/include/stringzilla/utf8_uncased/neon.h +++ b/include/stringzilla/utf8_uncased/neon.h @@ -111,7 +111,7 @@ SZ_HELPER_INLINE sz_u32_t sz_utf8_uncased_neon_mask_until_(sz_size_t n) { * probe inside a valid window and trip no alarm, so tail chunks reuse the main-loop logic * unchanged instead of branching into a separate epilogue. */ -SZ_HELPER_AUTO uint8x16x2_t sz_utf8_uncased_neon_load_padded_u8x16x2_(sz_cptr_t source, sz_size_t length) { +SZ_HELPER_INLINE uint8x16x2_t sz_utf8_uncased_neon_load_padded_u8x16x2_(sz_cptr_t source, sz_size_t length) { sz_u8_t buffer[32] = {0}; for (sz_size_t byte_index = 0; byte_index < length; ++byte_index) buffer[byte_index] = (sz_u8_t)source[byte_index]; return vld1q_u8_x2(buffer); @@ -122,7 +122,7 @@ SZ_HELPER_AUTO uint8x16x2_t sz_utf8_uncased_neon_load_padded_u8x16x2_(sz_cptr_t * haystack: the fast full load is taken whenever 16 bytes remain, and only the last few * candidates near the haystack end pay for the zero-padded stack copy. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_uncased_neon_load_window_u8x16_(sz_cptr_t source, sz_size_t available) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_uncased_neon_load_window_u8x16_(sz_cptr_t source, sz_size_t available) { if (available >= 16) return vld1q_u8((sz_u8_t const *)source); sz_u8_t buffer[16] = {0}; for (sz_size_t byte_index = 0; byte_index < available; ++byte_index) @@ -138,7 +138,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_uncased_neon_load_window_u8x16_(sz_cptr_t sour * @brief Fold a 32-byte chunk using ASCII case folding rules. * @sa sz_utf8_uncased_rune_ascii_invariant_k */ -SZ_HELPER_AUTO uint8x16x2_t sz_utf8_uncased_search_neon_ascii_fold_u8x16x2_(uint8x16x2_t text_u8x16x2) { +SZ_HELPER_INLINE uint8x16x2_t sz_utf8_uncased_search_neon_ascii_fold_u8x16x2_(uint8x16x2_t text_u8x16x2) { uint8x16x2_t result_u8x16x2; // Only fold bytes in range A-Z; the masked add stays branch-free across both registers result_u8x16x2.val[0] = sz_utf8_fold_neon_ascii_(text_u8x16x2.val[0]); @@ -155,10 +155,10 @@ SZ_HELPER_AUTO uint8x16x2_t sz_utf8_uncased_search_neon_ascii_fold_u8x16x2_(uint * 32-bit movemask integers: with windows ≤ 16 bytes every chunk still exposes â‰Ĩ 17 valid start * positions per iteration. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_ascii_3probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_ascii_3probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { sz_size_t const folded_window_length = needle_metadata->folded_slice_length; @@ -398,10 +398,10 @@ SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_scripted_( // * and no alarm - ASCII never changes byte width when folded, so the danger machinery * compiles away entirely and the step covers every valid start position. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_ascii_4probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_ascii_4probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_ascii_fold_u8x16x2_, @@ -517,10 +517,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_western_europe_alarm_u8x * @brief Western European uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_western_europe_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_western_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_western_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_western_europe_fold_u8x16x2_, @@ -631,10 +631,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_central_europe_alarm_u8x * @brief Central European uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_central_europe_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_central_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_central_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_central_europe_fold_u8x16x2_, @@ -723,10 +723,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_cyrillic_alarm_u8x16x2_( * @brief Cyrillic uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_cyrillic_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_cyrillic_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_cyrillic_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_cyrillic_fold_u8x16x2_, @@ -824,10 +824,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_armenian_alarm_u8x16x2_( * @brief Armenian uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_armenian_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_armenian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_armenian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_armenian_fold_u8x16x2_, @@ -971,10 +971,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_greek_alarm_u8x16x2_(uin * @brief Greek uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_greek_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_greek_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_greek_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_greek_fold_u8x16x2_, @@ -1110,10 +1110,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_vietnamese_alarm_u8x16x2 * @brief Vietnamese uncased search for needles with safe slices up to 16 bytes. * @sa sz_utf8_uncased_rune_safe_vietnamese_k */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_vietnamese_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_vietnamese_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_vietnamese_fold_u8x16x2_, @@ -1175,10 +1175,10 @@ SZ_HELPER_NOINLINE sz_u32_t sz_utf8_uncased_search_neon_georgian_alarm_u8x16x2_( * The fastest non-ASCII kernel: Mkhedruli is caseless, so the fold callback is just the * ASCII fold for mixed Latin text and the alarm only watches for the historical scripts. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_neon_georgian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_neon_georgian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { return sz_utf8_uncased_search_neon_scripted_( // sz_utf8_uncased_search_neon_ascii_fold_u8x16x2_, diff --git a/include/stringzilla/utf8_uncased/rvv.h b/include/stringzilla/utf8_uncased/rvv.h index b7c1f5de..ce33235f 100644 --- a/include/stringzilla/utf8_uncased/rvv.h +++ b/include/stringzilla/utf8_uncased/rvv.h @@ -685,8 +685,8 @@ SZ_HELPER_NOINLINE long sz_utf8_uncased_alarm_georgian_strip_rvv_(sz_u8_t const /* Loads up to `length` bytes from `source` into a zeroed 64-byte scratch buffer, returning a pointer into a * caller-provided buffer whose first `length` bytes are the data and the rest zero. The padding lets the * fold/alarm strips read `source_ptr[vector_length]` for their `next` carry and keeps range compares safe-negative. */ -SZ_HELPER_AUTO sz_u8_t const *sz_utf8_uncased_load_padded_rvv_(sz_cptr_t source, sz_size_t length, sz_u8_t *buffer, - sz_size_t buffer_capacity) { +SZ_HELPER_INLINE sz_u8_t const *sz_utf8_uncased_load_padded_rvv_(sz_cptr_t source, sz_size_t length, sz_u8_t *buffer, + sz_size_t buffer_capacity) { for (sz_size_t byte_index = 0; byte_index < buffer_capacity; ++byte_index) buffer[byte_index] = 0; for (sz_size_t byte_index = 0; byte_index < length; ++byte_index) buffer[byte_index] = (sz_u8_t)source[byte_index]; return buffer; @@ -841,72 +841,72 @@ SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_scripted_( // #pragma region Per Script Kernels -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_ascii_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_ascii_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_ascii_strip_rvv_, (sz_utf8_uncased_alarm_strip_rvv_t)SZ_NULL, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_western_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_western_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_( sz_utf8_uncased_fold_western_europe_strip_rvv_, sz_utf8_uncased_alarm_western_europe_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_central_europe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_central_europe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_( sz_utf8_uncased_fold_central_europe_strip_rvv_, sz_utf8_uncased_alarm_central_europe_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_cyrillic_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_cyrillic_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_cyrillic_strip_rvv_, sz_utf8_uncased_alarm_cyrillic_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_greek_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_greek_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_greek_strip_rvv_, sz_utf8_uncased_alarm_greek_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_armenian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_armenian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_armenian_strip_rvv_, sz_utf8_uncased_alarm_armenian_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_vietnamese_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_vietnamese_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_vietnamese_strip_rvv_, sz_utf8_uncased_alarm_vietnamese_strip_rvv_, haystack, haystack_length, needle, needle_length, needle_metadata, matched_length); } -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_rvv_georgian_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_rvv_georgian_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // sz_utf8_uncased_needle_metadata_t const *needle_metadata, sz_size_t *matched_length) { // Mkhedruli is caseless, so the fold is the bare ASCII fold; the alarm watches the historical scripts. return sz_utf8_uncased_search_rvv_scripted_(sz_utf8_uncased_fold_ascii_strip_rvv_, diff --git a/include/stringzilla/utf8_uncased/serial.h b/include/stringzilla/utf8_uncased/serial.h index bb785ec6..7204fc6e 100644 --- a/include/stringzilla/utf8_uncased/serial.h +++ b/include/stringzilla/utf8_uncased/serial.h @@ -16,166 +16,6 @@ extern "C" { #endif -/** - * @brief Iterator state for streaming through folded UTF-8 runes. - * Handles one-to-many case folding expansions (e.g., 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)) transparently. - */ -typedef struct { - sz_cptr_t ptr; // Current position in UTF-8 string - sz_cptr_t end; // End of string - sz_rune_t pending[4]; // Buffered folded runes from one-to-many expansions - sz_size_t pending_count; // Number of pending folded runes - sz_size_t pending_idx; // Current index into pending buffer -} sz_utf8_folded_iter_t; - -/** @brief Initialize a folded rune iterator. */ -SZ_HELPER_AUTO void sz_utf8_folded_iter_init_(sz_utf8_folded_iter_t *iterator, sz_cptr_t string, sz_size_t length) { - iterator->ptr = string; - iterator->end = string + length; - iterator->pending_count = 0; - iterator->pending_idx = 0; -} - -/** - * @brief Get next folded rune. Returns `sz_false_k` when exhausted. - * Malformed UTF-8 is handled losslessly: a byte that does not begin a well-formed codepoint is emitted as a - * single literal byte (tagged so it compares byte-for-byte and never collides with a real folded codepoint) and - * the iterator resyncs by one byte, never reading past `end`. - */ -SZ_HELPER_AUTO sz_bool_t sz_utf8_folded_iter_next_(sz_utf8_folded_iter_t *it, sz_rune_t *out_rune) { - // Refill pending buffer if exhausted - if (it->pending_idx >= it->pending_count) { - if (it->ptr >= it->end) return sz_false_k; - - // ASCII fast-path: fold inline without buffering - sz_u8_t lead = *(sz_u8_t const *)it->ptr; - if (lead < 0x80) { - *out_rune = sz_ascii_fold_(lead); - it->ptr++; - it->pending_count = 0; // Clear pending buffer - it->pending_idx = 0; // Signal first rune of new codepoint for source tracking - return sz_true_k; - } - - // Multi-byte UTF-8: decode (bounds-checked), fold, and buffer. A byte that does not begin a - // well-formed codepoint folds to itself (>= 0x80 bytes are unchanged by `sz_ascii_fold_`) and resyncs - // by one byte, never over-reading past `end`. - sz_rune_t rune; - sz_rune_length_t const rune_length = sz_rune_decode(it->ptr, it->end, &rune); - if (rune_length == sz_rune_invalid_k) { - *out_rune = sz_rune_malformed_byte_(lead); - it->ptr++; - it->pending_count = 0; - it->pending_idx = 0; - return sz_true_k; - } - - it->ptr += rune_length; - // Pre-fill pending buffer with sentinel values to prevent stale data from causing false matches. - // The fold function will overwrite positions it uses; unused positions keep the sentinel. - // This follows the same pattern as sz_utf8_uncased_search_2folded_serial_ and - // sz_utf8_uncased_search_3folded_serial_. - it->pending[0] = 0xFFFFFFFFu; - it->pending[1] = 0xFFFFFFFEu; - it->pending[2] = 0xFFFFFFFDu; - it->pending[3] = 0xFFFFFFFCu; - it->pending_count = sz_unicode_fold_codepoint_(rune, it->pending); - it->pending_idx = 0; - } - - *out_rune = it->pending[it->pending_idx++]; - return sz_true_k; -} - -/** - * @brief Reverse iterator state for streaming through folded UTF-8 runes backwards. - * Handles one-to-many case folding expansions (e.g., 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)) transparently - * in reverse order. - */ -typedef struct { - sz_cptr_t ptr; // Current position (points to byte AFTER current sequence) - sz_cptr_t start; // Start of string (stop when ptr reaches this) - sz_rune_t pending[4]; // Buffered folded runes from one-to-many expansions (in reverse order) - sz_size_t pending_count; // Number of pending folded runes - sz_size_t pending_idx; // Current index into pending buffer -} sz_utf8_folded_reverse_iter_t; - -/** @brief Initialize a reverse folded rune iterator. Iterates from end towards start. */ -SZ_HELPER_AUTO void sz_utf8_folded_reverse_iter_init_(sz_utf8_folded_reverse_iter_t *it, sz_cptr_t start, - sz_cptr_t end) { - it->ptr = end; - it->start = start; - it->pending_count = 0; - it->pending_idx = 0; -} - -/** - * @brief Get previous folded rune (walking backwards). Returns `sz_false_k` when exhausted. - * When a codepoint folds to multiple runes (like 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)), returns them in - * reverse order ('s', then 's'). Malformed UTF-8 is handled losslessly and byte-identically to the forward - * iterator: a byte that does not begin/end a well-formed codepoint is emitted as a single tagged literal byte and - * the iterator resyncs by one byte, so the backward rune stream is exactly the reverse of the forward stream. - */ -SZ_HELPER_AUTO sz_bool_t sz_utf8_folded_reverse_iter_prev_(sz_utf8_folded_reverse_iter_t *it, sz_rune_t *out_rune) { - // Return pending runes if any (stored in reverse order, consumed in reverse) - if (it->pending_idx < it->pending_count) { - *out_rune = it->pending[it->pending_count - 1 - it->pending_idx]; - it->pending_idx++; - return sz_true_k; - } - - // Refill: find previous codepoint - if (it->ptr <= it->start) return sz_false_k; - - // Remember one-past-the-end of the sequence we are about to decode, so the strict decode is bounded - // and a malformed run resyncs one byte at a time - mirroring the forward iterator byte-for-byte. - sz_cptr_t const sequence_end = it->ptr; - - // The byte immediately before `sequence_end` is the last byte of whatever codepoint ends here. - sz_u8_t const last_byte = *(sz_u8_t const *)(sequence_end - 1); - - // ASCII fast-path: a byte < 0x80 is always its own complete 1-byte codepoint. - if (last_byte < 0x80) { - it->ptr = sequence_end - 1; - *out_rune = sz_ascii_fold_(last_byte); - it->pending_count = 0; - it->pending_idx = 0; - return sz_true_k; - } - - // Otherwise walk backwards over up to 3 continuation bytes (0x80-0xBF) to locate a candidate lead. - // A well-formed multi-byte rune is at most 4 bytes, so stop after considering 4 positions. - sz_cptr_t candidate = sequence_end - 1; - for (sz_size_t back = 0; back < 3 && candidate > it->start && (*(sz_u8_t const *)candidate & 0xC0) == 0x80; ++back) - candidate--; - - // Multi-byte UTF-8: decode (bounded) and fold only if the bytes from the candidate lead form a well-formed - // codepoint that ends EXACTLY at `sequence_end`. Otherwise the last byte does not begin/end a valid rune, so - // treat it as a literal folded-to-itself byte and resync by one - matching the forward iterator byte-for-byte. - sz_rune_t rune; - sz_rune_length_t const rune_length = sz_rune_decode(candidate, sequence_end, &rune); - if (rune_length == sz_rune_invalid_k || candidate + rune_length != sequence_end) { - it->ptr = sequence_end - 1; - *out_rune = sz_rune_malformed_byte_(last_byte); - it->pending_count = 0; - it->pending_idx = 0; - return sz_true_k; - } - it->ptr = candidate; - - // Store folded runes in pending buffer - it->pending[0] = 0xFFFFFFFFu; - it->pending[1] = 0xFFFFFFFEu; - it->pending[2] = 0xFFFFFFFDu; - it->pending[3] = 0xFFFFFFFCu; - it->pending_count = sz_unicode_fold_codepoint_(rune, it->pending); - it->pending_idx = 1; // We'll return the last one now, then the rest in subsequent calls - - // Return the LAST folded rune first (since we're going backwards) - *out_rune = it->pending[it->pending_count - 1]; - return sz_true_k; -} - #pragma region Case Invariance & Ordering /** @@ -608,6 +448,96 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_1folded_serial_( // * @param match_length Haystack bytes consumed by the match. * @return Pointer to match start, or SZ_NULL_CHAR if not found in this region. */ +/** + * @brief Verifies the needle anchored at folded rune @p anchor_index of the codepoint at @p danger_cursor. + * @param haystack_folded_runes The codepoint's folded image; @p anchor_index selects the rune to anchor on. + * @return Match start, or `SZ_NULL_CHAR` when this anchor carries no match. + * + * Anchoring on rune zero starts on a codepoint boundary, which the shared validator already handles. Past + * that, the runes of this one codepoint on either side of the anchor never reach a folded iterator - the + * iterators step over the codepoint whole - so they are compared against the image directly. + */ +SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_verify_at_folded_rune_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_cptr_t danger_cursor, sz_size_t haystack_rune_length, // + sz_rune_t const *haystack_folded_runes, sz_size_t haystack_folded_runes_count, // + sz_size_t anchor_index, // + sz_rune_t needle_first_safe_folded_rune, // + sz_size_t needle_first_safe_folded_rune_offset, // + sz_size_t *match_length) { + + sz_cptr_t const haystack_end = haystack + haystack_length; + + if (anchor_index == 0) + return sz_utf8_uncased_verify_match_( // + haystack, haystack_length, // + needle, needle_length, // + (sz_size_t)(danger_cursor - haystack), 0, // No pre-matched middle + needle_first_safe_folded_rune_offset, // + needle_length - needle_first_safe_folded_rune_offset, // Verify everything after head serially + match_length); + + sz_cptr_t haystack_match_start = 0, haystack_match_end = 0; + + // Walk the needle head backwards against the haystack before the danger zone began. + sz_rune_t needle_riter_rune = 0, haystack_riter_rune = 0; + sz_utf8_folded_reverse_iter_t needle_riter, haystack_riter; + sz_utf8_folded_reverse_iter_init_(&needle_riter, needle, needle + needle_first_safe_folded_rune_offset); + sz_utf8_folded_reverse_iter_init_(&haystack_riter, haystack, danger_cursor); + + // This codepoint's own runes before the anchor, newest first. + for (sz_size_t before = anchor_index; before-- > 0;) { + if (!sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune)) break; + if (needle_riter_rune != haystack_folded_runes[before]) return SZ_NULL_CHAR; + } + + for (;;) { + // Needle exhausted - success! + if (!sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune)) { + haystack_match_start = haystack_riter.ptr; + break; + } + if (!sz_utf8_folded_reverse_iter_prev_(&haystack_riter, &haystack_riter_rune)) return SZ_NULL_CHAR; + if (needle_riter_rune != haystack_riter_rune) return SZ_NULL_CHAR; + } + + // Walk the needle tail forwards from the safe window's start. + sz_rune_t needle_iter_rune = 0, haystack_iter_rune = 0; + sz_utf8_folded_iter_t needle_iter, haystack_iter; + sz_utf8_folded_iter_init_(&needle_iter, needle + needle_first_safe_folded_rune_offset, + needle_length - needle_first_safe_folded_rune_offset); + sz_utf8_folded_iter_init_(&haystack_iter, danger_cursor + haystack_rune_length, + (sz_size_t)(haystack_end - (danger_cursor + haystack_rune_length))); + + // Pop the `needle_first_safe_folded_rune` from the forward iterator + { + sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); + sz_assert_(have_needle && needle_iter_rune == needle_first_safe_folded_rune); + sz_unused_(have_needle); + } + + // This codepoint's own runes after the anchor, oldest first. + for (sz_size_t after = anchor_index + 1; after < haystack_folded_runes_count; ++after) { + if (!sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune)) break; + if (needle_iter_rune != haystack_folded_runes[after]) return SZ_NULL_CHAR; + } + + for (;;) { + // Needle exhausted - success! + if (!sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune)) { + haystack_match_end = haystack_iter.ptr; + break; + } + if (!sz_utf8_folded_iter_next_(&haystack_iter, &haystack_iter_rune)) return SZ_NULL_CHAR; + if (needle_iter_rune != haystack_iter_rune) return SZ_NULL_CHAR; + } + + if (haystack_match_start == 0 || haystack_match_end == 0) return SZ_NULL_CHAR; + *match_length = (sz_size_t)(haystack_match_end - haystack_match_start); + return haystack_match_start; +} + SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_in_danger_zone_( // sz_cptr_t haystack, sz_size_t haystack_length, // sz_cptr_t needle, sz_size_t needle_length, // @@ -645,172 +575,22 @@ SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_in_danger_zone_( // } else { haystack_folded_runes_count = sz_unicode_fold_codepoint_(haystack_rune, haystack_folded_runes); } - // The simplest case is when the very first in `haystack_folded_runes` is our target: - if (haystack_folded_runes[0] == needle_first_safe_folded_rune) { - // Validate the full match using the unified validator - sz_cptr_t match = sz_utf8_uncased_verify_match_( // - haystack, haystack_length, // - needle, needle_length, // - danger_cursor - haystack, 0, // No pre-matched middle - needle_first_safe_folded_rune_offset, - needle_length - needle_first_safe_folded_rune_offset, // Verify everything after head serially + // The needle's anchor may sit at any rune of this codepoint's folded image, and a candidate that + // fails only rules out that rune - the next one is still open. + for (sz_size_t anchor_index = 0; anchor_index < haystack_folded_runes_count; ++anchor_index) { + if (haystack_folded_runes[anchor_index] != needle_first_safe_folded_rune) continue; + sz_cptr_t const match = sz_utf8_uncased_verify_at_folded_rune_( // + haystack, haystack_length, // + needle, needle_length, // + danger_cursor, (sz_size_t)haystack_rune_length, // + haystack_folded_runes, haystack_folded_runes_count, // + anchor_index, // + needle_first_safe_folded_rune, // + needle_first_safe_folded_rune_offset, // match_length); - if (match) return match; - else { goto consider_second_haystack_folded_rune; } // We fall through here anyways :) - } - - consider_second_haystack_folded_rune: - - // Check for a match at the second position in the folded haystack rune sequence - if (haystack_folded_runes_count > 1 && haystack_folded_runes[1] == needle_first_safe_folded_rune) { - sz_cptr_t haystack_match_start = 0, haystack_match_end = 0; - - // Check if the previous characters in the needle match the haystack before the danger zone began - sz_rune_t needle_riter_rune = 0, haystack_riter_rune = 0; - sz_utf8_folded_reverse_iter_t needle_riter, haystack_riter; - sz_utf8_folded_reverse_iter_init_(&needle_riter, needle, needle + needle_first_safe_folded_rune_offset); - sz_utf8_folded_reverse_iter_init_(&haystack_riter, haystack, danger_cursor); - - // Check if we even have needle bytes to check - { - sz_bool_t have_needle = sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune); - if (have_needle && needle_riter_rune != haystack_folded_runes[0]) - goto consider_third_haystack_folded_rune; - } - - // Loop backwards until we exhaust the needle head or find a mismatch - for (;;) { - sz_bool_t have_needle = sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune); - - // Needle exhausted - success! - if (!have_needle) { - haystack_match_start = haystack_riter.ptr; - break; - } - - sz_bool_t have_haystack = sz_utf8_folded_reverse_iter_prev_(&haystack_riter, &haystack_riter_rune); - if (!have_haystack) goto consider_third_haystack_folded_rune; - if (needle_riter_rune != haystack_riter_rune) goto consider_third_haystack_folded_rune; - } - - // First match the tail (from safe window start forward) - sz_rune_t needle_iter_rune = 0, haystack_iter_rune = 0; - sz_utf8_folded_iter_t needle_iter, haystack_iter; - sz_utf8_folded_iter_init_(&needle_iter, needle + needle_first_safe_folded_rune_offset, - needle_length - needle_first_safe_folded_rune_offset); - sz_utf8_folded_iter_init_(&haystack_iter, danger_cursor + haystack_rune_length, - haystack_end - (danger_cursor + haystack_rune_length)); - - // Pop the `needle_first_safe_folded_rune` from the forward iterator - { - sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); - sz_assert_(have_needle && needle_iter_rune == needle_first_safe_folded_rune); - } - - // In some cases we already have the first point of comparison in the `haystack_folded_runes[2]` - if (haystack_folded_runes_count == 3) { - sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); - if (have_needle && needle_iter_rune != haystack_folded_runes[2]) - goto consider_third_haystack_folded_rune; - } - - // Match the remaining tail runes - for (;;) { - sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); - - // Needle exhausted - success! - if (!have_needle) { - haystack_match_end = haystack_iter.ptr; - break; - } - - sz_bool_t have_haystack = sz_utf8_folded_iter_next_(&haystack_iter, &haystack_iter_rune); - if (!have_haystack) goto consider_third_haystack_folded_rune; - if (needle_iter_rune != haystack_iter_rune) goto consider_third_haystack_folded_rune; - } - - // Check if we have a match to report - if (haystack_match_start != 0 && haystack_match_end != 0) { - *match_length = (sz_size_t)(haystack_match_end - haystack_match_start); - return haystack_match_start; - } - } - - consider_third_haystack_folded_rune: - - // Check for a match at the second position in the folded haystack rune sequence - if (haystack_folded_runes_count > 2 && haystack_folded_runes[2] == needle_first_safe_folded_rune) { - sz_cptr_t haystack_match_start = 0, haystack_match_end = 0; - - // Check if the previous characters in the needle match the haystack before the danger zone began - sz_rune_t needle_riter_rune = 0, haystack_riter_rune = 0; - sz_utf8_folded_reverse_iter_t needle_riter, haystack_riter; - sz_utf8_folded_reverse_iter_init_(&needle_riter, needle, needle + needle_first_safe_folded_rune_offset); - sz_utf8_folded_reverse_iter_init_(&haystack_riter, haystack, danger_cursor); - - // Check if we even have needle bytes to check - { - sz_bool_t have_needle = sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune); - if (have_needle && needle_riter_rune != haystack_folded_runes[1]) - goto consider_following_haystack_runes; - have_needle = sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune); - if (have_needle && needle_riter_rune != haystack_folded_runes[0]) - goto consider_following_haystack_runes; - } - - // Loop backwards until we exhaust the needle head or find a mismatch - for (;;) { - sz_bool_t have_needle = sz_utf8_folded_reverse_iter_prev_(&needle_riter, &needle_riter_rune); - - // Needle exhausted - success! - if (!have_needle) { - haystack_match_start = haystack_riter.ptr; - break; - } - - sz_bool_t have_haystack = sz_utf8_folded_reverse_iter_prev_(&haystack_riter, &haystack_riter_rune); - if (!have_haystack) goto consider_following_haystack_runes; - if (needle_riter_rune != haystack_riter_rune) goto consider_following_haystack_runes; - } - - // First match the tail (from safe window start forward) - sz_rune_t needle_iter_rune = 0, haystack_iter_rune = 0; - sz_utf8_folded_iter_t needle_iter, haystack_iter; - sz_utf8_folded_iter_init_(&needle_iter, needle + needle_first_safe_folded_rune_offset, - needle_length - needle_first_safe_folded_rune_offset); - sz_utf8_folded_iter_init_(&haystack_iter, danger_cursor + haystack_rune_length, - haystack_end - (danger_cursor + haystack_rune_length)); - - // Pop the `needle_first_safe_folded_rune` from the forward iterator - { - sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); - sz_assert_(have_needle && needle_iter_rune == needle_first_safe_folded_rune); - } - - // Match the remaining tail runes - for (;;) { - sz_bool_t have_needle = sz_utf8_folded_iter_next_(&needle_iter, &needle_iter_rune); - - // Needle exhausted - success! - if (!have_needle) { - haystack_match_end = haystack_iter.ptr; - break; - } - - sz_bool_t have_haystack = sz_utf8_folded_iter_next_(&haystack_iter, &haystack_iter_rune); - if (!have_haystack) goto consider_following_haystack_runes; - if (needle_iter_rune != haystack_iter_rune) goto consider_following_haystack_runes; - } - - // Check if we have a match to report - if (haystack_match_start != 0 && haystack_match_end != 0) { - *match_length = (sz_size_t)(haystack_match_end - haystack_match_start); - return haystack_match_start; - } } - consider_following_haystack_runes: // Move to next candidate danger_cursor += haystack_rune_length; } diff --git a/include/stringzilla/utf8_uncased/sve2.h b/include/stringzilla/utf8_uncased/sve2.h index cecb5359..04fc5dfd 100644 --- a/include/stringzilla/utf8_uncased/sve2.h +++ b/include/stringzilla/utf8_uncased/sve2.h @@ -211,10 +211,10 @@ SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_sve2_scripted_( // /** @brief 3-probe ASCII uncased search: probes at 0, mid, last cover ALL bytes of windows up to 3 bytes, * so candidates skip window verification and go straight to head/tail validation. */ -SZ_HELPER_AUTO sz_cptr_t sz_utf8_uncased_search_sve2_ascii_3probe_( // - sz_cptr_t haystack, sz_size_t haystack_length, // - sz_cptr_t needle, sz_size_t needle_length, // - sz_utf8_uncased_needle_metadata_t const *needle_metadata, // +SZ_HELPER_INLINE sz_cptr_t sz_utf8_uncased_search_sve2_ascii_3probe_( // + sz_cptr_t haystack, sz_size_t haystack_length, // + sz_cptr_t needle, sz_size_t needle_length, // + sz_utf8_uncased_needle_metadata_t const *needle_metadata, // sz_size_t *matched_length) { sz_size_t const folded_window_length = needle_metadata->folded_slice_length; diff --git a/include/stringzilla/utf8_uncased/tables.h b/include/stringzilla/utf8_uncased/tables.h new file mode 100644 index 00000000..a075ad52 --- /dev/null +++ b/include/stringzilla/utf8_uncased/tables.h @@ -0,0 +1,485 @@ +/** + * @file include/stringzilla/utf8_uncased/tables.h + * @author Ash Vardanian + * @brief Fold-preimage table: given a folded codepoint sequence, which source codepoints fold to it. + * + * An Aho-Corasick automaton built over folded needles must, at construction time, add a transition for + * every codepoint that folds onto each edge of the trie - not just the literal codepoint already in the + * needle. This table inverts `sz_unicode_fold_codepoint_` (`include/stringzilla/utf8_uncased_fold/serial.h`): + * sorted ascending by source codepoint, `sz_utf8_fold_preimage_sources_` is binary-searchable, and each + * entry's fold image - 1 to 3 codepoints - lives in the matching run of `sz_utf8_fold_preimage_runs_`, + * addressed through `sz_utf8_fold_preimage_offsets_`. Every codepoint absent from `..._sources_` folds to + * itself (identity), so the table only lists the 1585 codepoints with a non-identity full fold. + * + * Derived from Unicode `CaseFolding.txt` 17.0.0, keeping only status `C` (common, 1:1) + * and `F` (full, 1:N) mappings - `S` (simple) and `T` (Turkic) are excluded, matching the full case folding + * StringZilla already implements in `sz_unicode_fold_codepoint_`. Of the 1585 cased codepoints, + * 104 produce multi-rune expansions; 35 of the remaining single-rune + * folds change the UTF-8 encoded length, and 96 of the multi-rune expansions do too + * (unsurprising, since turning one rune into two or three usually adds bytes). Generated by: + * + * @code{.py} + * # test/sz_helpers.py::get_uncased_folding_rules_as_codepoints() downloads and parses CaseFolding.txt, + * # keeping only status C and F, and returns {source_codepoint: [target_codepoint, ...]}. + * folding_rules = get_uncased_folding_rules_as_codepoints("17.0.0") + * sorted_sources = sorted(folding_rules.keys()) + * + * sources = sorted_sources + * offsets = [0] + * runs = [] + * for source in sorted_sources: + * runs.extend(folding_rules[source]) + * offsets.append(len(runs)) + * @endcode + */ +#ifndef STRINGZILLA_UTF8_UNCASED_TABLES_H_ +#define STRINGZILLA_UTF8_UNCASED_TABLES_H_ + +#include "stringzilla/types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#pragma region Fold preimage tables + +enum { + sz_utf8_fold_preimage_sources_count_k = 1585, + sz_utf8_fold_preimage_runs_count_k = 1705, +}; + +/** @brief Codepoints with a non-identity full fold, sorted ascending. Binary-searchable. */ +sz_align_(64) static sz_u32_t const sz_utf8_fold_preimage_sources_[1585] = { + 0x000041, 0x000042, 0x000043, 0x000044, 0x000045, 0x000046, 0x000047, 0x000048, 0x000049, 0x00004A, 0x00004B, + 0x00004C, 0x00004D, 0x00004E, 0x00004F, 0x000050, 0x000051, 0x000052, 0x000053, 0x000054, 0x000055, 0x000056, + 0x000057, 0x000058, 0x000059, 0x00005A, 0x0000B5, 0x0000C0, 0x0000C1, 0x0000C2, 0x0000C3, 0x0000C4, 0x0000C5, + 0x0000C6, 0x0000C7, 0x0000C8, 0x0000C9, 0x0000CA, 0x0000CB, 0x0000CC, 0x0000CD, 0x0000CE, 0x0000CF, 0x0000D0, + 0x0000D1, 0x0000D2, 0x0000D3, 0x0000D4, 0x0000D5, 0x0000D6, 0x0000D8, 0x0000D9, 0x0000DA, 0x0000DB, 0x0000DC, + 0x0000DD, 0x0000DE, 0x0000DF, 0x000100, 0x000102, 0x000104, 0x000106, 0x000108, 0x00010A, 0x00010C, 0x00010E, + 0x000110, 0x000112, 0x000114, 0x000116, 0x000118, 0x00011A, 0x00011C, 0x00011E, 0x000120, 0x000122, 0x000124, + 0x000126, 0x000128, 0x00012A, 0x00012C, 0x00012E, 0x000130, 0x000132, 0x000134, 0x000136, 0x000139, 0x00013B, + 0x00013D, 0x00013F, 0x000141, 0x000143, 0x000145, 0x000147, 0x000149, 0x00014A, 0x00014C, 0x00014E, 0x000150, + 0x000152, 0x000154, 0x000156, 0x000158, 0x00015A, 0x00015C, 0x00015E, 0x000160, 0x000162, 0x000164, 0x000166, + 0x000168, 0x00016A, 0x00016C, 0x00016E, 0x000170, 0x000172, 0x000174, 0x000176, 0x000178, 0x000179, 0x00017B, + 0x00017D, 0x00017F, 0x000181, 0x000182, 0x000184, 0x000186, 0x000187, 0x000189, 0x00018A, 0x00018B, 0x00018E, + 0x00018F, 0x000190, 0x000191, 0x000193, 0x000194, 0x000196, 0x000197, 0x000198, 0x00019C, 0x00019D, 0x00019F, + 0x0001A0, 0x0001A2, 0x0001A4, 0x0001A6, 0x0001A7, 0x0001A9, 0x0001AC, 0x0001AE, 0x0001AF, 0x0001B1, 0x0001B2, + 0x0001B3, 0x0001B5, 0x0001B7, 0x0001B8, 0x0001BC, 0x0001C4, 0x0001C5, 0x0001C7, 0x0001C8, 0x0001CA, 0x0001CB, + 0x0001CD, 0x0001CF, 0x0001D1, 0x0001D3, 0x0001D5, 0x0001D7, 0x0001D9, 0x0001DB, 0x0001DE, 0x0001E0, 0x0001E2, + 0x0001E4, 0x0001E6, 0x0001E8, 0x0001EA, 0x0001EC, 0x0001EE, 0x0001F0, 0x0001F1, 0x0001F2, 0x0001F4, 0x0001F6, + 0x0001F7, 0x0001F8, 0x0001FA, 0x0001FC, 0x0001FE, 0x000200, 0x000202, 0x000204, 0x000206, 0x000208, 0x00020A, + 0x00020C, 0x00020E, 0x000210, 0x000212, 0x000214, 0x000216, 0x000218, 0x00021A, 0x00021C, 0x00021E, 0x000220, + 0x000222, 0x000224, 0x000226, 0x000228, 0x00022A, 0x00022C, 0x00022E, 0x000230, 0x000232, 0x00023A, 0x00023B, + 0x00023D, 0x00023E, 0x000241, 0x000243, 0x000244, 0x000245, 0x000246, 0x000248, 0x00024A, 0x00024C, 0x00024E, + 0x000345, 0x000370, 0x000372, 0x000376, 0x00037F, 0x000386, 0x000388, 0x000389, 0x00038A, 0x00038C, 0x00038E, + 0x00038F, 0x000390, 0x000391, 0x000392, 0x000393, 0x000394, 0x000395, 0x000396, 0x000397, 0x000398, 0x000399, + 0x00039A, 0x00039B, 0x00039C, 0x00039D, 0x00039E, 0x00039F, 0x0003A0, 0x0003A1, 0x0003A3, 0x0003A4, 0x0003A5, + 0x0003A6, 0x0003A7, 0x0003A8, 0x0003A9, 0x0003AA, 0x0003AB, 0x0003B0, 0x0003C2, 0x0003CF, 0x0003D0, 0x0003D1, + 0x0003D5, 0x0003D6, 0x0003D8, 0x0003DA, 0x0003DC, 0x0003DE, 0x0003E0, 0x0003E2, 0x0003E4, 0x0003E6, 0x0003E8, + 0x0003EA, 0x0003EC, 0x0003EE, 0x0003F0, 0x0003F1, 0x0003F4, 0x0003F5, 0x0003F7, 0x0003F9, 0x0003FA, 0x0003FD, + 0x0003FE, 0x0003FF, 0x000400, 0x000401, 0x000402, 0x000403, 0x000404, 0x000405, 0x000406, 0x000407, 0x000408, + 0x000409, 0x00040A, 0x00040B, 0x00040C, 0x00040D, 0x00040E, 0x00040F, 0x000410, 0x000411, 0x000412, 0x000413, + 0x000414, 0x000415, 0x000416, 0x000417, 0x000418, 0x000419, 0x00041A, 0x00041B, 0x00041C, 0x00041D, 0x00041E, + 0x00041F, 0x000420, 0x000421, 0x000422, 0x000423, 0x000424, 0x000425, 0x000426, 0x000427, 0x000428, 0x000429, + 0x00042A, 0x00042B, 0x00042C, 0x00042D, 0x00042E, 0x00042F, 0x000460, 0x000462, 0x000464, 0x000466, 0x000468, + 0x00046A, 0x00046C, 0x00046E, 0x000470, 0x000472, 0x000474, 0x000476, 0x000478, 0x00047A, 0x00047C, 0x00047E, + 0x000480, 0x00048A, 0x00048C, 0x00048E, 0x000490, 0x000492, 0x000494, 0x000496, 0x000498, 0x00049A, 0x00049C, + 0x00049E, 0x0004A0, 0x0004A2, 0x0004A4, 0x0004A6, 0x0004A8, 0x0004AA, 0x0004AC, 0x0004AE, 0x0004B0, 0x0004B2, + 0x0004B4, 0x0004B6, 0x0004B8, 0x0004BA, 0x0004BC, 0x0004BE, 0x0004C0, 0x0004C1, 0x0004C3, 0x0004C5, 0x0004C7, + 0x0004C9, 0x0004CB, 0x0004CD, 0x0004D0, 0x0004D2, 0x0004D4, 0x0004D6, 0x0004D8, 0x0004DA, 0x0004DC, 0x0004DE, + 0x0004E0, 0x0004E2, 0x0004E4, 0x0004E6, 0x0004E8, 0x0004EA, 0x0004EC, 0x0004EE, 0x0004F0, 0x0004F2, 0x0004F4, + 0x0004F6, 0x0004F8, 0x0004FA, 0x0004FC, 0x0004FE, 0x000500, 0x000502, 0x000504, 0x000506, 0x000508, 0x00050A, + 0x00050C, 0x00050E, 0x000510, 0x000512, 0x000514, 0x000516, 0x000518, 0x00051A, 0x00051C, 0x00051E, 0x000520, + 0x000522, 0x000524, 0x000526, 0x000528, 0x00052A, 0x00052C, 0x00052E, 0x000531, 0x000532, 0x000533, 0x000534, + 0x000535, 0x000536, 0x000537, 0x000538, 0x000539, 0x00053A, 0x00053B, 0x00053C, 0x00053D, 0x00053E, 0x00053F, + 0x000540, 0x000541, 0x000542, 0x000543, 0x000544, 0x000545, 0x000546, 0x000547, 0x000548, 0x000549, 0x00054A, + 0x00054B, 0x00054C, 0x00054D, 0x00054E, 0x00054F, 0x000550, 0x000551, 0x000552, 0x000553, 0x000554, 0x000555, + 0x000556, 0x000587, 0x0010A0, 0x0010A1, 0x0010A2, 0x0010A3, 0x0010A4, 0x0010A5, 0x0010A6, 0x0010A7, 0x0010A8, + 0x0010A9, 0x0010AA, 0x0010AB, 0x0010AC, 0x0010AD, 0x0010AE, 0x0010AF, 0x0010B0, 0x0010B1, 0x0010B2, 0x0010B3, + 0x0010B4, 0x0010B5, 0x0010B6, 0x0010B7, 0x0010B8, 0x0010B9, 0x0010BA, 0x0010BB, 0x0010BC, 0x0010BD, 0x0010BE, + 0x0010BF, 0x0010C0, 0x0010C1, 0x0010C2, 0x0010C3, 0x0010C4, 0x0010C5, 0x0010C7, 0x0010CD, 0x0013F8, 0x0013F9, + 0x0013FA, 0x0013FB, 0x0013FC, 0x0013FD, 0x001C80, 0x001C81, 0x001C82, 0x001C83, 0x001C84, 0x001C85, 0x001C86, + 0x001C87, 0x001C88, 0x001C89, 0x001C90, 0x001C91, 0x001C92, 0x001C93, 0x001C94, 0x001C95, 0x001C96, 0x001C97, + 0x001C98, 0x001C99, 0x001C9A, 0x001C9B, 0x001C9C, 0x001C9D, 0x001C9E, 0x001C9F, 0x001CA0, 0x001CA1, 0x001CA2, + 0x001CA3, 0x001CA4, 0x001CA5, 0x001CA6, 0x001CA7, 0x001CA8, 0x001CA9, 0x001CAA, 0x001CAB, 0x001CAC, 0x001CAD, + 0x001CAE, 0x001CAF, 0x001CB0, 0x001CB1, 0x001CB2, 0x001CB3, 0x001CB4, 0x001CB5, 0x001CB6, 0x001CB7, 0x001CB8, + 0x001CB9, 0x001CBA, 0x001CBD, 0x001CBE, 0x001CBF, 0x001E00, 0x001E02, 0x001E04, 0x001E06, 0x001E08, 0x001E0A, + 0x001E0C, 0x001E0E, 0x001E10, 0x001E12, 0x001E14, 0x001E16, 0x001E18, 0x001E1A, 0x001E1C, 0x001E1E, 0x001E20, + 0x001E22, 0x001E24, 0x001E26, 0x001E28, 0x001E2A, 0x001E2C, 0x001E2E, 0x001E30, 0x001E32, 0x001E34, 0x001E36, + 0x001E38, 0x001E3A, 0x001E3C, 0x001E3E, 0x001E40, 0x001E42, 0x001E44, 0x001E46, 0x001E48, 0x001E4A, 0x001E4C, + 0x001E4E, 0x001E50, 0x001E52, 0x001E54, 0x001E56, 0x001E58, 0x001E5A, 0x001E5C, 0x001E5E, 0x001E60, 0x001E62, + 0x001E64, 0x001E66, 0x001E68, 0x001E6A, 0x001E6C, 0x001E6E, 0x001E70, 0x001E72, 0x001E74, 0x001E76, 0x001E78, + 0x001E7A, 0x001E7C, 0x001E7E, 0x001E80, 0x001E82, 0x001E84, 0x001E86, 0x001E88, 0x001E8A, 0x001E8C, 0x001E8E, + 0x001E90, 0x001E92, 0x001E94, 0x001E96, 0x001E97, 0x001E98, 0x001E99, 0x001E9A, 0x001E9B, 0x001E9E, 0x001EA0, + 0x001EA2, 0x001EA4, 0x001EA6, 0x001EA8, 0x001EAA, 0x001EAC, 0x001EAE, 0x001EB0, 0x001EB2, 0x001EB4, 0x001EB6, + 0x001EB8, 0x001EBA, 0x001EBC, 0x001EBE, 0x001EC0, 0x001EC2, 0x001EC4, 0x001EC6, 0x001EC8, 0x001ECA, 0x001ECC, + 0x001ECE, 0x001ED0, 0x001ED2, 0x001ED4, 0x001ED6, 0x001ED8, 0x001EDA, 0x001EDC, 0x001EDE, 0x001EE0, 0x001EE2, + 0x001EE4, 0x001EE6, 0x001EE8, 0x001EEA, 0x001EEC, 0x001EEE, 0x001EF0, 0x001EF2, 0x001EF4, 0x001EF6, 0x001EF8, + 0x001EFA, 0x001EFC, 0x001EFE, 0x001F08, 0x001F09, 0x001F0A, 0x001F0B, 0x001F0C, 0x001F0D, 0x001F0E, 0x001F0F, + 0x001F18, 0x001F19, 0x001F1A, 0x001F1B, 0x001F1C, 0x001F1D, 0x001F28, 0x001F29, 0x001F2A, 0x001F2B, 0x001F2C, + 0x001F2D, 0x001F2E, 0x001F2F, 0x001F38, 0x001F39, 0x001F3A, 0x001F3B, 0x001F3C, 0x001F3D, 0x001F3E, 0x001F3F, + 0x001F48, 0x001F49, 0x001F4A, 0x001F4B, 0x001F4C, 0x001F4D, 0x001F50, 0x001F52, 0x001F54, 0x001F56, 0x001F59, + 0x001F5B, 0x001F5D, 0x001F5F, 0x001F68, 0x001F69, 0x001F6A, 0x001F6B, 0x001F6C, 0x001F6D, 0x001F6E, 0x001F6F, + 0x001F80, 0x001F81, 0x001F82, 0x001F83, 0x001F84, 0x001F85, 0x001F86, 0x001F87, 0x001F88, 0x001F89, 0x001F8A, + 0x001F8B, 0x001F8C, 0x001F8D, 0x001F8E, 0x001F8F, 0x001F90, 0x001F91, 0x001F92, 0x001F93, 0x001F94, 0x001F95, + 0x001F96, 0x001F97, 0x001F98, 0x001F99, 0x001F9A, 0x001F9B, 0x001F9C, 0x001F9D, 0x001F9E, 0x001F9F, 0x001FA0, + 0x001FA1, 0x001FA2, 0x001FA3, 0x001FA4, 0x001FA5, 0x001FA6, 0x001FA7, 0x001FA8, 0x001FA9, 0x001FAA, 0x001FAB, + 0x001FAC, 0x001FAD, 0x001FAE, 0x001FAF, 0x001FB2, 0x001FB3, 0x001FB4, 0x001FB6, 0x001FB7, 0x001FB8, 0x001FB9, + 0x001FBA, 0x001FBB, 0x001FBC, 0x001FBE, 0x001FC2, 0x001FC3, 0x001FC4, 0x001FC6, 0x001FC7, 0x001FC8, 0x001FC9, + 0x001FCA, 0x001FCB, 0x001FCC, 0x001FD2, 0x001FD3, 0x001FD6, 0x001FD7, 0x001FD8, 0x001FD9, 0x001FDA, 0x001FDB, + 0x001FE2, 0x001FE3, 0x001FE4, 0x001FE6, 0x001FE7, 0x001FE8, 0x001FE9, 0x001FEA, 0x001FEB, 0x001FEC, 0x001FF2, + 0x001FF3, 0x001FF4, 0x001FF6, 0x001FF7, 0x001FF8, 0x001FF9, 0x001FFA, 0x001FFB, 0x001FFC, 0x002126, 0x00212A, + 0x00212B, 0x002132, 0x002160, 0x002161, 0x002162, 0x002163, 0x002164, 0x002165, 0x002166, 0x002167, 0x002168, + 0x002169, 0x00216A, 0x00216B, 0x00216C, 0x00216D, 0x00216E, 0x00216F, 0x002183, 0x0024B6, 0x0024B7, 0x0024B8, + 0x0024B9, 0x0024BA, 0x0024BB, 0x0024BC, 0x0024BD, 0x0024BE, 0x0024BF, 0x0024C0, 0x0024C1, 0x0024C2, 0x0024C3, + 0x0024C4, 0x0024C5, 0x0024C6, 0x0024C7, 0x0024C8, 0x0024C9, 0x0024CA, 0x0024CB, 0x0024CC, 0x0024CD, 0x0024CE, + 0x0024CF, 0x002C00, 0x002C01, 0x002C02, 0x002C03, 0x002C04, 0x002C05, 0x002C06, 0x002C07, 0x002C08, 0x002C09, + 0x002C0A, 0x002C0B, 0x002C0C, 0x002C0D, 0x002C0E, 0x002C0F, 0x002C10, 0x002C11, 0x002C12, 0x002C13, 0x002C14, + 0x002C15, 0x002C16, 0x002C17, 0x002C18, 0x002C19, 0x002C1A, 0x002C1B, 0x002C1C, 0x002C1D, 0x002C1E, 0x002C1F, + 0x002C20, 0x002C21, 0x002C22, 0x002C23, 0x002C24, 0x002C25, 0x002C26, 0x002C27, 0x002C28, 0x002C29, 0x002C2A, + 0x002C2B, 0x002C2C, 0x002C2D, 0x002C2E, 0x002C2F, 0x002C60, 0x002C62, 0x002C63, 0x002C64, 0x002C67, 0x002C69, + 0x002C6B, 0x002C6D, 0x002C6E, 0x002C6F, 0x002C70, 0x002C72, 0x002C75, 0x002C7E, 0x002C7F, 0x002C80, 0x002C82, + 0x002C84, 0x002C86, 0x002C88, 0x002C8A, 0x002C8C, 0x002C8E, 0x002C90, 0x002C92, 0x002C94, 0x002C96, 0x002C98, + 0x002C9A, 0x002C9C, 0x002C9E, 0x002CA0, 0x002CA2, 0x002CA4, 0x002CA6, 0x002CA8, 0x002CAA, 0x002CAC, 0x002CAE, + 0x002CB0, 0x002CB2, 0x002CB4, 0x002CB6, 0x002CB8, 0x002CBA, 0x002CBC, 0x002CBE, 0x002CC0, 0x002CC2, 0x002CC4, + 0x002CC6, 0x002CC8, 0x002CCA, 0x002CCC, 0x002CCE, 0x002CD0, 0x002CD2, 0x002CD4, 0x002CD6, 0x002CD8, 0x002CDA, + 0x002CDC, 0x002CDE, 0x002CE0, 0x002CE2, 0x002CEB, 0x002CED, 0x002CF2, 0x00A640, 0x00A642, 0x00A644, 0x00A646, + 0x00A648, 0x00A64A, 0x00A64C, 0x00A64E, 0x00A650, 0x00A652, 0x00A654, 0x00A656, 0x00A658, 0x00A65A, 0x00A65C, + 0x00A65E, 0x00A660, 0x00A662, 0x00A664, 0x00A666, 0x00A668, 0x00A66A, 0x00A66C, 0x00A680, 0x00A682, 0x00A684, + 0x00A686, 0x00A688, 0x00A68A, 0x00A68C, 0x00A68E, 0x00A690, 0x00A692, 0x00A694, 0x00A696, 0x00A698, 0x00A69A, + 0x00A722, 0x00A724, 0x00A726, 0x00A728, 0x00A72A, 0x00A72C, 0x00A72E, 0x00A732, 0x00A734, 0x00A736, 0x00A738, + 0x00A73A, 0x00A73C, 0x00A73E, 0x00A740, 0x00A742, 0x00A744, 0x00A746, 0x00A748, 0x00A74A, 0x00A74C, 0x00A74E, + 0x00A750, 0x00A752, 0x00A754, 0x00A756, 0x00A758, 0x00A75A, 0x00A75C, 0x00A75E, 0x00A760, 0x00A762, 0x00A764, + 0x00A766, 0x00A768, 0x00A76A, 0x00A76C, 0x00A76E, 0x00A779, 0x00A77B, 0x00A77D, 0x00A77E, 0x00A780, 0x00A782, + 0x00A784, 0x00A786, 0x00A78B, 0x00A78D, 0x00A790, 0x00A792, 0x00A796, 0x00A798, 0x00A79A, 0x00A79C, 0x00A79E, + 0x00A7A0, 0x00A7A2, 0x00A7A4, 0x00A7A6, 0x00A7A8, 0x00A7AA, 0x00A7AB, 0x00A7AC, 0x00A7AD, 0x00A7AE, 0x00A7B0, + 0x00A7B1, 0x00A7B2, 0x00A7B3, 0x00A7B4, 0x00A7B6, 0x00A7B8, 0x00A7BA, 0x00A7BC, 0x00A7BE, 0x00A7C0, 0x00A7C2, + 0x00A7C4, 0x00A7C5, 0x00A7C6, 0x00A7C7, 0x00A7C9, 0x00A7CB, 0x00A7CC, 0x00A7CE, 0x00A7D0, 0x00A7D2, 0x00A7D4, + 0x00A7D6, 0x00A7D8, 0x00A7DA, 0x00A7DC, 0x00A7F5, 0x00AB70, 0x00AB71, 0x00AB72, 0x00AB73, 0x00AB74, 0x00AB75, + 0x00AB76, 0x00AB77, 0x00AB78, 0x00AB79, 0x00AB7A, 0x00AB7B, 0x00AB7C, 0x00AB7D, 0x00AB7E, 0x00AB7F, 0x00AB80, + 0x00AB81, 0x00AB82, 0x00AB83, 0x00AB84, 0x00AB85, 0x00AB86, 0x00AB87, 0x00AB88, 0x00AB89, 0x00AB8A, 0x00AB8B, + 0x00AB8C, 0x00AB8D, 0x00AB8E, 0x00AB8F, 0x00AB90, 0x00AB91, 0x00AB92, 0x00AB93, 0x00AB94, 0x00AB95, 0x00AB96, + 0x00AB97, 0x00AB98, 0x00AB99, 0x00AB9A, 0x00AB9B, 0x00AB9C, 0x00AB9D, 0x00AB9E, 0x00AB9F, 0x00ABA0, 0x00ABA1, + 0x00ABA2, 0x00ABA3, 0x00ABA4, 0x00ABA5, 0x00ABA6, 0x00ABA7, 0x00ABA8, 0x00ABA9, 0x00ABAA, 0x00ABAB, 0x00ABAC, + 0x00ABAD, 0x00ABAE, 0x00ABAF, 0x00ABB0, 0x00ABB1, 0x00ABB2, 0x00ABB3, 0x00ABB4, 0x00ABB5, 0x00ABB6, 0x00ABB7, + 0x00ABB8, 0x00ABB9, 0x00ABBA, 0x00ABBB, 0x00ABBC, 0x00ABBD, 0x00ABBE, 0x00ABBF, 0x00FB00, 0x00FB01, 0x00FB02, + 0x00FB03, 0x00FB04, 0x00FB05, 0x00FB06, 0x00FB13, 0x00FB14, 0x00FB15, 0x00FB16, 0x00FB17, 0x00FF21, 0x00FF22, + 0x00FF23, 0x00FF24, 0x00FF25, 0x00FF26, 0x00FF27, 0x00FF28, 0x00FF29, 0x00FF2A, 0x00FF2B, 0x00FF2C, 0x00FF2D, + 0x00FF2E, 0x00FF2F, 0x00FF30, 0x00FF31, 0x00FF32, 0x00FF33, 0x00FF34, 0x00FF35, 0x00FF36, 0x00FF37, 0x00FF38, + 0x00FF39, 0x00FF3A, 0x010400, 0x010401, 0x010402, 0x010403, 0x010404, 0x010405, 0x010406, 0x010407, 0x010408, + 0x010409, 0x01040A, 0x01040B, 0x01040C, 0x01040D, 0x01040E, 0x01040F, 0x010410, 0x010411, 0x010412, 0x010413, + 0x010414, 0x010415, 0x010416, 0x010417, 0x010418, 0x010419, 0x01041A, 0x01041B, 0x01041C, 0x01041D, 0x01041E, + 0x01041F, 0x010420, 0x010421, 0x010422, 0x010423, 0x010424, 0x010425, 0x010426, 0x010427, 0x0104B0, 0x0104B1, + 0x0104B2, 0x0104B3, 0x0104B4, 0x0104B5, 0x0104B6, 0x0104B7, 0x0104B8, 0x0104B9, 0x0104BA, 0x0104BB, 0x0104BC, + 0x0104BD, 0x0104BE, 0x0104BF, 0x0104C0, 0x0104C1, 0x0104C2, 0x0104C3, 0x0104C4, 0x0104C5, 0x0104C6, 0x0104C7, + 0x0104C8, 0x0104C9, 0x0104CA, 0x0104CB, 0x0104CC, 0x0104CD, 0x0104CE, 0x0104CF, 0x0104D0, 0x0104D1, 0x0104D2, + 0x0104D3, 0x010570, 0x010571, 0x010572, 0x010573, 0x010574, 0x010575, 0x010576, 0x010577, 0x010578, 0x010579, + 0x01057A, 0x01057C, 0x01057D, 0x01057E, 0x01057F, 0x010580, 0x010581, 0x010582, 0x010583, 0x010584, 0x010585, + 0x010586, 0x010587, 0x010588, 0x010589, 0x01058A, 0x01058C, 0x01058D, 0x01058E, 0x01058F, 0x010590, 0x010591, + 0x010592, 0x010594, 0x010595, 0x010C80, 0x010C81, 0x010C82, 0x010C83, 0x010C84, 0x010C85, 0x010C86, 0x010C87, + 0x010C88, 0x010C89, 0x010C8A, 0x010C8B, 0x010C8C, 0x010C8D, 0x010C8E, 0x010C8F, 0x010C90, 0x010C91, 0x010C92, + 0x010C93, 0x010C94, 0x010C95, 0x010C96, 0x010C97, 0x010C98, 0x010C99, 0x010C9A, 0x010C9B, 0x010C9C, 0x010C9D, + 0x010C9E, 0x010C9F, 0x010CA0, 0x010CA1, 0x010CA2, 0x010CA3, 0x010CA4, 0x010CA5, 0x010CA6, 0x010CA7, 0x010CA8, + 0x010CA9, 0x010CAA, 0x010CAB, 0x010CAC, 0x010CAD, 0x010CAE, 0x010CAF, 0x010CB0, 0x010CB1, 0x010CB2, 0x010D50, + 0x010D51, 0x010D52, 0x010D53, 0x010D54, 0x010D55, 0x010D56, 0x010D57, 0x010D58, 0x010D59, 0x010D5A, 0x010D5B, + 0x010D5C, 0x010D5D, 0x010D5E, 0x010D5F, 0x010D60, 0x010D61, 0x010D62, 0x010D63, 0x010D64, 0x010D65, 0x0118A0, + 0x0118A1, 0x0118A2, 0x0118A3, 0x0118A4, 0x0118A5, 0x0118A6, 0x0118A7, 0x0118A8, 0x0118A9, 0x0118AA, 0x0118AB, + 0x0118AC, 0x0118AD, 0x0118AE, 0x0118AF, 0x0118B0, 0x0118B1, 0x0118B2, 0x0118B3, 0x0118B4, 0x0118B5, 0x0118B6, + 0x0118B7, 0x0118B8, 0x0118B9, 0x0118BA, 0x0118BB, 0x0118BC, 0x0118BD, 0x0118BE, 0x0118BF, 0x016E40, 0x016E41, + 0x016E42, 0x016E43, 0x016E44, 0x016E45, 0x016E46, 0x016E47, 0x016E48, 0x016E49, 0x016E4A, 0x016E4B, 0x016E4C, + 0x016E4D, 0x016E4E, 0x016E4F, 0x016E50, 0x016E51, 0x016E52, 0x016E53, 0x016E54, 0x016E55, 0x016E56, 0x016E57, + 0x016E58, 0x016E59, 0x016E5A, 0x016E5B, 0x016E5C, 0x016E5D, 0x016E5E, 0x016E5F, 0x016EA0, 0x016EA1, 0x016EA2, + 0x016EA3, 0x016EA4, 0x016EA5, 0x016EA6, 0x016EA7, 0x016EA8, 0x016EA9, 0x016EAA, 0x016EAB, 0x016EAC, 0x016EAD, + 0x016EAE, 0x016EAF, 0x016EB0, 0x016EB1, 0x016EB2, 0x016EB3, 0x016EB4, 0x016EB5, 0x016EB6, 0x016EB7, 0x016EB8, + 0x01E900, 0x01E901, 0x01E902, 0x01E903, 0x01E904, 0x01E905, 0x01E906, 0x01E907, 0x01E908, 0x01E909, 0x01E90A, + 0x01E90B, 0x01E90C, 0x01E90D, 0x01E90E, 0x01E90F, 0x01E910, 0x01E911, 0x01E912, 0x01E913, 0x01E914, 0x01E915, + 0x01E916, 0x01E917, 0x01E918, 0x01E919, 0x01E91A, 0x01E91B, 0x01E91C, 0x01E91D, 0x01E91E, 0x01E91F, 0x01E920, + 0x01E921, +}; + +/** @brief Start offset into `sz_utf8_fold_preimage_runs_` for `sz_utf8_fold_preimage_sources_[index]`; + * entry `index + 1` bounds the run, so the last entry equals the total run count. */ +sz_align_(64) static sz_u32_t const sz_utf8_fold_preimage_offsets_[1586] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, + 0x000E, 0x000F, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, 0x001B, + 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, + 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, + 0x0047, 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, 0x0051, 0x0052, 0x0053, 0x0055, + 0x0056, 0x0057, 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0062, 0x0063, 0x0064, + 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, + 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, 0x0080, + 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, + 0x008F, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, + 0x009D, 0x009E, 0x009F, 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, + 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, + 0x00B9, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, + 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, + 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x00F0, 0x00F1, + 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF, 0x0100, 0x0101, + 0x0102, 0x0103, 0x0104, 0x0105, 0x0106, 0x0107, 0x0108, 0x0109, 0x010A, 0x010B, 0x010C, 0x010D, 0x010E, 0x010F, + 0x0110, 0x0111, 0x0112, 0x0113, 0x0114, 0x0117, 0x0118, 0x0119, 0x011A, 0x011B, 0x011C, 0x011D, 0x011E, 0x011F, + 0x0120, 0x0121, 0x0122, 0x0123, 0x0124, 0x0125, 0x0126, 0x0127, 0x0128, 0x0129, 0x012A, 0x012B, 0x012C, 0x012D, + 0x012E, 0x012F, 0x0130, 0x0131, 0x0132, 0x0133, 0x0134, 0x0135, 0x0136, 0x0137, 0x0138, 0x0139, 0x013A, 0x013B, + 0x013C, 0x013D, 0x013E, 0x013F, 0x0140, 0x0141, 0x0142, 0x0143, 0x0144, 0x0145, 0x0146, 0x0147, 0x0148, 0x0149, + 0x014A, 0x014B, 0x014C, 0x014D, 0x014E, 0x014F, 0x0150, 0x0151, 0x0152, 0x0153, 0x0154, 0x0155, 0x0156, 0x0157, + 0x0158, 0x0159, 0x015A, 0x015B, 0x015C, 0x015D, 0x015E, 0x015F, 0x0160, 0x0161, 0x0162, 0x0163, 0x0164, 0x0165, + 0x0166, 0x0167, 0x0168, 0x0169, 0x016A, 0x016B, 0x016C, 0x016D, 0x016E, 0x016F, 0x0170, 0x0171, 0x0172, 0x0173, + 0x0174, 0x0175, 0x0176, 0x0177, 0x0178, 0x0179, 0x017A, 0x017B, 0x017C, 0x017D, 0x017E, 0x017F, 0x0180, 0x0181, + 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187, 0x0188, 0x0189, 0x018A, 0x018B, 0x018C, 0x018D, 0x018E, 0x018F, + 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197, 0x0198, 0x0199, 0x019A, 0x019B, 0x019C, 0x019D, + 0x019E, 0x019F, 0x01A0, 0x01A1, 0x01A2, 0x01A3, 0x01A4, 0x01A5, 0x01A6, 0x01A7, 0x01A8, 0x01A9, 0x01AA, 0x01AB, + 0x01AC, 0x01AD, 0x01AE, 0x01AF, 0x01B0, 0x01B1, 0x01B2, 0x01B3, 0x01B4, 0x01B5, 0x01B6, 0x01B7, 0x01B8, 0x01B9, + 0x01BA, 0x01BB, 0x01BC, 0x01BD, 0x01BE, 0x01BF, 0x01C0, 0x01C1, 0x01C2, 0x01C3, 0x01C4, 0x01C5, 0x01C6, 0x01C7, + 0x01C8, 0x01C9, 0x01CA, 0x01CB, 0x01CC, 0x01CD, 0x01CE, 0x01CF, 0x01D0, 0x01D1, 0x01D2, 0x01D3, 0x01D4, 0x01D5, + 0x01D6, 0x01D7, 0x01D8, 0x01D9, 0x01DA, 0x01DB, 0x01DC, 0x01DD, 0x01DE, 0x01DF, 0x01E0, 0x01E1, 0x01E2, 0x01E3, + 0x01E4, 0x01E5, 0x01E6, 0x01E7, 0x01E8, 0x01E9, 0x01EA, 0x01EB, 0x01EC, 0x01ED, 0x01EF, 0x01F0, 0x01F1, 0x01F2, + 0x01F3, 0x01F4, 0x01F5, 0x01F6, 0x01F7, 0x01F8, 0x01F9, 0x01FA, 0x01FB, 0x01FC, 0x01FD, 0x01FE, 0x01FF, 0x0200, + 0x0201, 0x0202, 0x0203, 0x0204, 0x0205, 0x0206, 0x0207, 0x0208, 0x0209, 0x020A, 0x020B, 0x020C, 0x020D, 0x020E, + 0x020F, 0x0210, 0x0211, 0x0212, 0x0213, 0x0214, 0x0215, 0x0216, 0x0217, 0x0218, 0x0219, 0x021A, 0x021B, 0x021C, + 0x021D, 0x021E, 0x021F, 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0226, 0x0227, 0x0228, 0x0229, 0x022A, + 0x022B, 0x022C, 0x022D, 0x022E, 0x022F, 0x0230, 0x0231, 0x0232, 0x0233, 0x0234, 0x0235, 0x0236, 0x0237, 0x0238, + 0x0239, 0x023A, 0x023B, 0x023C, 0x023D, 0x023E, 0x023F, 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, + 0x0247, 0x0248, 0x0249, 0x024A, 0x024B, 0x024C, 0x024D, 0x024E, 0x024F, 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, + 0x0255, 0x0256, 0x0257, 0x0258, 0x0259, 0x025A, 0x025B, 0x025C, 0x025D, 0x025E, 0x025F, 0x0260, 0x0261, 0x0262, + 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, 0x0268, 0x0269, 0x026A, 0x026B, 0x026C, 0x026D, 0x026E, 0x026F, 0x0270, + 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, 0x0278, 0x0279, 0x027A, 0x027B, 0x027C, 0x027D, 0x027E, + 0x027F, 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, 0x0288, 0x0289, 0x028A, 0x028B, 0x028C, + 0x028D, 0x028E, 0x028F, 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, 0x0298, 0x0299, 0x029A, + 0x029B, 0x029C, 0x029D, 0x029E, 0x029F, 0x02A0, 0x02A2, 0x02A4, 0x02A6, 0x02A8, 0x02AA, 0x02AB, 0x02AD, 0x02AE, + 0x02AF, 0x02B0, 0x02B1, 0x02B2, 0x02B3, 0x02B4, 0x02B5, 0x02B6, 0x02B7, 0x02B8, 0x02B9, 0x02BA, 0x02BB, 0x02BC, + 0x02BD, 0x02BE, 0x02BF, 0x02C0, 0x02C1, 0x02C2, 0x02C3, 0x02C4, 0x02C5, 0x02C6, 0x02C7, 0x02C8, 0x02C9, 0x02CA, + 0x02CB, 0x02CC, 0x02CD, 0x02CE, 0x02CF, 0x02D0, 0x02D1, 0x02D2, 0x02D3, 0x02D4, 0x02D5, 0x02D6, 0x02D7, 0x02D8, + 0x02D9, 0x02DA, 0x02DB, 0x02DC, 0x02DD, 0x02DE, 0x02DF, 0x02E0, 0x02E1, 0x02E2, 0x02E3, 0x02E4, 0x02E5, 0x02E6, + 0x02E7, 0x02E8, 0x02E9, 0x02EA, 0x02EB, 0x02EC, 0x02ED, 0x02EE, 0x02EF, 0x02F0, 0x02F1, 0x02F2, 0x02F3, 0x02F4, + 0x02F5, 0x02F6, 0x02F7, 0x02F8, 0x02F9, 0x02FA, 0x02FB, 0x02FC, 0x02FD, 0x02FE, 0x02FF, 0x0300, 0x0301, 0x0303, + 0x0306, 0x0309, 0x030C, 0x030D, 0x030E, 0x030F, 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + 0x0318, 0x031A, 0x031C, 0x031E, 0x0320, 0x0322, 0x0324, 0x0326, 0x0328, 0x032A, 0x032C, 0x032E, 0x0330, 0x0332, + 0x0334, 0x0336, 0x0338, 0x033A, 0x033C, 0x033E, 0x0340, 0x0342, 0x0344, 0x0346, 0x0348, 0x034A, 0x034C, 0x034E, + 0x0350, 0x0352, 0x0354, 0x0356, 0x0358, 0x035A, 0x035C, 0x035E, 0x0360, 0x0362, 0x0364, 0x0366, 0x0368, 0x036A, + 0x036C, 0x036E, 0x0370, 0x0372, 0x0374, 0x0376, 0x0378, 0x037A, 0x037C, 0x037E, 0x0380, 0x0383, 0x0384, 0x0385, + 0x0386, 0x0387, 0x0389, 0x038A, 0x038C, 0x038E, 0x0390, 0x0392, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039B, + 0x039E, 0x03A1, 0x03A3, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03AA, 0x03AD, 0x03B0, 0x03B2, 0x03B4, 0x03B7, 0x03B8, + 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BE, 0x03C0, 0x03C2, 0x03C4, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CD, + 0x03CE, 0x03CF, 0x03D0, 0x03D1, 0x03D2, 0x03D3, 0x03D4, 0x03D5, 0x03D6, 0x03D7, 0x03D8, 0x03D9, 0x03DA, 0x03DB, + 0x03DC, 0x03DD, 0x03DE, 0x03DF, 0x03E0, 0x03E1, 0x03E2, 0x03E3, 0x03E4, 0x03E5, 0x03E6, 0x03E7, 0x03E8, 0x03E9, + 0x03EA, 0x03EB, 0x03EC, 0x03ED, 0x03EE, 0x03EF, 0x03F0, 0x03F1, 0x03F2, 0x03F3, 0x03F4, 0x03F5, 0x03F6, 0x03F7, + 0x03F8, 0x03F9, 0x03FA, 0x03FB, 0x03FC, 0x03FD, 0x03FE, 0x03FF, 0x0400, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, + 0x0406, 0x0407, 0x0408, 0x0409, 0x040A, 0x040B, 0x040C, 0x040D, 0x040E, 0x040F, 0x0410, 0x0411, 0x0412, 0x0413, + 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, + 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, + 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, + 0x044C, 0x044D, 0x044E, 0x044F, 0x0450, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, + 0x045A, 0x045B, 0x045C, 0x045D, 0x045E, 0x045F, 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467, + 0x0468, 0x0469, 0x046A, 0x046B, 0x046C, 0x046D, 0x046E, 0x046F, 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, + 0x0476, 0x0477, 0x0478, 0x0479, 0x047A, 0x047B, 0x047C, 0x047D, 0x047E, 0x047F, 0x0480, 0x0481, 0x0482, 0x0483, + 0x0484, 0x0485, 0x0486, 0x0487, 0x0488, 0x0489, 0x048A, 0x048B, 0x048C, 0x048D, 0x048E, 0x048F, 0x0490, 0x0491, + 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497, 0x0498, 0x0499, 0x049A, 0x049B, 0x049C, 0x049D, 0x049E, 0x049F, + 0x04A0, 0x04A1, 0x04A2, 0x04A3, 0x04A4, 0x04A5, 0x04A6, 0x04A7, 0x04A8, 0x04A9, 0x04AA, 0x04AB, 0x04AC, 0x04AD, + 0x04AE, 0x04AF, 0x04B0, 0x04B1, 0x04B2, 0x04B3, 0x04B4, 0x04B5, 0x04B6, 0x04B7, 0x04B8, 0x04B9, 0x04BA, 0x04BB, + 0x04BC, 0x04BD, 0x04BE, 0x04BF, 0x04C0, 0x04C1, 0x04C2, 0x04C3, 0x04C4, 0x04C5, 0x04C6, 0x04C7, 0x04C8, 0x04C9, + 0x04CA, 0x04CB, 0x04CC, 0x04CD, 0x04CE, 0x04CF, 0x04D0, 0x04D1, 0x04D2, 0x04D3, 0x04D4, 0x04D5, 0x04D6, 0x04D7, + 0x04D8, 0x04D9, 0x04DA, 0x04DB, 0x04DC, 0x04DD, 0x04DE, 0x04DF, 0x04E0, 0x04E1, 0x04E2, 0x04E3, 0x04E4, 0x04E5, + 0x04E6, 0x04E7, 0x04E8, 0x04E9, 0x04EA, 0x04EB, 0x04EC, 0x04ED, 0x04EE, 0x04EF, 0x04F0, 0x04F1, 0x04F2, 0x04F3, + 0x04F4, 0x04F5, 0x04F6, 0x04F7, 0x04F8, 0x04F9, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF, 0x0500, 0x0501, + 0x0502, 0x0503, 0x0504, 0x0505, 0x0506, 0x0507, 0x0508, 0x0509, 0x050A, 0x050B, 0x050C, 0x050D, 0x050E, 0x050F, + 0x0510, 0x0511, 0x0512, 0x0513, 0x0514, 0x0515, 0x0516, 0x0517, 0x0518, 0x0519, 0x051A, 0x051B, 0x051C, 0x051D, + 0x051E, 0x051F, 0x0520, 0x0521, 0x0522, 0x0523, 0x0524, 0x0525, 0x0526, 0x0527, 0x0528, 0x0529, 0x052A, 0x052B, + 0x052C, 0x052D, 0x052E, 0x052F, 0x0530, 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, 0x0538, 0x0539, + 0x053A, 0x053B, 0x053C, 0x053D, 0x053E, 0x053F, 0x0540, 0x0541, 0x0542, 0x0544, 0x0546, 0x0548, 0x054B, 0x054E, + 0x0550, 0x0552, 0x0554, 0x0556, 0x0558, 0x055A, 0x055C, 0x055D, 0x055E, 0x055F, 0x0560, 0x0561, 0x0562, 0x0563, + 0x0564, 0x0565, 0x0566, 0x0567, 0x0568, 0x0569, 0x056A, 0x056B, 0x056C, 0x056D, 0x056E, 0x056F, 0x0570, 0x0571, + 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, 0x0578, 0x0579, 0x057A, 0x057B, 0x057C, 0x057D, 0x057E, 0x057F, + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0587, 0x0588, 0x0589, 0x058A, 0x058B, 0x058C, 0x058D, + 0x058E, 0x058F, 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, 0x0598, 0x0599, 0x059A, 0x059B, + 0x059C, 0x059D, 0x059E, 0x059F, 0x05A0, 0x05A1, 0x05A2, 0x05A3, 0x05A4, 0x05A5, 0x05A6, 0x05A7, 0x05A8, 0x05A9, + 0x05AA, 0x05AB, 0x05AC, 0x05AD, 0x05AE, 0x05AF, 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, + 0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05C4, 0x05C5, + 0x05C6, 0x05C7, 0x05C8, 0x05C9, 0x05CA, 0x05CB, 0x05CC, 0x05CD, 0x05CE, 0x05CF, 0x05D0, 0x05D1, 0x05D2, 0x05D3, + 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, + 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0x05EB, 0x05EC, 0x05ED, 0x05EE, 0x05EF, + 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0x05F5, 0x05F6, 0x05F7, 0x05F8, 0x05F9, 0x05FA, 0x05FB, 0x05FC, 0x05FD, + 0x05FE, 0x05FF, 0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x0606, 0x0607, 0x0608, 0x0609, 0x060A, 0x060B, + 0x060C, 0x060D, 0x060E, 0x060F, 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, 0x0618, 0x0619, + 0x061A, 0x061B, 0x061C, 0x061D, 0x061E, 0x061F, 0x0620, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, + 0x0636, 0x0637, 0x0638, 0x0639, 0x063A, 0x063B, 0x063C, 0x063D, 0x063E, 0x063F, 0x0640, 0x0641, 0x0642, 0x0643, + 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650, 0x0651, + 0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, 0x0658, 0x0659, 0x065A, 0x065B, 0x065C, 0x065D, 0x065E, 0x065F, + 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, 0x0668, 0x0669, 0x066A, 0x066B, 0x066C, 0x066D, + 0x066E, 0x066F, 0x0670, 0x0671, 0x0672, 0x0673, 0x0674, 0x0675, 0x0676, 0x0677, 0x0678, 0x0679, 0x067A, 0x067B, + 0x067C, 0x067D, 0x067E, 0x067F, 0x0680, 0x0681, 0x0682, 0x0683, 0x0684, 0x0685, 0x0686, 0x0687, 0x0688, 0x0689, + 0x068A, 0x068B, 0x068C, 0x068D, 0x068E, 0x068F, 0x0690, 0x0691, 0x0692, 0x0693, 0x0694, 0x0695, 0x0696, 0x0697, + 0x0698, 0x0699, 0x069A, 0x069B, 0x069C, 0x069D, 0x069E, 0x069F, 0x06A0, 0x06A1, 0x06A2, 0x06A3, 0x06A4, 0x06A5, + 0x06A6, 0x06A7, 0x06A8, 0x06A9, +}; + +/** @brief Fold images concatenated in source order; a multi-rune fold occupies 2-3 consecutive slots, + * addressed by `sz_utf8_fold_preimage_offsets_[index]` .. `sz_utf8_fold_preimage_offsets_[index + 1]`. */ +static sz_u32_t const sz_utf8_fold_preimage_runs_[1705] = { + 0x000061, 0x000062, 0x000063, 0x000064, 0x000065, 0x000066, 0x000067, 0x000068, 0x000069, 0x00006A, 0x00006B, + 0x00006C, 0x00006D, 0x00006E, 0x00006F, 0x000070, 0x000071, 0x000072, 0x000073, 0x000074, 0x000075, 0x000076, + 0x000077, 0x000078, 0x000079, 0x00007A, 0x0003BC, 0x0000E0, 0x0000E1, 0x0000E2, 0x0000E3, 0x0000E4, 0x0000E5, + 0x0000E6, 0x0000E7, 0x0000E8, 0x0000E9, 0x0000EA, 0x0000EB, 0x0000EC, 0x0000ED, 0x0000EE, 0x0000EF, 0x0000F0, + 0x0000F1, 0x0000F2, 0x0000F3, 0x0000F4, 0x0000F5, 0x0000F6, 0x0000F8, 0x0000F9, 0x0000FA, 0x0000FB, 0x0000FC, + 0x0000FD, 0x0000FE, 0x000073, 0x000073, 0x000101, 0x000103, 0x000105, 0x000107, 0x000109, 0x00010B, 0x00010D, + 0x00010F, 0x000111, 0x000113, 0x000115, 0x000117, 0x000119, 0x00011B, 0x00011D, 0x00011F, 0x000121, 0x000123, + 0x000125, 0x000127, 0x000129, 0x00012B, 0x00012D, 0x00012F, 0x000069, 0x000307, 0x000133, 0x000135, 0x000137, + 0x00013A, 0x00013C, 0x00013E, 0x000140, 0x000142, 0x000144, 0x000146, 0x000148, 0x0002BC, 0x00006E, 0x00014B, + 0x00014D, 0x00014F, 0x000151, 0x000153, 0x000155, 0x000157, 0x000159, 0x00015B, 0x00015D, 0x00015F, 0x000161, + 0x000163, 0x000165, 0x000167, 0x000169, 0x00016B, 0x00016D, 0x00016F, 0x000171, 0x000173, 0x000175, 0x000177, + 0x0000FF, 0x00017A, 0x00017C, 0x00017E, 0x000073, 0x000253, 0x000183, 0x000185, 0x000254, 0x000188, 0x000256, + 0x000257, 0x00018C, 0x0001DD, 0x000259, 0x00025B, 0x000192, 0x000260, 0x000263, 0x000269, 0x000268, 0x000199, + 0x00026F, 0x000272, 0x000275, 0x0001A1, 0x0001A3, 0x0001A5, 0x000280, 0x0001A8, 0x000283, 0x0001AD, 0x000288, + 0x0001B0, 0x00028A, 0x00028B, 0x0001B4, 0x0001B6, 0x000292, 0x0001B9, 0x0001BD, 0x0001C6, 0x0001C6, 0x0001C9, + 0x0001C9, 0x0001CC, 0x0001CC, 0x0001CE, 0x0001D0, 0x0001D2, 0x0001D4, 0x0001D6, 0x0001D8, 0x0001DA, 0x0001DC, + 0x0001DF, 0x0001E1, 0x0001E3, 0x0001E5, 0x0001E7, 0x0001E9, 0x0001EB, 0x0001ED, 0x0001EF, 0x00006A, 0x00030C, + 0x0001F3, 0x0001F3, 0x0001F5, 0x000195, 0x0001BF, 0x0001F9, 0x0001FB, 0x0001FD, 0x0001FF, 0x000201, 0x000203, + 0x000205, 0x000207, 0x000209, 0x00020B, 0x00020D, 0x00020F, 0x000211, 0x000213, 0x000215, 0x000217, 0x000219, + 0x00021B, 0x00021D, 0x00021F, 0x00019E, 0x000223, 0x000225, 0x000227, 0x000229, 0x00022B, 0x00022D, 0x00022F, + 0x000231, 0x000233, 0x002C65, 0x00023C, 0x00019A, 0x002C66, 0x000242, 0x000180, 0x000289, 0x00028C, 0x000247, + 0x000249, 0x00024B, 0x00024D, 0x00024F, 0x0003B9, 0x000371, 0x000373, 0x000377, 0x0003F3, 0x0003AC, 0x0003AD, + 0x0003AE, 0x0003AF, 0x0003CC, 0x0003CD, 0x0003CE, 0x0003B9, 0x000308, 0x000301, 0x0003B1, 0x0003B2, 0x0003B3, + 0x0003B4, 0x0003B5, 0x0003B6, 0x0003B7, 0x0003B8, 0x0003B9, 0x0003BA, 0x0003BB, 0x0003BC, 0x0003BD, 0x0003BE, + 0x0003BF, 0x0003C0, 0x0003C1, 0x0003C3, 0x0003C4, 0x0003C5, 0x0003C6, 0x0003C7, 0x0003C8, 0x0003C9, 0x0003CA, + 0x0003CB, 0x0003C5, 0x000308, 0x000301, 0x0003C3, 0x0003D7, 0x0003B2, 0x0003B8, 0x0003C6, 0x0003C0, 0x0003D9, + 0x0003DB, 0x0003DD, 0x0003DF, 0x0003E1, 0x0003E3, 0x0003E5, 0x0003E7, 0x0003E9, 0x0003EB, 0x0003ED, 0x0003EF, + 0x0003BA, 0x0003C1, 0x0003B8, 0x0003B5, 0x0003F8, 0x0003F2, 0x0003FB, 0x00037B, 0x00037C, 0x00037D, 0x000450, + 0x000451, 0x000452, 0x000453, 0x000454, 0x000455, 0x000456, 0x000457, 0x000458, 0x000459, 0x00045A, 0x00045B, + 0x00045C, 0x00045D, 0x00045E, 0x00045F, 0x000430, 0x000431, 0x000432, 0x000433, 0x000434, 0x000435, 0x000436, + 0x000437, 0x000438, 0x000439, 0x00043A, 0x00043B, 0x00043C, 0x00043D, 0x00043E, 0x00043F, 0x000440, 0x000441, + 0x000442, 0x000443, 0x000444, 0x000445, 0x000446, 0x000447, 0x000448, 0x000449, 0x00044A, 0x00044B, 0x00044C, + 0x00044D, 0x00044E, 0x00044F, 0x000461, 0x000463, 0x000465, 0x000467, 0x000469, 0x00046B, 0x00046D, 0x00046F, + 0x000471, 0x000473, 0x000475, 0x000477, 0x000479, 0x00047B, 0x00047D, 0x00047F, 0x000481, 0x00048B, 0x00048D, + 0x00048F, 0x000491, 0x000493, 0x000495, 0x000497, 0x000499, 0x00049B, 0x00049D, 0x00049F, 0x0004A1, 0x0004A3, + 0x0004A5, 0x0004A7, 0x0004A9, 0x0004AB, 0x0004AD, 0x0004AF, 0x0004B1, 0x0004B3, 0x0004B5, 0x0004B7, 0x0004B9, + 0x0004BB, 0x0004BD, 0x0004BF, 0x0004CF, 0x0004C2, 0x0004C4, 0x0004C6, 0x0004C8, 0x0004CA, 0x0004CC, 0x0004CE, + 0x0004D1, 0x0004D3, 0x0004D5, 0x0004D7, 0x0004D9, 0x0004DB, 0x0004DD, 0x0004DF, 0x0004E1, 0x0004E3, 0x0004E5, + 0x0004E7, 0x0004E9, 0x0004EB, 0x0004ED, 0x0004EF, 0x0004F1, 0x0004F3, 0x0004F5, 0x0004F7, 0x0004F9, 0x0004FB, + 0x0004FD, 0x0004FF, 0x000501, 0x000503, 0x000505, 0x000507, 0x000509, 0x00050B, 0x00050D, 0x00050F, 0x000511, + 0x000513, 0x000515, 0x000517, 0x000519, 0x00051B, 0x00051D, 0x00051F, 0x000521, 0x000523, 0x000525, 0x000527, + 0x000529, 0x00052B, 0x00052D, 0x00052F, 0x000561, 0x000562, 0x000563, 0x000564, 0x000565, 0x000566, 0x000567, + 0x000568, 0x000569, 0x00056A, 0x00056B, 0x00056C, 0x00056D, 0x00056E, 0x00056F, 0x000570, 0x000571, 0x000572, + 0x000573, 0x000574, 0x000575, 0x000576, 0x000577, 0x000578, 0x000579, 0x00057A, 0x00057B, 0x00057C, 0x00057D, + 0x00057E, 0x00057F, 0x000580, 0x000581, 0x000582, 0x000583, 0x000584, 0x000585, 0x000586, 0x000565, 0x000582, + 0x002D00, 0x002D01, 0x002D02, 0x002D03, 0x002D04, 0x002D05, 0x002D06, 0x002D07, 0x002D08, 0x002D09, 0x002D0A, + 0x002D0B, 0x002D0C, 0x002D0D, 0x002D0E, 0x002D0F, 0x002D10, 0x002D11, 0x002D12, 0x002D13, 0x002D14, 0x002D15, + 0x002D16, 0x002D17, 0x002D18, 0x002D19, 0x002D1A, 0x002D1B, 0x002D1C, 0x002D1D, 0x002D1E, 0x002D1F, 0x002D20, + 0x002D21, 0x002D22, 0x002D23, 0x002D24, 0x002D25, 0x002D27, 0x002D2D, 0x0013F0, 0x0013F1, 0x0013F2, 0x0013F3, + 0x0013F4, 0x0013F5, 0x000432, 0x000434, 0x00043E, 0x000441, 0x000442, 0x000442, 0x00044A, 0x000463, 0x00A64B, + 0x001C8A, 0x0010D0, 0x0010D1, 0x0010D2, 0x0010D3, 0x0010D4, 0x0010D5, 0x0010D6, 0x0010D7, 0x0010D8, 0x0010D9, + 0x0010DA, 0x0010DB, 0x0010DC, 0x0010DD, 0x0010DE, 0x0010DF, 0x0010E0, 0x0010E1, 0x0010E2, 0x0010E3, 0x0010E4, + 0x0010E5, 0x0010E6, 0x0010E7, 0x0010E8, 0x0010E9, 0x0010EA, 0x0010EB, 0x0010EC, 0x0010ED, 0x0010EE, 0x0010EF, + 0x0010F0, 0x0010F1, 0x0010F2, 0x0010F3, 0x0010F4, 0x0010F5, 0x0010F6, 0x0010F7, 0x0010F8, 0x0010F9, 0x0010FA, + 0x0010FD, 0x0010FE, 0x0010FF, 0x001E01, 0x001E03, 0x001E05, 0x001E07, 0x001E09, 0x001E0B, 0x001E0D, 0x001E0F, + 0x001E11, 0x001E13, 0x001E15, 0x001E17, 0x001E19, 0x001E1B, 0x001E1D, 0x001E1F, 0x001E21, 0x001E23, 0x001E25, + 0x001E27, 0x001E29, 0x001E2B, 0x001E2D, 0x001E2F, 0x001E31, 0x001E33, 0x001E35, 0x001E37, 0x001E39, 0x001E3B, + 0x001E3D, 0x001E3F, 0x001E41, 0x001E43, 0x001E45, 0x001E47, 0x001E49, 0x001E4B, 0x001E4D, 0x001E4F, 0x001E51, + 0x001E53, 0x001E55, 0x001E57, 0x001E59, 0x001E5B, 0x001E5D, 0x001E5F, 0x001E61, 0x001E63, 0x001E65, 0x001E67, + 0x001E69, 0x001E6B, 0x001E6D, 0x001E6F, 0x001E71, 0x001E73, 0x001E75, 0x001E77, 0x001E79, 0x001E7B, 0x001E7D, + 0x001E7F, 0x001E81, 0x001E83, 0x001E85, 0x001E87, 0x001E89, 0x001E8B, 0x001E8D, 0x001E8F, 0x001E91, 0x001E93, + 0x001E95, 0x000068, 0x000331, 0x000074, 0x000308, 0x000077, 0x00030A, 0x000079, 0x00030A, 0x000061, 0x0002BE, + 0x001E61, 0x000073, 0x000073, 0x001EA1, 0x001EA3, 0x001EA5, 0x001EA7, 0x001EA9, 0x001EAB, 0x001EAD, 0x001EAF, + 0x001EB1, 0x001EB3, 0x001EB5, 0x001EB7, 0x001EB9, 0x001EBB, 0x001EBD, 0x001EBF, 0x001EC1, 0x001EC3, 0x001EC5, + 0x001EC7, 0x001EC9, 0x001ECB, 0x001ECD, 0x001ECF, 0x001ED1, 0x001ED3, 0x001ED5, 0x001ED7, 0x001ED9, 0x001EDB, + 0x001EDD, 0x001EDF, 0x001EE1, 0x001EE3, 0x001EE5, 0x001EE7, 0x001EE9, 0x001EEB, 0x001EED, 0x001EEF, 0x001EF1, + 0x001EF3, 0x001EF5, 0x001EF7, 0x001EF9, 0x001EFB, 0x001EFD, 0x001EFF, 0x001F00, 0x001F01, 0x001F02, 0x001F03, + 0x001F04, 0x001F05, 0x001F06, 0x001F07, 0x001F10, 0x001F11, 0x001F12, 0x001F13, 0x001F14, 0x001F15, 0x001F20, + 0x001F21, 0x001F22, 0x001F23, 0x001F24, 0x001F25, 0x001F26, 0x001F27, 0x001F30, 0x001F31, 0x001F32, 0x001F33, + 0x001F34, 0x001F35, 0x001F36, 0x001F37, 0x001F40, 0x001F41, 0x001F42, 0x001F43, 0x001F44, 0x001F45, 0x0003C5, + 0x000313, 0x0003C5, 0x000313, 0x000300, 0x0003C5, 0x000313, 0x000301, 0x0003C5, 0x000313, 0x000342, 0x001F51, + 0x001F53, 0x001F55, 0x001F57, 0x001F60, 0x001F61, 0x001F62, 0x001F63, 0x001F64, 0x001F65, 0x001F66, 0x001F67, + 0x001F00, 0x0003B9, 0x001F01, 0x0003B9, 0x001F02, 0x0003B9, 0x001F03, 0x0003B9, 0x001F04, 0x0003B9, 0x001F05, + 0x0003B9, 0x001F06, 0x0003B9, 0x001F07, 0x0003B9, 0x001F00, 0x0003B9, 0x001F01, 0x0003B9, 0x001F02, 0x0003B9, + 0x001F03, 0x0003B9, 0x001F04, 0x0003B9, 0x001F05, 0x0003B9, 0x001F06, 0x0003B9, 0x001F07, 0x0003B9, 0x001F20, + 0x0003B9, 0x001F21, 0x0003B9, 0x001F22, 0x0003B9, 0x001F23, 0x0003B9, 0x001F24, 0x0003B9, 0x001F25, 0x0003B9, + 0x001F26, 0x0003B9, 0x001F27, 0x0003B9, 0x001F20, 0x0003B9, 0x001F21, 0x0003B9, 0x001F22, 0x0003B9, 0x001F23, + 0x0003B9, 0x001F24, 0x0003B9, 0x001F25, 0x0003B9, 0x001F26, 0x0003B9, 0x001F27, 0x0003B9, 0x001F60, 0x0003B9, + 0x001F61, 0x0003B9, 0x001F62, 0x0003B9, 0x001F63, 0x0003B9, 0x001F64, 0x0003B9, 0x001F65, 0x0003B9, 0x001F66, + 0x0003B9, 0x001F67, 0x0003B9, 0x001F60, 0x0003B9, 0x001F61, 0x0003B9, 0x001F62, 0x0003B9, 0x001F63, 0x0003B9, + 0x001F64, 0x0003B9, 0x001F65, 0x0003B9, 0x001F66, 0x0003B9, 0x001F67, 0x0003B9, 0x001F70, 0x0003B9, 0x0003B1, + 0x0003B9, 0x0003AC, 0x0003B9, 0x0003B1, 0x000342, 0x0003B1, 0x000342, 0x0003B9, 0x001FB0, 0x001FB1, 0x001F70, + 0x001F71, 0x0003B1, 0x0003B9, 0x0003B9, 0x001F74, 0x0003B9, 0x0003B7, 0x0003B9, 0x0003AE, 0x0003B9, 0x0003B7, + 0x000342, 0x0003B7, 0x000342, 0x0003B9, 0x001F72, 0x001F73, 0x001F74, 0x001F75, 0x0003B7, 0x0003B9, 0x0003B9, + 0x000308, 0x000300, 0x0003B9, 0x000308, 0x000301, 0x0003B9, 0x000342, 0x0003B9, 0x000308, 0x000342, 0x001FD0, + 0x001FD1, 0x001F76, 0x001F77, 0x0003C5, 0x000308, 0x000300, 0x0003C5, 0x000308, 0x000301, 0x0003C1, 0x000313, + 0x0003C5, 0x000342, 0x0003C5, 0x000308, 0x000342, 0x001FE0, 0x001FE1, 0x001F7A, 0x001F7B, 0x001FE5, 0x001F7C, + 0x0003B9, 0x0003C9, 0x0003B9, 0x0003CE, 0x0003B9, 0x0003C9, 0x000342, 0x0003C9, 0x000342, 0x0003B9, 0x001F78, + 0x001F79, 0x001F7C, 0x001F7D, 0x0003C9, 0x0003B9, 0x0003C9, 0x00006B, 0x0000E5, 0x00214E, 0x002170, 0x002171, + 0x002172, 0x002173, 0x002174, 0x002175, 0x002176, 0x002177, 0x002178, 0x002179, 0x00217A, 0x00217B, 0x00217C, + 0x00217D, 0x00217E, 0x00217F, 0x002184, 0x0024D0, 0x0024D1, 0x0024D2, 0x0024D3, 0x0024D4, 0x0024D5, 0x0024D6, + 0x0024D7, 0x0024D8, 0x0024D9, 0x0024DA, 0x0024DB, 0x0024DC, 0x0024DD, 0x0024DE, 0x0024DF, 0x0024E0, 0x0024E1, + 0x0024E2, 0x0024E3, 0x0024E4, 0x0024E5, 0x0024E6, 0x0024E7, 0x0024E8, 0x0024E9, 0x002C30, 0x002C31, 0x002C32, + 0x002C33, 0x002C34, 0x002C35, 0x002C36, 0x002C37, 0x002C38, 0x002C39, 0x002C3A, 0x002C3B, 0x002C3C, 0x002C3D, + 0x002C3E, 0x002C3F, 0x002C40, 0x002C41, 0x002C42, 0x002C43, 0x002C44, 0x002C45, 0x002C46, 0x002C47, 0x002C48, + 0x002C49, 0x002C4A, 0x002C4B, 0x002C4C, 0x002C4D, 0x002C4E, 0x002C4F, 0x002C50, 0x002C51, 0x002C52, 0x002C53, + 0x002C54, 0x002C55, 0x002C56, 0x002C57, 0x002C58, 0x002C59, 0x002C5A, 0x002C5B, 0x002C5C, 0x002C5D, 0x002C5E, + 0x002C5F, 0x002C61, 0x00026B, 0x001D7D, 0x00027D, 0x002C68, 0x002C6A, 0x002C6C, 0x000251, 0x000271, 0x000250, + 0x000252, 0x002C73, 0x002C76, 0x00023F, 0x000240, 0x002C81, 0x002C83, 0x002C85, 0x002C87, 0x002C89, 0x002C8B, + 0x002C8D, 0x002C8F, 0x002C91, 0x002C93, 0x002C95, 0x002C97, 0x002C99, 0x002C9B, 0x002C9D, 0x002C9F, 0x002CA1, + 0x002CA3, 0x002CA5, 0x002CA7, 0x002CA9, 0x002CAB, 0x002CAD, 0x002CAF, 0x002CB1, 0x002CB3, 0x002CB5, 0x002CB7, + 0x002CB9, 0x002CBB, 0x002CBD, 0x002CBF, 0x002CC1, 0x002CC3, 0x002CC5, 0x002CC7, 0x002CC9, 0x002CCB, 0x002CCD, + 0x002CCF, 0x002CD1, 0x002CD3, 0x002CD5, 0x002CD7, 0x002CD9, 0x002CDB, 0x002CDD, 0x002CDF, 0x002CE1, 0x002CE3, + 0x002CEC, 0x002CEE, 0x002CF3, 0x00A641, 0x00A643, 0x00A645, 0x00A647, 0x00A649, 0x00A64B, 0x00A64D, 0x00A64F, + 0x00A651, 0x00A653, 0x00A655, 0x00A657, 0x00A659, 0x00A65B, 0x00A65D, 0x00A65F, 0x00A661, 0x00A663, 0x00A665, + 0x00A667, 0x00A669, 0x00A66B, 0x00A66D, 0x00A681, 0x00A683, 0x00A685, 0x00A687, 0x00A689, 0x00A68B, 0x00A68D, + 0x00A68F, 0x00A691, 0x00A693, 0x00A695, 0x00A697, 0x00A699, 0x00A69B, 0x00A723, 0x00A725, 0x00A727, 0x00A729, + 0x00A72B, 0x00A72D, 0x00A72F, 0x00A733, 0x00A735, 0x00A737, 0x00A739, 0x00A73B, 0x00A73D, 0x00A73F, 0x00A741, + 0x00A743, 0x00A745, 0x00A747, 0x00A749, 0x00A74B, 0x00A74D, 0x00A74F, 0x00A751, 0x00A753, 0x00A755, 0x00A757, + 0x00A759, 0x00A75B, 0x00A75D, 0x00A75F, 0x00A761, 0x00A763, 0x00A765, 0x00A767, 0x00A769, 0x00A76B, 0x00A76D, + 0x00A76F, 0x00A77A, 0x00A77C, 0x001D79, 0x00A77F, 0x00A781, 0x00A783, 0x00A785, 0x00A787, 0x00A78C, 0x000265, + 0x00A791, 0x00A793, 0x00A797, 0x00A799, 0x00A79B, 0x00A79D, 0x00A79F, 0x00A7A1, 0x00A7A3, 0x00A7A5, 0x00A7A7, + 0x00A7A9, 0x000266, 0x00025C, 0x000261, 0x00026C, 0x00026A, 0x00029E, 0x000287, 0x00029D, 0x00AB53, 0x00A7B5, + 0x00A7B7, 0x00A7B9, 0x00A7BB, 0x00A7BD, 0x00A7BF, 0x00A7C1, 0x00A7C3, 0x00A794, 0x000282, 0x001D8E, 0x00A7C8, + 0x00A7CA, 0x000264, 0x00A7CD, 0x00A7CF, 0x00A7D1, 0x00A7D3, 0x00A7D5, 0x00A7D7, 0x00A7D9, 0x00A7DB, 0x00019B, + 0x00A7F6, 0x0013A0, 0x0013A1, 0x0013A2, 0x0013A3, 0x0013A4, 0x0013A5, 0x0013A6, 0x0013A7, 0x0013A8, 0x0013A9, + 0x0013AA, 0x0013AB, 0x0013AC, 0x0013AD, 0x0013AE, 0x0013AF, 0x0013B0, 0x0013B1, 0x0013B2, 0x0013B3, 0x0013B4, + 0x0013B5, 0x0013B6, 0x0013B7, 0x0013B8, 0x0013B9, 0x0013BA, 0x0013BB, 0x0013BC, 0x0013BD, 0x0013BE, 0x0013BF, + 0x0013C0, 0x0013C1, 0x0013C2, 0x0013C3, 0x0013C4, 0x0013C5, 0x0013C6, 0x0013C7, 0x0013C8, 0x0013C9, 0x0013CA, + 0x0013CB, 0x0013CC, 0x0013CD, 0x0013CE, 0x0013CF, 0x0013D0, 0x0013D1, 0x0013D2, 0x0013D3, 0x0013D4, 0x0013D5, + 0x0013D6, 0x0013D7, 0x0013D8, 0x0013D9, 0x0013DA, 0x0013DB, 0x0013DC, 0x0013DD, 0x0013DE, 0x0013DF, 0x0013E0, + 0x0013E1, 0x0013E2, 0x0013E3, 0x0013E4, 0x0013E5, 0x0013E6, 0x0013E7, 0x0013E8, 0x0013E9, 0x0013EA, 0x0013EB, + 0x0013EC, 0x0013ED, 0x0013EE, 0x0013EF, 0x000066, 0x000066, 0x000066, 0x000069, 0x000066, 0x00006C, 0x000066, + 0x000066, 0x000069, 0x000066, 0x000066, 0x00006C, 0x000073, 0x000074, 0x000073, 0x000074, 0x000574, 0x000576, + 0x000574, 0x000565, 0x000574, 0x00056B, 0x00057E, 0x000576, 0x000574, 0x00056D, 0x00FF41, 0x00FF42, 0x00FF43, + 0x00FF44, 0x00FF45, 0x00FF46, 0x00FF47, 0x00FF48, 0x00FF49, 0x00FF4A, 0x00FF4B, 0x00FF4C, 0x00FF4D, 0x00FF4E, + 0x00FF4F, 0x00FF50, 0x00FF51, 0x00FF52, 0x00FF53, 0x00FF54, 0x00FF55, 0x00FF56, 0x00FF57, 0x00FF58, 0x00FF59, + 0x00FF5A, 0x010428, 0x010429, 0x01042A, 0x01042B, 0x01042C, 0x01042D, 0x01042E, 0x01042F, 0x010430, 0x010431, + 0x010432, 0x010433, 0x010434, 0x010435, 0x010436, 0x010437, 0x010438, 0x010439, 0x01043A, 0x01043B, 0x01043C, + 0x01043D, 0x01043E, 0x01043F, 0x010440, 0x010441, 0x010442, 0x010443, 0x010444, 0x010445, 0x010446, 0x010447, + 0x010448, 0x010449, 0x01044A, 0x01044B, 0x01044C, 0x01044D, 0x01044E, 0x01044F, 0x0104D8, 0x0104D9, 0x0104DA, + 0x0104DB, 0x0104DC, 0x0104DD, 0x0104DE, 0x0104DF, 0x0104E0, 0x0104E1, 0x0104E2, 0x0104E3, 0x0104E4, 0x0104E5, + 0x0104E6, 0x0104E7, 0x0104E8, 0x0104E9, 0x0104EA, 0x0104EB, 0x0104EC, 0x0104ED, 0x0104EE, 0x0104EF, 0x0104F0, + 0x0104F1, 0x0104F2, 0x0104F3, 0x0104F4, 0x0104F5, 0x0104F6, 0x0104F7, 0x0104F8, 0x0104F9, 0x0104FA, 0x0104FB, + 0x010597, 0x010598, 0x010599, 0x01059A, 0x01059B, 0x01059C, 0x01059D, 0x01059E, 0x01059F, 0x0105A0, 0x0105A1, + 0x0105A3, 0x0105A4, 0x0105A5, 0x0105A6, 0x0105A7, 0x0105A8, 0x0105A9, 0x0105AA, 0x0105AB, 0x0105AC, 0x0105AD, + 0x0105AE, 0x0105AF, 0x0105B0, 0x0105B1, 0x0105B3, 0x0105B4, 0x0105B5, 0x0105B6, 0x0105B7, 0x0105B8, 0x0105B9, + 0x0105BB, 0x0105BC, 0x010CC0, 0x010CC1, 0x010CC2, 0x010CC3, 0x010CC4, 0x010CC5, 0x010CC6, 0x010CC7, 0x010CC8, + 0x010CC9, 0x010CCA, 0x010CCB, 0x010CCC, 0x010CCD, 0x010CCE, 0x010CCF, 0x010CD0, 0x010CD1, 0x010CD2, 0x010CD3, + 0x010CD4, 0x010CD5, 0x010CD6, 0x010CD7, 0x010CD8, 0x010CD9, 0x010CDA, 0x010CDB, 0x010CDC, 0x010CDD, 0x010CDE, + 0x010CDF, 0x010CE0, 0x010CE1, 0x010CE2, 0x010CE3, 0x010CE4, 0x010CE5, 0x010CE6, 0x010CE7, 0x010CE8, 0x010CE9, + 0x010CEA, 0x010CEB, 0x010CEC, 0x010CED, 0x010CEE, 0x010CEF, 0x010CF0, 0x010CF1, 0x010CF2, 0x010D70, 0x010D71, + 0x010D72, 0x010D73, 0x010D74, 0x010D75, 0x010D76, 0x010D77, 0x010D78, 0x010D79, 0x010D7A, 0x010D7B, 0x010D7C, + 0x010D7D, 0x010D7E, 0x010D7F, 0x010D80, 0x010D81, 0x010D82, 0x010D83, 0x010D84, 0x010D85, 0x0118C0, 0x0118C1, + 0x0118C2, 0x0118C3, 0x0118C4, 0x0118C5, 0x0118C6, 0x0118C7, 0x0118C8, 0x0118C9, 0x0118CA, 0x0118CB, 0x0118CC, + 0x0118CD, 0x0118CE, 0x0118CF, 0x0118D0, 0x0118D1, 0x0118D2, 0x0118D3, 0x0118D4, 0x0118D5, 0x0118D6, 0x0118D7, + 0x0118D8, 0x0118D9, 0x0118DA, 0x0118DB, 0x0118DC, 0x0118DD, 0x0118DE, 0x0118DF, 0x016E60, 0x016E61, 0x016E62, + 0x016E63, 0x016E64, 0x016E65, 0x016E66, 0x016E67, 0x016E68, 0x016E69, 0x016E6A, 0x016E6B, 0x016E6C, 0x016E6D, + 0x016E6E, 0x016E6F, 0x016E70, 0x016E71, 0x016E72, 0x016E73, 0x016E74, 0x016E75, 0x016E76, 0x016E77, 0x016E78, + 0x016E79, 0x016E7A, 0x016E7B, 0x016E7C, 0x016E7D, 0x016E7E, 0x016E7F, 0x016EBB, 0x016EBC, 0x016EBD, 0x016EBE, + 0x016EBF, 0x016EC0, 0x016EC1, 0x016EC2, 0x016EC3, 0x016EC4, 0x016EC5, 0x016EC6, 0x016EC7, 0x016EC8, 0x016EC9, + 0x016ECA, 0x016ECB, 0x016ECC, 0x016ECD, 0x016ECE, 0x016ECF, 0x016ED0, 0x016ED1, 0x016ED2, 0x016ED3, 0x01E922, + 0x01E923, 0x01E924, 0x01E925, 0x01E926, 0x01E927, 0x01E928, 0x01E929, 0x01E92A, 0x01E92B, 0x01E92C, 0x01E92D, + 0x01E92E, 0x01E92F, 0x01E930, 0x01E931, 0x01E932, 0x01E933, 0x01E934, 0x01E935, 0x01E936, 0x01E937, 0x01E938, + 0x01E939, 0x01E93A, 0x01E93B, 0x01E93C, 0x01E93D, 0x01E93E, 0x01E93F, 0x01E940, 0x01E941, 0x01E942, 0x01E943, +}; + +#pragma endregion Fold preimage tables + +#ifdef __cplusplus +} +#endif + +#endif // STRINGZILLA_UTF8_UNCASED_TABLES_H_ diff --git a/include/stringzilla/utf8_uncased/v128.h b/include/stringzilla/utf8_uncased/v128.h index ae156b5f..8552f7f7 100644 --- a/include/stringzilla/utf8_uncased/v128.h +++ b/include/stringzilla/utf8_uncased/v128.h @@ -50,17 +50,17 @@ SZ_HELPER_INLINE v128_t sz_utf8_uncased_inrange01_v128_(v128_t bytes_u8x16, sz_u } /** @brief Zero a scratch buffer then copy `length` bytes, so strip kernels can read one byte past. */ -SZ_HELPER_AUTO sz_u8_t const *sz_utf8_uncased_load_padded_v128_(sz_cptr_t source, sz_size_t length, sz_u8_t *buffer, - sz_size_t buffer_capacity) { +SZ_HELPER_INLINE sz_u8_t const *sz_utf8_uncased_load_padded_v128_(sz_cptr_t source, sz_size_t length, sz_u8_t *buffer, + sz_size_t buffer_capacity) { for (sz_size_t byte_index = 0; byte_index < buffer_capacity; ++byte_index) buffer[byte_index] = 0; for (sz_size_t byte_index = 0; byte_index < length; ++byte_index) buffer[byte_index] = (sz_u8_t)source[byte_index]; return buffer; } /** @brief Gather the C4/C5/C6 +1 parity delta for the continuation byte's low 6 bits (irregular flag kept). */ -SZ_HELPER_AUTO v128_t sz_utf8_uncased_latin_delta_v128_(v128_t source_u8x16, v128_t after_c4_u8x16, - v128_t after_c5_u8x16, v128_t after_c6_u8x16, - v128_t is_continuation_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_uncased_latin_delta_v128_(v128_t source_u8x16, v128_t after_c4_u8x16, + v128_t after_c5_u8x16, v128_t after_c6_u8x16, + v128_t is_continuation_u8x16) { v128_t low6_u8x16 = wasm_v128_and(source_u8x16, wasm_i8x16_splat(0x3F)); v128_t c4_u8x16 = sz_utf8_gather64_v128_(wasm_v128_load(&sz_utf8_fold_latin_c4_deltas_v128_[0]), wasm_v128_load(&sz_utf8_fold_latin_c4_deltas_v128_[16]), @@ -83,17 +83,17 @@ SZ_HELPER_AUTO v128_t sz_utf8_uncased_latin_delta_v128_(v128_t source_u8x16, v12 /** @brief A 16-byte fold window: the chunk at `src + pos` plus its cross-boundary neighbours. */ typedef struct { - v128_t source; // The 16 bytes at `src + pos`. - v128_t previous; // `source` slid up one lane, carrying the real predecessor byte (0 at `pos == 0`). - v128_t next; // `source` slid down one lane, carrying the real successor byte. + v128_t source_u8x16; // The 16 bytes at `src + pos`. + v128_t previous_u8x16; // `source_u8x16` slid up one lane, carrying the real predecessor byte (0 at `pos == 0`). + v128_t next_u8x16; // `source_u8x16` slid down one lane, carrying the real successor byte. } sz_utf8_uncased_window_v128_t; /** @brief Loads the fold window at `src + pos`, reading one byte on each side for cross-window folds. */ SZ_HELPER_INLINE sz_utf8_uncased_window_v128_t sz_utf8_uncased_load_window_v128_(sz_u8_t const *src, sz_size_t pos) { sz_utf8_uncased_window_v128_t window; - window.source = wasm_v128_load(src + pos); - window.previous = sz_utf8_uncased_slide1up_v128_(window.source, pos > 0 ? src[pos - 1] : 0); - window.next = sz_utf8_slide1down_v128_(window.source, src[pos + 16]); + window.source_u8x16 = wasm_v128_load(src + pos); + window.previous_u8x16 = sz_utf8_uncased_slide1up_v128_(window.source_u8x16, pos > 0 ? src[pos - 1] : 0); + window.next_u8x16 = sz_utf8_slide1down_v128_(window.source_u8x16, src[pos + 16]); return window; } @@ -114,7 +114,7 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_western_europe_strip_v128_(sz_u8_t for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t after_c3_u8x16 = wasm_i8x16_eq(previous_u8x16, wasm_i8x16_splat((sz_i8_t)0xC3)); v128_t folded_u8x16 = sz_ascii_fold_v128_(source_u8x16); v128_t latin1_range_u8x16 = sz_utf8_in_range_v128_(source_u8x16, 0x80, 0x1F); @@ -136,7 +136,7 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_central_europe_strip_v128_(sz_u8_t for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; sz_unused_(next_u8x16); v128_t is_continuation_u8x16 = wasm_i8x16_eq(wasm_v128_and(source_u8x16, wasm_i8x16_splat((sz_i8_t)0xC0)), wasm_i8x16_splat((sz_i8_t)0x80)); @@ -162,7 +162,7 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_cyrillic_strip_v128_(sz_u8_t const for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t after_d0_u8x16 = wasm_i8x16_eq(previous_u8x16, wasm_i8x16_splat((sz_i8_t)0xD0)); v128_t is_d0_u8x16 = wasm_i8x16_eq(source_u8x16, wasm_i8x16_splat((sz_i8_t)0xD0)); v128_t folded_u8x16 = sz_ascii_fold_v128_(source_u8x16); @@ -194,7 +194,7 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_greek_strip_v128_(sz_u8_t const *sr for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t is_continuation_u8x16 = wasm_i8x16_eq(wasm_v128_and(source_u8x16, wasm_i8x16_splat((sz_i8_t)0xC0)), wasm_i8x16_splat((sz_i8_t)0x80)); v128_t after_ce_u8x16 = wasm_i8x16_eq(previous_u8x16, wasm_i8x16_splat((sz_i8_t)0xCE)); @@ -243,7 +243,7 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_armenian_strip_v128_(sz_u8_t const for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t is_d4_u8x16 = wasm_i8x16_eq(source_u8x16, wasm_i8x16_splat((sz_i8_t)0xD4)); v128_t is_d5_u8x16 = wasm_i8x16_eq(source_u8x16, wasm_i8x16_splat((sz_i8_t)0xD5)); v128_t after_d4_u8x16 = wasm_i8x16_eq(previous_u8x16, wasm_i8x16_splat((sz_i8_t)0xD4)); @@ -299,8 +299,8 @@ SZ_HELPER_NOINLINE void sz_utf8_uncased_fold_vietnamese_strip_v128_(sz_u8_t cons #pragma region Per script alarm strips /** @brief Fold the first lead-danger from a window's 0/1 second-byte danger vector into `*best` (min). */ -SZ_HELPER_AUTO void sz_utf8_uncased_alarm_window_(v128_t danger_second_u8x16, sz_size_t pos, sz_size_t window, - long *best) { +SZ_HELPER_INLINE void sz_utf8_uncased_alarm_window_(v128_t danger_second_u8x16, sz_size_t pos, sz_size_t window, + long *best) { sz_u32_t bits = (sz_u32_t)wasm_i8x16_bitmask(wasm_i8x16_ne(danger_second_u8x16, wasm_i8x16_splat(0))); if (window < 16) bits &= ((sz_u32_t)1 << window) - 1; while (bits) { @@ -319,7 +319,7 @@ SZ_HELPER_NOINLINE long sz_utf8_uncased_alarm_western_europe_strip_v128_(sz_u8_t for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t after_c3_u8x16 = sz_utf8_uncased_eq01_v128_(previous_u8x16, 0xC3); v128_t after_c5_u8x16 = sz_utf8_uncased_eq01_v128_(previous_u8x16, 0xC5); v128_t danger_u8x16 = wasm_v128_and(wasm_v128_and(sz_utf8_uncased_eq01_v128_(source_u8x16, 0xBA), @@ -378,7 +378,7 @@ SZ_HELPER_NOINLINE long sz_utf8_uncased_alarm_cyrillic_strip_v128_(sz_u8_t const for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t danger_u8x16 = wasm_v128_and(wasm_v128_and(sz_utf8_uncased_eq01_v128_(source_u8x16, 0xB2), sz_utf8_uncased_eq01_v128_(previous_u8x16, 0xE1)), sz_utf8_uncased_inrange01_v128_(next_u8x16, 0x80, 0x09)); @@ -447,7 +447,7 @@ SZ_HELPER_NOINLINE long sz_utf8_uncased_alarm_vietnamese_strip_v128_(sz_u8_t con for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t danger_u8x16 = wasm_v128_and(wasm_v128_and(sz_utf8_uncased_eq01_v128_(source_u8x16, 0xBA), sz_utf8_uncased_eq01_v128_(previous_u8x16, 0xE1)), sz_utf8_uncased_inrange01_v128_(next_u8x16, 0x96, 0x0A)); @@ -480,7 +480,7 @@ SZ_HELPER_NOINLINE long sz_utf8_uncased_alarm_georgian_strip_v128_(sz_u8_t const for (sz_size_t pos = 0; pos < vector_length; pos += 16) { sz_size_t window = vector_length - pos < 16 ? vector_length - pos : 16; sz_utf8_uncased_window_v128_t chunk = sz_utf8_uncased_load_window_v128_(src, pos); - v128_t source_u8x16 = chunk.source, previous_u8x16 = chunk.previous, next_u8x16 = chunk.next; + v128_t source_u8x16 = chunk.source_u8x16, previous_u8x16 = chunk.previous_u8x16, next_u8x16 = chunk.next_u8x16; v128_t after_e1_u8x16 = sz_utf8_uncased_eq01_v128_(previous_u8x16, 0xE1); v128_t danger_u8x16 = wasm_v128_and(sz_utf8_uncased_eq01_v128_(source_u8x16, 0xB2), after_e1_u8x16); danger_u8x16 = wasm_v128_or( diff --git a/include/stringzilla/utf8_uncased_fold.h b/include/stringzilla/utf8_uncased_fold.h index 63fce6cb..58b3544e 100644 --- a/include/stringzilla/utf8_uncased_fold.h +++ b/include/stringzilla/utf8_uncased_fold.h @@ -24,7 +24,7 @@ extern "C" { * Case folding normalizes text for uncased comparisons by mapping uppercase letters * to their lowercase equivalents and handling special expansions defined in Unicode CaseFolding.txt. * - * @section Buffer Sizing + * @section utf8_uncased_fold_buffer_sizing Buffer Sizing * * The destination buffer must be at least `source_length * 3` bytes to guarantee sufficient space * for worst-case expansion. The maximum expansion ratio is 3:1 (3x), which occurs with Greek diff --git a/include/stringzilla/utf8_uncased_fold/haswell.h b/include/stringzilla/utf8_uncased_fold/haswell.h index 30044986..b4642c96 100644 --- a/include/stringzilla/utf8_uncased_fold/haswell.h +++ b/include/stringzilla/utf8_uncased_fold/haswell.h @@ -75,7 +75,7 @@ SZ_HELPER_INLINE sz_u32_t sz_haswell_mask_until_(sz_size_t n) { return (sz_u32_t * word still copies its longest caseless prefix vectorized. * @return Bytes consumed and written, or zero if the first character needs another handler. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_caseless_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_caseless_chunk_( // __m256i source_u8x32, sz_u32_t is_two_byte_lead_mask, sz_u32_t is_three_byte_lead_mask, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { @@ -119,7 +119,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_caseless_chunk_( // * vectorized instead of degrading to one-rune serial steps per chunk. * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_latin_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_latin_chunk_( // __m256i source_u8x32, sz_u32_t is_continuation_mask, sz_u32_t is_three_byte_lead_mask, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { @@ -297,7 +297,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_latin_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_cyrillic_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_cyrillic_chunk_( // __m256i source_u8x32, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { __m256i previous_bytes_u8x32 = sz_haswell_previous_bytes_(source_u8x32, 1); @@ -367,7 +367,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_cyrillic_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_greek_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_greek_chunk_( // __m256i source_u8x32, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { __m256i previous_bytes_u8x32 = sz_haswell_previous_bytes_(source_u8x32, 1); @@ -454,7 +454,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_greek_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs another handler. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_georgian_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_georgian_chunk_( // __m256i source_u8x32, sz_u32_t is_three_byte_lead_mask, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { __m256i previous_bytes_u8x32 = sz_haswell_previous_bytes_(source_u8x32, 1); @@ -531,7 +531,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_georgian_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_guarded_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_guarded_chunk_( // __m256i source_u8x32, sz_u32_t is_two_byte_lead_mask, sz_u32_t is_three_byte_lead_mask, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { @@ -590,7 +590,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_guarded_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_armenian_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_armenian_chunk_( // __m256i source_u8x32, sz_u32_t is_lead_mask, sz_u32_t malformed_lead_mask, sz_ptr_t target) { __m256i previous_bytes_u8x32 = sz_haswell_previous_bytes_(source_u8x32, 1); @@ -662,7 +662,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_armenian_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_supplementary_chunk_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_supplementary_chunk_( // __m256i source_u8x32, sz_u32_t is_complex_lead_mask, sz_u32_t is_four_byte_lead_mask, sz_u32_t is_foreign_lead_mask, sz_ptr_t target) { @@ -717,7 +717,7 @@ typedef struct sz_utf8_uncased_fold_haswell_leads_t { * flag byte the Ice Lake `VPERMB` LUT produces. The caseless family merges D7-DF and E0 into * one contiguous D7-E0 span. */ -SZ_HELPER_AUTO sz_utf8_uncased_fold_haswell_leads_t sz_utf8_uncased_fold_haswell_classify_leads_( +SZ_HELPER_INLINE sz_utf8_uncased_fold_haswell_leads_t sz_utf8_uncased_fold_haswell_classify_leads_( __m256i source_u8x32, sz_u32_t is_non_ascii_mask) { sz_utf8_uncased_fold_haswell_leads_t leads; @@ -804,9 +804,8 @@ SZ_HELPER_AUTO sz_utf8_uncased_fold_haswell_leads_t sz_utf8_uncased_fold_haswell * next family. * @return Bytes consumed and written, or zero if every handler declined the chunk. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_dispatch_chunk_(__m256i source_u8x32, - sz_utf8_uncased_fold_haswell_leads_t const *leads, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_dispatch_chunk_( + __m256i source_u8x32, sz_utf8_uncased_fold_haswell_leads_t const *leads, sz_ptr_t target) { // Malformed leads (overlong, surrogate, truncated, out-of-range, C0/C1, F5..FF) are foreign to // every family: ORing them into each handler's foreign/stop mask truncates the fold before them @@ -871,8 +870,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_dispatch_chunk_(__m256i so * @param rune_length Receives the number of source bytes consumed. * @return Bytes written to @p target (Unicode case folding produces at most 3 runes). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_haswell_one_rune_(sz_cptr_t source, sz_cptr_t source_end, sz_ptr_t target, - sz_rune_length_t *rune_length) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_haswell_one_rune_(sz_cptr_t source, sz_cptr_t source_end, + sz_ptr_t target, sz_rune_length_t *rune_length) { sz_rune_t rune; sz_rune_length_t const parsed_length = sz_rune_decode(source, source_end, &rune); if (parsed_length == sz_rune_invalid_k) { diff --git a/include/stringzilla/utf8_uncased_fold/icelake.h b/include/stringzilla/utf8_uncased_fold/icelake.h index 0d69bb6e..2dc57c08 100644 --- a/include/stringzilla/utf8_uncased_fold/icelake.h +++ b/include/stringzilla/utf8_uncased_fold/icelake.h @@ -15,17 +15,18 @@ extern "C" { #if SZ_USE_ICELAKE #if defined(__clang__) && SZ_CLANG_HAS_EVEX512_ -#pragma clang attribute push( \ - __attribute__((target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,popcnt,evex512"))), \ +#pragma clang attribute push( \ + __attribute__(( \ + target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,popcnt,evex512"))), \ apply_to = function) #elif defined(__clang__) -#pragma clang attribute push( \ +#pragma clang attribute push( \ __attribute__((target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,popcnt"))), \ apply_to = function) #elif defined(__GNUC__) #pragma GCC push_options #pragma GCC target("avx", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512vbmi", "avx512vbmi2", "bmi", "bmi2", \ - "lzcnt", "popcnt") + "lzcnt", "popcnt") #endif /** @@ -89,8 +90,8 @@ SZ_HELPER_INLINE sz_u8_t sz_utf8_fold_icelake_reduce_or_u8_(__m512i flags_u8x64) * Folds ASCII A-Z in place and copies everything else, trimming incomplete trailing sequences. * @return Bytes consumed and written, or zero if the chunk starts with an incomplete sequence. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_icelake_caseless_chunk_( // - __m512i source_u8x64, __mmask64 load_m64, sz_size_t chunk_size, // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_icelake_caseless_chunk_( // + __m512i source_u8x64, __mmask64 load_m64, sz_size_t chunk_size, // __mmask64 is_two_byte_lead_m64, __mmask64 is_three_byte_lead_m64, __mmask64 malformed_lead_m64, sz_ptr_t target) { __m512i const a_upper_u8x64 = _mm512_set1_epi8('A'); @@ -125,8 +126,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_icelake_caseless_chunk_( // * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_icelake_latin_chunk_( // - __m512i source_u8x64, __mmask64 load_m64, sz_size_t chunk_size, // +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_icelake_latin_chunk_( // + __m512i source_u8x64, __mmask64 load_m64, sz_size_t chunk_size, // __mmask64 is_continuation_m64, __mmask64 is_three_byte_lead_m64, __mmask64 malformed_lead_m64, sz_ptr_t target) { __m512i const a_upper_u8x64 = _mm512_set1_epi8('A'); diff --git a/include/stringzilla/utf8_uncased_fold/neon.h b/include/stringzilla/utf8_uncased_fold/neon.h index 1101f3d3..f0d481c2 100644 --- a/include/stringzilla/utf8_uncased_fold/neon.h +++ b/include/stringzilla/utf8_uncased_fold/neon.h @@ -22,7 +22,7 @@ extern "C" { #endif /** @brief Folds ASCII A-Z down to a-z in one register, leaving every other byte unchanged. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_fold_neon_ascii_(uint8x16_t source_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_fold_neon_ascii_(uint8x16_t source_u8x16) { // Unsigned wrap-around turns the two-sided 'A' ≤ x ≤ 'Z' test into one compare: bytes below // 'A' wrap past 0xE5 and bytes above 'Z' land at 26+, so only A-Z stay under 26. uint8x16_t is_ascii_upper_u8x16 = vcltq_u8(vsubq_u8(source_u8x16, vdupq_n_u8('A')), vdupq_n_u8(26)); @@ -55,7 +55,8 @@ SZ_HELPER_INLINE sz_u8_t sz_utf8_fold_neon_reduce_or_u8_(uint8x16_t flags_u8x16) * @brief Maps every lead byte in one register onto its folding-family flag; non-leads map to zero. * One `vqtbl4q_u8` covers the full 64-entry table - the NEON twin of Ice Lake's single VPERMB. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_fold_neon_classify_(uint8x16_t source_u8x16, uint8x16x4_t lead_families_lut_u8x16x4) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_fold_neon_classify_(uint8x16_t source_u8x16, + uint8x16x4_t lead_families_lut_u8x16x4) { uint8x16_t is_non_ascii_u8x16 = vcgeq_u8(source_u8x16, vdupq_n_u8(0x80)); // Continuations are 10xxxxxx, i.e. exactly the [0x80, 0xBF] range - one wrap-around compare uint8x16_t is_continuation_u8x16 = vcltq_u8(vsubq_u8(source_u8x16, vdupq_n_u8(0x80)), vdupq_n_u8(0x40)); @@ -81,7 +82,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_fold_neon_classify_(uint8x16_t source_u8x16, u * * @return Per-byte mask (0xFF) set on every lead byte that does NOT begin a well-formed rune. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_fold_neon_malformed_lead_(uint8x16_t source_u8x16, uint8x16_t next_register_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_fold_neon_malformed_lead_(uint8x16_t source_u8x16, uint8x16_t next_register_u8x16) { uint8x16_t const continuation_low_u8x16 = vdupq_n_u8(0x80); uint8x16_t const continuation_span_u8x16 = vdupq_n_u8(0x40); @@ -142,7 +143,7 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_fold_neon_malformed_lead_(uint8x16_t source_u8 * @return Bytes consumed; always 62..64, never zero - 62 bytes of any valid UTF-8 cover at * least one complete sequence, so the superchunk cannot start with an incomplete one. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_caseless_chunk_(uint8x16x4_t source_u8x16x4, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_caseless_chunk_(uint8x16x4_t source_u8x16x4, sz_ptr_t target) { uint8x16_t last_u8x16 = source_u8x16x4.val[3]; uint8x16_t is_two_byte_lead_u8x16 = vcltq_u8(vsubq_u8(last_u8x16, vdupq_n_u8(0xC0)), vdupq_n_u8(0x20)); @@ -188,8 +189,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_caseless_chunk_(uint8x16x4_t * * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_latin_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_latin_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { uint8x16x4_t const c4_deltas_lut_u8x16x4 = vld1q_u8_x4(sz_utf8_fold_c4_deltas_lut_); uint8x16x4_t const c5_deltas_lut_u8x16x4 = vld1q_u8_x4(sz_utf8_fold_c5_deltas_lut_); @@ -361,8 +362,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_latin_chunk_(uint8x16x4_t sou * * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_cyrillic_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_cyrillic_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { static sz_u8_t const second_byte_offsets_lut_[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x20, 0xE0, 0, 0, 0, 0, 0}; uint8x16_t const offsets_lut_u8x16 = vld1q_u8(second_byte_offsets_lut_); uint8x16_t const zero_u8x16 = vdupq_n_u8(0x00); @@ -438,8 +439,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_cyrillic_chunk_(uint8x16x4_t * * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_greek_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_greek_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { uint8x16_t const zero_u8x16 = vdupq_n_u8(0x00); uint8x16_t previous_is_ce_u8x16 = zero_u8x16, previous_is_cf_u8x16 = zero_u8x16; uint8x16_t stop_masks_u8x16[4]; @@ -534,8 +535,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_greek_chunk_(uint8x16x4_t sou * * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_armenian_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_armenian_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { uint8x16_t const zero_u8x16 = vdupq_n_u8(0x00); uint8x16_t previous_is_d4_u8x16 = zero_u8x16, previous_is_d5_u8x16 = zero_u8x16; uint8x16_t stop_masks_u8x16[4]; @@ -633,8 +634,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_armenian_chunk_(uint8x16x4_t * * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_georgian_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_georgian_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { uint8x16_t const zero_u8x16 = vdupq_n_u8(0x00); uint8x16_t previous_is_82_upper_lead_u8x16 = zero_u8x16, previous_is_83_upper_lead_u8x16 = zero_u8x16; uint8x16_t previous_is_82_upper_second_u8x16 = zero_u8x16, previous_is_83_upper_second_u8x16 = zero_u8x16; @@ -729,8 +730,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_georgian_chunk_(uint8x16x4_t * * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_neon_guarded_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, - sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_neon_guarded_chunk_(uint8x16x4_t source_u8x16x4, sz_cptr_t source, + sz_ptr_t target) { uint8x16_t const zero_u8x16 = vdupq_n_u8(0x00); uint8x16_t stop_masks_u8x16[4]; uint8x16_t any_stop_u8x16 = zero_u8x16; diff --git a/include/stringzilla/utf8_uncased_fold/rvv.h b/include/stringzilla/utf8_uncased_fold/rvv.h index e23d977c..a7be0695 100644 --- a/include/stringzilla/utf8_uncased_fold/rvv.h +++ b/include/stringzilla/utf8_uncased_fold/rvv.h @@ -63,8 +63,8 @@ SZ_HELPER_INLINE vuint8m8_t sz_utf8_fold_ascii_rvv_(vuint8m8_t source_u8m8, sz_s /* Largest strip length that does not split a trailing multi-byte sequence across strips. On the final strip * (`vector_length == remaining`) the whole input ends here, so nothing is trimmed; otherwise a last codepoint whose * declared length runs past `vector_length` is excluded and reprocessed in the next strip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_trim_incomplete_(sz_u8_t const *source_ptr, sz_size_t vector_length, - sz_size_t remaining) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_trim_incomplete_(sz_u8_t const *source_ptr, sz_size_t vector_length, + sz_size_t remaining) { if (vector_length >= remaining) return vector_length; sz_size_t boundary = vector_length; while (boundary && (source_ptr[boundary - 1] & 0xC0) == 0x80) --boundary; // back up to the last lead @@ -86,8 +86,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_trim_incomplete_(sz_u8_t const *source_ptr * indices (which is what pinned the gather-based form to `e8m4`). Sets `*needs_serial` when it stopped on a * non-handled codepoint (vs. merely trimming a trailing incomplete sequence). Returns the number of bytes * folded and written. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_latin_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_latin_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = __riscv_vsetvl_e8m8(remaining); vuint8m8_t source_u8m8 = __riscv_vle8_v_u8m8(source_ptr, vector_length); vuint8m8_t previous_u8m8 = __riscv_vslide1up_vx_u8m8(source_u8m8, 0, vector_length); @@ -190,8 +190,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_latin_strip_rvv_(sz_u8_t const *source_ptr * Runs at `e8m8`: the offset table is read from memory, so it no longer has to share a register group with * the indices the way a `vrgather` would, lifting the old `e8m4` ceiling. Same stop-and-serial contract as * `sz_utf8_fold_latin_strip_rvv_`. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_cyrillic_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_cyrillic_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { static sz_u8_t const second_byte_offsets[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x20, 0xE0, 0, 0, 0, 0, 0}; sz_size_t vector_length = __riscv_vsetvl_e8m8(remaining); vuint8m8_t source_u8m8 = __riscv_vle8_v_u8m8(source_ptr, vector_length); @@ -257,8 +257,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_cyrillic_strip_rvv_(sz_u8_t const *source_ * and 'ÎŖ'-'ÎĢ' fold `-0x20` with a CE->CF lead promotion (+1); final sigma 'Ī‚' (CF 82) folds to 'Īƒ' (+1). * Accented uppercase (CE 84-90), the expanding 'ΰ' (CE B0), the CF 8F+ symbols, and any non-CE/CF lead are * stops routed to serial. Same `e8m8` strip / stop-and-serial contract as the other handlers. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_greek_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_greek_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = __riscv_vsetvl_e8m8(remaining); vuint8m8_t source_u8m8 = __riscv_vle8_v_u8m8(source_ptr, vector_length); sz_u8_t next_carry = (vector_length < remaining) ? source_ptr[vector_length] : 0; @@ -341,8 +341,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_greek_strip_rvv_(sz_u8_t const *source_ptr * D5 90-96 fold `-0x10`, D5 80-8F folds `+0x30` — plus the lead `+1` rewrites D4->D5 (next B1-BF) and * D5->D6 (next 90-96). The 'և' ligature (D6 87, expands), the D4 Cyrillic-Supplement range (next < B1), and * any non-D4/D5/D6 lead are stops routed to serial. Same `e8m8` strip / stop-and-serial contract. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_armenian_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_armenian_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = __riscv_vsetvl_e8m8(remaining); vuint8m8_t source_u8m8 = __riscv_vle8_v_u8m8(source_ptr, vector_length); sz_u8_t next_carry = (vector_length < remaining) ? source_ptr[vector_length] : 0; @@ -422,8 +422,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_armenian_strip_rvv_(sz_u8_t const *source_ * uppercase flag is carried one lane to the second-byte rewrite and two lanes to the third-byte offset: * lead E1->E2, second 82/83->B4, third -0x20 (E1 82) or +0x20 (E1 83). Non-Georgian E1 second bytes and * any non-E1 lead are stops routed to serial. Same `e8m8` strip / stop-and-serial contract. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_georgian_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_georgian_strip_rvv_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = __riscv_vsetvl_e8m8(remaining); vuint8m8_t source_u8m8 = __riscv_vle8_v_u8m8(source_ptr, vector_length); sz_u8_t carry1 = (vector_length < remaining) ? source_ptr[vector_length] : 0; diff --git a/include/stringzilla/utf8_uncased_fold/serial.h b/include/stringzilla/utf8_uncased_fold/serial.h index a14bda26..ad496d41 100644 --- a/include/stringzilla/utf8_uncased_fold/serial.h +++ b/include/stringzilla/utf8_uncased_fold/serial.h @@ -19,6 +19,38 @@ extern "C" { */ SZ_HELPER_AUTO sz_u8_t sz_ascii_fold_(sz_u8_t c) { return c + (((sz_u8_t)(c - 'A') <= 25u) * 0x20); } +enum { + /** @brief Most source bytes one folded byte can come from - the Kelvin sign's three, folding to `k`. */ + sz_utf8_fold_max_contraction_k = 3, + /** @brief Most folded bytes one source byte can produce; the mirror bound. */ + sz_utf8_fold_max_expansion_k = 3, +}; + +/** + * @brief Whether any codepoint beginning with @p lead folds to something other than itself. + * + * 49 of the 256 lead bytes qualify, so the answer is a bit in a 256-bit mask carried as four immediates - + * no table, and therefore no memory access on either a CPU or a GPU. Generated from + * `sz_unicode_fold_codepoint_`, so it cannot disagree with the fold it guards: + * + * @code{.py} + * for codepoint in range(0x110000): + * if 0xD800 <= codepoint <= 0xDFFF: continue + * if fold(codepoint) != [codepoint]: + * lead = encode(codepoint)[0] + * mask[lead >> 6] |= 1 << (lead & 63) + * @endcode + */ +SZ_HELPER_AUTO sz_bool_t sz_utf8_lead_may_fold_(sz_u8_t lead) { + sz_u64_t word; + switch (lead >> 6) { + case 1: word = 0x0000000007FFFFFEull; break; // 0x40-0x7F: 'A'-'Z' + case 3: word = 0x00018406007FE3FCull; break; // 0xC0-0xFF: C2-D6 Latin through Cyrillic, E1 E2 EA EF, F0 + default: return sz_false_k; // 0x00-0x3F punctuation, 0x80-0xBF continuation bytes + } + return (sz_bool_t)((word >> (lead & 63)) & 1u); +} + /** * @brief Folded-rune representation of a byte that does not begin a well-formed codepoint. * @@ -27,7 +59,7 @@ SZ_HELPER_AUTO sz_u8_t sz_ascii_fold_(sz_u8_t c) { return c + (((sz_u8_t)(c - 'A * lone malformed byte 0xFC can only match another malformed 0xFC - never the valid rune U+00FC ('Ãŧ'). Two * equal malformed bytes still produce equal tagged runes, preserving byte-for-byte matching. */ -SZ_HELPER_INLINE sz_rune_t sz_rune_malformed_byte_(sz_u8_t byte) { return 0x80000000u | (sz_rune_t)byte; } +SZ_HELPER_AUTO sz_rune_t sz_rune_malformed_byte_(sz_u8_t byte) { return 0x80000000u | (sz_rune_t)byte; } /** * Bit flags describing which UTF-8 lead-byte families occur in a chunk, shared by every back-end. @@ -1385,7 +1417,7 @@ SZ_HELPER_AUTO sz_size_t sz_unicode_fold_codepoint_(sz_rune_t rune, sz_rune_t *f * @param bytes_consumed Number of bytes read from source. * @param bytes_exported Number of bytes written to destination. */ -SZ_HELPER_AUTO void sz_utf8_uncased_fold_upto_( // +SZ_HELPER_INLINE void sz_utf8_uncased_fold_upto_( // sz_cptr_t source, sz_size_t source_length, // sz_ptr_t destination, sz_size_t destination_length, // sz_size_t *codepoints_consumed, sz_size_t *codepoints_exported, // @@ -1480,6 +1512,182 @@ SZ_API_COMPTIME sz_size_t sz_utf8_uncased_fold_serial(sz_cptr_t source, sz_size_ return (sz_size_t)(destination_ptr - (sz_u8_t *)destination); } +#pragma region Folded Iterators + +/** + * @brief Iterator state for streaming through folded UTF-8 runes. + * Handles one-to-many case folding expansions (e.g., 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)) transparently. + */ +typedef struct { + sz_cptr_t ptr; // Current position in UTF-8 string + sz_cptr_t end; // End of string + sz_rune_t pending[4]; // Buffered folded runes from one-to-many expansions + sz_size_t pending_count; // Number of pending folded runes + sz_size_t pending_idx; // Current index into pending buffer + sz_cptr_t codepoint_begin; // Source codepoint that produced the pending runes; `ptr` has already left it + sz_size_t codepoint_length; // Its length in bytes, so its span is `[begin, begin + length)` +} sz_utf8_folded_iter_t; + +/** @brief Initialize a folded rune iterator. */ +SZ_HELPER_AUTO void sz_utf8_folded_iter_init_(sz_utf8_folded_iter_t *iterator, sz_cptr_t string, sz_size_t length) { + iterator->ptr = string; + iterator->end = string + length; + iterator->pending_count = 0; + iterator->pending_idx = 0; + iterator->codepoint_begin = string; + iterator->codepoint_length = 0; +} + +/** + * @brief Get next folded rune. Returns `sz_false_k` when exhausted. + * Malformed UTF-8 is handled losslessly: a byte that does not begin a well-formed codepoint is emitted as a + * single literal byte (tagged so it compares byte-for-byte and never collides with a real folded codepoint) and + * the iterator resyncs by one byte, never reading past `end`. + * `codepoint_begin` and `codepoint_length` name the source span every rune of one codepoint comes from, and + * stay put while the expansion drains - `ptr` cannot serve, having already moved past the codepoint. + */ +SZ_HELPER_AUTO sz_bool_t sz_utf8_folded_iter_next_(sz_utf8_folded_iter_t *it, sz_rune_t *out_rune) { + // Refill pending buffer if exhausted + if (it->pending_idx >= it->pending_count) { + if (it->ptr >= it->end) return sz_false_k; + + // ASCII fast-path: fold inline without buffering + sz_u8_t lead = *(sz_u8_t const *)it->ptr; + if (lead < 0x80) { + *out_rune = sz_ascii_fold_(lead); + it->codepoint_begin = it->ptr; + it->codepoint_length = 1; + it->ptr++; + it->pending_count = 0; // Clear pending buffer + it->pending_idx = 0; // Signal first rune of new codepoint for source tracking + return sz_true_k; + } + + // Multi-byte UTF-8: decode (bounds-checked), fold, and buffer. A byte that does not begin a + // well-formed codepoint folds to itself (>= 0x80 bytes are unchanged by `sz_ascii_fold_`) and resyncs + // by one byte, never over-reading past `end`. + sz_rune_t rune; + sz_rune_length_t const rune_length = sz_rune_decode(it->ptr, it->end, &rune); + if (rune_length == sz_rune_invalid_k) { + *out_rune = sz_rune_malformed_byte_(lead); + it->codepoint_begin = it->ptr; + it->codepoint_length = 1; + it->ptr++; + it->pending_count = 0; + it->pending_idx = 0; + return sz_true_k; + } + + it->codepoint_begin = it->ptr; + it->codepoint_length = (sz_size_t)rune_length; + it->ptr += rune_length; + // Pre-fill pending buffer with sentinel values to prevent stale data from causing false matches. + // The fold function will overwrite positions it uses; unused positions keep the sentinel. + // This follows the same pattern as sz_utf8_uncased_search_2folded_serial_ and + // sz_utf8_uncased_search_3folded_serial_. + it->pending[0] = 0xFFFFFFFFu; + it->pending[1] = 0xFFFFFFFEu; + it->pending[2] = 0xFFFFFFFDu; + it->pending[3] = 0xFFFFFFFCu; + it->pending_count = sz_unicode_fold_codepoint_(rune, it->pending); + it->pending_idx = 0; + } + + *out_rune = it->pending[it->pending_idx++]; + return sz_true_k; +} + +/** + * @brief Reverse iterator state for streaming through folded UTF-8 runes backwards. + * Handles one-to-many case folding expansions (e.g., 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)) transparently + * in reverse order. + */ +typedef struct { + sz_cptr_t ptr; // Current position (points to byte AFTER current sequence) + sz_cptr_t start; // Start of string (stop when ptr reaches this) + sz_rune_t pending[4]; // Buffered folded runes from one-to-many expansions (in reverse order) + sz_size_t pending_count; // Number of pending folded runes + sz_size_t pending_idx; // Current index into pending buffer +} sz_utf8_folded_reverse_iter_t; + +/** @brief Initialize a reverse folded rune iterator. Iterates from end towards start. */ +SZ_HELPER_AUTO void sz_utf8_folded_reverse_iter_init_(sz_utf8_folded_reverse_iter_t *it, sz_cptr_t start, + sz_cptr_t end) { + it->ptr = end; + it->start = start; + it->pending_count = 0; + it->pending_idx = 0; +} + +/** + * @brief Get previous folded rune (walking backwards). Returns `sz_false_k` when exhausted. + * When a codepoint folds to multiple runes (like 'ß' (U+00DF, C3 9F) → "ss" (U+0073 U+0073, 73 73)), returns them in + * reverse order ('s', then 's'). Malformed UTF-8 is handled losslessly and byte-identically to the forward + * iterator: a byte that does not begin/end a well-formed codepoint is emitted as a single tagged literal byte and + * the iterator resyncs by one byte, so the backward rune stream is exactly the reverse of the forward stream. + */ +SZ_HELPER_AUTO sz_bool_t sz_utf8_folded_reverse_iter_prev_(sz_utf8_folded_reverse_iter_t *it, sz_rune_t *out_rune) { + // Return pending runes if any (stored in reverse order, consumed in reverse) + if (it->pending_idx < it->pending_count) { + *out_rune = it->pending[it->pending_count - 1 - it->pending_idx]; + it->pending_idx++; + return sz_true_k; + } + + // Refill: find previous codepoint + if (it->ptr <= it->start) return sz_false_k; + + // Remember one-past-the-end of the sequence we are about to decode, so the strict decode is bounded + // and a malformed run resyncs one byte at a time - mirroring the forward iterator byte-for-byte. + sz_cptr_t const sequence_end = it->ptr; + + // The byte immediately before `sequence_end` is the last byte of whatever codepoint ends here. + sz_u8_t const last_byte = *(sz_u8_t const *)(sequence_end - 1); + + // ASCII fast-path: a byte < 0x80 is always its own complete 1-byte codepoint. + if (last_byte < 0x80) { + it->ptr = sequence_end - 1; + *out_rune = sz_ascii_fold_(last_byte); + it->pending_count = 0; + it->pending_idx = 0; + return sz_true_k; + } + + // Otherwise walk backwards over up to 3 continuation bytes (0x80-0xBF) to locate a candidate lead. + // A well-formed multi-byte rune is at most 4 bytes, so stop after considering 4 positions. + sz_cptr_t candidate = sequence_end - 1; + for (sz_size_t back = 0; back < 3 && candidate > it->start && (*(sz_u8_t const *)candidate & 0xC0) == 0x80; ++back) + candidate--; + + // Multi-byte UTF-8: decode (bounded) and fold only if the bytes from the candidate lead form a well-formed + // codepoint that ends EXACTLY at `sequence_end`. Otherwise the last byte does not begin/end a valid rune, so + // treat it as a literal folded-to-itself byte and resync by one - matching the forward iterator byte-for-byte. + sz_rune_t rune; + sz_rune_length_t const rune_length = sz_rune_decode(candidate, sequence_end, &rune); + if (rune_length == sz_rune_invalid_k || candidate + rune_length != sequence_end) { + it->ptr = sequence_end - 1; + *out_rune = sz_rune_malformed_byte_(last_byte); + it->pending_count = 0; + it->pending_idx = 0; + return sz_true_k; + } + it->ptr = candidate; + + // Store folded runes in pending buffer + it->pending[0] = 0xFFFFFFFFu; + it->pending[1] = 0xFFFFFFFEu; + it->pending[2] = 0xFFFFFFFDu; + it->pending[3] = 0xFFFFFFFCu; + it->pending_count = sz_unicode_fold_codepoint_(rune, it->pending); + it->pending_idx = 1; // We'll return the last one now, then the rest in subsequent calls + + // Return the LAST folded rune first (since we're going backwards) + *out_rune = it->pending[it->pending_count - 1]; + return sz_true_k; +} + +#pragma endregion Folded Iterators + #ifdef __cplusplus } #endif diff --git a/include/stringzilla/utf8_uncased_fold/sve2.h b/include/stringzilla/utf8_uncased_fold/sve2.h index c4cc19f7..aaae6519 100644 --- a/include/stringzilla/utf8_uncased_fold/sve2.h +++ b/include/stringzilla/utf8_uncased_fold/sve2.h @@ -38,7 +38,7 @@ SZ_HELPER_INLINE svuint8_t sz_utf8_fold_sve2_ascii_(svuint8_t source_u8x) { /** @brief Per-lead well-formedness mirror of `sz_rune_decode` - the SVE2 twin of * @ref sz_utf8_fold_neon_malformed_lead_ over one (chunk, peek) pair. * @return Predicate set on every lead byte that does NOT begin a well-formed rune. */ -SZ_HELPER_AUTO svbool_t sz_utf8_fold_sve2_malformed_lead_(svuint8_t source_u8x, svuint8_t peek_u8x) { +SZ_HELPER_INLINE svbool_t sz_utf8_fold_sve2_malformed_lead_(svuint8_t source_u8x, svuint8_t peek_u8x) { svbool_t const all_b8x = svptrue_b8(); svuint8_t const next1_u8x = svext_u8(source_u8x, peek_u8x, 1); svuint8_t const next2_u8x = svext_u8(source_u8x, peek_u8x, 2); @@ -81,7 +81,7 @@ SZ_HELPER_AUTO svbool_t sz_utf8_fold_sve2_malformed_lead_(svuint8_t source_u8x, /** @brief Folds a 64-byte superchunk containing only caseless multi-byte scripts mixed with ASCII - the SVE2 * twin of @ref sz_utf8_uncased_fold_neon_caseless_chunk_. * @return Bytes consumed; always 62..64, never zero. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_caseless_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_caseless_chunk_(sz_cptr_t source, sz_ptr_t target) { sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; for (sz_size_t chunk_base = 0; chunk_base < 64; chunk_base += chunk_bytes) { svbool_t const loaded_b8x = svwhilelt_b8_u64((sz_u64_t)chunk_base, 64); @@ -99,7 +99,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_caseless_chunk_(sz_cptr_t sou * @ref sz_utf8_uncased_fold_neon_latin_chunk_, with the same C4/C5/C6 delta tables read through * chunked `svtbl` walks and the same stop policy. * @return Bytes consumed and written, or zero if the first character needs the serial path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_latin_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_latin_chunk_(sz_cptr_t source, sz_ptr_t target) { svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; svuint8_t const lane_iota_u8x = svindex_u8(0, 1); @@ -240,7 +240,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_latin_chunk_(sz_cptr_t source /** @brief Folds a 64-byte superchunk of basic Cyrillic mixed with ASCII - the SVE2 twin of * @ref sz_utf8_uncased_fold_neon_cyrillic_chunk_. * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_cyrillic_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_cyrillic_chunk_(sz_cptr_t source, sz_ptr_t target) { static sz_u8_t const second_byte_offsets_lut_[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x20, 0xE0, 0, 0, 0, 0, 0}; svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; @@ -301,7 +301,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_cyrillic_chunk_(sz_cptr_t sou /** @brief Folds a 64-byte superchunk of basic Greek mixed with ASCII - the SVE2 twin of * @ref sz_utf8_uncased_fold_neon_greek_chunk_. * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_greek_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_greek_chunk_(sz_cptr_t source, sz_ptr_t target) { svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; svuint8_t const lane_iota_u8x = svindex_u8(0, 1); @@ -377,7 +377,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_greek_chunk_(sz_cptr_t source /** @brief Folds a 64-byte superchunk of Armenian (D4-D6 leads) mixed with ASCII - the SVE2 twin of * @ref sz_utf8_uncased_fold_neon_armenian_chunk_. * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_armenian_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_armenian_chunk_(sz_cptr_t source, sz_ptr_t target) { svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; svuint8_t const lane_iota_u8x = svindex_u8(0, 1); @@ -450,7 +450,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_armenian_chunk_(sz_cptr_t sou /** @brief Folds a 64-byte superchunk of Georgian (E1 82/83 content) mixed with ASCII - the SVE2 twin of * @ref sz_utf8_uncased_fold_neon_georgian_chunk_. * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_georgian_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_georgian_chunk_(sz_cptr_t source, sz_ptr_t target) { svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; svuint8_t const lane_iota_u8x = svindex_u8(0, 1); @@ -530,7 +530,7 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_georgian_chunk_(sz_cptr_t sou /** @brief Folds a 64-byte superchunk of caseless scripts mixed with SAFE guarded punctuation - the SVE2 twin * of @ref sz_utf8_uncased_fold_neon_guarded_chunk_. * @return Bytes consumed and written, or zero if the first character needs another path. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_uncased_fold_sve2_guarded_chunk_(sz_cptr_t source, sz_ptr_t target) { +SZ_HELPER_INLINE sz_size_t sz_utf8_uncased_fold_sve2_guarded_chunk_(sz_cptr_t source, sz_ptr_t target) { svbool_t const all_b8x = svptrue_b8(); sz_size_t const chunk_bytes = svcntb() < 64 ? svcntb() : 64; sz_u64_t stop_lanes = 0; diff --git a/include/stringzilla/utf8_uncased_fold/v128.h b/include/stringzilla/utf8_uncased_fold/v128.h index e446046e..badbc8ba 100644 --- a/include/stringzilla/utf8_uncased_fold/v128.h +++ b/include/stringzilla/utf8_uncased_fold/v128.h @@ -76,8 +76,8 @@ SZ_HELPER_INLINE v128_t sz_utf8_masked_add_v128_(v128_t bytes_u8x16, v128_t mask } /** @brief 64-entry table lookup via four 16-entry swizzles selected by the index's high two bits. */ -SZ_HELPER_AUTO v128_t sz_utf8_gather64_v128_(v128_t lut0_u8x16, v128_t lut1_u8x16, v128_t lut2_u8x16, v128_t lut3_u8x16, - v128_t index_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_gather64_v128_(v128_t lut0_u8x16, v128_t lut1_u8x16, v128_t lut2_u8x16, + v128_t lut3_u8x16, v128_t index_u8x16) { v128_t local_u8x16 = wasm_v128_and(index_u8x16, wasm_i8x16_splat(0x0F)); v128_t sub_u8x16 = wasm_u8x16_shr(index_u8x16, 4); // index in [0, 63] -> sub in [0, 3] v128_t result_u8x16 = wasm_i8x16_swizzle(lut0_u8x16, local_u8x16); @@ -112,8 +112,8 @@ SZ_HELPER_INLINE v128_t sz_utf8_load_window_v128_(sz_u8_t const *source_ptr, sz_ * malformed - coinciding with the incomplete-sequence trim, leaving valid output unchanged. The family * handlers OR this into their stop mask so overlong, surrogate, truncated, and out-of-range leads are * treated as foreign and resync one byte at a time, byte-for-byte with the serial reference. */ -SZ_HELPER_AUTO v128_t sz_utf8_malformed_lead_v128_(v128_t source_u8x16, v128_t next_u8x16, v128_t is_continuation_u8x16, - v128_t is_lead_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_malformed_lead_v128_(v128_t source_u8x16, v128_t next_u8x16, + v128_t is_continuation_u8x16, v128_t is_lead_u8x16) { v128_t continuation_plus1_u8x16 = sz_utf8_slide1down_v128_(is_continuation_u8x16, 0); v128_t continuation_plus2_u8x16 = sz_utf8_slide1down_v128_(continuation_plus1_u8x16, 0); v128_t continuation_plus3_u8x16 = sz_utf8_slide1down_v128_(continuation_plus2_u8x16, 0); @@ -147,8 +147,8 @@ SZ_HELPER_AUTO v128_t sz_utf8_malformed_lead_v128_(v128_t source_u8x16, v128_t n } /** @brief Largest prefix of a window that does not split a trailing multi-byte sequence (twin of RVV trim). */ -SZ_HELPER_AUTO sz_size_t sz_utf8_trim_incomplete_v128_(sz_u8_t const *source_ptr, sz_size_t vector_length, - sz_size_t remaining) { +SZ_HELPER_INLINE sz_size_t sz_utf8_trim_incomplete_v128_(sz_u8_t const *source_ptr, sz_size_t vector_length, + sz_size_t remaining) { if (vector_length >= remaining) return vector_length; sz_size_t boundary = vector_length; while (boundary && (source_ptr[boundary - 1] & 0xC0) == 0x80) --boundary; // back up to the last lead @@ -159,9 +159,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_trim_incomplete_v128_(sz_u8_t const *source_ptr } /** @brief Common tail of every strip handler: resolve `consumed` from the first stop, store, set the flag. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_strip_finish_v128_(sz_u8_t const *source_ptr, sz_size_t vector_length, - sz_size_t remaining, v128_t folded_u8x16, int first_stop, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_strip_finish_v128_(sz_u8_t const *source_ptr, sz_size_t vector_length, + sz_size_t remaining, v128_t folded_u8x16, int first_stop, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t consumed; if (first_stop >= 0) { consumed = (sz_size_t)first_stop; @@ -181,8 +181,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_strip_finish_v128_(sz_u8_t const *source_ptr, s #pragma region Per script strip handlers /** @brief Fold one window of Latin (ASCII + Latin-1 C2/C3 + Latin Extended-A/B C4-C6). @sa RVV latin strip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_latin_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_latin_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = remaining < 16 ? remaining : 16; v128_t source_u8x16 = sz_utf8_load_window_v128_(source_ptr, remaining); v128_t previous_u8x16 = sz_utf8_slide1up_v128_(source_u8x16); @@ -258,8 +258,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_latin_strip_v128_(sz_u8_t const *source_pt } /** @brief Fold one window of basic Cyrillic (D0/D1 leads). @sa RVV cyrillic strip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_cyrillic_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_cyrillic_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { static sz_align_(16) sz_u8_t const second_byte_offsets[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x20, 0xE0, 0, 0, 0, 0, 0}; sz_size_t vector_length = remaining < 16 ? remaining : 16; @@ -300,8 +300,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_cyrillic_strip_v128_(sz_u8_t const *source } /** @brief Fold one window of basic Greek (CE/CF leads). @sa RVV greek strip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_greek_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_greek_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = remaining < 16 ? remaining : 16; v128_t source_u8x16 = sz_utf8_load_window_v128_(source_ptr, remaining); v128_t next_u8x16 = sz_utf8_slide1down_v128_(source_u8x16, @@ -351,8 +351,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_greek_strip_v128_(sz_u8_t const *source_pt } /** @brief Fold one window of Armenian (D4/D5/D6 leads). @sa RVV armenian strip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_armenian_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_armenian_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = remaining < 16 ? remaining : 16; v128_t source_u8x16 = sz_utf8_load_window_v128_(source_ptr, remaining); v128_t next_u8x16 = sz_utf8_slide1down_v128_(source_u8x16, @@ -397,8 +397,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_fold_armenian_strip_v128_(sz_u8_t const *source } /** @brief Fold one window of Georgian (3-byte E1 82/83 sequences, uppercase keyed by the third byte). @sa RVV. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_fold_georgian_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, - sz_u8_t *destination_ptr, int *needs_serial) { +SZ_HELPER_INLINE sz_size_t sz_utf8_fold_georgian_strip_v128_(sz_u8_t const *source_ptr, sz_size_t remaining, + sz_u8_t *destination_ptr, int *needs_serial) { sz_size_t vector_length = remaining < 16 ? remaining : 16; v128_t source_u8x16 = sz_utf8_load_window_v128_(source_ptr, remaining); v128_t next_u8x16 = sz_utf8_slide1down_v128_(source_u8x16, diff --git a/include/stringzilla/utf8_wordbreaks.h b/include/stringzilla/utf8_wordbreaks.h index aa1a2c37..74106a46 100644 --- a/include/stringzilla/utf8_wordbreaks.h +++ b/include/stringzilla/utf8_wordbreaks.h @@ -1,6 +1,6 @@ /** * @brief Hardware-accelerated UAX-29 word boundary segmentation. - * @file utf8_wordbreaks.h + * @file include/stringzilla/utf8_wordbreaks.h * @author Ash Vardanian */ #ifndef STRINGZILLA_UTF8_WORDBREAKS_H_ diff --git a/include/stringzilla/utf8_wordbreaks/README.md b/include/stringzilla/utf8_wordbreaks/README.md index 4c96784c..914e46d5 100644 --- a/include/stringzilla/utf8_wordbreaks/README.md +++ b/include/stringzilla/utf8_wordbreaks/README.md @@ -5,7 +5,7 @@ Each operation has a serial baseline plus `haswell` and `icelake` SIMD backends ## Methodology -Numbers are throughput in MB/s, measured with `bench/utf8_iterate.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. +Numbers are throughput in MB/s, measured with `bench/utf8_segment.cpp` over the full multilingual `xlsum.csv` corpus, reporting the median of repeated runs. Each table fixes one input shape; its single column is the `sz_utf8_wordbreaks` operation and its rows are a backend on a chip, so reading down the column compares the backend ladder on one fixed input shape. Results are split into a Short Words workload (whitespace-delimited tokens averaging a few bytes) and a Long Lines workload (full text lines) to expose how each kernel scales with token length. A `↑` cell means there is no dedicated kernel at that backend, so the dispatcher reuses the tier above it. diff --git a/include/stringzilla/utf8_wordbreaks/haswell.h b/include/stringzilla/utf8_wordbreaks/haswell.h index 44d8d15d..a76b927c 100644 --- a/include/stringzilla/utf8_wordbreaks/haswell.h +++ b/include/stringzilla/utf8_wordbreaks/haswell.h @@ -47,7 +47,7 @@ extern "C" { /** @brief Word_Break class byte for thirty-two BMP codepoints (per-lane high = cp>>8, low = cp&0xFF): the `bmp_page_lut_` * page LUT selects one of the 52 distinct 256-byte pages, then `flat_bmp_` is fetched by `vpgatherdd`. * Bit-exact with `sz_rune_word_break_property` over the whole BMP. */ -SZ_HELPER_AUTO __m256i sz_utf8_word_break_bmp_class_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_word_break_bmp_class_haswell_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { return sz_utf8_rune_flat_lookup_haswell_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, high_bytes_u8x32, low_bytes_u8x32); } @@ -57,8 +57,8 @@ SZ_HELPER_AUTO __m256i sz_utf8_word_break_bmp_class_haswell_(__m256i high_bytes_ * @p plane_off_u8x32 = (offset>>16)&0xFF (low nibble meaningful), @p high_u8x32 = (offset>>8)&0xFF, * @p low_u8x32 = offset&0xFF. * Bit-exact with `sz_rune_word_break_property` over the Supplementary Planes. */ -SZ_HELPER_AUTO __m256i sz_utf8_word_break_astral_class_haswell_(__m256i plane_off_u8x32, __m256i high_u8x32, - __m256i low_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_word_break_astral_class_haswell_(__m256i plane_off_u8x32, __m256i high_u8x32, + __m256i low_u8x32) { __m256i const low_nibble_mask_u8x32 = _mm256_set1_epi8(0x0F); __m256i const n4_u8x32 = _mm256_and_si256(plane_off_u8x32, low_nibble_mask_u8x32); __m256i const n3_u8x32 = _mm256_and_si256(_mm256_srli_epi16(high_u8x32, 4), low_nibble_mask_u8x32); @@ -115,7 +115,7 @@ SZ_HELPER_INLINE __m256i sz_utf8_word_break_ascii_class_haswell_(__m256i bytes_u * bits of @p bmp_starts. Bit-identical to two full * @ref sz_utf8_word_break_bmp_class_haswell_ passes on every BMP-start lane; every other lane is a * don't-care left at its incoming value. */ -SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_haswell_( // +SZ_HELPER_INLINE void sz_utf8_word_break_bmp_compact_haswell_( // sz_u64_t bmp_starts, __m256i high_lo_u8x32, __m256i high_hi_u8x32, __m256i low_lo_u8x32, __m256i low_hi_u8x32, __m256i *out_lo_u8x32, __m256i *out_hi_u8x32) { sz_u8_t high_bytes[64], low_bytes[64]; @@ -161,9 +161,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_haswell_( // * @ref sz_utf8_word_break_classify_window_icelake_, bit-identical on every start lane. ASCII through the * property table, BMP through the nibble cascade, 4-byte leads through the astral cascade with the codepoint * high/low/plane reconstructed from the forward neighbours. */ -SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_haswell_( // +SZ_HELPER_INLINE void sz_utf8_word_break_classify_window_haswell_( // sz_utf8_rune_window_haswell_t window, __m256i *classes_lo_u8x32, __m256i *classes_hi_u8x32) { - __m256i const raw_lo_u8x32 = window.window_lo, raw_hi_u8x32 = window.window_hi; + __m256i const raw_lo_u8x32 = window.window_low_u8x32, raw_hi_u8x32 = window.window_high_u8x32; sz_u64_t const ascii_starts = window.codepoint_starts & ~window.two_byte_starts & ~window.three_byte_starts & ~window.four_byte_starts; __m256i const four_select_lo_u8x32 = sz_utf8_byte_mask_from_bits_haswell_((sz_u32_t)window.four_byte_starts); @@ -178,8 +178,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_haswell_( // __m256i out_hi_u8x32 = _mm256_setzero_si256(); sz_u64_t const bmp_starts = window.two_byte_starts | window.three_byte_starts; if (bmp_starts) - sz_utf8_word_break_bmp_compact_haswell_(bmp_starts, window.high_lo, window.high_hi, window.low_lo, - window.low_hi, &out_lo_u8x32, &out_hi_u8x32); + sz_utf8_word_break_bmp_compact_haswell_(bmp_starts, window.high_byte_low_u8x32, window.high_byte_high_u8x32, + window.low_byte_low_u8x32, window.low_byte_high_u8x32, &out_lo_u8x32, + &out_hi_u8x32); // ASCII lanes: read the 128-entry property table directly off the raw byte. out_lo_u8x32 = _mm256_blendv_epi8(out_lo_u8x32, sz_utf8_word_break_ascii_class_haswell_(raw_lo_u8x32), @@ -281,7 +282,7 @@ SZ_HELPER_INLINE __m256i sz_utf8_word_break_range16_one_haswell_(__m256i high_u8 /** @brief A 64-bit "(high,low) 16-bit value in any sorted `[lo, hi]` range" lane mask over both window halves, the * AVX2 twin of @ref sz_utf8_word_break_range16_mask_icelake_ (WSegSpace / Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_haswell_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_haswell_( // __m256i high_lo_u8x32, __m256i high_hi_u8x32, __m256i low_lo_u8x32, __m256i low_hi_u8x32, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { __m256i hit_lo_u8x32 = _mm256_setzero_si256(), hit_hi_u8x32 = _mm256_setzero_si256(); @@ -309,7 +310,7 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_haswe sz_size_t const loaded = window.loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); sz_u64_t const start_bytes = start_bytes_all & valid; - __m256i const raw_lo_u8x32 = window.window_lo, raw_hi_u8x32 = window.window_hi; + __m256i const raw_lo_u8x32 = window.window_low_u8x32, raw_hi_u8x32 = window.window_high_u8x32; // Truncated-edge U+FFFD reclassify (force the class to Other on a lead whose declared span runs past `loaded`). sz_u64_t const lead_two = length_two & start_bytes; @@ -368,8 +369,9 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_haswe sz_u64_t wseg_multibyte = 0ull; if (non_ascii_lanes) wseg_multibyte = sz_utf8_word_break_range16_mask_haswell_( - window.high_lo, window.high_hi, window.low_lo, window.low_hi, sz_utf8_word_break_wseg_lo_, - sz_utf8_word_break_wseg_hi_, sz_utf8_word_break_wseg_count_k) & + window.high_byte_low_u8x32, window.high_byte_high_u8x32, window.low_byte_low_u8x32, + window.low_byte_high_u8x32, sz_utf8_word_break_wseg_lo_, sz_utf8_word_break_wseg_hi_, + sz_utf8_word_break_wseg_count_k) & non_ascii_lanes; frame.wseg = (wseg_multibyte | (sz_utf8_word_break_byte_equal_haswell_(raw_lo_u8x32, raw_hi_u8x32, 0x20) & valid)); @@ -414,8 +416,9 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_haswe _mm256_set1_epi8((char)0xC0)), _mm256_and_si256(next3_hi_u8x32, _mm256_set1_epi8(0x3F))); sz_u64_t const pictographic_bmp = sz_utf8_word_break_range16_mask_haswell_( - window.high_lo, window.high_hi, window.low_lo, window.low_hi, sz_utf8_word_break_pict_bmp_lo_, - sz_utf8_word_break_pict_bmp_hi_, sz_utf8_word_break_pict_bmp_count_k); + window.high_byte_low_u8x32, window.high_byte_high_u8x32, window.low_byte_low_u8x32, + window.low_byte_high_u8x32, sz_utf8_word_break_pict_bmp_lo_, sz_utf8_word_break_pict_bmp_hi_, + sz_utf8_word_break_pict_bmp_count_k); sz_u64_t const pictographic_smp = sz_utf8_word_break_range16_mask_haswell_( smp_high_lo_u8x32, smp_high_hi_u8x32, smp_low_lo_u8x32, smp_low_hi_u8x32, sz_utf8_word_break_pict_smp_lo_, sz_utf8_word_break_pict_smp_hi_, sz_utf8_word_break_pict_smp_count_k); @@ -435,9 +438,9 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_haswe /** @brief Resolve one window into the maximal-subpart partition - the AVX2 twin of * @ref sz_utf8_word_break_partition_icelake_: compute the per-ISA `sz_u64_t` masks and delegate to the * portable @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_haswell_( +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_haswell_( sz_utf8_rune_window_haswell_t window, sz_u64_t valid, int at_end_of_text) { - __m256i const raw_lo_u8x32 = window.window_lo, raw_hi_u8x32 = window.window_hi; + __m256i const raw_lo_u8x32 = window.window_low_u8x32, raw_hi_u8x32 = window.window_high_u8x32; sz_u64_t const real_continuation = window.continuation & valid; __m256i const high_nibble_lo_u8x32 = sz_utf8_srl8_haswell_(raw_lo_u8x32, 4, 0x0F); __m256i const high_nibble_hi_u8x32 = sz_utf8_srl8_haswell_(raw_hi_u8x32, 4, 0x0F); diff --git a/include/stringzilla/utf8_wordbreaks/icelake.h b/include/stringzilla/utf8_wordbreaks/icelake.h index 8fc4f361..ddb8f8d0 100644 --- a/include/stringzilla/utf8_wordbreaks/icelake.h +++ b/include/stringzilla/utf8_wordbreaks/icelake.h @@ -74,8 +74,9 @@ extern "C" { #if SZ_USE_ICELAKE #if defined(__clang__) && SZ_CLANG_HAS_EVEX512_ -#pragma clang attribute push( \ - __attribute__((target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,evex512,popcnt"))), \ +#pragma clang attribute push( \ + __attribute__(( \ + target("avx,avx512f,avx512vl,avx512bw,avx512dq,avx512vbmi,avx512vbmi2,bmi,bmi2,lzcnt,evex512,popcnt"))), \ apply_to = function) #elif defined(__clang__) #pragma clang attribute push( \ @@ -84,8 +85,7 @@ extern "C" { #elif defined(__GNUC__) #pragma GCC push_options #pragma GCC target("avx", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512vbmi", "avx512vbmi2", "bmi", "bmi2", \ - "lzcnt", \ - "popcnt") + "lzcnt", "popcnt") #endif #pragma region Word_Break Classifier @@ -141,7 +141,7 @@ SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_class_mask_icelake_(__m512i classes /** @brief 64-lane mask of lanes whose `(high, low)` 16-bit value lies inside any sorted `[lo, hi]` range (WSegSpace * WB3d and Extended_Pictographic WB3c, which are NOT part of the 4-bit Word_Break model). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_icelake_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_icelake_( // __m512i high_u8x64, __m512i low_u8x64, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { __mmask64 hit_m64 = _cvtu64_mask64(0); for (int range = 0; range < count; ++range) { @@ -164,7 +164,7 @@ SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_icelake_( // /** @brief Look up one cp < 0x800 page over @ref sz_utf8_word_break_flat_lut_0800_ via an in-register `vpermi2b` * network (cheaper than the flat gather for the dense 2-byte scripts). */ -SZ_HELPER_AUTO __m512i sz_utf8_word_break_small_page_icelake_(__m512i high_u8x64, __m512i low_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_word_break_small_page_icelake_(__m512i high_u8x64, __m512i low_u8x64) { __m512i const in_seven_u8x64 = _mm512_and_si512(low_u8x64, _mm512_set1_epi8(0x7F)); __m512i const low_high_bit_u8x64 = sz_utf8_srl8_icelake_(low_u8x64, 7, 0x01); __m512i const page_u8x64 = _mm512_or_si512( @@ -195,8 +195,8 @@ SZ_HELPER_AUTO __m512i sz_utf8_word_break_small_page_icelake_(__m512i high_u8x64 /** @brief Classify the 4-byte (astral) lanes of a window via the aligned `.rodata` astral trie. Four 16-lane chunks * reconstruct the 21-bit codepoint and walk the 4-stage trie; the caller blends the result onto `is_four_byte` * lanes. Bit-exact with `sz_rune_word_break_property` over the astral planes. */ -SZ_HELPER_AUTO __m512i sz_utf8_word_break_classify_four_byte_icelake_(__m512i window_u8x64, __m512i next1_u8x64, - __m512i next2_u8x64, __m512i next3_u8x64) { +SZ_HELPER_INLINE __m512i sz_utf8_word_break_classify_four_byte_icelake_(__m512i window_u8x64, __m512i next1_u8x64, + __m512i next2_u8x64, __m512i next3_u8x64) { __m512i const byte0_u8x64 = _mm512_and_si512(window_u8x64, _mm512_set1_epi8(0x07)); __m512i const byte1_u8x64 = _mm512_and_si512(next1_u8x64, _mm512_set1_epi8(0x3F)); __m512i const byte2_u8x64 = _mm512_and_si512(next2_u8x64, _mm512_set1_epi8(0x3F)); @@ -234,7 +234,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_word_break_classify_four_byte_icelake_(__m512i wi * `vpexpandb`-scattered back onto @p classes at their original byte-lane positions. The second half only * runs when more than sixteen cold starts are present. Cold continuation lanes are don't-cares (`decide` * reads only start lanes) and keep their prior value. */ -SZ_HELPER_AUTO __m512i sz_utf8_word_break_cold_compact_icelake_( // +SZ_HELPER_INLINE __m512i sz_utf8_word_break_cold_compact_icelake_( // __m512i classes_u8x64, __m512i high_bytes_u8x64, __m512i low_bytes_u8x64, sz_u64_t cold_starts) { __mmask64 const cold_start_mask_m64 = _cvtu64_mask64(cold_starts); __m512i const high_packed_u8x64 = _mm512_maskz_compress_epi8(cold_start_mask_m64, high_bytes_u8x64); @@ -265,7 +265,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_word_break_cold_compact_icelake_( // * table for the residue; 4-byte leads through the aligned `.rodata` astral trie. All cheap paths are * rare-class gated. */ -SZ_HELPER_AUTO __m512i sz_utf8_word_break_classify_window_icelake_( // +SZ_HELPER_INLINE __m512i sz_utf8_word_break_classify_window_icelake_( // __m512i window_u8x64, __m512i high_u8x64, __m512i low_u8x64, __mmask64 is_four_byte_m64, __m512i next1_u8x64, __m512i next2_u8x64, __m512i next3_u8x64) { __mmask64 const is_ascii_m64 = ~_mm512_movepi8_mask(window_u8x64); @@ -324,7 +324,7 @@ SZ_HELPER_AUTO __m512i sz_utf8_word_break_classify_window_icelake_( // * portable @ref sz_utf8_word_break_partition_from_masks_. @p at_end_of_text distinguishes a benign interior * straddle (the next window completes it) from a true end-of-text truncation. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_icelake_( // +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_icelake_( // __m512i window_u8x64, __m512i next1_u8x64, sz_u64_t valid, int at_end_of_text) { sz_u64_t const real_continuation = _cvtmask64_u64(_mm512_cmpeq_epi8_mask( _mm512_and_si512(window_u8x64, _mm512_set1_epi8((char)0xC0)), @@ -516,10 +516,10 @@ SZ_API_COMPTIME sz_size_t sz_utf8_wordbreaks_icelake( // sz_utf8_rune_window_t const decoded = sz_utf8_rune_decode_window_icelake_(text_u8 + position, length - position, lane_identity_u8x64); sz_size_t const loaded = decoded.loaded; - __m512i const window_u8x64 = decoded.window; + __m512i const window_u8x64 = decoded.window_u8x64; sz_u64_t const valid = sz_u64_mask_until_(loaded); - __m512i const high_u8x64 = decoded.high; - __m512i const low_u8x64 = decoded.low; + __m512i const high_u8x64 = decoded.high_byte_u8x64; + __m512i const low_u8x64 = decoded.low_byte_u8x64; __m512i const next1_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(1)), window_u8x64); __m512i const next2_u8x64 = _mm512_permutexvar_epi8(_mm512_add_epi8(lane_identity_u8x64, _mm512_set1_epi8(2)), diff --git a/include/stringzilla/utf8_wordbreaks/lasx.h b/include/stringzilla/utf8_wordbreaks/lasx.h index daaa0b0f..c316e8f4 100644 --- a/include/stringzilla/utf8_wordbreaks/lasx.h +++ b/include/stringzilla/utf8_wordbreaks/lasx.h @@ -39,7 +39,7 @@ extern "C" { /** @brief Word_Break class byte for thirty-two BMP codepoints (per-lane high = cp>>8, low = cp&0xFF) from the flat * page-compressed table via @ref sz_utf8_rune_flat_lookup_lasx_, the LASX twin of * @ref sz_utf8_word_break_bmp_class_haswell_. Bit-exact with `sz_rune_word_break_property` over the BMP. */ -SZ_HELPER_AUTO __m256i sz_utf8_word_break_bmp_class_lasx_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_word_break_bmp_class_lasx_(__m256i high_bytes_u8x32, __m256i low_bytes_u8x32) { return sz_utf8_rune_flat_lookup_lasx_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, (int)sz_utf8_word_break_flat_pages_k, high_bytes_u8x32, low_bytes_u8x32); } @@ -48,8 +48,8 @@ SZ_HELPER_AUTO __m256i sz_utf8_word_break_bmp_class_lasx_(__m256i high_bytes_u8x * cascade), the LASX twin of @ref sz_utf8_word_break_astral_class_haswell_. The 256-entry stage-1 and * stage-4 tables resolve by a bounded scalar walk; the 16-byte stage-2/stage-3 rows by `xvshuf.b` over a * double-broadcast row. Bit-exact with `sz_rune_word_break_property` over the Supplementary Planes. */ -SZ_HELPER_AUTO __m256i sz_utf8_word_break_astral_class_lasx_(__m256i plane_offset_u8x32, __m256i high_byte_u8x32, - __m256i low_byte_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_word_break_astral_class_lasx_(__m256i plane_offset_u8x32, __m256i high_byte_u8x32, + __m256i low_byte_u8x32) { __m256i const low_nibble_mask_u8x32 = __lasx_xvreplgr2vr_b(0x0F); __m256i const nibble_4_u8x32 = __lasx_xvand_v(plane_offset_u8x32, low_nibble_mask_u8x32); __m256i const nibble_3_u8x32 = sz_utf8_high_nibble_lasx_(high_byte_u8x32); @@ -92,7 +92,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_word_break_astral_class_lasx_(__m256i plane_offse /** @brief Word_Break class byte for thirty-two ASCII codepoints (cp < 0x80) via the existing 128-entry property * table, read by a bounded scalar walk (the window byte equals the codepoint on ASCII lanes). The LASX twin * of @ref sz_utf8_word_break_ascii_class_haswell_. */ -SZ_HELPER_AUTO __m256i sz_utf8_word_break_ascii_class_lasx_(__m256i bytes_u8x32) { +SZ_HELPER_INLINE __m256i sz_utf8_word_break_ascii_class_lasx_(__m256i bytes_u8x32) { sz_u256_vec_t byte_vec, result_vec; byte_vec.lasx = bytes_u8x32; for (int lane = 0; lane < 32; ++lane) @@ -107,7 +107,7 @@ SZ_HELPER_AUTO __m256i sz_utf8_word_break_ascii_class_lasx_(__m256i bytes_u8x32) * populated 32-lane half/halves, then scatters the dense class bytes back. Bit-identical to a full * @ref sz_utf8_word_break_bmp_class_lasx_ over both halves on every BMP-start lane; every other lane is a * don't-care left at its incoming value. */ -SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_lasx_( // +SZ_HELPER_INLINE void sz_utf8_word_break_bmp_compact_lasx_( // sz_u64_t bmp_starts, __m256i high_byte_low_u8x32, __m256i high_byte_high_u8x32, __m256i low_byte_low_u8x32, __m256i low_byte_high_u8x32, __m256i *out_low_u8x32, __m256i *out_high_u8x32) { sz_u8_t high_bytes[64], low_bytes[64]; @@ -152,7 +152,7 @@ SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_lasx_( // * @ref sz_utf8_word_break_classify_window_haswell_, bit-identical on every start lane. ASCII through the * property table, BMP through the compacted flat lookup, 4-byte leads through the astral cascade with the * codepoint high/low/plane reconstructed from the forward neighbours. */ -SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_lasx_( // +SZ_HELPER_INLINE void sz_utf8_word_break_classify_window_lasx_( // sz_utf8_rune_window_lasx_t window, __m256i *classes_low_u8x32, __m256i *classes_high_u8x32) { __m256i const raw_low_u8x32 = window.window_low_u8x32, raw_high_u8x32 = window.window_high_u8x32; sz_u64_t const ascii_starts = window.codepoint_starts & ~window.two_byte_starts & ~window.three_byte_starts & @@ -278,7 +278,7 @@ SZ_HELPER_INLINE __m256i sz_utf8_word_break_range16_one_lasx_(__m256i high_byte_ /** @brief A 64-bit "(high_byte, low_byte) 16-bit value in any sorted `[floor, ceiling]` range" lane mask over both * window halves, the LASX twin of @ref sz_utf8_word_break_range16_mask_haswell_ (WSegSpace / * Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_lasx_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_lasx_( // __m256i high_byte_low_u8x32, __m256i high_byte_high_u8x32, __m256i low_byte_low_u8x32, __m256i low_byte_high_u8x32, sz_u16_t const *floor_table, sz_u16_t const *ceiling_table, int count) { __m256i hit_low_u8x32 = __lasx_xvreplgr2vr_b(0), hit_high_u8x32 = __lasx_xvreplgr2vr_b(0); @@ -299,7 +299,7 @@ SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_lasx_( // * the class halves, materializes every per-class lane mask + the raw-byte membership masks, the * Extended_Pictographic mask (BMP + SMP range scan), and the per-lane class byte array. */ -SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_lasx_( +SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_lasx_( sz_utf8_rune_window_lasx_t window, __m256i classes_low_u8x32, __m256i classes_high_u8x32, sz_u64_t start_bytes_all, sz_u64_t length_two, sz_u64_t length_three, sz_u64_t length_four, int want_pictographic) { @@ -430,8 +430,8 @@ SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_lasx_( /** @brief Resolve one window into the maximal-subpart partition - the LASX twin of * @ref sz_utf8_word_break_partition_haswell_: compute the per-ISA `sz_u64_t` masks and delegate to the * portable @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_lasx_(sz_utf8_rune_window_lasx_t window, - sz_u64_t valid, int at_end_of_text) { +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_lasx_(sz_utf8_rune_window_lasx_t window, + sz_u64_t valid, int at_end_of_text) { __m256i const raw_low_u8x32 = window.window_low_u8x32, raw_high_u8x32 = window.window_high_u8x32; sz_u64_t const real_continuation = window.continuation & valid; __m256i const high_nibble_low_u8x32 = sz_utf8_srl8_lasx_(raw_low_u8x32, 4, 0x0F); diff --git a/include/stringzilla/utf8_wordbreaks/neon.h b/include/stringzilla/utf8_wordbreaks/neon.h index 194d2d88..259d5ae9 100644 --- a/include/stringzilla/utf8_wordbreaks/neon.h +++ b/include/stringzilla/utf8_wordbreaks/neon.h @@ -58,7 +58,8 @@ SZ_HELPER_INLINE uint8x16_t sz_utf8_word_break_byte_mask_from_bits_neon_(sz_u64_ * page-compressed table via @ref sz_utf8_rune_flat_lookup_neon_, the NEON twin of * @ref sz_utf8_word_break_bmp_class_haswell_. Bit-exact with `sz_rune_word_break_property` over the whole * BMP. Addresses ONE quarter; the caller iterates the four quarters. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_word_break_bmp_class_neon_(uint8x16_t high_bytes_u8x16, uint8x16_t low_bytes_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_word_break_bmp_class_neon_(uint8x16_t high_bytes_u8x16, + uint8x16_t low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_neon_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, (int)sz_utf8_word_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -68,8 +69,8 @@ SZ_HELPER_AUTO uint8x16_t sz_utf8_word_break_bmp_class_neon_(uint8x16_t high_byt * @p plane_off_u8x16 = (offset>>16)&0xFF (low nibble meaningful), @p high_u8x16 = (offset>>8)&0xFF, * @p low_u8x16 = offset&0xFF. Bit-exact with `sz_rune_word_break_property` over the Supplementary Planes. * Addresses ONE quarter; the caller iterates the four quarters. */ -SZ_HELPER_AUTO uint8x16_t sz_utf8_word_break_astral_class_neon_(uint8x16_t plane_off_u8x16, uint8x16_t high_u8x16, - uint8x16_t low_u8x16) { +SZ_HELPER_INLINE uint8x16_t sz_utf8_word_break_astral_class_neon_(uint8x16_t plane_off_u8x16, uint8x16_t high_u8x16, + uint8x16_t low_u8x16) { uint8x16_t const low_nibble_mask_u8x16 = vdupq_n_u8(0x0F); uint8x16_t const n4_u8x16 = vandq_u8(plane_off_u8x16, low_nibble_mask_u8x16); uint8x16_t const n3_u8x16 = vandq_u8(vshrq_n_u8(high_u8x16, 4), low_nibble_mask_u8x16); @@ -126,8 +127,8 @@ SZ_HELPER_INLINE uint8x16_t sz_utf8_word_break_ascii_class_neon_(uint8x16_t byte * dense class bytes back to their original byte lanes in @p bmp_out_u8x16 (zeroed elsewhere). Bit-identical * to four full @ref sz_utf8_word_break_bmp_class_neon_ quarters on every BMP-start lane; every other lane is * a don't-care left at zero. */ -SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_neon_(sz_u64_t bmp_starts, uint8x16_t const *high_u8x16, - uint8x16_t const *low_u8x16, uint8x16_t *bmp_out_u8x16) { +SZ_HELPER_INLINE void sz_utf8_word_break_bmp_compact_neon_(sz_u64_t bmp_starts, uint8x16_t const *high_u8x16, + uint8x16_t const *low_u8x16, uint8x16_t *bmp_out_u8x16) { sz_u8_t high_bytes[64], low_bytes[64]; for (int quarter = 0; quarter < 4; ++quarter) { vst1q_u8(high_bytes + quarter * 16, high_u8x16[quarter]); @@ -169,9 +170,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_neon_(sz_u64_t bmp_starts, ui * @ref sz_utf8_word_break_classify_window_haswell_, bit-identical on every start lane. ASCII through the * property table, BMP through the nibble cascade, 4-byte leads through the astral cascade with the codepoint * high/low/plane reconstructed from the forward neighbours. */ -SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_neon_( // +SZ_HELPER_INLINE void sz_utf8_word_break_classify_window_neon_( // sz_utf8_rune_window_neon_t window, uint8x16_t *classes_u8x16) { - uint8x16_t const *raw_u8x16 = window.window; + uint8x16_t const *raw_u8x16 = window.window_u8x16s; sz_u64_t const ascii_starts = window.codepoint_starts & ~window.two_byte_starts & ~window.three_byte_starts & ~window.four_byte_starts; @@ -186,7 +187,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_neon_( // // don't-cares (overwritten or unread below), so the dense walk leaves them at zero. uint8x16_t bmp_out_u8x16[4] = {vdupq_n_u8(0), vdupq_n_u8(0), vdupq_n_u8(0), vdupq_n_u8(0)}; sz_u64_t const bmp_starts = window.two_byte_starts | window.three_byte_starts; - if (bmp_starts) sz_utf8_word_break_bmp_compact_neon_(bmp_starts, window.high, window.low, bmp_out_u8x16); + if (bmp_starts) + sz_utf8_word_break_bmp_compact_neon_(bmp_starts, window.high_byte_u8x16s, window.low_byte_u8x16s, + bmp_out_u8x16); for (int quarter = 0; quarter < 4; ++quarter) { uint8x16_t const raw_q_u8x16 = raw_u8x16[quarter]; @@ -267,7 +270,7 @@ SZ_HELPER_INLINE uint8x16_t sz_utf8_word_break_range16_one_neon_(uint8x16_t high /** @brief A 64-bit "(high,low) 16-bit value in any sorted `[lo, hi]` range" lane mask over the four window quarters, * the NEON twin of @ref sz_utf8_word_break_range16_mask_haswell_ (WSegSpace / Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_neon_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_neon_( // uint8x16_t const *high_u8x16, uint8x16_t const *low_u8x16, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { uint8x16_t hit_u8x16[4] = {vdupq_n_u8(0), vdupq_n_u8(0), vdupq_n_u8(0), vdupq_n_u8(0)}; @@ -292,7 +295,7 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_neon_ sz_size_t const loaded = window.loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); sz_u64_t const start_bytes = start_bytes_all & valid; - uint8x16_t const *raw_u8x16 = window.window; + uint8x16_t const *raw_u8x16 = window.window_u8x16s; // Truncated-edge U+FFFD reclassify (force the class to Other on a lead whose declared span runs past `loaded`). sz_u64_t const lead_two = length_two & start_bytes; @@ -335,8 +338,8 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_neon_ // WB3d WSegSpace raw membership: the ASCII U+0020 byte compare OR the multibyte (high,low) range scan. sz_u64_t wseg_multibyte = 0ull; if (non_ascii_lanes) - wseg_multibyte = sz_utf8_word_break_range16_mask_neon_(window.high, window.low, sz_utf8_word_break_wseg_lo_, - sz_utf8_word_break_wseg_hi_, + wseg_multibyte = sz_utf8_word_break_range16_mask_neon_(window.high_byte_u8x16s, window.low_byte_u8x16s, + sz_utf8_word_break_wseg_lo_, sz_utf8_word_break_wseg_hi_, sz_utf8_word_break_wseg_count_k) & non_ascii_lanes; frame.wseg = (wseg_multibyte | (sz_utf8_word_break_byte_equal_neon_(raw_u8x16, 0x20) & valid)); @@ -368,8 +371,8 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_neon_ vceqq_u8(plane_q_u8x16[0], one_u8x16), vceqq_u8(plane_q_u8x16[1], one_u8x16), vceqq_u8(plane_q_u8x16[2], one_u8x16), vceqq_u8(plane_q_u8x16[3], one_u8x16)); sz_u64_t const pictographic_bmp = sz_utf8_word_break_range16_mask_neon_( - window.high, window.low, sz_utf8_word_break_pict_bmp_lo_, sz_utf8_word_break_pict_bmp_hi_, - sz_utf8_word_break_pict_bmp_count_k); + window.high_byte_u8x16s, window.low_byte_u8x16s, sz_utf8_word_break_pict_bmp_lo_, + sz_utf8_word_break_pict_bmp_hi_, sz_utf8_word_break_pict_bmp_count_k); sz_u64_t const pictographic_smp = sz_utf8_word_break_range16_mask_neon_( smp_high_u8x16, smp_low_u8x16, sz_utf8_word_break_pict_smp_lo_, sz_utf8_word_break_pict_smp_hi_, sz_utf8_word_break_pict_smp_count_k); @@ -388,9 +391,9 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_neon_ /** @brief Resolve one window into the maximal-subpart partition - the NEON twin of * @ref sz_utf8_word_break_partition_haswell_: compute the per-ISA `sz_u64_t` masks and delegate to the * portable @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_neon_(sz_utf8_rune_window_neon_t window, - sz_u64_t valid, int at_end_of_text) { - uint8x16_t const *raw_u8x16 = window.window; +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_neon_(sz_utf8_rune_window_neon_t window, + sz_u64_t valid, int at_end_of_text) { + uint8x16_t const *raw_u8x16 = window.window_u8x16s; sz_u64_t const real_continuation = window.continuation & valid; // Declared length follows the serial high-nibble rule: 0xC/0xD → 2, 0xE → 3, 0xF → 4. The strict // `two`/`three_byte_starts` masks already match 0xC0-0xDF and 0xE0-0xEF; only `length_four` needs widening to fold diff --git a/include/stringzilla/utf8_wordbreaks/powervsx.h b/include/stringzilla/utf8_wordbreaks/powervsx.h index f15af6ea..3b99890d 100644 --- a/include/stringzilla/utf8_wordbreaks/powervsx.h +++ b/include/stringzilla/utf8_wordbreaks/powervsx.h @@ -63,8 +63,8 @@ SZ_HELPER_INLINE __vector unsigned char sz_utf8_word_break_byte_mask_from_bits_p * page-compressed table via @ref sz_utf8_rune_flat_lookup_powervsx_, the VSX twin of * @ref sz_utf8_word_break_bmp_class_neon_. Bit-exact with `sz_rune_word_break_property` over the whole BMP. * Addresses ONE quarter; the caller iterates the four quarters. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_bmp_class_powervsx_(__vector unsigned char high_bytes_u8x16, - __vector unsigned char low_bytes_u8x16) { +SZ_HELPER_INLINE __vector unsigned char sz_utf8_word_break_bmp_class_powervsx_(__vector unsigned char high_bytes_u8x16, + __vector unsigned char low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_powervsx_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, (int)sz_utf8_word_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -73,7 +73,7 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_bmp_class_powervsx_(__v * cascade), the VSX twin of @ref sz_utf8_word_break_astral_class_neon_. Per-lane bytes: @p plane_off = * (offset>>16)&0xFF (low nibble meaningful), @p high = (offset>>8)&0xFF, @p low = offset&0xFF. Bit-exact with * `sz_rune_word_break_property` over the Supplementary Planes. Addresses ONE quarter. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_astral_class_powervsx_( // +SZ_HELPER_INLINE __vector unsigned char sz_utf8_word_break_astral_class_powervsx_( // __vector unsigned char plane_off_u8x16, __vector unsigned char high_u8x16, __vector unsigned char low_u8x16) { __vector unsigned char const low_nibble_mask_u8x16 = vec_splats((unsigned char)0x0F); __vector unsigned char const shift_four_u8x16 = vec_splats((unsigned char)4); @@ -113,7 +113,7 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_astral_class_powervsx_( /** @brief Word_Break class byte for sixteen ASCII codepoints (cp < 0x80) via the existing 128-entry property table, * the VSX twin of @ref sz_utf8_word_break_ascii_class_neon_, read by a bounded scalar L1 walk (the window * byte equals the codepoint on ASCII lanes). Addresses ONE quarter. */ -SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_ascii_class_powervsx_(__vector unsigned char bytes_u8x16) { +SZ_HELPER_INLINE __vector unsigned char sz_utf8_word_break_ascii_class_powervsx_(__vector unsigned char bytes_u8x16) { sz_u128_vec_t bytes_vec, result_vec; bytes_vec.vsx_u8 = bytes_u8x16; for (int lane = 0; lane < 16; ++lane) @@ -127,10 +127,10 @@ SZ_HELPER_AUTO __vector unsigned char sz_utf8_word_break_ascii_class_powervsx_(_ * only over the populated quarters, then scatters the dense class bytes back to their original byte lanes in * @p bmp_out_u8x16 (zeroed elsewhere). Bit-identical to four full @ref sz_utf8_word_break_bmp_class_powervsx_ * quarters on every BMP-start lane. */ -SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_powervsx_(sz_u64_t bmp_starts, - __vector unsigned char const *high_u8x16, - __vector unsigned char const *low_u8x16, - __vector unsigned char *bmp_out_u8x16) { +SZ_HELPER_INLINE void sz_utf8_word_break_bmp_compact_powervsx_(sz_u64_t bmp_starts, + __vector unsigned char const *high_u8x16, + __vector unsigned char const *low_u8x16, + __vector unsigned char *bmp_out_u8x16) { sz_u512_vec_t high_bytes_vec, low_bytes_vec; for (int quarter = 0; quarter < 4; ++quarter) { vec_xst(high_u8x16[quarter], 0, high_bytes_vec.u8s + quarter * 16); @@ -169,9 +169,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_bmp_compact_powervsx_(sz_u64_t bmp_starts * twin of @ref sz_utf8_word_break_classify_window_neon_, bit-identical on every start lane. ASCII through * the property table, BMP through the flat lookup, 4-byte leads through the astral cascade with the * codepoint high/low/plane reconstructed from the forward neighbours. */ -SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_powervsx_( // +SZ_HELPER_INLINE void sz_utf8_word_break_classify_window_powervsx_( // sz_utf8_rune_window_powervsx_t window, __vector unsigned char *classes_u8x16) { - __vector unsigned char const *raw_u8x16 = window.window; + __vector unsigned char const *raw_u8x16 = window.window_u8x16s; sz_u64_t const ascii_starts = window.codepoint_starts & ~window.two_byte_starts & ~window.three_byte_starts & ~window.four_byte_starts; @@ -194,7 +194,9 @@ SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_powervsx_( // __vector unsigned char bmp_out_u8x16[4] = {vec_splats((unsigned char)0), vec_splats((unsigned char)0), vec_splats((unsigned char)0), vec_splats((unsigned char)0)}; sz_u64_t const bmp_starts = window.two_byte_starts | window.three_byte_starts; - if (bmp_starts) sz_utf8_word_break_bmp_compact_powervsx_(bmp_starts, window.high, window.low, bmp_out_u8x16); + if (bmp_starts) + sz_utf8_word_break_bmp_compact_powervsx_(bmp_starts, window.high_byte_u8x16s, window.low_byte_u8x16s, + bmp_out_u8x16); for (int quarter = 0; quarter < 4; ++quarter) { __vector unsigned char const raw_q_u8x16 = raw_u8x16[quarter]; @@ -292,7 +294,7 @@ SZ_HELPER_INLINE __vector unsigned char sz_utf8_word_break_range16_one_powervsx_ /** @brief A 64-bit "(high,low) 16-bit value in any sorted `[lo, hi]` range" lane mask over the four window quarters, * the VSX twin of @ref sz_utf8_word_break_range16_mask_neon_ (WSegSpace / Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_powervsx_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_powervsx_( // __vector unsigned char const *high_u8x16, __vector unsigned char const *low_u8x16, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { __vector unsigned char hit_u8x16[4] = {vec_splats((unsigned char)0), vec_splats((unsigned char)0), @@ -311,14 +313,14 @@ SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_powervsx_( // * class quarters, materializes every per-class lane mask + the raw-byte membership masks, the * Extended_Pictographic mask (BMP + SMP range scan), and the per-lane class byte array. */ -SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_powervsx_( +SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_powervsx_( sz_utf8_rune_window_powervsx_t window, __vector unsigned char *classes_u8x16, sz_u64_t start_bytes_all, sz_u64_t length_two, sz_u64_t length_three, sz_u64_t length_four, int want_pictographic) { sz_size_t const loaded = window.loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); sz_u64_t const start_bytes = start_bytes_all & valid; - __vector unsigned char const *raw_u8x16 = window.window; + __vector unsigned char const *raw_u8x16 = window.window_u8x16s; // Truncated-edge U+FFFD reclassify (force the class to Other on a lead whose declared span runs past `loaded`). sz_u64_t const lead_two = length_two & start_bytes; @@ -363,9 +365,9 @@ SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_powervs // WB3d WSegSpace raw membership: the ASCII U+0020 byte compare OR the multibyte (high,low) range scan. sz_u64_t wseg_multibyte = 0ull; if (non_ascii_lanes) - wseg_multibyte = sz_utf8_word_break_range16_mask_powervsx_(window.high, window.low, sz_utf8_word_break_wseg_lo_, - sz_utf8_word_break_wseg_hi_, - sz_utf8_word_break_wseg_count_k) & + wseg_multibyte = sz_utf8_word_break_range16_mask_powervsx_( + window.high_byte_u8x16s, window.low_byte_u8x16s, sz_utf8_word_break_wseg_lo_, + sz_utf8_word_break_wseg_hi_, sz_utf8_word_break_wseg_count_k) & non_ascii_lanes; frame.wseg = (wseg_multibyte | (sz_utf8_word_break_byte_equal_powervsx_(raw_u8x16, 0x20) & valid)); @@ -407,8 +409,8 @@ SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_powervs plane_one_bool_u8x16[quarter] = (__vector unsigned char)vec_cmpeq(plane_q_u8x16[quarter], one_u8x16); sz_u64_t const plane_one = sz_utf8_mask_combine_powervsx_(plane_one_bool_u8x16); sz_u64_t const pictographic_bmp = sz_utf8_word_break_range16_mask_powervsx_( - window.high, window.low, sz_utf8_word_break_pict_bmp_lo_, sz_utf8_word_break_pict_bmp_hi_, - sz_utf8_word_break_pict_bmp_count_k); + window.high_byte_u8x16s, window.low_byte_u8x16s, sz_utf8_word_break_pict_bmp_lo_, + sz_utf8_word_break_pict_bmp_hi_, sz_utf8_word_break_pict_bmp_count_k); sz_u64_t const pictographic_smp = sz_utf8_word_break_range16_mask_powervsx_( smp_high_u8x16, smp_low_u8x16, sz_utf8_word_break_pict_smp_lo_, sz_utf8_word_break_pict_smp_hi_, sz_utf8_word_break_pict_smp_count_k); @@ -427,9 +429,9 @@ SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_powervs /** @brief Resolve one window into the maximal-subpart partition, the VSX twin of * @ref sz_utf8_word_break_partition_neon_: compute the per-ISA `sz_u64_t` masks and delegate to the portable * @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_powervsx_( // +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_powervsx_( // sz_utf8_rune_window_powervsx_t window, sz_u64_t valid, int at_end_of_text) { - __vector unsigned char const *raw_u8x16 = window.window; + __vector unsigned char const *raw_u8x16 = window.window_u8x16s; sz_u64_t const real_continuation = window.continuation & valid; // Declared length follows the serial high-nibble rule: 0xC/0xD -> 2, 0xE -> 3, 0xF -> 4. The strict // `two`/`three_byte_starts` masks already match 0xC0-0xDF and 0xE0-0xEF; only `length_four` needs widening to fold diff --git a/include/stringzilla/utf8_wordbreaks/rvv.h b/include/stringzilla/utf8_wordbreaks/rvv.h index a4ff35bf..6560fb17 100644 --- a/include/stringzilla/utf8_wordbreaks/rvv.h +++ b/include/stringzilla/utf8_wordbreaks/rvv.h @@ -64,7 +64,7 @@ SZ_HELPER_INLINE vuint8m4_t sz_utf8_word_break_ascii_class_rvv_(vuint8m4_t raw_u * values < 14, stage-2 values below the stage-3 row count) except at stage 4, where `leaf_group` is * clamped for the address and the result zeroed on out-of-range groups, matching the NEON blend loop. * Bit-exact with `sz_rune_word_break_property` over the Supplementary Planes. */ -SZ_HELPER_AUTO vuint8m4_t sz_utf8_word_break_astral_class_rvv_( // +SZ_HELPER_INLINE vuint8m4_t sz_utf8_word_break_astral_class_rvv_( // vuint8m4_t plane_off_u8m4, vuint8m4_t high_u8m4, vuint8m4_t low_u8m4) { vuint8m4_t const n4_u8m4 = __riscv_vand_vx_u8m4(plane_off_u8m4, 0x0F, 64); vuint8m4_t const n3_u8m4 = __riscv_vsrl_vx_u8m4(high_u8m4, 4, 64); @@ -114,8 +114,8 @@ SZ_HELPER_AUTO vuint8m4_t sz_utf8_word_break_astral_class_rvv_( // * `loaded` clamp is an in-register `vid < loaded` compare (lane indices <= 63 fit `u8`). ASCII and BMP resolve * through UNCONDITIONAL masked gathers — inactive lanes perform no memory access, and index safety comes from table * totality, never from the mask — so only the rare 4-gather astral cascade keeps a branch. */ -SZ_HELPER_AUTO vuint8m4_t sz_utf8_word_break_classify_window_rvv_(vuint8m4_t const raw_u8m4, - sz_utf8_rune_window_rvv_t const *window) { +SZ_HELPER_INLINE vuint8m4_t sz_utf8_word_break_classify_window_rvv_(vuint8m4_t const raw_u8m4, + sz_utf8_rune_window_rvv_t const *window) { // In-register lead classes and the `loaded` clamp, mirroring the struct fields `decode_window` lowered for the // engine but kept as masks so no `sz_u64_t` is raised back to a `vbool2_t` here. vbool2_t const within_loaded_b2 = __riscv_vmsltu_vx_u8m4_b2(__riscv_vid_v_u8m4(64), (sz_u8_t)window->loaded, 64); @@ -188,7 +188,7 @@ SZ_HELPER_INLINE vuint16m8_t sz_utf8_word_break_codepoint16_rvv_(vuint8m4_t high /** @brief A 64-bit "codepoint16 in any sorted `[lo, hi]` range" lane mask, the RVV twin of * @ref sz_utf8_word_break_range16_mask_neon_ over native 16-bit lanes (WSegSpace / Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_rvv_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_rvv_( // vuint16m8_t values_u16m8, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { vbool2_t hit_b2 = __riscv_vmclr_m_b2(64); for (int range = 0; range < count; ++range) { @@ -206,7 +206,7 @@ SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_rvv_( // /** @brief Resolve one window into the maximal-subpart partition — the RVV twin of * @ref sz_utf8_word_break_partition_neon_: compute the per-ISA `sz_u64_t` masks and delegate to the * portable @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_rvv_( // +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_rvv_( // vuint8m4_t const raw_u8m4, sz_utf8_rune_window_rvv_t const *window, sz_u64_t valid, int at_end_of_text) { sz_u64_t const real_continuation = window->continuation & valid; // Declared length follows the serial high-nibble rule: 0xC/0xD -> 2, 0xE -> 3, 0xF -> 4. The strict @@ -245,7 +245,7 @@ SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_rvv_( * the truncated-edge U+FFFD reclassify to the class lanes, materializes every per-class lane mask + the * raw-byte membership masks, the Extended_Pictographic mask (BMP + SMP range scan), and the per-lane * class byte array. */ -SZ_HELPER_AUTO sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_rvv_( // +SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_rvv_( // vuint8m4_t const raw_u8m4, sz_utf8_rune_window_rvv_t const *window, vuint8m4_t classes_u8m4, sz_u64_t start_bytes_all, sz_u64_t length_two, sz_u64_t length_three, sz_u64_t length_four, int want_pictographic) { diff --git a/include/stringzilla/utf8_wordbreaks/serial.h b/include/stringzilla/utf8_wordbreaks/serial.h index 83b14408..947d6a7b 100644 --- a/include/stringzilla/utf8_wordbreaks/serial.h +++ b/include/stringzilla/utf8_wordbreaks/serial.h @@ -94,12 +94,12 @@ enum { }; /** @brief Check if a property is WB4-ignorable (Extend, Format, ZWJ). */ -SZ_HELPER_INLINE sz_bool_t sz_utf8_word_break_is_ignorable_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_utf8_word_break_is_ignorable_(sz_u8_t property) { return (sz_bool_t)((sz_utf8_word_break_ignorable_set_k >> property) & 1u); } /** @brief Check if a property is AHLetter (ALetter or Hebrew_Letter). */ -SZ_HELPER_INLINE sz_bool_t sz_utf8_word_break_is_aletter_or_hebrew_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_utf8_word_break_is_aletter_or_hebrew_(sz_u8_t property) { return (sz_bool_t)((sz_utf8_word_break_aletter_or_hebrew_set_k >> property) & 1u); } @@ -107,7 +107,7 @@ SZ_HELPER_INLINE sz_bool_t sz_utf8_word_break_is_aletter_or_hebrew_(sz_u8_t prop * @brief Check if a property is MidNumLetQ (MidNumLet or Single_Quote). * In our encoding, MID_QUOTES (15) covers MidNumLet + quotes. */ -SZ_HELPER_INLINE sz_bool_t sz_utf8_word_break_is_mid_quotes_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_utf8_word_break_is_mid_quotes_(sz_u8_t property) { return (sz_bool_t)((sz_utf8_word_break_mid_quotes_set_k >> property) & 1u); } @@ -133,7 +133,7 @@ typedef struct sz_word_element_t { } sz_word_element_t; /** @brief True for the newline family CR/LF/Newline, which neither absorb (WB4) nor are absorbed. */ -SZ_HELPER_INLINE sz_bool_t sz_word_is_newline_(sz_u8_t property) { +SZ_HELPER_AUTO sz_bool_t sz_word_is_newline_(sz_u8_t property) { return (sz_bool_t)(property == sz_utf8_word_break_cr_k || property == sz_utf8_word_break_lf_k || property == sz_utf8_word_break_newline_k); } @@ -141,13 +141,13 @@ SZ_HELPER_INLINE sz_bool_t sz_word_is_newline_(sz_u8_t property) { /* The 4-bit model lumps Single_Quote (U+0027), Double_Quote (U+0022), and MidNumLet into MID_QUOTES, so the * WB6/WB7/WB7a-c/WB11/WB12 distinctions are recovered from the codepoint. MidNumLetQ = MidNumLet + Single_Quote, * i.e. every MID_QUOTES codepoint that is NOT the Double_Quote. */ -SZ_HELPER_INLINE sz_bool_t sz_word_is_single_quote_(sz_u8_t property, sz_rune_t codepoint) { +SZ_HELPER_AUTO sz_bool_t sz_word_is_single_quote_(sz_u8_t property, sz_rune_t codepoint) { return (sz_bool_t)(property == sz_utf8_word_break_mid_quotes_k && codepoint == 0x0027u); } -SZ_HELPER_INLINE sz_bool_t sz_word_is_double_quote_(sz_u8_t property, sz_rune_t codepoint) { +SZ_HELPER_AUTO sz_bool_t sz_word_is_double_quote_(sz_u8_t property, sz_rune_t codepoint) { return (sz_bool_t)(property == sz_utf8_word_break_mid_quotes_k && codepoint == 0x0022u); } -SZ_HELPER_INLINE sz_bool_t sz_word_is_mid_num_let_q_(sz_u8_t property, sz_rune_t codepoint) { +SZ_HELPER_AUTO sz_bool_t sz_word_is_mid_num_let_q_(sz_u8_t property, sz_rune_t codepoint) { return (sz_bool_t)(property == sz_utf8_word_break_mid_quotes_k && codepoint != 0x0022u); } @@ -199,7 +199,7 @@ SZ_HELPER_AUTO sz_word_element_t sz_word_previous_element_(sz_cptr_t text, sz_si } /** @brief The element following the one whose base is at @p position (eot sentinel at end of text). */ -SZ_HELPER_AUTO sz_word_element_t sz_word_next_element_(sz_cptr_t text, sz_size_t length, sz_size_t position) { +SZ_HELPER_INLINE sz_word_element_t sz_word_next_element_(sz_cptr_t text, sz_size_t length, sz_size_t position) { sz_word_element_t element; element.valid = sz_false_k; sz_size_t cursor = position; @@ -238,8 +238,11 @@ SZ_HELPER_AUTO sz_size_t sz_word_regional_run_before_(sz_cptr_t text, sz_size_t /** * @brief Whether @p position is a UAX-29 word boundary. Direct, branch-per-rule transcription of WB1-WB16 over - * the WB4 element model; used unchanged by the forward and reverse drivers (and every backend that - * delegates here), so segmentation is identical in either direction. + * the WB4 element model, re-walking left context per position. + * + * Nothing in the library calls this: the segmenters run a streaming state machine instead, which carries that + * context forward. It is the independent second opinion the tests measure that machine against, which is what + * turns `sz_word_serial_boundary_`'s byte-identity claim into something checked rather than asserted. */ SZ_API_COMPTIME sz_bool_t sz_utf8_is_word_boundary_serial(sz_cptr_t text, sz_size_t length, sz_size_t position) { if (position == 0) return sz_true_k; // WB1 diff --git a/include/stringzilla/utf8_wordbreaks/sve2.h b/include/stringzilla/utf8_wordbreaks/sve2.h index 534415f6..f0f1edc6 100644 --- a/include/stringzilla/utf8_wordbreaks/sve2.h +++ b/include/stringzilla/utf8_wordbreaks/sve2.h @@ -48,14 +48,14 @@ extern "C" { * predicate/lane-mask bridge. The substrate LUT readers live in `utf8_runes/sve2.h`. */ /** @brief Word_Break class byte for sixteen-bit-wide BMP codepoints (per-lane high = cp>>8, low = cp&0xFF). */ -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_bmp_class_sve2_(svuint8_t high_u8x, svuint8_t low_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_bmp_class_sve2_(svuint8_t high_u8x, svuint8_t low_u8x) { return sz_utf8_rune_flat_lookup_sve2_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, high_u8x, low_u8x); } /** @brief Word_Break class byte for sixteen ASTRAL codepoints over the 20-bit offset = cp - 0x10000. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_astral_class_sve2_(svuint8_t plane_off_u8x, svuint8_t high_u8x, - svuint8_t low_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_astral_class_sve2_(svuint8_t plane_off_u8x, svuint8_t high_u8x, + svuint8_t low_u8x) { svbool_t const pg_b8x = svptrue_b8(); svuint8_t const n4_u8x = svand_n_u8_x(pg_b8x, plane_off_u8x, 0x0F); svuint8_t const n3_u8x = svand_n_u8_x(pg_b8x, svlsr_n_u8_x(pg_b8x, high_u8x, 4), 0x0F); @@ -89,7 +89,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_astral_class_sve2_(svuint8_t plane_o /** @brief Predicate -> 64-bit lane mask bridge: each active lane raises bit `lane & 7` inside its byte, the byte * groups OR-fold within every 64-bit element (bit sets are disjoint, so shifts never carry), and the * per-element mask bytes recombine through one shifted `svaddv` — no stack round-trip, no lane loop. */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_pred_to_u64_sve2_(svbool_t p_b8x, sz_size_t loaded) { +SZ_HELPER_INLINE sz_u64_t sz_utf8_pred_to_u64_sve2_(svbool_t p_b8x, sz_size_t loaded) { svbool_t const pg_b8x = svptrue_b8(); svbool_t const pg_b64x = svptrue_b64(); svbool_t const loaded_b8x = svwhilelt_b8_u64(0, (sz_u64_t)loaded); @@ -106,7 +106,7 @@ SZ_HELPER_AUTO sz_u64_t sz_utf8_pred_to_u64_sve2_(svbool_t p_b8x, sz_size_t load /** @brief 64-bit lane mask -> predicate bridge, the inverse of @ref sz_utf8_pred_to_u64_sve2_: each lane reads its * mask byte via `svtbl` over the broadcast mask and tests bit `lane & 7` — no stack round-trip. */ -SZ_HELPER_AUTO svbool_t sz_utf8_u64_to_pred_sve2_(sz_u64_t mask, sz_size_t loaded) { +SZ_HELPER_INLINE svbool_t sz_utf8_u64_to_pred_sve2_(sz_u64_t mask, sz_size_t loaded) { svbool_t const pg_b8x = svptrue_b8(); svuint8_t const iota_u8x = svindex_u8(0, 1); svuint8_t const mask_bytes_u8x = svtbl_u8(svreinterpret_u8_u64(svdup_n_u64(mask)), @@ -123,8 +123,8 @@ typedef struct sz_utf8_word_window_sve2_t { } sz_utf8_word_window_sve2_t; /** @brief Decode one window into per-lane (high, low, plane_off) substrate bytes + the codepoint partition. */ -SZ_HELPER_AUTO sz_utf8_word_window_sve2_t sz_utf8_word_decode_window_sve2_( // - sz_u8_t const *text, sz_size_t available, int at_end_of_text, // +SZ_HELPER_INLINE sz_utf8_word_window_sve2_t sz_utf8_word_decode_window_sve2_( // + sz_u8_t const *text, sz_size_t available, int at_end_of_text, // sz_u8_t *high_out, sz_u8_t *low_out, sz_u8_t *plane_off_out) { svbool_t const pg_b8x = svptrue_b8(); @@ -204,7 +204,7 @@ SZ_HELPER_AUTO sz_utf8_word_window_sve2_t sz_utf8_word_decode_window_sve2_( // } /** @brief Classify each lane of one window into its Word_Break class byte (ASCII/BMP/astral, forced-Other). */ -SZ_HELPER_AUTO void sz_utf8_word_classify_window_sve2_( // +SZ_HELPER_INLINE void sz_utf8_word_classify_window_sve2_( // sz_u8_t const *high, sz_u8_t const *low, sz_u8_t const *plane_off, // sz_u64_t four_byte_starts, sz_u64_t forced_other, sz_size_t loaded, sz_u8_t *out) { svbool_t const loaded_b8x = svwhilelt_b8_u64(0, (sz_u64_t)loaded); @@ -238,7 +238,7 @@ SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_lane_dn1_sve2_(svuint8_t a_u8x) { #define sz_utf8_word_break_lane_up_sve2_(v, k) svrev_u8(svext_u8(svrev_u8((v)), svdup_n_u8(0), (k))) #define sz_utf8_word_break_lane_dn_sve2_(v, k) svext_u8((v), svdup_n_u8(0), (k)) -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_fill_right_sve2_(svuint8_t seed_u8x, svuint8_t gate_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_fill_right_sve2_(svuint8_t seed_u8x, svuint8_t gate_u8x) { svbool_t const pg_b8x = svptrue_b8(); svuint8_t bits_u8x = seed_u8x, reach_u8x = gate_u8x; bits_u8x = svorr_u8_x(pg_b8x, bits_u8x, @@ -260,7 +260,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_fill_right_sve2_(svuint8_t seed_u8x, svand_u8_x(pg_b8x, sz_utf8_word_break_lane_up_sve2_(bits_u8x, 32), reach_u8x)); return bits_u8x; } -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_fill_left_sve2_(svuint8_t seed_u8x, svuint8_t gate_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_fill_left_sve2_(svuint8_t seed_u8x, svuint8_t gate_u8x) { svbool_t const pg_b8x = svptrue_b8(); svuint8_t bits_u8x = seed_u8x, reach_u8x = gate_u8x; bits_u8x = svorr_u8_x(pg_b8x, bits_u8x, @@ -282,15 +282,15 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_fill_left_sve2_(svuint8_t seed_u8x, svand_u8_x(pg_b8x, sz_utf8_word_break_lane_dn_sve2_(bits_u8x, 32), reach_u8x)); return bits_u8x; } -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_smear_right_sve2_(svuint8_t bits_u8x, svuint8_t reach_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_smear_right_sve2_(svuint8_t bits_u8x, svuint8_t reach_u8x) { svbool_t const pg_b8x = svptrue_b8(); for (int s = 0; s < sz_utf8_word_break_smear_steps_k; ++s) bits_u8x = svorr_u8_x(pg_b8x, bits_u8x, svand_u8_x(pg_b8x, sz_utf8_word_break_lane_up1_sve2_(bits_u8x), reach_u8x)); return bits_u8x; } -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_ri_join_sve2_(svuint8_t ri_u8x, svuint8_t run_gate_u8x, int inbound_parity, - svuint8_t *inclusive_out_u8x) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_ri_join_sve2_(svuint8_t ri_u8x, svuint8_t run_gate_u8x, + int inbound_parity, svuint8_t *inclusive_out_u8x) { svbool_t const pg_b8x = svptrue_b8(); svuint8_t bits_u8x = ri_u8x, reach_u8x = run_gate_u8x; bits_u8x = sveor_u8_x(pg_b8x, bits_u8x, @@ -331,7 +331,7 @@ SZ_HELPER_INLINE sz_u8_t sz_utf8_word_break_at_last_sve2_(svuint8_t m_u8x, svuin } SZ_HELPER_INLINE sz_u8_t sz_utf8_word_break_lane0_sve2_(svuint8_t v_u8x) { return svlasta_u8(svpfalse_b(), v_u8x); } -SZ_HELPER_AUTO sz_size_t sz_utf8_word_break_decide_window_sve2_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_word_break_decide_window_sve2_( // svuint8_t class_aletter_in_u8x, svuint8_t class_hebrew_in_u8x, svuint8_t class_numeric_in_u8x, // svuint8_t class_katakana_in_u8x, svuint8_t class_extendnumlet_in_u8x, svuint8_t class_extend_in_u8x, // svuint8_t class_zwj_in_u8x, svuint8_t class_format_in_u8x, svuint8_t class_midletter_in_u8x, // @@ -689,8 +689,8 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_word_break_decide_window_sve2_( * `sz_utf8_word_break_range16_one_neon_` (not-below: high greater, or equal-high with low at least lo; * symmetric for not-above). Used by the WSegSpace / Extended_Pictographic scans inside @ref * sz_utf8_word_break_resolve_window_sve2_. */ -SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_range16_sve2_(svuint8_t high_u8x, svuint8_t low_u8x, sz_u16_t const *lo_t, - sz_u16_t const *hi_t, int count) { +SZ_HELPER_INLINE svuint8_t sz_utf8_word_break_range16_sve2_(svuint8_t high_u8x, svuint8_t low_u8x, sz_u16_t const *lo_t, + sz_u16_t const *hi_t, int count) { svbool_t const pg_b8x = svptrue_b8(); svbool_t acc_b8x = svpfalse_b(); for (int i = 0; i < count; ++i) { @@ -711,7 +711,7 @@ SZ_HELPER_AUTO svuint8_t sz_utf8_word_break_range16_sve2_(svuint8_t high_u8x, sv * engine. Builds the 15 Word_Break class masks + WSegSpace / Extended_Pictographic / quote bytes + * partition masks, then decides for @p complete_limit and re-resolves the carry to @p adv when an open * bridge is still undecided at the edge -- the two-edge carry logic that mirrors the NEON driver. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_word_break_resolve_window_sve2_( // +SZ_HELPER_INLINE sz_size_t sz_utf8_word_break_resolve_window_sve2_( // sz_u8_t const *raw, sz_u8_t const *high_a, sz_u8_t const *low_a, sz_u8_t const *plane_a, sz_u8_t const *cls_a, // sz_u64_t start_bytes_all, sz_u64_t length_two, sz_u64_t length_three, sz_u64_t length_four, // sz_u64_t continuation_all, sz_u64_t forced_other, sz_u64_t four_byte_starts, // @@ -858,9 +858,9 @@ SZ_HELPER_AUTO sz_size_t sz_utf8_word_break_resolve_window_sve2_( /** @brief In-register boundary drain over a full byte-vector of lanes: each 32-bit quarter of @p boundary_b8x is * compacted, widened to absolute 64-bit positions, and chained through the carried open `word_start` with an * `svinsr` shift-in, so consecutive boundaries become (start, length) pairs without a stack round-trip. */ -SZ_HELPER_AUTO sz_size_t sz_utf8_word_drain_sve2_(svbool_t boundary_b8x, sz_size_t base, sz_size_t *starts, - sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, - sz_size_t *word_start_io) { +SZ_HELPER_INLINE sz_size_t sz_utf8_word_drain_sve2_(svbool_t boundary_b8x, sz_size_t base, sz_size_t *starts, + sz_size_t *lengths, sz_size_t produced, sz_size_t capacity, + sz_size_t *word_start_io) { svbool_t const pg_b32x = svptrue_b32(); svbool_t const pg_b64x = svptrue_b64(); sz_size_t const quarter_lanes = svcntw(); diff --git a/include/stringzilla/utf8_wordbreaks/v128.h b/include/stringzilla/utf8_wordbreaks/v128.h index 5169697d..de7f8ac4 100644 --- a/include/stringzilla/utf8_wordbreaks/v128.h +++ b/include/stringzilla/utf8_wordbreaks/v128.h @@ -56,7 +56,7 @@ SZ_HELPER_INLINE v128_t sz_utf8_word_break_byte_mask_from_bits_v128_(sz_u64_t bi * read by a bounded scalar L1 walk over `raw & 0x7F` (v128's `wasm_i8x16_swizzle` reaches only 16 B, so * NEON's in-register `vqtbl4q_u8` read does not port). The window byte equals the codepoint on ASCII lanes. * Addresses ONE quarter. */ -SZ_HELPER_AUTO v128_t sz_utf8_word_break_ascii_class_v128_(v128_t bytes_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_word_break_ascii_class_v128_(v128_t bytes_u8x16) { sz_align_(16) sz_u8_t index_lanes[16], class_lanes[16]; wasm_v128_store(index_lanes, wasm_v128_and(bytes_u8x16, wasm_i8x16_splat(0x7F))); for (int lane = 0; lane < 16; ++lane) class_lanes[lane] = sz_utf8_word_break_property_ascii_[index_lanes[lane]]; @@ -67,7 +67,7 @@ SZ_HELPER_AUTO v128_t sz_utf8_word_break_ascii_class_v128_(v128_t bytes_u8x16) { * page-compressed table via @ref sz_utf8_rune_flat_lookup_v128_, the v128 twin of * @ref sz_utf8_word_break_bmp_class_neon_. Bit-exact with `sz_rune_word_break_property` over the whole BMP. * Addresses ONE quarter. */ -SZ_HELPER_AUTO v128_t sz_utf8_word_break_bmp_class_v128_(v128_t high_bytes_u8x16, v128_t low_bytes_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_word_break_bmp_class_v128_(v128_t high_bytes_u8x16, v128_t low_bytes_u8x16) { return sz_utf8_rune_flat_lookup_v128_(sz_utf8_word_break_bmp_page_lut_, sz_utf8_word_break_flat_bmp_, (int)sz_utf8_word_break_flat_pages_k, high_bytes_u8x16, low_bytes_u8x16); } @@ -77,8 +77,8 @@ SZ_HELPER_AUTO v128_t sz_utf8_word_break_bmp_class_v128_(v128_t high_bytes_u8x16 * @p plane_off_u8x16 = (offset>>16)&0xFF (low nibble meaningful), @p high_u8x16 = (offset>>8)&0xFF, * @p low_u8x16 = offset&0xFF. Bit-exact with `sz_rune_word_break_property` over the Supplementary Planes. * Addresses ONE quarter. */ -SZ_HELPER_AUTO v128_t sz_utf8_word_break_astral_class_v128_(v128_t plane_off_u8x16, v128_t high_u8x16, - v128_t low_u8x16) { +SZ_HELPER_INLINE v128_t sz_utf8_word_break_astral_class_v128_(v128_t plane_off_u8x16, v128_t high_u8x16, + v128_t low_u8x16) { v128_t const low_nibble_mask_u8x16 = wasm_i8x16_splat(0x0F); v128_t const nibble4_u8x16 = wasm_v128_and(plane_off_u8x16, low_nibble_mask_u8x16); v128_t const nibble3_u8x16 = wasm_v128_and(wasm_u8x16_shr(high_u8x16, 4), low_nibble_mask_u8x16); @@ -118,9 +118,9 @@ SZ_HELPER_AUTO v128_t sz_utf8_word_break_astral_class_v128_(v128_t plane_off_u8x * at the 2-/3-byte start lanes: the NEON `bmp_compact` start-gather needs `ctz` (banned here), and the flat * lookup is index-safe on any byte, so the maskless full-window read matches the RVV backend and stays * bit-exact at every start lane (every other lane is a don't-care left at zero, exactly as NEON leaves it). */ -SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_v128_( // +SZ_HELPER_INLINE void sz_utf8_word_break_classify_window_v128_( // sz_utf8_rune_window_v128_t window, v128_t *classes_u8x16) { - v128_t const *raw_u8x16 = window.window; + v128_t const *raw_u8x16 = window.window_u8x16s; sz_u64_t const ascii_starts = window.codepoint_starts & ~window.two_byte_starts & ~window.three_byte_starts & ~window.four_byte_starts; sz_u64_t const bmp_starts = window.two_byte_starts | window.three_byte_starts; @@ -147,8 +147,8 @@ SZ_HELPER_AUTO void sz_utf8_word_break_classify_window_v128_( // if (bmp_starts) { v128_t const bmp_select_u8x16 = sz_utf8_word_break_byte_mask_from_bits_v128_(bmp_starts, lane_base); class_bytes_u8x16 = wasm_v128_bitselect( - sz_utf8_word_break_bmp_class_v128_(window.high[quarter], window.low[quarter]), class_bytes_u8x16, - bmp_select_u8x16); + sz_utf8_word_break_bmp_class_v128_(window.high_byte_u8x16s[quarter], window.low_byte_u8x16s[quarter]), + class_bytes_u8x16, bmp_select_u8x16); } // 4-byte (astral) lanes: reconstruct the codepoint from the lead + three forward neighbours, then the astral @@ -231,7 +231,7 @@ SZ_HELPER_INLINE v128_t sz_utf8_word_break_range16_one_v128_(v128_t high_u8x16, /** @brief A 64-bit "(high,low) 16-bit value in any sorted `[lo, hi]` range" lane mask over the four window quarters, * the v128 twin of @ref sz_utf8_word_break_range16_mask_neon_ (WSegSpace / Extended_Pictographic). */ -SZ_HELPER_AUTO sz_u64_t sz_utf8_word_break_range16_mask_v128_( // +SZ_HELPER_INLINE sz_u64_t sz_utf8_word_break_range16_mask_v128_( // v128_t const *high_u8x16, v128_t const *low_u8x16, sz_u16_t const *lo_table, sz_u16_t const *hi_table, int count) { v128_t hit_u8x16[4] = {wasm_i8x16_splat(0), wasm_i8x16_splat(0), wasm_i8x16_splat(0), wasm_i8x16_splat(0)}; for (int range = 0; range < count; ++range) @@ -255,7 +255,7 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_v128_ sz_size_t const loaded = window.loaded; sz_u64_t const valid = sz_u64_mask_until_serial_(loaded); sz_u64_t const start_bytes = start_bytes_all & valid; - v128_t const *raw_u8x16 = window.window; + v128_t const *raw_u8x16 = window.window_u8x16s; // Truncated-edge U+FFFD reclassify (force the class to Other on a lead whose declared span runs past `loaded`). sz_u64_t const lead_two = length_two & start_bytes; @@ -298,8 +298,8 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_v128_ // WB3d WSegSpace raw membership: the ASCII U+0020 byte compare OR the multibyte (high,low) range scan. sz_u64_t wseg_multibyte = 0ull; if (non_ascii_lanes) - wseg_multibyte = sz_utf8_word_break_range16_mask_v128_(window.high, window.low, sz_utf8_word_break_wseg_lo_, - sz_utf8_word_break_wseg_hi_, + wseg_multibyte = sz_utf8_word_break_range16_mask_v128_(window.high_byte_u8x16s, window.low_byte_u8x16s, + sz_utf8_word_break_wseg_lo_, sz_utf8_word_break_wseg_hi_, sz_utf8_word_break_wseg_count_k) & non_ascii_lanes; frame.wseg = (wseg_multibyte | (sz_utf8_word_break_byte_equal_v128_(raw_u8x16, 0x20) & valid)); @@ -336,8 +336,8 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_v128_ wasm_i8x16_eq(plane_q_u8x16[0], one_u8x16), wasm_i8x16_eq(plane_q_u8x16[1], one_u8x16), wasm_i8x16_eq(plane_q_u8x16[2], one_u8x16), wasm_i8x16_eq(plane_q_u8x16[3], one_u8x16)); sz_u64_t const pictographic_bmp = sz_utf8_word_break_range16_mask_v128_( - window.high, window.low, sz_utf8_word_break_pict_bmp_lo_, sz_utf8_word_break_pict_bmp_hi_, - sz_utf8_word_break_pict_bmp_count_k); + window.high_byte_u8x16s, window.low_byte_u8x16s, sz_utf8_word_break_pict_bmp_lo_, + sz_utf8_word_break_pict_bmp_hi_, sz_utf8_word_break_pict_bmp_count_k); sz_u64_t const pictographic_smp = sz_utf8_word_break_range16_mask_v128_( smp_high_u8x16, smp_low_u8x16, sz_utf8_word_break_pict_smp_lo_, sz_utf8_word_break_pict_smp_hi_, sz_utf8_word_break_pict_smp_count_k); @@ -357,9 +357,9 @@ SZ_HELPER_INLINE sz_utf8_word_break_frame_t sz_utf8_word_break_build_frame_v128_ /** @brief Resolve one window into the maximal-subpart partition — the v128 twin of * @ref sz_utf8_word_break_partition_neon_: compute the per-ISA `sz_u64_t` masks and delegate to the portable * @ref sz_utf8_word_break_partition_from_masks_. */ -SZ_HELPER_AUTO sz_utf8_word_break_partition_t sz_utf8_word_break_partition_v128_(sz_utf8_rune_window_v128_t window, - sz_u64_t valid, int at_end_of_text) { - v128_t const *raw_u8x16 = window.window; +SZ_HELPER_INLINE sz_utf8_word_break_partition_t sz_utf8_word_break_partition_v128_(sz_utf8_rune_window_v128_t window, + sz_u64_t valid, int at_end_of_text) { + v128_t const *raw_u8x16 = window.window_u8x16s; sz_u64_t const real_continuation = window.continuation & valid; // Declared length follows the serial high-nibble rule: 0xC/0xD → 2, 0xE → 3, 0xF → 4. The strict // `two`/`three_byte_starts` masks already match 0xC0-0xDF and 0xE0-0xEF; only `length_four` needs widening to fold diff --git a/include/stringzillas/README.md b/include/stringzillas/README.md index c2e80920..479e4cf5 100644 --- a/include/stringzillas/README.md +++ b/include/stringzillas/README.md @@ -113,8 +113,8 @@ Each engine exposes a call on each layout, for example `szs_levenshtein_distance ### Unified Memory -For zero-copy sharing between the CPU and the GPU, initialize the allocator with unified memory. -On CUDA-capable systems it uses `cudaMallocManaged`, so the same buffers are reachable from both host and device. +A GPU scope addresses its work from the device, so every buffer you hand an engine must be __device-accessible__. +That means unified memory, or plain device memory you allocated yourself. ```c sz_status_t sz_memory_allocator_init_unified(sz_memory_allocator_t *alloc, char const **error_message); @@ -122,6 +122,14 @@ void *szs_unified_alloc(sz_size_t size_bytes); void szs_unified_free(void *ptr, sz_size_t size_bytes); ``` +The rule covers __outputs as well as inputs__, and the small per-collection arrays with them. +Match counts, tape offsets, BM25 weights and lengths, and score vectors are no different from a result matrix. +Host memory is refused with `sz_device_memory_mismatch_k` and nothing is written, so a copy across the bus is yours to make rather than one the engine makes unasked. +Page-locked host memory counts as host memory here, since the driver reports it as such. + +The single-value out-parameters are the exception and take ordinary host addresses: `matches_total`, `matches_found`, `output_bytes_written`, and every `error_message`. +A CPU scope has no such requirement and reads and writes host memory throughout. + Pass the resulting `sz_memory_allocator_t *` as the `alloc` argument of any engine's `*_init`, or pass `NULL` to use the default allocator. ### The Cross Product Convention @@ -163,12 +171,8 @@ A self-contained byte-distance example over a tiny corpus: void run(void) { char const *strings[] = {"listen", "silent", "kitten"}; - sz_size_t lengths[] = {6, 6, 6}; sz_sequence_t queries; - queries.count = 3; - queries.handle = strings; - queries.get_start = NULL; // ... wire your own getters in real code - queries.get_length = NULL; + sz_sequence_from_null_terminated_strings(strings, 3, &queries); szs_device_scope_t device = NULL; char const *error = NULL; @@ -188,7 +192,6 @@ void run(void) { szs_levenshtein_distances_free(engine); szs_device_scope_free(device); - (void)lengths; } ``` @@ -261,6 +264,86 @@ void run(void) { } ``` +## Multi-Pattern Search + +The substrings engine compiles a whole needle set into one Aho-Corasick automaton and walks every haystack against all of them at once, so a dictionary of thousands of terms costs one pass rather than thousands. +Unlike the other engines it is constructed in two steps: `szs_substrings_init` picks the backend and allocates the handle, while `szs_substrings_index` compiles a needle set into it and sizes the automaton's hot and cold tiers against the cache that will walk it. +Re-indexing replaces the needle set, so one engine serves a new vocabulary or another device without being rebuilt. + +```c +sz_status_t szs_substrings_init( + sz_memory_allocator_t const *alloc, sz_capability_t capabilities, + szs_substrings_t *engine, char const **error_message); + +sz_status_t szs_substrings_index( + szs_substrings_t engine, sz_sequence_t const *needles, + szs_substrings_case_sensitivity_t case_sensitivity, szs_device_scope_t device, + char const **error_message); + +sz_status_t szs_substrings_count( + szs_substrings_t engine, szs_device_scope_t device, + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, + sz_size_t *counts, sz_size_t *matches_total, + char const **error_message); + +sz_status_t szs_substrings_find( + szs_substrings_t engine, szs_device_scope_t device, + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, + char const **error_message); + +void szs_substrings_free(szs_substrings_t engine); +``` + +`szs_substrings_cased_k` matches raw bytes and accepts any needle, including malformed UTF-8, while `szs_substrings_uncased_k` applies full Unicode case folding and requires valid UTF-8 on both sides. +Folding does not preserve byte length — needle `k` matches both the 1-byte `k` and the 3-byte Kelvin sign `U+212A` — which is why every `szs_substrings_match_t` carries its own `byte_length` rather than borrowing the needle's. + +The overlap policy decides how matches that collide are resolved: `szs_substrings_overlapping_k` reports all of them, while `szs_substrings_leftmost_longest_k` and `szs_substrings_leftmost_first_k` each keep one non-overlapping cover. +`szs_substrings_find` follows the size-query convention — passing `matches_capacity == 0` writes nothing and returns `sz_unexpected_dimensions_k` with `matches_found` holding the count a full call would need. + +Two more operations share the same automaton. +`szs_substrings_score_bm25` treats the dictionary itself as the query, taking one `needle_weights` entry per needle and returning one score per haystack; its term frequencies are raw overlapping counts, which is classic BM25, so no overlap policy applies. +`szs_substrings_replace_u32tape` and its 64-bit sibling rewrite every haystack, substituting each match with its needle's replacement, and accept only the tape layouts, since a rewrite's product is itself a tape. +Size their `output_data` with `szs_substrings_replace_bound`, which answers from the needle set alone and needs no haystacks, no walk, and no device. + +```c +#include <assert.h> +#include <stringzillas/stringzillas.h> + +void multi_pattern_example(void) { + char const *error = NULL; + szs_device_scope_t device = NULL; + sz_status_t status = szs_device_scope_init_default(&device, &error); + assert(status == sz_success_k); + + char const *needle_texts[] = {"he", "she", "his", "hers"}; + sz_size_t const needle_count = 4; + sz_sequence_t needles; + sz_sequence_from_null_terminated_strings(needle_texts, needle_count, &needles); + + szs_substrings_t engine = NULL; + status = szs_substrings_init(NULL, sz_cap_serial_k, &engine, &error); + assert(status == sz_success_k); + status = szs_substrings_index(engine, &needles, szs_substrings_cased_k, device, &error); + assert(status == sz_success_k); + + char const *haystack_texts[] = {"ushers", "hershey"}; + sz_sequence_t haystacks; + sz_sequence_from_null_terminated_strings(haystack_texts, 2, &haystacks); + + sz_size_t counts[2], matches_total = 0; + status = szs_substrings_count(engine, device, &haystacks, szs_substrings_overlapping_k, // + counts, &matches_total, &error); + assert(status == sz_success_k); + assert(matches_total == 7); + + szs_substrings_free(engine); + szs_device_scope_free(device); +} +``` + +On a GPU scope every buffer above is device-accessible, `counts` and the BM25 weights included, as [Unified Memory](#unified-memory) spells out; `matches_total`, `matches_found`, and `output_bytes_written` stay ordinary host addresses. + ## Fingerprints The fingerprints engine sketches each string into a fixed-width MinHash plus a Count-Min-Sketch, so two near-duplicate documents land on overlapping hash dimensions even after small edits. @@ -310,26 +393,32 @@ void run(void) { NULL, szs_capabilities(), &engine, &error); assert(status == sz_success_k); - char const *docs[] = {"the quick brown fox", "the quick brown dog"}; - sz_size_t lengths[] = {19, 19}; - sz_sequence_t texts; + // One tape of two documents, and both output arrays, in memory either backend can reach. + char *data = (char *)szs_unified_alloc(38); + sz_u32_t *offsets = (sz_u32_t *)szs_unified_alloc(3 * sizeof(sz_u32_t)); + sz_u32_t *hashes = (sz_u32_t *)szs_unified_alloc(2 * 256 * sizeof(sz_u32_t)); + sz_u32_t *counts = (sz_u32_t *)szs_unified_alloc(2 * 256 * sizeof(sz_u32_t)); + memcpy(data, "the quick brown foxthe quick brown dog", 38); + offsets[0] = 0, offsets[1] = 19, offsets[2] = 38; + + sz_sequence_u32tape_t texts; + texts.data = data; + texts.offsets = offsets; texts.count = 2; - texts.handle = docs; - texts.get_start = NULL; - texts.get_length = NULL; - sz_u32_t hashes[2 * 256]; - sz_u32_t counts[2 * 256]; - status = szs_fingerprints_sequence( + status = szs_fingerprints_u32tape( engine, device, &texts, hashes, 256 * sizeof(sz_u32_t), counts, 256 * sizeof(sz_u32_t), &error); assert(status == sz_success_k); + szs_unified_free(data, 38); + szs_unified_free(offsets, 3 * sizeof(sz_u32_t)); + szs_unified_free(hashes, 2 * 256 * sizeof(sz_u32_t)); + szs_unified_free(counts, 2 * 256 * sizeof(sz_u32_t)); szs_fingerprints_free(engine); szs_device_scope_free(device); - (void)lengths; } ``` @@ -345,7 +434,7 @@ The device scope is the single knob for __where__ and __how widely__ an engine r A CPU slice spreads the cross-product (or the corpus of texts) across a thread pool, so you can reserve cores for other work by asking for fewer than all of them. A GPU device offloads the whole batch to one CUDA device, where each engine routes string pairs into size-tiered kernels. -Unified memory is what makes the GPU path seamless: allocate inputs and outputs through the unified allocator, and the same pointers are valid on host and device with no explicit copies. +Every buffer a GPU scope touches must be device-accessible, which the __Unified Memory__ section above states in full. The underlying C++ engines are templated on an __executor__: `dummy_executor_t` runs serially, and `forkunion_executor_t` is the preferred library-grade thread pool, wrapping a [ForkUnion](https://github.com/ashvardanian/ForkUnion) pool through its C API so the compiled runtime handles NUMA-aware placement. The C ABI hides this choice behind the device scope, picking the right executor for the cores or GPU you requested. diff --git a/include/stringzillas/fingerprints/cuda.cuh b/include/stringzillas/fingerprints/cuda.cuh index 6be8c84a..20e26deb 100644 --- a/include/stringzillas/fingerprints/cuda.cuh +++ b/include/stringzillas/fingerprints/cuda.cuh @@ -23,9 +23,27 @@ namespace stringzillas { #pragma region CUDA Device Helpers +/** + * @brief Refuses a batch whose tape or output rows no kernel can reach, before any task is described. + * + * One probe per allocation rather than per text: the tape is contiguous and both output arrays are, so the + * first element of each decides for the batch. An empty batch reads and writes nothing and is accepted. + */ +template <typename texts_type_, typename min_hashes_per_text_type_, typename min_counts_per_text_type_> +inline status_t check_fingerprints_memory(texts_type_ const &texts, min_hashes_per_text_type_ &&min_hashes_per_text, + min_counts_per_text_type_ &&min_counts_per_text) noexcept { + if (texts.size() == 0) return status_t::success_k; + if (status_t const reachable = check_device_accessible_sequence(texts); reachable != status_t::success_k) + return reachable; + if (status_t const reachable = check_device_accessible_memory(to_span(min_hashes_per_text[0])); + reachable != status_t::success_k) + return reachable; + return check_device_accessible_memory(to_span(min_counts_per_text[0])); +} + /** * @brief Wraps a single task for the CUDA-based @b byte-level "fingerprint" kernels. - * @note Used to allow sorting/grouping inputs to differentiate device-wide and warp-wide tasks. + * @note Tasks are consumed in the order they were built, so nothing here records where a text came from. */ template <typename char_type_, typename min_hash_type_ = u32_t, typename min_count_type_ = u32_t> struct cuda_fingerprint_task { @@ -35,7 +53,6 @@ struct cuda_fingerprint_task { char_t const *text_ptr = nullptr; size_t text_length = 0; - size_t original_index = 0; min_hash_t *min_hashes = nullptr; min_count_t *min_counts = nullptr; warp_tasks_density_t density = warps_working_together_k; // ? Worst case, we have to sync final writes @@ -602,21 +619,12 @@ struct basic_rolling_hashers<hasher_type_, min_hash_type_, min_count_type_, unif // container-dependent step; the device kernels and timing then run from the container-independent `run()`. tasks_.clear(); auto &tasks = tasks_; - if (tasks.try_resize(texts.size()) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; - - // Ensure device-accessible buffers (Unified/Device memory) for inputs and outputs. Both the input - // texts (one contiguous tape) and the output fingerprints (contiguous arrays) live in single - // allocations, so probing every element with `cudaPointerGetAttributes` - a per-pointer driver - // round-trip - would cost three driver calls per text. We validate the base pointers of the first - // element once, which covers the whole tape and both output arrays. - if (texts.size()) { - auto first_min_hashes = to_span(min_hashes_per_text[0]); - auto first_min_counts = to_span(min_counts_per_text[0]); - if (!is_device_accessible_memory((void const *)texts[0].data()) || - !is_device_accessible_memory((void const *)first_min_hashes.data()) || - !is_device_accessible_memory((void const *)first_min_counts.data())) - return {status_t::device_memory_mismatch_k, cudaSuccess}; - } + // Every task is fully overwritten below, so constructing them first would write the array twice. + if (tasks.try_resize_uninitialized(texts.size()) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; + + if (status_t const reachable = check_fingerprints_memory(texts, min_hashes_per_text, min_counts_per_text); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; for (size_t task_index = 0; task_index < texts.size(); ++task_index) { auto const &text = texts[task_index]; @@ -625,7 +633,6 @@ struct basic_rolling_hashers<hasher_type_, min_hash_type_, min_count_type_, unif tasks[task_index] = device_task_t { .text_ptr = reinterpret_cast<byte_t const *>(text.data()), .text_length = text.size(), - .original_index = task_index, .min_hashes = min_hashes.data(), .min_counts = min_counts.data(), .density = four_warps_per_multiprocessor_k, @@ -837,6 +844,13 @@ struct floating_rolling_hashers<sz_cap_cuda_k, dimensions_> { sz_unused_(specs); + if (status_t const reachable = check_device_accessible_memory(text); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(min_hashes); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(min_counts); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + // Create the engine-owned timing events on first use; the kernel table resolves itself on first access. CUresult timer_error = timer_.ensure_created(executor.device_id()); if (timer_error != CUDA_SUCCESS) return make_cuda_status(timer_error); @@ -844,12 +858,11 @@ struct floating_rolling_hashers<sz_cap_cuda_k, dimensions_> { // Populate the tasks array with a single task for the entire device, reusing the hoisted buffer. tasks_.clear(); auto &tasks = tasks_; - if (tasks.try_resize(1) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; + if (tasks.try_resize_uninitialized(1) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; tasks[0] = device_task_t { .text_ptr = text.data(), .text_length = text.size(), - .original_index = 0, .min_hashes = min_hashes.data(), .min_counts = min_counts.data(), .density = one_warp_per_multiprocessor_k, @@ -920,7 +933,13 @@ struct floating_rolling_hashers<sz_cap_cuda_k, dimensions_> { // container-dependent step; the device kernel and timing then run from the container-independent `run()`. tasks_.clear(); auto &tasks = tasks_; - if (tasks.try_resize(texts.size()) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; + // Every task is fully overwritten below, so constructing them first would write the array twice. + if (tasks.try_resize_uninitialized(texts.size()) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; + + if (status_t const reachable = check_fingerprints_memory(texts, min_hashes_per_text, min_counts_per_text); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; + for (size_t task_index = 0; task_index < texts.size(); ++task_index) { auto const &text = texts[task_index]; auto min_hashes = to_span(min_hashes_per_text[task_index]); @@ -928,7 +947,6 @@ struct floating_rolling_hashers<sz_cap_cuda_k, dimensions_> { tasks[task_index] = device_task_t { .text_ptr = reinterpret_cast<byte_t const *>(text.data()), .text_length = text.size(), - .original_index = task_index, .min_hashes = min_hashes.data(), .min_counts = min_counts.data(), .density = four_warps_per_multiprocessor_k, diff --git a/include/stringzillas/fingerprints/serial.hpp b/include/stringzillas/fingerprints/serial.hpp index d411d461..8577044a 100644 --- a/include/stringzillas/fingerprints/serial.hpp +++ b/include/stringzillas/fingerprints/serial.hpp @@ -16,10 +16,11 @@ #include "stringzillas/types.hpp" // `sz::executor_like` #include <cstddef> -#include <limits> // `std::numeric_limits` for numeric types -#include <iterator> // `std::iterator_traits` for iterators -#include <cmath> // `std::fabsf` for `f32_rolling_hasher` -#include <numeric> // `std::gcd` for `choose_coprime_modulo` +#include <limits> // `std::numeric_limits` for numeric types +#include <iterator> // `std::iterator_traits` for iterators +#include <cmath> // `std::fabsf` for `f32_rolling_hasher` +#include <numeric> // `std::gcd` for `choose_coprime_modulo` +#include <type_traits> // `std::is_unsigned` for the rolling hashers' wraparound requirement namespace ashvardanian { namespace stringzillas { @@ -58,6 +59,10 @@ struct multiplying_rolling_hasher { using state_t = hash_type_; using hash_t = hash_type_; + // Wraparound is the modulo here, and only unsigned types define it - a signed state would make every + // multiply undefined at the width it is meant to overflow. + static_assert(std::is_unsigned<hash_t>::value, "A hash that relies on wraparound must be unsigned"); + explicit multiplying_rolling_hasher(size_t window_width, hash_t multiplier = static_cast<hash_t>(257)) noexcept : window_width_ {window_width}, multiplier_ {multiplier}, highest_power_ {1} { @@ -197,6 +202,10 @@ struct buz_rolling_hasher { using state_t = hash_type_; using hash_t = hash_type_; + // The rotation shifts both ways, and only unsigned types define either at full width - a signed left + // shift shifts a sign bit out, and a signed right shift replicates it instead of feeding in zeros. + static_assert(std::is_unsigned<hash_t>::value, "A hash that rotates its state must be unsigned"); + constexpr buz_rolling_hasher() noexcept : window_width_ {0}, table_ {} {} explicit buz_rolling_hasher(size_t window_width, u64_t seed = 0x9E3779B97F4A7C15ull) noexcept diff --git a/include/stringzillas/similarities/cuda.cuh b/include/stringzillas/similarities/cuda.cuh index 57edf5f5..92474c13 100644 --- a/include/stringzillas/similarities/cuda.cuh +++ b/include/stringzillas/similarities/cuda.cuh @@ -60,6 +60,24 @@ using affine_smith_waterman_hopper_t = #pragma region Common Helpers +/** + * @brief Refuses a cross-product whose inputs or result matrix no kernel can reach. + * + * One probe per allocation rather than per cell: both sides come from contiguous tapes, and the matrix is + * one region `rows * row_stride` wide - the widest a cell offset can name, since a caller may embed a + * narrower matrix in a wider allocation. + */ +template <typename queries_type_, typename candidates_type_, typename results_type_> +inline status_t check_similarities_memory(queries_type_ const &queries, candidates_type_ const &candidates, + results_type_ const &results) noexcept { + using results_value_t = typename std::remove_reference<results_type_>::type::value_type; + if (status_t const reachable = check_device_accessible_sequence(queries); reachable != status_t::success_k) + return reachable; + if (status_t const reachable = check_device_accessible_sequence(candidates); reachable != status_t::success_k) + return reachable; + return check_device_accessible_memory(span<results_value_t> {results.data, results.rows * results.row_stride}); +} + /** * @brief Dispatches min or max operation based on the compile-time objective. */ @@ -1762,10 +1780,6 @@ struct cuda_cross_buffers { */ safe_vector<u32_t, device_alloc<u32_t>> rune_offsets_ {}; - /** @brief Dense results staging for the host-output scatter fallback: the kernel writes the row-major matrix here - * and one `cuMemcpy2DAsync` strides it out. Byte-typed since the element width varies per call. */ - safe_vector<std::byte, device_alloc<std::byte>> results_staging_ {}; - cuda_cross_buffers() noexcept = default; cuda_cross_buffers(cuda_cross_buffers const &) = delete; @@ -1939,7 +1953,8 @@ cuda_status_t cuda_route_tasks_into_tiers_(buffers_type_ &buffers, rle_scratch_t executor.stream()); if (hist_status.status != status_t::success_k) return hist_status; cuda_status_t const scan_status = cuda_launch_exclusive_sum_( - scan_u32_shape, dense_tier_counts, static_cast<size_t>(tier_count), bucket_cursors, executor.stream()); + exclusive_sum_shapes_t {scan_u32_shape}, dense_tier_counts, static_cast<size_t>(tier_count), bucket_cursors, + span<u32_t> {}, gpu_specs_t {}, executor.stream()); if (scan_status.status != status_t::success_k) return scan_status; cuda_status_t const scatter_status = cuda_launch_scatter_tasks_by_bucket_( router_scatter_shape, buffers.tasks_.data(), count, dense_tier_functor, bucket_cursors, @@ -2152,74 +2167,6 @@ __global__ void similarity_scatter_results_(task_type_ const *tasks, size_t task if (task.mirror_offset != task.result_offset) results[task.mirror_offset] = value; } -/** - * @brief Host-output (non-device-accessible) scatter fallback shared by every CUDA cross-product engine: instead of a - * per-cell host loop, the device @ref similarity_scatter_results_ kernel writes the full row-major matrix into a - * hoisted device-resident staging buffer (laid out at the caller's `row_stride`, so each task's precomputed - * `result_offset` / `mirror_offset` stays valid), then a single strided `cudaMemcpy2DAsync` copies the valid - * `rows x columns` region into the caller's host `strided_rows`. Fully stream-async; staging grows-and-reuses. - * - * @param buffers Engine buffer bundle holding the grow-only `results_staging_` device buffer (passed by reference). - * @param tasks Device-resident task array (already scored); read-only. - * @param tasks_count Number of live tasks. - * @param results The caller's host-side strided output matrix. - */ -template <typename task_type_, typename value_type_, typename buffers_task_type_> -cuda_status_t cuda_scatter_results_to_host_strided_(cuda_cross_buffers<buffers_task_type_> &buffers, - task_type_ const *tasks, size_t tasks_count, - strided_rows<value_type_> const &results, - cuda_executor_t const &executor, unsigned block) noexcept { - - if (!tasks_count) return {status_t::success_k, cudaSuccess}; - - // Size the dense staging matrix at the caller's `row_stride` (not the tighter `columns`) so the device kernel's - // precomputed `result_offset = query_index * row_stride + candidate_index` indexes it without any remapping. - size_t const staging_elements = results.rows * results.row_stride; - if (buffers.results_staging_.try_resize_uninitialized(staging_elements * sizeof(value_type_)) == - status_t::bad_alloc_k) - return {status_t::bad_alloc_k}; - value_type_ *const staging_ptr = static_cast<value_type_ *>(static_cast<void *>(buffers.results_staging_.data())); - - kernel_shape_t scatter_shape; - cuda_status_t const scatter_resolve = resolve_kernel_shape( - scatter_shape, (void const *)&similarity_scatter_results_<task_type_, value_type_>, 256, 0, false); - if (scatter_resolve.status != status_t::success_k) return scatter_resolve; - - task_type_ const *tasks_ptr = tasks; - value_type_ *results_ptr = staging_ptr; - size_t tasks_size = tasks_count; - void *scatter_args[3] = {(void *)&tasks_ptr, (void *)&tasks_size, (void *)&results_ptr}; - unsigned const scatter_grid = static_cast<unsigned>((tasks_size + block - 1) / block); - CUresult const scatter_error = cuda_launch_t {} - .grid(scatter_grid) - .block(block) - .shared(0) - .stream(executor.stream()) - .launch(scatter_shape.function, scatter_args); - if (scatter_error != CUDA_SUCCESS) return make_cuda_status(scatter_error); - - // Strided copy: only the valid `columns`-wide prefix of each of the `rows` rows is transferred; the padding - // between `columns` and `row_stride` is skipped on both sides (matching the per-cell host loop's behavior). - size_t const valid_row_bytes = results.columns * sizeof(value_type_); - size_t const stride_bytes = results.row_stride * sizeof(value_type_); - CUDA_MEMCPY2D copy_descriptor {}; - copy_descriptor.srcMemoryType = CU_MEMORYTYPE_DEVICE; - copy_descriptor.srcDevice = (CUdeviceptr)staging_ptr; - copy_descriptor.srcPitch = stride_bytes; - copy_descriptor.dstMemoryType = CU_MEMORYTYPE_HOST; - copy_descriptor.dstHost = results.data; - copy_descriptor.dstPitch = stride_bytes; - copy_descriptor.WidthInBytes = valid_row_bytes; - copy_descriptor.Height = results.rows; - CUresult copy_error = cuMemcpy2DAsync(©_descriptor, executor.stream()); - if (copy_error != CUDA_SUCCESS) return make_cuda_status(copy_error); - { - CUresult sync_error = cuStreamSynchronize(executor.stream()); - if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); - } - return {status_t::success_k, cudaSuccess}; -} - /** * @brief Byte-wise SIMD helpers (4× `u8_t` packed in a `u32_t`) for the register-only Levenshtein kernel. * On the device they map to the `__vcmpeq4`/`__vminu4`/`__vaddus4`/`__byte_perm` video instructions; the @@ -4262,14 +4209,12 @@ struct levenshtein_distances<gap_costs_type_, allocator_type_, capability_, auto &tasks = buffers_.tasks_; if (tasks.try_resize_uninitialized(live_cells) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; - // Ensure inputs are device-accessible (Unified/Device memory). Both sides come from contiguous - // tapes/arrays, so we validate the base pointers of the first element once (covers the whole tape) - // instead of a per-cell `cudaPointerGetAttributes` driver round-trip. - if (queries_count != 0 && candidates_count != 0) { - if (!is_device_accessible_memory((void const *)queries[0].data()) || - !is_device_accessible_memory((void const *)candidates[0].data())) - return {status_t::device_memory_mismatch_k, cudaSuccess}; - } + // Inputs and outputs alike must be reachable from a kernel: materializing them is the caller's explicit + // choice, as in every other CUDA engine here. One probe per allocation, since both sides come from + // contiguous tapes and the matrix is one region. + if (status_t const reachable = check_similarities_memory(queries, candidates, results); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; // Export one task per live cell; the per-cell sizing/tiering is unchanged from the pairwise design. using diagonal_memory_requirements_t = diagonal_memory_requirements<size_t>; @@ -4282,8 +4227,8 @@ struct levenshtein_distances<gap_costs_type_, allocator_type_, capability_, // descriptors on the host and let one thread per live cell fill the O(queries*candidates) task array on // the GPU (the symmetric path maps the flat cell index into the lower triangle). The descriptor buffers // are unified, so the host writes them and the kernel reads them with no extra copy. - if (buffers_.query_descriptors_.try_resize(queries_count) == status_t::bad_alloc_k || - buffers_.candidate_descriptors_.try_resize(candidates_count) == status_t::bad_alloc_k) + if (buffers_.query_descriptors_.try_resize_uninitialized(queries_count) == status_t::bad_alloc_k || + buffers_.candidate_descriptors_.try_resize_uninitialized(candidates_count) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; for (size_t query_index = 0; query_index < queries_count; ++query_index) buffers_.query_descriptors_[query_index] = {queries[query_index].data(), queries[query_index].length()}; @@ -4294,18 +4239,34 @@ struct levenshtein_distances<gap_costs_type_, allocator_type_, capability_, if (candidate_length > max_candidate_length) max_candidate_length = candidate_length; } - // WORDS direct-score fast path: when every live cell is single-word Myers (unit-cost, both sides <= 64) and the - // result matrix is device-accessible, skip the task array, the tier sort/gather/RLE, and the result scatter - // entirely - one thread per cell reads the two strings from the descriptors, runs single-word Myers, and writes - // straight into the matrix. This removes the host-orchestration overhead that leaves the GPU >60% idle on - // tiny-token cross-products. Bit-identical to the tiered path (same Levenshtein recurrence, unique distance). + return cross_described_(queries_count, candidates_count, live_cells, max_candidate_length, row_stride, + is_symmetric, cross_kind, results, executor, specs); + } + + /** + * @brief Everything `cross_` does once the descriptors are built, with the input containers behind it. + * + * The descriptors are the type-erasure boundary, so this compiles once per result type rather than once + * per result type and input shape. + */ + template <typename results_type_> + cuda_status_t cross_described_(size_t queries_count, size_t candidates_count, size_t live_cells, + size_t max_candidate_length, size_t row_stride, bool is_symmetric, + cross_similarities_t cross_kind, results_type_ &&results, + cuda_executor_t const &executor, gpu_specs_t specs) noexcept { + auto &tasks = buffers_.tasks_; + + // WORDS direct-score fast path: when every live cell is single-word Myers (unit-cost, both sides <= 64), skip + // the task array, the tier sort/gather/RLE, and the result scatter entirely - one thread per cell reads the + // two strings from the descriptors, runs single-word Myers, and writes straight into the matrix. This removes + // the host-orchestration overhead that leaves the GPU >60% idle on tiny-token cross-products. Bit-identical to + // the tiered path (same Levenshtein recurrence, unique distance). constexpr bool is_affine_cross_k = is_same_type<gap_costs_t, affine_gap_costs_t>::value; if constexpr (!is_affine_cross_k) { bool const is_unit_cost = substituter_.match == 0 && substituter_.mismatch == 1 && gap_costs_.open_or_extend == 1; if (is_unit_cost && live_cells && cross_max_query_length_ <= levenshtein_myers_word1_cap_k && - max_candidate_length <= levenshtein_myers_word1_cap_k && - is_device_accessible_memory((void const *)results.data)) { + max_candidate_length <= levenshtein_myers_word1_cap_k) { using results_value_t = typename std::remove_reference_t<results_type_>::value_type; kernel_shape_t direct_shape; cuda_status_t const direct_resolve = resolve_kernel_shape( @@ -4378,10 +4339,10 @@ struct levenshtein_distances<gap_costs_type_, allocator_type_, capability_, cuda_status_t status = run_trampoline_(executor, specs); if (status.status != status_t::success_k) return status; - // Scatter on the device when the output is device-accessible (the common unified-memory case): the host - // then never reads the large GPU-resident task array, avoiding a full unified-memory page migration. The - // scatter kernel depends on `value_type_`, so it is resolved at the call site rather than the cached table. - if (tasks.size() && is_device_accessible_memory((void const *)results.data)) { + // Scatter on the device: the host never reads the large GPU-resident task array, avoiding a full + // unified-memory page migration. The scatter kernel depends on `value_type_`, so it is resolved at the call + // site rather than the cached table. + if (tasks.size()) { using results_value_t = typename std::remove_reference_t<results_type_>::value_type; kernel_shape_t scatter_shape; cuda_status_t scatter_resolve = resolve_kernel_shape( @@ -4405,14 +4366,6 @@ struct levenshtein_distances<gap_costs_type_, allocator_type_, capability_, if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); } } - else if (tasks.size()) { - // Host-only output: device-scatter into a dense staging matrix, then one strided `cudaMemcpy2DAsync` - // into the caller's host matrix - stream-async, no hot-path alloc. - using results_value_t = typename std::remove_reference_t<results_type_>::value_type; - cuda_status_t const fallback_status = cuda_scatter_results_to_host_strided_<task_t, results_value_t>( - buffers_, tasks.data(), tasks.size(), results, executor, 256u); - if (fallback_status.status != status_t::success_k) return fallback_status; - } return status; } @@ -4916,18 +4869,15 @@ struct levenshtein_distances_utf8<gap_costs_type_, allocator_type_, capability_, auto &tasks = buffers_.tasks_; if (tasks.try_resize_uninitialized(live_cells) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; - // Ensure inputs are device-accessible (Unified/Device memory); the base pointer of the first element of each - // contiguous tape covers the whole tape. - if (queries_count != 0 && candidates_count != 0) { - if (!is_device_accessible_memory((void const *)queries[0].data()) || - !is_device_accessible_memory((void const *)candidates[0].data())) - return {status_t::device_memory_mismatch_k, cudaSuccess}; - } + // Inputs and outputs alike must be reachable from a kernel; one probe per allocation covers each. + if (status_t const reachable = check_similarities_memory(queries, candidates, results); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; // Device-side materialization (all-pairs or symmetric): build the O(queries+candidates) descriptors on the // host, let one thread per live cell fill the O(queries*candidates) task array on the GPU. - if (buffers_.query_descriptors_.try_resize(queries_count) == status_t::bad_alloc_k || - buffers_.candidate_descriptors_.try_resize(candidates_count) == status_t::bad_alloc_k) + if (buffers_.query_descriptors_.try_resize_uninitialized(queries_count) == status_t::bad_alloc_k || + buffers_.candidate_descriptors_.try_resize_uninitialized(candidates_count) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; cross_max_query_length_ = 0; cross_max_candidate_length_ = 0; @@ -5142,9 +5092,8 @@ struct levenshtein_distances_utf8<gap_costs_type_, allocator_type_, capability_, if (execution_error != CUDA_SUCCESS) return make_cuda_status(execution_error); status.elapsed_milliseconds = timer_.elapsed_milliseconds(); - // Scatter on the device when the output is device-accessible (the common unified-memory case); otherwise scatter - // into a hoisted staging matrix and stream-copy it into the caller's host matrix. - if (tasks.size() && is_device_accessible_memory((void const *)results.data)) { + // Scatter on the device, so the host never reads the GPU-resident task array back. + if (tasks.size()) { using results_value_t = typename std::remove_reference_t<results_type_>::value_type; kernel_shape_t scatter_shape; cuda_status_t scatter_resolve = resolve_kernel_shape( @@ -5168,12 +5117,6 @@ struct levenshtein_distances_utf8<gap_costs_type_, allocator_type_, capability_, if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); } } - else if (tasks.size()) { - using results_value_t = typename std::remove_reference_t<results_type_>::value_type; - cuda_status_t const fallback_status = cuda_scatter_results_to_host_strided_<task_t, results_value_t>( - buffers_, tasks.data(), tasks.size(), results, executor, 256u); - if (fallback_status.status != status_t::success_k) return fallback_status; - } return status; } @@ -5937,19 +5880,16 @@ cuda_status_t cuda_weighted_cross_( auto &tasks = buffers.tasks_; if (tasks.try_resize_uninitialized(live_cells) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; - // Ensure inputs are device-accessible (Unified/Device memory). Both sides come from contiguous - // tapes/arrays, so we validate the base pointers of the first element once (covers the whole tape). - if (queries_count != 0 && candidates_count != 0) { - if (!is_device_accessible_memory((void const *)queries[0].data()) || - !is_device_accessible_memory((void const *)candidates[0].data())) - return {status_t::device_memory_mismatch_k, cudaSuccess}; - } + // Inputs and outputs alike must be reachable from a kernel; one probe per allocation covers each. + if (status_t const reachable = check_similarities_memory(queries, candidates, results); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; // Device-side materialization for BOTH all-pairs and symmetric (one thread per live cell; the symmetric // path maps the flat cell index into the lower triangle). Build the O(queries+candidates) descriptors on // the host; the kernel reads them from unified memory. Weighted cells start at 2 bytes (signed scores). - if (buffers.query_descriptors_.try_resize(queries_count) == status_t::bad_alloc_k || - buffers.candidate_descriptors_.try_resize(candidates_count) == status_t::bad_alloc_k) + if (buffers.query_descriptors_.try_resize_uninitialized(queries_count) == status_t::bad_alloc_k || + buffers.candidate_descriptors_.try_resize_uninitialized(candidates_count) == status_t::bad_alloc_k) return {status_t::bad_alloc_k}; for (size_t query_index = 0; query_index < queries_count; ++query_index) buffers.query_descriptors_[query_index] = {queries[query_index].data(), queries[query_index].length()}; @@ -5997,13 +5937,10 @@ cuda_status_t cuda_weighted_cross_( class_substitution_costs_buffer, executor, specs); if (status.status != status_t::success_k) return status; - // Scatter on the device when the output is device-accessible (the common unified-memory case): the device - // scatter kernel writes each result by its `result_offset` so the host never reads the large task array back. - // When the output is host-only, the same kernel scatters into a device-resident dense staging matrix (laid out - // at the caller's `row_stride` so the precomputed offsets stay valid), then a single strided `cudaMemcpy2DAsync` - // strides the valid `rows x columns` region into the host matrix - no per-cell host loop, fully stream-async. + // Scatter on the device: the kernel writes each result by its `result_offset`, so the host never reads the + // large task array back. using results_value_t = typename std::remove_reference_t<results_type_>::value_type; - if (tasks.size() && is_device_accessible_memory((void const *)results.data)) { + if (tasks.size()) { results_value_t *results_ptr = results.data; task_t const *tasks_const_ptr = tasks.data(); size_t tasks_size = tasks.size(); @@ -6023,11 +5960,6 @@ cuda_status_t cuda_weighted_cross_( if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); } } - else if (tasks.size()) { - cuda_status_t const fallback_status = cuda_scatter_results_to_host_strided_<task_t, results_value_t>( - buffers, tasks.data(), tasks.size(), results, executor, block); - if (fallback_status.status != status_t::success_k) return fallback_status; - } return status; } diff --git a/include/stringzillas/similarities/serial.hpp b/include/stringzillas/similarities/serial.hpp index 4810a5c1..0a3ecfad 100644 --- a/include/stringzillas/similarities/serial.hpp +++ b/include/stringzillas/similarities/serial.hpp @@ -9,9 +9,9 @@ * * Every backend specialization header (`icelake.hpp`, `cuda.cuh`, ...) must include this * file first, so that the primary templates are visible before any specialization. - * + * * This file is designed around several guiding principles: - * + * * - Avoid larger integral types, where smaller ones are enough. * - Larger kernels are assembled from smaller templates, so to keep binary size and compilation time * sane, type-invariant pieces are shielded from generic interfaces via "trampolines". @@ -423,32 +423,6 @@ struct diagonal_memory_requirements { using scratch_space_t = span<std::byte>; -/** - * @brief A running, cache-line-padded scratch byte amount, used to lay out a walker's sub-buffers. - * - * Each walker partitions its `scratch_space_t` into a handful of sub-buffers (score diagonals, a reversed - * copy of the shorter string, a Myers `match_masks` table, ...). Growing this amount once per sub-buffer keeps - * every offset cache-line aligned and yields the total scratch the walker needs - a single source of truth - * shared by the walker's `layout()` and its `operator()`. Cache-line width is `>=` any CPU register width, so - * the padding also keeps full-register SIMD over-reads near a buffer's end in bounds. - */ -struct scratch_amount_t { - // ? Deliberately a poison default (not `SZ_CACHE_LINE_WIDTH`): an instance built without an explicit - // ? `cpu_specs_t::cache_line_width` should produce an obviously-broken `total` (huge → `bad_alloc`/ASan), - // ? surfacing any place that forgot to propagate the alignment rather than silently assuming 64 bytes. - size_t alignment = std::numeric_limits<size_t>::max(); - size_t total = 0; // ? The accumulated, padded byte count == the next buffer's offset. - - /** @brief Reads the current end of the scratch, i.e. the offset where the next sub-buffer would start. */ - constexpr operator size_t() const noexcept { return total; } - - /** @brief Reserves @p bytes for the next sub-buffer, padded so the following offset stays aligned. */ - constexpr scratch_amount_t &operator+=(size_t bytes) noexcept { - total += round_up_to_multiple<size_t>(bytes, alignment); - return *this; - } -}; - /** * @brief Routes a runtime word-count @p bucket in `[current_k, high_k]` to the matching @b compile-time @p fixed * callback (invoked with `std::integral_constant<size_t, bucket>` so it can pick the right @@ -477,12 +451,21 @@ SZ_INLINE bool text_is_ascii_(span<char const> text) noexcept { return find_byteset_(text.data(), text.size(), &non_ascii) == SZ_NULL_CHAR; } -/** @brief Whether every string in @p corpus is ASCII, so rune distances equal byte distances. */ +/** + * @brief Whether every string in @p corpus is ASCII, so rune distances equal byte distances. + * + * A tape is one contiguous block, so it is scanned in a single pass rather than one byteset launch per + * element. Any terminator bytes the block carries between elements are themselves ASCII, so the answer is + * the same either way. + */ template <sz_find_byteset_t find_byteset_, typename corpus_type_> SZ_INLINE bool corpus_is_ascii_(corpus_type_ const &corpus) noexcept { - for (size_t index = 0; index != corpus.size(); ++index) - if (!text_is_ascii_<find_byteset_>(to_view(corpus[index]))) return false; - return true; + if constexpr (is_tape_like<corpus_type_>::value) { return text_is_ascii_<find_byteset_>(corpus.tape_bytes()); } + else { + for (size_t index = 0; index != corpus.size(); ++index) + if (!text_is_ascii_<find_byteset_>(to_view(corpus[index]))) return false; + return true; + } } #pragma region Core Templates diff --git a/include/stringzillas/stringzillas.h b/include/stringzillas/stringzillas.h index 073d0585..385cd693 100644 --- a/include/stringzillas/stringzillas.h +++ b/include/stringzillas/stringzillas.h @@ -109,6 +109,20 @@ SZ_API_RUNTIME sz_status_t sz_memory_allocator_init_unified(sz_memory_allocator_ * * Set `cpu_cores` to 0 to target all available CPU cores, to -1 to avoid CPUs, to 1 to use only calling thread. * Set `gpu_device` to -1 to avoid GPUs, or to a positive device ID to target a specific GPU. + * + * @section szs_unified_memory Unified Memory + * + * A GPU scope addresses its work from the device, so every @b buffer an operation reads or writes must be + * device-accessible: unified memory from @ref szs_unified_alloc or @ref sz_memory_allocator_init_unified, or + * plain device memory. Outputs are held to this as firmly as inputs, and the small per-collection arrays + * with them - match counts, tape offsets, BM25 weights and lengths, and score vectors are no different from + * a result matrix. Host memory is refused with `sz_device_memory_mismatch_k` and nothing is written, so a + * copy across the bus is the caller's to make rather than one the engine makes unasked. Page-locked host + * memory is host memory here - the driver reports it as such - and is refused too. + * + * The single-value out-parameters are the exception and take ordinary host addresses: `matches_total`, + * `matches_found`, `output_bytes_written`, and every `error_message` are written by the host once the device + * has finished. A CPU scope has no requirement at all and reads and writes host memory throughout. */ typedef void *szs_device_scope_t; @@ -491,7 +505,7 @@ SZ_API_RUNTIME void szs_smith_waterman_scores_free(szs_smith_waterman_scores_t e * APIs for computing fingerprints, Min-Hashes, and Count-Min-Sketches of binary and UTF-8 strings. * Supports `sz_sequence_t`, `sz_sequence_u32tape_t`, and `sz_sequence_u64tape_t` inputs. * - * @section Speed Considerations + * @section szs_speed_considerations Speed Considerations * * For each window width you should aim for a multiple of 64 dimensions. Rolling hashes with identical window widths * will share the same memory access pattern and can be effectively parallelized. For each platform, different minimum @@ -595,6 +609,312 @@ SZ_API_RUNTIME sz_status_t szs_fingerprints_u32tape( // */ SZ_API_RUNTIME void szs_fingerprints_free(szs_fingerprints_t engine); +/** + * @brief Multi-pattern search: one compiled dictionary of needles applied to many haystacks in one pass. + * + * Compiles a needle set into an Aho-Corasick automaton once, then reuses it across every later call, so + * the build cost is paid per dictionary rather than per haystack. Every needle is tested against every + * haystack in a single walk, and matches are reported under a caller-chosen overlap policy: every + * overlapping match, or a non-overlapping leftmost cover. + * + * Supports `sz_sequence_t`, `sz_sequence_u32tape_t`, and `sz_sequence_u64tape_t` inputs. + * + * Beyond counting and locating matches, the engine scores haystacks against per-needle weights with BM25 + * and rewrites haystacks by substituting matches, on every backend. A rewrite needs a cover whose matches + * share no bytes, so `szs_substrings_overlapping_k` is the one policy it refuses. + * + * @section szs_case_sensitivity Case Sensitivity + * + * `szs_substrings_cased_k` matches raw bytes and accepts any needle, including malformed UTF-8. + * + * `szs_substrings_uncased_k` applies full Unicode case folding as defined by `CaseFolding.txt` status codes + * `C` and `F`, the same contract `sz_utf8_uncased_find` implements, so the two agree. Needles must be + * well-formed UTF-8 and are rejected with `sz_invalid_utf8_k` otherwise. Folding applies no normalization, + * so a precomposed character does not match its decomposed spelling; compose `sz_utf8_norm` ahead of the + * search when canonical equivalence is wanted. + */ +typedef void *szs_substrings_t; + +/** @brief Whether a dictionary matches needles byte-for-byte or folds both sides to a shared case first. */ +typedef enum szs_substrings_case_sensitivity_t { + /** Byte-exact matching; needles may be arbitrary bytes. */ + szs_substrings_cased_k = 0, + /** Full Unicode case folding; needles must be valid UTF-8. */ + szs_substrings_uncased_k = 1, +} szs_substrings_case_sensitivity_t; + +/** + * @brief How overlapping matches resolve: reported in full, or reduced to a non-overlapping cover. + * + * The three states map one-to-one onto the `MatchKind` trio of the reference Rust engines. The policy + * travels per call rather than per engine, since it never shapes the compiled automaton - one dictionary + * serves all three. + */ +typedef enum szs_substrings_overlap_policy_t { + /** Every match of every needle, including overlapping and nested ones. */ + szs_substrings_overlapping_k = 0, + /** Non-overlapping cover: earliest start, then longest span, then lower needle index. */ + szs_substrings_leftmost_longest_k = 1, + /** Non-overlapping cover: earliest start, then lower needle index, even when a longer needle matches. */ + szs_substrings_leftmost_first_k = 2, +} szs_substrings_overlap_policy_t; + +/** + * @brief One reported match, locating it by haystack, by needle, and by byte span. + * + * Under case folding a needle's own byte length is not the length of every match - needle "k" matches both + * the 1-byte "k" and the 3-byte Kelvin sign - so the span is carried per match rather than looked up. + */ +typedef struct szs_substrings_match_t { + /** Which haystack the match was found in. */ + sz_size_t haystack_index; + /** Which needle matched. */ + sz_size_t needle_index; + /** Offset of the match within its haystack, in bytes. */ + sz_size_t byte_offset; + /** Length of the matched span, in bytes. */ + sz_size_t byte_length; +} szs_substrings_match_t; + +/** @brief Classic BM25's continuous parameters. */ +typedef struct szs_substrings_bm25_t { + /** The literature's `k1`: how slowly repeated occurrences stop adding score; 1.2 is customary. */ + sz_f32_t term_frequency_saturation; + /** The literature's `b`, in [0, 1]: 0 ignores document length and every length input with it, 1 + * normalizes fully; 0.75 is customary. */ + sz_f32_t length_normalization; + /** Corpus-wide mean of `document_lengths`, in the same unit; never derived from the batch, and read + * only when `length_normalization` is positive. A non-positive mean is then refused rather than + * silently discarding both it and `document_lengths`. */ + sz_f32_t average_document_length; +} szs_substrings_bm25_t; + +/** + * @brief Create a multi-pattern search engine on the backend @p capabilities names. + * + * Unlike the other engines, this one is constructed in two steps: the automaton's hot/cold tier is sized + * from the cache that will walk it, and no scope reaches a constructor. This call picks the backend and + * allocates the handle; @ref szs_substrings_index compiles a needle set into it. Every operation refuses an + * engine that has not been indexed. + * + * @param[in] alloc Memory allocator (NULL for default). + * @param[in] capabilities Hardware capabilities mask, which selects the backend. + * @param[out] engine Pointer to initialized engine handle. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_init( // + sz_memory_allocator_t const *alloc, sz_capability_t capabilities, // + szs_substrings_t *engine, char const **error_message); + +/** + * @brief Compile @p needles into @p engine, tiered for @p device. + * + * Replaces any needle set the engine already held, so one engine can be re-indexed for a new vocabulary or + * re-tiered for another device. The state-id width is not a caller's choice: it follows from the needle set, + * so construction derives the automaton wide and keeps the narrower one whenever its ceilings hold. + * + * Operations may later name a different scope - the automaton is reachable from every device - but the tier + * stays as this call sized it. + * + * @param[in] engine Engine handle from @ref szs_substrings_init. + * @param[in] needles Needle collection to compile into the automaton. + * @param[in] case_sensitivity Byte-exact or case-folded matching. + * @param[in] device Device scope whose cache sizes the hot tier, and whose backend must match the engine's. + * @param[out] error_message Optional output pointer for detailed error information. + * @retval `sz_invalid_utf8_k` A needle is not well-formed UTF-8 while folding is requested. + * @retval `sz_unexpected_dimensions_k` A needle is empty. + * @retval `sz_device_code_mismatch_k` @p device names a backend the engine was not created for. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_index( // + szs_substrings_t engine, sz_sequence_t const *needles, // + szs_substrings_case_sensitivity_t case_sensitivity, szs_device_scope_t device, // + char const **error_message); + +/** + * @brief Count matches of every needle in every haystack. + * @param[in] engine Initialized search engine. + * @param[in] device Device scope for execution. + * @param[in] haystacks Input haystack collection. + * @param[in] overlap_policy Whether overlapping matches all count, or only a leftmost cover. + * @param[out] counts Output array of per-haystack match counts, one entry per haystack. + * @param[out] matches_total Matches across every haystack, the sum @p counts would produce. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_count( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message); + +/** + * @brief Locate matches of every needle in every haystack, resolved under @p overlap_policy. + * @param[in] engine Initialized search engine. + * @param[in] device Device scope for execution. + * @param[in] haystacks Input haystack collection. + * @param[in] overlap_policy Whether overlapping matches are all reported, or only a leftmost cover. + * @param[out] matches Output match array, filled in ascending haystack order. + * @param[in] matches_capacity Number of entries @p matches can hold. + * @param[out] matches_found Matches written, or - when capacity is short - the count that would be. + * @retval `sz_unexpected_dimensions_k` @p matches_capacity is too small; @p matches_found holds the + * need and nothing was written, so `matches_capacity == 0` is the canonical size query. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_find( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message); + +/** + * @brief Score every haystack against the compiled dictionary in one automaton walk. + * + * The dictionary @b is the query: `needle_weights[needle_index]` is that needle's IDF or boost, and one + * score comes back per haystack. Term frequencies are raw overlapping counts, which is classic BM25 - a + * leftmost cover would suppress genuine occurrences of any needle nested in another, so no overlap policy + * applies here. Frequencies are integers, and each backend reaches a total no run of it can perturb - the + * CPU sums in ascending needle order, the GPU sums fixed-point integers, whose addition is associative - so + * scores are bit-stable across runs of one backend, and agree numerically rather than bitwise between two. + * + * @param[in] engine Initialized search engine. + * @param[in] device Device scope for execution. + * @param[in] haystacks Input haystack collection. + * @param[in] document_lengths One per haystack, in whatever unit the pipeline normalizes by; + * NULL uses byte lengths, the only unit a byte-level engine can own. + * @param[in] parameters BM25's continuous parameters. + * @param[in] needle_weights One IDF or boost per needle; as many entries as the dictionary has needles. + * @param[out] scores One per haystack. + * @retval `sz_unexpected_dimensions_k` @p needle_weights is NULL, or `length_normalization` is positive + * while `average_document_length` is not - there would be no mean to divide by. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message); + +/** + * @brief Bound the bytes a rewrite can produce, from the dictionary and @p replacements alone. + * + * A cover's matches share no bytes and each consumes at least its needle's shortest matchable span, so + * every input byte emits at most `max over needles of replacement_bytes / min_source_match_bytes`. Sizing an + * output tape to this bound makes `szs_substrings_replace` a single call that cannot be refused, at the + * cost of over-allocating whenever the corpus does not consist entirely of the widest-expanding needle. + * + * Needs no haystacks, no walk, and no device: the answer is arithmetic over the needle set. + * + * @param[in] engine Initialized search engine. + * @param[in] replacements One replacement per needle; count must equal the dictionary's needle count. + * @param[in] input_bytes Total bytes of the haystacks to be rewritten. + * @param[out] output_bytes_bound Bytes an output tape must hold to accept any such rewrite. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_replace_bound( // + szs_substrings_t engine, // + sz_sequence_t const *replacements, // + sz_size_t input_bytes, sz_size_t *output_bytes_bound, // + char const **error_message); + +/** + * @brief Rewrite every haystack of a tape into another tape, substituting matches with replacements. + * + * Matches resolve under @p overlap_policy, which must name a non-overlapping cover - an overlapping rewrite + * is not a function, so `szs_substrings_overlapping_k` is rejected. Replacements are indexed by needle and + * inserted verbatim, even in uncased mode where the removed span's byte length varies per match, and an + * empty replacement deletes. + * + * Tape in, tape out: only the Apache Arrow-like shapes are accepted, since a rewrite's product is itself a + * tape and a callback-addressed `sz_sequence_t` has nowhere to put one. Size @p output_data with + * `szs_substrings_replace_bound` for a call that cannot be refused. + * + * @param[in] engine Initialized search engine. + * @param[in] device Device scope for execution. + * @param[in] haystacks Input haystack tape. + * @param[in] overlap_policy Cover policy; `szs_substrings_overlapping_k` is rejected. + * @param[in] replacements One replacement per needle; count must equal the dictionary's needle count. + * @param[out] output_data Byte buffer receiving every rewritten haystack back to back. + * @param[in] output_data_capacity Bytes @p output_data can hold; a short buffer is refused, never overrun. + * @param[out] output_offsets Receives `haystacks->count + 1` boundaries partitioning the output tape: + * `output_offsets[0]` is zero, they ascend, and `output_offsets[haystacks->count]` is the total, + * so the array can be wrapped as a tape view directly. They are written before the capacity + * check, so a refused call still names every boundary it would have produced. + * @param[out] output_bytes_written Bytes written, or - when the buffer is short - the bytes needed. + * @retval `sz_unexpected_dimensions_k` @p output_data_capacity is too small and nothing was written. + * @param[out] error_message Optional output pointer for detailed error information. + */ +SZ_API_RUNTIME sz_status_t szs_substrings_replace_u32tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u32tape_t const *haystacks, // + szs_substrings_overlap_policy_t overlap_policy, // + sz_sequence_t const *replacements, // + sz_ptr_t output_data, sz_size_t output_data_capacity, sz_size_t *output_offsets, // + sz_size_t *output_bytes_written, // + char const **error_message); + +/** @copydoc szs_substrings_replace_u32tape */ +SZ_API_RUNTIME sz_status_t szs_substrings_replace_u64tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u64tape_t const *haystacks, // + szs_substrings_overlap_policy_t overlap_policy, // + sz_sequence_t const *replacements, // + sz_ptr_t output_data, sz_size_t output_data_capacity, sz_size_t *output_offsets, // + sz_size_t *output_bytes_written, // + char const **error_message); + +/** @copydoc szs_substrings_count */ +SZ_API_RUNTIME sz_status_t szs_substrings_count_u32tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u32tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message); + +/** @copydoc szs_substrings_count */ +SZ_API_RUNTIME sz_status_t szs_substrings_count_u64tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u64tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + sz_size_t *counts, sz_size_t *matches_total, // + char const **error_message); + +/** @copydoc szs_substrings_find */ +SZ_API_RUNTIME sz_status_t szs_substrings_find_u32tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u32tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message); + +/** @copydoc szs_substrings_find */ +SZ_API_RUNTIME sz_status_t szs_substrings_find_u64tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u64tape_t const *haystacks, szs_substrings_overlap_policy_t overlap_policy, // + szs_substrings_match_t *matches, sz_size_t matches_capacity, sz_size_t *matches_found, // + char const **error_message); + +/** @copydoc szs_substrings_score_bm25 */ +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25_u32tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u32tape_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message); + +/** @copydoc szs_substrings_score_bm25 */ +SZ_API_RUNTIME sz_status_t szs_substrings_score_bm25_u64tape( // + szs_substrings_t engine, szs_device_scope_t device, // + sz_sequence_u64tape_t const *haystacks, // + sz_f32_t const *document_lengths, // + szs_substrings_bm25_t parameters, // + sz_f32_t const *needle_weights, sz_f32_t *scores, // + char const **error_message); + +/** + * @brief Free multi-pattern search engine resources. + * @param[in] engine Engine handle to free. + */ +SZ_API_RUNTIME void szs_substrings_free(szs_substrings_t engine); + /** * @brief Allocates memory using unified memory allocator. * @param[in] size_bytes Number of bytes to allocate. diff --git a/include/stringzillas/substrings.cuh b/include/stringzillas/substrings.cuh new file mode 100644 index 00000000..af52c46c --- /dev/null +++ b/include/stringzillas/substrings.cuh @@ -0,0 +1,20 @@ +/** + * @brief Multi-pattern exact and case-folded substring search for CUDA GPUs. + * @file include/stringzillas/substrings.cuh + * @author Ash Vardanian + * @sa include/stringzillas/substrings.hpp for the CPU backends. + * + * This is a thin hub aggregating the GPU backend on top of the CPU one, since the dictionary is built on the + * host and only its published view is uploaded. + * + * A thread walks one chunk of the tape and the hot tier is staged into shared memory per block. + * `substrings/cuda.cuh` documents the chunking and staging rules. + */ +#ifndef STRINGZILLAS_SUBSTRINGS_CUH_ +#define STRINGZILLAS_SUBSTRINGS_CUH_ + +#include "stringzillas/substrings.hpp" // Host-side dictionary, built before any upload + +#include "stringzillas/substrings/cuda.cuh" // GPU engine over the uploaded view + +#endif // STRINGZILLAS_SUBSTRINGS_CUH_ diff --git a/include/stringzillas/substrings.hpp b/include/stringzillas/substrings.hpp new file mode 100644 index 00000000..1901bac0 --- /dev/null +++ b/include/stringzillas/substrings.hpp @@ -0,0 +1,20 @@ +/** + * @brief Multi-pattern exact and case-folded substring search for CPUs. + * @file include/stringzillas/substrings.hpp + * @author Ash Vardanian + * @sa include/stringzillas/substrings.cuh for the GPU backends. + * + * This is a thin hub aggregating the per-backend headers. The shared vocabulary, the automaton, its + * construction, and the CPU engines all live in `substrings/serial.hpp`, matching how the `similarities` and + * `fingerprints` families are laid out. + * + * There are no per-ISA kernels here, unlike the other families. A transition is one data-dependent load on + * a serial dependency chain, so parallelism comes from advancing many haystacks at once rather than from + * vector instructions, and per-ISA variation reduces to the chain count taken from `cpu_specs_t`. + */ +#ifndef STRINGZILLAS_SUBSTRINGS_HPP_ +#define STRINGZILLAS_SUBSTRINGS_HPP_ + +#include "stringzillas/substrings/serial.hpp" // Automaton, dictionary builder, and CPU engines + +#endif // STRINGZILLAS_SUBSTRINGS_HPP_ diff --git a/include/stringzillas/substrings/README.md b/include/stringzillas/substrings/README.md new file mode 100644 index 00000000..3095f303 --- /dev/null +++ b/include/stringzillas/substrings/README.md @@ -0,0 +1,71 @@ +# Substrings for StringZillas + +The substrings engine matches a __whole dictionary of needles__ against a __whole collection of haystacks__ in a single pass, the workhorse of __log scanning__, __protocol dispatch__, __content filtering__, and __signature matching__. +It compiles the needle set once into an Aho-Corasick automaton and reuses it across every later call, so the dictionary is paid for once rather than per haystack. +The automaton is __goto-completed__ and split into two tiers: a dense 256-wide row per frequently-visited state, and a double array for the rest, so a step is one load with no failure-following at runtime. +`substrings_cased_k` matches raw bytes and accepts any needle, while `substrings_uncased_k` applies __full Unicode case folding__ to both sides of the comparison: the needle is folded once at build time, and the haystack one codepoint at a time as the walk consumes it, never into a buffer. +A match is therefore any contiguous run of the folded haystack, so it may begin or end part-way through an expansion - `"s"` matches inside `"ß"` - with both ends reported in original haystack bytes, snapped outward to the codepoint they fall in. +That is the same rule `sz_utf8_uncased_search` follows, so the single-pattern and multi-pattern engines agree. +Every engine runs across a slice of CPU cores or a CUDA GPU, advancing many haystacks concurrently rather than making any single haystack faster. + +Throughput is reported in __MB/s__ of haystack bytes consumed, the rate at which the automaton advances over the corpus. + +## Methodology + +Each table fixes one input shape: the `xlsum.csv` corpus split into lines as haystacks, searched with one slice of that corpus's own frequency-ordered vocabulary as the dictionary. +Slices are taken by term count, so the frequent and the rare slice of the same percentage hold the same number of needles and differ only in byte totals, since Zipf makes frequent terms short and their automata correspondingly shallower. +Cells carry the benchmark's `Throughput` line for each of the four capabilities — counting, which only tallies matches; finding, which materializes each one; replacing, which rewrites every haystack; and scoring, which reduces each haystack to one BM25 float — with the fastest backend of each column in bold. +A `-` cell is a measurement not yet taken. +The replace columns resolve a __leftmost__ cover, since a rewrite must, while counting and finding are __overlapping__ passes; the two are not the same walk and should not be differenced. +The score columns weight every needle uniformly, so each slice heading doubles as a query size — scoring counts into a table sized by the document rather than by the dictionary, so its cost tracks the walk, and the widest slice no longer collapses the way a counter per needle made it. +One run produced every cell: 64 MiB of the corpus, `STRINGWARS_DATASET_LIMIT=64mb` with `STRINGWARS_TOKENS=lines`, on one idle H100 80GB HBM3 and one Xeon Platinum 8468, built through the `cuda_clang` preset. +The box is shared, so the run waits for it to go quiet; the counting columns are the control, and they reproduce the previous table's within 1.6x end to end, which is what makes the score columns comparable against it. +That preset is not a preference — GCC miscompiles the engines' return-by-value in these translation units, handing the caller the kernel timing where the status belongs, so a GCC-hosted benchmark aborts on its first CUDA cell. + +The tier split dominates these numbers more than the backend does, so `bench/substrings.cpp` prints the automaton's state count, how many of those states fit the hot tier, and the double array's byte size beside every result. +Rows name a threading tier rather than an ISA because there are no per-ISA kernels: a transition is one data-dependent load on a serial dependency chain, so throughput comes from independent scalar chains over many haystacks, and an AVX-512 `vpgatherdd` formulation measured slower than plain scalar chains at every dictionary size. +These numbers should not be compared against `sz_find`, which searches for one needle and solves a strictly easier problem. + +Uncased matching follows `CaseFolding.txt` status codes `C` and `F`, excluding `S` and `T`, which is the same contract `sz_utf8_uncased_find` implements and is what keeps results locale-independent. +No normalization is applied, so precomposed `Ê` does not match `e` followed by a combining acute; callers who need canonical equivalence run `sz_utf8_norm` first. +Uncased needles must be well-formed UTF-8, and needles of either mode must be non-empty. + +## Most Frequent 1% of the Vocabulary + +| Backend | Cased Count | Cased Find | Cased Replace | Cased Score | Uncased Count | Uncased Find | Uncased Replace | Uncased Score | +| :--------------- | ---------------: | ---------------: | --------------: | ---------------: | --------------: | --------------: | --------------: | --------------: | +| Serial @ Xeon4 | 210.2 MB/s | 100.1 MB/s | 55.1 MB/s | 184.0 MB/s | 90.2 MB/s | 44.7 MB/s | 36.2 MB/s | 91.9 MB/s | +| Parallel @ Xeon4 | 2974.3 MB/s | 1256.3 MB/s | 467.8 MB/s | 2448.1 MB/s | 1065.9 MB/s | 512.9 MB/s | 390.7 MB/s | 1038.7 MB/s | +| CUDA @ H100 | __19198.5 MB/s__ | __17995.9 MB/s__ | __8504.0 MB/s__ | __15461.9 MB/s__ | __7688.0 MB/s__ | __4949.9 MB/s__ | __4359.4 MB/s__ | __4273.5 MB/s__ | + +## Most Frequent 10% of the Vocabulary + +| Backend | Cased Count | Cased Find | Cased Replace | Cased Score | Uncased Count | Uncased Find | Uncased Replace | Uncased Score | +| :--------------- | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | +| Serial @ Xeon4 | 63.1 MB/s | 29.3 MB/s | 20.7 MB/s | 55.8 MB/s | 45.3 MB/s | 20.6 MB/s | 17.1 MB/s | 41.9 MB/s | +| Parallel @ Xeon4 | 901.4 MB/s | 423.4 MB/s | 248.4 MB/s | 796.7 MB/s | 610.5 MB/s | 292.2 MB/s | 226.9 MB/s | 568.4 MB/s | +| CUDA @ H100 | __9051.6 MB/s__ | __8654.4 MB/s__ | __5798.2 MB/s__ | __7913.5 MB/s__ | __7108.2 MB/s__ | __3908.4 MB/s__ | __2856.2 MB/s__ | __2652.1 MB/s__ | + +## Least Frequent 1% of the Vocabulary + +| Backend | Cased Count | Cased Find | Cased Replace | Cased Score | Uncased Count | Uncased Find | Uncased Replace | Uncased Score | +| :--------------- | ---------------: | ---------------: | ---------------: | ---------------: | --------------: | --------------: | --------------: | --------------: | +| Serial @ Xeon4 | 353.3 MB/s | 177.9 MB/s | 146.3 MB/s | 356.7 MB/s | 107.6 MB/s | 53.9 MB/s | 50.7 MB/s | 107.2 MB/s | +| Parallel @ Xeon4 | 5486.8 MB/s | 2491.1 MB/s | 1975.7 MB/s | 4541.9 MB/s | 1127.4 MB/s | 560.7 MB/s | 524.2 MB/s | 1116.7 MB/s | +| CUDA @ H100 | __26274.5 MB/s__ | __29893.0 MB/s__ | __16256.5 MB/s__ | __34971.8 MB/s__ | __8300.0 MB/s__ | __5379.4 MB/s__ | __4617.1 MB/s__ | __5991.5 MB/s__ | + +## Least Frequent 10% of the Vocabulary + +| Backend | Cased Count | Cased Find | Cased Replace | Cased Score | Uncased Count | Uncased Find | Uncased Replace | Uncased Score | +| :--------------- | ---------------: | ---------------: | --------------: | ---------------: | --------------: | --------------: | --------------: | --------------: | +| Serial @ Xeon4 | 140.3 MB/s | 71.2 MB/s | 54.4 MB/s | 122.8 MB/s | 76.5 MB/s | 38.0 MB/s | 32.5 MB/s | 75.5 MB/s | +| Parallel @ Xeon4 | 2083.1 MB/s | 991.4 MB/s | 589.9 MB/s | 1675.0 MB/s | 958.7 MB/s | 476.3 MB/s | 405.9 MB/s | 941.3 MB/s | +| CUDA @ H100 | __11220.6 MB/s__ | __10930.7 MB/s__ | __7065.2 MB/s__ | __13636.5 MB/s__ | __7086.7 MB/s__ | __4348.7 MB/s__ | __3425.2 MB/s__ | __4380.9 MB/s__ | + +## Entire Vocabulary + +| Backend | Cased Count | Cased Find | Cased Replace | Cased Score | Uncased Count | Uncased Find | Uncased Replace | Uncased Score | +| :--------------- | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | --------------: | +| Serial @ Xeon4 | 34.7 MB/s | 17.5 MB/s | 12.0 MB/s | 31.9 MB/s | 27.2 MB/s | 12.0 MB/s | 9.7 MB/s | 23.6 MB/s | +| Parallel @ Xeon4 | 563.7 MB/s | 269.3 MB/s | 150.4 MB/s | 442.9 MB/s | 396.5 MB/s | 166.3 MB/s | 133.6 MB/s | 324.7 MB/s | +| CUDA @ H100 | __7698.7 MB/s__ | __4649.3 MB/s__ | __2652.1 MB/s__ | __3972.8 MB/s__ | __4831.8 MB/s__ | __2566.2 MB/s__ | __1395.9 MB/s__ | __1567.7 MB/s__ | diff --git a/include/stringzillas/substrings/cuda.cuh b/include/stringzillas/substrings/cuda.cuh new file mode 100644 index 00000000..92251390 --- /dev/null +++ b/include/stringzillas/substrings/cuda.cuh @@ -0,0 +1,2192 @@ +/** + * @brief CUDA backend for multi-pattern exact and case-folded substring search. + * @file include/stringzillas/substrings/cuda.cuh + * @author Ash Vardanian + * @sa include/stringzillas/substrings/serial.hpp + * + * `try_build` uploads a host-built `aho_corasick_view`, so this backend consumes the published contract + * rather than the builder's internals. + * + * Each haystack travels as its own pointer-and-length descriptor, so no layout is assumed: one packed + * tape and scattered device allocations chunk identically. One thread owns one contiguous slice of one + * haystack, sized against the corpus's total byte count rather than the haystack count, so occupancy does + * not depend on how the corpus is split, and no needle can straddle the seam between two documents. + * + * A thread starts its walk `max_source_match_bytes - 1` bytes before its chunk, clamped to its own haystack's + * start. Aho-Corasick is self-synchronizing, so that warm-up makes chunking exact rather than approximate: + * the automaton reaches the same state at the chunk boundary wherever the walk began. + * + * A transition is one data-dependent load, so the walk is latency-bound rather than bandwidth-bound and + * resident warps are what hides it. The automaton is therefore staged into shared memory only when the + * whole of it fits without costing a resident block, and read through the cache hierarchy otherwise. + */ +#ifndef STRINGZILLAS_SUBSTRINGS_CUDA_CUH_ +#define STRINGZILLAS_SUBSTRINGS_CUDA_CUH_ + +#include <cuda.h> +#include <cuda_runtime.h> + +#include "stringzillas/types.cuh" // `unified_alloc_t`, `cuda_status_t` +#include "stringzillas/substrings/serial.hpp" // `aho_corasick_view`, the host-built contract + +namespace ashvardanian { +namespace stringzillas { + +// Per-symbol: a using-directive re-exports our `memcpy` and nvcc then finds the call ambiguous. +using ashvardanian::stringzilla::byte_t; +using ashvardanian::stringzilla::size_t; +using ashvardanian::stringzilla::small_size_t; +using ashvardanian::stringzilla::span; +using ashvardanian::stringzilla::status_t; +using ashvardanian::stringzilla::to_bytes_view; + +#pragma region Device Kernels + +/** @brief Block size every `substrings_cuda` kernel launches with; occupancy is shared-memory-bound, not + * thread-count-bound, so a modest fixed block keeps the launch geometry simple. */ +static constexpr unsigned substrings_threads_per_block_k = 256; + +/** @brief Candidates one thread will scan quadratically before a segment falls back to emitted order. */ +static constexpr size_t substrings_cover_segment_limit_k = 4096; + +/** @brief Output bytes one block of a rewrite's copy owns, so no block's work scales with a run's width. */ +static constexpr size_t substrings_rewrite_tile_bytes_k = 4096; + +/** @brief Counter slots a scoring block keeps in shared memory, sized by the document rather than by the + * dictionary. 64 KiB is the largest power of two that still leaves two blocks resident per + * multiprocessor, and it seats the distinct needles of a 40 KiB document below half load. */ +static constexpr size_t substrings_bm25_slots_k = 8192; + +/** @brief Probe distance a scoring insert gives up at, spilling to the overflow row. */ +static constexpr size_t substrings_bm25_probes_k = 16; + +/** @brief Fractional bits a block's running score carries, leaving three orders of headroom over the largest + * score a unit-weighted full vocabulary reaches. */ +static constexpr int substrings_bm25_scale_k = 32; + +/** @brief A counter slot no needle has claimed. Needle indices are dense from zero, so the top value is + * never one of them. */ +static constexpr u32_t substrings_bm25_empty_slot_k = 0xFFFFFFFFu; + +/** + * @brief Selects what a chunk walk does at each match: size the output, write it, or count per needle. + * + * `sizing_k` only sizes the output so the caller can scan it, `writing_k` writes each match at its + * chunk's precomputed offset, and `counting_k` increments a per-needle counter instead of reporting + * anything - the shape BM25 needs, which wants how often each needle occurred and never where. + * + * Consumed with `if constexpr`, matching `tile_march_t` in `stringzillas/types.cuh`. + */ +enum class substrings_pass_t : u8_t { sizing_k = 0, writing_k = 1, counting_k = 2 }; + +/** + * @brief Adds to a counter without reading it back - a reduction, not an atomic exchange. + * + * `red` is issued to the L2 slice and retires without a round-trip, so the warp never stalls on it, which + * is what counting wants: the old count is never the question. `atomicAdd` lowers to the same + * instruction only when the compiler proves the returned value is dead, which is a property of the + * optimizer rather than of the source; spelling the reduction out states the intent and cannot regress. + */ +SZ_DEVICE_INLINE void cuda_increment_global_(u32_t *counter, u32_t addend) noexcept { + asm volatile("red.global.add.u32 [%0], %1;" ::"l"(__cvta_generic_to_global(counter)), "r"(addend) : "memory"); +} + +/** @brief The same reduction against a block's own shared memory, for counters that never leave it. */ +SZ_DEVICE_INLINE void cuda_increment_shared_(u32_t *counter, u32_t addend) noexcept { + asm volatile("red.shared.add.u32 [%0], %1;" ::"r"((u32_t)__cvta_generic_to_shared(counter)), "r"(addend) + : "memory"); +} + +SZ_DEVICE_INLINE void cuda_increment_shared_(u64_t *counter, u64_t addend) noexcept { + asm volatile("red.shared.add.u64 [%0], %1;" ::"r"((u32_t)__cvta_generic_to_shared(counter)), "l"(addend) + : "memory"); +} + +/** @brief Where a counting walk puts its counts: the block's own table first, the overflow row when full. + * An empty @p overflow means the dictionary fits the table, which is what lifts the probe bound. */ +struct substrings_bm25_counters_t { + substrings_bm25_counter_t *slots = nullptr; + span<u32_t> overflow {}; + int *overflowed = nullptr; +}; + +/** + * @brief Counts one occurrence of @p needle_index into the block's table, or into the overflow row. + * + * Every lane of the block counts into one table, so every write here races and every write here is atomic. + * The probe re-reads its slot rather than hoisting it: a cached read would make a full table look like one + * slot reused forever, which scores wrong rather than hanging. + */ +SZ_DEVICE_INLINE void substrings_bm25_count_(substrings_bm25_counters_t const &counters, u32_t needle_index) noexcept { + // A dictionary that fits the table gets a slot per needle, so the index @b is the slot - no hash, no probe. + if (counters.overflow.empty()) return cuda_increment_shared_(&counters.slots[needle_index].frequency, 1u); + + substrings_bm25_counter_t volatile *const slots = counters.slots; // ? Volatile: re-read on every probe + size_t slot = substrings_bm25_probe_of_(needle_index) & (substrings_bm25_slots_k - 1u); + for (size_t probe = 0; probe != substrings_bm25_probes_k; + ++probe, slot = (slot + 1u) & (substrings_bm25_slots_k - 1u)) { + // Whoever the exchange hands the slot to owns it; a rival wanting the same needle joins them. + u32_t seated = slots[slot].needle_index; + if (seated == substrings_bm25_empty_slot_k) + seated = atomicCAS(&counters.slots[slot].needle_index, substrings_bm25_empty_slot_k, needle_index); + if (seated != substrings_bm25_empty_slot_k && seated != needle_index) continue; + return cuda_increment_shared_(&counters.slots[slot].frequency, 1u); + } + cuda_increment_global_(counters.overflow.data() + needle_index, 1u); + *counters.overflowed = 1; +} + +/** @brief One contribution as a fixed-point integer. The scaling runs in double so the `f32` contribution + * survives it exactly; scaling in `f32` would quantize back to 24 significant bits and waste it. */ +SZ_DEVICE_INLINE i64_t substrings_bm25_to_fixed_(f32_t contribution) noexcept { + return (i64_t)__double2ll_rn((double)contribution * (double)(1ull << (unsigned)substrings_bm25_scale_k)); +} + +/** @brief The block's fixed-point total, back as the score the caller reads. */ +SZ_DEVICE_INLINE f32_t substrings_bm25_from_fixed_(i64_t total) noexcept { + return (f32_t)((double)total / (double)(1ull << (unsigned)substrings_bm25_scale_k)); +} + +/** + * @brief Cooperatively fills @p shared_hot_rows from the head of the hot tier and @p shared_accepts_words + * from the acceptance bitmap, once per block. The hot tier's out-degree ordering makes its head the + * best prefix to stage; the bitmap span is empty when `try_build` budgeted it out of shared memory. + */ +template <typename state_id_type_> +SZ_DEVICE_INLINE void substrings_stage_automaton_(aho_corasick_view<state_id_type_> const &view, + span<state_id_type_> shared_hot_rows, span<u32_t const> accepts_words, + span<u32_t> shared_accepts_words) noexcept { + for (size_t cell = threadIdx.x; cell < shared_hot_rows.size(); cell += blockDim.x) + shared_hot_rows[cell] = view.hot_rows[cell]; + for (size_t word = threadIdx.x; word < shared_accepts_words.size(); word += blockDim.x) + shared_accepts_words[word] = accepts_words[word]; + __syncthreads(); +} + +/** + * @brief One byte's transition, staged-shared-memory-first: the staged prefix resolves branch-free from + * shared memory, and everything else - hot tier beyond the prefix, and the whole cold tier - defers + * to @ref aho_corasick_step, the single transition definition every backend shares. + * + * A single cold lane still makes the whole warp pay that lane's failure-chase depth, which is the cost the + * shared-memory staging exists to shrink. + */ +template <typename state_id_type_> +SZ_DEVICE_INLINE state_id_type_ substrings_step_device_(aho_corasick_view<state_id_type_> const &view, + span<state_id_type_ const> shared_hot_rows, + small_size_t staged_rows_count, state_id_type_ state, + u8_t byte) noexcept { + if (static_cast<small_size_t>(state) < staged_rows_count) + return hot_row_of<small_size_t>(shared_hot_rows, state)[byte]; + return aho_corasick_step(view, state, byte); +} + +/** + * @brief Finds, via binary search, which haystack owns global chunk @p chunk_index, given the exclusive + * prefix sum of chunk counts per haystack (`haystack_chunk_offsets[haystack_count]` is the grand + * total chunk count, mirroring how `outputs_offsets` bounds `outputs_counts`). + */ +SZ_DEVICE_INLINE size_t substrings_resolve_haystack_(span<size_t const> haystack_chunk_offsets, + size_t chunk_index) noexcept { + size_t low = 0, high = haystack_chunk_offsets.size() - 1; + while (low + 1 < high) { + size_t const mid = low + (high - low) / 2; + if (haystack_chunk_offsets[mid] <= chunk_index) low = mid; + else high = mid; + } + return low; +} + +/** + * @brief Walks one chunk's transitions, warming up `max_source_match_bytes - 1` bytes before @p chunk_begin - + * clamped to the haystack's own start, never earlier - so a match ending inside the chunk is found + * regardless of where its needle started, without reading another haystack. Counts or writes + * every match ending in `[chunk_begin, chunk_end)` whose start offset is still within this haystack, + * per @p pass_. + * @return The number of matches found in the chunk. + */ +template <typename state_id_type_, substrings_pass_t pass_> +SZ_DEVICE_INLINE size_t substrings_walk_chunk_( // + aho_corasick_view<state_id_type_> const &view, span<state_id_type_ const> shared_hot_rows, + span<u32_t const> accepts_words, span<byte_t const> haystack, size_t chunk_begin, size_t chunk_end, + size_t haystack_index, size_t output_base_offset, span<substrings_match_t> matches_out, + substrings_bm25_counters_t counters = {}) noexcept { + + // Offsets are relative to this haystack, so the warm-up clamps against its own start at zero. Sized in + // source bytes, the unit a haystack window is measured in, not in the folded bytes a needle is. + size_t const warm_up_bytes = view.max_source_match_bytes > 0 ? (size_t)view.max_source_match_bytes - 1 : 0; + size_t const walk_begin = chunk_begin >= warm_up_bytes ? chunk_begin - warm_up_bytes : 0; + + // Every 64-bit quantity is resolved here, once, and the per-byte loops below ride 32-bit deltas from it. + byte_t const *const walk_base = haystack.data() + walk_begin; + substrings_match_t *const matches_at_chunk = matches_out.data() + output_base_offset; + small_size_t const walk_span = static_cast<small_size_t>(chunk_end - walk_begin); + small_size_t const emit_from = static_cast<small_size_t>(chunk_begin - walk_begin); + sz_assert_(shared_hot_rows.size() / substrings_alphabet_size_k <= std::numeric_limits<small_size_t>::max() && + "The staged prefix is budgeted against one multiprocessor's shared memory in `try_build`"); + small_size_t const staged_rows_count = static_cast<small_size_t>(shared_hot_rows.size() / + substrings_alphabet_size_k); + + state_id_type_ state = view.root; // ? Fresh at walk_begin - no state ever crosses a haystack boundary. + small_size_t matches_found = 0; + + // Every match ending at `position` under the state just entered. The bit answers "does anything end + // here" - the common no-match byte never touches the global counts array, which at scale costs nearly + // as much as the tape read itself. Counts ride the state id; offsets index a pool that is O(states + // squared) and so stays 64-bit, but only as a base hoisted out of the inner loop. + auto const emit_matches_at = [&](small_size_t position) { + if (((accepts_words[state >> 5] >> (state & 31u)) & 1u) == 0) return; + state_id_type_ const output_count = view.outputs_counts[state]; + substrings_output<state_id_type_> const *const outputs_at_state = view.outputs + view.outputs_offsets[state]; + for (state_id_type_ output_index = 0; output_index < output_count; ++output_index) { + substrings_output<state_id_type_> const &output = outputs_at_state[output_index]; + // `walk_begin` is clamped to the haystack's own start, so underflowing the walk and underflowing + // the haystack are the same test - and this one needs no absolute offset. + if (position + 1 < static_cast<small_size_t>(output.folded_match_bytes)) continue; + if constexpr (pass_ == substrings_pass_t::writing_k) { + size_t const match_end = walk_begin + position + 1; + matches_at_chunk[matches_found] = substrings_match_t {haystack_index, (size_t)output.needle_index, + match_end - output.folded_match_bytes, + (size_t)output.folded_match_bytes}; + } + else if constexpr (pass_ == substrings_pass_t::counting_k) + substrings_bm25_count_(counters, (u32_t)output.needle_index); + ++matches_found; + } + }; + + // The warm-up primes the state and reports nothing, so once it ends the emit test vanishes from the + // loop rather than being re-asked on every byte. + small_size_t delta = 0; + for (; delta < emit_from; ++delta) + state = substrings_step_device_(view, shared_hot_rows, staged_rows_count, state, walk_base[delta]); + + // Peeled to the load's own alignment, so the body pays one 4-byte load per four transitions - the + // transition chain stays strictly serial; only the tape reads widen. + for (; delta < walk_span && ((size_t)(walk_base + delta) & 3u) != 0; ++delta) { + state = substrings_step_device_(view, shared_hot_rows, staged_rows_count, state, walk_base[delta]); + emit_matches_at(delta); + } + for (; delta + 4 <= walk_span; delta += 4) { + u32_vec_t const quad = sz_u32_load_aligned(walk_base + delta); +#pragma unroll + for (small_size_t lane = 0; lane < 4; ++lane) { + state = substrings_step_device_(view, shared_hot_rows, staged_rows_count, state, + static_cast<u8_t>(quad.u32 >> (lane * 8))); + emit_matches_at(delta + lane); + } + } + for (; delta < walk_span; ++delta) { + state = substrings_step_device_(view, shared_hot_rows, staged_rows_count, state, walk_base[delta]); + emit_matches_at(delta); + } + return matches_found; +} + +/** + * @brief Walks one chunk as folded bytes, the case-insensitive twin of `substrings_walk_chunk_`. + * + * Folding makes the walk restart-safe only at a codepoint start, so the warm-up snaps back to one before it + * begins - three bytes at most, and always earlier, so the extra transitions only prime state further. Match + * ends are reported at the source codepoint's end, which is what keeps chunk ownership comparable against + * the unsnapped `[chunk_begin, chunk_end)` the planner handed out. + */ +template <typename state_id_type_, substrings_pass_t pass_> +SZ_DEVICE_INLINE size_t substrings_walk_chunk_uncased_( // + aho_corasick_view<state_id_type_> const &view, span<state_id_type_ const> shared_hot_rows, + span<u32_t const> accepts_words, span<byte_t const> haystack, size_t chunk_begin, size_t chunk_end, + size_t haystack_index, size_t output_base_offset, span<substrings_match_t> matches_out, + substrings_bm25_counters_t counters = {}) noexcept { + + size_t const warm_up_bytes = view.max_source_match_bytes > 0 ? (size_t)view.max_source_match_bytes - 1 : 0; + size_t walk_begin = chunk_begin >= warm_up_bytes ? chunk_begin - warm_up_bytes : 0; + walk_begin = sz_utf8_rune_start_at_((cptr_t)haystack.data(), haystack.size(), walk_begin); + + substrings_match_t *const matches_at_chunk = matches_out.data() + output_base_offset; + small_size_t const staged_rows_count = static_cast<small_size_t>(shared_hot_rows.size() / + substrings_alphabet_size_k); + state_id_type_ state = view.root; + small_size_t matches_found = 0; + + span<byte_t const> const walked {haystack.data() + walk_begin, haystack.size() - walk_begin}; + substrings_folded_cursor_t cursor; + substrings_folded_cursor_init(cursor, walked.cast<char const>()); + + size_t folded = 0, last_break_folded_end = 0; + substrings_folded_byte_t step; + while (substrings_folded_cursor_next(cursor, step)) { + size_t const source_end = walk_begin + step.codepoint_end; + if (source_end > chunk_end) break; + ++folded; + if (step.malformed) { + state = view.root; + continue; + } + + state = substrings_step_device_(view, shared_hot_rows, staged_rows_count, state, step.byte); + if (!step.rune_end) continue; + if (step.breaks_boundary) last_break_folded_end = folded + step.trailing; + // The warm-up primes state without reporting, exactly as the byte-exact walk's prefix does. + if (source_end <= chunk_begin) continue; + if (((accepts_words[state >> 5] >> (state & 31u)) & 1u) == 0) continue; + + state_id_type_ const output_count = view.outputs_counts[state]; + substrings_output<state_id_type_> const *const outputs_at_state = view.outputs + view.outputs_offsets[state]; + for (state_id_type_ output_index = 0; output_index < output_count; ++output_index) { + substrings_output<state_id_type_> const &output = outputs_at_state[output_index]; + size_t const folded_length = output.folded_match_bytes; + if (folded < folded_length) continue; + + substrings_resolved_match_t const resolved = substrings_folded_span(walked.cast<char const>(), step, folded, + last_break_folded_end, folded_length); + if (resolved.repeats) continue; + size_t const match_offset = resolved.source_offset; + if constexpr (pass_ == substrings_pass_t::writing_k) + matches_at_chunk[matches_found] = substrings_match_t {haystack_index, (size_t)output.needle_index, + walk_begin + match_offset, + step.codepoint_end - match_offset}; + else if constexpr (pass_ == substrings_pass_t::counting_k) + substrings_bm25_count_(counters, (u32_t)output.needle_index); + ++matches_found; + } + } + return matches_found; +} + +/** + * @brief How many equal-sized chunks of `chunk_bytes` a haystack of @p haystack_length bytes needs - at + * least one, so even an empty or shorter-than-a-chunk haystack still gets a thread. + */ +constexpr size_t substrings_chunks_for_haystack_(size_t haystack_length, size_t chunk_bytes) noexcept { + return haystack_length == 0 ? (size_t)1 : divide_round_up(haystack_length, chunk_bytes); +} + +/** + * @brief Walks every chunk of every haystack, one thread per chunk, in whichever pass @p pass_ names. + * + * Both passes share one @p chunk_match_slots buffer, because the host's in-place exclusive scan already + * makes them the same allocation: the counting pass writes each chunk's match count into its slot, and the + * scattering pass reads the exclusive offset the scan left there. Every chunk owns a private, + * non-overlapping output range, so placing a write needs no atomics. + */ +template <typename state_id_type_, substrings_pass_t pass_> +__global__ void substrings_walk_per_cuda_chunk_(aho_corasick_view<state_id_type_> view, + state_id_type_ staged_rows_count, span<u32_t const> accepts_words, + u32_t staged_accepts_words, span<span<byte_t const> const> haystacks, + span<size_t const> haystack_chunk_offsets, size_t chunk_bytes, + size_t chunk_count, span<size_t> chunk_match_slots, + span<substrings_match_t> matches_out) { + extern __shared__ unsigned char substrings_shared_bytes_[]; + span<state_id_type_> const shared_hot_rows {reinterpret_cast<state_id_type_ *>(substrings_shared_bytes_), + (size_t)staged_rows_count * substrings_alphabet_size_k}; + // The bitmap words land right after the rows, whose byte count is a multiple of four at either id width. + span<u32_t> const shared_accepts_words { + reinterpret_cast<u32_t *>(substrings_shared_bytes_ + shared_hot_rows.size() * sizeof(state_id_type_)), + staged_accepts_words}; + substrings_stage_automaton_(view, shared_hot_rows, accepts_words, shared_accepts_words); + // Resolved once per block: the staged copy when `try_build` budgeted it in, the global array otherwise. + span<u32_t const> const accepts = staged_accepts_words + ? span<u32_t const> {shared_accepts_words.data(), accepts_words.size()} + : accepts_words; + + for (size_t chunk_index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; chunk_index < chunk_count; + chunk_index += (size_t)gridDim.x * blockDim.x) { + size_t const haystack_index = substrings_resolve_haystack_(haystack_chunk_offsets, chunk_index); + span<byte_t const> const haystack = haystacks[haystack_index]; + size_t const local_chunk_index = chunk_index - haystack_chunk_offsets[haystack_index]; + size_t const chunk_begin = local_chunk_index * chunk_bytes; + size_t const chunk_end = sz_min_of_two(chunk_begin + chunk_bytes, haystack.size()); + + size_t const output_base_offset = pass_ == substrings_pass_t::writing_k ? chunk_match_slots[chunk_index] + : (size_t)0; + // One dictionary is byte-exact or folded for its whole lifetime, so every thread takes the same + // side and the branch costs no divergence. Policy no longer reaches here: the walk emits every + // match, and a cover - when one is asked for - is resolved afterwards over what it emitted. + size_t const matches_in_chunk = view.case_sensitivity == substrings_uncased_k + ? substrings_walk_chunk_uncased_<state_id_type_, pass_>( // + view, shared_hot_rows, accepts, haystack, chunk_begin, chunk_end, + haystack_index, output_base_offset, matches_out) + : substrings_walk_chunk_<state_id_type_, pass_>( // + view, shared_hot_rows, accepts, haystack, chunk_begin, chunk_end, + haystack_index, output_base_offset, matches_out); + if constexpr (pass_ == substrings_pass_t::sizing_k) chunk_match_slots[chunk_index] = matches_in_chunk; + else sz_unused_(matches_in_chunk); + } +} + +/** + * @brief Decides which overlapping matches survive a leftmost cover, one segment per thread. + * + * A cover is a property of the matches, not of the bytes, so it is resolved here rather than inside the + * walk - where it cost every thread a ring wide enough for the longest match, and a second walk to find a + * safe place to start. Both are gone: the walk emits every match and this pass decides between them. + * + * Within a haystack the walk emits in non-decreasing end order, so the running maximum end is simply the + * previous match's end. A boundary sits there when nothing still to come reaches back across it - and only + * matches ending within `max_source_match_bytes` of it can, since no match is longer than that. So the test + * is bounded: look ahead while ends stay inside that window and check that no start falls behind. Ends + * alone would not do, because the list is ordered by end and a later match can begin earlier. + * + * Nothing before such a boundary can reach past it, so each segment resolves against a cursor of zero, + * independently of every other. Segments are short in real text - a needle set drawn from a vocabulary + * leaves a median of one match between boundaries - so one thread takes a whole one. + * + * That is a measurement, not a guarantee. A dictionary of a needle and its own suffixes over a repetitive + * haystack makes one segment of the whole document, and the greedy below is quadratic in a segment, so the + * scan is capped: past `substrings_cover_segment_limit_k` candidates a segment falls back to accepting in + * emitted order, which is the same cover whenever starts ascend with ends and a documented approximation + * when they do not. Without the cap one thread could hold the grid for the length of a document. + */ +static __global__ void substrings_cover_resolve_(span<substrings_match_t const> matches, size_t longest_match_bytes, + substrings_overlap_policy_t policy, span<size_t> keep) { + + // Whether the boundary before `index` is real: no match still to come starts before the maximum end + // already reached. Only matches ending within one match's length of it can, which bounds the look-ahead. + auto const boundary_before = [&](size_t index) noexcept { + if (index == 0) return true; + substrings_match_t const &previous = matches[index - 1]; + if (previous.haystack_index != matches[index].haystack_index) return true; + size_t const reached = previous.byte_offset + previous.byte_length; + for (size_t ahead = index; ahead < matches.size(); ++ahead) { + substrings_match_t const &candidate = matches[ahead]; + if (candidate.haystack_index != previous.haystack_index) break; + if (candidate.byte_offset + candidate.byte_length >= reached + longest_match_bytes) break; + if (candidate.byte_offset < reached) return false; + } + return true; + }; + + for (size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; index < matches.size(); + index += (size_t)gridDim.x * blockDim.x) { + + // Only a segment's first match works; the rest are decided by whoever owns their segment. + if (!boundary_before(index)) continue; + + size_t segment_end = index + 1; + for (; segment_end < matches.size(); ++segment_end) + if (boundary_before(segment_end)) break; + + // A segment past the cap is resolved in one linear sweep instead, so no thread can stall the grid. + if (segment_end - index > substrings_cover_segment_limit_k) { + size_t reached = 0; + for (size_t slot = index; slot < segment_end; ++slot) { + substrings_match_t const &candidate = matches[slot]; + bool const accepted = candidate.byte_offset >= reached; + keep[slot] = accepted; + if (accepted) reached = candidate.byte_offset + candidate.byte_length; + } + continue; + } + + // The greedy cover: take the earliest start at or past the cursor, breaking ties by policy, and + // repeat. Quadratic in the segment, which is why the segment is one thread's worth and no more. + for (size_t slot = index; slot < segment_end; ++slot) keep[slot] = 0; + size_t cursor = 0; + for (;;) { + size_t chosen = segment_end; + for (size_t slot = index; slot < segment_end; ++slot) { + substrings_match_t const &candidate = matches[slot]; + if (candidate.byte_offset < cursor) continue; + if (chosen == segment_end) { + chosen = slot; + continue; + } + substrings_match_t const &incumbent = matches[chosen]; + if (candidate.byte_offset != incumbent.byte_offset) { + if (candidate.byte_offset < incumbent.byte_offset) chosen = slot; + continue; + } + if (policy == substrings_leftmost_longest_k && candidate.byte_length != incumbent.byte_length) { + if (candidate.byte_length > incumbent.byte_length) chosen = slot; + continue; + } + if (candidate.needle_index < incumbent.needle_index) chosen = slot; + } + if (chosen == segment_end) break; + keep[chosen] = 1; + cursor = matches[chosen].byte_offset + matches[chosen].byte_length; + } + } +} + +/** + * @brief Gathers the surviving matches into their scanned slots, order preserved. + * @param[in] keep_offsets The scanned keep flags, one longer than @p matches so the last one has a successor. + * + * The scan overwrote the flags it summed, so survival is read back out of it: a match was kept exactly when + * the scan steps across it. + */ +static __global__ void substrings_cover_compact_(span<substrings_match_t const> matches, + span<size_t const> keep_offsets, span<substrings_match_t> survivors) { + for (size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; index < matches.size(); + index += (size_t)gridDim.x * blockDim.x) + if (keep_offsets[index + 1] > keep_offsets[index]) survivors[keep_offsets[index]] = matches[index]; +} + +/** + * @brief Maps each haystack's match range onto the boundaries its reported matches occupy. + * @param[in] keep_offsets The cover's scanned keep flags, or empty when every emitted match is reported. + */ +static __global__ void substrings_haystack_match_offsets_(span<size_t const> haystack_chunk_offsets, + span<size_t const> chunk_match_offsets, + span<size_t const> keep_offsets, + span<size_t> haystack_match_offsets) { + for (size_t haystack_index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + haystack_index < haystack_match_offsets.size(); haystack_index += (size_t)gridDim.x * blockDim.x) { + size_t const emitted_before = chunk_match_offsets[haystack_chunk_offsets[haystack_index]]; + haystack_match_offsets[haystack_index] = keep_offsets.size() ? keep_offsets[emitted_before] : emitted_before; + } +} + +/** @brief Writes how many matches each haystack owns, as the gap between its two boundaries. */ +static __global__ void substrings_counts_from_boundaries_(span<size_t const> haystack_match_offsets, + span<size_t> counts_per_haystack) { + for (size_t haystack_index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + haystack_index < counts_per_haystack.size(); haystack_index += (size_t)gridDim.x * blockDim.x) + counts_per_haystack[haystack_index] = haystack_match_offsets[haystack_index + 1] - + haystack_match_offsets[haystack_index]; +} + +/** + * @brief Writes where each match's preceding gap lands, and how long each haystack becomes. + * + * One block per haystack, threads striding its match range. A rewrite is a tiling of gaps and + * replacements, and every boundary in that tiling follows from one running quantity: how far the output + * has drifted from the input by the time a match is reached. So that drift is all this stores - one + * scanned offset per match - and the copy kernel derives the rest from the match list it already has. + * + * Offsets are relative to the haystack's own start, because the base is only known after the scan across + * haystacks that this kernel feeds. The copy kernel adds it, having looked the haystack up anyway. + */ +static __global__ void substrings_rewrite_offsets_per_haystack_( // + span<span<byte_t const> const> haystacks, span<size_t const> match_offsets, span<substrings_match_t const> matches, + span<size_t const> replacement_offsets, span<size_t> match_gap_offsets, span<size_t> output_sizes) { + using scan_t = cub::BlockScan<size_t, substrings_threads_per_block_k>; + __shared__ typename scan_t::TempStorage scan_storage; + __shared__ size_t drift_carry; + + for (size_t haystack_index = blockIdx.x; haystack_index < haystacks.size(); haystack_index += gridDim.x) { + size_t const first = match_offsets[haystack_index], last = match_offsets[haystack_index + 1]; + if (threadIdx.x == 0) drift_carry = 0; + __syncthreads(); + + for (size_t tile_first = first; tile_first < last; tile_first += blockDim.x) { + size_t const match_index = tile_first + threadIdx.x; + bool const owns_match = match_index < last; + size_t drift_here = 0, previous_end = 0; + if (owns_match) { + substrings_match_t const &match = matches[match_index]; + // Shrinking matches make this wrap, which is exactly right: only the prefix sums are ever + // read, every one of them names a real offset, and modular arithmetic reproduces each. + drift_here = replacement_offsets[match.needle_index + 1] - replacement_offsets[match.needle_index] - + match.byte_length; + previous_end = match_index == first + ? 0 + : matches[match_index - 1].byte_offset + matches[match_index - 1].byte_length; + } + + size_t drift_before = 0, drift_in_tile = 0; + scan_t(scan_storage).ExclusiveSum(drift_here, drift_before, drift_in_tile); + if (owns_match) match_gap_offsets[match_index] = previous_end + drift_carry + drift_before; + __syncthreads(); + if (threadIdx.x == 0) drift_carry += drift_in_tile; + __syncthreads(); + } + + // The scan's own aggregate is the haystack's total drift, so no second pass reduces what it knows. + if (threadIdx.x == 0) output_sizes[haystack_index] = haystacks[haystack_index].size() + drift_carry; + __syncthreads(); // ! The next haystack resets the carry this one is still reading. + } +} + +/** + * @brief Copies one stretch, clipped to `[tile_begin, tile_end)`, with @p lane striding the surviving bytes. + * + * A stretch that misses the tile entirely costs the clip and nothing else, which is what lets the caller + * hand every warp a stretch without first working out which ones land inside. + */ +SZ_DEVICE_INLINE void substrings_copy_clipped_(char *output, size_t tile_begin, size_t tile_end, size_t output_offset, + byte_t const *source, size_t bytes, unsigned lane) noexcept { + size_t const copy_begin = sz_max_of_two(output_offset, tile_begin); + size_t const copy_end = sz_min_of_two(output_offset + bytes, tile_end); + for (size_t position = copy_begin + lane; position < copy_end; position += 32) + output[position] = (char)source[position - output_offset]; +} + +/** @brief Index of the last entry at or below @p value, in an ascending array; zero when none is. */ +SZ_DEVICE_INLINE size_t substrings_last_not_above_(span<size_t const> ascending, size_t value) noexcept { + size_t low = 0, high = ascending.size(); + while (low + 1 < high) { + size_t const middle = low + (high - low) / 2; + if (ascending[middle] <= value) low = middle; + else high = middle; + } + return low; +} + +/** + * @brief Copies the rewritten tape, one fixed-width output tile per block, one warp per gap or replacement. + * + * Tiling the output rather than the matches bounds how long any one block works: a corpus of one huge + * document with a single match and a corpus of a million tiny ones give every block the same slice. Within + * a block the warps take stretches in parallel, because a rewrite over prose has stretches of tens of bytes + * and striding a whole block across one of them would leave most lanes idle. + */ +static __global__ void substrings_rewrite_copy_( // + span<span<byte_t const> const> haystacks, span<size_t const> match_offsets, span<substrings_match_t const> matches, + span<size_t const> match_gap_offsets, byte_t const *replacement_bytes, span<size_t const> replacement_offsets, + span<size_t const> output_offsets, size_t tile_bytes, char *output) { + + // Read on the device, so nothing about the tape's size has to reach the host before this launch. + size_t const output_bytes_total = output_offsets[output_offsets.size() - 1]; + size_t const tile_count = divide_round_up(output_bytes_total, tile_bytes); + unsigned const warp_index = threadIdx.x / 32u, warps_per_block = blockDim.x / 32u, lane = threadIdx.x % 32u; + + for (size_t tile_index = blockIdx.x; tile_index < tile_count; tile_index += gridDim.x) { + size_t const tile_begin = tile_index * tile_bytes; + size_t const tile_end = sz_min_of_two(tile_begin + tile_bytes, output_bytes_total); + + for (size_t haystack_index = substrings_last_not_above_(output_offsets, tile_begin); + haystack_index < haystacks.size() && output_offsets[haystack_index] < tile_end; ++haystack_index) { + + span<byte_t const> const haystack = haystacks[haystack_index]; + size_t const base = output_offsets[haystack_index]; + size_t const first = match_offsets[haystack_index], last = match_offsets[haystack_index + 1]; + + // Every match contributes a gap and a replacement; one more stretch closes the haystack. + size_t const stretches = last - first + 1; + size_t const wanted = tile_begin > base ? tile_begin - base : 0; + size_t const skip = first == last ? 0 + : substrings_last_not_above_( + {match_gap_offsets.data() + first, last - first}, wanted); + + // Past the last match the drift is whatever the whole haystack accumulated, which its rewritten + // length already names - so the closing stretch needs no offset of its own. + size_t const total_drift = (output_offsets[haystack_index + 1] - base) - haystack.size(); + + for (size_t stretch = skip + warp_index; stretch < stretches; stretch += warps_per_block) { + size_t const match_index = first + stretch; + bool const closes_haystack = match_index == last; + size_t const previous_end = match_index == first ? 0 + : matches[match_index - 1].byte_offset + + matches[match_index - 1].byte_length; + size_t const gap_source_end = closes_haystack ? haystack.size() : matches[match_index].byte_offset; + size_t const gap_begin = base + (closes_haystack ? previous_end + total_drift + : match_gap_offsets[match_index]); + + substrings_copy_clipped_(output, tile_begin, tile_end, gap_begin, haystack.data() + previous_end, + gap_source_end - previous_end, lane); + if (closes_haystack) continue; + + size_t const needle_index = matches[match_index].needle_index; + size_t const replacement_first = replacement_offsets[needle_index]; + substrings_copy_clipped_(output, tile_begin, tile_end, gap_begin + (gap_source_end - previous_end), + replacement_bytes + replacement_first, + replacement_offsets[needle_index + 1] - replacement_first, lane); + } + } + } +} + +/** + * @brief One BM25 score per haystack: a block tallies its haystack's needle frequencies, then reduces them. + * + * A block owns a haystack and its threads stride that haystack's chunks, so one long document still spreads + * across 256 lanes while the counter table stays private to the block and needs no cross-block traffic. + * Term frequencies are raw overlapping counts, which is what classic BM25 asks for - a cover would suppress + * genuine occurrences of a needle nested inside another - so no policy reaches here. + * + * The counters are the block's own shared table, sized by the document rather than by the dictionary, so + * scoring reads back what this haystack hit. Contributions accumulate as fixed-point integers into one + * shared total, whose addition is associative, so the total does not depend on the order lanes finish in. + */ +template <typename state_id_type_> +__global__ void substrings_score_bm25_per_haystack_(aho_corasick_view<state_id_type_> view, + state_id_type_ staged_rows_count, span<u32_t const> accepts_words, + u32_t staged_accepts_words, + span<span<byte_t const> const> haystacks, + span<f32_t const> document_lengths, substrings_bm25_t parameters, + span<f32_t const> needle_weights, span<u32_t> overflow_per_block, + span<f32_t> scores) { + + extern __shared__ unsigned char substrings_shared_bytes_[]; + span<state_id_type_> const shared_hot_rows {reinterpret_cast<state_id_type_ *>(substrings_shared_bytes_), + (size_t)staged_rows_count * substrings_alphabet_size_k}; + span<u32_t> const shared_accepts_words { + reinterpret_cast<u32_t *>(substrings_shared_bytes_ + shared_hot_rows.size() * sizeof(state_id_type_)), + (size_t)staged_accepts_words}; + substrings_stage_automaton_(view, shared_hot_rows, accepts_words, shared_accepts_words); + span<u32_t const> const accepts = staged_accepts_words + ? span<u32_t const> {shared_accepts_words.data(), shared_accepts_words.size()} + : accepts_words; + + size_t const needle_count = needle_weights.size(); + size_t const table_slots = sz_min_of_two(needle_count, substrings_bm25_slots_k); + span<substrings_match_t> const no_matches; + + // The table sits after the staged automaton, in the same dynamic allocation the host sized for both. + substrings_bm25_counter_t *const slots = reinterpret_cast<substrings_bm25_counter_t *>( + substrings_shared_bytes_ + shared_hot_rows.size() * sizeof(state_id_type_) + + (size_t)staged_accepts_words * sizeof(u32_t)); + __shared__ u64_t block_score_fixed; + __shared__ int block_overflowed; + + // Allocated only for a dictionary wider than the table; empty otherwise, draining this clear to nothing. + span<u32_t> const overflow = overflow_per_block.size() + ? span<u32_t> {overflow_per_block.data() + (size_t)blockIdx.x * needle_count, + needle_count} + : span<u32_t> {}; + for (size_t needle_index = threadIdx.x; needle_index < overflow.size(); needle_index += blockDim.x) + overflow[needle_index] = 0; + __syncthreads(); + + for (size_t haystack_index = blockIdx.x; haystack_index < haystacks.size(); haystack_index += gridDim.x) { + span<byte_t const> const haystack = haystacks[haystack_index]; + + for (size_t slot = threadIdx.x; slot < table_slots; slot += blockDim.x) + slots[slot] = substrings_bm25_counter_t {substrings_bm25_empty_slot_k, 0u}; + if (threadIdx.x == 0) { + block_score_fixed = 0ull; + block_overflowed = 0; + } + __syncthreads(); + + substrings_bm25_counters_t const counters {slots, overflow, &block_overflowed}; + + // One chunk per lane where the haystack allows it, floored at one match width so no chunk re-walks + // more warm-up than it covers. A width derived from the corpus cannot answer this: sized from the + // mean, a haystack shorter than the mean leaves most of the block with nothing to walk. + size_t const chunk_bytes = sz_max_of_two(divide_round_up(haystack.size(), (size_t)blockDim.x), + sz_max_of_two((size_t)view.max_source_match_bytes, (size_t)1)); + size_t const chunks = substrings_chunks_for_haystack_(haystack.size(), chunk_bytes); + for (size_t chunk = threadIdx.x; chunk < chunks; chunk += blockDim.x) { + size_t const chunk_begin = chunk * chunk_bytes; + size_t const chunk_end = sz_min_of_two(chunk_begin + chunk_bytes, haystack.size()); + if (view.case_sensitivity == substrings_uncased_k) + substrings_walk_chunk_uncased_<state_id_type_, substrings_pass_t::counting_k>( + view, shared_hot_rows, accepts, haystack, chunk_begin, chunk_end, haystack_index, 0, no_matches, + counters); + else + substrings_walk_chunk_<state_id_type_, substrings_pass_t::counting_k>( + view, shared_hot_rows, accepts, haystack, chunk_begin, chunk_end, haystack_index, 0, no_matches, + counters); + } + __syncthreads(); + + // A needle this document never hit scores `+0`, so skipping free slots is exact, and it keeps + // `substrings_bm25_term` away from a zero frequency under a zero normalized length, where it is `0/0`. + f32_t const document_length = document_lengths.size() ? document_lengths[haystack_index] + : (f32_t)haystack.size(); + auto contribution_of = [&](u32_t needle_index, u32_t frequency) noexcept { + return substrings_bm25_to_fixed_(needle_weights[needle_index] * + substrings_bm25_term(parameters, (f32_t)frequency, document_length)); + }; + + // A seated slot is always incremented before this sync, so a zero frequency means untouched in both + // layouts - and a slot's needle is its own index when the dictionary got one slot each. + i64_t partial_fixed = 0; + for (size_t slot = threadIdx.x; slot < table_slots; slot += blockDim.x) { + u32_t const frequency = slots[slot].frequency; + if (frequency == 0u) continue; + partial_fixed += contribution_of(overflow.empty() ? (u32_t)slot : slots[slot].needle_index, frequency); + } + + // Only a document that outgrew the table ever dirties the overflow row, so only that document pays a + // pass over the vocabulary - and a document that large already amortizes it over its own bytes. + if (block_overflowed) + for (size_t needle_index = threadIdx.x; needle_index < overflow.size(); needle_index += blockDim.x) { + u32_t const frequency = overflow[needle_index]; + if (frequency == 0u) continue; + partial_fixed += contribution_of((u32_t)needle_index, frequency); + overflow[needle_index] = 0u; + } + + cuda_increment_shared_(&block_score_fixed, (u64_t)partial_fixed); + __syncthreads(); + if (threadIdx.x == 0) scores[haystack_index] = substrings_bm25_from_fixed_((i64_t)block_score_fixed); + __syncthreads(); // ! The next haystack clears this table and reuses this total. + } +} + +#pragma endregion Device Kernels + +#pragma region Engine + +/** + * @brief Aho-Corasick-based @b GPU multi-pattern exact/case-folded substring search. + * @tparam allocator_type_ The allocator backing this engine's automaton and device-resident scratch; unified + * memory by default, so the host can read match totals straight back after a stream synchronize. + * @tparam capability_ Any capability including `sz_cap_cuda_k` - the kernels need no generation-specific + * instructions, so every combination shares this specialization. + * + * The automaton needs no upload: `allocator_t` already places the dictionary's arrays where the kernels read + * them, so the same `aho_corasick_dictionary` the host builds is the one the device walks. The state-id width + * follows from the needle set rather than from a template argument, so the engine holds whichever of the two + * automatons `try_build` settled on. + * + * Move-only and owns its scratch: the automaton, and the per-call chunk-planning buffers. A moved-from engine + * holds no device memory and must not be used before another build. + */ +template < // + typename allocator_type_ = unified_alloc_t, // + sz_capability_t capability_ = sz_cap_cuda_k, // + typename enable_ = void // + > +struct substrings_cuda; + +template <typename allocator_type_, sz_capability_t capability_> +struct substrings_cuda<allocator_type_, capability_, std::enable_if_t<(capability_ & sz_cap_cuda_k) != 0>> { + + using allocator_t = allocator_type_; + using narrow_dictionary_t = aho_corasick_dictionary<u16_t, allocator_t>; + using wide_dictionary_t = aho_corasick_dictionary<u32_t, allocator_t>; + using match_t = substrings_match_t; + static constexpr sz_capability_t capability_k = capability_; + + /** + * @brief How far beyond `state_count` the cold tier's `base`/`check`/`fail`/`outputs_counts`/ + * `outputs_offsets` arrays must extend. + * + * A cold transition's target is `base[state] + byte` for `byte` in `[0, 256)`, and the builder guarantees + * `base[state] < state_count`, so the highest slot ever addressed is `state_count + 254`. + */ + static constexpr size_t substrings_cold_slot_headroom_k = substrings_alphabet_size_k - 1; + + private: + using allocator_traits_t = std::allocator_traits<allocator_t>; + /** @brief Rebinds to `size_t`, for the output CSR and the chunk offsets, neither of which has a ceiling. */ + using offset_allocator_t = typename allocator_traits_t::template rebind_alloc<size_t>; + using descriptor_allocator_t = typename allocator_traits_t::template rebind_alloc<span<byte_t const>>; + using word_allocator_t = typename allocator_traits_t::template rebind_alloc<u32_t>; + using byte_allocator_t = typename allocator_traits_t::template rebind_alloc<byte_t>; + /* Scratch no host ever touches - written by one kernel, read by the next, or drained by a copy. Unified + * memory would fault it in on first touch and migrate it again on the drain, so it is device-resident by + * the algorithm's nature rather than by the caller's allocator choice. @sa `similarities/cuda.cuh`. */ + using device_match_allocator_t = device_alloc<substrings_match_t>; + using device_offset_allocator_t = device_alloc<size_t>; + using device_byte_allocator_t = device_alloc<byte_t>; + using device_word_allocator_t = device_alloc<u32_t>; + + /** + * @brief Dense acceptance bitmap: bit `state` is set when some needle ends at that state, hot or cold. + * + * Derived from `outputs_counts` once the automaton is built, so the per-byte walk gate never touches that + * 32x larger array. Words are 32-bit because that is one shared-memory bank: the gate reads exactly one + * bank-wide word, and lanes clustered near the root share it as a broadcast rather than a conflict. + */ + safe_vector<u32_t, word_allocator_t> accepts_words_ {}; + + /** @brief One descriptor per haystack. Unified, as the host writes them and every chunk thread reads them. */ + safe_vector<span<byte_t const>, descriptor_allocator_t> haystack_descriptors_ {}; + /** @brief Per-call scratch: chunk count per haystack, then - in place - the exclusive chunk-index offset + * per haystack, with the grand total chunk count trailing at `[haystack_count]`. */ + safe_vector<size_t, offset_allocator_t> haystack_chunk_offsets_ {}; + /** @brief Per-call scratch: holds per-chunk counts, then - in place - per-chunk exclusive offsets, with the + * grand total in the trailing slot. Grown as needed, reused across calls. */ + safe_vector<size_t, offset_allocator_t> chunk_match_offsets_ {}; + /** + * @brief Every match the walk emitted, before any cover has been applied to them. + * + * Under a cover this is an intermediate rather than a result - all three entry points walk into it and + * then decide between what it holds - so it is engine scratch the caller never sees. + */ + safe_vector<substrings_match_t, device_match_allocator_t> emitted_matches_ {}; + /** @brief One flag per emitted match going in, its scanned slot coming out, with the survivor count + * trailing - the same in-place trick `chunk_match_offsets_` plays, and for the same reason. */ + safe_vector<size_t, device_offset_allocator_t> cover_keep_ {}; + /** @brief Tile totals for the multi-block route of `cuda_launch_exclusive_sum_`, sized once at its grid + * ceiling so no scan ever has to fall back to one block for want of scratch. */ + safe_vector<size_t, device_offset_allocator_t> scan_partials_ {}; + /** @brief The survivors themselves, gathered out of the emitted list. */ + safe_vector<substrings_match_t, device_match_allocator_t> cover_survivors_ {}; + /** @brief Per-haystack match boundaries, the reported twin of `haystack_chunk_offsets_`. */ + safe_vector<size_t, offset_allocator_t> haystack_match_offsets_ {}; + /** @brief Where each match's preceding gap begins, relative to its haystack's rewritten start. The only + * thing the copy kernel cannot recompute, since it is the running drift the scan produced. */ + safe_vector<size_t, device_offset_allocator_t> rewrite_gap_offsets_ {}; + /** @brief The replacements as one tape, uploaded per call: the caller's container is host-addressed. */ + safe_vector<byte_t, byte_allocator_t> replacement_bytes_ {}; + /** @brief Where each needle's replacement starts in `replacement_bytes_`, with a trailing terminator. */ + safe_vector<size_t, offset_allocator_t> replacement_offsets_ {}; + + /** @brief One `needle_count`-wide row per resident block, catching what will not seat in that block's + * table. Empty for any dictionary the table can hold, which is most of them. */ + safe_vector<u32_t, device_word_allocator_t> bm25_overflow_ {}; + + /** + * @brief The automaton this engine compiles from its needles, at whichever state-id width it fits. + * + * `allocator_t` places its arrays where the kernels read them, so `dictionary_.view()` is already the + * device view - there is no second copy to keep in step with it. + */ + std::variant<narrow_dictionary_t, wide_dictionary_t> dictionary_; + + /** @brief Hot rows staged into shared memory at block start - all of them, or zero when they would cost + * a resident block and the kernels read them through the cache instead. */ + u32_t staged_rows_ {}; + /** @brief Acceptance bitmap words staged alongside `staged_rows_`, under the same all-or-nothing rule. */ + u32_t staged_accepts_words_ {}; + /** @brief Resident blocks per multiprocessor to budget shared memory against; zero derives it from the + * occupancy the kernel reaches with none, which is the default the accessor below overrides. */ + unsigned target_blocks_per_multiprocessor_ = 0; + allocator_t alloc_ {}; + cuda_timer_t timer_ {}; + + public: + substrings_cuda() noexcept = default; + substrings_cuda(substrings_cuda const &) = delete; + substrings_cuda &operator=(substrings_cuda const &) = delete; + substrings_cuda(substrings_cuda &&) noexcept = default; + substrings_cuda &operator=(substrings_cuda &&) noexcept = default; + + /** @brief Releases the automaton and every device-resident buffer this engine owns; a fresh + * `try_insert_all` is required after. */ + void reset() noexcept { + std::visit([](auto &dictionary) noexcept { dictionary.reset(); }, dictionary_); + accepts_words_.reset(); + haystack_descriptors_.reset(); + haystack_chunk_offsets_.reset(); + chunk_match_offsets_.reset(); + emitted_matches_.reset(); + cover_keep_.reset(); + scan_partials_.reset(); + cover_survivors_.reset(); + haystack_match_offsets_.reset(); + rewrite_gap_offsets_.reset(); + replacement_bytes_.reset(); + replacement_offsets_.reset(); + bm25_overflow_.reset(); + staged_rows_ = u32_t {}; + staged_accepts_words_ = u32_t {}; + } + + /** @brief Overrides the resident-blocks-per-multiprocessor target the hot tier is budgeted against when the + * engine finalizes; call before the first operation, with zero restoring the automatic choice. */ + void target_blocks_per_multiprocessor(unsigned desired) noexcept { target_blocks_per_multiprocessor_ = desired; } + unsigned target_blocks_per_multiprocessor() const noexcept { return target_blocks_per_multiprocessor_; } + + /** @brief The state-id width this engine's automaton settled on, once finalized. */ + substrings_state_width_t state_width() const noexcept { + return std::holds_alternative<narrow_dictionary_t>(dictionary_) ? substrings_state_width_t::u16_k + : substrings_state_width_t::u32_k; + } + size_t count_needles() const noexcept { + return std::visit([](auto const &dictionary) noexcept { return dictionary.count_needles(); }, dictionary_); + } + size_t count_states() const noexcept { + return std::visit([](auto const &dictionary) noexcept { return dictionary.count_states(); }, dictionary_); + } + size_t max_source_match_bytes() const noexcept { + return std::visit([](auto const &dictionary) noexcept { return (size_t)dictionary.max_source_match_bytes(); }, + dictionary_); + } + size_t min_source_match_bytes() const noexcept { + return std::visit([](auto const &dictionary) noexcept { return (size_t)dictionary.min_source_match_bytes(); }, + dictionary_); + } + size_t hot_count() const noexcept { + return std::visit([](auto const &dictionary) noexcept { return dictionary.hot_count(); }, dictionary_); + } + + /** @brief Runs @p callable against the automaton at whichever state-id width it settled on. */ + template <typename callable_type_> + auto visit_dictionary(callable_type_ &&callable) const noexcept { + return std::visit(std::forward<callable_type_>(callable), dictionary_); + } + +#pragma region Kernel Table + + struct kernels_t { + /** @brief One shape per state-id width, for the three kernels that walk the automaton. The cover and + * rewrite kernels below take no view, so they are shared. @sa `levenshtein_distances::kernels_t`, + * which lists its cell widths the same way. */ + struct by_width_t { + kernel_shape_t u16, u32; + + kernel_shape_t const &for_width(substrings_state_width_t width) const noexcept { + return width == substrings_state_width_t::u16_k ? u16 : u32; + } + }; + by_width_t count_chunk; + by_width_t scatter_chunk; + exclusive_sum_shapes_t exclusive_sum; + kernel_shape_t cover_resolve; + kernel_shape_t cover_compact; + kernel_shape_t haystack_match_offsets; + kernel_shape_t counts_from_boundaries; + kernel_shape_t rewrite_offsets; + kernel_shape_t rewrite_copy; + by_width_t score_bm25; + }; + + /** @brief Resolves every kernel handle for @p device_id into @p table, raising the dynamic shared-memory + * ceiling on the two chunk kernels to the device's opt-in maximum. The per-launch allocation + * depends on the dictionary's hot-tier size, so occupancy is queried per launch. + * Both state-id widths are resolved into one table, since the automaton picks its own. */ + static cuda_status_t resolve_kernels_(kernels_t &table, int device_id) noexcept { + CUdevice const device = device_id; + int shared_memory_ceiling = 0; + cuDeviceGetAttribute(&shared_memory_ceiling, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, device); + + cuda_status_t status {status_t::success_k, cudaSuccess}; + + // One lambda per kernel family, invoked once per width, as `resolve_warp` does in `similarities/cuda.cuh`. + auto const resolve_walk = + [&]<typename state_id_type_, substrings_pass_t pass_>(kernel_shape_t &shape) noexcept -> cuda_status_t { + return resolve_kernel_shape( + shape, reinterpret_cast<void const *>(&substrings_walk_per_cuda_chunk_<state_id_type_, pass_>), 0, + static_cast<unsigned>(shared_memory_ceiling), false); + }; + status = resolve_walk.template operator()<u16_t, substrings_pass_t::sizing_k>(table.count_chunk.u16); + if (status.status != status_t::success_k) return status; + status = resolve_walk.template operator()<u32_t, substrings_pass_t::sizing_k>(table.count_chunk.u32); + if (status.status != status_t::success_k) return status; + status = resolve_walk.template operator()<u16_t, substrings_pass_t::writing_k>(table.scatter_chunk.u16); + if (status.status != status_t::success_k) return status; + status = resolve_walk.template operator()<u32_t, substrings_pass_t::writing_k>(table.scatter_chunk.u32); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.exclusive_sum.whole, + reinterpret_cast<void const *>(&exclusive_sum_across_cuda_device_<size_t>), 0, 0, + false); + if (status.status != status_t::success_k) return status; + + // The tiled phases pick their grid from this occupancy, so unlike the whole-array kernel they precompute it. + status = resolve_kernel_shape( + table.exclusive_sum.reduce_tiles, + reinterpret_cast<void const *>(&exclusive_sum_reduce_tiles_across_cuda_device_<size_t>), + cuda_device_collective_threads_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape( + table.exclusive_sum.apply_tiles, + reinterpret_cast<void const *>(&exclusive_sum_apply_tiles_across_cuda_device_<size_t>), + cuda_device_collective_threads_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.cover_resolve, reinterpret_cast<void const *>(&substrings_cover_resolve_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.cover_compact, reinterpret_cast<void const *>(&substrings_cover_compact_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.haystack_match_offsets, + reinterpret_cast<void const *>(&substrings_haystack_match_offsets_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.counts_from_boundaries, + reinterpret_cast<void const *>(&substrings_counts_from_boundaries_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + // The rewrite kernels carry no automaton, so they stage nothing, need no raised ceiling, and - unlike + // the walk - are not shared-memory-bound. Their own occupancy is precomputed here so neither has to + // borrow a grid sized for a dictionary's hot tier. + status = resolve_kernel_shape(table.rewrite_offsets, + reinterpret_cast<void const *>(&substrings_rewrite_offsets_per_haystack_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + status = resolve_kernel_shape(table.rewrite_copy, reinterpret_cast<void const *>(&substrings_rewrite_copy_), + substrings_threads_per_block_k, 0, true); + if (status.status != status_t::success_k) return status; + + // Scoring stages the same automaton the walk does, so it wants the same opt-in shared ceiling. + auto const resolve_score = [&]<typename state_id_type_>(kernel_shape_t &shape) noexcept -> cuda_status_t { + return resolve_kernel_shape( + shape, reinterpret_cast<void const *>(&substrings_score_bm25_per_haystack_<state_id_type_>), 0, + static_cast<unsigned>(shared_memory_ceiling), false); + }; + status = resolve_score.template operator()<u16_t>(table.score_bm25.u16); + if (status.status != status_t::success_k) return status; + return resolve_score.template operator()<u32_t>(table.score_bm25.u32); + } + + /** @brief This device's kernel table, resolved on first use. Check the status before reading it - a full + * cache hands back an unresolved table. @sa cuda_device_kernels */ + static expected<kernels_t const &, cuda_status_t> kernels(int device_id) noexcept { + static cuda_device_kernels<kernels_t> per_device; + auto *entry = per_device.acquire(device_id); + if (!entry) return {per_device.unusable(), {status_t::missing_gpu_k, cudaSuccess, CUDA_ERROR_INVALID_DEVICE}}; + if (!entry->resolved) { + cuda_status_t const status = resolve_kernels_(entry->table, device_id); + if (status.status != status_t::success_k) { + per_device.release(); + return {per_device.unusable(), status}; + } + entry->resolved = true; + } + per_device.release(); + return {entry->table, {}}; + } + +#pragma endregion Kernel Table + + /** + * @brief Indexes all of the @p needles strings into the FSM, at whichever state id it ends up fitting. + * + * No upload follows: `allocator_t` is unified memory, reachable from every device, so the arrays this + * builds are already the ones the kernels walk. Construction runs wide, because a dictionary's state + * count is only known once it is built, and the narrowing attempt is itself the ceiling test. + * @param[in] executor Names the device whose context the non-unified scan scratch is allocated under. + * @param[in] specs Sizes the hot tier against that device's L2, the cache its walk reads through. + * @note Replaces any previously indexed needle set: the automaton is rebuilt from scratch and the old one + * released, so an engine can be re-indexed for a different vocabulary or a different device. + * @sa `aho_corasick_dictionary::try_insert` for the status codes this forwards. + */ + template <typename needles_type_> + cuda_status_t try_index(needles_type_ const &needles, + substrings_case_sensitivity_t case_sensitivity = substrings_cased_k, + cuda_executor_t const &executor = {}, gpu_specs_t const &specs = {}) noexcept { + // The scan scratch below is device-resident rather than unified, so it lands wherever the context + // points; binding the named device first is what keeps it off whichever one happened to be current. + if (cuda_status_t const current = executor.ensure_current(); current.status != status_t::success_k) + return current; + wide_dictionary_t wide(alloc_); + wide.case_sensitivity(case_sensitivity); + for (auto const &needle : needles) { + status_t const status = wide.try_insert(to_bytes_view(needle)); + if (status != status_t::success_k) return {status, cudaSuccess}; + } + // The tier split follows the cache the device walks through rather than the host's last level, which + // a default `cpu_specs_t` would put at 8 MB whatever the GPU. + wide.hot_count(specs.l2_bytes / (substrings_alphabet_size_k * sizeof(u32_t))); + if (status_t const built = wide.try_build(); built != status_t::success_k) return {built, cudaSuccess}; + + narrow_dictionary_t narrow(alloc_); + status_t const narrowed = narrow.try_build(wide); + if (narrowed != status_t::success_k && narrowed != status_t::overflow_risk_k) return {narrowed, cudaSuccess}; + if (narrowed == status_t::success_k) dictionary_.template emplace<narrow_dictionary_t>(std::move(narrow)); + else dictionary_.template emplace<wide_dictionary_t>(std::move(wide)); + + // Sized at the scan's grid ceiling rather than per call: it is 8 KB whatever the corpus, and a scan + // that found it short would silently drop back to one block. + if (scan_partials_.try_resize_uninitialized(cuda_device_collective_max_blocks_k + 1) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + return try_derive_accepts_(); + } + + private: + /** @brief One cell's width in the settled automaton, which is what a staged hot row costs per entry. */ + size_t bytes_per_state_id_() const noexcept { + return state_width() == substrings_state_width_t::u16_k ? sizeof(u16_t) : sizeof(u32_t); + } + + /** @brief Shared bytes a scoring block's counter table costs, read by the occupancy query and by the + * launch alike so the two cannot disagree about the footprint they are sizing. A dictionary + * narrower than the table gets a slot per needle and pays for no more than that. */ + size_t substrings_bm25_counters_bytes_() const noexcept { + return sz_min_of_two(count_needles(), substrings_bm25_slots_k) * sizeof(substrings_bm25_counter_t); + } + + /** @brief The settled automaton at @p state_id_type_, which `try_build` has already pinned. */ + template <typename state_id_type_> + aho_corasick_dictionary<state_id_type_, allocator_t> const &settled_dictionary_() const noexcept { + return std::get<aho_corasick_dictionary<state_id_type_, allocator_t>>(dictionary_); + } + + /** + * @brief Builds the dense acceptance bitmap the per-byte walk gate reads. + * + * A pure function of the built automaton - no device involved - so it belongs to the build rather than + * to whichever executor happens to arrive first. The gate reads it instead of `outputs_counts` because + * that array is 32x larger. + */ + cuda_status_t try_derive_accepts_() noexcept { + return visit_dictionary([&](auto const &dictionary) noexcept -> cuda_status_t { + auto const view = dictionary.view(); + size_t const state_capacity = (size_t)view.state_count + substrings_cold_slot_headroom_k; + // `try_resize` leaves trivial words uninitialized, so every word is written before any bit is set. + size_t const words_count = divide_round_up<size_t>(state_capacity, 32); + if (accepts_words_.try_resize(words_count) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + for (size_t word = 0; word < words_count; ++word) accepts_words_[word] = 0; + for (size_t slot = 0; slot < state_capacity; ++slot) + if (view.outputs_counts[slot] != 0) accepts_words_[slot >> 5] |= u32_t(1) << (slot & 31u); + return {status_t::success_k, cudaSuccess}; + }); + } + + /** + * @brief Settles how much of the automaton a block stages in shared memory on @p executor 's device. + * + * Budgeted per call rather than once per build, because occupancy and the shared-memory ceiling belong to + * the device the caller names - and nothing stops two calls on one engine from naming different ones. + */ + cuda_status_t try_budget_staging_(cuda_executor_t const &executor) noexcept { + auto [kernel_table, kernels_status] = kernels(executor.device_id()); + if (kernels_status.status != status_t::success_k) return kernels_status; + CUfunction const walk_function = + kernel_table.count_chunk.for_width(state_width()).function; // ? The scatter kernel shares its shape + + // Occupancy first, staging only out of what is left over. The walk chases a data-dependent transition + // load, so resident warps are the only thing hiding its latency, while the rows it would stage are + // cache-resident already - which makes a block traded away for shared memory a straight loss. + unsigned target_blocks = target_blocks_per_multiprocessor_; + if (target_blocks == 0) { + int blocks_without_staging = 0; + CUresult const occupancy_error = cuOccupancyMaxActiveBlocksPerMultiprocessor( + &blocks_without_staging, walk_function, (int)substrings_threads_per_block_k, 0); + if (occupancy_error != CUDA_SUCCESS) return make_cuda_status(occupancy_error); + target_blocks = (unsigned)sz_max_of_two(blocks_without_staging, 1); + } + + size_t shared_memory_budget = 0; + cuda_status_t const budget_status = shared_memory_budget_for_resident_blocks( + shared_memory_budget, walk_function, substrings_threads_per_block_k, target_blocks, executor.device_id()); + if (budget_status.status != status_t::success_k) return budget_status; + + // Staging is all-or-nothing: a partial prefix still pays the copy per block and the bounds test per + // byte, while most transitions miss it and fall through to memory anyway. + size_t const bytes_per_state_id = state_width() == substrings_state_width_t::u16_k ? sizeof(u16_t) + : sizeof(u32_t); + size_t const hot_rows = hot_count(); + size_t const accepts_bytes = accepts_words_.size() * sizeof(u32_t); + size_t const whole_automaton_bytes = hot_rows * substrings_alphabet_size_k * bytes_per_state_id + accepts_bytes; + // Scoring carries its counter table in the same allocation, so staging must fit beside it or the two + // would compete for one budget. They do not today - staging is refused for every real dictionary - + // but that is luck rather than design, and this keeps it true by construction. + size_t const staging_budget = shared_memory_budget > substrings_bm25_counters_bytes_() + ? shared_memory_budget - substrings_bm25_counters_bytes_() + : 0; + bool const stages_whole_automaton = whole_automaton_bytes <= staging_budget; + staged_rows_ = stages_whole_automaton ? static_cast<u32_t>(hot_rows) : u32_t {0}; + staged_accepts_words_ = stages_whole_automaton ? static_cast<u32_t>(accepts_words_.size()) : u32_t {0}; + + return {status_t::success_k, cudaSuccess}; + } + + /** @brief Everything the counting pass establishes that a following scatter pass still needs. */ + struct planned_pass_t { + kernels_t kernel_table {}; + unsigned shared_memory_bytes = 0; + unsigned blocks_per_grid = 0; + size_t chunk_bytes = 0; + size_t chunk_count = 0; + /** @brief False when the corpus is empty, so the caller returns success without launching anything. */ + bool has_work = false; + }; + + /** + * @brief Copies the caller's replacements into one unified tape the kernels can address. + * + * Unlike haystacks, which are validated in place, replacements arrive through a callback-addressed + * container in host memory, so they have to be materialized. The needle set is query-sized and this + * runs against a whole corpus walk, so it is re-uploaded per call rather than cached and invalidated. + */ + template <typename replacements_type_> + cuda_status_t upload_replacements_(replacements_type_ const &replacements) noexcept { + size_t const needle_count = count_needles(); + if (replacements.size() != needle_count) return {status_t::unexpected_dimensions_k, cudaSuccess}; + + size_t total_bytes = 0; + for (size_t needle_index = 0; needle_index < needle_count; ++needle_index) + total_bytes += to_bytes_view(replacements[needle_index]).size(); + + if (replacement_offsets_.try_resize_uninitialized(needle_count + 1) != status_t::success_k || + replacement_bytes_.try_resize_uninitialized(sz_max_of_two(total_bytes, (size_t)1)) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + size_t written = 0; + for (size_t needle_index = 0; needle_index < needle_count; ++needle_index) { + span<byte_t const> const replacement = to_bytes_view(replacements[needle_index]); + replacement_offsets_[needle_index] = written; + if (replacement.size()) + sz_copy((sz_ptr_t)(replacement_bytes_.data() + written), (sz_cptr_t)replacement.data(), + replacement.size()); + written += replacement.size(); + } + replacement_offsets_[needle_count] = written; + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Records one pointer-and-length descriptor per haystack, sums their bytes, and reports the + * longest of them - which is what bounds the widest chunk any single haystack can ask for. + * + * No layout is assumed: haystacks may sit in one tape or in separate allocations, and the input bytes + * are validated rather than copied, as in every other CUDA engine here. + */ + template <typename haystacks_type_> + cuda_status_t describe_haystacks_(haystacks_type_ const &haystacks, size_t &total_bytes, + size_t &longest_bytes) noexcept { + if (haystack_descriptors_.try_resize_uninitialized(haystacks.size()) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + total_bytes = 0; + longest_bytes = 0; + + bool probed = false; + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + span<byte_t const> const haystack = to_bytes_view(haystacks[haystack_index]); + haystack_descriptors_[haystack_index] = haystack; + total_bytes += haystack.size(); + longest_bytes = sz_max_of_two(longest_bytes, haystack.size()); + // The probe is a driver round-trip, so one non-empty element decides for the whole batch. + if (!probed && haystack.size() != 0) { + if (!is_device_accessible_memory((void const *)haystack.data())) + return {status_t::device_memory_mismatch_k, cudaSuccess}; + probed = true; + } + } + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Runs everything `try_count` and `try_find` share: the guards, the kernel table, the timer's + * start, the chunk plan, and the counting pass whose exclusive scan both then read. + */ + cuda_status_t plan_and_count_(size_t total_bytes, cuda_executor_t const &executor, gpu_specs_t const &specs, + planned_pass_t &pass) noexcept { + cuda_status_t const current_status = executor.ensure_current(); + if (current_status.status != status_t::success_k) return current_status; + if (haystack_descriptors_.size() == 0 || total_bytes == 0) return {status_t::success_k, cudaSuccess}; + + // The staging budget belongs to the device this call names, so it is settled here rather than at build. + if (cuda_status_t const budgeted = try_budget_staging_(executor); budgeted.status != status_t::success_k) + return budgeted; + + auto [kernel_table, kernels_status] = kernels(executor.device_id()); + if (kernels_status.status != status_t::success_k) return kernels_status; + pass.kernel_table = kernel_table; + + CUresult const timer_error = timer_.ensure_created(executor.device_id()); + if (timer_error != CUDA_SUCCESS) return make_cuda_status(timer_error); + CUresult const start_error = timer_.record_start(executor.stream()); + if (start_error != CUDA_SUCCESS) return make_cuda_status(start_error); + + cuda_status_t const plan_status = plan_haystack_chunks_(total_bytes, specs, pass.kernel_table, + pass.shared_memory_bytes, pass.blocks_per_grid, + pass.chunk_bytes, pass.chunk_count); + if (plan_status.status != status_t::success_k) return plan_status; + + cuda_status_t const count_status = count_into_offsets_(executor, specs, pass.kernel_table, + pass.shared_memory_bytes, pass.blocks_per_grid, + pass.chunk_bytes, pass.chunk_count); + if (count_status.status != status_t::success_k) return count_status; + + pass.has_work = true; + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Bytes a rewrite of @p input_bytes can reach, from the dictionary and @p replacements alone. + * @sa `szs_substrings_replace_bound`, whose arithmetic this mirrors. + * + * A caller sizing its output span to this cannot be refused, which lets `try_replace` skip the host + * round-trip its capacity check would otherwise need. + */ + template <typename replacements_type_> + size_t replace_bound_host_(size_t input_bytes, replacements_type_ const &replacements) const noexcept { + // The densest rewrite tiles the input with the shortest match there is and swaps each one for the + // widest replacement there is. The bytes past the last whole match survive verbatim, so they are + // added rather than dropped - integer division alone under-counts, and a caller sizing to an + // under-count would be handed an overflowing write by the pre-sized path below. + size_t widest_replacement = 0; + for (size_t needle_index = 0; needle_index < replacements.size(); ++needle_index) + widest_replacement = sz_max_of_two(widest_replacement, to_bytes_view(replacements[needle_index]).size()); + size_t const shortest_match = sz_max_of_two(min_source_match_bytes(), (size_t)1); + size_t const whole_matches = input_bytes / shortest_match; + + // A dictionary that only ever shrinks still bounds at the input length, never below it. + return sz_max_of_two(input_bytes, whole_matches * widest_replacement + input_bytes % shortest_match); + } + + /** + * @brief Grid for a kernel launched with one block per work item, from that kernel's own occupancy. + * + * Clamped to the item count, because a block that finds nothing to do still costs its scratch - the + * BM25 frequency rows are sized from this, so an unclamped grid would allocate rows nobody fills. + */ + static unsigned grid_for_items_(kernel_shape_t const &shape, size_t items, gpu_specs_t const &specs) noexcept { + size_t const resident = (size_t)shape.blocks_per_multiprocessor * specs.streaming_multiprocessors; + size_t const wanted = sz_min_of_two(sz_max_of_two(resident, (size_t)1), sz_max_of_two(items, (size_t)1)); + return (unsigned)wanted; + } + + /** + * @brief Refuses a chunk width whose warm-up prefix or worst-case match count outgrows @ref small_size_t. + * + * The walkers carry a chunk's reach and its match count in that narrow type, so both bounds belong to the + * chunk width rather than to the input: haystack offsets themselves stay 64-bit. Shared by the chunked + * plan and by the scoring pass, which sizes its own chunks and never builds a chunk-offsets map. + */ + cuda_status_t check_chunk_bytes_fit_(size_t chunk_bytes) const noexcept { + size_t const longest = max_source_match_bytes(); + size_t const warm_up_bytes = longest > 0 ? longest - 1 : 0; + if (chunk_bytes + warm_up_bytes > (size_t)std::numeric_limits<small_size_t>::max()) + return {status_t::overflow_risk_k, cudaSuccess}; + + // Worst case is a repeated byte against a nested-suffix dictionary, where every position emits + // `max_outputs_per_state` merged outputs. + size_t const outputs_per_state = visit_dictionary([](auto const &dictionary) noexcept { // + return (size_t)dictionary.view().max_outputs_per_state; + }); + size_t const worst_case_matches_per_chunk = chunk_bytes * outputs_per_state; + if (outputs_per_state != 0 && (worst_case_matches_per_chunk / outputs_per_state != chunk_bytes || + worst_case_matches_per_chunk > (size_t)std::numeric_limits<small_size_t>::max())) + return {status_t::overflow_risk_k, cudaSuccess}; + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Sizes the chunk grid from the counting kernel's occupancy under the dictionary's actual + * shared-memory footprint, then lays out every haystack's chunks against that target. + * + * The layout is arithmetic over lengths the descriptor walk already read, so the host writes + * `haystack_chunk_offsets_` outright rather than counting on the device and scanning back. The array stays + * unified because every chunk thread reads it. + */ + cuda_status_t plan_haystack_chunks_(size_t total_bytes, gpu_specs_t const &specs, kernels_t const &kernel_table, + unsigned &shared_memory_bytes, unsigned &blocks_per_grid, size_t &chunk_bytes, + size_t &chunk_count) noexcept { + size_t const haystack_count = haystack_descriptors_.size(); + + size_t const staged_bytes = (size_t)staged_rows_ * substrings_alphabet_size_k * bytes_per_state_id_() + + (size_t)staged_accepts_words_ * sizeof(u32_t); // ? Settled per call + sz_assert_(staged_bytes <= std::numeric_limits<unsigned>::max() && + "The staged prefix is budgeted against one multiprocessor's shared memory in `try_build`"); + shared_memory_bytes = static_cast<unsigned>(staged_bytes); + cuda_status_t const occupancy_status = occupancy_grid_for( + blocks_per_grid, kernel_table.count_chunk.for_width(state_width()).function, substrings_threads_per_block_k, + shared_memory_bytes, specs); + if (occupancy_status.status != status_t::success_k) return occupancy_status; + + size_t const target_threads = sz_max_of_two((size_t)blocks_per_grid * substrings_threads_per_block_k, + (size_t)1); + chunk_bytes = sz_max_of_two(divide_round_up(total_bytes, target_threads), (size_t)1); + + if (cuda_status_t const fits = check_chunk_bytes_fit_(chunk_bytes); fits.status != status_t::success_k) + return fits; + + if (haystack_chunk_offsets_.try_resize_uninitialized(haystack_count + 1) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + chunk_count = 0; + for (size_t haystack_index = 0; haystack_index < haystack_count; ++haystack_index) { + haystack_chunk_offsets_[haystack_index] = chunk_count; + chunk_count += substrings_chunks_for_haystack_(haystack_descriptors_[haystack_index].size(), chunk_bytes); + } + haystack_chunk_offsets_[haystack_count] = chunk_count; + return {status_t::success_k, cudaSuccess}; + } + + /** @brief Runs the counting pass, then the in-place exclusive scan, so `chunk_match_offsets_` holds every + * chunk's write offset with the grand total trailing at `[chunk_count]` - the shared core of + * `try_count` and `try_find`. */ + cuda_status_t count_into_offsets_(cuda_executor_t const &executor, gpu_specs_t const &specs, + kernels_t const &kernel_table, unsigned shared_memory_bytes, + unsigned blocks_per_grid, size_t chunk_bytes, size_t chunk_count) noexcept { + if (chunk_match_offsets_.try_resize_uninitialized(chunk_count + 1) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + // Sizing hands the walk no output span: the pass writes each chunk's count into its slot instead. + CUresult const count_error = launch_walk_(substrings_pass_t::sizing_k, kernel_table, blocks_per_grid, + shared_memory_bytes, chunk_bytes, chunk_count, span<match_t> {}, + executor); + if (count_error != CUDA_SUCCESS) return make_cuda_status(count_error); + + // In place: each chunk's slot holds its raw count going in and its exclusive offset coming out - the + // scan kernel reads `input[i]` into a register before any thread writes `output[i]`, so reusing one + // buffer for both is safe and skips a second chunk_count-sized allocation. + return cuda_launch_exclusive_sum_(kernel_table.exclusive_sum, chunk_match_offsets_.data(), chunk_count, + chunk_match_offsets_.data(), {scan_partials_.data(), scan_partials_.size()}, + specs, executor.stream()); + } + + /** + * @brief Lays each haystack's match range onto the boundaries its reported matches occupy. + * @param[in] keep_offsets The cover's scanned keep flags, or empty when every emitted match is reported. + */ + cuda_status_t publish_haystack_match_offsets_(planned_pass_t const &pass, span<size_t const> keep_offsets, + size_t haystack_count, cuda_executor_t const &executor, + gpu_specs_t const &specs) noexcept { + if (haystack_match_offsets_.try_resize_uninitialized(haystack_count + 1) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + span<size_t const> haystack_chunk_offsets_argument {haystack_chunk_offsets_.data(), + haystack_chunk_offsets_.size()}; + span<size_t const> chunk_match_offsets_argument {chunk_match_offsets_.data(), chunk_match_offsets_.size()}; + span<size_t const> keep_offsets_argument = keep_offsets; + span<size_t> boundaries_argument {haystack_match_offsets_.data(), haystack_count + 1}; + void *boundary_arguments[4] = {&haystack_chunk_offsets_argument, &chunk_match_offsets_argument, + &keep_offsets_argument, &boundaries_argument}; + unsigned const boundary_grid = grid_for_items_(pass.kernel_table.haystack_match_offsets, haystack_count + 1, + specs); + CUresult const boundary_error = cuda_launch_t {} + .grid(boundary_grid) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.haystack_match_offsets.function, + boundary_arguments); + if (boundary_error != CUDA_SUCCESS) return make_cuda_status(boundary_error); + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Fences, then reads the rewritten tape's length out of the caller's own offsets array. + * + * The trailing boundary is the total, and the array may be plain device memory, so it comes back through + * a driver copy rather than a dereference. + */ + cuda_status_t read_rewritten_bytes_(span<size_t> output_offsets, size_t haystack_count, + cuda_executor_t const &executor, size_t &rewritten_bytes) noexcept { + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + CUresult const read_error = cuMemcpyDtoH(&rewritten_bytes, + (CUdeviceptr)(output_offsets.data() + haystack_count), sizeof(size_t)); + if (read_error != CUDA_SUCCESS) return make_cuda_status(read_error); + return {status_t::success_k, cudaSuccess}; + } + + /** @brief Zeroes the caller's counts through the driver, for a corpus the walk never reaches. */ + cuda_status_t clear_counts_(span<size_t> counts_per_haystack, cuda_executor_t const &executor) noexcept { + if (counts_per_haystack.size() == 0) return {status_t::success_k, cudaSuccess}; + CUresult const clear_error = cuMemsetD8Async((CUdeviceptr)counts_per_haystack.data(), 0, + counts_per_haystack.size() * sizeof(size_t), executor.stream()); + if (clear_error != CUDA_SUCCESS) return make_cuda_status(clear_error); + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + return {status_t::success_k, cudaSuccess}; + } + + /** @brief Zeroes the caller's scores through the driver, for a batch no walk reaches. */ + cuda_status_t clear_scores_(span<f32_t> scores, cuda_executor_t const &executor) noexcept { + if (scores.size() == 0) return {status_t::success_k, cudaSuccess}; + CUresult const clear_error = cuMemsetD8Async((CUdeviceptr)scores.data(), 0, scores.size() * sizeof(f32_t), + executor.stream()); + if (clear_error != CUDA_SUCCESS) return make_cuda_status(clear_error); + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + return {status_t::success_k, cudaSuccess}; + } + + /** @brief Differences the published boundaries into the caller's per-haystack counts, then fences once. */ + cuda_status_t count_from_boundaries_(planned_pass_t const &pass, span<size_t> counts_per_haystack, + cuda_executor_t const &executor, gpu_specs_t const &specs) noexcept { + span<size_t const> boundaries_argument {haystack_match_offsets_.data(), haystack_match_offsets_.size()}; + span<size_t> counts_argument = counts_per_haystack; + void *counts_arguments[2] = {&boundaries_argument, &counts_argument}; + unsigned const counts_grid = grid_for_items_(pass.kernel_table.counts_from_boundaries, + counts_per_haystack.size(), specs); + CUresult const counts_error = cuda_launch_t {} + .grid(counts_grid) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.counts_from_boundaries.function, counts_arguments); + if (counts_error != CUDA_SUCCESS) return make_cuda_status(counts_error); + + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + return {status_t::success_k, cudaSuccess, CUDA_SUCCESS, timer_.elapsed_milliseconds()}; + } + + /** + * @brief Resolves a leftmost cover over an already-emitted match list, and compacts the survivors. + * + * The walk emits every match, which is the only thing it is fast at; deciding between them is a pass + * over a few million records rather than a few dozen million bytes, and it needs no per-thread ring and + * no second walk to find a safe cursor. + * + * @param[in,out] matches In: every match, ascending by haystack and end. Out: the survivors, in place. + * @param[out] surviving How many survived, which is what sizes everything downstream. + */ + cuda_status_t resolve_cover_(planned_pass_t const &pass, span<match_t> matches, + substrings_overlap_policy_t overlap_policy, size_t haystack_count, + cuda_executor_t const &executor, gpu_specs_t const &specs, + size_t &surviving) noexcept { + + size_t const emitted = matches.size(); + surviving = emitted; + if (overlap_policy == substrings_overlapping_k) return {status_t::success_k, cudaSuccess}; + + // A corpus nothing matched still owes its caller a boundary per haystack; an empty keep span makes the + // boundary kernel the identity over the emitted offsets, which are all zero in that case. + if (emitted == 0) { + surviving = 0; + return publish_haystack_match_offsets_(pass, {}, haystack_count, executor, specs); + } + + if (cover_keep_.try_resize_uninitialized(emitted + 1) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + span<match_t const> matches_argument {matches.data(), emitted}; + size_t longest_argument = max_source_match_bytes(); + substrings_overlap_policy_t policy_argument = overlap_policy; + span<size_t> keep_argument {cover_keep_.data(), emitted}; + void *resolve_arguments[4] = {&matches_argument, &longest_argument, &policy_argument, &keep_argument}; + unsigned const resolve_grid = grid_for_items_(pass.kernel_table.cover_resolve, emitted, specs); + CUresult const resolve_error = cuda_launch_t {} + .grid(resolve_grid) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.cover_resolve.function, resolve_arguments); + if (resolve_error != CUDA_SUCCESS) return make_cuda_status(resolve_error); + + cuda_status_t const scan_status = cuda_launch_exclusive_sum_( + pass.kernel_table.exclusive_sum, cover_keep_.data(), emitted, cover_keep_.data(), + {scan_partials_.data(), scan_partials_.size()}, specs, executor.stream()); + if (scan_status.status != status_t::success_k) return scan_status; + + if (cuda_status_t const boundary_status = publish_haystack_match_offsets_( + pass, {cover_keep_.data(), emitted + 1}, haystack_count, executor, specs); + boundary_status.status != status_t::success_k) + return boundary_status; + + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + // The scan lives on the device, where it belongs - only its last element is the host's business. + CUresult const total_error = cuMemcpyDtoH(&surviving, (CUdeviceptr)(cover_keep_.data() + emitted), + sizeof(size_t)); + if (total_error != CUDA_SUCCESS) return make_cuda_status(total_error); + return {status_t::success_k, cudaSuccess}; + } + + /** + * @brief Gathers the cover's survivors into @p destination, which the caller sizes and owns. + * + * Separate from `resolve_cover_` because the three callers want them in three different places: + * counting wants them nowhere, finding wants them in the caller's own span, and rewriting wants them in + * scratch its kernels read. Fusing this into the resolve would cost finding a whole extra copy. + */ + CUresult compact_cover_(planned_pass_t const &pass, span<match_t const> matches, span<match_t> destination, + cuda_executor_t const &executor, gpu_specs_t const &specs) noexcept { + span<size_t const> keep_argument {cover_keep_.data(), matches.size() + 1}; + span<match_t const> matches_argument = matches; + span<match_t> destination_argument = destination; + void *arguments[3] = {&matches_argument, &keep_argument, &destination_argument}; + return cuda_launch_t {} + .grid(grid_for_items_(pass.kernel_table.cover_compact, matches.size(), specs)) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.cover_compact.function, arguments); + } + + /** + * @brief Runs the scattering pass into @p target, which the counting pass has already sized and scanned. + * + * Shared by all three entry points, so the ten arguments are written out once and stay in one order. + */ + CUresult launch_scatter_(planned_pass_t const &pass, span<match_t> target, + cuda_executor_t const &executor) noexcept { + // Bounded to what the counting pass actually found, not to the caller's capacity, so a debug index + // assert inside the kernel catches an over-write rather than merely staying inside the allocation. + return launch_walk_(substrings_pass_t::writing_k, pass.kernel_table, pass.blocks_per_grid, + pass.shared_memory_bytes, pass.chunk_bytes, pass.chunk_count, target, executor); + } + + /** + * @brief Launches one walk over every chunk, in whichever pass @p pass_kind names. + * + * Both passes take the same ten arguments in the same order - the counting one simply leaves @p target + * empty and tallies into the chunk slots - so the argument block is written here once. + */ + CUresult launch_walk_(substrings_pass_t pass_kind, kernels_t const &kernel_table, unsigned blocks_per_grid, + unsigned shared_memory_bytes, size_t chunk_bytes, size_t chunk_count, span<match_t> target, + cuda_executor_t const &executor) noexcept { + return state_width() == substrings_state_width_t::u16_k + ? launch_walk_at_<u16_t>(pass_kind, kernel_table, blocks_per_grid, shared_memory_bytes, chunk_bytes, + chunk_count, target, executor) + : launch_walk_at_<u32_t>(pass_kind, kernel_table, blocks_per_grid, shared_memory_bytes, chunk_bytes, + chunk_count, target, executor); + } + + template <typename state_id_type_> + CUresult launch_walk_at_(substrings_pass_t pass_kind, kernels_t const &kernel_table, unsigned blocks_per_grid, + unsigned shared_memory_bytes, size_t chunk_bytes, size_t chunk_count, span<match_t> target, + cuda_executor_t const &executor) noexcept { + aho_corasick_view<state_id_type_> view_argument = settled_dictionary_<state_id_type_>().view(); + state_id_type_ staged_rows_argument = static_cast<state_id_type_>(staged_rows_); + span<u32_t const> accepts_words_argument {accepts_words_.data(), accepts_words_.size()}; + u32_t staged_accepts_words_argument = staged_accepts_words_; + span<span<byte_t const> const> haystacks_argument {haystack_descriptors_.data(), haystack_descriptors_.size()}; + span<size_t const> haystack_chunk_offsets_argument {haystack_chunk_offsets_.data(), + haystack_chunk_offsets_.size()}; + size_t chunk_bytes_argument = chunk_bytes; + size_t chunk_count_argument = chunk_count; + span<size_t> chunk_match_slots_argument {chunk_match_offsets_.data(), chunk_match_offsets_.size()}; + span<match_t> matches_out_argument = target; + void *walk_arguments[10] = {&view_argument, + &staged_rows_argument, + &accepts_words_argument, + &staged_accepts_words_argument, + &haystacks_argument, + &haystack_chunk_offsets_argument, + &chunk_bytes_argument, + &chunk_count_argument, + &chunk_match_slots_argument, + &matches_out_argument}; + auto const &shapes = pass_kind == substrings_pass_t::sizing_k ? kernel_table.count_chunk + : kernel_table.scatter_chunk; + return cuda_launch_t {} + .grid(blocks_per_grid) + .block(substrings_threads_per_block_k) + .shared(shared_memory_bytes) + .stream(executor.stream()) + .launch(shapes.for_width(state_width()).function, walk_arguments); + } + + public: + /** + * @brief Occurrences of all needles in each of the @p haystacks, for filtering and ranking. + * @param[in] haystacks Device-accessible, contiguously laid out; no needle is ever reported straddling + * two of them. + * @param[out] matches_total Sum of @p counts_per_haystack, which is what sizes a later `try_find` buffer. + * + * Under `substrings_overlapping_k` this is strictly cheaper than `try_find` - the plan and the counting + * pass, no scatter - because chunks never cross a haystack boundary and the breakdown is a subtraction + * over the counting pass's offsets. A cover costs the same as finding one: the walk cannot know which + * matches survive, so they have to be emitted before anything can be counted. + */ + template <typename haystacks_type_> + cuda_status_t try_count(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<size_t> counts_per_haystack, size_t &matches_total, + cuda_executor_t const &executor = {}, gpu_specs_t specs = {}) noexcept { + matches_total = 0; + if (counts_per_haystack.size() != haystacks.size()) return {status_t::unexpected_dimensions_k, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(counts_per_haystack); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; + + size_t total_bytes = 0; + [[maybe_unused]] size_t longest_bytes = 0; + cuda_status_t const describe_status = describe_haystacks_(haystacks, total_bytes, longest_bytes); + if (describe_status.status != status_t::success_k) return describe_status; + + return count_described_(haystacks.size(), total_bytes, overlap_policy, counts_per_haystack, matches_total, + executor, specs); + } + + /** + * @brief Everything `try_count` does once the haystacks are described, with the container behind it. + * + * The descriptors are the type-erasure boundary, so the work below compiles once rather than once per + * input shape. @sa `describe_haystacks_`. + */ + cuda_status_t count_described_(size_t haystack_count, size_t total_bytes, + substrings_overlap_policy_t overlap_policy, span<size_t> counts_per_haystack, + size_t &matches_total, cuda_executor_t const &executor, gpu_specs_t specs) noexcept { + planned_pass_t pass; + cuda_status_t const pass_status = plan_and_count_(total_bytes, executor, specs, pass); + if (pass_status.status != status_t::success_k) return pass_status; + // A corpus with nothing to walk still owes its caller a count per haystack, all of them zero, and the + // caller's span may be plain device memory - so the driver clears it rather than a host loop. + if (!pass.has_work) return clear_counts_(counts_per_haystack, executor); + + CUresult const stop_error = timer_.record_stop(executor.stream()); + if (stop_error != CUDA_SUCCESS) return make_cuda_status(stop_error); + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + + matches_total = chunk_match_offsets_[pass.chunk_count]; + if (overlap_policy == substrings_overlapping_k) { + if (cuda_status_t const boundary_status = publish_haystack_match_offsets_(pass, {}, haystack_count, + executor, specs); + boundary_status.status != status_t::success_k) + return boundary_status; + } + else { + // A cover is decided between matches, so counting one means emitting them first - the walk cannot + // know which survive. That makes a counted cover cost what a found one does. + if (emitted_matches_.try_resize_uninitialized(sz_max_of_two(matches_total, (size_t)1)) != + status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + CUresult const scatter_error = launch_scatter_(pass, {emitted_matches_.data(), matches_total}, executor); + if (scatter_error != CUDA_SUCCESS) return make_cuda_status(scatter_error); + + // No gather: a count wants the boundaries, and those the resolve already wrote. + cuda_status_t const cover_status = resolve_cover_(pass, {emitted_matches_.data(), matches_total}, + overlap_policy, haystack_count, executor, specs, + matches_total); + if (cover_status.status != status_t::success_k) return cover_status; + } + + return count_from_boundaries_(pass, counts_per_haystack, executor, specs); + } + + /** + * @brief Finds all occurrences of all needles in all the @p haystacks: count, then scatter every match + * at its chunk's precomputed offset without atomics, then - under a cover - resolve and gather. + * @param[out] matches_found Matches written, in ascending haystack order. + * @retval `status_t::unexpected_dimensions_k` @p matches_out is too small; nothing is written in that + * case. See `try_count` for the @p haystacks contract. + */ + template <typename haystacks_type_> + cuda_status_t try_find(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<match_t> matches_out, size_t &matches_found, cuda_executor_t const &executor = {}, + gpu_specs_t specs = {}) noexcept { + matches_found = 0; + if (status_t const reachable = check_device_accessible_memory(matches_out); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + + size_t total_bytes = 0; + [[maybe_unused]] size_t longest_bytes = 0; + cuda_status_t const describe_status = describe_haystacks_(haystacks, total_bytes, longest_bytes); + if (describe_status.status != status_t::success_k) return describe_status; + + return find_described_(haystacks.size(), total_bytes, overlap_policy, matches_out, matches_found, executor, + specs); + } + + /** @brief Everything `try_find` does once the haystacks are described. @sa `count_described_`. */ + cuda_status_t find_described_(size_t haystack_count, size_t total_bytes, substrings_overlap_policy_t overlap_policy, + span<match_t> matches_out, size_t &matches_found, cuda_executor_t const &executor, + gpu_specs_t specs) noexcept { + size_t const matches_capacity = matches_out.size(); + + planned_pass_t pass; + cuda_status_t const pass_status = plan_and_count_(total_bytes, executor, specs, pass); + if (pass_status.status != status_t::success_k || !pass.has_work) return pass_status; + + // The emitted total is only host-visible after a fence, and it sizes the scratch the walk writes to. + CUresult const mid_sync_error = timer_.synchronize(executor.stream()); + if (mid_sync_error != CUDA_SUCCESS) return make_cuda_status(mid_sync_error); + size_t const emitted = chunk_match_offsets_[pass.chunk_count]; + bool const covering = overlap_policy != substrings_overlapping_k; + + // Under a cover the walk's output is an intermediate, so it lands in scratch and only the survivors + // reach the caller. Without one it is the answer, and the caller's span takes it directly. + size_t matches_in_batch = emitted; + span<match_t> scatter_target; + if (covering) { + if (emitted_matches_.try_resize_uninitialized(sz_max_of_two(emitted, (size_t)1)) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + scatter_target = {emitted_matches_.data(), emitted}; + } + else { + if (emitted > matches_capacity) + return matches_found = emitted, cuda_status_t {status_t::unexpected_dimensions_k, cudaSuccess}; + scatter_target = {matches_out.data(), emitted}; + } + + CUresult const scatter_error = launch_scatter_(pass, scatter_target, executor); + if (scatter_error != CUDA_SUCCESS) return make_cuda_status(scatter_error); + + if (covering) { + cuda_status_t const cover_status = resolve_cover_(pass, scatter_target, overlap_policy, haystack_count, + executor, specs, matches_in_batch); + if (cover_status.status != status_t::success_k) return cover_status; + // The survivors' count survives the refusal, so a caller that brought no buffer learns its size. + if (matches_in_batch > matches_capacity) + return matches_found = matches_in_batch, cuda_status_t {status_t::unexpected_dimensions_k, cudaSuccess}; + + if (matches_in_batch) { + CUresult const gather_error = compact_cover_(pass, {scatter_target.data(), emitted}, + {matches_out.data(), matches_in_batch}, executor, specs); + if (gather_error != CUDA_SUCCESS) return make_cuda_status(gather_error); + } + } + + CUresult const stop_error = timer_.record_stop(executor.stream()); + if (stop_error != CUDA_SUCCESS) return make_cuda_status(stop_error); + + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + + matches_found = matches_in_batch; + return {status_t::success_k, cudaSuccess, CUDA_SUCCESS, timer_.elapsed_milliseconds()}; + } + + /** + * @brief Rewrites every haystack into one tape, substituting each match with its needle's replacement. + * @param[in] overlap_policy Must name a leftmost policy; an overlapping rewrite is not a function. + * @param[in] replacements One per needle, inserted verbatim. + * @param[out] output_offsets Rewritten boundaries, `haystacks.size() + 1` entries; always filled. + * @param[out] output_bytes_written Bytes written, or - when @p output_bytes is short - the size needed. + * @retval `status_t::unexpected_dimensions_k` @p output_bytes is too small, and nothing was written. + * + * Where the host walks each haystack twice, once to size and once to write, the device walks once and + * keeps the matches: the automaton is latency-bound on dependent transition loads, while everything + * after it is bandwidth. The scratch that buys is about 96 bytes per match, so a corpus matching every + * tenth byte should be handed over in batches. + * + * Sizing @p output_bytes to `szs_substrings_replace_bound` makes the capacity check unfailable, which + * drops the host round-trip it would otherwise need; a device-accessible @p output_bytes then makes the + * whole call one uninterrupted stream of launches. + */ + template <typename haystacks_type_, typename replacements_type_> + cuda_status_t try_replace(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + replacements_type_ const &replacements, span<char> output_bytes, + span<size_t> output_offsets, size_t &output_bytes_written, + cuda_executor_t const &executor = {}, gpu_specs_t specs = {}) noexcept { + output_bytes_written = 0; + if (output_offsets.size() != haystacks.size() + 1) return {status_t::unexpected_dimensions_k, cudaSuccess}; + if (status_t const rewritable = substrings_check_rewritable(overlap_policy); rewritable != status_t::success_k) + return {rewritable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(output_offsets); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(output_bytes); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + + size_t total_bytes = 0; + [[maybe_unused]] size_t longest_bytes = 0; + cuda_status_t const describe_status = describe_haystacks_(haystacks, total_bytes, longest_bytes); + if (describe_status.status != status_t::success_k) return describe_status; + cuda_status_t const upload_status = upload_replacements_(replacements); + if (upload_status.status != status_t::success_k) return upload_status; + + // The bound is arithmetic over the needle set, so it is settled here and the rewrite below never + // names either container again. + return replace_described_(haystacks.size(), total_bytes, replace_bound_host_(total_bytes, replacements), + overlap_policy, output_bytes, output_offsets, output_bytes_written, executor, specs); + } + + /** @brief Everything `try_replace` does once the haystacks and replacements are staged. @sa `count_described_`. */ + cuda_status_t replace_described_(size_t haystack_count, size_t total_bytes, size_t bound, + substrings_overlap_policy_t overlap_policy, span<char> output_bytes, + span<size_t> output_offsets, size_t &output_bytes_written, + cuda_executor_t const &executor, gpu_specs_t specs) noexcept { + planned_pass_t pass; + cuda_status_t const pass_status = plan_and_count_(total_bytes, executor, specs, pass); + if (pass_status.status != status_t::success_k) return pass_status; + // A corpus with nothing to rewrite still owes its caller a boundary per haystack, all of them zero. + if (!pass.has_work) return clear_counts_({output_offsets.data(), output_offsets.size()}, executor); + + // The match count sizes three buffers, so it has to reach the host before they can be allocated. + CUresult const count_sync_error = timer_.synchronize(executor.stream()); + if (count_sync_error != CUDA_SUCCESS) return make_cuda_status(count_sync_error); + + size_t const matches_in_batch = chunk_match_offsets_[pass.chunk_count]; + if (emitted_matches_.try_resize_uninitialized(sz_max_of_two(matches_in_batch, (size_t)1)) != + status_t::success_k || + rewrite_gap_offsets_.try_resize_uninitialized(sz_max_of_two(matches_in_batch, (size_t)1)) != + status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + CUresult const scatter_error = launch_scatter_(pass, {emitted_matches_.data(), matches_in_batch}, executor); + if (scatter_error != CUDA_SUCCESS) return make_cuda_status(scatter_error); + + size_t surviving = 0; + cuda_status_t const cover_status = resolve_cover_(pass, {emitted_matches_.data(), matches_in_batch}, + overlap_policy, haystack_count, executor, specs, surviving); + if (cover_status.status != status_t::success_k) return cover_status; + if (cover_survivors_.try_resize_uninitialized(sz_max_of_two(surviving, (size_t)1)) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + if (surviving) { + CUresult const gather_error = compact_cover_(pass, {emitted_matches_.data(), matches_in_batch}, + {cover_survivors_.data(), surviving}, executor, specs); + if (gather_error != CUDA_SUCCESS) return make_cuda_status(gather_error); + } + + span<span<byte_t const> const> haystacks_argument {haystack_descriptors_.data(), haystack_descriptors_.size()}; + span<size_t const> match_offsets_argument {haystack_match_offsets_.data(), haystack_count + 1}; + span<match_t const> matches_argument {cover_survivors_.data(), surviving}; + span<size_t const> replacement_offsets_argument {replacement_offsets_.data(), replacement_offsets_.size()}; + span<size_t> gap_offsets_argument {rewrite_gap_offsets_.data(), surviving}; + span<size_t> output_sizes_argument {output_offsets.data(), haystack_count}; + void *offsets_arguments[6] = {&haystacks_argument, &match_offsets_argument, &matches_argument, + &replacement_offsets_argument, &gap_offsets_argument, &output_sizes_argument}; + unsigned const offsets_grid = grid_for_items_(pass.kernel_table.rewrite_offsets, haystack_count, specs); + CUresult const offsets_error = cuda_launch_t {} + .grid(offsets_grid) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.rewrite_offsets.function, offsets_arguments); + if (offsets_error != CUDA_SUCCESS) return make_cuda_status(offsets_error); + + // The scan writes the caller's own array, so the boundaries are complete before any capacity check - + // which is what lets a refused call name the exact size it wanted. + cuda_status_t const scan_status = cuda_launch_exclusive_sum_( + pass.kernel_table.exclusive_sum, output_offsets.data(), haystack_count, output_offsets.data(), + {scan_partials_.data(), scan_partials_.size()}, specs, executor.stream()); + if (scan_status.status != status_t::success_k) return scan_status; + + // A caller sized against the dictionary's own bound cannot be refused, so neither the check nor the + // fence it needs happens at all - the copy kernel reads the tape's length from the scan itself. + bool const pre_sized = output_bytes.size() >= bound; + size_t rewritten_bytes = 0; + if (!pre_sized) { + cuda_status_t const size_status = read_rewritten_bytes_(output_offsets, haystack_count, executor, + rewritten_bytes); + if (size_status.status != status_t::success_k) return size_status; + if (rewritten_bytes > output_bytes.size()) + return output_bytes_written = rewritten_bytes, + cuda_status_t {status_t::unexpected_dimensions_k, cudaSuccess}; + } + + size_t const copy_bytes = pre_sized ? bound : rewritten_bytes; + char *copy_target = output_bytes.data(); + + span<size_t const> gap_offsets_const_argument {rewrite_gap_offsets_.data(), surviving}; + byte_t const *replacement_bytes_argument = replacement_bytes_.data(); + span<size_t const> output_offsets_argument {output_offsets.data(), haystack_count + 1}; + size_t tile_bytes_argument = substrings_rewrite_tile_bytes_k; + void *copy_arguments[9] = { + &haystacks_argument, &match_offsets_argument, &matches_argument, + &gap_offsets_const_argument, &replacement_bytes_argument, &replacement_offsets_argument, + &output_offsets_argument, &tile_bytes_argument, ©_target}; + unsigned const copy_grid = grid_for_items_( + pass.kernel_table.rewrite_copy, + divide_round_up(sz_max_of_two(copy_bytes, (size_t)1), substrings_rewrite_tile_bytes_k), specs); + + CUresult const copy_error = cuda_launch_t {} + .grid(copy_grid) + .block(substrings_threads_per_block_k) + .shared(0) + .stream(executor.stream()) + .launch(pass.kernel_table.rewrite_copy.function, copy_arguments); + if (copy_error != CUDA_SUCCESS) return make_cuda_status(copy_error); + + CUresult const stop_error = timer_.record_stop(executor.stream()); + if (stop_error != CUDA_SUCCESS) return make_cuda_status(stop_error); + + // A pre-sized caller never fetched the tape's length, so it is read past the one fence this call + // always pays - the same fence that makes the caller's offsets readable. + cuda_status_t const size_status = read_rewritten_bytes_(output_offsets, haystack_count, executor, + rewritten_bytes); + if (size_status.status != status_t::success_k) return size_status; + + output_bytes_written = rewritten_bytes; + return {status_t::success_k, cudaSuccess, CUDA_SUCCESS, timer_.elapsed_milliseconds()}; + } + + /** + * @brief Scores every haystack against the compiled needle set in one walk. + * @param[in] document_lengths One per haystack; an empty span uses byte lengths. + * @param[in] needle_weights One IDF or boost per needle. + * @param[out] scores One per haystack. + * + * One launch and one synchronize: nothing here is sized by a device result, so unlike `try_find` and + * `try_replace` this never stalls mid-call. Scores are bit-stable run to run because the block sums + * fixed-point integers, and integer addition is associative - no grid size, lane order or scheduling + * order can perturb the total. + */ + template <typename haystacks_type_> + cuda_status_t try_score_bm25(haystacks_type_ const &haystacks, span<f32_t const> document_lengths, + substrings_bm25_t parameters, span<f32_t const> needle_weights, span<f32_t> scores, + cuda_executor_t const &executor = {}, gpu_specs_t specs = {}) noexcept { + size_t total_bytes = 0, longest_bytes = 0; + cuda_status_t const describe_status = describe_haystacks_(haystacks, total_bytes, longest_bytes); + if (describe_status.status != status_t::success_k) return describe_status; + return score_bm25_described_(haystacks.size(), total_bytes, longest_bytes, document_lengths, parameters, + needle_weights, scores, executor, specs); + } + + /** @brief Everything `try_score_bm25` does once the haystacks are described. @sa `count_described_`. */ + cuda_status_t score_bm25_described_(size_t haystack_count, size_t total_bytes, size_t longest_bytes, + span<f32_t const> document_lengths, substrings_bm25_t parameters, + span<f32_t const> needle_weights, span<f32_t> scores, + cuda_executor_t const &executor, gpu_specs_t specs) noexcept { + size_t const needle_count = needle_weights.size(); + if (needle_count != count_needles() || scores.size() != haystack_count || + (document_lengths.size() != 0 && document_lengths.size() != haystack_count)) + return {status_t::unexpected_dimensions_k, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(needle_weights); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(document_lengths); + reachable != status_t::success_k) + return {reachable, cudaSuccess}; + if (status_t const reachable = check_device_accessible_memory(scores); reachable != status_t::success_k) + return {reachable, cudaSuccess}; + + cuda_status_t const current_status = executor.ensure_current(); + if (current_status.status != status_t::success_k) return current_status; + // A dictionary with no needles scores every haystack zero, and the caller's span may be plain device + // memory, so the driver clears it rather than a host loop. + if (haystack_count == 0 || needle_count == 0) return clear_scores_(scores, executor); + // Scoring bypasses `plan_and_count_`, so it budgets its own staging against this call's device. + if (cuda_status_t const budgeted = try_budget_staging_(executor); budgeted.status != status_t::success_k) + return budgeted; + auto [kernel_table, kernels_status] = kernels(executor.device_id()); + if (kernels_status.status != status_t::success_k) return kernels_status; + + // The counter table shares the dynamic allocation with the staged automaton, so it must be counted + // before the occupancy query - that query settles `blocks_per_grid`, which sizes the overflow rows. + size_t const staged_bytes = (size_t)staged_rows_ * substrings_alphabet_size_k * bytes_per_state_id_() + + (size_t)staged_accepts_words_ * sizeof(u32_t) + + substrings_bm25_counters_bytes_(); // ? Settled per call + sz_assert_(staged_bytes <= std::numeric_limits<unsigned>::max() && + "A block's shared footprint must fit the launch parameter"); + unsigned const shared_memory_bytes = static_cast<unsigned>(staged_bytes); + unsigned blocks_per_grid = 0; + cuda_status_t const occupancy_status = occupancy_grid_for( + blocks_per_grid, kernel_table.score_bm25.for_width(state_width()).function, substrings_threads_per_block_k, + shared_memory_bytes, specs); + if (occupancy_status.status != status_t::success_k) return occupancy_status; + // One block owns one haystack, and each block owns an overflow row, so a grid wider than the corpus + // buys nothing and costs `needle_count` counters per surplus block. + blocks_per_grid = (unsigned)sz_min_of_two((size_t)blocks_per_grid, sz_max_of_two(haystack_count, (size_t)1)); + + // Each block splits the one haystack it owns, so the chunk width is derived per haystack in the kernel. + // Only the widest one any haystack can ask for has to clear the fit test, and that is the longest + // haystack's share of a block. + size_t const widest_chunk_bytes = sz_max_of_two( + divide_round_up(longest_bytes, (size_t)substrings_threads_per_block_k), + sz_max_of_two(max_source_match_bytes(), (size_t)1)); + if (cuda_status_t const fits = check_chunk_bytes_fit_(widest_chunk_bytes); fits.status != status_t::success_k) + return fits; + + // A dictionary the table can hold never overflows, so most calls allocate nothing at all here. + size_t const overflow_total = needle_count > substrings_bm25_slots_k ? (size_t)blocks_per_grid * needle_count + : 0; + if (bm25_overflow_.try_resize_uninitialized(overflow_total) != status_t::success_k) + return {status_t::bad_alloc_k, cudaSuccess}; + + span<f32_t const> weights_argument = needle_weights; + span<f32_t const> lengths_argument = document_lengths; + span<f32_t> scores_argument = scores; + + CUresult const timer_error = timer_.ensure_created(executor.device_id()); + if (timer_error != CUDA_SUCCESS) return make_cuda_status(timer_error); + CUresult const start_error = timer_.record_start(executor.stream()); + if (start_error != CUDA_SUCCESS) return make_cuda_status(start_error); + + CUresult const score_error = + state_width() == substrings_state_width_t::u16_k + ? launch_score_bm25_at_<u16_t>(kernel_table, blocks_per_grid, shared_memory_bytes, parameters, + lengths_argument, weights_argument, scores_argument, executor) + : launch_score_bm25_at_<u32_t>(kernel_table, blocks_per_grid, shared_memory_bytes, parameters, + lengths_argument, weights_argument, scores_argument, executor); + if (score_error != CUDA_SUCCESS) return make_cuda_status(score_error); + + CUresult const stop_error = timer_.record_stop(executor.stream()); + if (stop_error != CUDA_SUCCESS) return make_cuda_status(stop_error); + + CUresult const sync_error = timer_.synchronize(executor.stream()); + if (sync_error != CUDA_SUCCESS) return make_cuda_status(sync_error); + return {status_t::success_k, cudaSuccess, CUDA_SUCCESS, timer_.elapsed_milliseconds()}; + } + + private: + /** @brief The scoring launch, at the width the automaton settled on; the ten arguments stay in one order. */ + template <typename state_id_type_> + CUresult launch_score_bm25_at_(kernels_t const &kernel_table, unsigned blocks_per_grid, + unsigned shared_memory_bytes, substrings_bm25_t parameters, + span<f32_t const> lengths_argument, span<f32_t const> weights_argument, + span<f32_t> scores_argument, cuda_executor_t const &executor) noexcept { + aho_corasick_view<state_id_type_> view_argument = settled_dictionary_<state_id_type_>().view(); + state_id_type_ staged_rows_argument = static_cast<state_id_type_>(staged_rows_); + span<u32_t const> accepts_words_argument {accepts_words_.data(), accepts_words_.size()}; + u32_t staged_accepts_words_argument = staged_accepts_words_; + span<span<byte_t const> const> haystacks_argument {haystack_descriptors_.data(), haystack_descriptors_.size()}; + substrings_bm25_t parameters_argument = parameters; + span<u32_t> overflow_argument {bm25_overflow_.data(), bm25_overflow_.size()}; + void *score_arguments[10] = { + &view_argument, &staged_rows_argument, &accepts_words_argument, &staged_accepts_words_argument, + &haystacks_argument, &lengths_argument, ¶meters_argument, &weights_argument, + &overflow_argument, &scores_argument}; + return cuda_launch_t {} + .grid(blocks_per_grid) + .block(substrings_threads_per_block_k) + .shared(shared_memory_bytes) + .stream(executor.stream()) + .launch(kernel_table.score_bm25.for_width(state_width()).function, score_arguments); + } +}; + +using substrings_cuda_t = substrings_cuda<unified_alloc_t, sz_cap_cuda_k>; + +#pragma endregion Engine + +} // namespace stringzillas +} // namespace ashvardanian + +#endif // STRINGZILLAS_SUBSTRINGS_CUDA_CUH_ diff --git a/include/stringzillas/substrings/serial.hpp b/include/stringzillas/substrings/serial.hpp new file mode 100644 index 00000000..baec665f --- /dev/null +++ b/include/stringzillas/substrings/serial.hpp @@ -0,0 +1,3402 @@ +/** + * @brief Hardware-accelerated multi-pattern exact and case-folded substring search (serial backend). + * @file include/stringzillas/substrings/serial.hpp + * @author Ash Vardanian + * + * Implements the Aho-Corasick automaton at the core of the multi-pattern search engine: + * + * - `aho_corasick_dictionary` builds the trie, derives the walking automaton (below), splits the result + * into a shallow-first, out-degree-ordered hot tier and a double-array cold tier, and exposes CSR-flattened + * match outputs through the shared `aho_corasick_view` contract. Construction allocates a constant + * number of flat buffers rather than one per state, and none of them outlives `try_build`. + * - `substrings` wraps the dictionary behind a `try_build` / `try_count` / `try_find` triple, with a + * single-threaded serial specialization and a two-level parallel one - one haystack per core below the + * L2 size, all cores on one haystack above it. + * + * Every other StringZillas engine exposes a single `operator()`; the triple exists here because a + * dictionary is compiled once and reused across many later calls, and counting is useful in its own right. + * + * Case folding lives in the @b stream, not in the automaton. A needle is folded once at build time and + * inserted byte for byte, exactly like a cased one, so the trie stays a tree and Aho-Corasick failure links + * stay single-valued. A haystack is folded one codepoint at a time as the walk consumes it - never into a + * buffer - by a cursor built on the same folded iterator the single-pattern engine walks with. + * + * Inverting the fold onto the needle instead cannot express a match beginning or ending part-way through an + * expansion, and cannot be repaired: an automaton whose alphabet is raw haystack bytes and which must report + * match spans is exponential in needle length, since a self-overlapping needle's span encodes the byte width + * of every spelling behind it. + * + * Folded byte offsets equal source byte offsets everywhere except across the 139 codepoints whose fold + * breaks a codepoint boundary, so a match's source span is plain arithmetic unless it touches one of those. + */ +#ifndef STRINGZILLAS_SUBSTRINGS_SERIAL_HPP_ +#define STRINGZILLAS_SUBSTRINGS_SERIAL_HPP_ + +#include "stringzilla/types.hpp" // `status_t::status_t` +#include "stringzilla/memory.h" // `sz_copy` +#include "stringzilla/utf8_runes/serial.h" // `sz_rune_decode`, `sz_rune_encode` +#include "stringzilla/utf8_uncased.h" // `sz_utf8_folded_reverse_iter_t`, via its own backends +#include "stringzilla/utf8_uncased_fold/serial.h" // `sz_unicode_fold_codepoint_`, `sz_ascii_fold_` +#include "stringzillas/types.hpp" // `dummy_executor_t` + +#include <forkunion/types.hpp> // `indexed_split_t` - the balanced range split every executor uses + +#include <limits> // `std::numeric_limits` for numeric types +#include <memory> // `std::allocator_traits` to re-bind the allocator +#include <type_traits> // `std::enable_if_t` for meta-programming +#include <variant> // `std::variant` holds the automaton at whichever state-id width it fits + +namespace ashvardanian { +namespace stringzillas { + +namespace fu = ashvardanian::forkunion; + +// Per-symbol: a using-directive re-exports our `memcpy` and nvcc then finds the call ambiguous. +using ashvardanian::stringzilla::byte_t; +using ashvardanian::stringzilla::dummy_alloc_t; +using ashvardanian::stringzilla::size_t; +using ashvardanian::stringzilla::span; +using ashvardanian::stringzilla::to_bytes_view; + +#pragma region Vocabulary + +/** @brief Whether a dictionary matches needles byte-for-byte or folds both sides to a shared case first. */ +enum substrings_case_sensitivity_t { + /** @brief Byte-exact matching; needles may be arbitrary bytes. */ + substrings_cased_k, + /** @brief Full Unicode case folding; needles must be valid UTF-8. */ + substrings_uncased_k, +}; + +/** @brief How matches that share bytes resolve: reported in full, or thinned to a leftmost run. */ +enum substrings_overlap_policy_t { + /** @brief Every match of every needle, including ones that share bytes and ones nested in others. */ + substrings_overlapping_k, + /** @brief Matches sharing no bytes: earliest start, then longest span, then lower needle index. */ + substrings_leftmost_longest_k, + /** @brief Matches sharing no bytes: earliest start, then lower needle index, however long the rival. */ + substrings_leftmost_first_k, +}; + +/** + * @brief One reported match: which needle matched, and how long it is in the bytes the automaton walked. + * + * The automaton walks @b folded bytes, so this length is the folded one - the needle's own, identical for + * every match of that needle. The @b source span it corresponds to is not: needle "k" matches both the + * 1-byte "k" and the 3-byte Kelvin sign U+212A. Recovering that span is the walk's job, not this struct's. + */ +template <typename state_id_type_> +struct substrings_output { + using state_id_t = state_id_type_; + + state_id_t needle_index {}; + /** @brief Folded bytes this match spans; a walk traverses one edge per byte, so it fits a state id. */ + state_id_t folded_match_bytes {}; +}; + +/** + * @brief One reported match, locating it by haystack, by needle, and by byte span - the one match shape + * every backend emits, layout-identical to the C ABI's `szs_substrings_match_t`. + * + * Under case folding a needle's own byte length is not the length of every match - needle "k" matches both + * the 1-byte "k" and the 3-byte Kelvin sign - so the span is carried per match. The matched bytes, when + * needed, are `to_bytes_view(haystacks[match.haystack_index]).subspan(byte_offset, byte_length)`. + */ +struct substrings_match_t { + size_t haystack_index {}; + size_t needle_index {}; + size_t byte_offset {}; + size_t byte_length {}; + + /** @brief All four fields, since two matches agreeing on three of them are still different matches. */ + friend bool operator==(substrings_match_t const &first, substrings_match_t const &second) noexcept { + return first.haystack_index == second.haystack_index && first.needle_index == second.needle_index && + first.byte_offset == second.byte_offset && first.byte_length == second.byte_length; + } + friend bool operator!=(substrings_match_t const &first, substrings_match_t const &second) noexcept { + return !(first == second); + } +}; + +/** + * @brief One start position's incumbent match while a leftmost policy is still deciding it. + * + * Separate from `substrings_output` because the units differ: an output carries the folded length the + * automaton walked, a pending start carries the @b source span that length resolved to. Sharing one struct + * would let a folded length be compared against a source one, which the two leftmost policies would then + * silently resolve wrong. + */ +struct substrings_pending_start { + u32_t needle_index {}; + /** @brief Haystack bytes this match spans; zero means no match has claimed that start yet. */ + u32_t source_match_bytes {}; +}; + +/** + * @brief Whether @p challenger outranks @p incumbent among matches sharing one start position. + * @param[in] incumbent A zero `source_match_bytes` means no match has claimed that start yet. + * + * The only place either leftmost policy is consulted, and it runs once per discovered match rather than + * once per byte, so the policy stays an ordinary argument. Lengths compared here are source bytes, since + * a needle that folds shorter can still outspan a rival whose folded form is longer. + */ +constexpr bool substrings_leftmost_wins( // + substrings_pending_start const &challenger, // + substrings_pending_start const &incumbent, // + substrings_overlap_policy_t policy) noexcept { + + if (incumbent.source_match_bytes == 0) return true; + if (policy == substrings_leftmost_longest_k && challenger.source_match_bytes != incumbent.source_match_bytes) + return challenger.source_match_bytes > incumbent.source_match_bytes; + return challenger.needle_index < incumbent.needle_index; +} + +/** + * @brief How many starts a leftmost walk can hold undecided, given the longest match in the dictionary. + * @param[in] max_source_match_bytes The longest match in @b source bytes, which is what the slots are keyed + * by; passing the folded bound instead would undersize the ring wherever a fold contracts. + * + * A start can no longer be outbid once the walk is that many bytes past it, so that many slots suffice. + * Rounded up to a power of two, which turns the walk's slot lookup into a mask instead of a division by a + * value only known at runtime. + */ +inline size_t substrings_pending_starts_width(size_t max_source_match_bytes) noexcept { + return sz_size_bit_ceil(sz_max_of_two(max_source_match_bytes, (size_t)1)); +} + +/** @brief BM25's continuous parameters. */ +struct substrings_bm25_t { + /** @brief The literature's `k1`: how slowly repeated occurrences stop adding score. */ + f32_t term_frequency_saturation = 1.2f; + /** @brief The literature's `b`, in [0, 1]: 0 ignores document length, 1 normalizes fully. */ + f32_t length_normalization = 0.75f; + /** @brief Corpus-wide mean document length, read only when `length_normalization` is positive. The C + * SDK refuses a non-positive mean beside a positive `b`; this floor is the kernel's own. */ + f32_t average_document_length = 0.0f; +}; + +/** @brief One needle's saturated contribution, before its weight. */ +constexpr f32_t substrings_bm25_term(substrings_bm25_t const ¶meters, f32_t term_frequency, + f32_t document_length) noexcept { + // A zero mean length has no normalizer to divide by, so the length term collapses to one. + f32_t const normalized_length = parameters.average_document_length > 0.0f + ? 1.0f - parameters.length_normalization + + parameters.length_normalization * document_length / + parameters.average_document_length + : 1.0f; + return term_frequency * (parameters.term_frequency_saturation + 1.0f) / + (term_frequency + parameters.term_frequency_saturation * normalized_length); +} + +/** + * @brief One slot of a document-sized counter table: which needle, and how often this document hit it. + * + * A row with a counter per needle carries no key, since a needle's index is its position. A table sized by + * the document has to, and that key is the one field this adds over the bare count. + */ +struct substrings_bm25_counter_t { + u32_t needle_index {}; + u32_t frequency {}; +}; + +/** @brief Scrambles a needle index into a starting probe, as `probe_of_` does for a `(parent, byte)` pair. + * The caller masks with `capacity - 1`, so a capacity of one needs no `>> 64` special case. */ +constexpr size_t substrings_bm25_probe_of_(u32_t needle_index) noexcept { + return (size_t)((((u64_t)needle_index + 1) * 0x9E3779B97F4A7C15ull) >> 32); +} + +#pragma endregion Vocabulary + +#pragma region Published View + +/** @brief Number of columns in a hot-tier row, one per possible input byte. */ +static constexpr size_t substrings_alphabet_size_k = 256; + +/** + * @brief One goto-completed row of @p rows, whichever memory space holds them. + * @tparam index_type_ `size_t` for the global tier; `small_size_t` for a shared-memory staged prefix, whose + * bound makes the narrower multiply safe. Both operands convert first, so the alphabet constant + * cannot widen the product back. + */ +template <typename index_type_ = size_t, typename state_id_type_> +constexpr span<state_id_type_ const, substrings_alphabet_size_k> hot_row_of( // + span<state_id_type_ const> rows, state_id_type_ state) noexcept { + static_assert(sizeof(index_type_) >= sizeof(state_id_type_), + "The row index must be at least as wide as the state id, so this conversion never narrows"); + index_type_ const first_cell = static_cast<index_type_>(state) * + static_cast<index_type_>(substrings_alphabet_size_k); + sz_assert_(static_cast<size_t>(first_cell) + substrings_alphabet_size_k <= rows.size()); + return {rows.data() + first_cell}; +} + +/** + * @brief Immutable, trivially-copyable view of a built dictionary, safe to pass to a CUDA kernel by value. + * + * Owns nothing. Every pointer refers to storage held by the `aho_corasick_dictionary` that produced the + * view, which must outlive it. A device-side view points at device memory holding the same layout. + * + * Transitions are split into two tiers by how often a state is visited. Text keeps resetting the walk toward + * the root, so a small set of states absorbs most byte steps whatever the dictionary size, and the tiers are + * sized to that skew rather than to the automaton as a whole. + * + * The @b hot tier is a dense goto-completed table, one row of 256 targets per state. A step is a single load + * with no branch and no failure chasing. The @b cold tier is a double array: `base` and `check` encode + * transitions as address arithmetic plus an ownership test, and `fail` restores the failure links that + * goto-completion would otherwise have folded away. Completing the cold tier the same way would require + * every one of a state's 256 slots to be free, which is a dense row again, so its failure links stay live. + * + * States are numbered so the hot ones come first, making the tier test `state < hot_count` with no lookup. + */ +template <typename state_id_type_> +struct aho_corasick_view { + using state_id_t = state_id_type_; + using output_t = substrings_output<state_id_t>; + + /** @brief Hot tier: `hot_count * 256` goto-completed targets, row-major, shallow states first and each + * depth band ordered by out-degree - a build-time proxy for how often text visits a state. */ + state_id_t const *hot_rows {}; + + /** @brief Cold tier: transition target for `state` on `byte` is `base[state] + byte`, if owned. */ + state_id_t const *base {}; + /** @brief Cold tier: owner of each slot, so a collision reads as a missing edge rather than a wrong one. */ + state_id_t const *check {}; + /** @brief Cold tier: failure link, followed when `check` denies ownership. */ + state_id_t const *fail {}; + + /** + * @brief Matches ending at each state, flattened; already merged along failure chains at build time. + * + * Offsets are `size_t` rather than a narrower width because a state's outputs are its own plus its + * failure state's whole run, so a nested-suffix dictionary drives the pool to O(states squared). + */ + output_t const *outputs {}; + state_id_t const *outputs_counts {}; + size_t const *outputs_offsets {}; + /** @brief Length of `outputs`, so a consumer never has to rescan the CSR to recover it. */ + size_t outputs_total {}; + + /** @brief States `[0, hot_count)` live in `hot_rows`; the rest live in the double array. */ + state_id_t hot_count {}; + state_id_t state_count {}; + state_id_t root {}; + + /** @brief Longest needle, in the folded bytes the automaton walks. */ + state_id_t max_folded_match_bytes {}; + + /** @brief Shortest needle, in folded bytes; bounds how densely a rewrite can fire. */ + state_id_t min_folded_match_bytes {}; + + /** + * @brief Most @b haystack bytes one match can span, which is what every slice, halo and warm-up needs. + * + * A fold can contract three source bytes into one - the Kelvin sign into `k` - so a folded length says + * nothing directly about how far back into the haystack a match reaches. Sizing a window from + * `max_folded_match_bytes` instead would silently drop matches straddling a boundary. + */ + state_id_t max_source_match_bytes {}; + + /** @brief Fewest haystack bytes one match can span; the mirror bound. */ + state_id_t min_source_match_bytes {}; + + /** @brief Whether a walk folds the haystack as it consumes it, or steps it byte for byte. */ + substrings_case_sensitivity_t case_sensitivity {substrings_cased_k}; + + /** + * @brief Most merged outputs any single state carries, so a consumer can bound one pass's match count: + * `n` bytes report at most `n * max_outputs_per_state` matches. A nested-suffix vocabulary puts + * every shorter needle on the deepest state's run, and that is the worst case. + */ + state_id_t max_outputs_per_state {}; + + /** @brief Whether the cold tier is empty, so every step takes the branch-free hot path. */ + constexpr bool all_hot() const noexcept { return state_count <= hot_count; } + + /** @brief The whole hot tier as one span, so a row lookup can bounds-check itself. */ + constexpr span<state_id_t const> all_hot_rows() const noexcept { + return {hot_rows, hot_count * substrings_alphabet_size_k}; + } + + /** @brief One goto-completed row, whose width is the alphabet and therefore known at compile time. */ + constexpr span<state_id_t const, substrings_alphabet_size_k> hot_row(state_id_t state) const noexcept { + return hot_row_of(all_hot_rows(), state); + } +}; + +#pragma endregion Published View + +#pragma region Transition + +/** + * @brief Advances one state by one byte. The single definition shared by every CPU and GPU kernel. + * + * Hot states resolve in one load. Cold states probe the double array and, when the slot is owned by + * somebody else, hop to the failure link and retry the same byte. The root is total - every one of its + * slots resolves, self-looping where the trie has no edge - which is what terminates the retry loop. + * + * `constexpr` rather than host-device annotated, so CUDA kernels reach it through + * @b `--expt-relaxed-constexpr`, which every build path already passes. + */ +template <typename state_id_type_> +constexpr state_id_type_ aho_corasick_step( // + aho_corasick_view<state_id_type_> const &view, state_id_type_ state, u8_t byte) noexcept { + + for (;;) { + if (state < view.hot_count) return view.hot_row(state)[byte]; + // The probe index stays wide: `base + byte` can exceed the id ceiling on a slot this state does not + // own, and narrowing first would wrap onto a slot `check` might accept. The arrays carry + // `alphabet_size_k - 1` slots of headroom past the last state for exactly this reach. + size_t const candidate = (size_t)view.base[state] + byte; + if (view.check[candidate] == state) return (state_id_type_)candidate; + if (state == view.root) return view.root; + state = view.fail[state]; + } +} + +/** + * @brief Advances @p state by one byte and reports how many needles end on the new state. + * + * The pair every walk repeats: the transition, then the output count that decides whether the walk stops + * to enumerate matches. Sharing it keeps `find`, `count`, and the per-core counters reading one array. + */ +template <typename state_id_type_> +constexpr state_id_type_ aho_corasick_step_counting( // + aho_corasick_view<state_id_type_> const &view, state_id_type_ &state, u8_t byte) noexcept { + + state = aho_corasick_step(view, state, byte); + return view.outputs_counts[state]; +} + +#pragma endregion Transition + +#pragma region Folding Filter + +/** @brief Most folded bytes one source codepoint can produce: three runes of three bytes each. */ +static constexpr size_t substrings_folded_image_max_k = 9; + +/** @brief One folded byte, and everything the walk needs about the codepoint it came from. */ +struct substrings_folded_byte_t { + u8_t byte {}; + /** @brief Whether this byte ends a folded rune; a needle is valid UTF-8, so only there can a match end. */ + bool rune_end {}; + /** @brief Whether this codepoint's fold leaves a folded byte at an offset no source byte owns. */ + bool breaks_boundary {}; + /** @brief Whether the source byte began no well-formed codepoint, so the walk must resynchronize. */ + bool malformed {}; + /** @brief Offset just past the source codepoint; every folded byte of it reports the same end. */ + size_t codepoint_end {}; + /** @brief Folded bytes of this codepoint still to come, which a backward walk has to step over first. */ + u8_t trailing {}; + /** @brief Folded bytes back to this codepoint's previous rune end, zero at its first. */ + u8_t shift {}; +}; + +/** + * @brief Streams a haystack as folded bytes, one source codepoint at a time and never into a buffer. + * + * The automaton's alphabet is bytes while `sz_utf8_folded_iter_t` yields runes, so one codepoint's runes + * are drained and re-encoded into a nine-byte image, then handed out a byte at a time. The ASCII fast path, + * the expansion buffering and the one-byte malformed resynchronization all remain the iterator's. + */ +struct substrings_folded_cursor_t { + sz_utf8_folded_iter_t runes {}; + cptr_t origin {}; + u8_t image[substrings_folded_image_max_k] {}; + /** @brief Bit `index` marks the byte at `index` as ending a folded rune. */ + u16_t rune_end_mask {}; + u8_t image_length {}; + u8_t image_index {}; + u8_t previous_rune_end {}; + size_t codepoint_end {}; + bool breaks_boundary {}; + bool malformed {}; +}; + +SZ_HELPER_AUTO void substrings_folded_cursor_init(substrings_folded_cursor_t &cursor, + span<char const> haystack) noexcept { + sz_utf8_folded_iter_init_(&cursor.runes, haystack.data(), haystack.size()); + cursor.origin = haystack.data(); + cursor.image_length = 0; + cursor.image_index = 0; +} + +/** @brief Next folded byte, or false once the haystack is spent. */ +SZ_HELPER_AUTO bool substrings_folded_cursor_next(substrings_folded_cursor_t &cursor, + substrings_folded_byte_t &folded) noexcept { + + if (cursor.image_index == cursor.image_length) { + // ASCII is its own codepoint and folds with one add, so it never decodes and never consults a table. + if (cursor.runes.ptr < cursor.runes.end && (sz_u8_t)*cursor.runes.ptr < 0x80) { + cursor.image[0] = (u8_t)sz_ascii_fold_((sz_u8_t)*cursor.runes.ptr); + cursor.image_length = 1; + cursor.rune_end_mask = 1; + cursor.runes.codepoint_begin = cursor.runes.ptr; + cursor.runes.codepoint_length = 1; + ++cursor.runes.ptr; + cursor.breaks_boundary = false; + cursor.malformed = false; + } + else { + // Nothing under this lead byte folds, so the codepoint's own bytes already are its folded image: + // no rune is assembled, no fold ladder walked, no re-encode. This is every CJK, Arabic, Hebrew, + // Devanagari and emoji sequence, and it is what holds them at byte-exact speed. + rune_t rune; + rune_length_t verbatim = sz_rune_invalid_k; + if (cursor.runes.ptr < cursor.runes.end && !sz_utf8_lead_may_fold_((sz_u8_t)*cursor.runes.ptr)) + verbatim = sz_rune_decode(cursor.runes.ptr, cursor.runes.end, &rune); + + if (verbatim != sz_rune_invalid_k) { + for (size_t index = 0; index < (size_t)verbatim; ++index) + cursor.image[index] = (u8_t)cursor.runes.ptr[index]; + cursor.image_length = (u8_t)verbatim; + cursor.rune_end_mask = (u16_t)(1u << (verbatim - 1)); + cursor.runes.codepoint_begin = cursor.runes.ptr; + cursor.runes.codepoint_length = (size_t)verbatim; + cursor.runes.ptr += verbatim; + cursor.malformed = false; + cursor.breaks_boundary = false; + } + else { + if (!sz_utf8_folded_iter_next_(&cursor.runes, &rune)) return false; + + // A malformed byte arrives tagged, outside the scalar range `sz_rune_encode` accepts, so it + // passes through as the literal byte it was. + cursor.malformed = (rune & 0x80000000u) != 0; + cursor.image_length = 0; + cursor.rune_end_mask = 0; + size_t rune_count = 0; + for (;;) { + if (cursor.malformed) cursor.image[cursor.image_length++] = (u8_t)(rune & 0xFF); + else cursor.image_length += (u8_t)sz_rune_encode(rune, cursor.image + cursor.image_length); + cursor.rune_end_mask |= (u16_t)(1u << (cursor.image_length - 1)); + ++rune_count; + // More pending runes belong to this codepoint; the iterator refills only once they are spent. + if (cursor.runes.pending_idx >= cursor.runes.pending_count) break; + sz_utf8_folded_iter_next_(&cursor.runes, &rune); + } + cursor.breaks_boundary = rune_count != 1 || + (size_t)cursor.image_length != cursor.runes.codepoint_length; + } + } + + cursor.codepoint_end = (size_t)(cursor.runes.codepoint_begin - cursor.origin) + cursor.runes.codepoint_length; + cursor.image_index = 0; + cursor.previous_rune_end = 0; + } + + u8_t const index = cursor.image_index++; + folded.byte = cursor.image[index]; + folded.rune_end = (cursor.rune_end_mask >> index) & 1u; + folded.breaks_boundary = cursor.breaks_boundary; + folded.malformed = cursor.malformed; + folded.codepoint_end = cursor.codepoint_end; + folded.trailing = (u8_t)(cursor.image_length - cursor.image_index); + folded.shift = (u8_t)(cursor.image_index - cursor.previous_rune_end); + if (folded.rune_end) { + // A codepoint's first rune end has nothing before it to repeat, which a zero shift names. + if (cursor.previous_rune_end == 0) folded.shift = 0; + cursor.previous_rune_end = cursor.image_index; + } + return true; +} + +/** + * @brief Where a match resolved by walking backwards from the codepoint it ends in landed. + * @see `substrings_folded_span` for the cheap path that skips the walk entirely. + * + * `repeats` marks a span an earlier rune end of that same codepoint already reported: needle "s" ends at + * both runes the sharp S folds to, and both spans are the whole codepoint. Equal spans are what makes a + * repeat, not equal content - `"ss"` ends at both runes of the second sharp S in `"ßß"` too, and those are + * two genuinely different spans. + */ +struct substrings_resolved_match_t { + size_t source_offset {}; + bool repeats {}; +}; + +/** + * @brief Resolves a match's source start, and whether it repeats an earlier rune end's span, in one pass. + * @param[in] source_end End of the codepoint the match ends in; every rune end of it shares this. + * @param[in] trailing Folded bytes of that codepoint sitting past the match's end. + * @param[in] shift Folded bytes back to the previous rune end of the same codepoint, zero at the first. + * + * Reached only when the match starts at or before the last boundary-breaking codepoint, which is the only + * place either question is open: a boundary-preserving match starts on a lead byte, so distinct starts sit + * in distinct codepoints and no span can repeat. Two windows one length apart hold the same bytes exactly + * when the folded stream is periodic with that period, so the repeat test rides a `shift`-sized ring that + * one codepoint's image bounds, and shares the single backward walk with the start it recovers. + */ +SZ_HELPER_AUTO substrings_resolved_match_t substrings_resolve_match(span<char const> haystack, size_t source_end, + size_t trailing, size_t folded_match_bytes, + size_t shift) noexcept { + + sz_utf8_folded_reverse_iter_t iterator; + sz_utf8_folded_reverse_iter_init_(&iterator, haystack.data(), haystack.data() + source_end); + + substrings_resolved_match_t resolved; + resolved.source_offset = source_end; + resolved.repeats = false; + + u8_t pending[4]; + size_t pending_count = 0; + cptr_t codepoint_begin = haystack.data() + source_end; + u8_t ring[substrings_folded_image_max_k] = {}; + cptr_t start_here = nullptr, start_earlier = nullptr; + size_t const wanted = shift != 0 ? folded_match_bytes + shift : folded_match_bytes; + bool periodic = shift != 0; + + for (size_t stepped = 0; stepped < trailing + wanted; ++stepped) { + if (pending_count == 0) { + rune_t image; + if (!sz_utf8_folded_reverse_iter_prev_(&iterator, &image)) break; + // A malformed byte arrives tagged above the encodable range, so it passes through as itself. + if (image & 0x80000000u) pending[0] = (u8_t)(image & 0xFF), pending_count = 1; + else pending_count = (size_t)sz_rune_encode(image, pending); + // The cursor sits on the codepoint's own start as soon as any of its runes is yielded, which is + // the outward snap a match beginning mid-expansion needs. + codepoint_begin = iterator.ptr; + } + u8_t const byte = pending[--pending_count]; + if (stepped < trailing) continue; // ? Folded bytes of the ending codepoint past the match's own end + + size_t const taken = stepped - trailing + 1; + if (shift != 0) { + if (taken > shift && ring[(taken - shift) % substrings_folded_image_max_k] != byte) periodic = false; + ring[taken % substrings_folded_image_max_k] = byte; + } + if (taken == folded_match_bytes) start_here = codepoint_begin; + if (taken == wanted) start_earlier = codepoint_begin; + if (!periodic && taken >= folded_match_bytes) break; + } + + if (start_here != nullptr) resolved.source_offset = (size_t)(start_here - haystack.data()); + resolved.repeats = periodic && start_here != nullptr && start_here == start_earlier; + return resolved; +} + +/** + * @brief Resolves the source span of one output at the rune end @p step stands on. + * @param[in] folded Folded bytes consumed so far, the ending offset the output's length is taken back from. + * @param[in] last_break_folded_end Folded end of the last codepoint whose fold broke a boundary. + * + * A match starting at or after the last break lies where folded and source offsets still agree, so its start + * is one subtraction; anything earlier pays the backward walk. Every walk resolves matches this way, so the + * cheap test and the expensive fallback stay one decision rather than four copies of one. + */ +SZ_HELPER_AUTO substrings_resolved_match_t substrings_folded_span(span<char const> haystack, + substrings_folded_byte_t const &step, size_t folded, + size_t last_break_folded_end, + size_t folded_match_bytes) noexcept { + if (folded - folded_match_bytes >= last_break_folded_end) return {step.codepoint_end - folded_match_bytes, false}; + return substrings_resolve_match(haystack, step.codepoint_end, step.trailing, folded_match_bytes, step.shift); +} + +#pragma endregion Folding Filter + +#pragma region Engine + +/** + * @brief Multi-pattern search engine: one compiled dictionary applied to many haystacks in a single pass. + * + * Declared without a body so every backend supplies its own specialization, guarded on @p capability_ - + * the shape `levenshtein_distances` and the other StringZillas engines already use. The state-id width is + * not a parameter here: it follows from the needle set, so `try_build` settles it and stores whichever + * automaton won, the way the similarity engines choose a cell width from the inputs. + */ +template <typename allocator_type_ = dummy_alloc_t, sz_capability_t capability_ = sz_cap_serial_k, + typename enable_ = void> +struct substrings; + +/** @brief Which state-id width a built automaton settled on. @sa `substrings::state_width`. */ +enum class substrings_state_width_t : bool { u16_k, u32_k }; + +#pragma endregion Engine + +#pragma region Dictionary + +/** + * @brief Two-tier @b byte-level Aho-Corasick dictionary for multi-pattern exact and case-folded substring + * search: a dense goto-completed hot tier plus a double-array cold tier. + * @tparam state_id_type_ The type of the state ID. Default is `u32_t`; `u16_t` fits dictionaries under + * 65534 states into half the row width. + * @tparam allocator_type_ The type of the allocator. Default is `dummy_alloc_t`. + * + * Holds no STL container and throws nothing, reporting through `status_t` and `try_`-prefixed functions. + * Construction allocates a constant number of flat buffers rather than one per state, none of which + * survives `try_build`, and no phase ever materializes a dense row per state. + * + * `hot_rows_` is a plain dense 256-wide goto-completed table for the `hot_count_` most-visited states - + * raw state IDs, no byte-class compression and no premultiplication, so a lookup is + * `hot_rows_[state * 256 + byte]`. Every other state lives in the double array `base_` / `check_` / + * `fail_`, exactly as `aho_corasick_view` publishes it. + */ +template <typename state_id_type_ = u32_t, typename allocator_type_ = dummy_alloc_t> +struct aho_corasick_dictionary { + + using state_id_t = state_id_type_; + using allocator_t = allocator_type_; + using output_t = substrings_output<state_id_t>; + using pending_start_t = substrings_pending_start; + static_assert(std::is_unsigned<state_id_t>::value, "State ID should be unsigned"); + + static constexpr size_t alphabet_size_k = 256; + /** + * @brief The one ceiling of the automaton, shared by everything stored in `state_id_t` cells - states, + * needles, edges, slot capacity, match byte-lengths, and merged output runs. Each is checked for + * `overflow_risk_k` at its own growth site, so a `u16` dictionary caps needles at 65535 bytes. + */ + static constexpr state_id_t invalid_state_k = std::numeric_limits<state_id_t>::max(); + /** @brief `hot_count_`'s "not chosen yet" state, so `hot_count(0)` stays a real all-cold request. */ + static constexpr size_t derive_hot_count_k = std::numeric_limits<size_t>::max(); + /** @brief Interior vacancies one row may reject before it settles on the arena frontier instead. */ + static constexpr size_t max_interior_probes_k = 256; + + /** + * @brief Narrows @p value to a state id, asserting in debug that the pool ceiling still holds. + * + * Only for values whose ceiling `allocate_raw_state_` already enforced at insertion time. @b Not for + * caller-controlled sizes - those report `overflow_risk_k` in release builds too. + */ + static constexpr state_id_t state_id_of_(size_t value) noexcept { + sz_assert_(value <= static_cast<size_t>(invalid_state_k) && "State id does not fit state_id_t"); + return static_cast<state_id_t>(value); + } + + /** @brief One literal trie edge, appended as it is created and counting-sorted by `parent` at build. */ + struct trie_edge_t { + state_id_t parent; + state_id_t child; + u8_t byte; + }; + + /** @brief One trie edge again, narrowed for the build-time CSR where `parent` is the row index. */ + struct csr_edge_t { + state_id_t child; + u8_t byte; + }; + + /** @brief One pending match at a raw state, threaded into that state's own list by `output_run_t`. */ + struct pending_output_t { + state_id_t needle_index; + state_id_t folded_match_bytes; + size_t next; + }; + + /** @brief Per spelling state: the matches ending on it, shared by every walking state that spells it. */ + struct output_run_t { + /** @brief Head of this state's own match list, most recent first. */ + size_t own_head = SZ_SIZE_MAX; + /** @brief Matches ending exactly on this state, before failure-chain inheritance. */ + state_id_t own_count = 0; + }; + + /** + * @brief Per trie state: its failure link, its published slot, and its merged output run. + * + * One entry per raw state, addressed by that state's own id. Insertion builds a tree, so a state has + * exactly one spelling and therefore exactly one failure link. + */ + struct trie_state_t { + /** @brief The failure state, always strictly shallower, so depth order finishes it first. */ + state_id_t failure_state = 0; + /** @brief Published double-array slot for this state. */ + state_id_t published_id = invalid_state_k; + /** @brief Into `outputs_`: own matches followed by the failure state's whole run. */ + size_t total_offset = 0; + state_id_t total_count = 0; + }; + + private: + using allocator_traits_t = std::allocator_traits<allocator_t>; + using state_id_allocator_t = typename allocator_traits_t::template rebind_alloc<state_id_t>; + using output_allocator_t = typename allocator_traits_t::template rebind_alloc<output_t>; + using edge_allocator_t = typename allocator_traits_t::template rebind_alloc<trie_edge_t>; + using pending_output_allocator_t = typename allocator_traits_t::template rebind_alloc<pending_output_t>; + using output_run_allocator_t = typename allocator_traits_t::template rebind_alloc<output_run_t>; + using trie_state_allocator_t = typename allocator_traits_t::template rebind_alloc<trie_state_t>; + using csr_edge_allocator_t = typename allocator_traits_t::template rebind_alloc<csr_edge_t>; + using word_allocator_t = typename allocator_traits_t::template rebind_alloc<u64_t>; + using byte_allocator_t = typename allocator_traits_t::template rebind_alloc<std::byte>; + /** @brief Rebinds to `size_t`, for the edge index, `outputs_counts`, and `outputs_offsets` alike. */ + using offset_allocator_t = typename allocator_traits_t::template rebind_alloc<size_t>; + + /** @brief Every literal trie edge, in creation order; `compact_edges_into_csr_` consumes it. */ + safe_vector<trie_edge_t, edge_allocator_t> edges_; + /** @brief Open-addressed indices into `edges_`, giving insertion and the failure chase an O(1) lookup + * of `(parent, byte)` without storing a key of its own. Released alongside `edges_`. */ + safe_vector<size_t, offset_allocator_t> edge_index_; + /** @brief Matches ending at each raw state, before failure-chain inheritance merges them. */ + safe_vector<pending_output_t, pending_output_allocator_t> own_outputs_; + /** @brief One entry per raw state; grows with the state pool and survives into the output pass. */ + safe_vector<output_run_t, output_run_allocator_t> output_runs_; + /** @brief The needle under construction, folded once into canonical bytes; uncased mode only. */ + safe_vector<byte_t, byte_allocator_t> folded_needle_; + + /** @brief One block carved by `build_layout_` into the buffers whose size is fixed once insertion ends. */ + safe_vector<std::byte, byte_allocator_t> build_scratch_; + + /** @brief One entry per raw trie state, addressed by that state's own id. */ + safe_vector<trie_state_t, trie_state_allocator_t> trie_states_; + /** @brief Trie states in depth-band, out-degree-descending order: the input to the published numbering. + * Insertion order is not depth order, so this permutation - not the id itself - is what a band + * is a contiguous range of. */ + safe_vector<state_id_t, state_id_allocator_t> trie_order_; + /** @brief Scratch the band sort permutes through, since a band's states are not a contiguous id range. */ + safe_vector<state_id_t, state_id_allocator_t> trie_order_scratch_; + /** @brief The root's goto-completed row, dense over the alphabet, so a failure chase ends in one lookup + * rather than a scan of the root's whole edge list. */ + safe_vector<state_id_t, state_id_allocator_t> trie_root_row_; + + /** @brief Published slot -> walking state; grows with the double array, so it lives outside the layout. */ + safe_vector<state_id_t, state_id_allocator_t> old_of_final_; + /** @brief One bit per double-array slot; the only record of what the packing search has claimed. */ + safe_vector<u64_t, word_allocator_t> occupied_bits_; + /** @brief Lowest slot that could still be free; packing only fills forward, so it never moves back. */ + size_t lowest_free_cursor_ = 0; + /** @brief One past the highest claimed slot, so every slot at or above it is free by construction. */ + size_t arena_frontier_ = 0; + + /** @brief Hot tier: `hot_count_ * alphabet_size_k` goto-completed targets, row-major, shallow states + * first and each depth band ordered by out-degree descending. */ + safe_vector<state_id_t, state_id_allocator_t> hot_rows_; + /** @brief Cold tier: transition target for `state` on `byte` is `base_[state] + byte`, if `check_` confirms + * ownership. Sized `count_states_ + (alphabet_size_k - 1)`, since a child's ID is address + * arithmetic and can exceed the real state count; only entries `>= hot_count_` are meaningful. */ + safe_vector<state_id_t, state_id_allocator_t> base_; + /** @brief Cold tier: owner of each double-array slot; a mismatch means "no such edge", not "wrong edge". + * Same length as `base_`. */ + safe_vector<state_id_t, state_id_allocator_t> check_; + /** @brief Cold tier: failure link, followed when `check_` denies ownership. Same length as `base_`. */ + safe_vector<state_id_t, state_id_allocator_t> fail_; + /** @brief CSR-flattened match outputs, addressed through `outputs_offsets_` / `outputs_counts_`. */ + safe_vector<output_t, output_allocator_t> outputs_; + /** @brief Number of outputs per state, in the published band-ordered numbering, same length as `base_`. */ + safe_vector<state_id_t, state_id_allocator_t> outputs_counts_; + /** @brief Exclusive prefix sum of `outputs_counts_`, same length as `base_`. */ + safe_vector<size_t, offset_allocator_t> outputs_offsets_; + + size_t count_states_ = 0; + size_t count_needles_ = 0; + state_id_t max_folded_match_bytes_ = 0; + state_id_t min_folded_match_bytes_ = 0; + /** @brief Worst-case haystack span of one match; a fold contracting 3 bytes into 1 is what widens it. */ + state_id_t max_source_match_bytes_ = 0; + state_id_t min_source_match_bytes_ = 0; + state_id_t max_outputs_per_state_ = 0; + substrings_case_sensitivity_t case_sensitivity_ = substrings_cased_k; + + /** @brief States `[0, hot_count_)` live in `hot_rows_`; `derive_hot_count_k` asks `try_build` to size + * the tier from `cpu_specs_t` instead, which `hot_count` overrides with any explicit value. */ + size_t hot_count_ = derive_hot_count_k; + /** @brief The root's ID in the published numbering; always `0`, since the root is the unique + * shallowest state and therefore always sorts first. */ + state_id_t root_ = 0; + + allocator_t alloc_; + +#pragma region Construction Helpers + + /** @brief Scrambles a `(parent, byte)` pair into a starting probe; the index stores no key of its own. */ + static size_t probe_of_(state_id_t parent, u8_t byte) noexcept { + u64_t const mixed = ((((u64_t)parent << 8) | byte) + 1) * 0x9E3779B97F4A7C15ull; + return mixed >> 32; + } + + /** @brief Rebuilds `edge_index_` at @p new_capacity slots, reinserting every edge already in `edges_`. */ + status_t rehash_edge_index_(size_t new_capacity) noexcept { + if (edge_index_.try_resize(new_capacity) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t slot = 0; slot < new_capacity; ++slot) edge_index_[slot] = SZ_SIZE_MAX; + size_t const mask = new_capacity - 1; + for (size_t edge = 0; edge < edges_.size(); ++edge) { + trie_edge_t const &record = edges_[edge]; + size_t slot = probe_of_(record.parent, record.byte) & mask; + while (edge_index_[slot] != SZ_SIZE_MAX) slot = (slot + 1) & mask; + edge_index_[slot] = edge; + } + return status_t::success_k; + } + + /** @brief Index into `edges_` of the `(parent, byte)` edge, or the invalid sentinel when absent. */ + size_t find_edge_(state_id_t parent, u8_t byte) const noexcept { + size_t const mask = edge_index_.size() - 1; + for (size_t slot = probe_of_(parent, byte) & mask;; slot = (slot + 1) & mask) { + size_t const edge = edge_index_[slot]; + if (edge == SZ_SIZE_MAX) return edge; + trie_edge_t const &record = edges_[edge]; + if (record.parent == parent && record.byte == byte) return edge; + } + } + + /** @brief Records a `(parent, byte) -> child` edge that `find_edge_` has just reported missing. */ + status_t add_edge_(state_id_t parent, u8_t byte, state_id_t child) noexcept { + // Keep the load factor under one half, so linear probing stays a couple of slots deep. + if ((edges_.size() + 1) * 2 >= edge_index_.size()) + if (rehash_edge_index_(sz_size_bit_ceil(edge_index_.size() + 1) * 2) != status_t::success_k) + return status_t::bad_alloc_k; + if (edges_.try_push_back(trie_edge_t {parent, child, byte}) != status_t::success_k) + return status_t::bad_alloc_k; + + size_t const mask = edge_index_.size() - 1; + size_t slot = probe_of_(parent, byte) & mask; + while (edge_index_[slot] != SZ_SIZE_MAX) slot = (slot + 1) & mask; + edge_index_[slot] = (edges_.size() - 1); + return status_t::success_k; + } + + status_t allocate_raw_state_(state_id_t &new_state) noexcept { + if (count_states_ >= (size_t)invalid_state_k) return status_t::overflow_risk_k; + // `try_reserve` allocates exactly what is asked, so a bare `try_resize(count + 1)` per state would + // re-move the whole array on every allocation - quadratic in the state count. Reserving in + // power-of-two steps keeps the growth amortized-linear; the reserve is a no-op once capacity holds. + if (output_runs_.try_reserve(sz_size_bit_ceil(count_states_ + 1)) != status_t::success_k) + return status_t::bad_alloc_k; + if (output_runs_.try_resize(count_states_ + 1) != status_t::success_k) return status_t::bad_alloc_k; + new_state = static_cast<state_id_t>(count_states_); + ++count_states_; + return status_t::success_k; + } + + status_t ensure_root_() noexcept { + if (edge_index_.size() == 0 && rehash_edge_index_(1024) != status_t::success_k) return status_t::bad_alloc_k; + if (count_states_ != 0) return status_t::success_k; + state_id_t root; + status_t const status = allocate_raw_state_(root); + sz_assert_(status != status_t::success_k || root == 0); + return status; + } + + /** + * @brief Appends a match ending at @p state, onto that state's own list. + * + * A match traverses one edge per byte along a trie path whose positions strictly increase, so it can + * never span more bytes than the automaton has states - which is why the length rides `state_id_t`. + */ + status_t add_output_(state_id_t state, state_id_t needle_index, size_t folded_match_bytes) noexcept { + // Caller-controlled length, so the ceiling is a real status in every build rather than an assert. + if (folded_match_bytes > static_cast<size_t>(invalid_state_k)) return status_t::overflow_risk_k; + output_run_t &run = output_runs_[state]; + state_id_t const folded_narrow = static_cast<state_id_t>(folded_match_bytes); + if (own_outputs_.try_push_back(pending_output_t {needle_index, folded_narrow, run.own_head}) != + status_t::success_k) + return status_t::bad_alloc_k; + run.own_head = (own_outputs_.size() - 1); + ++run.own_count; + max_folded_match_bytes_ = sz_max_of_two(max_folded_match_bytes_, folded_narrow); + min_folded_match_bytes_ = min_folded_match_bytes_ ? sz_min_of_two(min_folded_match_bytes_, folded_narrow) + : folded_narrow; + + // One folded byte can stand for up to `sz_utf8_fold_max_contraction_k` source bytes, and one source + // byte for up to `sz_utf8_fold_max_expansion_k` folded ones, so a folded length brackets rather than + // fixes the source span. Cased needles fold to themselves, so their bounds stay exact. + size_t const contraction = case_sensitivity_ == substrings_uncased_k ? (size_t)sz_utf8_fold_max_contraction_k + : (size_t)1; + size_t const expansion = case_sensitivity_ == substrings_uncased_k ? (size_t)sz_utf8_fold_max_expansion_k + : (size_t)1; + size_t const source_ceiling = folded_match_bytes * contraction; + size_t const source_floor = (folded_match_bytes + expansion - 1) / expansion; + if (source_ceiling > static_cast<size_t>(invalid_state_k)) return status_t::overflow_risk_k; + max_source_match_bytes_ = sz_max_of_two(max_source_match_bytes_, static_cast<state_id_t>(source_ceiling)); + min_source_match_bytes_ = min_source_match_bytes_ + ? sz_min_of_two(min_source_match_bytes_, static_cast<state_id_t>(source_floor)) + : static_cast<state_id_t>(source_floor); + return status_t::success_k; + } + + /** @brief Canonical UTF-8 length of @p rune, as the encoder itself reports it. */ + static rune_length_t utf8_length_of_rune_(rune_t rune) noexcept { + u8_t scratch[4]; + return sz_rune_encode(rune, scratch); + } + + /** @brief Follows @p parent's @p byte edge, minting a state and wiring the edge when it is missing. */ + status_t follow_or_create_(state_id_t parent, u8_t byte, state_id_t &child) noexcept { + size_t const edge = find_edge_(parent, byte); + if (edge != SZ_SIZE_MAX) { + child = edges_[edge].child; + return status_t::success_k; + } + status_t const status = allocate_raw_state_(child); + if (status != status_t::success_k) return status; + return add_edge_(parent, byte, child); + } + + /** + * @brief Byte-exact trie insertion: standard follow-or-create walk, one state per byte consumed. + */ + status_t try_insert_cased_(span<byte_t const> needle, state_id_t needle_index) noexcept { + status_t status = ensure_root_(); + if (status != status_t::success_k) return status; + + state_id_t current_state = 0; + for (size_t offset = 0; offset < needle.size(); ++offset) { + status = follow_or_create_(current_state, needle[offset], current_state); + if (status != status_t::success_k) return status; + } + + return add_output_(current_state, needle_index, needle.size()); + } + + /** @brief Decodes and fully folds @p needle into `folded_needle_`, in canonical UTF-8 bytes. */ + status_t fold_needle_(span<byte_t const> needle) noexcept { + folded_needle_.clear(); + byte_t const *cursor = needle.begin(); + byte_t const *const needle_end = needle.end(); + while (cursor != needle_end) { + rune_t rune; + rune_length_t const consumed = sz_rune_decode(reinterpret_cast<cptr_t>(cursor), + reinterpret_cast<cptr_t>(needle_end), &rune); + if (consumed == sz_rune_invalid_k) return status_t::invalid_utf8_k; + rune_t images[3]; + size_t const runes = sz_unicode_fold_codepoint_(rune, images); + for (size_t index = 0; index < runes; ++index) { + u8_t encoded[4]; + rune_length_t const encoded_length = sz_rune_encode(images[index], encoded); + for (size_t byte = 0; byte < (size_t)encoded_length; ++byte) + if (folded_needle_.try_push_back((byte_t)encoded[byte]) != status_t::success_k) + return status_t::bad_alloc_k; + } + cursor += consumed; + } + return status_t::success_k; + } + + /** + * @brief Case-folded trie insertion: fold the needle once, then insert those bytes literally. + * + * The haystack is folded as the walk consumes it, so both sides meet in one canonical byte stream and + * the trie has nothing case-specific left in it - which is what keeps it a tree, and its failure links + * single-valued. A needle that folds to the same bytes as an earlier one simply shares its path. + */ + status_t try_insert_uncased_(span<byte_t const> needle, state_id_t needle_index) noexcept { + status_t status = ensure_root_(); + if (status != status_t::success_k) return status; + + // Reject malformed UTF-8 outright: the walk resets to the root on a malformed haystack byte, so a + // needle carrying one could never match, and accepting it would only hide the caller's mistake. + status = fold_needle_(needle); + if (status != status_t::success_k) return status; + if (folded_needle_.size() == 0) return status_t::success_k; + + return try_insert_cased_({folded_needle_.data(), folded_needle_.size()}, needle_index); + } + +#pragma endregion Construction Helpers + +#pragma region Build Phases + + /** + * @brief Byte offsets of the spelling-automaton CSR, the only construction buffers whose size is fixed + * once insertion ends. Everything the splitting pass produces is sized by the walking-state + * count, which it only learns as it runs, so those buffers are growable members instead. + */ + struct layout_t { + size_t edges = 0, edge_offsets = 0, total = 0; + }; + + layout_t build_layout_(cpu_specs_t const &specs) const noexcept { + scratch_amount_t amount {specs.cache_line_width}; + layout_t layout; + layout.edges = amount, amount += edges_.size() * sizeof(csr_edge_t); + layout.edge_offsets = amount, amount += (count_states_ + 1) * sizeof(state_id_t); + layout.total = amount; + return layout; + } + + csr_edge_t *edges_at_(layout_t const &layout) noexcept { + return (csr_edge_t *)(build_scratch_.data() + layout.edges); + } + state_id_t *edge_offsets_at_(layout_t const &layout) noexcept { + return (state_id_t *)(build_scratch_.data() + layout.edge_offsets); + } + + /** + * @brief Turns the insertion-time edge pool into a state-major CSR - one counting sort, no comparisons. + * @note Byte order inside a row is left alone; the packing search anchors on a `sz_byteset_t` child mask. + */ + void compact_edges_into_csr_(layout_t const &layout) noexcept { + state_id_t *const offsets = edge_offsets_at_(layout); + csr_edge_t *const rows = edges_at_(layout); + for (size_t state = 0; state <= count_states_; ++state) offsets[state] = 0; + for (size_t edge = 0; edge < edges_.size(); ++edge) ++offsets[(size_t)edges_[edge].parent + 1]; + for (size_t state = 0; state < count_states_; ++state) offsets[state + 1] += offsets[state]; + // Scatter with `offsets[parent]` doubling as that row's fill cursor, which leaves every entry + // holding its row's END; one backward shift turns them into starts again. + for (size_t edge = 0; edge < edges_.size(); ++edge) { + trie_edge_t const &record = edges_[edge]; + rows[offsets[record.parent]++] = csr_edge_t {record.child, record.byte}; + } + for (size_t state = count_states_; state > 0; --state) offsets[state] = offsets[state - 1]; + offsets[0] = 0; + } + + /** + * @brief Reorders one depth band of `trie_order_` by out-degree descending, so shallow high-fan-out + * states - the ones text keeps returning to - land in the hot tier regardless of dictionary + * content. A counting sort over a bounded degree. + * @note A band is the position range `[band_first, band_last)` within `trie_order_`, not an id range: + * insertion numbers states in needle order, so depth order lives only in this permutation. + */ + void order_band_by_out_degree_(state_id_t const *offsets, size_t band_first, size_t band_last) noexcept { + if (band_last - band_first < 2) return; + + state_id_t histogram[alphabet_size_k + 1] = {}; + for (size_t index = band_first; index < band_last; ++index) { + state_id_t const state = trie_order_[index]; + ++histogram[offsets[state + 1] - offsets[state]]; + } + // Suffix-summed, so the highest degree claims the lowest positions in the band. + state_id_t running = state_id_of_(band_first); + for (size_t degree = alphabet_size_k + 1; degree-- > 0;) { + state_id_t const count = histogram[degree]; + histogram[degree] = running; + running += count; + } + // Scattered through scratch rather than in place: the source is the band itself, so writing a + // position before reading it would overwrite a state still waiting to be placed. + for (size_t index = band_first; index < band_last; ++index) trie_order_scratch_[index] = trie_order_[index]; + for (size_t index = band_first; index < band_last; ++index) { + state_id_t const state = trie_order_scratch_[index]; + trie_order_[histogram[offsets[state + 1] - offsets[state]]++] = state; + } + } + + /** @brief Fills the dense root row from the root's edges, defaulting every other byte to the + * self-looping root, so a failure chase that falls all the way back resolves in one lookup. */ + status_t fill_root_row_(state_id_t const *offsets, csr_edge_t const *rows) noexcept { + if (trie_root_row_.try_resize(alphabet_size_k) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t byte = 0; byte < alphabet_size_k; ++byte) trie_root_row_[byte] = 0; + for (size_t edge = offsets[0]; edge < offsets[1]; ++edge) trie_root_row_[rows[edge].byte] = rows[edge].child; + return status_t::success_k; + } + + /** @brief Child of @p state on @p byte among its literal edges, or `invalid_state_k` if none. */ + state_id_t find_trie_edge_(state_id_t const *offsets, csr_edge_t const *rows, state_id_t state, + u8_t byte) const noexcept { + for (size_t edge = offsets[state]; edge < offsets[state + 1]; ++edge) + if (rows[edge].byte == byte) return rows[edge].child; + return invalid_state_k; + } + + /** @brief Goto-completed target for @p state on @p byte; the root answers from its dense row. */ + state_id_t chase_trie_(state_id_t const *offsets, csr_edge_t const *rows, state_id_t state, + u8_t byte) const noexcept { + for (state_id_t current = state;;) { + if (current == 0) return trie_root_row_[byte]; + state_id_t const child = find_trie_edge_(offsets, rows, current, byte); + if (child != invalid_state_k) return child; + current = trie_states_[current].failure_state; + } + } + + /** + * @brief Assigns every state's failure link and lays `trie_order_` out depth band by depth band. + * + * The classic Aho-Corasick construction: a child's failure link is found by chasing its parent's, and a + * failure link is always strictly shallower, so one shallow-to-deep pass finishes with no fixpoint. The + * trie is a tree, so each state is reached by exactly one edge and this visits each exactly once - which + * is why the state count is final at insertion and nothing is minted here. + */ + status_t build_failure_links_(layout_t const &layout) noexcept { + state_id_t const *const offsets = edge_offsets_at_(layout); + csr_edge_t const *const rows = edges_at_(layout); + + if (trie_states_.try_resize(count_states_) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t state = 0; state < count_states_; ++state) trie_states_[state] = trie_state_t {}; + if (trie_order_.try_resize(count_states_) != status_t::success_k) return status_t::bad_alloc_k; + if (trie_order_scratch_.try_resize(count_states_) != status_t::success_k) return status_t::bad_alloc_k; + + trie_order_[0] = 0; // ? The root fails to itself, which `trie_state_t` already defaults to. + size_t discovered = 1; + for (size_t band_first = 0, band_last = 1; band_first != band_last;) { + for (size_t index = band_first; index < band_last; ++index) { + state_id_t const parent = trie_order_[index]; + state_id_t const parent_failure = trie_states_[parent].failure_state; + for (size_t edge = offsets[parent]; edge < offsets[parent + 1]; ++edge) { + state_id_t const child = rows[edge].child; + // A depth-one state fails to the root; anything deeper chases its parent's failure link. + trie_states_[child].failure_state = + parent == 0 ? 0 : chase_trie_(offsets, rows, parent_failure, rows[edge].byte); + trie_order_[discovered++] = child; + } + // The root's own row has to be dense before any chase consults it, and the root is the only + // state in the first band, so filling it here is still ahead of every lookup. + if (parent == 0) { + status_t const status = fill_root_row_(offsets, rows); + if (status != status_t::success_k) return status; + } + } + order_band_by_out_degree_(offsets, band_first, band_last); + band_first = band_last, band_last = discovered; + } + // Every state is reachable from the root by construction, so a short walk means the trie is malformed. + sz_assert_(discovered == count_states_ && "A tree trie reaches every state exactly once"); + return status_t::success_k; + } + + /** @brief Grows every slot-indexed array to hold @p minimum slots. */ + status_t ensure_slot_capacity_(size_t minimum) noexcept { + if (minimum <= check_.size()) return status_t::success_k; + size_t const old_capacity = check_.size(); + size_t const new_capacity = sz_size_bit_ceil(minimum); + if (new_capacity >= (size_t)invalid_state_k) return status_t::overflow_risk_k; + + if (base_.try_resize(new_capacity) != status_t::success_k) return status_t::bad_alloc_k; + if (check_.try_resize(new_capacity) != status_t::success_k) return status_t::bad_alloc_k; + if (old_of_final_.try_resize(new_capacity) != status_t::success_k) return status_t::bad_alloc_k; + + // Four words of headroom so a 256-bit feasibility window never reads past the end. `try_resize` + // skips construction for trivially-constructible types, so every new word is explicitly cleared - + // the bitmap is the only record of what is claimed, and a stale set bit would hide a free slot. + size_t const old_words = occupied_bits_.size(); + size_t const new_words = (new_capacity >> 6) + 8; + if (occupied_bits_.try_resize(new_words) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t word = old_words; word < new_words; ++word) occupied_bits_[word] = 0; + + for (size_t slot = old_capacity; slot < new_capacity; ++slot) { + base_[slot] = 0; + check_[slot] = invalid_state_k; + old_of_final_[slot] = invalid_state_k; + } + return status_t::success_k; + } + + /** @brief Marks @p slot taken and carries the frontier past it; the bitmap is the only record of what + * is free, so the frontier has to move with every single claim to stay a valid bound. */ + void claim_slot_(size_t slot) noexcept { + occupied_bits_[slot >> 6] |= (u64_t)1 << (slot & 63); + arena_frontier_ = sz_max_of_two(arena_frontier_, slot + 1); + } + + /** + * @brief First free slot at or after @p from, growing the arena when the search runs off the end. + * + * Packing only ever fills forward, so `lowest_free_cursor_` never moves back and the whole walk across + * one build is amortized linear in the slot count - but only while every packing phase advances it. + * A phase that scans from the cursor without advancing it is quadratic in the states it places. + */ + status_t next_free_slot_(size_t from, size_t &found) noexcept { + for (size_t word = from >> 6;; ++word) { + if ((word << 6) >= check_.size()) { + status_t const status = ensure_slot_capacity_(check_.size() * 2); + if (status != status_t::success_k) return status; + } + u64_t vacancies = ~occupied_bits_[word]; + // Slots before `from` are not candidates, even when the word says they are free. + if (word == (from >> 6)) vacancies &= ~(u64_t)0 << (from & 63); + if (vacancies == 0) continue; + found = (word << 6) + (size_t)sz_u64_ctz(vacancies); + return status_t::success_k; + } + } + + /** @brief True when every byte set in @p wanted lands on a currently-free slot at @p base. */ + bool slots_are_free_(size_t base, sz_byteset_t const &wanted) const noexcept { + size_t const word = base >> 6, shift = base & 63; + for (size_t quarter = 0; quarter < 4; ++quarter) { + u64_t const low = occupied_bits_[word + quarter]; + u64_t const high = occupied_bits_[word + quarter + 1]; + u64_t const window = shift == 0 ? low : (low >> shift) | (high << (64 - shift)); + if (window & wanted._u64s[quarter]) return false; + } + return true; + } + + /** + * @brief Assigns every live state its published ID: the hot tier takes `[0, hot_count_)` in frequency + * order, and the rest are packed into the double array by a free list walked in the same + * depth-ascending order, so a state's literal parent always resolves before the state itself. + * + * Every state has exactly one parent edge, so every state is placed exactly once and each published slot + * names a distinct state. + */ + status_t pack_cold_tier_(state_id_t const *offsets, csr_edge_t const *rows) noexcept { + // Every state is fresh with `published_id == invalid_state_k`, which the packing reads as + // "not placed yet", so no reset loop is needed. + lowest_free_cursor_ = hot_count_; // ? Hot states own the low IDs outright. + status_t status = ensure_slot_capacity_(hot_count_ + alphabet_size_k); + if (status != status_t::success_k) return status; + + for (size_t hot_index = 0; hot_index < hot_count_; ++hot_index) { + state_id_t const state = trie_order_[hot_index]; + trie_states_[state].published_id = state_id_of_(hot_index); + old_of_final_[hot_index] = state; + claim_slot_(hot_index); + } + + if (hot_count_ == 0) { // ? The root has no parent to assign it a cold ID. + size_t assigned; + status = next_free_slot_(lowest_free_cursor_, assigned); + if (status != status_t::success_k) return status; + claim_slot_(assigned); + check_[assigned] = state_id_of_(assigned); + old_of_final_[assigned] = 0; + trie_states_[0].published_id = state_id_of_(assigned); + lowest_free_cursor_ = assigned + 1; + } + + for (size_t index = 0; index < count_states_; ++index) { + state_id_t const parent = trie_order_[index]; + status = index < hot_count_ ? pack_hot_children_(offsets, rows, parent) + : pack_cold_children_(offsets, rows, parent); + if (status != status_t::success_k) return status; + } + return status_t::success_k; + } + + /** + * @brief Places a hot parent's children on whatever free slots come next. + * + * `hot_rows_` addresses every child unconditionally, so nothing has to verify ownership through + * `check_` for them and they need no shared base. A child some other parent already placed keeps the + * slot it has, since both routes resolve to the same target through the completed row. + */ + status_t pack_hot_children_(state_id_t const *offsets, csr_edge_t const *rows, state_id_t parent) noexcept { + for (size_t edge = offsets[parent]; edge < offsets[parent + 1]; ++edge) { + state_id_t const child = rows[edge].child; + if (trie_states_[child].published_id != invalid_state_k) continue; + size_t assigned; + status_t const status = next_free_slot_(lowest_free_cursor_, assigned); + if (status != status_t::success_k) return status; + claim_slot_(assigned); + check_[assigned] = state_id_of_(assigned); + old_of_final_[assigned] = child; + trie_states_[child].published_id = state_id_of_(assigned); + lowest_free_cursor_ = assigned + 1; + } + return status_t::success_k; + } + + /** + * @brief Places a cold parent's children on one shared base, so `base_[parent] + byte` addresses each. + * + * Candidates are scanned out of the occupancy bitmap anchored on the parent's smallest child byte, and + * the whole row is tested at once rather than probed child by child. After `max_interior_probes_k` + * rejections the row settles on the arena frontier instead: the vacancies a packed arena leaves behind + * are mostly singletons no multi-byte row can ever cover, and a search that keeps re-walking them is + * quadratic in the states it places rather than linear. + */ + status_t pack_cold_children_(state_id_t const *offsets, csr_edge_t const *rows, state_id_t parent) noexcept { + if (offsets[parent] == offsets[parent + 1]) return status_t::success_k; + + safe_array<state_id_t, alphabet_size_k> child_of_byte; + sz_byteset_t child_mask; + sz_byteset_init(&child_mask); + for (size_t edge = offsets[parent]; edge < offsets[parent + 1]; ++edge) { + sz_byteset_add_u8(&child_mask, rows[edge].byte); + child_of_byte[rows[edge].byte] = rows[edge].child; + } + + u8_t anchor_byte = 0; + for (size_t quarter = 0; quarter < 4; ++quarter) + if (child_mask._u64s[quarter]) { + anchor_byte = (u8_t)(quarter * 64 + sz_u64_ctz(child_mask._u64s[quarter])); + break; + } + + // The arena's lowest vacancy, and the cursor this call publishes: a rejected row leaves it free. + size_t first_free = lowest_free_cursor_; + if (status_t const status = next_free_slot_(first_free, first_free); status != status_t::success_k) + return status; + + for (size_t candidate = first_free, rejected = 0;;) { + status_t status = next_free_slot_(candidate, candidate); + if (status != status_t::success_k) return status; + if (candidate < anchor_byte) { + ++candidate; + continue; + } + size_t const candidate_base = candidate - anchor_byte; + status = ensure_slot_capacity_(candidate_base + alphabet_size_k); + if (status != status_t::success_k) return status; + if (!slots_are_free_(candidate_base, child_mask)) { + // Landing on the frontier itself, rather than an anchor byte past it, is what keeps the + // fallback free: every slot from there up is unclaimed, so the row strands nothing behind it. + if (++rejected >= max_interior_probes_k) candidate = arena_frontier_; + else ++candidate; + continue; + } + + state_id_t const parent_final = trie_states_[parent].published_id; + base_[parent_final] = state_id_of_(candidate_base); + for (size_t quarter = 0; quarter < 4; ++quarter) + for (u64_t bits = child_mask._u64s[quarter]; bits; bits &= bits - 1) { + u8_t const byte = (u8_t)(quarter * 64 + sz_u64_ctz(bits)); + size_t const slot = candidate_base + byte; + state_id_t const child = child_of_byte[byte]; + claim_slot_(slot); + check_[slot] = parent_final; + old_of_final_[slot] = child; + // A tree gives every state exactly one parent edge, so no state is ever claimed twice and + // no slot is a relay for another's identity. + sz_assert_(trie_states_[child].published_id == invalid_state_k && "One parent edge per state"); + trie_states_[child].published_id = state_id_of_(slot); + } + // `candidate` is the lowest slot just claimed, so only a row landing on `first_free` frees it. + lowest_free_cursor_ = first_free == candidate ? first_free + 1 : first_free; + return status_t::success_k; + } + } + + /** + * @brief Sizes and fills the published output pool in one pass each. + * + * A state's matches are its own plus its failure state's complete set, and BFS order guarantees the + * failure state is finished first, so the pool is written once at exactly the right size. + */ + status_t size_and_fill_outputs_() noexcept { + size_t running = 0; + for (size_t index = 0; index < count_states_; ++index) { + state_id_t const state = trie_order_[index]; + trie_state_t &entry = trie_states_[state]; + output_run_t const &own = output_runs_[state]; + // A state's matches are its own plus its failure state's whole run. Read on every byte step, so + // the total rides `state_id_t` and the ceiling is refused here rather than assumed. Depth-band + // order finished the failure state first, so its total is already set. + size_t const total = static_cast<size_t>(own.own_count) + + (state == 0 ? size_t {0} + : static_cast<size_t>(trie_states_[entry.failure_state].total_count)); + if (total > static_cast<size_t>(invalid_state_k)) return status_t::overflow_risk_k; + entry.total_count = static_cast<state_id_t>(total); + entry.total_offset = running; + running += entry.total_count; + } + + if (outputs_.try_resize(running) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t index = 0; index < count_states_; ++index) { + state_id_t const state = trie_order_[index]; + trie_state_t const &entry = trie_states_[state]; + output_run_t const &own = output_runs_[state]; + output_t *const destination = outputs_.data() + entry.total_offset; + + // The per-state list is most-recent-first, so filling it backwards restores insertion order. + size_t written = own.own_count; + for (size_t walk = own.own_head; walk != SZ_SIZE_MAX; walk = own_outputs_[walk].next) { + pending_output_t const &pending = own_outputs_[walk]; + destination[--written] = output_t {pending.needle_index, pending.folded_match_bytes}; + } + + if (state == 0) continue; + trie_state_t const &inherited = trie_states_[entry.failure_state]; + for (size_t position = 0; position < inherited.total_count; ++position) + destination[own.own_count + position] = outputs_[inherited.total_offset + position]; + } + return status_t::success_k; + } + + /** + * @brief Materializes the hot tier's goto-completed rows by inheritance. + * + * Goto completion means `goto(state, byte) == goto(fail(state), byte)` wherever `state` has no literal + * edge on `byte`. The depth-primary ordering puts `fail(state)` at a strictly smaller index, so a hot + * state's failure state is always hot and always already materialized - one row copy plus one store per + * literal edge, instead of a failure chase per cell. + */ + status_t materialize_hot_rows_(state_id_t const *offsets, csr_edge_t const *rows) noexcept { + if (hot_rows_.try_resize(hot_count_ * alphabet_size_k) != status_t::success_k) return status_t::bad_alloc_k; + if (hot_count_ == 0) return status_t::success_k; + + state_id_t *const root_row = hot_rows_.data(); + for (size_t byte = 0; byte < alphabet_size_k; ++byte) root_row[byte] = root_; + for (size_t edge = offsets[0]; edge < offsets[1]; ++edge) + root_row[rows[edge].byte] = trie_states_[rows[edge].child].published_id; + + for (size_t hot_index = 1; hot_index < hot_count_; ++hot_index) { + state_id_t const state = trie_order_[hot_index]; + // A failure state is strictly shallower, so depth-primary order places it earlier and its row is + // already final. A real return rather than an assert: violated in a release build this would read + // an unwritten row and bake wrong transitions into the published automaton, silently. + size_t const inherited_index = (size_t)trie_states_[trie_states_[state].failure_state].published_id; + if (inherited_index >= hot_index) return status_t::unexpected_dimensions_k; + state_id_t const *const inherited = hot_rows_.data() + inherited_index * alphabet_size_k; + state_id_t *const row = hot_rows_.data() + hot_index * alphabet_size_k; + for (size_t byte = 0; byte < alphabet_size_k; ++byte) row[byte] = inherited[byte]; + for (size_t edge = offsets[state]; edge < offsets[state + 1]; ++edge) + row[rows[edge].byte] = trie_states_[rows[edge].child].published_id; + } + return status_t::success_k; + } + + /** @brief Fills `fail_`, `outputs_counts_`, and `outputs_offsets_` over the published slot range. */ + status_t publish_(size_t cold_capacity_published) noexcept { + + if (fail_.try_resize(cold_capacity_published) != status_t::success_k) return status_t::bad_alloc_k; + if (outputs_counts_.try_resize(cold_capacity_published) != status_t::success_k) return status_t::bad_alloc_k; + if (outputs_offsets_.try_resize(cold_capacity_published) != status_t::success_k) return status_t::bad_alloc_k; + + for (size_t slot = 0; slot < cold_capacity_published; ++slot) { + state_id_t const raw_state = slot < old_of_final_.size() ? old_of_final_[slot] : invalid_state_k; + if (raw_state == invalid_state_k) { + outputs_counts_[slot] = 0, outputs_offsets_[slot] = 0; + if (slot >= hot_count_) fail_[slot] = root_; + continue; + } + trie_state_t const &entry = trie_states_[raw_state]; + outputs_counts_[slot] = entry.total_count; + outputs_offsets_[slot] = entry.total_offset; + max_outputs_per_state_ = sz_max_of_two(max_outputs_per_state_, entry.total_count); + if (slot < hot_count_) continue; + // Each state owns exactly one slot, so a slot's failure link is simply its state's, published. + sz_assert_(entry.published_id == state_id_of_(slot) && "One published slot per state"); + fail_[slot] = trie_states_[entry.failure_state].published_id; + } + return status_t::success_k; + } + +#pragma endregion Build Phases + + public: + aho_corasick_dictionary() = default; + ~aho_corasick_dictionary() noexcept { reset(); } + + explicit aho_corasick_dictionary(allocator_t alloc) noexcept + : edges_(alloc), edge_index_(alloc), own_outputs_(alloc), output_runs_(alloc), folded_needle_(alloc), + build_scratch_(alloc), trie_states_(alloc), trie_order_(alloc), trie_order_scratch_(alloc), + trie_root_row_(alloc), old_of_final_(alloc), occupied_bits_(alloc), hot_rows_(alloc), base_(alloc), + check_(alloc), fail_(alloc), outputs_(alloc), outputs_counts_(alloc), outputs_offsets_(alloc), alloc_(alloc) { + } + + aho_corasick_dictionary(aho_corasick_dictionary &&) noexcept = default; + aho_corasick_dictionary &operator=(aho_corasick_dictionary &&) noexcept = default; + aho_corasick_dictionary(aho_corasick_dictionary const &) = delete; + aho_corasick_dictionary &operator=(aho_corasick_dictionary const &) = delete; + + /** @brief Frees every buffer construction needs and matching never touches. */ + void release_construction_scratch_() noexcept { + edges_.reset(); + edge_index_.reset(); + own_outputs_.reset(); + output_runs_.reset(); + folded_needle_.reset(); + build_scratch_.reset(); + trie_states_.reset(); + trie_order_.reset(); + trie_order_scratch_.reset(); + trie_root_row_.reset(); + old_of_final_.reset(); + occupied_bits_.reset(); + lowest_free_cursor_ = 0; + arena_frontier_ = 0; + } + + void reset() noexcept { + release_construction_scratch_(); + hot_rows_.reset(); + base_.reset(); + check_.reset(); + fail_.reset(); + outputs_.reset(); + outputs_counts_.reset(); + outputs_offsets_.reset(); + count_states_ = 0; + count_needles_ = 0; + max_folded_match_bytes_ = 0; + min_folded_match_bytes_ = 0; + max_source_match_bytes_ = 0; + min_source_match_bytes_ = 0; + max_outputs_per_state_ = 0; + case_sensitivity_ = substrings_cased_k; + hot_count_ = derive_hot_count_k; + root_ = 0; + } + + /** @brief Selects byte-exact or case-folded matching; must be called before the first `try_insert`. */ + void case_sensitivity(substrings_case_sensitivity_t desired) noexcept { + sz_assert_(count_needles_ == 0 && "Case sensitivity can't change once needles have been inserted"); + case_sensitivity_ = desired; + } + substrings_case_sensitivity_t case_sensitivity() const noexcept { return case_sensitivity_; } + + /** @brief Forces the hot-tier size instead of deriving it from `cpu_specs_t` in `try_build`. */ + void hot_count(size_t desired) noexcept { hot_count_ = desired; } + + size_t count_states() const noexcept { return count_states_; } + size_t count_needles() const noexcept { return count_needles_; } + state_id_t max_folded_match_bytes() const noexcept { return max_folded_match_bytes_; } + state_id_t min_folded_match_bytes() const noexcept { return min_folded_match_bytes_; } + state_id_t max_source_match_bytes() const noexcept { return max_source_match_bytes_; } + state_id_t min_source_match_bytes() const noexcept { return min_source_match_bytes_; } + size_t hot_count() const noexcept { return hot_count_; } + allocator_t const &allocator() const noexcept { return alloc_; } + + /** @brief Bytes held by both transition tiers together, hot rows plus the double array. */ + size_t transitions_bytes() const noexcept { + return (hot_rows_.size() + base_.size() + check_.size() + fail_.size()) * sizeof(state_id_t); + } + + /** + * @brief Adds a single @p needle to the vocabulary, assigning it a unique, insertion-order needle ID. + * @note Can't be called after `try_build`. Can't be called from multiple threads at the same time. + * @retval `status_t::success_k` The needle was successfully added. + * @retval `status_t::bad_alloc_k` Memory allocation failed. + * @retval `status_t::overflow_risk_k` Too many needles or states for the current state ID type. + * @retval `status_t::invalid_utf8_k` In `substrings_uncased_k` mode, the needle was not valid UTF-8. + * @retval `status_t::unexpected_dimensions_k` The needle was empty. + * + * An empty needle is rejected rather than skipped: it would match at every one of `haystack_length + 1` + * positions, and dropping it silently would shift every later needle's reported `needle_index`. + */ + status_t try_insert(span<byte_t const> needle) noexcept { + if (needle.size() == 0) return status_t::unexpected_dimensions_k; + if (count_needles_ >= (size_t)invalid_state_k) return status_t::overflow_risk_k; + + state_id_t const needle_index = static_cast<state_id_t>(count_needles_); + status_t const status = case_sensitivity_ == substrings_uncased_k ? try_insert_uncased_(needle, needle_index) + : try_insert_cased_(needle, needle_index); + if (status != status_t::success_k) return status; + ++count_needles_; + return status_t::success_k; + } + + status_t try_insert(span<char const> needle) noexcept { return try_insert(needle.template cast<byte_t const>()); } + + /** + * @brief Constructs the automaton from the vocabulary. Can only be called @b once. + * @param[in] specs Sizes the hot tier from the host's last-level cache, unless `hot_count` forced it. + * + * Seven phases, each named below: the edge pool becomes the spelling CSR, the splitting pass derives the + * walking automaton one depth band at a time while ordering each band by out-degree, the hot/cold split + * falls out of that ordering, the double array packs the rest, and the outputs and hot rows materialize. + */ + status_t try_build(cpu_specs_t const &specs = {}) noexcept { + status_t status = ensure_root_(); + if (status != status_t::success_k) return status; + + // Uncased preimages add edges into pre-existing states, so the edge pool is not bounded by the state + // pool - yet the CSR row offsets below store edge ordinals in `state_id_t` cells. + if (edges_.size() > (size_t)invalid_state_k) return status_t::overflow_risk_k; + + // Every sub-buffer is written before it is read, so zeroing the block first would be pure waste. + layout_t const layout = build_layout_(specs); + if (build_scratch_.try_resize_uninitialized(layout.total) != status_t::success_k) return status_t::bad_alloc_k; + compact_edges_into_csr_(layout); + + // The insertion pools have no reader past compaction; the CSR in `build_scratch_` carries every edge, + // and every phase below walks it rather than `find_edge_`. + edges_.reset(); + edge_index_.reset(); + + state_id_t const *const offsets = edge_offsets_at_(layout); + csr_edge_t const *const rows = edges_at_(layout); + + status = build_failure_links_(layout); + if (status != status_t::success_k) return status; + + // Hot rows are shared, read-mostly, and re-entered on nearly every byte, so they're sized against + // the last-level cache rather than a private L2 slice. + if (hot_count_ == derive_hot_count_k) hot_count_ = specs.l3_bytes / (alphabet_size_k * sizeof(state_id_t)); + hot_count_ = sz_min_of_two(hot_count_, count_states_); + + status = pack_cold_tier_(offsets, rows); + if (status != status_t::success_k) return status; + + root_ = trie_states_[0].published_id; + sz_assert_(root_ == 0 && "The root is the unique shallowest state, so it always sorts first"); + + // The exclusive published bound: not `hot_count_` plus the cold-state count, since a packed child's + // ID is address arithmetic and can skip past slots no state ever ended up owning. + size_t state_count_published = hot_count_; + for (size_t state = 0; state < count_states_; ++state) + state_count_published = sz_max_of_two(state_count_published, (size_t)trie_states_[state].published_id + 1); + size_t const cold_capacity_published = state_count_published + (alphabet_size_k - 1); + + // `base_` and `check_` were written in place by the packing above; widening them here only extends + // the address-arithmetic headroom a `base_[state] + byte` lookup can reach. + status = ensure_slot_capacity_(cold_capacity_published); + if (status != status_t::success_k) return status; + + status = size_and_fill_outputs_(); + if (status != status_t::success_k) return status; + status = materialize_hot_rows_(offsets, rows); + if (status != status_t::success_k) return status; + status = publish_(cold_capacity_published); + if (status != status_t::success_k) return status; + + count_states_ = state_count_published; + release_construction_scratch_(); + return status_t::success_k; + } + + /** + * @brief Adopts an already-built @p wider dictionary, narrowing every published array to this width. + * + * The walking automaton is derived once at the widest id and narrowed here, so a vocabulary that fits a + * smaller id pays no second derivation - only a copy of the published arrays, whose rows then halve. + * A `u16` row is 512 bytes against `u32`'s 1024, so twice the automaton stays cache-resident, which is + * what the tier split is sized against in the first place. + * + * Reads @p wider through its public view and accessors alone, so the two widths need no friendship. + * @retval `status_t::overflow_risk_k` Some published value exceeds this width; @p wider stays usable. + * @retval `status_t::bad_alloc_k` Memory allocation failed. + */ + template <typename wider_id_type_, typename wider_allocator_type_> + status_t try_build(aho_corasick_dictionary<wider_id_type_, wider_allocator_type_> const &wider) noexcept { + static_assert(sizeof(state_id_t) <= sizeof(wider_id_type_), "This overload only ever narrows"); + auto const source = wider.view(); + + // Every ceiling this width imposes, tested before a single element is copied. Slot ids reach past the + // state count by the alphabet's headroom, since a packed child's id is address arithmetic. + size_t const slots_published = (size_t)source.state_count + (alphabet_size_k - 1); + if (slots_published > (size_t)invalid_state_k) return status_t::overflow_risk_k; + if (wider.count_needles() > (size_t)invalid_state_k) return status_t::overflow_risk_k; + if ((size_t)source.max_source_match_bytes > (size_t)invalid_state_k) return status_t::overflow_risk_k; + if ((size_t)source.max_outputs_per_state > (size_t)invalid_state_k) return status_t::overflow_risk_k; + + size_t const hot_cells = (size_t)source.hot_count * alphabet_size_k; + if (hot_rows_.try_resize(hot_cells) != status_t::success_k) return status_t::bad_alloc_k; + if (base_.try_resize(slots_published) != status_t::success_k) return status_t::bad_alloc_k; + if (check_.try_resize(slots_published) != status_t::success_k) return status_t::bad_alloc_k; + if (fail_.try_resize(slots_published) != status_t::success_k) return status_t::bad_alloc_k; + if (outputs_.try_resize(source.outputs_total) != status_t::success_k) return status_t::bad_alloc_k; + if (outputs_counts_.try_resize(slots_published) != status_t::success_k) return status_t::bad_alloc_k; + if (outputs_offsets_.try_resize(slots_published) != status_t::success_k) return status_t::bad_alloc_k; + + for (size_t cell = 0; cell < hot_cells; ++cell) + hot_rows_[cell] = static_cast<state_id_t>(source.hot_rows[cell]); + for (size_t slot = 0; slot < slots_published; ++slot) { + base_[slot] = static_cast<state_id_t>(source.base[slot]); + check_[slot] = static_cast<state_id_t>(source.check[slot]); + fail_[slot] = static_cast<state_id_t>(source.fail[slot]); + outputs_counts_[slot] = static_cast<state_id_t>(source.outputs_counts[slot]); + outputs_offsets_[slot] = source.outputs_offsets[slot]; // ? Indexes a pool, so it stays `size_t` + } + for (size_t output = 0; output < source.outputs_total; ++output) + outputs_[output] = output_t {static_cast<state_id_t>(source.outputs[output].needle_index), + static_cast<state_id_t>(source.outputs[output].folded_match_bytes)}; + + count_states_ = source.state_count; + count_needles_ = wider.count_needles(); + hot_count_ = source.hot_count; + root_ = static_cast<state_id_t>(source.root); + max_folded_match_bytes_ = static_cast<state_id_t>(source.max_folded_match_bytes); + min_folded_match_bytes_ = static_cast<state_id_t>(source.min_folded_match_bytes); + max_source_match_bytes_ = static_cast<state_id_t>(source.max_source_match_bytes); + min_source_match_bytes_ = static_cast<state_id_t>(source.min_source_match_bytes); + max_outputs_per_state_ = static_cast<state_id_t>(source.max_outputs_per_state); + case_sensitivity_ = wider.case_sensitivity(); + return status_t::success_k; + } + +#pragma region Published View + + using view_t = aho_corasick_view<state_id_t>; + + /** + * @brief Immutable, trivially-copyable view of this dictionary, safe to pass to a CUDA kernel by value. + * @note Every pointer refers to storage owned by `*this`, which must outlive the view. + */ + view_t view() const noexcept { + view_t result; + result.hot_rows = hot_rows_.data(); + result.base = base_.data(); + result.check = check_.data(); + result.fail = fail_.data(); + result.outputs = outputs_.data(); + result.outputs_counts = outputs_counts_.data(); + result.outputs_offsets = outputs_offsets_.data(); + result.outputs_total = outputs_.size(); + result.hot_count = state_id_of_(hot_count_); + result.state_count = state_id_of_(count_states_); + result.root = root_; + result.max_folded_match_bytes = max_folded_match_bytes_; + result.min_folded_match_bytes = min_folded_match_bytes_; + result.max_source_match_bytes = max_source_match_bytes_; + result.min_source_match_bytes = min_source_match_bytes_; + result.max_outputs_per_state = max_outputs_per_state_; + result.case_sensitivity = case_sensitivity_; + return result; + } + +#pragma endregion Published View + +#pragma region Matching + + /** + * @brief Finds all occurrences of all needles in the @p haystack, byte for byte. + * @note This is the serial reference oracle: obvious correctness over speed. + * @param[in] callback Invoked as `callback(needle_index, match_offset, match_length)` with offsets + * relative to the span handed in, returning `true` to continue. + */ + template <typename callback_type_> + void find_cased_(span<byte_t const> haystack, callback_type_ &&callback) const noexcept { + view_t const automaton = view(); + state_id_t current_state = automaton.root; + for (size_t offset = 0; offset < haystack.size(); ++offset) { + u8_t const byte = haystack[offset]; + size_t const output_count = aho_corasick_step_counting(automaton, current_state, byte); + if (output_count == 0) continue; + size_t const output_offset = automaton.outputs_offsets[current_state]; + + for (size_t index = 0; index < output_count; ++index) { + output_t const &output = automaton.outputs[output_offset + index]; + size_t const match_length = output.folded_match_bytes; + // Tested by addition rather than by subtracting the length from the position: the walk + // always restarts at the root at this span's own start, so a match can never reach behind + // it, but a subtraction would wrap and read as in-bounds if that ever stopped holding. + if (offset + 1 < match_length) continue; + if (!callback((size_t)output.needle_index, offset + 1 - match_length, match_length)) return; + } + } + } + + /** + * @brief Finds all occurrences of all needles in the @p haystack, folding it one codepoint at a time. + * @param[in] callback Invoked as `callback(needle_index, match_offset, match_length)` with offsets + * relative to the span handed in, returning `true` to continue. + * + * Only a byte ending a folded rune can end a match, so a reported end is always a whole codepoint's. + */ + template <typename callback_type_> + void find_uncased_(span<byte_t const> haystack, callback_type_ &&callback) const noexcept { + view_t const automaton = view(); + state_id_t current_state = automaton.root; + + substrings_folded_cursor_t cursor; + substrings_folded_cursor_init(cursor, haystack.template cast<char const>()); + + size_t folded = 0, last_break_folded_end = 0; + substrings_folded_byte_t step; + while (substrings_folded_cursor_next(cursor, step)) { + ++folded; + if (step.malformed) { + // A malformed byte can never sit inside a match, so the walk drops back to the root and + // resynchronizes one byte at a time, exactly as the folded iterators do. + current_state = automaton.root; + continue; + } + + current_state = aho_corasick_step(automaton, current_state, step.byte); + if (!step.rune_end) continue; + // Claimed before this rune end reports: a match ending inside a boundary-breaking codepoint + // starts inside it too, and subtracting its folded length would land mid-codepoint. + if (step.breaks_boundary) last_break_folded_end = folded + step.trailing; + + size_t const output_count = automaton.outputs_counts[current_state]; + if (output_count == 0) continue; + size_t const output_offset = automaton.outputs_offsets[current_state]; + for (size_t index = 0; index < output_count; ++index) { + output_t const &output = automaton.outputs[output_offset + index]; + size_t const folded_length = output.folded_match_bytes; + if (folded < folded_length) continue; + + substrings_resolved_match_t const resolved = substrings_folded_span( + haystack.template cast<char const>(), step, folded, last_break_folded_end, folded_length); + if (resolved.repeats) continue; + size_t const match_offset = resolved.source_offset; + if (!callback((size_t)output.needle_index, match_offset, step.codepoint_end - match_offset)) return; + } + } + } + + /** @brief Finds all occurrences of all needles in the @p haystack, folding it when the mode asks. */ + template <typename callback_type_> + void find(span<byte_t const> haystack, callback_type_ &&callback) const noexcept { + if (case_sensitivity_ == substrings_uncased_k) + return find_uncased_(haystack, std::forward<callback_type_>(callback)); + find_cased_(haystack, std::forward<callback_type_>(callback)); + } + + template <typename callback_type_> + void find(span<char const> haystack, callback_type_ &&callback) const noexcept { + find(haystack.template cast<byte_t const>(), std::forward<callback_type_>(callback)); + } + + /** + * @brief Counts the number of occurrences of all the needles in the @p haystack. + * @return The number of potentially-overlapping occurrences. + */ + size_t count(span<byte_t const> haystack) const noexcept { + // The folded walk reports at folded-rune ends rather than at every byte, and collapses spans that + // resolve to one, so a byte-per-step count would not agree with what `find` emits. Counting through + // the same walk is what keeps `try_find`'s count-then-write pass consistent. + if (case_sensitivity_ == substrings_uncased_k) { + size_t total = 0; + find_uncased_(haystack, [&](size_t, size_t, size_t) noexcept { return ++total, true; }); + return total; + } + + view_t const automaton = view(); + size_t total = 0; + state_id_t current_state = automaton.root; + // One 4-byte load feeds four transitions - the state chain stays strictly serial, and `sz_u32_load` + // absorbs misalignment itself, so only a tail loop remains. + size_t offset = 0; + for (; offset + 4 <= haystack.size(); offset += 4) { + sz_u32_vec_t const quad = sz_u32_load((sz_cptr_t)(haystack.data() + offset)); + total += aho_corasick_step_counting(automaton, current_state, quad.u8s[0]); + total += aho_corasick_step_counting(automaton, current_state, quad.u8s[1]); + total += aho_corasick_step_counting(automaton, current_state, quad.u8s[2]); + total += aho_corasick_step_counting(automaton, current_state, quad.u8s[3]); + } + for (; offset < haystack.size(); ++offset) + total += aho_corasick_step_counting(automaton, current_state, haystack[offset]); + return total; + } + + /** + * @brief Emits the matches of @p haystack that share no bytes, one per accepted start position. + * @param[in] pending_starts Scratch of `substrings_pending_starts_width` entries, at least one; + * contents on entry are ignored. + * @param[in] callback Invoked as `callback(needle_index, match_offset, match_length)` in ascending + * start order, returning `true` to continue. + * + * Matches surface at their end, so the earliest start is not the first seen: over "abcd" against + * {"bc", "abcd"}, "bc" completes first and "abcd" starts before it. A start settles only once the walk + * is `max_source_match_bytes` past it, which is what the pending starts hold. + */ + template <typename callback_type_> + void find_leftmost(span<byte_t const> haystack, span<pending_start_t> pending_starts, + substrings_overlap_policy_t policy, callback_type_ &&callback) const noexcept { + + sz_assert_(policy != substrings_overlapping_k && "Overlapping matches are reported through `find`"); + size_t const width = pending_starts.size(); + sz_assert_(width >= substrings_pending_starts_width(max_source_match_bytes_)); + sz_assert_((width & (width - 1)) == 0 && "A power-of-two width is what turns the lookup into a mask"); + size_t const mask = width - 1; + for (size_t slot_index = 0; slot_index < width; ++slot_index) pending_starts[slot_index] = {}; + + size_t cursor = 0, settled = 0; + bool keep_going = true; + auto const accept_start = [&](size_t start) noexcept { + pending_start_t &slot = pending_starts[start & mask]; + if (slot.source_match_bytes != 0 && start >= cursor) { + keep_going = callback((size_t)slot.needle_index, start, (size_t)slot.source_match_bytes); + cursor = start + slot.source_match_bytes; + } + slot = {}; + }; + + // `find` reports in non-decreasing end order, so the starts drain from the end each match reaches - + // no second walk, and the cursor test waits until a start can no longer be outbid. + size_t undrained_end = 0; + find(haystack, [&](size_t needle_index, size_t start, size_t length) noexcept { + size_t const settles_before = start + length > width ? start + length - width : 0; + for (; settled < settles_before && keep_going; ++settled) accept_start(settled); + if (!keep_going) return false; + pending_start_t const challenger {(u32_t)needle_index, (u32_t)length}; + pending_start_t &slot = pending_starts[start & mask]; + if (substrings_leftmost_wins(challenger, slot, policy)) slot = challenger; + undrained_end = sz_max_of_two(undrained_end, start + 1); + return true; + }); + + // Draining stops at the last start any match claimed rather than at the haystack's end: a slot is + // only ever occupied by a start a match reported, and visiting positions past the final one would + // both waste a pass over the whole haystack and re-report a stale slot at a position it never + // matched. A haystack no needle hits leaves this at zero and drains nothing at all. + for (; settled < undrained_end && keep_going; ++settled) accept_start(settled); + } + + template <typename callback_type_> + void find_leftmost(span<char const> haystack, span<pending_start_t> pending_starts, + substrings_overlap_policy_t policy, callback_type_ &&callback) const noexcept { + find_leftmost(haystack.template cast<byte_t const>(), pending_starts, policy, + std::forward<callback_type_>(callback)); + } + + /** + * @brief Every match of @p haystack, in the order @p policy reports them. + * @param[in] pending_starts Scratch the leftmost policies settle starts in; unused when they overlap. + * + * This is the one place the policy picks a walk. Engines forward their argument here rather than + * branching on it themselves, so a new backend implements this pair and nothing else. + */ + template <typename callback_type_> + void visit(span<byte_t const> haystack, substrings_overlap_policy_t policy, span<pending_start_t> pending_starts, + callback_type_ &&callback) const noexcept { + if (policy == substrings_overlapping_k) return find(haystack, std::forward<callback_type_>(callback)); + find_leftmost(haystack, pending_starts, policy, std::forward<callback_type_>(callback)); + } + + /** + * @brief How many matches `visit` would report, without enumerating them where it need not. + * @param[in] pending_starts Scratch the leftmost policies settle starts in; unused when they overlap. + */ + size_t count(span<byte_t const> haystack, substrings_overlap_policy_t policy, + span<pending_start_t> pending_starts) const noexcept { + // The overlapping count has a dedicated four-byte-load walk that never enumerates an output run. + if (policy == substrings_overlapping_k) return count(haystack); + size_t total = 0; + find_leftmost(haystack, pending_starts, policy, [&](size_t, size_t, size_t) noexcept { + ++total; + return true; + }); + return total; + } + +#pragma endregion Matching +}; + +using substrings_u16_dictionary_t = aho_corasick_dictionary<u16_t, std::allocator<char>>; +using substrings_u32_dictionary_t = aho_corasick_dictionary<u32_t, std::allocator<char>>; + +#pragma endregion Dictionary + +#pragma region Rewriting + +/** @brief Bytes @p haystack becomes once every match is swapped for its needle's replacement. */ +template <typename dictionary_type_, typename replacements_type_> +size_t substrings_rewritten_size(dictionary_type_ const &dictionary, span<byte_t const> haystack, + span<typename dictionary_type_::pending_start_t> pending_starts, + substrings_overlap_policy_t policy, replacements_type_ const &replacements) noexcept { + size_t removed = 0, added = 0; + dictionary.find_leftmost(haystack, pending_starts, policy, + [&](size_t needle_index, size_t, size_t length) noexcept { + removed += length; + added += to_bytes_view(replacements[needle_index]).size(); + return true; + }); + // Accumulated apart rather than netted per match, so a shrinking rewrite never wraps the unsigned sum. + return haystack.size() - removed + added; +} + +/** @brief Writes the rewritten @p haystack at @p output, whose room the caller has already reserved. */ +template <typename dictionary_type_, typename replacements_type_> +void substrings_rewrite(dictionary_type_ const &dictionary, span<byte_t const> haystack, + span<typename dictionary_type_::pending_start_t> pending_starts, + substrings_overlap_policy_t policy, replacements_type_ const &replacements, + char *output) noexcept { + size_t cursor = 0; + dictionary.find_leftmost(haystack, pending_starts, policy, + [&](size_t needle_index, size_t offset, size_t length) noexcept { + span<byte_t const> const replacement = to_bytes_view(replacements[needle_index]); + sz_copy(output, (sz_cptr_t)(haystack.data() + cursor), offset - cursor); + output += offset - cursor; + sz_copy(output, (sz_cptr_t)replacement.data(), replacement.size()); + output += replacement.size(); + cursor = offset + length; + return true; + }); + sz_copy(output, (sz_cptr_t)(haystack.data() + cursor), haystack.size() - cursor); +} + +/** @brief Rejects a rewrite under a policy that leaves no non-overlapping cover to substitute. */ +inline status_t substrings_check_rewritable(substrings_overlap_policy_t policy) noexcept { + return policy == substrings_overlapping_k ? status_t::unknown_k : status_t::success_k; +} + +/** @brief Turns per-haystack sizes into `size() - 1` boundaries plus a terminator, in place. */ +inline void substrings_sizes_into_offsets(span<size_t> offsets, size_t &total) noexcept { + total = 0; + for (size_t index = 0; index + 1 < offsets.size(); ++index) { + size_t const size = offsets[index]; + offsets[index] = total; + total += size; + } + offsets[offsets.size() - 1] = total; +} + +#pragma endregion Rewriting + +#pragma region Scoring + +/** @brief Whether a caller supplied its own length for a document, or wants the haystack's byte count. */ +enum class substrings_document_length_t : bool { + /** @brief Take @p document_length as given, in whatever unit the pipeline normalizes by. */ + given_k, + /** @brief Ignore @p document_length and measure the haystack in bytes, the only unit this engine owns. */ + haystack_bytes_k, +}; + +/** + * @brief Counts every occurrence of every needle in @p haystack into @p frequencies. + * @param[in,out] touched_count Grows by the needles this call hits for the first time. + * + * Split from the scoring below so several cores can count slices of one haystack into their own rows and + * merge afterwards, which is the only way a single long document reaches more than one core. + */ +template <typename dictionary_type_> +void substrings_bm25_count(dictionary_type_ const &dictionary, span<byte_t const> haystack, span<u32_t> frequencies, + span<u32_t> touched, size_t &touched_count) noexcept { + dictionary.find(haystack, [&](size_t needle_index, size_t, size_t) noexcept { + if (frequencies[needle_index]++ == 0) touched[touched_count++] = (u32_t)needle_index; + return true; + }); +} + +/** + * @brief Scores one counted document, leaving @p frequencies zeroed again for whoever runs next. + * @param[in] touched The needles the document hit, so the reset skips the rest of the vocabulary. + */ +inline f32_t substrings_bm25_total(span<f32_t const> needle_weights, substrings_bm25_t parameters, + f32_t document_length, span<u32_t> frequencies, span<u32_t> touched, + size_t touched_count) noexcept { + + // Float addition is not associative, so the summation order is part of the answer, and the order this + // engine publishes is ascending by needle. A walk touches needles in whatever order the haystack spells + // them, so they are ordered here before anything is added. + u32_t const *ordered = touched.data(); + if (touched_count * 2 <= touched.size()) { + // A least-significant-byte radix sort, with the list's own unused tail as the second buffer - which + // the half-full test above is what guarantees. Passes cover the widest needle index and no more, so + // a dictionary under 65,536 needles is two passes rather than four. + size_t const highest_index = frequencies.size() ? frequencies.size() - 1 : 0; + size_t key_bytes = 1; + while (key_bytes < sizeof(u32_t) && (highest_index >> (key_bytes * 8)) != 0) ++key_bytes; + + u32_t *source = touched.data(); + u32_t *target = touched.data() + touched_count; + for (size_t byte = 0; byte != key_bytes; ++byte) { + u32_t histogram[256] {}; + size_t const shift = byte * 8; + for (size_t slot = 0; slot != touched_count; ++slot) ++histogram[(source[slot] >> shift) & 0xFFu]; + for (size_t bucket = 0, offset = 0; bucket != 256; ++bucket) { + u32_t const count = histogram[bucket]; + histogram[bucket] = (u32_t)offset; + offset += count; + } + for (size_t slot = 0; slot != touched_count; ++slot) + target[histogram[(source[slot] >> shift) & 0xFFu]++] = source[slot]; + u32_t *const previous = source; + source = target, target = previous; + } + ordered = source; + } + else { + // Past half the vocabulary the row is the cheaper index: walking it ascending costs about what + // ordering the list would, and arrives already sorted. + size_t written = 0; + for (size_t needle_index = 0; needle_index != frequencies.size(); ++needle_index) + if (frequencies[needle_index]) touched[written++] = (u32_t)needle_index; + } + + f32_t score = 0; + for (size_t slot = 0; slot < touched_count; ++slot) { + size_t const needle_index = ordered[slot]; + score += needle_weights[needle_index] * + substrings_bm25_term(parameters, (f32_t)frequencies[needle_index], document_length); + frequencies[needle_index] = 0; + } + return score; +} + +/** + * @brief One haystack's BM25 score, leaving @p frequencies zeroed again for whoever runs next. + * @param[in] length_source Whether @p document_length is the caller's own or should be measured here. + * @param[in] frequencies One counter per needle, zero on entry and on return. + * @param[in] touched Room for the needles this haystack hits, so the reset skips the rest of the vocabulary. + */ +template <typename dictionary_type_> +f32_t substrings_bm25_score(dictionary_type_ const &dictionary, span<byte_t const> haystack, + substrings_document_length_t length_source, f32_t document_length, + substrings_bm25_t parameters, span<f32_t const> needle_weights, span<u32_t> frequencies, + span<u32_t> touched) noexcept { + size_t touched_count = 0; + substrings_bm25_count(dictionary, haystack, frequencies, touched, touched_count); + if (length_source == substrings_document_length_t::haystack_bytes_k) document_length = (f32_t)haystack.size(); + return substrings_bm25_total(needle_weights, parameters, document_length, frequencies, touched, touched_count); +} + +#pragma endregion Scoring + +#pragma region Primary API + +/** + * @brief Aho-Corasick-based @b single-threaded multi-pattern exact/case-folded substring search. + * @tparam state_id_type_ The type of the state ID. + * @tparam allocator_type_ The type of the allocator. + * @tparam capability_ Matches `sz_cap_serial_k` and any other capability that isn't parallel or CUDA; the + * two-level parallel specialization below claims `sz_caps_sp_k` specifically. + */ +template <typename allocator_type_, sz_capability_t capability_> +struct substrings<allocator_type_, capability_, + std::enable_if_t<(capability_ & (sz_cap_parallel_k | sz_cap_cuda_k)) == 0>> { + using allocator_t = allocator_type_; + using narrow_dictionary_t = aho_corasick_dictionary<u16_t, allocator_t>; + using wide_dictionary_t = aho_corasick_dictionary<u32_t, allocator_t>; + using match_t = substrings_match_t; + using pending_start_t = substrings_pending_start; + static constexpr sz_capability_t capability_k = capability_; + + using size_allocator_t = typename std::allocator_traits<allocator_t>::template rebind_alloc<size_t>; + using pending_start_allocator_t = + typename std::allocator_traits<allocator_t>::template rebind_alloc<pending_start_t>; + using u32_allocator_t = typename std::allocator_traits<allocator_t>::template rebind_alloc<u32_t>; + + explicit substrings(allocator_t alloc = allocator_t()) noexcept + : alloc_(alloc), dict_(std::in_place_type_t<wide_dictionary_t>(), alloc) {} + void reset() noexcept { + std::visit([](auto &dict) noexcept { dict.reset(); }, dict_); + } + + /** @brief The state-id width `try_index` settled on, which the needle set alone decides. */ + substrings_state_width_t state_width() const noexcept { + return std::holds_alternative<narrow_dictionary_t>(dict_) ? substrings_state_width_t::u16_k + : substrings_state_width_t::u32_k; + } + size_t count_needles() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.count_needles(); }, dict_); + } + size_t count_states() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.count_states(); }, dict_); + } + size_t max_source_match_bytes() const noexcept { + return std::visit([](auto const &dict) noexcept { return (size_t)dict.max_source_match_bytes(); }, dict_); + } + size_t min_source_match_bytes() const noexcept { + return std::visit([](auto const &dict) noexcept { return (size_t)dict.min_source_match_bytes(); }, dict_); + } + size_t hot_count() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.hot_count(); }, dict_); + } + substrings_case_sensitivity_t case_sensitivity() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.case_sensitivity(); }, dict_); + } + + /** + * @brief Runs @p callable against the automaton at whichever state-id width it settled on. + * + * One dispatch per call rather than per byte: the walks it hands the dictionary to stay monomorphic, + * so this costs a branch where a width-templated engine cost an instantiation. + */ + template <typename callable_type_> + auto visit_dictionary(callable_type_ &&callable) const noexcept { + return std::visit(std::forward<callable_type_>(callable), dict_); + } + + /** + * @brief Indexes all of the @p needles strings into the FSM, at whichever state id it ends up fitting. + * + * Construction runs wide, because a dictionary's state count is only known once it is built; the narrowing + * attempt is then itself the ceiling test, and only `overflow_risk_k` means "does not fit". + * @param[in] executor Taken for one shape across every entry point; construction stays on the calling + * thread, as building the FSM is not generally a bottleneck next to walking it. + * @param[in] specs Sizes the hot tier from the host's last-level cache. + * @note Replaces any previously indexed needle set: the automaton is rebuilt from scratch and the old one + * released, so an engine can be re-indexed for a different vocabulary or a different machine. + * @sa `aho_corasick_dictionary::try_insert` for the status codes this forwards. + */ + template <typename needles_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_index(needles_type_ &&needles, substrings_case_sensitivity_t case_sensitivity = substrings_cased_k, + executor_type_ &&executor = {}, cpu_specs_t const &specs = {}) noexcept { + sz_unused_(executor); + wide_dictionary_t wide(alloc_); + wide.case_sensitivity(case_sensitivity); + for (auto const &needle : needles) { + status_t const status = wide.try_insert(to_bytes_view(needle)); + if (status != status_t::success_k) return status; + } + if (status_t const built = wide.try_build(specs); built != status_t::success_k) return built; + + narrow_dictionary_t narrow(alloc_); + status_t const narrowed = narrow.try_build(wide); + if (narrowed == status_t::success_k) { + dict_.template emplace<narrow_dictionary_t>(std::move(narrow)); + return status_t::success_k; + } + if (narrowed != status_t::overflow_risk_k) return narrowed; + dict_.template emplace<wide_dictionary_t>(std::move(wide)); + return status_t::success_k; + } + + /** + * @brief Occurrences of all needles in each of the @p haystacks, for filtering and ranking. + * @param[in] overlap_policy Whether every overlapping match counts, or only the leftmost ones. + * @param[out] matches_total Sum of @p counts_per_haystack, which is what sizes a later `try_find` buffer. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> + status_t try_count(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<size_t> counts_per_haystack, size_t &matches_total, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + // A single-threaded walk has no split to schedule and no cache size to compare against. + sz_unused_(executor); + sz_unused_(specs); + sz_assert_(counts_per_haystack.size() == haystacks.size()); + if (status_t const reserved = try_reserve_pending_starts_(overlap_policy); reserved != status_t::success_k) + return reserved; + + matches_total = 0; + std::visit( + [&](auto const &dict) noexcept { + for (size_t index = 0; index < counts_per_haystack.size(); ++index) + matches_total += counts_per_haystack[index] = dict.count(to_bytes_view(haystacks[index]), + overlap_policy, pending_starts_span_()); + }, + dict_); + return status_t::success_k; + } + + /** + * @brief Finds all occurrences of all needles in all the @p haystacks, resolved under @p overlap_policy. + * @param[out] matches_found Matches written, or - when @p matches is too small - the count that would be, + * so `matches.size() == 0` is a size query rather than a wasted call. + * @retval `status_t::unexpected_dimensions_k` @p matches is too small; nothing is written in that case. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> + status_t try_find(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<substrings_match_t> matches, size_t &matches_found, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + + // Counting first is what makes the capacity refusable before any write. + matches_found = 0; + if (counts_per_haystack_.try_resize(haystacks.size()) != status_t::success_k) return status_t::bad_alloc_k; + span<size_t> const counts_per_haystack {counts_per_haystack_.data(), haystacks.size()}; + if (status_t const status = try_count(haystacks, overlap_policy, counts_per_haystack, matches_found, executor, + specs); + status != status_t::success_k) + return status; + + // The count survives the refusal, so a caller that brought no buffer still learns what to allocate. + if (matches_found > matches.size()) return status_t::unexpected_dimensions_k; + + size_t count_written = 0; + std::visit( + [&](auto const &dict) noexcept { + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) + dict.visit(to_bytes_view(haystacks[haystack_index]), overlap_policy, pending_starts_span_(), + [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + matches[count_written] = {haystack_index, needle_index, match_offset, match_length}; + count_written++; + return true; + }); + }, + dict_); + sz_assert_(count_written == matches_found); + return status_t::success_k; + } + + /** + * @brief Rewrites every haystack into one tape, substituting each match with its needle's replacement. + * @param[in] overlap_policy Must name a leftmost policy; an overlapping rewrite is not a function. + * @param[in] replacements One per needle, inserted verbatim. + * @param[out] output_offsets Rewritten boundaries, `haystacks.size() + 1` entries; always filled. + * @param[out] output_bytes_written Bytes written, or - when @p output_bytes is short - the size needed. + * @retval `status_t::unexpected_dimensions_k` @p output_bytes is too small, and nothing was written. + */ + template <typename haystacks_type_, typename replacements_type_, typename executor_type_ = dummy_executor_t> + status_t try_replace(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + replacements_type_ const &replacements, span<char> output_bytes, span<size_t> output_offsets, + size_t &output_bytes_written, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + sz_unused_(executor), sz_unused_(specs); + sz_assert_(output_offsets.size() == haystacks.size() + 1); + output_bytes_written = 0; + if (status_t const rewritable = substrings_check_rewritable(overlap_policy); rewritable != status_t::success_k) + return rewritable; + if (status_t const reserved = try_reserve_pending_starts_(overlap_policy); reserved != status_t::success_k) + return reserved; + + // Sizes land in the offsets array and become boundaries in place, so this needs no scratch of its own. + std::visit( + [&](auto const &dict) noexcept { + for (size_t index = 0; index < haystacks.size(); ++index) + output_offsets[index] = substrings_rewritten_size( + dict, to_bytes_view(haystacks[index]), pending_starts_span_(), overlap_policy, replacements); + }, + dict_); + substrings_sizes_into_offsets(output_offsets, output_bytes_written); + if (output_bytes_written > output_bytes.size()) return status_t::unexpected_dimensions_k; + + std::visit( + [&](auto const &dict) noexcept { + for (size_t index = 0; index < haystacks.size(); ++index) + substrings_rewrite(dict, to_bytes_view(haystacks[index]), pending_starts_span_(), overlap_policy, + replacements, output_bytes.data() + output_offsets[index]); + }, + dict_); + return status_t::success_k; + } + + /** + * @brief Scores every haystack against the compiled needle set in one walk. + * @param[in] document_lengths One per haystack; an empty span uses byte lengths. + * @param[in] needle_weights One IDF or boost per needle. + * @param[out] scores One per haystack. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> + status_t try_score_bm25(haystacks_type_ const &haystacks, span<f32_t const> document_lengths, + substrings_bm25_t parameters, span<f32_t const> needle_weights, span<f32_t> scores, + executor_type_ &&executor = {}, cpu_specs_t const &specs = {}) noexcept { + sz_unused_(executor), sz_unused_(specs); + sz_assert_(scores.size() == haystacks.size()); + sz_assert_(needle_weights.size() == count_needles()); + sz_assert_(document_lengths.size() == 0 || document_lengths.size() == haystacks.size()); + + if (status_t const reserved = try_reserve_frequencies_(); reserved != status_t::success_k) return reserved; + span<u32_t> const frequencies {frequencies_.data(), frequencies_.size()}; + span<u32_t> const touched {touched_needles_.data(), touched_needles_.size()}; + substrings_document_length_t const length_source = document_lengths.size() + ? substrings_document_length_t::given_k + : substrings_document_length_t::haystack_bytes_k; + std::visit( + [&](auto const &dict) noexcept { + for (size_t index = 0; index < haystacks.size(); ++index) + scores[index] = substrings_bm25_score(dict, to_bytes_view(haystacks[index]), length_source, + document_lengths.size() ? document_lengths[index] : 0.0f, + parameters, needle_weights, frequencies, touched); + }, + dict_); + return status_t::success_k; + } + + private: + allocator_t alloc_ {}; + std::variant<narrow_dictionary_t, wide_dictionary_t> dict_; + + /** @brief Grow-only per-call scratch, reused across calls; concurrent calls on one engine are unsafe. */ + safe_vector<size_t, size_allocator_t> counts_per_haystack_ {}; + /** @brief The undecided starts a leftmost walk keeps; stays empty until a leftmost policy asks for it. */ + safe_vector<pending_start_t, pending_start_allocator_t> pending_starts_ {}; + + /** @brief Sizes the pending-start scratch to the built dictionary, a no-op for the overlapping policy. */ + status_t try_reserve_pending_starts_(substrings_overlap_policy_t policy) noexcept { + if (policy == substrings_overlapping_k) return status_t::success_k; + size_t const width = substrings_pending_starts_width(max_source_match_bytes()); + return pending_starts_.size() == width ? status_t::success_k : pending_starts_.try_resize(width); + } + + span<pending_start_t> pending_starts_span_() noexcept { return {pending_starts_.data(), pending_starts_.size()}; } + + /** @brief Sizes the frequency counters to the dictionary, leaving every one of them at zero. */ + status_t try_reserve_frequencies_() noexcept { + size_t const needles = count_needles(); + if (frequencies_.size() == needles) return status_t::success_k; + // `try_resize` leaves trivial types uninitialized, and the counters must start - and stay - zero. + if (frequencies_.try_resize(needles) != status_t::success_k || + touched_needles_.try_resize(needles) != status_t::success_k) + return status_t::bad_alloc_k; + for (size_t index = 0; index < needles; ++index) frequencies_[index] = 0; + return status_t::success_k; + } + + /** @brief How often each needle hit the haystack being scored, and which needles those were. */ + safe_vector<u32_t, u32_allocator_t> frequencies_ {}; + safe_vector<u32_t, u32_allocator_t> touched_needles_ {}; +}; + +#pragma endregion Primary API + +#pragma region Parallel Backend + +/** + * @brief Aho-Corasick-based @b multi-threaded multi-pattern exact/case-folded substring search. + * @note Construction of the FSM is not parallelized, as it is not generally a bottleneck. + * + * Two levels of parallelism: one core per input below the L2 size, all cores on one input above it. + * + * Matches straddling two threads' slices are the hard part. Rather than firing a callback concurrently + * and leaving the user to synchronize, this counts each slice first and then writes the slices in + * parallel into disjoint output ranges, so neither a mutex nor an atomic sits on the write path. + */ +template <typename allocator_type_, typename enable_> +struct substrings<allocator_type_, sz_caps_sp_k, enable_> { + + using allocator_t = allocator_type_; + using narrow_dictionary_t = aho_corasick_dictionary<u16_t, allocator_t>; + using wide_dictionary_t = aho_corasick_dictionary<u32_t, allocator_t>; + using match_t = substrings_match_t; + using pending_start_t = substrings_pending_start; + static constexpr sz_capability_t capability_k = sz_caps_sp_k; + + using size_allocator_t = typename std::allocator_traits<allocator_t>::template rebind_alloc<size_t>; + using pending_start_allocator_t = + typename std::allocator_traits<allocator_t>::template rebind_alloc<pending_start_t>; + using u32_allocator_t = typename std::allocator_traits<allocator_t>::template rebind_alloc<u32_t>; + + explicit substrings(allocator_t alloc = allocator_t()) noexcept + : alloc_(alloc), dict_(std::in_place_type_t<wide_dictionary_t>(), alloc) {} + void reset() noexcept { + std::visit([](auto &dict) noexcept { dict.reset(); }, dict_); + } + + /** @brief The state-id width `try_build` settled on, which the needle set alone decides. */ + substrings_state_width_t state_width() const noexcept { + return std::holds_alternative<narrow_dictionary_t>(dict_) ? substrings_state_width_t::u16_k + : substrings_state_width_t::u32_k; + } + size_t count_needles() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.count_needles(); }, dict_); + } + size_t count_states() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.count_states(); }, dict_); + } + size_t max_source_match_bytes() const noexcept { + return std::visit([](auto const &dict) noexcept { return (size_t)dict.max_source_match_bytes(); }, dict_); + } + size_t min_source_match_bytes() const noexcept { + return std::visit([](auto const &dict) noexcept { return (size_t)dict.min_source_match_bytes(); }, dict_); + } + size_t hot_count() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.hot_count(); }, dict_); + } + substrings_case_sensitivity_t case_sensitivity() const noexcept { + return std::visit([](auto const &dict) noexcept { return dict.case_sensitivity(); }, dict_); + } + + /** + * @brief Runs @p callable against the automaton at whichever state-id width it settled on. + * + * One dispatch per call rather than per byte: the walks it hands the dictionary to stay monomorphic, + * so this costs a branch where a width-templated engine cost an instantiation. + */ + template <typename callable_type_> + auto visit_dictionary(callable_type_ &&callable) const noexcept { + return std::visit(std::forward<callable_type_>(callable), dict_); + } + + /** + * @brief Indexes all of the @p needles strings into the FSM, at whichever state id it ends up fitting. + * + * Construction runs wide, because a dictionary's state count is only known once it is built; the narrowing + * attempt is then itself the ceiling test, and only `overflow_risk_k` means "does not fit". + * @param[in] executor Taken for one shape across every entry point; construction stays on the calling + * thread, as building the FSM is not generally a bottleneck next to walking it. + * @param[in] specs Sizes the hot tier from the host's last-level cache. + * @note Replaces any previously indexed needle set: the automaton is rebuilt from scratch and the old one + * released, so an engine can be re-indexed for a different vocabulary or a different machine. + * @sa `aho_corasick_dictionary::try_insert` for the status codes this forwards. + */ + template <typename needles_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_index(needles_type_ &&needles, substrings_case_sensitivity_t case_sensitivity = substrings_cased_k, + executor_type_ &&executor = {}, cpu_specs_t const &specs = {}) noexcept { + sz_unused_(executor); + wide_dictionary_t wide(alloc_); + wide.case_sensitivity(case_sensitivity); + for (auto const &needle : needles) { + status_t const status = wide.try_insert(to_bytes_view(needle)); + if (status != status_t::success_k) return status; + } + if (status_t const built = wide.try_build(specs); built != status_t::success_k) return built; + + narrow_dictionary_t narrow(alloc_); + status_t const narrowed = narrow.try_build(wide); + if (narrowed == status_t::success_k) { + dict_.template emplace<narrow_dictionary_t>(std::move(narrow)); + return status_t::success_k; + } + if (narrowed != status_t::overflow_risk_k) return narrowed; + dict_.template emplace<wide_dictionary_t>(std::move(wide)); + return status_t::success_k; + } + + /** + * @brief Occurrences of all needles in each of the @p haystacks, for filtering and ranking. + * @param[out] matches_total Sum of @p counts_per_haystack, which is what sizes a later `try_find` buffer. + * @param[in] specs Picks the threading strategy per haystack, by comparing its size against the L2. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_count(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<size_t> counts_per_haystack, size_t &matches_total, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + sz_assert_(counts_per_haystack.size() == haystacks.size()); + if (overlap_policy != substrings_overlapping_k) + return count_all_leftmost_(haystacks, overlap_policy, counts_per_haystack, matches_total, executor); + return count_all_overlapping_(haystacks, counts_per_haystack, matches_total, executor, specs); + } + + /** + * @brief Finds all occurrences of all needles in all the @p haystacks, resolved under @p overlap_policy. + * @param[out] matches_found Matches written, or - when @p matches is too small - the count that would be. + * @retval `status_t::unexpected_dimensions_k` @p matches is too small; nothing is written in that case. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_find(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + span<substrings_match_t> matches, size_t &matches_found, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + if (overlap_policy != substrings_overlapping_k) + return find_all_leftmost_(haystacks, overlap_policy, matches, matches_found, executor); + return find_all_overlapping_(haystacks, matches, matches_found, executor, specs); + } + + /** + * @brief Rewrites every haystack into one tape, substituting each match with its needle's replacement. + * @param[in] overlap_policy Must name a leftmost policy; an overlapping rewrite is not a function. + * @param[in] replacements One per needle, inserted verbatim. + * @param[out] output_offsets Rewritten boundaries, `haystacks.size() + 1` entries; always filled. + * @param[out] output_bytes_written Bytes written, or - when @p output_bytes is short - the size needed. + * @retval `status_t::unexpected_dimensions_k` @p output_bytes is too small, and nothing was written. + */ + template <typename haystacks_type_, typename replacements_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_replace(haystacks_type_ const &haystacks, substrings_overlap_policy_t overlap_policy, + replacements_type_ const &replacements, span<char> output_bytes, span<size_t> output_offsets, + size_t &output_bytes_written, executor_type_ &&executor = {}, + cpu_specs_t const &specs = {}) noexcept { + sz_assert_(output_offsets.size() == haystacks.size() + 1); + output_bytes_written = 0; + if (status_t const rewritable = substrings_check_rewritable(overlap_policy); rewritable != status_t::success_k) + return rewritable; + size_t const cores = executor.threads_count(); + // One share row per large haystack, so the sizing pass's cover survives into the writing pass. + size_t large_total = 0; + for (size_t index = 0; index < haystacks.size(); ++index) + if (is_large_(to_bytes_view(haystacks[index]).size(), specs)) ++large_total; + if (status_t const reserved = try_reserve_pending_starts_(cores); reserved != status_t::success_k) + return reserved; + if (status_t const reserved = try_reserve_rewrite_shares_(cores, large_total); reserved != status_t::success_k) + return reserved; + if (status_t const reserved = try_reserve_spanned_(cores); reserved != status_t::success_k) return reserved; + + using prong_t = typename std::decay<executor_type_>::type::prong_t; + + // Sizes land in the offsets array and become boundaries in place, so this needs no scratch of its own. + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(haystacks.size(), [&](prong_t prong) noexcept { + span<byte_t const> const haystack = to_bytes_view(haystacks[prong.task]); + if (is_large_(haystack.size(), specs)) return; + output_offsets[prong.task] = substrings_rewritten_size(dict, haystack, pending_starts_of_(prong.thread), + overlap_policy, replacements); + }); + }); + size_t large_index = 0; + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + span<byte_t const> const haystack = to_bytes_view(haystacks[haystack_index]); + if (!is_large_(haystack.size(), specs)) continue; // ? Already sized above. + output_offsets[haystack_index] = size_one_large_(haystack, overlap_policy, replacements, executor, + shares_of_large_(large_index, cores)); + ++large_index; + } + substrings_sizes_into_offsets(output_offsets, output_bytes_written); + if (output_bytes_written > output_bytes.size()) return status_t::unexpected_dimensions_k; + + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(haystacks.size(), [&](prong_t prong) noexcept { + span<byte_t const> const haystack = to_bytes_view(haystacks[prong.task]); + if (is_large_(haystack.size(), specs)) return; + substrings_rewrite(dict, haystack, pending_starts_of_(prong.thread), overlap_policy, replacements, + output_bytes.data() + output_offsets[prong.task]); + }); + }); + large_index = 0; + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + span<byte_t const> const haystack = to_bytes_view(haystacks[haystack_index]); + if (!is_large_(haystack.size(), specs)) continue; + span<rewrite_share_t> const shares = shares_of_large_(large_index, cores); + executor.for_threads([&](size_t core_index) noexcept { + rewrite_share_t const &share = shares[core_index]; + if (share.output_bytes == 0) return; + rewrite_share_(haystack, share, overlap_policy, replacements, pending_starts_of_(core_index), + spanned_of_(core_index), + output_bytes.data() + output_offsets[haystack_index] + share.output_offset); + }); + ++large_index; + } + return status_t::success_k; + } + + /** + * @brief Scores every haystack against the compiled needle set in one walk. + * @param[in] document_lengths One per haystack; an empty span uses byte lengths. + * @param[in] needle_weights One IDF or boost per needle. + * @param[out] scores One per haystack. + */ + template <typename haystacks_type_, typename executor_type_ = dummy_executor_t> +#if SZ_HAS_CONCEPTS_ + requires executor_like<executor_type_> +#endif + status_t try_score_bm25(haystacks_type_ const &haystacks, span<f32_t const> document_lengths, + substrings_bm25_t parameters, span<f32_t const> needle_weights, span<f32_t> scores, + executor_type_ &&executor = {}, cpu_specs_t const &specs = {}) noexcept { + sz_assert_(scores.size() == haystacks.size()); + sz_assert_(needle_weights.size() == count_needles()); + sz_assert_(document_lengths.size() == 0 || document_lengths.size() == haystacks.size()); + + if (status_t const reserved = try_reserve_frequencies_(executor.threads_count()); + reserved != status_t::success_k) + return reserved; + // Carries how many needles each core touched when a long haystack is split across all of them. + if (counts_per_core_.try_resize(executor.threads_count()) != status_t::success_k) return status_t::bad_alloc_k; + + substrings_document_length_t const length_source = document_lengths.size() + ? substrings_document_length_t::given_k + : substrings_document_length_t::haystack_bytes_k; + using prong_t = typename std::decay<executor_type_>::type::prong_t; + + // A haystack per core, for everything that fits a core's cache. + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(haystacks.size(), [&](prong_t prong) noexcept { + if (is_large_(to_bytes_view(haystacks[prong.task]).size(), specs)) return; + scores[prong.task] = substrings_bm25_score( + dict, to_bytes_view(haystacks[prong.task]), length_source, + document_lengths.size() ? document_lengths[prong.task] : 0.0f, parameters, needle_weights, + frequencies_of_(prong.thread), touched_needles_of_(prong.thread)); + }); + }); + + // Every core on each long haystack: frequencies are order-independent, so the rows merge by addition. + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + span<byte_t const> const haystack = to_bytes_view(haystacks[haystack_index]); + if (!is_large_(haystack.size(), specs)) continue; // ? Already scored above. + scores[haystack_index] = score_one_large_(haystack, length_source, + document_lengths.size() ? document_lengths[haystack_index] : 0.0f, + parameters, needle_weights, executor); + } + return status_t::success_k; + } + + private: + /** + * @brief Counts every overlapping match, splitting one haystack across cores once it outgrows the L2. + * @param[in] specs Picks the threading strategy per haystack, by comparing its size against the L2. + */ + template <typename haystacks_type_, typename executor_type_> + status_t count_all_overlapping_(haystacks_type_ const &haystacks, span<size_t> counts_per_haystack, + size_t &matches_total, executor_type_ &executor, + cpu_specs_t const &specs) noexcept { + + matches_total = 0; + + using haystack_t = typename haystacks_type_::value_type; + static_assert(std::is_trivially_copyable<haystack_t>::value, + "The haystack should be trivially copyable for higher compatibility."); + + // On small strings, individually compute the counts. + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(counts_per_haystack.size(), [&](size_t haystack_index) noexcept { + haystack_t const &haystack = haystacks[haystack_index]; + if (is_large_(haystack.size(), specs)) return; + counts_per_haystack[haystack_index] = dict.count(to_bytes_view(haystack)); + }); + }); + + // On longer strings, throw all cores on each haystack. + if (counts_per_core_.try_resize(executor.threads_count()) != status_t::success_k) return status_t::bad_alloc_k; + for (size_t haystack_index = 0; haystack_index < counts_per_haystack.size(); ++haystack_index) { + haystack_t const &haystack = haystacks[haystack_index]; + if (!is_large_(haystack.size(), specs)) continue; // ? Already processed above. + auto const haystack_bytes = to_bytes_view(haystack); + + count_matches_per_core_(haystack_bytes, executor, counts_per_core_); + size_t total = 0; + for (size_t core_index = 0; core_index < counts_per_core_.size(); ++core_index) + total += counts_per_core_[core_index]; + counts_per_haystack[haystack_index] = total; + } + + for (size_t haystack_index = 0; haystack_index < counts_per_haystack.size(); ++haystack_index) + matches_total += counts_per_haystack[haystack_index]; + return status_t::success_k; + } + + /** + * @brief Locates every overlapping match, splitting one haystack across cores once it outgrows the L2. + * @param[out] matches_found Matches written, in ascending haystack order. + */ + template <typename haystacks_type_, typename executor_type_> + status_t find_all_overlapping_(haystacks_type_ const &haystacks, span<substrings_match_t> matches, + size_t &matches_found, executor_type_ &executor, cpu_specs_t const &specs) noexcept { + + using haystack_t = typename haystacks_type_::value_type; + + matches_found = 0; + if (haystacks.size() == 0) return status_t::success_k; + size_t const cores_total = executor.threads_count(); + + // Counting here rather than through `try_count` is what keeps every large haystack to two walks + // instead of three: `try_count` may attribute a straddling match to the core it ends on, while the + // scatter reserves each core's slot by where a match starts, so its split cannot be reused. + size_t large_total = 0; + for (size_t index = 0; index < haystacks.size(); ++index) + if (is_large_(haystacks[index].size(), specs)) ++large_total; + + if (counts_per_haystack_.try_resize(haystacks.size()) != status_t::success_k || + offsets_per_haystack_.try_resize(haystacks.size()) != status_t::success_k || + counts_per_core_per_large_.try_resize(large_total * cores_total) != status_t::success_k) + return status_t::bad_alloc_k; + + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(haystacks.size(), [&](size_t haystack_index) noexcept { + haystack_t const &haystack = haystacks[haystack_index]; + if (is_large_(haystack.size(), specs)) return; + counts_per_haystack_[haystack_index] = dict.count(to_bytes_view(haystack)); + }); + }); + + size_t large_index = 0; + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + haystack_t const &haystack = haystacks[haystack_index]; + if (!is_large_(haystack.size(), specs)) continue; + auto const haystack_bytes = to_bytes_view(haystack); + + span<size_t> const counts_per_core = counts_of_large_(large_index, cores_total); + count_matches_per_core_by_start_(haystack_bytes, executor, counts_per_core); + size_t total = 0; + for (size_t core_index = 0; core_index < cores_total; ++core_index) + counts_per_core[core_index] = (total += counts_per_core[core_index]); + counts_per_haystack_[haystack_index] = total; + ++large_index; + } + + status_t const prologue = prefix_and_check_(counts_per_haystack_, offsets_per_haystack_, matches.size(), + matches_found); + if (prologue != status_t::success_k) return prologue; + + scatter_matches_of_small_(haystacks, counts_per_haystack_, offsets_per_haystack_, matches, executor, specs); + + large_index = 0; + for (size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + haystack_t const &haystack = haystacks[haystack_index]; + if (!is_large_(haystack.size(), specs)) continue; + auto const haystack_bytes = to_bytes_view(haystack); + + span<size_t const> const counts_per_core = counts_of_large_(large_index, cores_total); + scatter_matches_of_one_large_(haystack_bytes, haystack_index, offsets_per_haystack_[haystack_index], + counts_per_core, matches, executor); + ++large_index; + } + + return status_t::success_k; + } + + /** @brief Counts a leftmost walk's matches; the recurrence keeps one core per haystack, whatever its size. */ + template <typename haystacks_type_, typename executor_type_> + status_t count_all_leftmost_(haystacks_type_ const &haystacks, substrings_overlap_policy_t policy, + span<size_t> counts_per_haystack, size_t &matches_total, + executor_type_ &executor) noexcept { + + matches_total = 0; + if (status_t const reserved = try_reserve_pending_starts_(executor.threads_count()); + reserved != status_t::success_k) + return reserved; + + using prong_t = typename std::decay<executor_type_>::type::prong_t; + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(counts_per_haystack.size(), [&](prong_t prong) noexcept { + size_t total = 0; + dict.find_leftmost(to_bytes_view(haystacks[prong.task]), pending_starts_of_(prong.thread), policy, + [&](size_t, size_t, size_t) noexcept { + ++total; + return true; + }); + counts_per_haystack[prong.task] = total; + }); + }); + + for (size_t haystack_index = 0; haystack_index < counts_per_haystack.size(); ++haystack_index) + matches_total += counts_per_haystack[haystack_index]; + return status_t::success_k; + } + + /** @brief Locates a leftmost walk's matches, counting into offsets so the scatter needs no atomics. */ + template <typename haystacks_type_, typename executor_type_> + status_t find_all_leftmost_(haystacks_type_ const &haystacks, substrings_overlap_policy_t policy, + span<substrings_match_t> matches, size_t &matches_found, + executor_type_ &executor) noexcept { + + matches_found = 0; + if (haystacks.size() == 0) return status_t::success_k; + if (counts_per_haystack_.try_resize(haystacks.size()) != status_t::success_k || + offsets_per_haystack_.try_resize(haystacks.size()) != status_t::success_k) + return status_t::bad_alloc_k; + + span<size_t> const counts_per_haystack {counts_per_haystack_.data(), haystacks.size()}; + size_t counted_total = 0; + if (status_t const counted = count_all_leftmost_(haystacks, policy, counts_per_haystack, counted_total, + executor); + counted != status_t::success_k) + return counted; + + status_t const prologue = prefix_and_check_(counts_per_haystack_, offsets_per_haystack_, matches.size(), + matches_found); + if (prologue != status_t::success_k) return prologue; + + using prong_t = typename std::decay<executor_type_>::type::prong_t; + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(haystacks.size(), [&](prong_t prong) noexcept { + size_t written = 0; + size_t const base_offset = offsets_per_haystack_[prong.task]; + dict.find_leftmost(to_bytes_view(haystacks[prong.task]), pending_starts_of_(prong.thread), policy, + [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + matches[base_offset + written] = {prong.task, needle_index, match_offset, + match_length}; + ++written; + return true; + }); + sz_assert_(written == counts_per_haystack_[prong.task]); + }); + }); + + return status_t::success_k; + } + + /** @brief Sizes one pending-start row per core, laid end to end so a core's slice needs no allocation. */ + status_t try_reserve_pending_starts_(size_t cores_total) noexcept { + size_t const width = substrings_pending_starts_width(max_source_match_bytes()); + if (pending_starts_width_ == width && pending_starts_.size() == width * cores_total) return status_t::success_k; + if (pending_starts_.try_resize(width * cores_total) != status_t::success_k) return status_t::bad_alloc_k; + pending_starts_width_ = width; + return status_t::success_k; + } + + span<pending_start_t> pending_starts_of_(size_t core_index) noexcept { + return {pending_starts_.data() + core_index * pending_starts_width_, pending_starts_width_}; + } + + /** @brief One core's share of a long haystack's cover: the source it owns, and what it rewrites into. */ + struct rewrite_share_t { + /** @brief Ownership bounds, by match @b start: this core substitutes the matches starting in here. + * Both passes test against these, so they can never disagree about who owns a match. */ + size_t slice_begin {}; + size_t slice_end {}; + /** @brief First source byte this core writes; the end of whatever match crossed into its slice. */ + size_t source_begin {}; + /** @brief One past the last source byte, which a match crossing out of the slice can push forward. */ + size_t source_end {}; + size_t output_bytes {}; + /** @brief Where this core writes, relative to its haystack's own base in the output tape. */ + size_t output_offset {}; + }; + + /** @brief Sizes one share record per core for each large haystack, so both passes read the same cover. */ + status_t try_reserve_rewrite_shares_(size_t cores_total, size_t large_total) noexcept { + size_t const wanted = cores_total * large_total; + if (rewrite_shares_.size() == wanted) return status_t::success_k; + return rewrite_shares_.try_resize(wanted); + } + + /** + * @brief Sizes one coverage row per core, the difference array the restart search marks matches into. + * + * As wide as the restart search's window - four times the longest match - since that is the span a + * difference array has to mark before a position can be judged unspanned. + */ + status_t try_reserve_spanned_(size_t cores_total) noexcept { + size_t const width = max_source_match_bytes() * 4 + 1; + if (spanned_width_ == width && spanned_.size() == width * cores_total) return status_t::success_k; + if (spanned_.try_resize(width * cores_total) != status_t::success_k) return status_t::bad_alloc_k; + spanned_width_ = width; + return status_t::success_k; + } + + /** @brief One large haystack's row of per-core shares, laid end to end like every other per-core scratch. */ + span<rewrite_share_t> shares_of_large_(size_t large_index, size_t cores_total) noexcept { + return {rewrite_shares_.data() + large_index * cores_total, cores_total}; + } + + /** @brief One large haystack's row of per-core match counts, which the scatter reads back as offsets. */ + span<size_t> counts_of_large_(size_t large_index, size_t cores_total) noexcept { + return {counts_per_core_per_large_.data() + large_index * cores_total, cores_total}; + } + + span<i32_t> spanned_of_(size_t core_index) noexcept { + return {spanned_.data() + core_index * spanned_width_, spanned_width_}; + } + + /** @brief One core's counter row, indexed by @b needle index; row zero doubles as the merge target. */ + span<u32_t> frequencies_of_(size_t core_index) noexcept { + return {frequencies_.data() + core_index * frequencies_width_, frequencies_width_}; + } + + /** @brief One core's list of the needles it touched, indexed by @b slot - not by needle index. */ + span<u32_t> touched_needles_of_(size_t core_index) noexcept { + return {touched_needles_.data() + core_index * frequencies_width_, frequencies_width_}; + } + + /** @brief Sizes one frequency row per core, leaving every counter at zero. */ + status_t try_reserve_frequencies_(size_t cores_total) noexcept { + frequencies_width_ = count_needles(); + size_t const wanted = cores_total * frequencies_width_; + if (frequencies_.size() == wanted) return status_t::success_k; + // `try_resize` leaves trivial types uninitialized, and the counters must start - and stay - zero. + if (frequencies_.try_resize(wanted) != status_t::success_k || + touched_needles_.try_resize(wanted) != status_t::success_k) + return status_t::bad_alloc_k; + for (size_t index = 0; index < wanted; ++index) frequencies_[index] = 0; + return status_t::success_k; + } + + allocator_t alloc_ {}; + /** @brief The compiled automaton, at whichever state-id width `try_index` settled on. */ + std::variant<narrow_dictionary_t, wide_dictionary_t> dict_; + + /** @brief Grow-only per-call scratch, reused across calls; concurrent calls on one engine are unsafe. */ + safe_vector<size_t, size_allocator_t> counts_per_core_ {}; + safe_vector<size_t, size_allocator_t> counts_per_haystack_ {}; + /** @brief One pending-start row per core, laid end to end so a core's slice needs no separate allocation. */ + safe_vector<pending_start_t, pending_start_allocator_t> pending_starts_ {}; + size_t pending_starts_width_ {}; + /** @brief One frequency row and one touched-needle row per core, laid out the same way. */ + safe_vector<u32_t, u32_allocator_t> frequencies_ {}; + safe_vector<u32_t, u32_allocator_t> touched_needles_ {}; + size_t frequencies_width_ {}; + /** @brief One share per core per large haystack, settled while sizing and read back while writing, so the + * cover is resolved once and both passes agree on who owns a match straddling a slice boundary. */ + safe_vector<rewrite_share_t, typename std::allocator_traits<allocator_t>::template rebind_alloc<rewrite_share_t>> + rewrite_shares_ {}; + /** @brief One coverage row per core, the difference array the restart search marks matches into. */ + safe_vector<i32_t, typename std::allocator_traits<allocator_t>::template rebind_alloc<i32_t>> spanned_ {}; + size_t spanned_width_ {}; + safe_vector<size_t, size_allocator_t> offsets_per_haystack_ {}; + safe_vector<size_t, size_allocator_t> counts_per_core_per_large_ {}; + + /** @brief Whether a haystack is big enough to deserve every core, rather than one core of its own. */ + static bool is_large_(size_t haystack_bytes, cpu_specs_t const &specs) noexcept { + return haystack_bytes > specs.l2_bytes; + } + + /** + * @brief Turns per-haystack @p counts into the exclusive prefix @p offsets, refusing a short output. + * + * Nothing downstream bounds its writes against the output, so the whole call is refused up front + * rather than half-written. + */ + static status_t prefix_and_check_(span<size_t const> counts, span<size_t> offsets, size_t matches_capacity, + size_t &matches_total) noexcept { + offsets[0] = 0; + for (size_t index = 1; index < counts.size(); ++index) offsets[index] = offsets[index - 1] + counts[index - 1]; + matches_total = offsets[counts.size() - 1] + counts[counts.size() - 1]; + if (matches_total <= matches_capacity) return status_t::success_k; + // The total survives the refusal, so a caller that brought no buffer still learns what to allocate. + return status_t::unexpected_dimensions_k; + } + + /** + * @brief Fills @p counts_per_core for one haystack, attributing a straddling match to the core it + * @b starts on - the rule the scatter reserves slots by, unlike `count_matches_per_core_`. + */ + template <typename executor_type_> + void count_matches_per_core_by_start_(span<byte_t const> haystack, executor_type_ &executor, + span<size_t> counts_per_core) const noexcept { + fu::indexed_split_t const optimal_split {haystack.size(), counts_per_core.size()}; + executor.for_threads([&](size_t core_index) noexcept { + counts_per_core[core_index] = count_matches_in_one_part( + haystack, snapped_subrange_(haystack, optimal_split, core_index)); + }); + } + + /** + * @brief The last position at or before @p limit that no match spans, so a cover can restart there. + * + * A greedy cursor never runs past the largest end among the matches before it, so at a position no match + * spans the cursor is exactly that position, whatever came earlier. That is what lets one core resolve + * its own slice of a leftmost cover without knowing what the core before it decided. + * + * Restarts are dense in real text, so a window of a few times the longest match almost always holds one; + * a slice that finds none falls back to the haystack's start, where the cursor is zero by definition. + * + * @param[in] spanned Scratch as wide as the searched window, holding a difference array of coverage. + */ + size_t restart_before_(span<byte_t const> haystack, size_t limit, span<i32_t> spanned) const noexcept { + size_t const longest = max_source_match_bytes(); + size_t const search_begin = limit >= longest * 3 ? limit - longest * 3 : 0; + if (search_begin == 0 || longest == 0) return 0; + + // A match spanning a judged position starts within `longest` of it, so judging only past that much of + // the window guarantees every such match was seen; anything starting earlier is invisible here. + size_t const judge_begin = search_begin + longest; + size_t const walk_end = sz_min_of_two(limit + longest, haystack.size()); + size_t const window = walk_end - search_begin; + if (window + 1 > spanned.size()) return 0; + for (size_t index = 0; index <= window; ++index) spanned[index] = 0; + + // A position is spanned when it lies strictly inside a match, so each match marks `(start, end)` and + // the running sum below reads zero exactly where nothing reaches across. + visit_dictionary([&](auto const &dict) noexcept { + dict.find({haystack.data() + search_begin, window}, [&](size_t needle_index, size_t match_offset, + size_t match_length) noexcept { + sz_unused_(needle_index); + size_t const start = search_begin + match_offset + 1, end = search_begin + match_offset + match_length; + size_t const from = sz_max_of_two(start, search_begin); + size_t const to = sz_min_of_two(end, walk_end); + if (from < to) ++spanned[from - search_begin], --spanned[to - search_begin]; + return true; + }); + }); + + size_t restart = 0; + i32_t reaching = 0; + for (size_t position = search_begin; position <= limit && position < walk_end; ++position) { + reaching += spanned[position - search_begin]; + if (position >= judge_begin && reaching == 0) restart = position; + } + return restart; + } + + /** + * @brief What one core owns of a long haystack's rewrite, resolved without talking to its neighbours. + * + * Ownership is by match @b start, so the core holding a match writes all of it even when it ends past + * the slice, and the next core starts after it. Both cores derive that boundary from the same cover, so + * they agree without a handshake. + */ + template <typename replacements_type_> + rewrite_share_t rewrite_share_of_core_(span<byte_t const> haystack, size_t slice_begin, size_t slice_end, + substrings_overlap_policy_t policy, replacements_type_ const &replacements, + span<pending_start_t> pending_starts, span<i32_t> spanned) noexcept { + + size_t const restart = restart_before_(haystack, slice_begin, spanned); + rewrite_share_t share; + share.slice_begin = slice_begin; + share.slice_end = slice_end; + share.source_begin = slice_begin; + share.source_end = slice_end; + size_t removed = 0, added = 0; + + visit_dictionary([&](auto const &dict) noexcept { + dict.find_leftmost({haystack.data() + restart, haystack.size() - restart}, pending_starts, policy, + [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + size_t const start = restart + match_offset, end = start + match_length; + // A match starting before this slice belongs to an earlier core, and pushes + // this one's first byte past whatever it consumed. + if (start < slice_begin) { + share.source_begin = sz_max_of_two(share.source_begin, end); + return true; + } + if (start >= slice_end) return false; // ? The next core's, and every one after. + removed += match_length; + added += to_bytes_view(replacements[needle_index]).size(); + share.source_end = sz_max_of_two(share.source_end, end); + return true; + }); + }); + + share.source_begin = sz_min_of_two(share.source_begin, share.source_end); + share.output_bytes = (share.source_end - share.source_begin) - removed + added; + return share; + } + + /** + * @brief Settles every core's share of one long haystack and returns the bytes they rewrite to. + * + * The shares survive into the writing pass, so the cover is resolved once rather than twice - which also + * keeps the two passes from disagreeing about who owns a match that straddles a slice boundary. + */ + template <typename replacements_type_, typename executor_type_> + size_t size_one_large_(span<byte_t const> haystack, substrings_overlap_policy_t policy, + replacements_type_ const &replacements, executor_type_ &executor, + span<rewrite_share_t> shares) noexcept { + + size_t const cores = executor.threads_count(); + fu::indexed_split_t const optimal_split {haystack.size(), cores}; + executor.for_threads([&](size_t core_index) noexcept { + fu::indexed_range_t const slice = snapped_subrange_(haystack, optimal_split, core_index); + shares[core_index] = rewrite_share_of_core_(haystack, slice.first, slice.first + slice.count, policy, + replacements, pending_starts_of_(core_index), + spanned_of_(core_index)); + }); + + size_t total = 0; + for (size_t core_index = 0; core_index < cores; ++core_index) { + shares[core_index].output_offset = total; + total += shares[core_index].output_bytes; + } + return total; + } + + /** @brief Writes one core's share, spliced exactly as `substrings_rewrite` does for a whole haystack. */ + template <typename replacements_type_> + void rewrite_share_(span<byte_t const> haystack, rewrite_share_t const &share, substrings_overlap_policy_t policy, + replacements_type_ const &replacements, span<pending_start_t> pending_starts, + span<i32_t> spanned, char *output) noexcept { + + size_t const restart = restart_before_(haystack, share.slice_begin, spanned); + size_t copied_through = share.source_begin; + visit_dictionary([&](auto const &dict) noexcept { + dict.find_leftmost({haystack.data() + restart, haystack.size() - restart}, pending_starts, policy, + [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + size_t const start = restart + match_offset; + // The same bounds the sizing pass used, so the two passes substitute exactly + // the same matches - anything else would write bytes nobody accounted for. + if (start < share.slice_begin) return true; + if (start >= share.slice_end) return false; + sz_copy((sz_ptr_t)output, (sz_cptr_t)(haystack.data() + copied_through), + start - copied_through); + output += start - copied_through; + span<byte_t const> const replacement = to_bytes_view(replacements[needle_index]); + sz_copy((sz_ptr_t)output, (sz_cptr_t)replacement.data(), replacement.size()); + output += replacement.size(); + copied_through = start + match_length; + return true; + }); + }); + sz_copy((sz_ptr_t)output, (sz_cptr_t)(haystack.data() + copied_through), share.source_end - copied_through); + } + + /** + * @brief Scores one haystack too long for a single core, counting its slices in parallel. + * + * Each core walks its own slice preceded by a warm-up of the longest match, so a match straddling a cut + * is still spelled, and claims it only when it @b ends inside the slice - the same one-owner rule + * `count_matches_per_core_` uses. Frequencies are integers, so the rows merge by plain addition and the + * merged row scores exactly as a single-core count would. + */ + template <typename executor_type_> + f32_t score_one_large_(span<byte_t const> haystack, substrings_document_length_t length_source, + f32_t document_length, substrings_bm25_t parameters, span<f32_t const> needle_weights, + executor_type_ &executor) noexcept { + + size_t const cores = executor.threads_count(); + fu::indexed_split_t const optimal_split {haystack.size(), cores}; + size_t const longest = max_source_match_bytes(); + + executor.for_threads([&](size_t core_index) noexcept { + fu::indexed_range_t const slice = snapped_subrange_(haystack, optimal_split, core_index); + size_t const slice_begin = slice.first, slice_end = slice.first + slice.count; + size_t const walk_begin = slice_begin >= longest ? slice_begin - longest : 0; + span<byte_t const> const walked {haystack.data() + walk_begin, slice_end - walk_begin}; + + span<u32_t> const frequencies = frequencies_of_(core_index); + span<u32_t> const touched = touched_needles_of_(core_index); + size_t touched_count = 0; + visit_dictionary([&](auto const &dict) noexcept { + dict.find(walked, [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + // A match belongs to the core its end falls in, so the warm-up never double-counts. + if (walk_begin + match_offset + match_length <= slice_begin) return true; + if (frequencies[needle_index]++ == 0) touched[touched_count++] = (u32_t)needle_index; + return true; + }); + }); + counts_per_core_[core_index] = touched_count; + }); + + // Merge into the first core's row, in ascending needle order so the sum is the one the header names. + span<u32_t> const merged = frequencies_of_(0); + span<u32_t> const merged_touched = touched_needles_of_(0); + size_t merged_count = counts_per_core_[0]; + for (size_t core_index = 1; core_index < cores; ++core_index) { + span<u32_t> const frequencies = frequencies_of_(core_index); + span<u32_t> const touched = touched_needles_of_(core_index); + for (size_t slot = 0; slot < counts_per_core_[core_index]; ++slot) { + // The touched row is keyed by slot, the frequency row by needle index - two rows, two keys. + u32_t const needle_index = touched[slot]; + if (merged[needle_index] == 0) merged_touched[merged_count++] = needle_index; + merged[needle_index] += frequencies[needle_index]; + frequencies[needle_index] = 0; + } + } + + if (length_source == substrings_document_length_t::haystack_bytes_k) document_length = (f32_t)haystack.size(); + return substrings_bm25_total(needle_weights, parameters, document_length, merged, merged_touched, merged_count); + } + + /** @brief Writes every small haystack's matches, one core per haystack into disjoint output ranges. */ + template <typename haystacks_type_, typename executor_type_> + void scatter_matches_of_small_(haystacks_type_ const &haystacks, span<size_t const> counts, + span<size_t const> offsets, span<substrings_match_t> matches, + executor_type_ &executor, cpu_specs_t const &specs) const noexcept { + + using haystack_t = typename haystacks_type_::value_type; + sz_unused_(counts); + + visit_dictionary([&](auto const &dict) noexcept { + executor.for_n_dynamic(offsets.size(), [&](size_t haystack_index) noexcept { + haystack_t const &haystack = haystacks[haystack_index]; + auto const haystack_bytes = to_bytes_view(haystack); + if (is_large_(haystack_bytes.size(), specs)) return; + + size_t matches_found = 0; + dict.find(haystack_bytes, [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + matches[offsets[haystack_index] + matches_found] = {haystack_index, needle_index, match_offset, + match_length}; + ++matches_found; + return true; + }); + sz_assert_(counts[haystack_index] == matches_found); + }); + }); + } + + /** + * @brief Writes one large haystack's matches, each core into the slot its own prefix sum reserves. + * @param[in] counts_per_core Inclusive prefix sums of each core's match count, attributed by start. + */ + template <typename executor_type_> + void scatter_matches_of_one_large_(span<byte_t const> haystack, size_t haystack_index, size_t base_offset, + span<size_t const> counts_per_core, span<substrings_match_t> matches, + executor_type_ &executor) const noexcept { + + fu::indexed_split_t const optimal_split {haystack.size(), counts_per_core.size()}; + executor.for_threads([&](size_t core_index) noexcept { + size_t const count_matches_before_this_core = core_index ? counts_per_core[core_index - 1] : 0; + size_t const count_matches_expected_on_this_core = counts_per_core[core_index] - + count_matches_before_this_core; + + fu::indexed_range_t const optimal_subrange = snapped_subrange_(haystack, optimal_split, core_index); + byte_t const *optimal_begin = haystack.begin() + optimal_subrange.first; + byte_t const *const optimal_end = optimal_begin + optimal_subrange.count; + // An empty dictionary reports zero, where `optimal_end + 0 - 1` would step past the end. + size_t const longest = max_source_match_bytes(); + size_t const overlap_bytes = longest > 0 ? longest - 1 : 0; + byte_t const *const overlapping_end = sz_min_of_two(optimal_end + overlap_bytes, haystack.end()); + + // Offsets arrive relative to the slice the dictionary was handed, not to the whole haystack. + size_t const slice_offset_in_haystack = (size_t)(optimal_begin - haystack.begin()); + size_t const owned_bytes = optimal_subrange.count; + size_t count_matches_found_on_this_core = 0; + visit_dictionary([&](auto const &dict) noexcept { + dict.find({optimal_begin, overlapping_end}, [&](size_t needle_index, size_t match_offset, + size_t match_length) noexcept { + bool const belongs_to_this_core = match_offset < owned_bytes; + if (!belongs_to_this_core) return true; + matches[base_offset + count_matches_before_this_core + count_matches_found_on_this_core] = { + haystack_index, needle_index, slice_offset_in_haystack + match_offset, match_length}; + count_matches_found_on_this_core++; + return true; + }); + }); + sz_assert_(count_matches_found_on_this_core == count_matches_expected_on_this_core); + }); + } + + /** + * @brief One core's slice, with both ends pulled back to codepoint starts whenever the walk folds. + * + * Attribution alone already makes an unsnapped cut safe: a match starts on a lead byte, so no match can + * start inside the codepoint a cut lands in, and that codepoint's own matches start before the cut and + * belong to the previous core, which reaches them through its overlap. Snapping keeps that argument from + * having to be made - the walk begins on a real codepoint - and saves the resynchronization it would + * otherwise spend treating a continuation byte as malformed. Both ends go through the same snap and one + * core's end is the next one's start, so the slices stay an exact partition however the boundaries move. + */ + fu::indexed_range_t snapped_subrange_(span<byte_t const> haystack, fu::indexed_split_t const &split, + size_t core_index) const noexcept { + fu::indexed_range_t subrange = split[core_index]; + if (case_sensitivity() != substrings_uncased_k) return subrange; + size_t const begin = sz_utf8_rune_start_at_((cptr_t)haystack.data(), haystack.size(), subrange.first); + size_t const end = sz_utf8_rune_start_at_((cptr_t)haystack.data(), haystack.size(), + subrange.first + subrange.count); + subrange.first = begin; + subrange.count = end - begin; // ? Zero when a whole slice fell inside one codepoint + return subrange; + } + + /** + * @brief Fills @p counts_per_core for one haystack, picking the cheaper of the two counting strategies. + * + * The two strategies attribute a straddling match to different cores - `count_matches_in_one_part` to + * the core the match starts on, `count_short_matches_in_one_part` to the core it ends on - so only a + * caller that sums across cores may choose freely. + */ + template <typename executor_type_> + void count_matches_per_core_(span<byte_t const> haystack, executor_type_ &executor, + span<size_t> counts_per_core) const noexcept { + fu::indexed_split_t const optimal_split {haystack.size(), counts_per_core.size()}; + // The short-match strategy steps raw haystack bytes through the automaton, which only spells a match + // when the dictionary is byte-exact; a folded walk has to go through `find`. + bool const longest_match_fits_on_one_core = optimal_split.smallest_size() >= max_source_match_bytes() && + case_sensitivity() == substrings_cased_k; + + if (!longest_match_fits_on_one_core) + executor.for_threads([&](size_t core_index) noexcept { + counts_per_core[core_index] = count_matches_in_one_part( + haystack, snapped_subrange_(haystack, optimal_split, core_index)); + }); + else + executor.for_threads([&](size_t core_index) noexcept { + size_t matches_in_prefix = 0; + size_t const matches_in_part = count_short_matches_in_one_part(haystack, optimal_split[core_index], + matches_in_prefix); + counts_per_core[core_index] = matches_in_part - non_zero_if<size_t>(matches_in_prefix, core_index > 0); + }); + } + + /** + * @brief Helper method implementing the core logic of the parallel `try_count` and part of `try_find`. + * @return Number of matches that @b begin in this core's slice and may end in another core's slice. + */ + size_t count_matches_in_one_part(span<byte_t const> haystack, + fu::indexed_range_t const optimal_subrange) const noexcept { + + size_t const max_source_match_bytes = sz_min_of_two(this->max_source_match_bytes(), haystack.size()); + + byte_t const *optimal_begin = haystack.begin() + optimal_subrange.first; + byte_t const *const optimal_end = optimal_begin + optimal_subrange.count; + + size_t const count_matches_non_overlapping = visit_dictionary( + [&](auto const &dict) noexcept { return dict.count({optimal_begin, optimal_end}); }); + + byte_t const *overlapping_start; + byte_t const *overlapping_end; + if (optimal_begin + max_source_match_bytes >= optimal_end) { + overlapping_start = optimal_begin; + overlapping_end = sz_min_of_two(optimal_end + max_source_match_bytes, haystack.end()); + } + else { + overlapping_start = sz_max_of_two(optimal_end - max_source_match_bytes + 1, optimal_begin); + overlapping_end = sz_min_of_two(optimal_end + max_source_match_bytes - 1, haystack.end()); + } + + // Both branches place `overlapping_start` at or after `optimal_begin`, so offsets relative to it need + // no lower-bound test. + size_t const slice_end_offset = (size_t)(optimal_end - overlapping_start); + size_t count_matches_overlapping = 0; + visit_dictionary([&](auto const &dict) noexcept { + dict.find({overlapping_start, overlapping_end}, + [&](size_t needle_index, size_t match_offset, size_t match_length) noexcept { + sz_unused_(needle_index); + bool const belongs_to_this_core = // + match_offset < slice_end_offset && // ? Starts before this slice ends. + match_offset + match_length > slice_end_offset; // ? Ends beyond this slice. + count_matches_overlapping += belongs_to_this_core; + return true; + }); + }); + + return count_matches_non_overlapping + count_matches_overlapping; + } + + /** + * @brief More optimized alternative to `count_matches_in_one_part`, assuming the longest match fits + * within a single core's slice, so a match can only spill into 2 core regions at most. + * @param[out] matches_in_prefix Matches ending within the first `max_source_match_bytes` of this core's slice, + * which the preceding core has already counted as its own overlapping tail. + * @return Total matches ending anywhere in this core's slice or its overlapping tail. + */ + size_t count_short_matches_in_one_part(span<byte_t const> haystack, fu::indexed_range_t const optimal_subrange, + size_t &matches_in_prefix) const noexcept { + + // One dispatch per core per haystack, outside the walk, so the transition chain below stays monomorphic. + return std::visit( + [&](auto const &dict) noexcept -> size_t { + using walked_state_id_t = typename std::decay<decltype(dict)>::type::state_id_t; + auto const automaton = dict.view(); + sz_assert_(automaton.case_sensitivity == substrings_cased_k && + "A folded walk cannot step raw haystack bytes; the uncased path counts through `find`"); + size_t const max_source_match_bytes = (size_t)dict.max_source_match_bytes(); + byte_t const *optimal_begin = haystack.begin() + optimal_subrange.first; + byte_t const *const optimal_end = optimal_begin + optimal_subrange.count; + byte_t const *const prefix_end = sz_min_of_two(optimal_begin + max_source_match_bytes, haystack.end()); + byte_t const *const overlapping_end = sz_min_of_two(optimal_end + max_source_match_bytes, + haystack.end()); + + size_t matches_in_part = 0; + matches_in_prefix = 0; + walked_state_id_t current_state = automaton.root; + // The prefix window spans at most `max_source_match_bytes` bytes and is the only region needing the + // per-byte attribution test, so it walks scalar; `prefix_end` never exceeds `overlapping_end`, + // both being clamped by the same haystack end. + for (; optimal_begin != prefix_end; ++optimal_begin) { + walked_state_id_t const output_count = aho_corasick_step_counting(automaton, current_state, + *optimal_begin); + matches_in_part += output_count; + matches_in_prefix += output_count; + } + // One 4-byte load feeds four transitions - the state chain stays strictly serial, and `sz_u32_load` + // absorbs misalignment itself, so only a tail loop remains. + for (; optimal_begin + 4 <= overlapping_end; optimal_begin += 4) { + sz_u32_vec_t const quad = sz_u32_load((sz_cptr_t)optimal_begin); + matches_in_part += aho_corasick_step_counting(automaton, current_state, quad.u8s[0]); + matches_in_part += aho_corasick_step_counting(automaton, current_state, quad.u8s[1]); + matches_in_part += aho_corasick_step_counting(automaton, current_state, quad.u8s[2]); + matches_in_part += aho_corasick_step_counting(automaton, current_state, quad.u8s[3]); + } + for (; optimal_begin != overlapping_end; ++optimal_begin) + matches_in_part += aho_corasick_step_counting(automaton, current_state, *optimal_begin); + + return matches_in_part; + }, + dict_); + } +}; + +using substrings_serial_t = substrings<std::allocator<char>, sz_cap_serial_k>; +using substrings_parallel_t = substrings<std::allocator<char>, sz_caps_sp_k>; + +#pragma endregion Parallel Backend + +} // namespace stringzillas +} // namespace ashvardanian + +#endif // STRINGZILLAS_SUBSTRINGS_SERIAL_HPP_ diff --git a/include/stringzillas/types.cuh b/include/stringzillas/types.cuh index 9791f255..7896d145 100644 --- a/include/stringzillas/types.cuh +++ b/include/stringzillas/types.cuh @@ -13,8 +13,7 @@ #ifndef STRINGZILLAS_TYPES_CUH_ #define STRINGZILLAS_TYPES_CUH_ -#include "stringzilla/types.hpp" -#include "stringzillas/types.hpp" // `bytes_per_cell_t`, `one_byte_per_cell_k` +#include <numeric> // `std::midpoint` #include <forkunion/types.hpp> // `limited_array` — inline storage for the per-device caches @@ -24,6 +23,9 @@ #include <cub/block/block_reduce.cuh> // `cub::BlockReduce` — block collective behind our `_across_cuda_device_` reductions #include <cub/block/block_scan.cuh> // `cub::BlockScan` — block collective behind our `exclusive_sum_across_cuda_device_` +#include "stringzilla/types.hpp" +#include "stringzillas/types.hpp" // `bytes_per_cell_t`, `one_byte_per_cell_k` + namespace ashvardanian { namespace stringzillas { @@ -264,7 +266,7 @@ using pinned_alloc_t = pinned_alloc<char>; /** @brief Returns `true` if the pointer refers to device-accessible memory (Device or Managed/Unified). */ inline bool is_device_accessible_memory(void const *ptr) noexcept { - if (!ptr) return true; + if (!ptr) return false; // Without a current context the query fails and every pointer reads back as host, including device memory. [[maybe_unused]] CUcontext const context = ensure_primary_context_(); // Driver query: `CU_POINTER_ATTRIBUTE_MEMORY_TYPE` collapses both device and managed/unified memory onto @@ -277,6 +279,36 @@ inline bool is_device_accessible_memory(void const *ptr) noexcept { return memory_type == CU_MEMORYTYPE_DEVICE; } +/** + * @brief Refuses a region no kernel can reach, so a caller's buffer is never written across the bus. + * + * An empty region is accepted, because nothing is read or written through it - which is what keeps a + * zero-capacity call a size query rather than a refusal. + */ +template <typename value_type_, sz_size_t extent_> +inline status_t check_device_accessible_memory(span<value_type_, extent_> region) noexcept { + if (region.size() == 0) return status_t::success_k; + return is_device_accessible_memory((void const *)region.data()) ? status_t::success_k + : status_t::device_memory_mismatch_k; +} + +/** + * @brief Refuses a sequence whose bytes no kernel can reach, probing its first non-empty element. + * + * One driver round-trip decides for the whole batch: a tape is one allocation, and scattered elements come + * from one allocator. An empty element carries no address to probe, so the walk steps past it. + */ +template <typename sequence_type_> +inline status_t check_device_accessible_sequence(sequence_type_ const &sequence) noexcept { + for (size_t index = 0; index < sequence.size(); ++index) { + span<byte_t const> const element = to_bytes_view(sequence[index]); + if (element.size() == 0) continue; + return is_device_accessible_memory((void const *)element.data()) ? status_t::success_k + : status_t::device_memory_mismatch_k; + } + return status_t::success_k; +} + struct cuda_status_t { status_t status = status_t::success_k; cudaError_t cuda_error = cudaSuccess; @@ -371,7 +403,7 @@ inline cuda_status_t gpu_specs_fetch(gpu_specs_t &specs, int device_id = 0) noex int multiprocessor_count = 0, warp_size = 0, major = 0, minor = 0; int constant_memory_bytes = 0, shared_per_multiprocessor = 0; - int max_blocks_per_multiprocessor = 0, reserved_shared_per_block = 0; + int max_blocks_per_multiprocessor = 0, reserved_shared_per_block = 0, l2_bytes = 0; size_t total_global_memory = 0; cuDeviceGetAttribute(&multiprocessor_count, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device); cuDeviceGetAttribute(&constant_memory_bytes, CU_DEVICE_ATTRIBUTE_TOTAL_CONSTANT_MEMORY, device); @@ -381,12 +413,14 @@ inline cuda_status_t gpu_specs_fetch(gpu_specs_t &specs, int device_id = 0) noex cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device); cuDeviceGetAttribute(&max_blocks_per_multiprocessor, CU_DEVICE_ATTRIBUTE_MAX_BLOCKS_PER_MULTIPROCESSOR, device); cuDeviceGetAttribute(&reserved_shared_per_block, CU_DEVICE_ATTRIBUTE_RESERVED_SHARED_MEMORY_PER_BLOCK, device); + cuDeviceGetAttribute(&l2_bytes, CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE, device); cuDeviceTotalMem(&total_global_memory, device); // Set the GPU specs specs.streaming_multiprocessors = multiprocessor_count; specs.constant_memory_bytes = constant_memory_bytes; specs.vram_bytes = total_global_memory; + specs.l2_bytes = static_cast<size_t>(l2_bytes); specs.warp_size = warp_size; // Infer other global settings, that CUDA doesn't expose directly @@ -551,6 +585,37 @@ inline cuda_status_t occupancy_grid_for(unsigned &blocks_per_grid, CUfunction fu return {status_t::success_k, cudaSuccess}; } +/** + * @brief Largest dynamic shared-memory allocation keeping @p target_blocks resident, for an already-resolved + * `CUfunction` whose opt-in ceiling is raised. `0` when no allocation admits that many. + * + * Dividing an SM's shared memory by the target silently yields one block fewer, since the driver charges each + * block a reserve of its own and rounds to an unreported granularity. `cuOccupancyAvailableDynamicSMemPerBlock` + * repeats that same division, so the occupancy query answers instead. + */ +inline cuda_status_t shared_memory_budget_for_resident_blocks(size_t &budget, CUfunction function, + unsigned threads_per_block, unsigned target_blocks, + int device_id) noexcept { + int optin_ceiling = 0; + CUresult const attribute_error = cuDeviceGetAttribute( + &optin_ceiling, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, (CUdevice)device_id); + if (attribute_error != CUDA_SUCCESS) return make_cuda_status(attribute_error); + + // Residency falls monotonically as the allocation grows, so binary search finds the largest admissible one. + budget = 0; + for (size_t low = 0, high = (size_t)optin_ceiling; low <= high;) { + size_t const middle = std::midpoint(low, high); + int resident_blocks = 0; + CUresult const occupancy_error = cuOccupancyMaxActiveBlocksPerMultiprocessor(&resident_blocks, function, + (int)threads_per_block, middle); + if (occupancy_error != CUDA_SUCCESS) return make_cuda_status(occupancy_error); + if (resident_blocks >= (int)target_blocks) budget = middle, low = middle + 1; + else if (middle == 0) break; // ? `high = middle - 1` would wrap on an unsigned zero + else high = middle - 1; + } + return {status_t::success_k, cudaSuccess}; +} + /** * @brief A resolved launch shape: a kernel's driver handle plus its co-resident block count. * @@ -725,6 +790,9 @@ struct cuda_launch_t { /** @brief Block dimension of every collective primitive; the block collectives need it as a compile-time constant. */ static constexpr unsigned cuda_device_collective_threads_k = 256; +/** @brief Grid ceiling of every collective primitive; past it a wider grid only adds contention on the merge slot. */ +static constexpr size_t cuda_device_collective_max_blocks_k = 1024; + /** @brief @b sz_max_of_two as a reduction operator; `cuda::maximum` needs CUDA 12.9 and `cub::Max` is deprecated. */ struct max_of_two_t { template <typename value_type_> @@ -782,6 +850,60 @@ __global__ void exclusive_sum_across_cuda_device_(value_type_ const *input, size if (threadIdx.x == 0) output[count] = running; } +/** + * @brief Reduces block @p blockIdx.x 's tile of @p input[0, count) into `partials[blockIdx.x]` - the first of the + * three phases @ref cuda_launch_exclusive_sum_ splits a long scan into. + * + * Tile `t` is `[t * elements_per_tile, min(count, (t + 1) * elements_per_tile))`, owned outright rather than + * grid-strided, so this phase and @ref exclusive_sum_apply_tiles_across_cuda_device_ partition identically + * without the host telling either one anything but the tile width. + */ +template <typename value_type_> +__global__ void exclusive_sum_reduce_tiles_across_cuda_device_(value_type_ const *input, size_t count, + size_t elements_per_tile, value_type_ *partials) { + using block_reduce_t = cub::BlockReduce<value_type_, cuda_device_collective_threads_k>; + __shared__ typename block_reduce_t::TempStorage temp_storage; + size_t const begin = (size_t)blockIdx.x * elements_per_tile; + size_t const end = sz_min_of_two(begin + elements_per_tile, count); + value_type_ running = value_type_(0); + for (size_t base = begin; base < end; base += blockDim.x) { + size_t const i = base + threadIdx.x; + value_type_ const value = i < end ? input[i] : value_type_(0); + value_type_ const block_aggregate = block_reduce_t(temp_storage).Sum(value); + if (threadIdx.x == 0) running += block_aggregate; + __syncthreads(); // reuse of `temp_storage` on the next tile must wait for this tile's readers + } + if (threadIdx.x == 0) partials[blockIdx.x] = running; +} + +/** + * @brief Scans block @p blockIdx.x 's tile of @p input[0, count) into @p output, seeded by `partials[blockIdx.x]`, + * and writes the grand total `partials[gridDim.x]` to @p output[count]. + * + * Each thread reads `input[i]` into a register before writing `output[i]`, and tiles are disjoint, so one buffer + * may serve as both - the same in-place contract the single-block @ref exclusive_sum_across_cuda_device_ offers. + */ +template <typename value_type_> +__global__ void exclusive_sum_apply_tiles_across_cuda_device_(value_type_ const *input, size_t count, + size_t elements_per_tile, value_type_ const *partials, + value_type_ *output) { + using block_scan_t = cub::BlockScan<value_type_, cuda_device_collective_threads_k>; + __shared__ typename block_scan_t::TempStorage temp_storage; + size_t const begin = (size_t)blockIdx.x * elements_per_tile; + size_t const end = sz_min_of_two(begin + elements_per_tile, count); + value_type_ running = partials[blockIdx.x]; + for (size_t base = begin; base < end; base += blockDim.x) { + size_t const i = base + threadIdx.x; + value_type_ const value = i < end ? input[i] : value_type_(0); + value_type_ exclusive = value_type_(0), block_aggregate = value_type_(0); + block_scan_t(temp_storage).ExclusiveSum(value, exclusive, block_aggregate); + if (i < end) output[i] = running + exclusive; + running += block_aggregate; + __syncthreads(); // reuse of `temp_storage` on the next tile must wait for this tile's readers + } + if (blockIdx.x == 0 && threadIdx.x == 0) output[count] = partials[gridDim.x]; +} + /** @brief Three `u32` values reduced together — one per output field of a fused maxima pass. */ struct u32x3_t { u32_t a, b, c; @@ -826,8 +948,8 @@ inline cuda_status_t cuda_launch_reduce_maxima3_(kernel_shape_t const &shape, ta extractor_ extract_arg = extract; u32_t *output_arg = output; void *args[4] = {(void *)&tasks_arg, (void *)&count_arg, (void *)&extract_arg, (void *)&output_arg}; - unsigned const blocks = static_cast<unsigned>( - sz_min_of_two((count + cuda_device_collective_threads_k - 1) / cuda_device_collective_threads_k, size_t(1024))); + unsigned const blocks = static_cast<unsigned>(sz_min_of_two( + divide_round_up<size_t>(count, cuda_device_collective_threads_k), cuda_device_collective_max_blocks_k)); CUresult error = cuda_launch_t {} .grid(blocks ? blocks : 1u) .block(cuda_device_collective_threads_k) @@ -875,8 +997,8 @@ inline cuda_status_t cuda_launch_reduce_minmax_(kernel_shape_t const &shape, inp value_type_ *min_arg = out_min; value_type_ *max_arg = out_max; void *args[4] = {(void *)&input_arg, (void *)&count_arg, (void *)&min_arg, (void *)&max_arg}; - unsigned const blocks = static_cast<unsigned>( - sz_min_of_two((count + cuda_device_collective_threads_k - 1) / cuda_device_collective_threads_k, size_t(1024))); + unsigned const blocks = static_cast<unsigned>(sz_min_of_two( + divide_round_up<size_t>(count, cuda_device_collective_threads_k), cuda_device_collective_max_blocks_k)); CUresult error = cuda_launch_t {} .grid(blocks ? blocks : 1u) .block(cuda_device_collective_threads_k) @@ -908,10 +1030,88 @@ inline cuda_status_t cuda_launch_segmented_reduce_max_(kernel_shape_t const &sha return {status_t::success_k, cudaSuccess}; } -/** @brief Launches the single-block @ref exclusive_sum_across_cuda_device_ (writes directly, no pre-init). */ +/** + * @brief The three resolved handles @ref cuda_launch_exclusive_sum_ picks between - one whole-array scan on one + * block, or a reduce-then-scan across many when the array is long enough to make that block the bottleneck. + */ +struct exclusive_sum_shapes_t { + kernel_shape_t whole {}; + kernel_shape_t reduce_tiles {}; // ? Left unresolved by callers whose counts never leave one tile + kernel_shape_t apply_tiles {}; +}; + +/** + * @brief Exclusive prefix sum of @p input[0,count) into @p output[0,count), inclusive total at @p output[count]. + * + * A single block scans the whole array in one launch, which is O(count / 256) dependent block-scan rounds on one + * multiprocessor. Past one tile's worth of work that latency chain is the bottleneck rather than the bandwidth, so + * a long array is split into one tile per block and folded in three launches: reduce each tile, scan the tile + * totals on one block, then re-scan each tile seeded by its total. @p partials holds those totals and needs + * `blocks + 1` elements, at most @ref cuda_device_collective_max_blocks_k + 1; too small a span - an empty one + * included - simply keeps the single-block route, so the primitive can never fail for want of scratch. + * + * @p input and @p output may be the same buffer on either route; every element is read into a register by the same + * thread that writes it back. The value type must be integral for the two routes to agree bit for bit, since they + * group the additions differently. + */ template <typename value_type_> -inline cuda_status_t cuda_launch_exclusive_sum_(kernel_shape_t const &shape, value_type_ const *input, size_t count, - value_type_ *output, CUstream stream) noexcept { +inline cuda_status_t cuda_launch_exclusive_sum_(exclusive_sum_shapes_t const &shapes, value_type_ const *input, + size_t count, value_type_ *output, span<value_type_> partials, + gpu_specs_t const &specs, CUstream stream) noexcept { + + static_assert(std::is_integral<value_type_>::value, "Tiling regroups the additions, which only integers survive"); + + // Every block wants at least one full tile of its own, so the grid follows the work rather than the device + // once the array is short - and the tile stays a multiple of the block so no thread sits out a whole round. + size_t const target_blocks = sz_min_of_two( + sz_max_of_two((size_t)shapes.reduce_tiles.blocks_per_multiprocessor * specs.streaming_multiprocessors, + (size_t)1), + cuda_device_collective_max_blocks_k); + size_t const elements_per_tile = divide_round_up<size_t>(divide_round_up<size_t>(count, target_blocks), + cuda_device_collective_threads_k) * + cuda_device_collective_threads_k; + size_t const blocks = elements_per_tile ? divide_round_up<size_t>(count, elements_per_tile) : (size_t)0; + + if (blocks > 1 && partials.size() > blocks) { + value_type_ const *input_arg = input; + size_t count_arg = count; + size_t elements_per_tile_arg = elements_per_tile; + value_type_ *partials_arg = partials.data(); + void *reduce_args[4] = {(void *)&input_arg, (void *)&count_arg, (void *)&elements_per_tile_arg, + (void *)&partials_arg}; + CUresult error = cuda_launch_t {} + .grid((unsigned)blocks) + .block(cuda_device_collective_threads_k) + .shared(0) + .stream(stream) + .launch(shapes.reduce_tiles.function, reduce_args); + if (error != CUDA_SUCCESS) return make_cuda_status(error); + + // In place, so `partials[blocks]` comes out holding the grand total the apply phase publishes. + value_type_ const *partials_input_arg = partials.data(); + size_t blocks_arg = blocks; + void *scan_args[3] = {(void *)&partials_input_arg, (void *)&blocks_arg, (void *)&partials_arg}; + error = cuda_launch_t {} + .grid(1u) + .block(cuda_device_collective_threads_k) + .shared(0) + .stream(stream) + .launch(shapes.whole.function, scan_args); + if (error != CUDA_SUCCESS) return make_cuda_status(error); + + value_type_ *output_arg = output; + void *apply_args[5] = {(void *)&input_arg, (void *)&count_arg, (void *)&elements_per_tile_arg, + (void *)&partials_input_arg, (void *)&output_arg}; + error = cuda_launch_t {} + .grid((unsigned)blocks) + .block(cuda_device_collective_threads_k) + .shared(0) + .stream(stream) + .launch(shapes.apply_tiles.function, apply_args); + if (error != CUDA_SUCCESS) return make_cuda_status(error); + return {status_t::success_k, cudaSuccess}; + } + value_type_ const *input_arg = input; size_t count_arg = count; value_type_ *output_arg = output; @@ -921,7 +1121,7 @@ inline cuda_status_t cuda_launch_exclusive_sum_(kernel_shape_t const &shape, val .block(cuda_device_collective_threads_k) .shared(0) .stream(stream) - .launch(shape.function, args); + .launch(shapes.whole.function, args); if (error != CUDA_SUCCESS) return make_cuda_status(error); return {status_t::success_k, cudaSuccess}; } @@ -966,8 +1166,8 @@ inline cuda_status_t cuda_launch_scatter_tasks_by_bucket_(kernel_shape_t const & task_type_ *output_arg = output; void *args[5] = {(void *)&tasks_arg, (void *)&count_arg, (void *)&bucket_arg, (void *)&cursors_arg, (void *)&output_arg}; - unsigned const blocks = static_cast<unsigned>( - sz_min_of_two((count + cuda_device_collective_threads_k - 1) / cuda_device_collective_threads_k, size_t(1024))); + unsigned const blocks = static_cast<unsigned>(sz_min_of_two( + divide_round_up<size_t>(count, cuda_device_collective_threads_k), cuda_device_collective_max_blocks_k)); CUresult error = cuda_launch_t {} .grid(blocks ? blocks : 1u) .block(cuda_device_collective_threads_k) @@ -1041,8 +1241,8 @@ inline cuda_status_t cuda_launch_histogram_tasks_by_bucket_(kernel_shape_t const u32_t *counts_arg = bucket_counts; void *args[5] = {(void *)&tasks_arg, (void *)&count_arg, (void *)&bucket_arg, (void *)&bucket_count_arg, (void *)&counts_arg}; - unsigned const blocks = static_cast<unsigned>( - sz_min_of_two((count + cuda_device_collective_threads_k - 1) / cuda_device_collective_threads_k, size_t(1024))); + unsigned const blocks = static_cast<unsigned>(sz_min_of_two( + divide_round_up<size_t>(count, cuda_device_collective_threads_k), cuda_device_collective_max_blocks_k)); CUresult error = cuda_launch_t {} .grid(blocks ? blocks : 1u) .block(cuda_device_collective_threads_k) @@ -1102,8 +1302,8 @@ inline cuda_status_t cuda_launch_histogram_dense_(kernel_shape_t const &shape, i u32_t bucket_count_arg = bucket_count; u32_t *out_arg = out; void *args[4] = {(void *)&buckets_arg, (void *)&count_arg, (void *)&bucket_count_arg, (void *)&out_arg}; - unsigned const blocks = static_cast<unsigned>( - sz_min_of_two((count + cuda_device_collective_threads_k - 1) / cuda_device_collective_threads_k, size_t(1024))); + unsigned const blocks = static_cast<unsigned>(sz_min_of_two( + divide_round_up<size_t>(count, cuda_device_collective_threads_k), cuda_device_collective_max_blocks_k)); CUresult error = cuda_launch_t {} .grid(blocks ? blocks : 1u) .block(cuda_device_collective_threads_k) @@ -1138,6 +1338,19 @@ SZ_DEVICE_INLINE u32_vec_t sz_u32_load_unaligned(void const *ptr) noexcept { return result; } +/** + * @brief Loads 32 bits from a 4-byte-aligned address as one plain load. + * + * PTX traps on a misaligned `ld.u32` rather than slowing down, so the compiler lowers a 4-byte `memcpy` + * from an unproven pointer into four byte loads plus three `prmt` merges. A caller that peeled its cursor + * to a 4-byte boundary states the guarantee here and gets the single `ld.u32` it earned. + */ +SZ_DEVICE_INLINE u32_vec_t sz_u32_load_aligned(void const *ptr) noexcept { + u32_vec_t result; + asm("ld.u32 %0, [%1];" : "=r"(result.u32) : "l"(ptr)); + return result; +} + /** @brief Number of threads per warp on the GPU. */ enum warp_size_t : unsigned { warp_size_nvidia_k = 32, // ? NVIDIA GPUs use 32 threads per warp @@ -1344,8 +1557,9 @@ warp_tasks_groups<task_type_> warp_tasks_grouping( // if (cuda_launch_histogram_dense_(tier_histogram_shape, tier_bucket_iterator, total_tasks, 3u, tier_counts, stream) .status != status_t::success_k) return result; - if (cuda_launch_exclusive_sum_(exclusive_sum_u32_shape, tier_counts, 3u, tier_cursors, stream).status != - status_t::success_k) + if (cuda_launch_exclusive_sum_(exclusive_sum_shapes_t {exclusive_sum_u32_shape}, tier_counts, 3u, tier_cursors, + span<u32_t> {}, gpu_specs_t {}, stream) + .status != status_t::success_k) return result; if (cuda_launch_scatter_tasks_by_bucket_(tier_scatter_shape, tasks.data(), total_tasks, tier_of, tier_cursors, partition_buffer, stream) diff --git a/include/stringzillas/types.hpp b/include/stringzillas/types.hpp index 856867b0..4937251a 100644 --- a/include/stringzillas/types.hpp +++ b/include/stringzillas/types.hpp @@ -6,11 +6,12 @@ #ifndef STRINGZILLAS_TYPES_HPP_ #define STRINGZILLAS_TYPES_HPP_ -#include <thread> // `std::thread::hardware_concurrency` +#include <cstdlib> // `std::malloc`, `std::free` + #include <atomic> // `std::atomic`, `std::memory_order` #include <concepts> // `std::convertible_to`, `std::same_as` -#include <cstdlib> // `std::malloc`, `std::free` #include <memory> // `std::addressof` +#include <thread> // `std::thread::hardware_concurrency` #include <forkunion.h> // `fu_pool_t`, `fu_topology_t`, capability-dispatched parallel loops @@ -193,6 +194,34 @@ class forkunion_executor_t { size_t threads_count() const noexcept { return fu_pool_threads_count(pool_); } mutex_t make_mutex() const noexcept { return {}; } + /** + * @brief The specs of the machine this executor spawned on, read from its own detected topology. + * Fills the shared-cache volume and core counts; `l1_bytes` and `l2_bytes` keep their conservative + * defaults - the ForkUnion C API exposes no per-core cache-level query. Defaults throughout when + * the pool was never spawned or the platform reports nothing. + */ + cpu_specs_t specs() const noexcept { + cpu_specs_t specs; + if (!topology_) return specs; + // The deepest cache confined to each compute domain - the shared L3 on uniform machines. The + // smallest nonzero domain wins so cache-resident chunk sizing never overshoots the tightest cluster. + size_t const compute_domains = fu_compute_domains_count(topology_); + size_t confined_cache_bytes = 0; + for (size_t domain = 0; domain != compute_domains; ++domain) { + size_t const domain_cache_bytes = fu_compute_cache_bytes_in(topology_, domain); + if (domain_cache_bytes && (confined_cache_bytes == 0 || domain_cache_bytes < confined_cache_bytes)) + confined_cache_bytes = domain_cache_bytes; + } + if (confined_cache_bytes) specs.l3_bytes = confined_cache_bytes; + size_t const logical_cores = fu_logical_cores_count(topology_); + size_t const memory_domains = fu_memory_domains_count(topology_); + if (logical_cores) { + specs.sockets = memory_domains ? memory_domains : 1; + specs.cores_per_socket = sz_max_of_two(logical_cores / specs.sockets, (size_t)1); + } + return specs; + } + /** * @brief Calls the @p function for each index from 0 to @p (n) in such * a way that consecutive elements are likely to be processed by @@ -456,8 +485,34 @@ concept continuous_like = requires(continuous_type_ container) { static_assert(continuous_like<span<char>>); static_assert(!continuous_like<int>); + #endif +/** + * @brief Whether a container's elements are slices of one contiguous block, addressed by an offsets array. + * + * Detects lengths rather than offsets on purpose: the packed and NULL-terminated flavors disagree on what an + * element's length is, and a caller doing its own offset arithmetic would silently get one of them wrong. + * + * A trait rather than a concept, so the engines can branch on it at C++17 - which the Python bindings build + * these headers at - without a preprocessor conditional cutting through the control flow that uses it. + */ +template <typename tape_type_, typename = void> +struct is_tape_like { + static constexpr bool value = false; +}; + +template <typename tape_type_> +struct is_tape_like<tape_type_, std::void_t<decltype(std::declval<tape_type_ const &>().tape_bytes()), + decltype(std::declval<tape_type_ const &>().tape_total_bytes()), + decltype(std::declval<tape_type_ const &>().tape_length_at(size_t {}))>> { + static constexpr bool value = true; +}; + +static_assert(is_tape_like<arrow_strings_view<char, u32_t>>::value); +static_assert(is_tape_like<arrow_packed_view<char, u32_t>>::value); +static_assert(!is_tape_like<span<char>>::value); + /** * @brief A function that takes a range of elements and a @p callback function and groups the elements * that @p equality function considers equal. Analogous to `std::ranges::group_by`. @@ -484,6 +539,32 @@ size_t group_by(begin_iterator_type_ const begin, end_iterator_type_ const end, return group_count; } +/** + * @brief A running, cache-line-padded scratch byte amount, used to lay out an engine's sub-buffers. + * + * An engine partitions one flat scratch block into a handful of sub-buffers - score diagonals, a reversed + * copy of the shorter string, an automaton's edge CSR, ... Growing this amount once per sub-buffer keeps + * every offset cache-line aligned and yields the total the engine needs, a single source of truth shared + * by its `layout()` and the code that reads those buffers back. Cache-line width is `>=` any CPU register + * width, so the padding also keeps full-register SIMD over-reads near a buffer's end in bounds. + */ +struct scratch_amount_t { + // ? Deliberately a poison default (not `SZ_CACHE_LINE_WIDTH`): an instance built without an explicit + // ? `cpu_specs_t::cache_line_width` should produce an obviously-broken `total` (huge → `bad_alloc`/ASan), + // ? surfacing any place that forgot to propagate the alignment rather than silently assuming 64 bytes. + size_t alignment = std::numeric_limits<size_t>::max(); + size_t total = 0; // ? The accumulated, padded byte count == the next buffer's offset. + + /** @brief Reads the current end of the scratch, i.e. the offset where the next sub-buffer would start. */ + constexpr operator size_t() const noexcept { return total; } + + /** @brief Reserves @p bytes for the next sub-buffer, padded so the following offset stays aligned. */ + constexpr scratch_amount_t &operator+=(size_t bytes) noexcept { + total += round_up_to_multiple<size_t>(bytes, alignment); + return *this; + } +}; + /** * @brief Safer alternative to `std::vector`, that avoids exceptions, copy constructors, * and provides alternative `try_push_back` and `try_reserve` for faulty memory allocations. diff --git a/java/README.md b/java/README.md index 42c45353..6cd3f7fe 100644 --- a/java/README.md +++ b/java/README.md @@ -9,7 +9,7 @@ SIMD-accelerated search, comparison, hashing, UTF-8 segmentation, case-folding, <dependency> <groupId>com.github.ashvardanian</groupId> <artifactId>stringzilla</artifactId> - <version>4.6.2</version> + <version>5.1.1</version> </dependency> ``` diff --git a/javascript/lib.c b/javascript/lib.c index 3001cfe9..ec1422db 100644 --- a/javascript/lib.c +++ b/javascript/lib.c @@ -1,5 +1,5 @@ /** - * @file lib.c + * @file javascript/lib.c * @brief JavaScript bindings for StringZilla. * @author Ash Vardanian * @date September 18, 2023 diff --git a/probes/cuda_native_arch.cu b/probes/cuda_native_arch.cu new file mode 100644 index 00000000..f6f907e1 --- /dev/null +++ b/probes/cuda_native_arch.cu @@ -0,0 +1,9 @@ +/* StringZilla CUDA probe: building a `.cu` source for this machine's own instruction set, which turns on the + * host compiler's widest intrinsic headers. NVCC reads those headers on its device pass even though only host + * functions use them, and some pairs of toolkit and host disagree over the builtins inside, so the header is + * what the probe carries - the flag alone would compile anywhere. */ +#include <immintrin.h> + +__global__ void sz_probe_kernel_(void) {} + +int main(void) { return 0; } diff --git a/pyproject.toml b/pyproject.toml index c8d8aeac..c87175e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,13 +31,18 @@ python_files = ["*.py"] testpaths = ["test"] xfail_strict = true -# Third-party oracles the suite differentials against when they import and skips over when they do not. +# Packages the pytest suite validates against when they import and skips over when they do not. PEP 735 +# groups, so they are development tooling: never published in wheel metadata, never installed by anyone +# consuming StringZilla, and reachable only through `pip install --group <name>`, which the CI jobs run +# rather than repeating the names. Rust spells the same idea as `[dev-dependencies]` in `Cargo.toml`. # Deliberately kept out of `[tool.cibuildwheel] test-requires`, which installs on QEMU-emulated # aarch64, ppc64le and s390x, where a missing wheel means compiling the oracle's own C under emulation. -# `pycryptodome` ships its own C implementation linking only libc, so it is an independent witness to -# the OpenSSL the C++ suite already compares against. +# `pycryptodome` and `pyahocorasick` each ship their own C, so they are independent witnesses to the +# OpenSSL and the multi-pattern walk the C++ suite already compares against. `cupy` is not an oracle but +# a producer, sourcing `__cuda_array_interface__` buffers that are not the module's own `unified_array`. [dependency-groups] -differential = ["pycryptodome"] +tests-oracles = ["pycryptodome", "pyahocorasick"] +tests-cuda = ["cupy-cuda12x"] [tool.black] line-length = 120 diff --git a/python/README.md b/python/README.md index 91e9db6e..5b086c47 100644 --- a/python/README.md +++ b/python/README.md @@ -474,7 +474,8 @@ Every engine is called as `engine(queries, candidates=None, device=None, out=Non - `queries` — the collection forming the matrix rows, either a `sz.Strs` or any string sequence. - `candidates` — the collection forming the matrix columns; when omitted or `None`, the engine computes the symmetric self-similarity of `queries`. - `device` — an optional `DeviceScope` overriding the constructor's. -- `out` — an optional pre-allocated 2-D NumPy output buffer of shape `(len(queries), len(candidates))`. +- `out` — an optional pre-allocated 2-D output buffer of shape `(len(queries), len(candidates))`. + On a CPU scope this is a NumPy array; on a GPU scope it must be a CUDA buffer, as described under [Unified Memory](#unified-memory). The result `result[i, j]` is the distance or score between `queries[i]` and `candidates[j]`. @@ -548,7 +549,79 @@ distances = engine(a, b, device=gpu) ``` Each engine also exposes a read-only `__capabilities__` property reporting the backends it selected at runtime. -The module-level `szs.to_device(strs)` converts a `sz.Strs` to use a unified/device-accessible allocator, forcing the allocator swap that normally happens during GPU kernel execution. + +### Unified Memory + +On a GPU scope every array an engine reads or writes lives on the device — unified or plain CUDA memory, never page-locked host memory. +Most of that is handled for you: `sz.Strs` inputs are moved to a device-accessible allocator on the way in, and every array an engine returns is already allocated there. +The rule is therefore only visible on the arrays _you_ supply, of which there are two: the optional `out` matrix of an alignment engine, and `Substrings.score_bm25`'s `needle_weights` and `document_lengths`. +A host array in one of those positions raises `BufferError` rather than being copied behind your back. +A CPU scope imposes no requirement at all. + +```python +gpu = szs.DeviceScope(gpu_device=0) +engine = szs.LevenshteinDistances(capabilities=gpu) + +distances = engine(a, b, device=gpu) # returned array is already device-resident +distances = engine(a, b, device=gpu, out=np.zeros((len(a), len(b)), dtype=np.uint64)) # BufferError + +out = szs.unified_array((len(a), len(b)), dtype=np.uint64) # this one qualifies +distances = engine(a, b, device=gpu, out=out) +``` + +`szs.unified_array(shape, dtype)` is how you produce a qualifying array without CuPy or Torch; anything exposing `__cuda_array_interface__` or `__dlpack__` works too. +The module-level `szs.to_device(strs)` does the same for a `sz.Strs`, which is worth doing to pay the swap once for a collection reused across many calls. + +## Multi-Pattern Matching + +`Substrings(needles, case_sensitivity='cased', device=None, capabilities=None)` matches a whole dictionary of needles against a whole collection of haystacks in one pass. +The needle set is compiled once into an Aho-Corasick automaton and reused across every later call, so the dictionary is paid for once rather than per haystack. +Construction is itself a device operation — the automaton's tier split is sized against the cache the walk reads through, and a CUDA automaton is uploaded to the device — which is why `device` belongs on the constructor as well as on each call. + +| Argument | Default | Meaning | +| ------------------ | ---------- | ------------------------------------------------------------- | +| `needles` | required | `sz.Strs` of needles, non-empty, valid UTF-8 when folding. | +| `case_sensitivity` | `'cased'` | `'cased'` matches bytes, `'uncased'` folds both sides. | +| `device` | `None` | `DeviceScope` the automaton is built for and uploaded to. | +| `capabilities` | `None` | Capabilities tuple restricting the engine, may include `'cuda'`. | + +Four operations share that automaton, each taking an optional `device` overriding the constructor's: + +| Call | Returns | +| --------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| `count(haystacks, policy='overlapping')` | A `uint64` count per haystack, and the corpus-wide total as an `int`. | +| `find(haystacks, policy='overlapping')` | Four `uint64` arrays — haystack indices, needle indices, offsets, lengths. | +| `score_bm25(haystacks, needle_weights, average_document_length, ...)` | One `float32` score per haystack. | +| `replace(haystacks, replacements, policy='leftmost-longest')` | The rewritten tape as `bytes`, and its `uint64` offsets, one per haystack plus one. | + +`policy` picks how overlaps resolve — `'overlapping'` reports every match, while `'leftmost-longest'` and `'leftmost-first'` each keep a non-overlapping cover. +`replace` accepts only the two cover policies, since a rewrite cannot substitute two matches at the same byte. +`replace_bound(replacements, input_bytes)` bounds the bytes a rewrite can produce from the needle set alone, needing no haystacks, no walk, and no device. + +```python +import stringzilla as sz +import stringzillas as szs + +engine = szs.Substrings(sz.Strs(["he", "she", "his", "hers"])) +haystacks = sz.Strs(["ushers", "hershey"]) + +counts, total = engine.count(haystacks) +assert total == 7 + +tape, offsets = engine.replace(haystacks, sz.Strs(["H", "SH", "HIS", "HERS"])) +assert bytes(tape[offsets[0]:offsets[1]]) == b"uSHrs" +``` + +The same engine runs on a GPU by passing a GPU `DeviceScope`, at construction and per call: + +```python +gpu = szs.DeviceScope(gpu_device=0) +engine = szs.Substrings(sz.Strs(["he", "she"]), device=gpu) +counts, total = engine.count(haystacks, device=gpu) +``` + +`score_bm25` is the one operation whose device rule you have to meet yourself: its `needle_weights` and optional `document_lengths` are read on the device, so on a GPU scope they must be CUDA buffers rather than plain NumPy arrays. +See [Unified Memory](#unified-memory) above. ## Rolling Fingerprints @@ -607,7 +680,7 @@ Each yields `Str` views into the original buffer, so segmentation stays allocati | `utf8_split_delimiters(string, skip_empty=False, with_separators=False)` | punctuation/symbol/separator | content BETWEEN any Unicode delimiter (superset of whitespace). | | `utf8_delimiters(string, skip_empty=False)` | punctuation/symbol/separator | the delimiter runs themselves (the separators). | -Naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the **separators**, while `split_*` yields the content **between** them. +Naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the __separators__, while `split_*` yields the content __between__ them. `skip_empty` drops empty segments; `with_separators=True` interleaves both losslessly (concatenation reproduces the input), replacing the old `keepends`. ```python diff --git a/python/stringzilla/compare.c b/python/stringzilla/compare.c index b35009f7..518cc4bd 100644 --- a/python/stringzilla/compare.c +++ b/python/stringzilla/compare.c @@ -28,7 +28,7 @@ PyObject *Str_like_equal(PyObject *self, PyObject *const *args, Py_ssize_t posit // Check minimum arguments int is_member = self != NULL && PyObject_TypeCheck(self, &StrType); if (positional_args_count < !is_member || positional_args_count > !is_member + 1 || args_names_tuple) { - PyErr_SetString(PyExc_TypeError, "equals() expects exactly two positional arguments"); + PyErr_SetString(PyExc_TypeError, "equal() expects exactly two positional arguments"); return NULL; } diff --git a/python/stringzilla/memory.c b/python/stringzilla/memory.c index 5322a0c9..df7bdd93 100644 --- a/python/stringzilla/memory.c +++ b/python/stringzilla/memory.c @@ -7,24 +7,25 @@ */ #include "stringzilla.h" -char const doc_translate[] = // - "Perform transformation of a string using a look-up table.\n" // - "\n" // - "Args:\n" // - " text (Str or str or bytes): The string object.\n" // - " table (str or dict): A 256-character string or a dictionary mapping bytes to bytes.\n" // - " inplace (bool, optional): If True, the string is modified in place (default is False).\n" // - "\n" // - " start (int, optional): The starting index for translation (default is 0).\n" // - " end (int, optional): The ending index for translation (default is the string length).\n" // - "Returns:\n" // - " Union[None, str, bytes]: If inplace is False, a new string is returned, otherwise None.\n" // - "Raises:\n" // - " ValueError: If the table is not 256 bytes long.\n" // - " TypeError: If the table is not a string or dictionary.\n" // - "\n" // - "Example:\n" // - " >>> sz.Str('abc').translate({'a': 'A'}) == b'Abc'\n" // +char const doc_translate[] = // + "Perform transformation of a string using a look-up table.\n" // + "\n" // + "Args:\n" // + " text (Str or str or bytes): The string object.\n" // + " table (str or dict): A 256-character string or a dict mapping single characters to single characters.\n" // + " inplace (bool, optional): If True, the string is modified in place (default is False).\n" // + "\n" // + " start (int, optional): The starting index for translation (default is 0).\n" // + " end (int, optional): The ending index for translation (default is the string length).\n" // + "Returns:\n" // + " Union[None, str, bytes]: If inplace is False, a translated copy of the [start, end) slice is\n" // + " returned, otherwise None.\n" // + "Raises:\n" // + " ValueError: If the table is not 256 bytes long.\n" // + " TypeError: If the table is not a string or dictionary.\n" // + "\n" // + "Example:\n" // + " >>> sz.Str('abc').translate({'a': 'A'}) == b'Abc'\n" // " True"; PyObject *Str_like_translate(PyObject *self, PyObject *const *args, Py_ssize_t positional_args_count, diff --git a/python/stringzilla/stringzilla.c b/python/stringzilla/stringzilla.c index bea12dac..4decf33e 100644 --- a/python/stringzilla/stringzilla.c +++ b/python/stringzilla/stringzilla.c @@ -105,8 +105,9 @@ static int parse_and_intersect_capabilities(PyObject *caps_obj, sz_capability_t // degrading to serial would hide it. *result = requested_caps & sz_capabilities(); if (*result == 0) { - PyErr_Format(PyExc_ValueError, "No requested capability is available here; available: %s", - sz_capabilities_to_string_implementation_(sz_capabilities())); + char available[256]; + sz_capabilities_to_string_implementation_(sz_capabilities(), available, sizeof(available)); + PyErr_Format(PyExc_ValueError, "No requested capability is available here; available: %s", available); return -1; } return 0; diff --git a/python/stringzilla/strs.c b/python/stringzilla/strs.c index 53df8553..09c63311 100644 --- a/python/stringzilla/strs.c +++ b/python/stringzilla/strs.c @@ -62,26 +62,6 @@ sz_size_t Strs_get_length_(void const *handle, sz_size_t i) { return 0; } -static sz_cptr_t sz_py_strs_sequence_member_start_if_fragmented(void const *sequence_punned, sz_size_t index) { - Strs *strs = (Strs *)sequence_punned; - sz_assert_(strs->layout == STRS_FRAGMENTED && "Expected a reordered Strs layout"); - if (index < 0 || index >= strs->data.fragmented.count) { - PyErr_SetString(PyExc_IndexError, "Index out of bounds"); - return NULL; - } - return strs->data.fragmented.spans[index].start; -} - -static sz_size_t sz_py_strs_sequence_member_length_if_fragmented(void const *sequence_punned, sz_size_t index) { - Strs *strs = (Strs *)sequence_punned; - sz_assert_(strs->layout == STRS_FRAGMENTED && "Expected a reordered Strs layout"); - if (index < 0 || index >= strs->data.fragmented.count) { - PyErr_SetString(PyExc_IndexError, "Index out of bounds"); - return 0; - } - return strs->data.fragmented.spans[index].length; -} - /** * @brief Helper function to export a `Strs` or similar sequence objects into a `sz_sequence_t`. */ @@ -90,12 +70,14 @@ sz_bool_t sz_py_export_strings_as_sequence(PyObject *object, sz_sequence_t *sequ if (PyObject_TypeCheck(object, &StrsType)) { Strs *strs = (Strs *)object; - sz_assert_(strs->layout == STRS_FRAGMENTED && "View as tapes!"); + // Every layout, not just the reordered one. A caller holding a tape normally prefers the tape + // exporters, which hand the kernel its offsets directly - but an entry point with no tape overload, + // like the multi-pattern engine's needle set, has nowhere else to go and must get a sequence here. sequence->handle = strs; - sequence->count = strs->data.fragmented.count; - sequence->get_start = sz_py_strs_sequence_member_start_if_fragmented; - sequence->get_length = sz_py_strs_sequence_member_length_if_fragmented; + sequence->count = (sz_size_t)Strs_len(strs); + sequence->get_start = Strs_get_start_; + sequence->get_length = Strs_get_length_; return sz_true_k; } @@ -2472,8 +2454,7 @@ static PyMethodDef Strs_methods[] = { {"argsort", Strs_argsort, SZ_METHOD_FLAGS, doc_argsort}, // {"sample", Strs_sample, SZ_METHOD_FLAGS, doc_Strs_sample}, // {"intersect", Strs_intersect, SZ_METHOD_FLAGS, doc_Strs_intersect}, // - // {"to_pylist", Strs_to_pylist, SZ_METHOD_FLAGS, "Exports string-views to a native list of native strings."}, // - {NULL, NULL, 0, NULL} // Sentinel + {NULL, NULL, 0, NULL} // Sentinel }; static char const doc_Strs[] = // diff --git a/python/stringzillas/device_scope.c b/python/stringzillas/device_scope.c index bc88b409..2d6304c3 100644 --- a/python/stringzillas/device_scope.c +++ b/python/stringzillas/device_scope.c @@ -88,9 +88,18 @@ static char const doc_DeviceScope[] = "\n" // "Note: Cannot specify both cpu_cores and gpu_device.\n" // "\n" // + "Unified memory:\n" // + " On a GPU scope every array an engine reads or writes lives on the device -\n" // + " unified or plain CUDA memory, never page-locked host memory. `Strs` inputs\n" // + " and the arrays engines return are placed there for you, so the rule is only\n" // + " visible on arrays you supply: an `out=` matrix, or `score_bm25`'s weights and\n" // + " lengths. A host array in those positions raises BufferError. A CPU scope\n" // + " imposes no requirement at all.\n" // + "\n" // "Examples:\n" // " >>> import stringzillas as szs\n" // - " >>> scope = szs.DeviceScope(cpu_cores=4) # restrict engines to 4 CPU cores"; + " >>> scope = szs.DeviceScope(cpu_cores=4) # restrict engines to 4 CPU cores\n" // + " >>> gpu = szs.DeviceScope(gpu_device=0) if 'cuda' in szs.__capabilities__ else scope"; PyTypeObject DeviceScopeType = { PyVarObject_HEAD_INIT(NULL, 0).tp_name = "stringzillas.DeviceScope", diff --git a/python/stringzillas/fingerprints.c b/python/stringzillas/fingerprints.c index a45a3342..854a59f1 100644 --- a/python/stringzillas/fingerprints.c +++ b/python/stringzillas/fingerprints.c @@ -153,7 +153,7 @@ static PyObject *Fingerprints_call(Fingerprints *self, PyObject *args, PyObject } // Swap allocators only when using CUDA with a GPU device (inputs must be unified) - sz_bool_t need_unified = requires_unified_memory(self->capabilities); + sz_bool_t need_unified = requires_device_memory(self->capabilities); if (need_unified) if (!try_swap_to_unified_allocator(texts_obj)) return NULL; diff --git a/python/stringzillas/similarities.c b/python/stringzillas/similarities.c index d8ae0bf4..7c30bbc3 100644 --- a/python/stringzillas/similarities.c +++ b/python/stringzillas/similarities.c @@ -265,7 +265,7 @@ static PyObject *LevenshteinDistances_vectorcall(PyObject *callable, PyObject *c sz_size_t, char const **) = NULL; // Swap allocators only when using CUDA with a GPU device (inputs must be unified) - if (requires_unified_memory(self->capabilities)) { + if (requires_device_memory(self->capabilities)) { if (!try_swap_to_unified_allocator(queries_obj)) return NULL; if (candidates_obj && !try_swap_to_unified_allocator(candidates_obj)) return NULL; } @@ -345,9 +345,31 @@ static PyObject *LevenshteinDistances_vectorcall(PyObject *callable, PyObject *c candidates_count = candidates_any_count; } - // Allocate a fresh 2-D matrix or validate the provided `out` array, deriving the row stride in ELEMENTS. + // A GPU scope writes its results from the device, so it allocates and accepts device memory; a CPU scope + // keeps speaking NumPy. Either way the row stride below is in ELEMENTS. PyObject *results_array = NULL; - if (!out_obj || out_obj == Py_None) { + if (requires_device_memory(self->capabilities)) { + if (!out_obj || out_obj == Py_None) { + npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; + results_array = new_unified_array(2, results_shape, NPY_UINT64, sizeof(sz_size_t)); + if (!results_array) goto cleanup; + kernel_results = (sz_size_t *)PyArray_DATA((PyArrayObject *)results_array); + kernel_results_row_stride = candidates_count; + } + else { + device_buffer_t out_buffer; + if (parse_device_buffer(out_obj, sizeof(sz_size_t), &out_buffer) != 0) goto cleanup; + if (out_buffer.rows < queries_count || out_buffer.columns < candidates_count) { + PyErr_SetString(PyExc_ValueError, "out buffer is too small for results"); + goto cleanup; + } + kernel_results = (sz_size_t *)out_buffer.data; + kernel_results_row_stride = out_buffer.row_stride; + results_array = out_obj; + Py_INCREF(results_array); + } + } + else if (!out_obj || out_obj == Py_None) { npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; results_array = PyArray_SimpleNew(2, results_shape, NPY_UINT64); if (!results_array) { @@ -430,6 +452,7 @@ static char const doc_LevenshteinDistances[] = " (or None), computes the symmetric self-similarity of queries.\n" // " device (DeviceScope, optional): Device execution context.\n" // " out (np.ndarray, optional): 2-D uint64 output buffer of shape (len(queries), len(candidates)).\n" // + " On a GPU scope this must be a CUDA buffer; a host array raises BufferError.\n" // "\n" // "Returns:\n" // " np.ndarray: 2-D uint64 matrix where result[query_index, candidate_index] is the distance\n" // @@ -651,7 +674,7 @@ static PyObject *LevenshteinDistancesUTF8_vectorcall(PyObject *callable, PyObjec sz_size_t, char const **) = NULL; // Swap allocators when engine supports CUDA - if (requires_unified_memory(self->capabilities)) { + if (requires_device_memory(self->capabilities)) { if (!try_swap_to_unified_allocator(queries_obj)) return NULL; if (candidates_obj && !try_swap_to_unified_allocator(candidates_obj)) return NULL; } @@ -731,9 +754,31 @@ static PyObject *LevenshteinDistancesUTF8_vectorcall(PyObject *callable, PyObjec candidates_count = candidates_any_count; } - // Allocate a fresh 2-D matrix or validate the provided `out` array, deriving the row stride in ELEMENTS. + // A GPU scope writes its results from the device, so it allocates and accepts device memory; a CPU scope + // keeps speaking NumPy. Either way the row stride below is in ELEMENTS. PyObject *results_array = NULL; - if (!out_obj || out_obj == Py_None) { + if (requires_device_memory(self->capabilities)) { + if (!out_obj || out_obj == Py_None) { + npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; + results_array = new_unified_array(2, results_shape, NPY_UINT64, sizeof(sz_size_t)); + if (!results_array) goto cleanup; + kernel_results = (sz_size_t *)PyArray_DATA((PyArrayObject *)results_array); + kernel_results_row_stride = candidates_count; + } + else { + device_buffer_t out_buffer; + if (parse_device_buffer(out_obj, sizeof(sz_size_t), &out_buffer) != 0) goto cleanup; + if (out_buffer.rows < queries_count || out_buffer.columns < candidates_count) { + PyErr_SetString(PyExc_ValueError, "out buffer is too small for results"); + goto cleanup; + } + kernel_results = (sz_size_t *)out_buffer.data; + kernel_results_row_stride = out_buffer.row_stride; + results_array = out_obj; + Py_INCREF(results_array); + } + } + else if (!out_obj || out_obj == Py_None) { npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; results_array = PyArray_SimpleNew(2, results_shape, NPY_UINT64); if (!results_array) { @@ -816,6 +861,7 @@ static char const doc_LevenshteinDistancesUTF8[] = " omitted (or None), computes symmetric self-similarity of queries.\n" // " device (DeviceScope, optional): Device execution context.\n" // " out (np.ndarray, optional): 2-D uint64 output buffer of shape (len(queries), len(candidates)).\n" // + " On a GPU scope this must be a CUDA buffer; a host array raises BufferError.\n" // "\n" // "Returns:\n" // " np.ndarray: 2-D uint64 matrix where result[query_index, candidate_index] is the distance\n" // @@ -1074,7 +1120,7 @@ static PyObject *NeedlemanWunsch_vectorcall(PyObject *callable, PyObject *const sz_ssize_t *, sz_size_t, char const **) = NULL; // Swap allocators only when using CUDA with a GPU device (inputs must be unified) - if (requires_unified_memory(self->capabilities)) { + if (requires_device_memory(self->capabilities)) { if (!try_swap_to_unified_allocator(queries_obj)) return NULL; if (candidates_obj && !try_swap_to_unified_allocator(candidates_obj)) return NULL; } @@ -1154,12 +1200,34 @@ static PyObject *NeedlemanWunsch_vectorcall(PyObject *callable, PyObject *const candidates_count = candidates_any_count; } - // Allocate a fresh 2-D matrix or validate the provided `out` array, deriving the row stride in ELEMENTS. + // A GPU scope writes its results from the device, so it allocates and accepts device memory; a CPU scope + // keeps speaking NumPy. Either way the row stride below is in ELEMENTS. PyObject *results_array = NULL; sz_ssize_t *kernel_results = NULL; sz_size_t kernel_results_row_stride = 0; - if (!out_obj || out_obj == Py_None) { + if (requires_device_memory(self->capabilities)) { + if (!out_obj || out_obj == Py_None) { + npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; + results_array = new_unified_array(2, results_shape, NPY_INT64, sizeof(sz_ssize_t)); + if (!results_array) goto cleanup; + kernel_results = (sz_ssize_t *)PyArray_DATA((PyArrayObject *)results_array); + kernel_results_row_stride = candidates_count; + } + else { + device_buffer_t out_buffer; + if (parse_device_buffer(out_obj, sizeof(sz_ssize_t), &out_buffer) != 0) goto cleanup; + if (out_buffer.rows < queries_count || out_buffer.columns < candidates_count) { + PyErr_SetString(PyExc_ValueError, "out buffer is too small for results"); + goto cleanup; + } + kernel_results = (sz_ssize_t *)out_buffer.data; + kernel_results_row_stride = out_buffer.row_stride; + results_array = out_obj; + Py_INCREF(results_array); + } + } + else if (!out_obj || out_obj == Py_None) { npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; results_array = PyArray_SimpleNew(2, results_shape, NPY_INT64); if (!results_array) { @@ -1242,6 +1310,7 @@ static char const doc_NeedlemanWunsch[] = " (or None), computes the symmetric self-similarity of queries.\n" // " device (DeviceScope, optional): Device execution context.\n" // " out (np.ndarray, optional): 2-D int64 output buffer of shape (len(queries), len(candidates)).\n" // + " On a GPU scope this must be a CUDA buffer; a host array raises BufferError.\n" // "\n" // "Returns:\n" // " np.ndarray: 2-D int64 matrix where result[query_index, candidate_index] is the score\n" // @@ -1493,7 +1562,7 @@ static PyObject *SmithWaterman_vectorcall(PyObject *callable, PyObject *const *a sz_ssize_t *, sz_size_t, char const **) = NULL; // Swap allocators only when using CUDA with a GPU device (inputs must be unified) - if (requires_unified_memory(self->capabilities)) { + if (requires_device_memory(self->capabilities)) { if (!try_swap_to_unified_allocator(queries_obj)) return NULL; if (candidates_obj && !try_swap_to_unified_allocator(candidates_obj)) return NULL; } @@ -1573,12 +1642,34 @@ static PyObject *SmithWaterman_vectorcall(PyObject *callable, PyObject *const *a candidates_count = candidates_any_count; } - // Allocate a fresh 2-D matrix or validate the provided `out` array, deriving the row stride in ELEMENTS. + // A GPU scope writes its results from the device, so it allocates and accepts device memory; a CPU scope + // keeps speaking NumPy. Either way the row stride below is in ELEMENTS. PyObject *results_array = NULL; sz_ssize_t *kernel_results = NULL; sz_size_t kernel_results_row_stride = 0; - if (!out_obj || out_obj == Py_None) { + if (requires_device_memory(self->capabilities)) { + if (!out_obj || out_obj == Py_None) { + npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; + results_array = new_unified_array(2, results_shape, NPY_INT64, sizeof(sz_ssize_t)); + if (!results_array) goto cleanup; + kernel_results = (sz_ssize_t *)PyArray_DATA((PyArrayObject *)results_array); + kernel_results_row_stride = candidates_count; + } + else { + device_buffer_t out_buffer; + if (parse_device_buffer(out_obj, sizeof(sz_ssize_t), &out_buffer) != 0) goto cleanup; + if (out_buffer.rows < queries_count || out_buffer.columns < candidates_count) { + PyErr_SetString(PyExc_ValueError, "out buffer is too small for results"); + goto cleanup; + } + kernel_results = (sz_ssize_t *)out_buffer.data; + kernel_results_row_stride = out_buffer.row_stride; + results_array = out_obj; + Py_INCREF(results_array); + } + } + else if (!out_obj || out_obj == Py_None) { npy_intp results_shape[2] = {(npy_intp)queries_count, (npy_intp)candidates_count}; results_array = PyArray_SimpleNew(2, results_shape, NPY_INT64); if (!results_array) { @@ -1674,6 +1765,7 @@ static char const doc_SmithWaterman[] = " (or None), computes the symmetric self-similarity of queries.\n" // " device (DeviceScope, optional): Device execution context.\n" // " out (np.ndarray, optional): 2-D int64 output buffer of shape (len(queries), len(candidates)).\n" // + " On a GPU scope this must be a CUDA buffer; a host array raises BufferError.\n" // "\n" // "Returns:\n" // " np.ndarray: 2-D int64 matrix where result[query_index, candidate_index] is the score\n" // diff --git a/python/stringzillas/stringzillas.c b/python/stringzillas/stringzillas.c index 164ffa9a..e2d33b3e 100644 --- a/python/stringzillas/stringzillas.c +++ b/python/stringzillas/stringzillas.c @@ -40,9 +40,11 @@ void set_stringzilla_error(sz_status_t status, char const *error_detail, char co case sz_invalid_utf8_k: PyErr_Format(PyExc_ValueError, "%s: %s", context, error_detail); break; case sz_overflow_risk_k: PyErr_Format(PyExc_OverflowError, "%s: %s", context, error_detail); break; case sz_unexpected_dimensions_k: PyErr_Format(PyExc_ValueError, "%s: %s", context, error_detail); break; + // A backstop: the binding stages every buffer it hands an engine under a CUDA capability, so a reachable + // one of these means the binding has a bug rather than the caller. + case sz_device_memory_mismatch_k: PyErr_Format(PyExc_BufferError, "%s: %s", context, error_detail); break; case sz_missing_gpu_k: case sz_device_code_mismatch_k: - case sz_device_memory_mismatch_k: default: PyErr_Format(PyExc_RuntimeError, "%s: %s", context, error_detail); break; } } @@ -141,8 +143,9 @@ int parse_and_intersect_capabilities(PyObject *caps_obj, sz_capability_t *result // degrading to serial would hide it, and the `DeviceScope` arm above already raises for the same case. *result = requested_caps & ceiling; if (*result == 0) { - PyErr_Format(PyExc_ValueError, "No requested capability is available here; available: %s", - sz_capabilities_to_string_implementation_(ceiling)); + char available[256]; + sz_capabilities_to_string_implementation_(ceiling, available, sizeof(available)); + PyErr_Format(PyExc_ValueError, "No requested capability is available here; available: %s", available); return -1; } @@ -189,21 +192,24 @@ static PyObject *module_reset_capabilities(PyObject *self, PyObject *args) { } Py_DECREF(caps_tuple); - sz_cptr_t caps_str = sz_capabilities_to_string_implementation_(caps); + char caps_str[256]; + sz_capabilities_to_string_implementation_(caps, caps_str, sizeof(caps_str)); if (PyObject_SetAttrString(self, "__capabilities_str__", PyUnicode_FromString(caps_str)) != 0) { return NULL; } Py_RETURN_NONE; } -static char const doc_to_device[] = // - "to_device(strs: sz.Strs) -> sz.Strs\n\n" // - "Converts a Strs object to use unified/device-accessible memory allocator.\n" // - "This function forces the allocator swap that would normally happen during\n" // - "GPU kernel execution. Useful for testing slice handling after re-allocation.\n" // - "\n" // - "Examples:\n" // - " >>> import stringzilla as sz, stringzillas as szs\n" // - " >>> strs = sz.Strs(['alpha', 'beta'])\n" // +static char const doc_to_device[] = // + "to_device(strs: sz.Strs) -> sz.Strs\n\n" // + "Move a Strs onto device-accessible unified memory, in place.\n" // + "\n" // + "Engines do this themselves for the inputs of a GPU-scoped call, so it is only\n" // + "worth calling ahead of time to pay the swap once for a collection reused across\n" // + "many calls, or to hold a slice's identity across the swap.\n" // + "\n" // + "Examples:\n" // + " >>> import stringzilla as sz, stringzillas as szs\n" // + " >>> strs = sz.Strs(['alpha', 'beta'])\n" // " >>> device_strs = szs.to_device(strs) if 'cuda' in szs.__capabilities__ else strs"; static PyObject *module_to_device(PyObject *self, PyObject *strs_obj) { @@ -213,6 +219,87 @@ static PyObject *module_to_device(PyObject *self, PyObject *strs_obj) { return strs_obj; } +static char const doc_unified_array[] = // + "unified_array(shape, dtype=numpy.float32) -> numpy.ndarray\n\n" // + "Allocate a NumPy array backed by device-accessible unified memory.\n" // + "\n" // + "A GPU scope refuses host buffers, so the arrays a caller supplies there - an\n" // + "engine's `out=`, or `Substrings.score_bm25`'s weights - come from here when\n" // + "CuPy or Torch is not in play. A CPU scope needs none of this.\n" // + "\n" // + "Args:\n" // + " shape (int or tuple): Length of a vector, or (rows, columns) of a matrix.\n" // + " dtype (numpy.dtype, optional): Element type; float32 by default.\n" // + "\n" // + "Returns:\n" // + " numpy.ndarray: Uninitialized, wrapping unified memory freed with the array.\n" // + "\n" // + "Examples:\n" // + " >>> import numpy as np, stringzillas as szs\n" // + " >>> weights = szs.unified_array(2, dtype=np.float32)\n" // + " >>> weights[:] = 1.0"; + +static PyObject *module_unified_array(PyObject *self, PyObject *args, PyObject *kwargs) { + sz_unused_(self); + static char *kwlist[] = {"shape", "dtype", NULL}; + PyObject *shape_obj = NULL; + PyObject *dtype_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, &shape_obj, &dtype_obj)) return NULL; + if (!numpy_available) { + PyErr_SetString(PyExc_RuntimeError, "NumPy is required to allocate unified arrays"); + return NULL; + } + + // One length or a (rows, columns) pair, matching what the engines accept as a vector or a matrix. + npy_intp shape[2] = {0, 0}; + int dimensions = 0; + if (PyIndex_Check(shape_obj)) { + Py_ssize_t const length = PyNumber_AsSsize_t(shape_obj, PyExc_OverflowError); + if (length == -1 && PyErr_Occurred()) return NULL; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, "Negative dimensions are not allowed"); + return NULL; + } + shape[0] = (npy_intp)length, dimensions = 1; + } + else if (PyTuple_Check(shape_obj)) { + dimensions = (int)PyTuple_GET_SIZE(shape_obj); + if (dimensions < 1 || dimensions > 2) { + PyErr_SetString(PyExc_ValueError, "Only 1-D and 2-D unified arrays are supported"); + return NULL; + } + for (int axis = 0; axis < dimensions; ++axis) { + Py_ssize_t const length = PyNumber_AsSsize_t(PyTuple_GET_ITEM(shape_obj, axis), PyExc_OverflowError); + if (length == -1 && PyErr_Occurred()) return NULL; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, "Negative dimensions are not allowed"); + return NULL; + } + shape[axis] = (npy_intp)length; + } + } + else { + PyErr_SetString(PyExc_TypeError, "shape must be an integer or a tuple of one or two integers"); + return NULL; + } + + // `PyArray_Descr::elsize` is private in NumPy 2, so the width comes from the accessor rather than the struct. + PyArray_Descr *descr = NULL; + if (dtype_obj == NULL || dtype_obj == Py_None) { descr = PyArray_DescrFromType(NPY_FLOAT32); } + else if (!PyArray_DescrConverter(dtype_obj, &descr)) { return NULL; } + if (!descr) return NULL; + + int const type_number = descr->type_num; + sz_size_t const element_bytes = (sz_size_t)PyDataType_ELSIZE(descr); + Py_DECREF(descr); + if (element_bytes == 0) { + PyErr_SetString(PyExc_TypeError, "dtype must have a fixed, non-zero element width"); + return NULL; + } + + return new_unified_array(dimensions, shape, type_number, element_bytes); +} + static void stringzillas_cleanup(PyObject *m) { sz_unused_(m); if (default_device_scope) { @@ -224,6 +311,7 @@ static void stringzillas_cleanup(PyObject *m) { static PyMethodDef stringzillas_methods[] = { {"reset_capabilities", (PyCFunction)module_reset_capabilities, METH_VARARGS, doc_reset_capabilities}, {"to_device", (PyCFunction)module_to_device, METH_O, doc_to_device}, + {"unified_array", (PyCFunction)module_unified_array, METH_VARARGS | METH_KEYWORDS, doc_unified_array}, {NULL, NULL, 0, NULL}}; static PyModuleDef stringzillas_module = { @@ -320,6 +408,7 @@ PyMODINIT_FUNC PyInit_stringzillas(void) { if (PyType_Ready(&NeedlemanWunschType) < 0) return NULL; if (PyType_Ready(&SmithWatermanType) < 0) return NULL; if (PyType_Ready(&FingerprintsType) < 0) return NULL; + if (PyType_Ready(&SubstringsType) < 0) return NULL; m = PyModule_Create(&stringzillas_module); if (m == NULL) return NULL; @@ -359,7 +448,8 @@ PyMODINIT_FUNC PyInit_stringzillas(void) { } // Also keep the old comma-separated string version for backward compatibility - sz_cptr_t caps_str = sz_capabilities_to_string_implementation_(caps); + char caps_str[256]; + sz_capabilities_to_string_implementation_(caps, caps_str, sizeof(caps_str)); PyModule_AddStringConstant(m, "__capabilities_str__", caps_str); } @@ -420,5 +510,18 @@ PyMODINIT_FUNC PyInit_stringzillas(void) { return NULL; } + Py_INCREF(&SubstringsType); + if (PyModule_AddObject(m, "Substrings", (PyObject *)&SubstringsType) < 0) { + Py_XDECREF(&SubstringsType); + Py_XDECREF(&FingerprintsType); + Py_XDECREF(&SmithWatermanType); + Py_XDECREF(&NeedlemanWunschType); + Py_XDECREF(&LevenshteinDistancesUTF8Type); + Py_XDECREF(&LevenshteinDistancesType); + Py_XDECREF(&DeviceScopeType); + Py_XDECREF(m); + return NULL; + } + return m; } diff --git a/python/stringzillas/stringzillas.h b/python/stringzillas/stringzillas.h index e69b668b..e313bf47 100644 --- a/python/stringzillas/stringzillas.h +++ b/python/stringzillas/stringzillas.h @@ -133,6 +133,7 @@ extern PyTypeObject LevenshteinDistancesUTF8Type; extern PyTypeObject NeedlemanWunschType; extern PyTypeObject SmithWatermanType; extern PyTypeObject FingerprintsType; +extern PyTypeObject SubstringsType; /** Shared documentation of the `__capabilities__` getter every engine exposes. */ extern char const doc_capabilities[]; @@ -177,7 +178,14 @@ extern int parse_and_intersect_capabilities(PyObject *caps_obj, sz_capability_t * @note Sets Pythonic error on failure. */ SZ_HELPER_AUTO sz_bool_t try_swap_to_unified_allocator(PyObject *strs_obj) { - if (!strs_obj || !sz_py_replace_strings_allocator) return sz_false_k; + if (!strs_obj || !sz_py_replace_strings_allocator || !sz_py_export_strings_as_sequence) return sz_false_k; + + // Nothing but a `Strs` owns an allocator, so anything else is a type error rather than a memory one. + sz_sequence_t probe_sequence; + if (!sz_py_export_strings_as_sequence(strs_obj, &probe_sequence)) { + PyErr_Format(PyExc_TypeError, "Expected a Strs collection, received %s", Py_TYPE(strs_obj)->tp_name); + return sz_false_k; + } // Try to swap to unified allocator - this will be a no-op if already using it sz_bool_t success = sz_py_replace_strings_allocator(strs_obj, &unified_allocator); @@ -185,7 +193,7 @@ SZ_HELPER_AUTO sz_bool_t try_swap_to_unified_allocator(PyObject *strs_obj) { if (!success) { // Always fatal: GPU kernels require unified/device-accessible memory PyErr_SetString( // - PyExc_RuntimeError, + PyExc_BufferError, "Device memory mismatch: GPU kernels require unified/device-accessible memory. " // "Consider reducing input size, freeing memory, or using CPU capabilities."); return sz_false_k; @@ -194,11 +202,296 @@ SZ_HELPER_AUTO sz_bool_t try_swap_to_unified_allocator(PyObject *strs_obj) { } /** - * @brief Helper function to determine if unified memory is required based on capabilities and device scope. + * @brief Whether an engine addresses its buffers from the device, so every one of them must be reachable there. + * + * Unified memory is one way to satisfy this and plain device memory is another, so the question is device + * accessibility rather than any single allocator. + * * @param[in] capabilities The capabilities bitmask of the current engine. */ -SZ_HELPER_AUTO sz_bool_t requires_unified_memory(sz_capability_t capabilities) { +SZ_HELPER_AUTO sz_bool_t requires_device_memory(sz_capability_t capabilities) { return (capabilities & sz_cap_cuda_k) != 0; } +/** + * @brief Frees the unified allocation a NumPy array was built over, once that array is collected. + * @param[in] capsule Holds the pointer under the name `"szs_unified"`, with the byte count as its context. + */ +SZ_HELPER_AUTO void free_unified_capsule(PyObject *capsule) { + void *const allocation = PyCapsule_GetPointer(capsule, "szs_unified"); + if (allocation) + unified_allocator.free(allocation, (sz_size_t)(uintptr_t)PyCapsule_GetContext(capsule), + unified_allocator.handle); +} + +/** + * @brief A NumPy array over freshly allocated unified memory, which both the host and a kernel can address. + * + * Keeps the engines' outputs zero-copy on a GPU scope without changing what callers receive: the result is + * an ordinary `ndarray`, and the allocation dies with it. + * + * @param[in] dimensions One or two, matching @p shape. + * @param[in] element_bytes Width of one element, which must agree with @p type_number. + * @return A new reference, or NULL with a Python exception set. + */ +SZ_HELPER_AUTO PyObject *new_unified_array(int dimensions, npy_intp const *shape, int type_number, + sz_size_t element_bytes) { + sz_size_t elements = 1; + for (int axis = 0; axis < dimensions; ++axis) elements *= (sz_size_t)shape[axis]; + // An empty result still allocates one element, so the array never wraps a null pointer. + sz_size_t const total_bytes = elements ? elements * element_bytes : element_bytes; + + void *const allocation = unified_allocator.allocate(total_bytes, unified_allocator.handle); + if (!allocation) return PyErr_NoMemory(); + + PyObject *const array = PyArray_SimpleNewFromData(dimensions, (npy_intp *)shape, type_number, allocation); + if (!array) { + unified_allocator.free(allocation, total_bytes, unified_allocator.handle); + return PyErr_NoMemory(); + } + + // The array does not own foreign memory, so a capsule base carries the lifetime and the size to free. + PyObject *const owner = PyCapsule_New(allocation, "szs_unified", free_unified_capsule); + if (!owner || PyCapsule_SetContext(owner, (void *)(uintptr_t)total_bytes) != 0 || + PyArray_SetBaseObject((PyArrayObject *)array, owner) != 0) { + Py_XDECREF(owner); + Py_DECREF(array); + unified_allocator.free(allocation, total_bytes, unified_allocator.handle); + return NULL; + } + return array; +} + +/** @brief One caller-supplied device buffer, as `__cuda_array_interface__` or `__dlpack__` described it. */ +typedef struct device_buffer_t { + void *data; + sz_size_t rows; //? One for a vector, the query count for a matrix + sz_size_t columns; //? Elements per row + sz_size_t row_stride; //? Elements between consecutive rows, at least `columns` + sz_size_t element_bytes; //? Width of one element, matched against what the engine writes +} device_buffer_t; + +/** @brief The DLPack device kinds a kernel can reach; the rest are host memory as far as we are concerned. */ +enum { dlpack_device_cuda_k = 2, dlpack_device_cuda_managed_k = 13 }; + +/** @brief DLPack's tensor descriptor, as its stable C ABI lays it out. */ +typedef struct dlpack_tensor_t { + void *data; + struct { + int32_t device_type; + int32_t device_id; + } device; + int32_t dimensions; + struct { + uint8_t code; + uint8_t bits; + uint16_t lanes; + } dtype; + int64_t *shape; + int64_t *strides; //? Null means C-contiguous + uint64_t byte_offset; +} dlpack_tensor_t; + +/** @brief DLPack's owning wrapper, whose deleter this binding calls once it has read the descriptor. */ +typedef struct dlpack_managed_tensor_t { + dlpack_tensor_t tensor; + void *manager_context; + void (*deleter)(struct dlpack_managed_tensor_t *self); +} dlpack_managed_tensor_t; + +/** + * @brief Fills @p result from a shape, an element width and a row stride, refusing shapes the engines cannot write. + * + * One or two dimensions only, rows at least as wide as they are long, and a stride that is a whole number of + * elements - the C ABI expresses a row stride in elements and has no way to say "half an element". + */ +SZ_HELPER_AUTO int fill_device_buffer(device_buffer_t *result, void *data, sz_size_t dimensions, int64_t const *shape, + sz_size_t row_stride_bytes, sz_size_t element_bytes) { + if (dimensions != 1 && dimensions != 2) { + PyErr_SetString(PyExc_ValueError, "Device buffers must be 1- or 2-dimensional"); + return -1; + } + result->data = data; + result->element_bytes = element_bytes; + result->rows = dimensions == 2 ? (sz_size_t)shape[0] : 1; + result->columns = dimensions == 2 ? (sz_size_t)shape[1] : (sz_size_t)shape[0]; + if (row_stride_bytes % element_bytes != 0) { + PyErr_SetString(PyExc_ValueError, "Device buffer rows must be strided by a whole number of elements"); + return -1; + } + result->row_stride = row_stride_bytes / element_bytes; + if (result->row_stride < result->columns) { + PyErr_SetString(PyExc_ValueError, "Device buffer rows overlap; the row stride is narrower than a row"); + return -1; + } + return 0; +} + +/** + * @brief Reads a device pointer out of an object exposing `__cuda_array_interface__`. + * + * Version 2 and above, C-contiguous along the last axis, and a non-null `mask` is refused - the engines write + * dense rows and honour no mask. + * + * @param[in] buffer_obj The candidate; one exposing no interface reports `sz_false_k` with no exception set, + * so the caller can fall through to another protocol. + */ +SZ_HELPER_AUTO sz_bool_t try_read_cuda_array_interface(PyObject *buffer_obj, sz_size_t expected_element_bytes, + device_buffer_t *result) { + PyObject *interface = PyObject_GetAttrString(buffer_obj, "__cuda_array_interface__"); + if (!interface) { + PyErr_Clear(); + return sz_false_k; + } + sz_bool_t parsed = sz_false_k; + PyObject *shape_obj = NULL, *data_obj = NULL, *typestr_obj = NULL, *strides_obj = NULL, *mask_obj = NULL; + if (!PyDict_Check(interface)) { + PyErr_SetString(PyExc_TypeError, "__cuda_array_interface__ must be a dict"); + goto done; + } + + mask_obj = PyDict_GetItemString(interface, "mask"); //? Borrowed + if (mask_obj && mask_obj != Py_None) { + PyErr_SetString(PyExc_ValueError, "Masked device buffers are not supported"); + goto done; + } + shape_obj = PyDict_GetItemString(interface, "shape"); + data_obj = PyDict_GetItemString(interface, "data"); + typestr_obj = PyDict_GetItemString(interface, "typestr"); + strides_obj = PyDict_GetItemString(interface, "strides"); + if (!shape_obj || !data_obj || !typestr_obj || !PyTuple_Check(shape_obj) || !PyTuple_Check(data_obj)) { + PyErr_SetString(PyExc_ValueError, "__cuda_array_interface__ is missing shape, data or typestr"); + goto done; + } + + Py_ssize_t const dimensions = PyTuple_GET_SIZE(shape_obj); + int64_t shape[2] = {0, 0}; + if (dimensions < 1 || dimensions > 2) { + PyErr_SetString(PyExc_ValueError, "Device buffers must be 1- or 2-dimensional"); + goto done; + } + for (Py_ssize_t axis = 0; axis < dimensions; ++axis) + shape[axis] = (int64_t)PyLong_AsLongLong(PyTuple_GET_ITEM(shape_obj, axis)); + if (PyErr_Occurred()) goto done; + + void *const data = PyLong_AsVoidPtr(PyTuple_GET_ITEM(data_obj, 0)); + if (PyErr_Occurred()) goto done; + + // A dtype narrower or wider than the engine writes would silently reinterpret every cell. + char const *const typestr = PyUnicode_AsUTF8(typestr_obj); + if (!typestr) goto done; + sz_size_t const element_bytes = (sz_size_t)strtoul(typestr + 2, NULL, 10); + if (element_bytes != expected_element_bytes) { + PyErr_Format(PyExc_TypeError, "Device buffer has %zu-byte elements, expected %zu", (size_t)element_bytes, + (size_t)expected_element_bytes); + goto done; + } + + // A null `strides` is the protocol's way of saying C-contiguous, which makes the row stride the row width. + sz_size_t row_stride_bytes = (sz_size_t)shape[dimensions - 1] * element_bytes; + if (strides_obj && strides_obj != Py_None) { + if (!PyTuple_Check(strides_obj) || PyTuple_GET_SIZE(strides_obj) != dimensions) { + PyErr_SetString(PyExc_ValueError, "__cuda_array_interface__ strides must match the shape"); + goto done; + } + if ((sz_size_t)PyLong_AsSsize_t(PyTuple_GET_ITEM(strides_obj, dimensions - 1)) != element_bytes) { + PyErr_SetString(PyExc_ValueError, "Device buffer rows must be contiguous along the last axis"); + goto done; + } + if (dimensions == 2) row_stride_bytes = (sz_size_t)PyLong_AsSsize_t(PyTuple_GET_ITEM(strides_obj, 0)); + if (PyErr_Occurred()) goto done; + } + + parsed = fill_device_buffer(result, data, (sz_size_t)dimensions, shape, row_stride_bytes, element_bytes) == 0 + ? sz_true_k + : sz_false_k; + +done: + Py_DECREF(interface); + return parsed; +} + +/** + * @brief Reads a device pointer out of an object exposing `__dlpack__`, gated on `__dlpack_device__`. + * @param[in] buffer_obj The candidate; see @ref try_read_cuda_array_interface for the return contract. + */ +SZ_HELPER_AUTO sz_bool_t try_read_dlpack(PyObject *buffer_obj, sz_size_t expected_element_bytes, + device_buffer_t *result) { + PyObject *device_method = PyObject_GetAttrString(buffer_obj, "__dlpack_device__"); + if (!device_method) { + PyErr_Clear(); + return sz_false_k; + } + PyObject *const device_pair = PyObject_CallNoArgs(device_method); + Py_DECREF(device_method); + if (!device_pair) return sz_false_k; + if (!PyTuple_Check(device_pair) || PyTuple_GET_SIZE(device_pair) != 2) { + Py_DECREF(device_pair); + PyErr_SetString(PyExc_ValueError, "__dlpack_device__ must return a (device_type, device_id) pair"); + return sz_false_k; + } + long const device_type = PyLong_AsLong(PyTuple_GET_ITEM(device_pair, 0)); + Py_DECREF(device_pair); + if (device_type != dlpack_device_cuda_k && device_type != dlpack_device_cuda_managed_k) { + PyErr_SetString(PyExc_BufferError, "A GPU scope needs a CUDA device buffer; this one lives on the host"); + return sz_false_k; + } + + PyObject *const capsule = PyObject_CallMethod(buffer_obj, "__dlpack__", NULL); + if (!capsule) return sz_false_k; + dlpack_managed_tensor_t *const managed = (dlpack_managed_tensor_t *)PyCapsule_GetPointer(capsule, "dltensor"); + if (!managed) { + Py_DECREF(capsule); + return sz_false_k; + } + + dlpack_tensor_t const *const tensor = &managed->tensor; + sz_bool_t parsed = sz_false_k; + sz_size_t const element_bytes = (sz_size_t)(tensor->dtype.bits / 8) * tensor->dtype.lanes; + if (element_bytes != expected_element_bytes) + PyErr_Format(PyExc_TypeError, "Device buffer has %zu-byte elements, expected %zu", (size_t)element_bytes, + (size_t)expected_element_bytes); + else if (tensor->strides && tensor->strides[tensor->dimensions - 1] != 1) + PyErr_SetString(PyExc_ValueError, "Device buffer rows must be contiguous along the last axis"); + else { + // DLPack counts strides in elements, and a null `strides` means C-contiguous. + sz_size_t const row_stride_bytes = (tensor->dimensions == 2 && tensor->strides) + ? (sz_size_t)tensor->strides[0] * element_bytes + : (sz_size_t)tensor->shape[tensor->dimensions - 1] * element_bytes; + void *const data = (char *)tensor->data + tensor->byte_offset; + parsed = fill_device_buffer(result, data, (sz_size_t)tensor->dimensions, tensor->shape, row_stride_bytes, + element_bytes) == 0 + ? sz_true_k + : sz_false_k; + } + + // The capsule is ours once read, so rename it spent and run the deleter rather than leaking the tensor. + if (managed->deleter) managed->deleter(managed); + PyCapsule_SetName(capsule, "used_dltensor"); + Py_DECREF(capsule); + return parsed; +} + +/** + * @brief Reads a caller's device buffer through whichever protocol it speaks, refusing host memory. + * + * `__cuda_array_interface__` is tried first because it is a plain dictionary, where `__dlpack__` allocates a + * capsule that has to be consumed. + * + * @param[in] expected_element_bytes Width the engine will write; another dtype is refused rather than reinterpreted. + * @return 0 when @p result was filled, -1 with a Python exception set otherwise. + */ +SZ_HELPER_AUTO int parse_device_buffer(PyObject *buffer_obj, sz_size_t expected_element_bytes, + device_buffer_t *result) { + if (try_read_cuda_array_interface(buffer_obj, expected_element_bytes, result)) return 0; + if (PyErr_Occurred()) return -1; + if (try_read_dlpack(buffer_obj, expected_element_bytes, result)) return 0; + if (PyErr_Occurred()) return -1; + PyErr_Format(PyExc_BufferError, + "A GPU scope writes its results from the device, so '%s' must expose " // + "__cuda_array_interface__ or __dlpack__ over CUDA memory", + Py_TYPE(buffer_obj)->tp_name); + return -1; +} + #endif // STRINGZILLAS_PYTHON_STRINGZILLAS_H_ diff --git a/python/stringzillas/substrings.c b/python/stringzillas/substrings.c new file mode 100644 index 00000000..0156254d --- /dev/null +++ b/python/stringzillas/substrings.c @@ -0,0 +1,863 @@ +/** + * @brief Multi-pattern Aho-Corasick search engine over a whole dictionary of needles. + * @file python/stringzillas/substrings.c + * @author Ash Vardanian + */ +#include "stringzillas.h" + +/** + * @brief Multi-pattern search engine, compiled once from a needle set and reused across calls. + * + * Unlike the other engines here, construction is a device operation - the automaton's tier split is sized + * against the cache the walk reads through, and a CUDA automaton is uploaded to a device - so `__init__` + * takes a scope of its own. The device buffers are managed memory, so a later call may name a different + * GPU; only a CPU scope on a GPU-built engine is refused, and the engine reports that itself. + */ +typedef struct { + PyObject ob_base; + szs_substrings_t handle; + char description[64]; + sz_capability_t capabilities; + sz_size_t needles_count; //? Fixed once built; every per-needle array is validated against it. + SZS_LOCK_FIELD_ +} Substrings; + +#pragma region Argument Parsing + +/** @brief Narrows a policy name to the C enumeration, so callers never spell an integer. */ +static int parse_overlap_policy(PyObject *policy_obj, szs_substrings_overlap_policy_t *result) { + if (policy_obj == NULL || policy_obj == Py_None) { + *result = szs_substrings_overlapping_k; + return 0; + } + char const *name = PyUnicode_AsUTF8(policy_obj); + if (!name) { + PyErr_SetString(PyExc_TypeError, "overlap policy must be a string"); + return -1; + } + if (strcmp(name, "overlapping") == 0) { *result = szs_substrings_overlapping_k; } + else if (strcmp(name, "leftmost-longest") == 0) { *result = szs_substrings_leftmost_longest_k; } + else if (strcmp(name, "leftmost-first") == 0) { *result = szs_substrings_leftmost_first_k; } + else { + PyErr_Format(PyExc_ValueError, + "Unknown overlap policy '%s', expected 'overlapping', 'leftmost-longest', or 'leftmost-first'", + name); + return -1; + } + return 0; +} + +/** @brief Narrows a sensitivity name to the C enumeration. */ +static int parse_case_sensitivity(PyObject *sensitivity_obj, szs_substrings_case_sensitivity_t *result) { + if (sensitivity_obj == NULL || sensitivity_obj == Py_None) { + *result = szs_substrings_cased_k; + return 0; + } + char const *name = PyUnicode_AsUTF8(sensitivity_obj); + if (!name) { + PyErr_SetString(PyExc_TypeError, "case_sensitivity must be a string"); + return -1; + } + if (strcmp(name, "cased") == 0) { *result = szs_substrings_cased_k; } + else if (strcmp(name, "uncased") == 0) { *result = szs_substrings_uncased_k; } + else { + PyErr_Format(PyExc_ValueError, "Unknown case sensitivity '%s', expected 'cased' or 'uncased'", name); + return -1; + } + return 0; +} + +/** @brief Resolves an optional `DeviceScope` argument to its handle, or the module default. */ +static int parse_device_scope(PyObject *device_obj, DeviceScope **scope_out, szs_device_scope_t *handle_out) { + *scope_out = NULL; + if (device_obj != NULL && device_obj != Py_None) { + if (!PyObject_TypeCheck(device_obj, &DeviceScopeType)) { + PyErr_SetString(PyExc_TypeError, "device must be a DeviceScope instance"); + return -1; + } + *scope_out = (DeviceScope *)device_obj; + } + *handle_out = *scope_out ? (*scope_out)->handle : default_device_scope; + return 0; +} + +/** + * @brief Points @p kernels at whichever haystack shape @p texts_obj carries, or fails with a type error. + * + * The probe order matches every other engine here: the two tape layouts hand their offsets straight to the + * kernel, and the callback-addressed sequence is the fallback a reordered `Strs` takes. + */ +typedef struct { + void *punned; + sz_size_t count; + int is_u32tape; + int is_u64tape; + sz_sequence_u32tape_t u32tape; + sz_sequence_u64tape_t u64tape; + sz_sequence_t sequence; +} haystacks_view_t; + +/** + * @brief Views a `Strs` as whichever shape the engine takes, re-pinning it to unified memory first when the + * engine holds CUDA capabilities. + * + * The sequence exporter accepts every layout, so it doubles as the type test - and it has to run before the + * swap, which reallocates the tape a pointer-capturing view would otherwise be left pointing into. + */ +static int parse_haystacks(PyObject *texts_obj, sz_capability_t capabilities, haystacks_view_t *view) { + if (!sz_py_export_strings_as_sequence(texts_obj, &view->sequence)) { + PyErr_Format(PyExc_TypeError, + "Expected stringzilla.Strs object, got %s. Convert using: stringzilla.Strs(your_string_list)", + Py_TYPE(texts_obj)->tp_name); + return -1; + } + if (requires_device_memory(capabilities)) + if (!try_swap_to_unified_allocator(texts_obj)) return -1; + + view->is_u32tape = sz_py_export_strings_as_u32tape( // + texts_obj, &view->u32tape.data, &view->u32tape.offsets, &view->u32tape.count); + if (view->is_u32tape) { + view->punned = &view->u32tape; + view->count = view->u32tape.count; + return 0; + } + view->is_u64tape = sz_py_export_strings_as_u64tape( // + texts_obj, &view->u64tape.data, &view->u64tape.offsets, &view->u64tape.count); + if (view->is_u64tape) { + view->punned = &view->u64tape; + view->count = view->u64tape.count; + return 0; + } + view->punned = &view->sequence; + view->count = view->sequence.count; + return 0; +} + +#pragma endregion Argument Parsing + +#pragma region Lifetime + +static void Substrings_dealloc(Substrings *self) { + if (self->handle) { + szs_substrings_free(self->handle); + self->handle = NULL; + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject *Substrings_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + Substrings *self = (Substrings *)type->tp_alloc(type, 0); + if (self != NULL) { + self->handle = NULL; + self->description[0] = '\0'; + self->capabilities = 0; + self->needles_count = 0; + } + return (PyObject *)self; +} + +static int Substrings_init(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *needles_obj = NULL, *sensitivity_obj = NULL, *device_obj = NULL, *capabilities_tuple = NULL; + sz_capability_t capabilities = active_capabilities_; + + static char *kwlist[] = {"needles", "case_sensitivity", "device", "capabilities", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOO", kwlist, &needles_obj, &sensitivity_obj, &device_obj, + &capabilities_tuple)) + return -1; + + if (capabilities_tuple) + if (parse_and_intersect_capabilities(capabilities_tuple, &capabilities) != 0) return -1; + + szs_substrings_case_sensitivity_t sensitivity; + if (parse_case_sensitivity(sensitivity_obj, &sensitivity) != 0) return -1; + + DeviceScope *device_scope = NULL; + szs_device_scope_t device_handle = NULL; + if (parse_device_scope(device_obj, &device_scope, &device_handle) != 0) return -1; + + // The mask alone picks the backend, so a CPU scope must not leave the CUDA bit standing: the engine would + // be built for a device the operations then refuse to run on. + if (device_handle) { + sz_capability_t scope_capabilities = 0; + char const *scope_error = NULL; + sz_status_t const scope_status = szs_device_scope_get_capabilities(device_handle, &scope_capabilities, + &scope_error); + if (scope_status != sz_success_k) { + set_stringzilla_error(scope_status, scope_error, "Device scope capabilities"); + return -1; + } + capabilities &= scope_capabilities; + } + + // Needles reach the engine as a callback-addressed sequence whatever layout they arrived in - the + // automaton has no tape overload, since it walks the needles once at construction and never again. + sz_sequence_t needles_sequence; + if (!sz_py_export_strings_as_sequence(needles_obj, &needles_sequence)) { + PyErr_Format(PyExc_TypeError, + "Expected stringzilla.Strs object, got %s. Convert using: stringzilla.Strs(your_string_list)", + Py_TYPE(needles_obj)->tp_name); + return -1; + } + if (requires_device_memory(capabilities)) + if (!try_swap_to_unified_allocator(needles_obj)) return -1; + + char const *error_detail = NULL; + sz_status_t status = szs_substrings_init(NULL, capabilities, &self->handle, &error_detail); + if (status != sz_success_k) { + set_stringzilla_error(status, error_detail, "Substrings construction"); + return -1; + } + + // Indexing is where the automaton is tiered against a device, so it takes the scope's lock like every + // operation does. A failure leaves `needles_count` at zero and the handle for `dealloc` to release. + if (device_scope) SZS_LOCK_(&device_scope->lock); + status = szs_substrings_index(self->handle, &needles_sequence, sensitivity, device_handle, &error_detail); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k) { + set_stringzilla_error(status, error_detail, "Substrings indexing"); + return -1; + } + + self->capabilities = capabilities; + self->needles_count = needles_sequence.count; + snprintf(self->description, sizeof(self->description), "%zu needles, %s", (size_t)self->needles_count, + sensitivity == szs_substrings_uncased_k ? "uncased" : "cased"); + return 0; +} + +static PyObject *Substrings_repr(Substrings *self) { + return PyUnicode_FromFormat("stringzillas.Substrings(%s)", self->description); +} + +static PyObject *Substrings_get_capabilities(Substrings *self, void *closure) { + return capabilities_to_tuple(self->capabilities); +} + +#pragma endregion Lifetime + +#pragma region Operations + +static PyObject *Substrings_count(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *haystacks_obj = NULL, *device_obj = NULL, *policy_obj = NULL; + static char *kwlist[] = {"haystacks", "device", "policy", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OO", kwlist, &haystacks_obj, &device_obj, &policy_obj)) + return NULL; + + szs_substrings_overlap_policy_t policy; + if (parse_overlap_policy(policy_obj, &policy) != 0) return NULL; + DeviceScope *device_scope = NULL; + szs_device_scope_t device_handle = NULL; + if (parse_device_scope(device_obj, &device_scope, &device_handle) != 0) return NULL; + + sz_bool_t const need_unified = requires_device_memory(self->capabilities); + haystacks_view_t haystacks; + if (parse_haystacks(haystacks_obj, self->capabilities, &haystacks) != 0) return NULL; + + npy_intp dims[1] = {(npy_intp)haystacks.count}; + PyArrayObject *counts_array = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINT64); + if (!counts_array) return PyErr_NoMemory(); + + sz_memory_allocator_t *out_alloc = need_unified ? &unified_allocator : &default_allocator; + sz_size_t const counts_bytes = haystacks.count * sizeof(sz_size_t); + sz_size_t matches_total = 0; + if (counts_bytes > 0) { + sz_size_t *counts = (sz_size_t *)out_alloc->allocate(counts_bytes, out_alloc->handle); + if (!counts) { + Py_DECREF(counts_array); + return PyErr_NoMemory(); + } + + char const *error_detail = NULL; + if (device_scope) SZS_LOCK_(&device_scope->lock); + SZS_LOCK_(&self->lock); + sz_status_t status = haystacks.is_u32tape + ? szs_substrings_count_u32tape(self->handle, device_handle, &haystacks.u32tape, policy, + counts, &matches_total, &error_detail) + : (haystacks.is_u64tape + ? szs_substrings_count_u64tape(self->handle, device_handle, &haystacks.u64tape, + policy, counts, &matches_total, &error_detail) + : szs_substrings_count(self->handle, device_handle, &haystacks.sequence, policy, + counts, &matches_total, &error_detail)); + SZS_UNLOCK_(&self->lock); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k) { + out_alloc->free(counts, counts_bytes, out_alloc->handle); + Py_DECREF(counts_array); + set_stringzilla_error(status, error_detail, "Substrings counting"); + return NULL; + } + + memcpy(PyArray_DATA(counts_array), counts, counts_bytes); + out_alloc->free(counts, counts_bytes, out_alloc->handle); + } + + PyObject *result = PyTuple_New(2); + if (!result) { + Py_DECREF(counts_array); + return NULL; + } + PyTuple_SET_ITEM(result, 0, (PyObject *)counts_array); + PyTuple_SET_ITEM(result, 1, PyLong_FromSize_t((size_t)matches_total)); + return result; +} + +static PyObject *Substrings_find(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *haystacks_obj = NULL, *device_obj = NULL, *policy_obj = NULL; + static char *kwlist[] = {"haystacks", "device", "policy", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OO", kwlist, &haystacks_obj, &device_obj, &policy_obj)) + return NULL; + + szs_substrings_overlap_policy_t policy; + if (parse_overlap_policy(policy_obj, &policy) != 0) return NULL; + DeviceScope *device_scope = NULL; + szs_device_scope_t device_handle = NULL; + if (parse_device_scope(device_obj, &device_scope, &device_handle) != 0) return NULL; + + sz_bool_t const need_unified = requires_device_memory(self->capabilities); + haystacks_view_t haystacks; + if (parse_haystacks(haystacks_obj, self->capabilities, &haystacks) != 0) return NULL; + + // A zero capacity is the engine's own size query: it reports what it wanted and writes nothing, so the + // buffer below is sized from the walk rather than from a guess the caller would have to make. + char const *error_detail = NULL; + sz_size_t matches_found = 0; + if (device_scope) SZS_LOCK_(&device_scope->lock); + SZS_LOCK_(&self->lock); + sz_status_t status = haystacks.is_u32tape + ? szs_substrings_find_u32tape(self->handle, device_handle, &haystacks.u32tape, policy, + NULL, 0, &matches_found, &error_detail) + : (haystacks.is_u64tape + ? szs_substrings_find_u64tape(self->handle, device_handle, &haystacks.u64tape, + policy, NULL, 0, &matches_found, &error_detail) + : szs_substrings_find(self->handle, device_handle, &haystacks.sequence, policy, + NULL, 0, &matches_found, &error_detail)); + SZS_UNLOCK_(&self->lock); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k && status != sz_unexpected_dimensions_k) { + set_stringzilla_error(status, error_detail, "Substrings sizing"); + return NULL; + } + + npy_intp dims[1] = {(npy_intp)matches_found}; + PyArrayObject *haystack_indices = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINT64); + PyArrayObject *needle_indices = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINT64); + PyArrayObject *byte_offsets = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINT64); + PyArrayObject *byte_lengths = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINT64); + if (!haystack_indices || !needle_indices || !byte_offsets || !byte_lengths) { + Py_XDECREF(haystack_indices); + Py_XDECREF(needle_indices); + Py_XDECREF(byte_offsets); + Py_XDECREF(byte_lengths); + return PyErr_NoMemory(); + } + + sz_memory_allocator_t *out_alloc = need_unified ? &unified_allocator : &default_allocator; + sz_size_t const matches_bytes = matches_found * sizeof(szs_substrings_match_t); + if (matches_bytes > 0) { + szs_substrings_match_t *matches = (szs_substrings_match_t *)out_alloc->allocate(matches_bytes, + out_alloc->handle); + if (!matches) { + Py_DECREF(haystack_indices); + Py_DECREF(needle_indices); + Py_DECREF(byte_offsets); + Py_DECREF(byte_lengths); + return PyErr_NoMemory(); + } + + sz_size_t written = 0; + if (device_scope) SZS_LOCK_(&device_scope->lock); + SZS_LOCK_(&self->lock); + status = haystacks.is_u32tape + ? szs_substrings_find_u32tape(self->handle, device_handle, &haystacks.u32tape, policy, matches, + matches_found, &written, &error_detail) + : (haystacks.is_u64tape + ? szs_substrings_find_u64tape(self->handle, device_handle, &haystacks.u64tape, policy, + matches, matches_found, &written, &error_detail) + : szs_substrings_find(self->handle, device_handle, &haystacks.sequence, policy, matches, + matches_found, &written, &error_detail)); + SZS_UNLOCK_(&self->lock); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k) { + out_alloc->free(matches, matches_bytes, out_alloc->handle); + Py_DECREF(haystack_indices); + Py_DECREF(needle_indices); + Py_DECREF(byte_offsets); + Py_DECREF(byte_lengths); + set_stringzilla_error(status, error_detail, "Substrings search"); + return NULL; + } + + // One column per field, so a caller wanting only the needle identities takes that array alone. + sz_u64_t *haystack_data = (sz_u64_t *)PyArray_DATA(haystack_indices); + sz_u64_t *needle_data = (sz_u64_t *)PyArray_DATA(needle_indices); + sz_u64_t *offset_data = (sz_u64_t *)PyArray_DATA(byte_offsets); + sz_u64_t *length_data = (sz_u64_t *)PyArray_DATA(byte_lengths); + for (sz_size_t index = 0; index < written; ++index) { + haystack_data[index] = (sz_u64_t)matches[index].haystack_index; + needle_data[index] = (sz_u64_t)matches[index].needle_index; + offset_data[index] = (sz_u64_t)matches[index].byte_offset; + length_data[index] = (sz_u64_t)matches[index].byte_length; + } + out_alloc->free(matches, matches_bytes, out_alloc->handle); + } + + PyObject *result = PyTuple_New(4); + if (!result) { + Py_DECREF(haystack_indices); + Py_DECREF(needle_indices); + Py_DECREF(byte_offsets); + Py_DECREF(byte_lengths); + return NULL; + } + PyTuple_SET_ITEM(result, 0, (PyObject *)haystack_indices); + PyTuple_SET_ITEM(result, 1, (PyObject *)needle_indices); + PyTuple_SET_ITEM(result, 2, (PyObject *)byte_offsets); + PyTuple_SET_ITEM(result, 3, (PyObject *)byte_lengths); + return result; +} + +static PyObject *Substrings_score_bm25(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *haystacks_obj = NULL, *weights_obj = NULL, *device_obj = NULL, *lengths_obj = NULL; + // The corpus mean is a property of the corpus, not a tunable, so it has no default and stays required; + // `length_normalization=0.0` is how a caller opts out of needing one, and the engine refuses the pair + // that asks to normalize without a mean. + double average_document_length = 0.0; + double term_frequency_saturation = 1.2, length_normalization = 0.75; + static char *kwlist[] = {"haystacks", + "needle_weights", + "average_document_length", + "device", + "document_lengths", + "term_frequency_saturation", + "length_normalization", + NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOd|OOdd", kwlist, &haystacks_obj, &weights_obj, + &average_document_length, &device_obj, &lengths_obj, &term_frequency_saturation, + &length_normalization)) + return NULL; + + DeviceScope *device_scope = NULL; + szs_device_scope_t device_handle = NULL; + if (parse_device_scope(device_obj, &device_scope, &device_handle) != 0) return NULL; + + if (!numpy_available || !PyArray_Check(weights_obj)) { + PyErr_SetString(PyExc_TypeError, "needle_weights must be a NumPy array of float32, one entry per needle"); + return NULL; + } + PyArrayObject *weights_array = (PyArrayObject *)weights_obj; + if (PyArray_TYPE(weights_array) != NPY_FLOAT32 || PyArray_NDIM(weights_array) != 1 || + !PyArray_ISCARRAY_RO(weights_array)) { + PyErr_SetString(PyExc_TypeError, "needle_weights must be a contiguous 1-D float32 array"); + return NULL; + } + if ((sz_size_t)PyArray_DIM(weights_array, 0) != self->needles_count) { + PyErr_Format(PyExc_ValueError, "Expected one weight per needle: %zu given, %zu needed", + (size_t)PyArray_DIM(weights_array, 0), (size_t)self->needles_count); + return NULL; + } + + sz_bool_t const need_unified = requires_device_memory(self->capabilities); + haystacks_view_t haystacks; + if (parse_haystacks(haystacks_obj, self->capabilities, &haystacks) != 0) return NULL; + + // Validated after the haystacks, because one length per haystack is the shape the kernel reads and a + // shorter array would otherwise be read past its end. + sz_f32_t const *document_lengths = NULL; + if (lengths_obj != NULL && lengths_obj != Py_None) { + if (!PyArray_Check(lengths_obj) || PyArray_TYPE((PyArrayObject *)lengths_obj) != NPY_FLOAT32 || + PyArray_NDIM((PyArrayObject *)lengths_obj) != 1 || !PyArray_ISCARRAY_RO((PyArrayObject *)lengths_obj)) { + PyErr_SetString(PyExc_TypeError, "document_lengths must be a contiguous 1-D float32 array"); + return NULL; + } + if ((sz_size_t)PyArray_DIM((PyArrayObject *)lengths_obj, 0) != haystacks.count) { + PyErr_Format(PyExc_ValueError, "Expected one length per haystack: %zu given, %zu needed", + (size_t)PyArray_DIM((PyArrayObject *)lengths_obj, 0), (size_t)haystacks.count); + return NULL; + } + document_lengths = (sz_f32_t const *)PyArray_DATA((PyArrayObject *)lengths_obj); + } + + npy_intp dims[1] = {(npy_intp)haystacks.count}; + PyArrayObject *scores_array = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT32); + if (!scores_array) return PyErr_NoMemory(); + + szs_substrings_bm25_t parameters; + parameters.term_frequency_saturation = (sz_f32_t)term_frequency_saturation; + parameters.length_normalization = (sz_f32_t)length_normalization; + parameters.average_document_length = (sz_f32_t)average_document_length; + + sz_memory_allocator_t *out_alloc = need_unified ? &unified_allocator : &default_allocator; + sz_size_t const scores_bytes = haystacks.count * sizeof(sz_f32_t); + if (scores_bytes > 0) { + sz_f32_t *scores = (sz_f32_t *)out_alloc->allocate(scores_bytes, out_alloc->handle); + if (!scores) { + Py_DECREF(scores_array); + return PyErr_NoMemory(); + } + + sz_f32_t const *weights = (sz_f32_t const *)PyArray_DATA(weights_array); + char const *error_detail = NULL; + if (device_scope) SZS_LOCK_(&device_scope->lock); + SZS_LOCK_(&self->lock); + sz_status_t status = + haystacks.is_u32tape + ? szs_substrings_score_bm25_u32tape(self->handle, device_handle, &haystacks.u32tape, document_lengths, + parameters, weights, scores, &error_detail) + : (haystacks.is_u64tape + ? szs_substrings_score_bm25_u64tape(self->handle, device_handle, &haystacks.u64tape, + document_lengths, parameters, weights, scores, &error_detail) + : szs_substrings_score_bm25(self->handle, device_handle, &haystacks.sequence, document_lengths, + parameters, weights, scores, &error_detail)); + SZS_UNLOCK_(&self->lock); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k) { + out_alloc->free(scores, scores_bytes, out_alloc->handle); + Py_DECREF(scores_array); + set_stringzilla_error(status, error_detail, "Substrings BM25 scoring"); + return NULL; + } + + memcpy(PyArray_DATA(scores_array), scores, scores_bytes); + out_alloc->free(scores, scores_bytes, out_alloc->handle); + } + + return (PyObject *)scores_array; +} + +/** @brief What a rewrite of nothing returns: empty bytes beside the one boundary an empty tape carries. */ +static PyObject *empty_rewrite_result(void) { + npy_intp dims[1] = {1}; + PyArrayObject *offsets_array = (PyArrayObject *)PyArray_SimpleNew(1, dims, NPY_UINTP); + if (!offsets_array) return PyErr_NoMemory(); + *(sz_size_t *)PyArray_DATA(offsets_array) = 0; + + PyObject *rewritten = PyBytes_FromStringAndSize(NULL, 0); + if (!rewritten) { + Py_DECREF(offsets_array); + return NULL; + } + PyObject *result = PyTuple_New(2); + if (!result) { + Py_DECREF(rewritten); + Py_DECREF(offsets_array); + return NULL; + } + PyTuple_SET_ITEM(result, 0, rewritten); + PyTuple_SET_ITEM(result, 1, (PyObject *)offsets_array); + return result; +} + +static PyObject *Substrings_replace(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *haystacks_obj = NULL, *replacements_obj = NULL, *device_obj = NULL, *policy_obj = NULL; + static char *kwlist[] = {"haystacks", "replacements", "device", "policy", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|OO", kwlist, &haystacks_obj, &replacements_obj, &device_obj, + &policy_obj)) + return NULL; + + // A rewrite needs a cover whose matches share no bytes, so the default is the leftmost one rather than + // the overlapping walk the search entry points default to. An overlapping rewrite is not a function, so + // naming one is an argument error here rather than an engine refusal further down. + szs_substrings_overlap_policy_t policy = szs_substrings_leftmost_longest_k; + if (policy_obj != NULL && policy_obj != Py_None) + if (parse_overlap_policy(policy_obj, &policy) != 0) return NULL; + if (policy == szs_substrings_overlapping_k) { + PyErr_SetString(PyExc_ValueError, "Rewriting needs a cover whose matches share no bytes: " // + "policy must be 'leftmost-longest' or 'leftmost-first'"); + return NULL; + } + + DeviceScope *device_scope = NULL; + szs_device_scope_t device_handle = NULL; + if (parse_device_scope(device_obj, &device_scope, &device_handle) != 0) return NULL; + + sz_sequence_t replacements_sequence; + if (!sz_py_export_strings_as_sequence(replacements_obj, &replacements_sequence)) { + PyErr_Format(PyExc_TypeError, + "Expected stringzilla.Strs object, got %s. Convert using: stringzilla.Strs(your_string_list)", + Py_TYPE(replacements_obj)->tp_name); + return NULL; + } + + sz_bool_t const need_unified = requires_device_memory(self->capabilities); + haystacks_view_t haystacks; + if (parse_haystacks(haystacks_obj, self->capabilities, &haystacks) != 0) return NULL; + + // An empty batch is a legal batch with nothing to rewrite, and its layout is not observable - an empty + // `Strs` is fragmented until an allocator swap turns it into a tape whose offsets are still NULL - so it + // is answered here rather than being sorted into a layout that then reads a trailing offset that is not + // there. + if (haystacks.count == 0) return empty_rewrite_result(); + + if (haystacks.is_u32tape == 0 && haystacks.is_u64tape == 0) { + PyErr_SetString(PyExc_TypeError, + "Rewriting takes a tape in and produces a tape out, so a reordered Strs is not accepted"); + return NULL; + } + + // The bound is arithmetic over the needle set, so one call sizes the tape and the rewrite below cannot + // then be refused for capacity. + sz_size_t input_bytes = haystacks.is_u32tape ? (sz_size_t)haystacks.u32tape.offsets[haystacks.u32tape.count] + : (sz_size_t)haystacks.u64tape.offsets[haystacks.u64tape.count]; + sz_size_t output_bound = 0; + char const *error_detail = NULL; + SZS_LOCK_(&self->lock); + sz_status_t status = szs_substrings_replace_bound(self->handle, &replacements_sequence, input_bytes, &output_bound, + &error_detail); + SZS_UNLOCK_(&self->lock); + if (status != sz_success_k) { + set_stringzilla_error(status, error_detail, "Substrings rewrite bound"); + return NULL; + } + + // The boundaries are written from the device on a GPU scope, so they come back as a device matrix the + // caller can hand onward; a CPU scope keeps returning NumPy. + PyObject *offsets_array = NULL; + sz_size_t *output_offsets = NULL; + if (need_unified) { + npy_intp offsets_dims[1] = {(npy_intp)(haystacks.count + 1)}; + offsets_array = new_unified_array(1, offsets_dims, NPY_UINTP, sizeof(sz_size_t)); + if (!offsets_array) return NULL; + output_offsets = (sz_size_t *)PyArray_DATA((PyArrayObject *)offsets_array); + } + else { + npy_intp offsets_dims[1] = {(npy_intp)(haystacks.count + 1)}; + offsets_array = PyArray_SimpleNew(1, offsets_dims, NPY_UINTP); + if (!offsets_array) return PyErr_NoMemory(); + output_offsets = (sz_size_t *)PyArray_DATA((PyArrayObject *)offsets_array); + } + + sz_memory_allocator_t *out_alloc = need_unified ? &unified_allocator : &default_allocator; + sz_ptr_t output_data = (sz_ptr_t)out_alloc->allocate(output_bound ? output_bound : 1, out_alloc->handle); + if (!output_data) { + Py_DECREF(offsets_array); + return PyErr_NoMemory(); + } + + sz_size_t output_bytes_written = 0; + if (device_scope) SZS_LOCK_(&device_scope->lock); + SZS_LOCK_(&self->lock); + status = haystacks.is_u32tape + ? szs_substrings_replace_u32tape(self->handle, device_handle, &haystacks.u32tape, policy, + &replacements_sequence, output_data, output_bound, output_offsets, + &output_bytes_written, &error_detail) + : szs_substrings_replace_u64tape(self->handle, device_handle, &haystacks.u64tape, policy, + &replacements_sequence, output_data, output_bound, output_offsets, + &output_bytes_written, &error_detail); + SZS_UNLOCK_(&self->lock); + if (device_scope) SZS_UNLOCK_(&device_scope->lock); + if (status != sz_success_k) { + out_alloc->free(output_data, output_bound ? output_bound : 1, out_alloc->handle); + Py_DECREF(offsets_array); + set_stringzilla_error(status, error_detail, "Substrings rewriting"); + return NULL; + } + + PyObject *rewritten = PyBytes_FromStringAndSize(output_data, (Py_ssize_t)output_bytes_written); + out_alloc->free(output_data, output_bound ? output_bound : 1, out_alloc->handle); + if (!rewritten) { + Py_DECREF(offsets_array); + return NULL; + } + + PyObject *result = PyTuple_New(2); + if (!result) { + Py_DECREF(rewritten); + Py_DECREF(offsets_array); + return NULL; + } + PyTuple_SET_ITEM(result, 0, rewritten); + PyTuple_SET_ITEM(result, 1, (PyObject *)offsets_array); + return result; +} + +static PyObject *Substrings_replace_bound(Substrings *self, PyObject *args, PyObject *kwargs) { + PyObject *replacements_obj = NULL; + sz_size_t input_bytes = 0; + static char *kwlist[] = {"replacements", "input_bytes", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "On", kwlist, &replacements_obj, &input_bytes)) return NULL; + + sz_sequence_t replacements_sequence; + if (!sz_py_export_strings_as_sequence(replacements_obj, &replacements_sequence)) { + PyErr_Format(PyExc_TypeError, + "Expected stringzilla.Strs object, got %s. Convert using: stringzilla.Strs(your_string_list)", + Py_TYPE(replacements_obj)->tp_name); + return NULL; + } + + sz_size_t output_bound = 0; + char const *error_detail = NULL; + SZS_LOCK_(&self->lock); + sz_status_t status = szs_substrings_replace_bound(self->handle, &replacements_sequence, input_bytes, &output_bound, + &error_detail); + SZS_UNLOCK_(&self->lock); + if (status != sz_success_k) { + set_stringzilla_error(status, error_detail, "Substrings rewrite bound"); + return NULL; + } + return PyLong_FromSize_t((size_t)output_bound); +} + +#pragma endregion Operations + +#pragma region Type Registration + +static char const doc_count[] = // + "count(haystacks, device=None, policy='overlapping')\n" "\n" "Count matches of every needle in every haystack.\n" "\n" "Args:\n" " haystacks (Strs): Haystack collection to scan.\n" " device (DeviceScope, optional): Execution scope. Defaults to the module scope.\n" " policy (str, optional): 'overlapping', 'leftmost-longest', or 'leftmost-first'.\n" "\n" "Returns:\n" " tuple[numpy.ndarray, int]: Per-haystack counts as uint64, and the corpus-wide total.\n" "\n" "Examples:\n" // + " >>> import stringzilla as sz, stringzillas as szs\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']))\n" // + " >>> counts, total = engine.count(sz.Strs(['hershey', 'nothing']))\n" // + " >>> total\n" // + " 3\n" // + " >>> # GPU example; falls back to CPU when CUDA is unavailable\n" // + " >>> scope = szs.DeviceScope(gpu_device=0) if 'cuda' in szs.__capabilities__ else szs.DeviceScope()\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']), device=scope)\n" // + " >>> counts, total = engine.count(sz.Strs(['hershey', 'nothing']), device=scope)"; + +static char const doc_find[] = // + "find(haystacks, device=None, policy='overlapping')\n" "\n" "Locate every match, resolved under `policy`.\n" "\n" "Args:\n" " haystacks (Strs): Haystack collection to scan.\n" " device (DeviceScope, optional): Execution scope. Defaults to the module scope.\n" " policy (str, optional): 'overlapping', 'leftmost-longest', or 'leftmost-first'.\n" "\n" "Returns:\n" " tuple: Four uint64 arrays - haystack indices, needle indices, byte offsets, byte lengths.\n" "\n" "Examples:\n" // + " >>> import stringzilla as sz, stringzillas as szs\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']))\n" // + " >>> haystack_ix, needle_ix, offsets, lengths = engine.find(sz.Strs(['hershey']))\n" // + " >>> len(offsets)\n" // + " 3\n" // + " >>> # GPU example; falls back to CPU when CUDA is unavailable\n" // + " >>> scope = szs.DeviceScope(gpu_device=0) if 'cuda' in szs.__capabilities__ else szs.DeviceScope()\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']), device=scope)\n" // + " >>> haystack_ix, needle_ix, offsets, lengths = engine.find(sz.Strs(['hershey']), device=scope)"; + +static char const doc_score_bm25[] = // + "score_bm25(haystacks, needle_weights, average_document_length, device=None,\n" // + " document_lengths=None, term_frequency_saturation=1.2, length_normalization=0.75)\n" // + "\n" // + "Score every haystack against the dictionary, which is the query.\n" // + "\n" // + "Term frequencies are raw overlapping counts, which is classic BM25, so no overlap policy applies.\n" // + "\n" // + "Args:\n" // + " haystacks (Strs): Documents to score.\n" // + " needle_weights (numpy.ndarray): One float32 IDF or boost per needle. A GPU scope reads this on\n" // + " the device, so allocate it with stringzillas.unified_array; a host array raises BufferError.\n" // + " average_document_length (float): Corpus-wide mean of the lengths below, in the same unit. A\n" // + " property of the corpus rather than a tunable, so it has no default; pass 0.0 only alongside\n" // + " length_normalization=0.0, since normalizing without a mean is refused.\n" // + " device (DeviceScope, optional): Execution scope. Defaults to the module scope.\n" // + " document_lengths (numpy.ndarray, optional): One float32 length per haystack; byte lengths when\n" // + " None. Held to the same device rule as needle_weights.\n" // + " term_frequency_saturation (float, optional): The literature's k1, how slowly repeated occurrences\n" // + " stop adding score.\n" // + " length_normalization (float, optional): The literature's b, in [0, 1]; 0 ignores document length\n" // + " and every length input with it, 1 normalizes fully.\n" // + "\n" // + "Returns:\n" // + " numpy.ndarray: One float32 score per haystack.\n" // + "\n" // + "Examples:\n" // + " >>> import numpy as np, stringzilla as sz, stringzillas as szs\n" // + " >>> engine = szs.Substrings(sz.Strs(['cat', 'dog']))\n" // + " >>> weights = np.ones(2, dtype=np.float32)\n" // + " >>> scores = engine.score_bm25(sz.Strs(['cat and dog', 'nothing here']), weights, 10.0)\n" // + " >>> float(scores[1])\n" // + " 0.0"; + +static char const doc_replace[] = // + "replace(haystacks, replacements, device=None, policy='leftmost-longest')\n" // + "\n" // + "Rewrite every haystack, substituting each match with its needle's replacement.\n" // + "\n" // + "Tape in, tape out: a rewrite's product is itself a tape, so a reordered `Strs` is refused. The output\n" // + "buffer is sized from `replace_bound`, so the call cannot be refused for capacity.\n" // + "\n" // + "Args:\n" // + " haystacks (Strs): Haystacks to rewrite, in a tape layout.\n" // + " replacements (Strs): One replacement per needle.\n" // + " device (DeviceScope, optional): Execution scope. Defaults to the module scope.\n" // + " policy (str, optional): 'leftmost-longest' or 'leftmost-first'; overlapping is refused.\n" // + "\n" // + "Returns:\n" // + " tuple[bytes, numpy.ndarray]: The rewritten tape and its uintp offsets, one per haystack plus one.\n" // + "\n" // + "Examples:\n" // + " >>> import stringzilla as sz, stringzillas as szs\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']))\n" // + " >>> tape, offsets = engine.replace(sz.Strs(['hershey']), sz.Strs(['HE', 'SHE']))\n" // + " >>> bytes(tape[offsets[0]:offsets[1]])\n" // + " b'HErSHEy'\n" // + " >>> # GPU example; falls back to CPU when CUDA is unavailable\n" // + " >>> scope = szs.DeviceScope(gpu_device=0) if 'cuda' in szs.__capabilities__ else szs.DeviceScope()\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she']), device=scope)\n" // + " >>> tape, offsets = engine.replace(sz.Strs(['hershey']), sz.Strs(['HE', 'SHE']), device=scope)"; + +static char const doc_replace_bound[] = // + "replace_bound(replacements, input_bytes)\n" // + "\n" // + "Bound the bytes a rewrite can produce, from the dictionary and replacements alone.\n" // + "\n" // + "Needs no haystacks, no walk, and no device: the answer is arithmetic over the needle set.\n" // + "\n" // + "Returns:\n" // + " int: Bytes an output tape must hold to accept any such rewrite."; + +static PyMethodDef Substrings_methods[] = { + {"count", (PyCFunction)Substrings_count, SZ_METHOD_FLAGS, doc_count}, + {"find", (PyCFunction)Substrings_find, SZ_METHOD_FLAGS, doc_find}, + {"score_bm25", (PyCFunction)Substrings_score_bm25, SZ_METHOD_FLAGS, doc_score_bm25}, + {"replace", (PyCFunction)Substrings_replace, SZ_METHOD_FLAGS, doc_replace}, + {"replace_bound", (PyCFunction)Substrings_replace_bound, SZ_METHOD_FLAGS, doc_replace_bound}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static char const doc_Substrings[] = // + "Substrings(needles, case_sensitivity='cased', device=None, capabilities=None)\n" // + "\n" // + "Match a whole dictionary of needles against a whole collection of haystacks in one pass.\n" // + "\n" // + "The needle set is compiled once into an Aho-Corasick automaton and reused across every later call,\n" // + "so the dictionary is paid for once rather than per haystack. Matches are reported under a\n" // + "caller-chosen overlap policy, and the same automaton also scores haystacks with BM25 and rewrites\n" // + "them.\n" // + "\n" // + "Construction is a device operation: the automaton's tier split is sized against the cache the walk\n" // + "reads through, and a CUDA automaton is uploaded to a device, so `device` belongs here as well as on\n" // + "each call.\n" // + "\n" // + "Args:\n" // + " needles (Strs): Needle collection to compile. Non-empty, and valid UTF-8 when folding.\n" // + " case_sensitivity (str, optional): 'cased' matches bytes, 'uncased' folds both sides.\n" // + " device (DeviceScope, optional): Scope the automaton is built for and uploaded to.\n" // + " capabilities (tuple, optional): Hardware capabilities to restrict the engine to.\n" // + "\n" // + "On a GPU scope every array an operation reads or writes lives on the device. `Strs` inputs and the\n" // + "arrays these methods return are moved there for you; only `score_bm25`'s weights and lengths must\n" // + "already be CUDA buffers - stringzillas.unified_array makes one - and a host array raises\n" // + "BufferError.\n" // + "\n" // + "Examples:\n" // + " >>> import stringzilla as sz, stringzillas as szs\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she', 'his', 'hers']))\n" // + " >>> counts, total = engine.count(sz.Strs(['ushers', 'nothing']))\n" // + " >>> total\n" // + " 3\n" // + " >>> # GPU example; falls back to CPU when CUDA is unavailable\n" // + " >>> scope = szs.DeviceScope(gpu_device=0) if 'cuda' in szs.__capabilities__ else szs.DeviceScope()\n" // + " >>> engine = szs.Substrings(sz.Strs(['he', 'she', 'his', 'hers']), device=scope)\n" // + " >>> counts, total = engine.count(sz.Strs(['ushers', 'nothing']), device=scope)"; + +static PyGetSetDef Substrings_getsetters[] = { + {"__capabilities__", (getter)Substrings_get_capabilities, NULL, doc_capabilities, NULL}, // + {NULL} /* Sentinel */ +}; + +PyTypeObject SubstringsType = { + PyVarObject_HEAD_INIT(NULL, 0).tp_name = "stringzillas.Substrings", + .tp_doc = doc_Substrings, + .tp_basicsize = sizeof(Substrings), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = Substrings_new, + .tp_init = (initproc)Substrings_init, + .tp_methods = Substrings_methods, + .tp_getset = Substrings_getsetters, + .tp_repr = (reprfunc)Substrings_repr, + .tp_dealloc = (destructor)Substrings_dealloc, +}; + +#pragma endregion Type Registration diff --git a/rust/README.md b/rust/README.md index 87f2315a..2554f9fd 100644 --- a/rust/README.md +++ b/rust/README.md @@ -21,7 +21,7 @@ Or declare it in `Cargo.toml`: ```toml [dependencies] -stringzilla = "4" +stringzilla = "5" ``` The crate ships the C/C++ sources and compiles them through a `build.rs` via `cc`, so no system StringZilla install is required. @@ -42,8 +42,8 @@ The entire `stringzillas` module is compiled only when at least one of `cpus`, ` ```toml [dependencies] -stringzilla = { version = "4", features = ["cpus"] } # CPU batch engines -# stringzilla = { version = "4", features = ["cuda"] } # CUDA-accelerated batch engines +stringzilla = { version = "5", features = ["cpus"] } # CPU batch engines +# stringzilla = { version = "5", features = ["cuda"] } # CUDA-accelerated batch engines ``` Import either by full module name or by alias: @@ -82,7 +82,7 @@ Disable it by opting out of default features (re-adding the ones you still want) ```toml [dependencies] # Compile-time dispatch: smaller, faster, but pinned to the build machine's best ISA. -stringzilla = { version = "4", default-features = false, features = ["std"] } +stringzilla = { version = "5", default-features = false, features = ["std"] } ``` Every tier can be forced on or off with its `SZ_USE_*` environment variable (`SZ_USE_SVE2=0 cargo build`), overriding the run gate but never the compile gate; the CMake build honors the same names as cache options (`-D SZ_USE_SVE2=0`). @@ -772,6 +772,97 @@ println!("Estimated Jaccard similarity: {similarity:.3}"); `Fingerprints` also exposes `compute_into` for writing into caller-provided buffers via an `AnyBytesTape`. +## Multi-Pattern Search + +`Substrings` compiles a whole needle set into one Aho-Corasick automaton and walks every haystack against all of them at once, so a dictionary of thousands of terms costs one pass rather than thousands. +Building it is the expensive half, and the engine is reusable, so a long-lived `Substrings` amortizes that across every later batch. + +```rust +fn new<S>(device: &DeviceScope, needles: &[S], case_sensitivity: CaseSensitivity) -> Result<Substrings, Error>; + +fn count_into(&self, device: &DeviceScope, haystacks: &AnyBytesTape<'_>, policy: OverlapPolicy, + counts: &mut [usize]) -> Result<usize, Error>; +fn find_into(&self, device: &DeviceScope, haystacks: &AnyBytesTape<'_>, policy: OverlapPolicy, + matches: &mut [SubstringsMatch]) -> Result<usize, Error>; +fn score_bm25_into(&self, device: &DeviceScope, haystacks: &AnyBytesTape<'_>, needle_weights: &[f32], + document_lengths: Option<&[f32]>, parameters: Bm25Params, scores: &mut [f32]) -> Result<(), Error>; +fn replace_bound<R>(&self, replacements: &[R], input_bytes: usize) -> Result<usize, Error>; +fn replace_into<R>(&self, device: &DeviceScope, haystacks: &AnyBytesTape<'_>, policy: OverlapPolicy, + replacements: &[R], output_data: &mut [u8], output_offsets: &mut [u64]) -> Result<usize, Error>; +``` + +Every verb writes into caller-owned buffers, so a pipeline allocates once and reuses those buffers across every batch. +On a GPU scope those buffers must be device-accessible, which [Unified Memory](#unified-memory) spells out. +Haystacks arrive as an `AnyBytesTape`, which spells all three shapes the C API accepts — a 32- or 64-bit tape built by `AnyBytesTape::from_sequences`, or borrowed slices addressed by callback through `AnyBytesTape::from_slices`, which copies nothing. +Only `replace_into` narrows that to the tapes, since a rewrite's product is itself a tape and there is nowhere to put one otherwise. + +`CaseSensitivity::Cased` matches bytes exactly and accepts arbitrary needles, while `CaseSensitivity::Uncased` folds both sides under full Unicode case folding and requires valid UTF-8. +Folding is not a byte-length-preserving operation, so a 1-byte needle can match a 3-byte span — the Kelvin sign `U+212A` folds to `k` — which is why every `SubstringsMatch` carries its own `byte_length` rather than borrowing the needle's. + +`OverlapPolicy` decides what a walk reports, and travels per call rather than per engine, since one automaton serves all three: + +- `Overlapping` — every match of every needle, nested and overlapping ones included. +- `LeftmostLongest` — a non-overlapping cover taking the widest match at the earliest start. +- `LeftmostFirst` — a non-overlapping cover taking the lowest needle index at the earliest start. + +```rust +use stringzilla::szs::{AnyBytesTape, CaseSensitivity, DeviceScope, OverlapPolicy, Substrings, SubstringsMatch}; + +let device = DeviceScope::default().unwrap(); +let engine = Substrings::new(&device, &["cat", "catalog"], CaseSensitivity::Cased).unwrap(); +let documents = vec!["a catalog of cats", "nothing here"]; +let haystacks = AnyBytesTape::from_slices(&documents); + +let mut counts = vec![0usize; documents.len()]; +let total = engine.count_into(&device, &haystacks, OverlapPolicy::Overlapping, &mut counts).unwrap(); +assert_eq!(counts, vec![3, 0]); // "catalog", the "cat" inside it, and the "cat" of "cats" +assert_eq!(total, 3); + +// A cover keeps no two matches sharing a byte, so the longer needle shadows the shorter one. +let mut cover = vec![SubstringsMatch::default(); total]; +let found = engine.find_into(&device, &haystacks, OverlapPolicy::LeftmostLongest, &mut cover).unwrap(); +assert_eq!(found, 2); +``` + +### Scoring and Rewriting + +The same automaton scores documents with BM25 and rewrites them, both in a single walk. +`score_bm25_into` treats the dictionary itself as the query — `needle_weights[i]` is needle `i`'s IDF or boost — and writes one score per haystack, so a many-term query over a large corpus never materializes per-term frequency rows. +Term frequencies are raw overlapping counts, which is classic BM25, and `document_lengths` defaults to byte lengths when `None`. + +`Bm25Params` has no `Default`, because a corpus mean has no correct default value. +Its two constructors name the two configurations that exist: `Bm25Params::normalized(mean)` is the literature's `k1 = 1.2` and `b = 0.75` against a corpus whose mean document length you know, and `Bm25Params::unnormalized()` switches length normalization off and leaves `document_lengths` unread. +A positive `b` beside a non-positive mean is refused rather than quietly discarding both it and the lengths you computed. + +```rust +use stringzilla::szs::{AnyBytesTape, Bm25Params, CaseSensitivity, DeviceScope, OverlapPolicy, Substrings}; + +let device = DeviceScope::default().unwrap(); +let engine = Substrings::new(&device, &["cat", "dog"], CaseSensitivity::Cased).unwrap(); +let documents = vec!["cat and dog", "nothing here"]; +let haystacks = AnyBytesTape::from_slices(&documents); + +let mut scores = vec![0.0f32; documents.len()]; +let parameters = Bm25Params::normalized(10.0); +engine.score_bm25_into(&device, &haystacks, &[1.0, 1.0], None, parameters, &mut scores).unwrap(); +assert_eq!(scores[1], 0.0); + +// One replacement per needle, inserted verbatim; an empty replacement deletes its match. Tape in, +// tape out, so this arm takes a materialized tape rather than borrowed slices. +let tape = AnyBytesTape::from_sequences(&documents).unwrap(); +let replacements = ["feline", "canine"]; +let input_bytes = documents.iter().map(|document| document.len()).sum(); +let mut data = vec![0u8; engine.replace_bound(&replacements, input_bytes).unwrap()]; +let mut offsets = vec![0u64; documents.len() + 1]; +engine + .replace_into(&device, &tape, OverlapPolicy::LeftmostLongest, &replacements, &mut data, &mut offsets) + .unwrap(); +assert_eq!(&data[offsets[0] as usize..offsets[1] as usize], b"feline and canine"); +``` + +Rewriting is defined only under a cover, so `OverlapPolicy::Overlapping` is rejected there — an overlapping rewrite is not a function. +`replace_bound` sizes an output tape from the needle set alone, without a walk, making the rewrite one call that cannot be refused for capacity. + ## UTF-8 Segmentation A single emoji such as the flag `đŸ‡ē🇸` is 8 bytes and 2 codepoints, yet a reader sees one character — one grapheme cluster. @@ -786,13 +877,13 @@ fn sz_utf8_split_whitespaces(&self) -> Utf8SplitWhitespaces<'_>; // content BETW fn sz_utf8_whitespaces(&self) -> Utf8Whitespaces<'_>; // the whitespace runs themselves fn sz_utf8_split_delimiters(&self) -> Utf8SplitDelimiters<'_>; // content BETWEEN whitespace/punctuation fn sz_utf8_delimiters(&self) -> Utf8Delimiters<'_>; // the delimiter runs themselves -fn sz_utf8_wordbreaks(&self) -> Utf8Wordbreaks<'_>; // all UAX-29 word segments (tiling) -fn sz_utf8_graphemes(&self) -> Utf8Graphemes<'_>; // UAX-29 grapheme clusters -fn sz_utf8_sentences(&self) -> Utf8Sentences<'_>; // UAX-29 sentences -fn sz_utf8_linebreaks(&self) -> Utf8Linebreaks<'_>; // UAX-14 line-break opportunities +fn sz_utf8_wordbreaks(&self) -> Utf8Wordbreaks<'_>; // all UAX-29 word segments (tiling) +fn sz_utf8_graphemes(&self) -> Utf8Graphemes<'_>; // UAX-29 grapheme clusters +fn sz_utf8_sentences(&self) -> Utf8Sentences<'_>; // UAX-29 sentences +fn sz_utf8_linebreaks(&self) -> Utf8Linebreaks<'_>; // UAX-14 line-break opportunities ``` -The naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the **separators** the kernel finds, while `split_*` yields the content **between** them. +The naming follows one rule: the bare name (`newlines`/`whitespaces`/`delimiters`) yields the __separators__ the kernel finds, while `split_*` yields the content __between__ them. Chain `.with_separators()` on a `split_*` iterator to interleave both losslessly. Every member of this family is lazy and zero-copy. @@ -914,3 +1005,23 @@ match DeviceScope::gpu_device(0) { `DeviceScope` API: `default()`, `cpu_cores(usize)`, `gpu_device(usize)`, `get_capabilities() -> Result<Capability, Error>`, `get_cpu_cores() -> Result<usize, Error>`, `get_gpu_device() -> Result<usize, Error>`, `is_gpu() -> bool`. Batch-engine failures surface as `szs::Error`, a `status` plus an optional message, which converts from the shared `Status` enum. + +### Unified Memory + +On a GPU scope every buffer an engine reads or writes must live on the device — unified or plain CUDA memory, never page-locked host memory. +A host buffer is refused with `Status::DeviceMemoryMismatch` rather than copied behind your back, so the cost of a stray `Vec` is a visible error instead of a hidden transfer. +A CPU scope imposes no requirement at all. + +The allocating verbs already satisfy this, since `Fingerprints::compute` returns `UnifiedVec<u32>` and the similarity engines return a `UnifiedMat`. +The `*_into` verbs take `&mut [T]`, which is what lets a caller write into a subrange of a larger buffer, so meeting the contract there means backing that slice with unified memory: + +```rust +use stringzilla::szs::{DeviceScope, UnifiedVec, UnifiedAlloc}; + +let gpu = DeviceScope::gpu_device(0).unwrap(); +let mut counts = UnifiedVec::with_capacity_in(haystacks_count, UnifiedAlloc); +counts.resize(haystacks_count, 0usize); +engine.count_into(&gpu, &haystacks, OverlapPolicy::Overlapping, &mut counts[..])?; +``` + +`&mut counts[..]` is the way in, and the same pattern covers `find_into`, `score_bm25_into` — whose `needle_weights` and `document_lengths` are read on the device too — and `replace_into`'s output data and offsets. diff --git a/rust/stringzillas.rs b/rust/stringzillas.rs index cb082864..1c510746 100644 --- a/rust/stringzillas.rs +++ b/rust/stringzillas.rs @@ -5,15 +5,18 @@ //! - Needleman-Wunsch global alignment //! - Smith-Waterman local alignment //! - Min-Hash fingerprinting +//! - Multi-pattern substring search mod device_scope; mod fingerprints; mod similarities; +mod substrings; mod types; pub use device_scope::*; pub use fingerprints::*; pub use similarities::*; +pub use substrings::*; pub use types::*; extern crate alloc; diff --git a/rust/stringzillas/fingerprints.rs b/rust/stringzillas/fingerprints.rs index 7e2ac694..61ef060c 100644 --- a/rust/stringzillas/fingerprints.rs +++ b/rust/stringzillas/fingerprints.rs @@ -707,6 +707,7 @@ impl Fingerprints { AnyBytesTape::View64(v) => SzSequenceU64Tape::from(v).count, AnyBytesTape::Tape32(t) => SzSequenceU32Tape::from(t).count, AnyBytesTape::View32(v) => SzSequenceU32Tape::from(v).count, + AnyBytesTape::Slices(s) => s.count(), }; let need = count * dimensions; if min_hashes.len() < need || min_counts.len() < need { @@ -775,6 +776,18 @@ impl Fingerprints { ) } } + AnyBytesTape::Slices(s) => unsafe { + szs_fingerprints_sequence( + self.handle, + device.handle, + s as *const _ as *const c_void, + min_hashes.as_mut_ptr(), + hashes_stride, + min_counts.as_mut_ptr(), + counts_stride, + &mut error_msg, + ) + }, }; match status { Status::Success => Ok(()), diff --git a/rust/stringzillas/similarities.rs b/rust/stringzillas/similarities.rs index d1962b9a..a6b95735 100644 --- a/rust/stringzillas/similarities.rs +++ b/rust/stringzillas/similarities.rs @@ -484,6 +484,39 @@ impl LevenshteinDistances { }; } + // The callback-addressed arm, walked in place with no tape to build. Both sides must name it: + // pairing borrowed slices with a tape would mean copying one of them behind the caller's back. + if let AnyBytesTape::Slices(queries_sequence) = &queries { + let candidates_sequence = match &candidates { + Some(AnyBytesTape::Slices(sequence)) => Some(sequence), + Some(_) => return Err(Error::from(SzStatus::UnexpectedDimensions)), + None => None, + }; + let candidates_count = candidates_sequence.map_or(queries_sequence.count(), |sequence| sequence.count()); + if matrix.queries_count != queries_sequence.count() || matrix.candidates_count != candidates_count { + return Err(Error::from(SzStatus::UnexpectedDimensions)); + } + let candidates_ptr = match candidates_sequence { + Some(sequence) => sequence as *const _ as *const c_void, + None => ptr::null(), + }; + let status = unsafe { + szs_levenshtein_distances( + self.handle, + device.handle, + queries_sequence as *const _ as *const c_void, + candidates_ptr, + matrix.data.as_mut_ptr(), + matrix.row_stride, + &mut error_msg, + ) + }; + return match status { + Status::Success => Ok(()), + err => Err(rust_error_from_c_message(err, error_msg)), + }; + } + // Mixed widths are unsupported to avoid implicit widening and extra copies Err(Error::from(SzStatus::UnexpectedDimensions)) } @@ -802,6 +835,39 @@ impl LevenshteinDistancesUtf8 { }; } + // The callback-addressed arm, walked in place with no tape to build. Both sides must name it: + // pairing borrowed slices with a tape would mean copying one of them behind the caller's back. + if let AnyCharsTape::Slices(queries_sequence) = &queries { + let candidates_sequence = match &candidates { + Some(AnyCharsTape::Slices(sequence)) => Some(sequence), + Some(_) => return Err(Error::from(SzStatus::UnexpectedDimensions)), + None => None, + }; + let candidates_count = candidates_sequence.map_or(queries_sequence.count(), |sequence| sequence.count()); + if matrix.queries_count != queries_sequence.count() || matrix.candidates_count != candidates_count { + return Err(Error::from(SzStatus::UnexpectedDimensions)); + } + let candidates_ptr = match candidates_sequence { + Some(sequence) => sequence as *const _ as *const c_void, + None => ptr::null(), + }; + let status = unsafe { + szs_levenshtein_distances_utf8( + self.handle, + device.handle, + queries_sequence as *const _ as *const c_void, + candidates_ptr, + matrix.data.as_mut_ptr(), + matrix.row_stride, + &mut error_msg, + ) + }; + return match status { + Status::Success => Ok(()), + err => Err(rust_error_from_c_message(err, error_msg)), + }; + } + Err(Error::from(SzStatus::UnexpectedDimensions)) } } @@ -1129,6 +1195,39 @@ impl NeedlemanWunschScores { err => Err(rust_error_from_c_message(err, error_msg)), }; } + + // The callback-addressed arm, walked in place with no tape to build. Both sides must name it: + // pairing borrowed slices with a tape would mean copying one of them behind the caller's back. + if let AnyBytesTape::Slices(queries_sequence) = &queries { + let candidates_sequence = match &candidates { + Some(AnyBytesTape::Slices(sequence)) => Some(sequence), + Some(_) => return Err(Error::from(SzStatus::UnexpectedDimensions)), + None => None, + }; + let candidates_count = candidates_sequence.map_or(queries_sequence.count(), |sequence| sequence.count()); + if matrix.queries_count != queries_sequence.count() || matrix.candidates_count != candidates_count { + return Err(Error::from(SzStatus::UnexpectedDimensions)); + } + let candidates_ptr = match candidates_sequence { + Some(sequence) => sequence as *const _ as *const c_void, + None => ptr::null(), + }; + let status = unsafe { + szs_needleman_wunsch_scores( + self.handle, + device.handle, + queries_sequence as *const _ as *const c_void, + candidates_ptr, + matrix.data.as_mut_ptr(), + matrix.row_stride, + &mut error_msg, + ) + }; + return match status { + Status::Success => Ok(()), + err => Err(rust_error_from_c_message(err, error_msg)), + }; + } Err(Error::from(SzStatus::UnexpectedDimensions)) } } @@ -1524,6 +1623,39 @@ impl SmithWatermanScores { err => Err(rust_error_from_c_message(err, error_msg)), }; } + + // The callback-addressed arm, walked in place with no tape to build. Both sides must name it: + // pairing borrowed slices with a tape would mean copying one of them behind the caller's back. + if let AnyBytesTape::Slices(queries_sequence) = &queries { + let candidates_sequence = match &candidates { + Some(AnyBytesTape::Slices(sequence)) => Some(sequence), + Some(_) => return Err(Error::from(SzStatus::UnexpectedDimensions)), + None => None, + }; + let candidates_count = candidates_sequence.map_or(queries_sequence.count(), |sequence| sequence.count()); + if matrix.queries_count != queries_sequence.count() || matrix.candidates_count != candidates_count { + return Err(Error::from(SzStatus::UnexpectedDimensions)); + } + let candidates_ptr = match candidates_sequence { + Some(sequence) => sequence as *const _ as *const c_void, + None => ptr::null(), + }; + let status = unsafe { + szs_smith_waterman_scores( + self.handle, + device.handle, + queries_sequence as *const _ as *const c_void, + candidates_ptr, + matrix.data.as_mut_ptr(), + matrix.row_stride, + &mut error_msg, + ) + }; + return match status { + Status::Success => Ok(()), + err => Err(rust_error_from_c_message(err, error_msg)), + }; + } Err(Error::from(SzStatus::UnexpectedDimensions)) } } @@ -1791,6 +1923,42 @@ mod tests { assert_eq!(matrix[(1, 1)], 3); } + #[test] + fn levenshtein_compute_into_borrowed_slices() { + let Some(device) = device_or_skip("levenshtein_compute_into_borrowed_slices") else { + return; + }; + let engine = LevenshteinDistances::new(&device, 0, 1, 1, 1).expect("Levenshtein engine should build on CPU"); + + let queries = [b"kitten".as_ref(), b"saturday".as_ref()]; + let candidates = [b"sitting".as_ref(), b"sunday".as_ref()]; + + // The callback-addressed arm reaches `szs_levenshtein_distances` with no tape built, and must + // reach the same answer the tape arms do. + let mut matrix = UnifiedMat::<usize>::try_allocate(2, 2).expect("matrix allocation"); + engine + .compute_into( + &device, + AnyBytesTape::from_slices(&queries), + Some(AnyBytesTape::from_slices(&candidates)), + &mut matrix, + ) + .expect("Levenshtein compute_into should accept borrowed slices"); + assert_eq!(matrix[(0, 0)], 3); + assert_eq!(matrix[(1, 1)], 3); + + // Mixing a tape with borrowed slices would need one of them copied, so it is refused. + let mut candidates_tape = BytesTape::<u32, UnifiedAlloc>::new_in(UnifiedAlloc); + candidates_tape.extend(candidates).unwrap(); + let mixed = engine.compute_into( + &device, + AnyBytesTape::from_slices(&queries), + Some(AnyBytesTape::Tape32(candidates_tape)), + &mut matrix, + ); + assert_eq!(mixed.unwrap_err().status, SzStatus::UnexpectedDimensions); + } + #[test] fn levenshtein_compute_into_u64_bytes() { let Some(device) = device_or_skip("levenshtein_compute_into_u64_bytes") else { diff --git a/rust/stringzillas/substrings.rs b/rust/stringzillas/substrings.rs new file mode 100644 index 00000000..b60dd1ed --- /dev/null +++ b/rust/stringzillas/substrings.rs @@ -0,0 +1,1261 @@ +//! Multi-pattern substring search engines - one compiled dictionary of needles walked over many haystacks. + +use super::types::{rust_error_from_c_message, SzSequenceFromBytes}; +use super::*; +use core::ffi::{c_char, c_void}; +use core::ptr; + +/// Opaque handle for the multi-pattern search engine +pub type SubstringsHandle = *mut c_void; + +/// Whether a dictionary matches needles byte-for-byte or folds both sides to a shared case first. +/// +/// Corresponds to `szs_substrings_case_sensitivity_t` in the C API. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaseSensitivity { + /// Byte-exact matching; needles may be arbitrary bytes. + Cased = 0, + /// Full Unicode case folding; needles must be valid UTF-8. + Uncased = 1, +} + +/// How overlapping matches resolve: reported in full, or reduced to a non-overlapping cover. +/// +/// The three states map one-to-one onto the `MatchKind` trio of the reference Rust engines. The +/// policy travels per call rather than per engine, since it never shapes the compiled automaton - +/// one dictionary serves all three. +/// +/// Corresponds to `szs_substrings_overlap_policy_t` in the C API. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OverlapPolicy { + /// Every match of every needle, including overlapping and nested ones. + Overlapping = 0, + /// Non-overlapping cover: earliest start, then longest span, then lower needle index. + LeftmostLongest = 1, + /// Non-overlapping cover: earliest start, then lower needle index, even when a longer needle matches. + LeftmostFirst = 2, +} + +/// One reported match, locating it by haystack, by needle, and by byte span. +/// +/// Under case folding a needle's own byte length is not the length of every match - needle "k" matches +/// both the 1-byte "k" and the 3-byte Kelvin sign - so the span is carried per match. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct SubstringsMatch { + /// Which haystack the match was found in. + pub haystack_index: usize, + /// Which needle matched. + pub needle_index: usize, + /// Offset of the match within its haystack, in bytes. + pub byte_offset: usize, + /// Length of the matched span, in bytes. + pub byte_length: usize, +} + +/// Classic BM25's continuous parameters. +/// +/// There is no correct default for the corpus mean, so there is no `Default`: reach for +/// [`Bm25Params::normalized`] or [`Bm25Params::unnormalized`], which name the two configurations that +/// exist. A positive `length_normalization` with a non-positive `average_document_length` is refused, +/// since it would divide by a mean that is not there. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Bm25Params { + /// The literature's `k1`: how slowly repeated occurrences stop adding score; 1.2 is customary. + pub term_frequency_saturation: f32, + /// The literature's `b`, in [0, 1]: 0 ignores document length and every length input with it, + /// 1 normalizes fully; 0.75 is customary. + pub length_normalization: f32, + /// Corpus-wide mean of the document lengths, in the same unit; never derived from the batch, and + /// read only when `length_normalization` is positive. + pub average_document_length: f32, +} + +impl Bm25Params { + /// Classic BM25 with the literature's customary `k1 = 1.2` and `b = 0.75`, normalized against a + /// corpus whose mean document length is `average_document_length`, in the unit the per-document + /// lengths use. + pub fn normalized(average_document_length: f32) -> Self { + Self { + term_frequency_saturation: 1.2, + length_normalization: 0.75, + average_document_length, + } + } + + /// BM25 with length normalization switched off, for a corpus whose mean length is unknown or whose + /// documents are uniform enough not to need it; the per-document lengths go unread. + pub fn unnormalized() -> Self { + Self { + term_frequency_saturation: 1.2, + length_normalization: 0.0, + average_document_length: 0.0, + } + } +} + +// C API bindings +extern "C" { + + fn szs_substrings_init( + alloc: *const c_void, // MemoryAllocator - using null for default + capabilities: Capability, + engine: *mut SubstringsHandle, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_index( + engine: SubstringsHandle, + needles: *const c_void, // SzSequence + case_sensitivity: CaseSensitivity, + device: *mut c_void, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_count( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequence + overlap_policy: OverlapPolicy, + counts: *mut usize, + matches_total: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_count_u32tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU32Tape + overlap_policy: OverlapPolicy, + counts: *mut usize, + matches_total: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_count_u64tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU64Tape + overlap_policy: OverlapPolicy, + counts: *mut usize, + matches_total: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_find( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequence + overlap_policy: OverlapPolicy, + matches: *mut SubstringsMatch, + matches_capacity: usize, + matches_found: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_find_u32tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU32Tape + overlap_policy: OverlapPolicy, + matches: *mut SubstringsMatch, + matches_capacity: usize, + matches_found: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_find_u64tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU64Tape + overlap_policy: OverlapPolicy, + matches: *mut SubstringsMatch, + matches_capacity: usize, + matches_found: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_score_bm25( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequence + document_lengths: *const f32, + parameters: Bm25Params, + needle_weights: *const f32, + scores: *mut f32, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_score_bm25_u32tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU32Tape + document_lengths: *const f32, + parameters: Bm25Params, + needle_weights: *const f32, + scores: *mut f32, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_score_bm25_u64tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU64Tape + document_lengths: *const f32, + parameters: Bm25Params, + needle_weights: *const f32, + scores: *mut f32, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_replace_bound( + engine: SubstringsHandle, + replacements: *const c_void, // SzSequence + input_bytes: usize, + output_bytes_bound: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_replace_u32tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU32Tape + overlap_policy: OverlapPolicy, + replacements: *const c_void, // SzSequence + output_data: *mut u8, + output_data_capacity: usize, + output_offsets: *mut usize, + output_bytes_written: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_replace_u64tape( + engine: SubstringsHandle, + device: *mut c_void, + haystacks: *const c_void, // SzSequenceU64Tape + overlap_policy: OverlapPolicy, + replacements: *const c_void, // SzSequence + output_data: *mut u8, + output_data_capacity: usize, + output_offsets: *mut usize, + output_bytes_written: *mut usize, + error_message: *mut *const c_char, + ) -> Status; + + fn szs_substrings_free(engine: SubstringsHandle); +} + +/// Multi-pattern exact and case-folded substring search engine. +/// +/// Compiles a needle set into an Aho-Corasick automaton once, then reuses it across every later +/// call, so the dictionary is paid for once rather than per haystack. Every needle is tested +/// against every haystack in a single walk, and matches are resolved under a caller-chosen +/// [`OverlapPolicy`] - every overlapping match, or a non-overlapping leftmost cover. The same +/// automaton also scores haystacks with BM25 and rewrites them by substituting their matches. +/// +/// Every verb writes into caller-owned buffers, so a pipeline allocates once and reuses those buffers +/// across every batch. Haystacks arrive as an [`AnyBytesTape`], which spells all three shapes the C API +/// accepts - a 32- or 64-bit tape, or borrowed slices addressed by callback and copied nowhere. Only +/// [`Substrings::replace_into`] narrows that to the tapes, since a rewrite's product is itself a tape. +/// +/// # Examples +/// +/// ```rust +/// # use stringzilla::szs::{AnyBytesTape, Substrings, CaseSensitivity, DeviceScope, OverlapPolicy}; +/// let device = DeviceScope::cpu_cores(1).unwrap(); +/// let engine = Substrings::new(&device, &["needle", "haystack"], CaseSensitivity::Cased).unwrap(); +/// +/// let documents = vec!["a needle in a haystack", "no matches here"]; +/// let haystacks = AnyBytesTape::from_slices(&documents); +/// +/// let mut counts = vec![0usize; documents.len()]; +/// let total = engine.count_into(&device, &haystacks, OverlapPolicy::Overlapping, &mut counts).unwrap(); +/// assert_eq!(counts, vec![2, 0]); +/// assert_eq!(total, 2); +/// +/// let mut matches = vec![Default::default(); total]; +/// let found = engine.find_into(&device, &haystacks, OverlapPolicy::Overlapping, &mut matches).unwrap(); +/// assert_eq!(found, 2); +/// ``` +pub struct Substrings { + handle: SubstringsHandle, + /// Fixed once the automaton is built, and what every per-needle array is validated against. + needles_count: usize, +} + +impl Substrings { + /// Compile a dictionary of `needles` into a search engine on `device`'s capabilities. + /// + /// Needles must be non-empty, and under [`CaseSensitivity::Uncased`] must be valid UTF-8. + pub fn new<Sequence>( + device: &DeviceScope, + needles: &[Sequence], + case_sensitivity: CaseSensitivity, + ) -> Result<Self, Error> + where + Sequence: AsRef<[u8]>, + { + let capabilities = device.get_capabilities()?; + let sequence = SzSequenceFromBytes::to_sz_sequence(needles); + let mut handle: SubstringsHandle = ptr::null_mut(); + let mut error_msg: *const c_char = ptr::null(); + let status = unsafe { szs_substrings_init(ptr::null(), capabilities, &mut handle, &mut error_msg) }; + if status != Status::Success { + return Err(rust_error_from_c_message(status, error_msg)); + } + + // The automaton is tiered against the cache `device` walks through, so indexing names it explicitly. + let status = unsafe { + szs_substrings_index( + handle, + &sequence as *const _ as *const c_void, + case_sensitivity, + device.handle, + &mut error_msg, + ) + }; + match status { + Status::Success => Ok(Substrings { + handle, + needles_count: needles.len(), + }), + // `Substrings` is never constructed on this path, so its `Drop` never runs and the handle + // would leak; the engine owns allocations from `init` alone, so it has to be freed here. + err => { + unsafe { szs_substrings_free(handle) }; + Err(rust_error_from_c_message(err, error_msg)) + } + } + } + + /// Needles compiled into the automaton; every reported `needle_index` is below this. + pub fn needles_count(&self) -> usize { + self.needles_count + } + + /// Count matches of every needle in every haystack into `counts`, one total per haystack. + /// + /// `counts` must hold one entry per haystack. The tape is borrowed, so one materialized corpus + /// serves a whole count-then-find-then-replace pipeline without being rebuilt. + pub fn count_into( + &self, + device: &DeviceScope, + haystacks: &AnyBytesTape<'_>, + overlap_policy: OverlapPolicy, + counts: &mut [usize], + ) -> Result<usize, Error> { + if counts.len() < Self::haystacks_count(haystacks) { + return Err(Error::from(Status::UnexpectedDimensions)); + } + + let mut matches_total: usize = 0; + let mut error_msg: *const c_char = ptr::null(); + let status = match &haystacks { + AnyBytesTape::Tape64(tape) => { + let view = SzSequenceU64Tape::from(tape); + unsafe { + szs_substrings_count_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + counts.as_mut_ptr(), + &mut matches_total, + &mut error_msg, + ) + } + } + AnyBytesTape::View64(borrowed) => { + let view = SzSequenceU64Tape::from(borrowed); + unsafe { + szs_substrings_count_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + counts.as_mut_ptr(), + &mut matches_total, + &mut error_msg, + ) + } + } + AnyBytesTape::Tape32(tape) => { + let view = SzSequenceU32Tape::from(tape); + unsafe { + szs_substrings_count_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + counts.as_mut_ptr(), + &mut matches_total, + &mut error_msg, + ) + } + } + AnyBytesTape::View32(borrowed) => { + let view = SzSequenceU32Tape::from(borrowed); + unsafe { + szs_substrings_count_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + counts.as_mut_ptr(), + &mut matches_total, + &mut error_msg, + ) + } + } + AnyBytesTape::Slices(sequence) => unsafe { + szs_substrings_count( + self.handle, + device.handle, + sequence as *const _ as *const c_void, + overlap_policy, + counts.as_mut_ptr(), + &mut matches_total, + &mut error_msg, + ) + }, + }; + match status { + Status::Success => Ok(matches_total), + err => Err(rust_error_from_c_message(err, error_msg)), + } + } + + /// Locate matches of every needle in every haystack into `matches`, under `overlap_policy`. + /// + /// `matches` is sized by the total [`Substrings::count_into`] returns under the same policy. Returns + /// the number of matches written; a short buffer fails with [`Status::UnexpectedDimensions`] and + /// writes nothing. + pub fn find_into( + &self, + device: &DeviceScope, + haystacks: &AnyBytesTape<'_>, + overlap_policy: OverlapPolicy, + matches: &mut [SubstringsMatch], + ) -> Result<usize, Error> { + let mut matches_found = 0usize; + let mut error_msg: *const c_char = ptr::null(); + let status = match &haystacks { + AnyBytesTape::Tape64(tape) => { + let view = SzSequenceU64Tape::from(tape); + unsafe { + szs_substrings_find_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + matches.as_mut_ptr(), + matches.len(), + &mut matches_found, + &mut error_msg, + ) + } + } + AnyBytesTape::View64(borrowed) => { + let view = SzSequenceU64Tape::from(borrowed); + unsafe { + szs_substrings_find_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + matches.as_mut_ptr(), + matches.len(), + &mut matches_found, + &mut error_msg, + ) + } + } + AnyBytesTape::Tape32(tape) => { + let view = SzSequenceU32Tape::from(tape); + unsafe { + szs_substrings_find_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + matches.as_mut_ptr(), + matches.len(), + &mut matches_found, + &mut error_msg, + ) + } + } + AnyBytesTape::View32(borrowed) => { + let view = SzSequenceU32Tape::from(borrowed); + unsafe { + szs_substrings_find_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + matches.as_mut_ptr(), + matches.len(), + &mut matches_found, + &mut error_msg, + ) + } + } + AnyBytesTape::Slices(sequence) => unsafe { + szs_substrings_find( + self.handle, + device.handle, + sequence as *const _ as *const c_void, + overlap_policy, + matches.as_mut_ptr(), + matches.len(), + &mut matches_found, + &mut error_msg, + ) + }, + }; + match status { + Status::Success => Ok(matches_found), + err => Err(rust_error_from_c_message(err, error_msg)), + } + } + + /// Score every haystack against the compiled dictionary in one automaton walk. + /// + /// `needle_weights` holds one IDF or boost per needle, and `document_lengths` defaults to byte + /// lengths when `None`. Term frequencies are raw overlapping counts, which is classic BM25. + pub fn score_bm25_into( + &self, + device: &DeviceScope, + haystacks: &AnyBytesTape<'_>, + needle_weights: &[f32], + document_lengths: Option<&[f32]>, + parameters: Bm25Params, + scores: &mut [f32], + ) -> Result<(), Error> { + let haystacks_count = Self::haystacks_count(haystacks); + if needle_weights.len() < self.needles_count + || scores.len() < haystacks_count + || document_lengths.is_some_and(|lengths| lengths.len() < haystacks_count) + { + return Err(Error::from(Status::UnexpectedDimensions)); + } + + let document_lengths_ptr = document_lengths.map_or(ptr::null(), |lengths| lengths.as_ptr()); + let mut error_msg: *const c_char = ptr::null(); + let status = match haystacks { + AnyBytesTape::Tape64(tape) => { + let view = SzSequenceU64Tape::from(tape); + unsafe { + szs_substrings_score_bm25_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + document_lengths_ptr, + parameters, + needle_weights.as_ptr(), + scores.as_mut_ptr(), + &mut error_msg, + ) + } + } + AnyBytesTape::View64(borrowed) => { + let view = SzSequenceU64Tape::from(borrowed); + unsafe { + szs_substrings_score_bm25_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + document_lengths_ptr, + parameters, + needle_weights.as_ptr(), + scores.as_mut_ptr(), + &mut error_msg, + ) + } + } + AnyBytesTape::Tape32(tape) => { + let view = SzSequenceU32Tape::from(tape); + unsafe { + szs_substrings_score_bm25_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + document_lengths_ptr, + parameters, + needle_weights.as_ptr(), + scores.as_mut_ptr(), + &mut error_msg, + ) + } + } + AnyBytesTape::View32(borrowed) => { + let view = SzSequenceU32Tape::from(borrowed); + unsafe { + szs_substrings_score_bm25_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + document_lengths_ptr, + parameters, + needle_weights.as_ptr(), + scores.as_mut_ptr(), + &mut error_msg, + ) + } + } + AnyBytesTape::Slices(sequence) => unsafe { + szs_substrings_score_bm25( + self.handle, + device.handle, + sequence as *const _ as *const c_void, + document_lengths_ptr, + parameters, + needle_weights.as_ptr(), + scores.as_mut_ptr(), + &mut error_msg, + ) + }, + }; + match status { + Status::Success => Ok(()), + err => Err(rust_error_from_c_message(err, error_msg)), + } + } + + /// Bound the bytes a rewrite of `input_bytes` can produce, from the dictionary and `replacements`. + /// + /// Needs no haystacks, no walk, and no device. Sizing an output buffer to this bound makes + /// [`Substrings::replace_into`] a single call that cannot be refused, at the cost of + /// over-allocating whenever the corpus is not entirely the widest-expanding needle. + pub fn replace_bound<Replacement>(&self, replacements: &[Replacement], input_bytes: usize) -> Result<usize, Error> + where + Replacement: AsRef<[u8]>, + { + let replacements_sequence = SzSequenceFromBytes::to_sz_sequence(replacements); + let mut output_bytes_bound = 0usize; + let mut error_msg: *const c_char = ptr::null(); + let status = unsafe { + szs_substrings_replace_bound( + self.handle, + &replacements_sequence as *const _ as *const c_void, + input_bytes, + &mut output_bytes_bound, + &mut error_msg, + ) + }; + match status { + Status::Success => Ok(output_bytes_bound), + err => Err(rust_error_from_c_message(err, error_msg)), + } + } + + /// Rewrite every haystack of a tape into caller-owned buffers, substituting matches. + /// + /// Tape in, tape out: `overlap_policy` must name a non-overlapping cover, replacements are indexed + /// by needle and inserted verbatim, and an empty replacement deletes. `output_offsets` takes + /// `haystacks.len() + 1` entries, in the width the engines address the output tape with. + /// Returns the bytes written; size `output_data` with [`Substrings::replace_bound`] to be sure the + /// call cannot be refused. + pub fn replace_into<Replacement>( + &self, + device: &DeviceScope, + haystacks: &AnyBytesTape<'_>, + overlap_policy: OverlapPolicy, + replacements: &[Replacement], + output_data: &mut [u8], + output_offsets: &mut [usize], + ) -> Result<usize, Error> + where + Replacement: AsRef<[u8]>, + { + // The engine writes one boundary per haystack plus a trailing total, and one replacement is read per + // needle - neither is negotiable, and a short slice here would be an out-of-bounds write. The + // callback-addressed arm is refused outright: a rewrite's product is a tape, and the C API offers + // this verb no `sz_sequence_t` overload to put one in. + if output_offsets.len() < Self::haystacks_count(haystacks) + 1 + || replacements.len() != self.needles_count + || matches!(haystacks, AnyBytesTape::Slices(_)) + { + return Err(Error::from(Status::UnexpectedDimensions)); + } + + let replacements_sequence = SzSequenceFromBytes::to_sz_sequence(replacements); + let mut bytes_written = 0usize; + let mut error_msg: *const c_char = ptr::null(); + let status = match haystacks { + AnyBytesTape::Tape64(tape) => { + let view = SzSequenceU64Tape::from(tape); + unsafe { + szs_substrings_replace_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + &replacements_sequence as *const _ as *const c_void, + output_data.as_mut_ptr(), + output_data.len(), + output_offsets.as_mut_ptr(), + &mut bytes_written, + &mut error_msg, + ) + } + } + AnyBytesTape::View64(borrowed) => { + let view = SzSequenceU64Tape::from(borrowed); + unsafe { + szs_substrings_replace_u64tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + &replacements_sequence as *const _ as *const c_void, + output_data.as_mut_ptr(), + output_data.len(), + output_offsets.as_mut_ptr(), + &mut bytes_written, + &mut error_msg, + ) + } + } + AnyBytesTape::Tape32(tape) => { + let view = SzSequenceU32Tape::from(tape); + unsafe { + szs_substrings_replace_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + &replacements_sequence as *const _ as *const c_void, + output_data.as_mut_ptr(), + output_data.len(), + output_offsets.as_mut_ptr(), + &mut bytes_written, + &mut error_msg, + ) + } + } + AnyBytesTape::View32(borrowed) => { + let view = SzSequenceU32Tape::from(borrowed); + unsafe { + szs_substrings_replace_u32tape( + self.handle, + device.handle, + &view as *const _ as *const c_void, + overlap_policy, + &replacements_sequence as *const _ as *const c_void, + output_data.as_mut_ptr(), + output_data.len(), + output_offsets.as_mut_ptr(), + &mut bytes_written, + &mut error_msg, + ) + } + } + AnyBytesTape::Slices(_) => unreachable!("refused above"), + }; + match status { + Status::Success => Ok(bytes_written), + err => Err(rust_error_from_c_message(err, error_msg)), + } + } + + /// Haystacks the input carries, which every output buffer above is sized against. + fn haystacks_count(haystacks: &AnyBytesTape<'_>) -> usize { + match haystacks { + AnyBytesTape::Tape64(tape) => SzSequenceU64Tape::from(tape).count, + AnyBytesTape::View64(view) => SzSequenceU64Tape::from(view).count, + AnyBytesTape::Tape32(tape) => SzSequenceU32Tape::from(tape).count, + AnyBytesTape::View32(view) => SzSequenceU32Tape::from(view).count, + AnyBytesTape::Slices(sequence) => sequence.count(), + } + } +} + +impl Drop for Substrings { + fn drop(&mut self) { + if !self.handle.is_null() { + unsafe { szs_substrings_free(self.handle) }; + } + } +} + +unsafe impl Send for Substrings {} +unsafe impl Sync for Substrings {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::stringzillas::fixtures::device_or_skip; + use crate::stringzillas::types::copy_bytes_into_tape; + use alloc::format; + use alloc::vec; + use alloc::vec::Vec; + + #[test] + fn substrings_count_and_find_known_answers() { + let Some(device) = device_or_skip("substrings_count_and_find_known_answers") else { + return; + }; + let engine = Substrings::new(&device, &["aa", "b"], CaseSensitivity::Cased).unwrap(); + let haystacks = ["aaab", "xyz"]; + + // Overlapping matches are all reported: "aa" at 0 and 1, "b" at 3. + let (counts, matches) = count_and_find(&engine, &device, &haystacks, OverlapPolicy::Overlapping); + assert_eq!(counts[0], 3); + assert_eq!(counts[1], 0); + + assert_eq!(matches.len(), 3); + for one_match in matches.iter() { + assert_eq!(one_match.haystack_index, 0); + let expected_needle = ["aa", "b"][one_match.needle_index].as_bytes(); + let span = &haystacks[0].as_bytes()[one_match.byte_offset..one_match.byte_offset + one_match.byte_length]; + assert_eq!(span, expected_needle); + } + } + + #[test] + fn substrings_uncased_kelvin_sign() { + let Some(device) = device_or_skip("substrings_uncased_kelvin_sign") else { + return; + }; + let engine = Substrings::new(&device, &["k"], CaseSensitivity::Uncased).unwrap(); + // The 3-byte Kelvin sign U+212A folds to "k", so the 1-byte needle matches a 3-byte span. + let kelvin_sign = "\u{212A}"; + let haystacks = ["K", "k", kelvin_sign]; + + let (counts, matches) = count_and_find(&engine, &device, &haystacks, OverlapPolicy::Overlapping); + assert_eq!(&counts[..], &[1, 1, 1]); + assert_eq!(matches.len(), 3); + let kelvin_match = matches.iter().find(|one_match| one_match.haystack_index == 2).unwrap(); + assert_eq!(kelvin_match.byte_length, 3); + } + + #[test] + fn substrings_needles_count() { + let Some(device) = device_or_skip("substrings_needles_count") else { + return; + }; + let engine = Substrings::new(&device, &["aa", "b"], CaseSensitivity::Cased).unwrap(); + assert_eq!(engine.needles_count(), 2); + } + + #[test] + fn substrings_empty_needle_rejected() { + let Some(device) = device_or_skip("substrings_empty_needle_rejected") else { + return; + }; + let Err(error) = Substrings::new(&device, &["good", ""], CaseSensitivity::Cased) else { + panic!("an empty needle must be rejected"); + }; + assert_eq!(error.status, Status::UnexpectedDimensions); + } + + /// Counts, then locates, sizing the match buffer from the counts exactly as a caller would. + fn count_and_find<Sequence: AsRef<[u8]>>( + engine: &Substrings, + device: &DeviceScope, + haystacks: &[Sequence], + overlap_policy: OverlapPolicy, + ) -> (Vec<usize>, Vec<SubstringsMatch>) { + let tape = AnyBytesTape::from_sequences(haystacks).unwrap(); + let mut counts = vec![0usize; haystacks.len()]; + engine.count_into(device, &tape, overlap_policy, &mut counts).unwrap(); + + let mut matches = vec![SubstringsMatch::default(); counts.iter().sum()]; + let found = engine.find_into(device, &tape, overlap_policy, &mut matches).unwrap(); + assert_eq!(found, matches.len(), "counting and locating must agree on the total"); + (counts, matches) + } + + /// Matches sorted by start offset, so a test asserts a cover's shape without depending on the + /// walk's own emission order, which follows match ends. + fn matches_by_offset(matches: &[SubstringsMatch]) -> Vec<(usize, usize, usize)> { + let mut ordered: Vec<(usize, usize, usize)> = matches + .iter() + .map(|one_match| (one_match.byte_offset, one_match.byte_length, one_match.needle_index)) + .collect(); + ordered.sort_unstable(); + ordered + } + + #[test] + fn substrings_leftmost_longest_shadows_shorter_needles() { + let Some(device) = device_or_skip("substrings_leftmost_longest_shadows_shorter_needles") else { + return; + }; + let engine = Substrings::new(&device, &["cat", "catalog"], CaseSensitivity::Cased).unwrap(); + let haystacks = ["catalog"]; + + // Both needles start at 0, so the longer span wins and "cat" never surfaces. + let (counts, matches) = count_and_find(&engine, &device, &haystacks, OverlapPolicy::LeftmostLongest); + assert_eq!(counts[0], 1); + assert_eq!(matches_by_offset(&matches), vec![(0, 7, 1)]); + + // The same dictionary under the overlapping policy reports both. + let (_, overlapping) = count_and_find(&engine, &device, &haystacks, OverlapPolicy::Overlapping); + assert_eq!(matches_by_offset(&overlapping), vec![(0, 3, 0), (0, 7, 1)]); + } + + #[test] + fn substrings_leftmost_first_prefers_the_lower_needle_index() { + let Some(device) = device_or_skip("substrings_leftmost_first_prefers_the_lower_needle_index") else { + return; + }; + let engine = Substrings::new(&device, &["cat", "catalog"], CaseSensitivity::Cased).unwrap(); + + // Same start, so the lower needle index wins even though "catalog" spans further; the walk then + // resumes past "cat", where "alog" matches nothing. + let (_, matches) = count_and_find(&engine, &device, &["catalog"], OverlapPolicy::LeftmostFirst); + assert_eq!(matches_by_offset(&matches), vec![(0, 3, 0)]); + } + + #[test] + fn substrings_cover_skips_matches_starting_inside_a_commit() { + let Some(device) = device_or_skip("substrings_cover_skips_matches_starting_inside_a_commit") else { + return; + }; + let engine = Substrings::new(&device, &["ab", "ba"], CaseSensitivity::Cased).unwrap(); + + // "ab" commits at 0, so "ba" at 1 starts inside the committed span and is dropped. + let (_, matches) = count_and_find(&engine, &device, &["aba"], OverlapPolicy::LeftmostLongest); + assert_eq!(matches_by_offset(&matches), vec![(0, 2, 0)]); + + // A self-overlapping needle tiles its haystack instead: commits at 0 and 2, not at 1. + let tiling = Substrings::new(&device, &["aa"], CaseSensitivity::Cased).unwrap(); + let (_, tiled) = count_and_find(&tiling, &device, &["aaaa"], OverlapPolicy::LeftmostLongest); + assert_eq!(matches_by_offset(&tiled), vec![(0, 2, 0), (2, 2, 0)]); + } + + #[test] + fn substrings_score_bm25_saturates_term_frequency() { + let Some(device) = device_or_skip("substrings_score_bm25_saturates_term_frequency") else { + return; + }; + let engine = Substrings::new(&device, &["cat"], CaseSensitivity::Cased).unwrap(); + let haystacks = ["cat cat", "dog"]; + let parameters = Bm25Params { + term_frequency_saturation: 1.2, + length_normalization: 0.75, + average_document_length: 5.0, + }; + + // Byte lengths stand in for document lengths, so haystack 0 is 7 bytes against a 5-byte mean: + // 1.0 * 2 * (1.2 + 1) / (2 + 1.2 * (1 - 0.75 + 0.75 * 7 / 5)) = 4.4 / 3.56 + let tape = AnyBytesTape::from_sequences(&haystacks).unwrap(); + let mut scores = vec![0.0f32; haystacks.len()]; + engine + .score_bm25_into(&device, &tape, &[1.0], None, parameters, &mut scores) + .unwrap(); + assert!((scores[0] - 4.4f32 / 3.56f32).abs() < 1e-4); + assert_eq!(scores[1], 0.0); // ? A needle absent from a haystack contributes nothing. + } + + /// Rewrites @p haystacks into buffers sized by `replace_bound`, returning each output string. + /// + /// The whole point of the bound is that this is one call, so a test that needed two would be + /// testing something the API does not promise. + fn replace_all<Sequence, Replacement>( + engine: &Substrings, + device: &DeviceScope, + haystacks: &[Sequence], + overlap_policy: OverlapPolicy, + replacements: &[Replacement], + ) -> Vec<Vec<u8>> + where + Sequence: AsRef<[u8]>, + Replacement: AsRef<[u8]>, + { + let input_bytes: usize = haystacks.iter().map(|one| one.as_ref().len()).sum(); + let bound = engine.replace_bound(replacements, input_bytes).unwrap(); + + let mut output_data = vec![0u8; bound]; + let mut output_offsets = vec![0usize; haystacks.len() + 1]; + let tape = AnyBytesTape::from_sequences(haystacks).unwrap(); + engine + .replace_into( + device, + &tape, + overlap_policy, + replacements, + &mut output_data, + &mut output_offsets, + ) + .unwrap(); + + (0..haystacks.len()) + .map(|index| { + let start = output_offsets[index]; + let end = output_offsets[index + 1]; + output_data[start..end].to_vec() + }) + .collect() + } + + #[test] + fn substrings_replace_rewrites_every_match() { + let Some(device) = device_or_skip("substrings_replace_rewrites_every_match") else { + return; + }; + let engine = Substrings::new(&device, &["cat", "dog"], CaseSensitivity::Cased).unwrap(); + let haystacks = ["cat and dog", "nothing here"]; + + let rewritten = replace_all( + &engine, + &device, + &haystacks, + OverlapPolicy::LeftmostLongest, + &["feline", "canine"], + ); + assert_eq!(rewritten[0], b"feline and canine"); + assert_eq!(rewritten[1], b"nothing here"); // ? An unmatched haystack passes through byte for byte. + } + + #[test] + fn substrings_replace_deletes_on_an_empty_replacement() { + let Some(device) = device_or_skip("substrings_replace_deletes_on_an_empty_replacement") else { + return; + }; + let engine = Substrings::new(&device, &["bad "], CaseSensitivity::Cased).unwrap(); + + // An empty replacement is legal and means deletion. + let rewritten = replace_all(&engine, &device, &["bad word"], OverlapPolicy::LeftmostLongest, &[""]); + assert_eq!(rewritten[0], b"word"); + } + + #[test] + fn substrings_replace_inserts_verbatim_under_folding() { + let Some(device) = device_or_skip("substrings_replace_inserts_verbatim_under_folding") else { + return; + }; + let engine = Substrings::new(&device, &["k"], CaseSensitivity::Uncased).unwrap(); + + // No case adaptation: every fold preimage - including the 3-byte Kelvin sign - takes the + // replacement's exact bytes, so the rewritten haystack shrinks. + let kelvin_sign = "\u{212A}"; + let haystacks = [format!("K{kelvin_sign}k")]; + let rewritten = replace_all(&engine, &device, &haystacks, OverlapPolicy::LeftmostLongest, &["x"]); + assert_eq!(rewritten[0], b"xxx"); + } + + #[test] + fn substrings_replace_shadows_under_leftmost_longest() { + let Some(device) = device_or_skip("substrings_replace_shadows_under_leftmost_longest") else { + return; + }; + let engine = Substrings::new(&device, &["cat", "catalog"], CaseSensitivity::Cased).unwrap(); + + // The longer needle shadows the shorter at the same start, so "cat" never fires. + let rewritten = replace_all( + &engine, + &device, + &["catalog"], + OverlapPolicy::LeftmostLongest, + &["feline", "directory"], + ); + assert_eq!(rewritten[0], b"directory"); + } + + #[test] + fn substrings_replace_bound_covers_the_widest_expansion() { + let Some(device) = device_or_skip("substrings_replace_bound_covers_the_widest_expansion") else { + return; + }; + let engine = Substrings::new(&device, &["a", "bb"], CaseSensitivity::Cased).unwrap(); + + // "a" expands 1 byte into 4 and "bb" expands 2 into 4, so the widest ratio is 4 and a fully + // tiled haystack of 8 bytes cannot exceed 32. + let bound = engine.replace_bound(&["wxyz", "wxyz"], 8).unwrap(); + assert_eq!(bound, 32); + + // A dictionary that only ever shrinks still bounds at the input length, never below it. + let shrinking = Substrings::new(&device, &["aaaa"], CaseSensitivity::Cased).unwrap(); + assert_eq!(shrinking.replace_bound(&["z"], 8).unwrap(), 8); + } + + #[test] + fn substrings_input_variants_agree() { + let Some(device) = device_or_skip("substrings_input_variants_agree") else { + return; + }; + let engine = Substrings::new(&device, &["cat", "dog"], CaseSensitivity::Cased).unwrap(); + let corpus = ["cat and dog", "nothing here", "dog dog"]; + + // The three shapes the C API accepts reach three different entry points - `_u32tape`, + // `_u64tape` and the callback-addressed `sz_sequence_t` - and must reach one answer. + let variants = [ + AnyBytesTape::from_sequences(&corpus).unwrap(), + copy_bytes_into_tape(&corpus, true).unwrap(), + AnyBytesTape::from_slices(&corpus), + ]; + let mut answers = Vec::new(); + for haystacks in &variants { + let mut counts = vec![0usize; corpus.len()]; + let total = engine + .count_into(&device, haystacks, OverlapPolicy::Overlapping, &mut counts) + .unwrap(); + let mut matches = vec![SubstringsMatch::default(); total]; + engine + .find_into(&device, haystacks, OverlapPolicy::Overlapping, &mut matches) + .unwrap(); + + let parameters = Bm25Params::normalized(10.0); + let mut scores = vec![0.0f32; corpus.len()]; + engine + .score_bm25_into(&device, haystacks, &[1.0, 1.0], None, parameters, &mut scores) + .unwrap(); + answers.push((counts, matches_by_offset(&matches), scores)); + } + assert_eq!(answers[1], answers[0], "the 64-bit tape must agree with the 32-bit one"); + assert_eq!(answers[2], answers[0], "borrowed slices must agree with a tape"); + assert_eq!(answers[0].2[1], 0.0); // ? "nothing here" holds neither needle. + + // Tape in, tape out: a rewrite has nowhere to put its product when the input is addressed by + // callback, and the C API gives `replace` no sequence overload to try. + let mut output_data = vec![0u8; 256]; + let mut output_offsets = vec![0usize; corpus.len() + 1]; + let refused = engine.replace_into( + &device, + &AnyBytesTape::from_slices(&corpus), + OverlapPolicy::LeftmostLongest, + &["feline", "canine"], + &mut output_data, + &mut output_offsets, + ); + assert_eq!(refused.unwrap_err().status, Status::UnexpectedDimensions); + } + + #[test] + fn substrings_parallel_scope_agrees_with_the_default() { + // `device_or_skip` only ever yields the default scope, so nothing else here reaches the + // multi-core arm of `szs_substrings_init` or the parallel walkers behind it. + let Ok(parallel) = DeviceScope::cpu_cores(2) else { + return; + }; + let Some(device) = device_or_skip("substrings_parallel_scope_agrees_with_the_default") else { + return; + }; + let haystacks = ["cat and dog", "nothing here", "dog dog", "a catalog of cats"]; + let needles = ["cat", "dog", "catalog"]; + + let serial_engine = Substrings::new(&device, &needles, CaseSensitivity::Cased).unwrap(); + let parallel_engine = Substrings::new(¶llel, &needles, CaseSensitivity::Cased).unwrap(); + for policy in [ + OverlapPolicy::Overlapping, + OverlapPolicy::LeftmostLongest, + OverlapPolicy::LeftmostFirst, + ] { + let (serial_counts, serial_matches) = count_and_find(&serial_engine, &device, &haystacks, policy); + let (parallel_counts, parallel_matches) = count_and_find(¶llel_engine, ¶llel, &haystacks, policy); + assert_eq!(parallel_counts, serial_counts); + assert_eq!(matches_by_offset(¶llel_matches), matches_by_offset(&serial_matches)); + } + } + + /// A tiny xorshift, so a seeded corpus needs no dependency and reproduces across platforms. + fn next_random(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state + } + + /// A needle set and corpus drawn from a five-letter alphabet, small enough that short needles hit + /// often rather than measuring an empty walk. + fn random_corpus(seed: u64) -> (Vec<String>, Vec<String>) { + let mut state = seed | 1; + let alphabet = b"abcde"; + let mut draw = |max_length: usize, state: &mut u64| -> String { + let length = 1 + (next_random(state) as usize) % max_length; + (0..length) + .map(|_| alphabet[(next_random(state) as usize) % alphabet.len()] as char) + .collect() + }; + let mut needles: Vec<String> = (0..12).map(|_| draw(4, &mut state)).collect(); + needles.sort_unstable(); + needles.dedup(); + let haystacks: Vec<String> = (0..20).map(|_| draw(180, &mut state)).collect(); + (needles, haystacks) + } + + #[test] + fn substrings_matches_aho_corasick() { + use aho_corasick::{AhoCorasick, MatchKind}; + + let Some(device) = device_or_skip("substrings_matches_aho_corasick") else { + return; + }; + for seed in [42u64, 1, 314159, 2718281828] { + let (needles, haystacks) = random_corpus(seed); + let engine = Substrings::new(&device, &needles, CaseSensitivity::Cased).unwrap(); + + // The three `MatchKind`s map one-to-one onto `OverlapPolicy`, so an independent automaton + // witnesses every cover rule rather than only the overlapping walk. + for (policy, kind) in [ + (OverlapPolicy::Overlapping, MatchKind::Standard), + (OverlapPolicy::LeftmostLongest, MatchKind::LeftmostLongest), + (OverlapPolicy::LeftmostFirst, MatchKind::LeftmostFirst), + ] { + let oracle_automaton = AhoCorasick::builder().match_kind(kind).build(&needles).unwrap(); + let mut oracle: Vec<(usize, usize, usize, usize)> = Vec::new(); + for (haystack_index, haystack) in haystacks.iter().enumerate() { + let found: Vec<_> = match kind { + MatchKind::Standard => oracle_automaton.find_overlapping_iter(haystack).collect(), + _ => oracle_automaton.find_iter(haystack).collect(), + }; + for one_match in found { + oracle.push(( + haystack_index, + one_match.pattern().as_usize(), + one_match.start(), + one_match.len(), + )); + } + } + oracle.sort_unstable(); + + let (_, matches) = count_and_find(&engine, &device, &haystacks, policy); + let mut found: Vec<(usize, usize, usize, usize)> = matches + .iter() + .map(|one| (one.haystack_index, one.needle_index, one.byte_offset, one.byte_length)) + .collect(); + found.sort_unstable(); + assert_eq!( + found, oracle, + "seed {seed} disagrees with aho-corasick under {policy:?}" + ); + } + } + } + + #[test] + fn substrings_replace_matches_aho_corasick() { + use aho_corasick::{AhoCorasick, MatchKind}; + + let Some(device) = device_or_skip("substrings_replace_matches_aho_corasick") else { + return; + }; + for seed in [7u64, 99, 123456789] { + let (needles, haystacks) = random_corpus(seed); + let replacements: Vec<String> = needles.iter().map(|needle| format!("<{needle}>")).collect(); + let engine = Substrings::new(&device, &needles, CaseSensitivity::Cased).unwrap(); + + for (policy, kind) in [ + (OverlapPolicy::LeftmostLongest, MatchKind::LeftmostLongest), + (OverlapPolicy::LeftmostFirst, MatchKind::LeftmostFirst), + ] { + let oracle_automaton = AhoCorasick::builder().match_kind(kind).build(&needles).unwrap(); + let rewritten = replace_all(&engine, &device, &haystacks, policy, &replacements); + for (haystack, rewritten_one) in haystacks.iter().zip(rewritten.iter()) { + let expected = oracle_automaton.replace_all_bytes(haystack.as_bytes(), &replacements); + assert_eq!( + rewritten_one, &expected, + "seed {seed} rewrote differently under {policy:?}" + ); + } + } + } + } +} diff --git a/rust/stringzillas/types.rs b/rust/stringzillas/types.rs index 06adffe7..f9dae4c1 100644 --- a/rust/stringzillas/types.rs +++ b/rust/stringzillas/types.rs @@ -1,8 +1,8 @@ //! Shared value types, unified-memory containers, and library introspection. use core::ffi::{c_char, c_void, CStr}; +use core::marker::PhantomData; use core::ops::Index; -use core::ptr; use allocator_api2::{alloc::AllocError, alloc::Allocator, alloc::Layout}; use stringtape::{BytesTape, BytesTapeView, CharsTape, CharsTapeView}; @@ -50,13 +50,18 @@ pub(crate) fn rust_error_from_c_message(status: Status, error_msg: *const c_char Error { status, message } } -/// Tape variant that can hold either 32-bit or 64-bit string tapes with unsigned offsets +/// Every shape a string collection can reach the engines in, one variant per layout the C API accepts. +/// +/// Named for the tapes because most of it is tape: the four tape arms spell `sz_sequence_u32tape_t` and +/// `sz_sequence_u64tape_t`, owned and borrowed collapsing onto the same pair. [`AnyCharsTape::Slices`] is +/// the exception - `sz_sequence_t`, addressed by callback and copied nowhere. pub enum AnyCharsTape<'a> { Tape32(CharsTape<u32, UnifiedAlloc>), Tape64(CharsTape<u64, UnifiedAlloc>), // Zero-copy FFI views (UTF-8) View32(CharsTapeView<'a, u32>), View64(CharsTapeView<'a, u64>), + Slices(SzSequence<'a>), } impl<'a> AnyCharsTape<'a> { @@ -66,15 +71,28 @@ impl<'a> AnyCharsTape<'a> { pub fn from_sequences<Sequence: AsRef<str>>(sequences: &[Sequence]) -> Result<Self, Error> { copy_chars_into_tape(sequences, false) } + + /// Borrow the sequences where they already are, building no tape and copying nothing. + /// + /// A GPU scope needs its inputs in unified memory and refuses this arm, so reach for + /// [`AnyCharsTape::from_sequences`] there. + pub fn from_slices<Sequence: AsRef<str>>(sequences: &'a [Sequence]) -> Self { + Self::Slices(SzSequenceFromChars::to_sz_sequence(sequences)) + } } -/// Tape variant that can hold either 32-bit or 64-bit byte tapes with unsigned offsets +/// Every shape a byte collection can reach the engines in, one variant per layout the C API accepts. +/// +/// Named for the tapes because most of it is tape: the four tape arms spell `sz_sequence_u32tape_t` and +/// `sz_sequence_u64tape_t`, owned and borrowed collapsing onto the same pair. [`AnyBytesTape::Slices`] is +/// the exception - `sz_sequence_t`, addressed by callback and copied nowhere. pub enum AnyBytesTape<'a> { Tape32(BytesTape<u32, UnifiedAlloc>), Tape64(BytesTape<u64, UnifiedAlloc>), // Zero-copy FFI views (bytes) View32(BytesTapeView<'a, u32>), View64(BytesTapeView<'a, u64>), + Slices(SzSequence<'a>), } impl<'a> AnyBytesTape<'a> { @@ -84,18 +102,35 @@ impl<'a> AnyBytesTape<'a> { pub fn from_sequences<Sequence: AsRef<[u8]>>(sequences: &[Sequence]) -> Result<Self, Error> { copy_bytes_into_tape(sequences, false) } + + /// Borrow the sequences where they already are, building no tape and copying nothing. + /// + /// A GPU scope needs its inputs in unified memory and refuses this arm, so reach for + /// [`AnyBytesTape::from_sequences`] there. + pub fn from_slices<Sequence: AsRef<[u8]>>(sequences: &'a [Sequence]) -> Self { + Self::Slices(SzSequenceFromBytes::to_sz_sequence(sequences)) + } } -/// Internal representation of `sz_sequence_t` for passing to C +/// A collection the engines reach through two accessors rather than a contiguous tape, mirroring +/// `sz_sequence_t`. +/// +/// Opaque: built by [`AnyBytesTape::from_slices`] or [`AnyCharsTape::from_slices`], whose lifetime is what +/// keeps the borrowed slices alive for as long as the engine can address them. #[repr(C)] -pub(crate) struct SzSequence { +pub struct SzSequence<'a> { handle: *mut c_void, count: usize, get_start: extern "C" fn(*mut c_void, usize) -> *const u8, get_length: extern "C" fn(*mut c_void, usize) -> usize, - // Additional fields for our implementation - starts: *const *const u8, - lengths: *const usize, + borrowed: PhantomData<&'a ()>, +} + +impl SzSequence<'_> { + /// Members the sequence carries, which every per-element output buffer is sized against. + pub(crate) fn count(&self) -> usize { + self.count + } } /// Raw C API tape structure for 32-bit offsets (data < 4GB), @@ -286,36 +321,34 @@ extern "C" fn sz_sequence_get_length_str<Sequence: AsRef<str>>(handle: *mut c_vo /// Trait for types that can be converted to SzSequence for byte sequences pub(crate) trait SzSequenceFromBytes { - fn to_sz_sequence(&self) -> SzSequence; + fn to_sz_sequence(&self) -> SzSequence<'_>; } impl<Sequence: AsRef<[u8]>> SzSequenceFromBytes for [Sequence] { - fn to_sz_sequence(&self) -> SzSequence { + fn to_sz_sequence(&self) -> SzSequence<'_> { SzSequence { handle: self.as_ptr() as *mut c_void, count: self.len(), get_start: sz_sequence_get_start_generic::<Sequence>, get_length: sz_sequence_get_length_generic::<Sequence>, - starts: ptr::null(), - lengths: ptr::null(), + borrowed: PhantomData, } } } /// Trait for types that can be converted to SzSequence for string sequences pub(crate) trait SzSequenceFromChars { - fn to_sz_sequence(&self) -> SzSequence; + fn to_sz_sequence(&self) -> SzSequence<'_>; } impl<Sequence: AsRef<str>> SzSequenceFromChars for [Sequence] { - fn to_sz_sequence(&self) -> SzSequence { + fn to_sz_sequence(&self) -> SzSequence<'_> { SzSequence { handle: self.as_ptr() as *mut c_void, count: self.len(), get_start: sz_sequence_get_start_str::<Sequence>, get_length: sz_sequence_get_length_str::<Sequence>, - starts: ptr::null(), - lengths: ptr::null(), + borrowed: PhantomData, } } } @@ -328,6 +361,8 @@ extern "C" { fn szs_version_minor() -> i32; fn szs_version_patch() -> i32; fn szs_capabilities() -> u32; + fn szs_capabilities_comptime() -> u32; + fn szs_capabilities_runtime() -> u32; // Unified allocator functions fn szs_unified_alloc(size_bytes: usize) -> *mut c_void; @@ -482,6 +517,21 @@ pub fn capabilities() -> crate::stringzilla::SmallCString { crate::stringzilla::capabilities_from_enum(caps) } +/// What this binary *ships*: the CPU kernels compiled in, and the GPU tiers the toolkit could build. +/// +/// Differs from [`capabilities`], which intersects this with what the machine offers - so a missing GPU +/// bit here means "not compiled", while a bit present here and absent there means "no driver". +pub fn capabilities_comptime() -> crate::stringzilla::SmallCString { + let caps = unsafe { szs_capabilities_comptime() }; + crate::stringzilla::capabilities_from_enum(caps) +} + +/// What this machine *offers*: the CPU's instruction set, its usable cores, and the first GPU's tier. +pub fn capabilities_runtime() -> crate::stringzilla::SmallCString { + let caps = unsafe { szs_capabilities_runtime() }; + crate::stringzilla::capabilities_from_enum(caps) +} + /// Check if either byte collection requires 64-bit tapes pub(crate) fn should_use_64bit_for_bytes<Sequence: AsRef<[u8]>>(seq_a: &[Sequence], seq_b: &[Sequence]) -> bool { let total_size_a: usize = seq_a.iter().map(|s| s.as_ref().len()).sum(); diff --git a/setup.py b/setup.py index 3712a437..9de17f13 100644 --- a/setup.py +++ b/setup.py @@ -684,6 +684,7 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "c/stringzillas/needleman_wunsch.cpp", "c/stringzillas/smith_waterman.cpp", "c/stringzillas/fingerprints.cpp", + "c/stringzillas/substrings.cpp", ] STRINGZILLAS_API_CU_SOURCES = [ "c/stringzillas/runtime.cu", @@ -691,6 +692,7 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "c/stringzillas/needleman_wunsch.cu", "c/stringzillas/smith_waterman.cu", "c/stringzillas/fingerprints.cu", + "c/stringzillas/substrings.cu", ] # Per-ISA CPU instantiation units, host C++ in every wheel; off-platform files compile to empty objects. STRINGZILLAS_CPUS_SOURCES = [ @@ -709,6 +711,7 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "c/stringzillas/smith_waterman_haswell.cpp", "c/stringzillas/smith_waterman_neon.cpp", "c/stringzillas/smith_waterman_rvv.cpp", + "c/stringzillas/substrings_serial.cpp", ] # Per-tier GPU instantiation units, grouped by architecture floor: Hopper DPX needs sm_90, the rest run # from the base set. @@ -716,6 +719,7 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "c/stringzillas/levenshtein_cuda.cu", "c/stringzillas/needleman_wunsch_cuda.cu", "c/stringzillas/smith_waterman_cuda.cu", + "c/stringzillas/substrings_cuda.cu", ] STRINGZILLAS_KEPLER_SOURCES = [ "c/stringzillas/levenshtein_kepler.cu", @@ -782,10 +786,14 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "python/stringzillas/device_scope.c", "python/stringzillas/similarities.c", "python/stringzillas/fingerprints.c", + "python/stringzillas/substrings.c", ] + STRINGZILLAS_API_CPP_SOURCES + STRINGZILLAS_CPUS_SOURCES - + STRINGZILLAS_RUNTIME_SOURCES, + + STRINGZILLAS_RUNTIME_SOURCES + # The multi-pattern rewrite path calls the dispatched `sz_copy`, so this wheel carries the core + # runtime the way every CMake target links `stringzilla_static` rather than leaving it undefined. + + STRINGZILLA_CORE_SOURCES, include_dirs=["include", "c/stringzillas", "forkunion/include"], extra_compile_args=compile_args, extra_link_args=link_args, @@ -824,13 +832,15 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ "python/stringzillas/device_scope.c", "python/stringzillas/similarities.c", "python/stringzillas/fingerprints.c", + "python/stringzillas/substrings.c", ] + STRINGZILLAS_API_CU_SOURCES + STRINGZILLAS_CPUS_SOURCES + STRINGZILLAS_CUDA_SOURCES + STRINGZILLAS_KEPLER_SOURCES + STRINGZILLAS_HOPPER_SOURCES - + STRINGZILLAS_RUNTIME_SOURCES, + + STRINGZILLAS_RUNTIME_SOURCES + + STRINGZILLA_CORE_SOURCES, include_dirs=["include", "c/stringzillas", "forkunion/include", f"{cuda_home}/include"], extra_compile_args=compile_args, extra_link_args=cuda_link_args, @@ -871,7 +881,8 @@ def windows_settings(use_cpp: bool = False) -> Tuple[List[str], List[str], List[ install_requires = [] if sz_target != "stringzilla": # Keep versions in lockstep to ensure ABI compatibility - install_requires = [f"stringzilla=={__version__}"] + # The parallel modules call `import_array()`, so NumPy is a runtime dependency and not merely a build one. + install_requires = [f"stringzilla=={__version__}", "numpy"] setup( name=__lib_name__, diff --git a/swift/README.md b/swift/README.md index da5db2b5..ff58d128 100644 --- a/swift/README.md +++ b/swift/README.md @@ -16,7 +16,7 @@ Add StringZilla as a Swift Package Manager dependency in your `Package.swift`. let package = Package( name: "MyApp", dependencies: [ - .package(url: "https://github.com/ashvardanian/StringZilla.git", from: "4.0.0") + .package(url: "https://github.com/ashvardanian/StringZilla.git", from: "5.0.0") ], targets: [ .target( diff --git a/swift/StringProtocol+StringZilla.swift b/swift/StringProtocol+StringZilla.swift index 233a67c5..8e206ddd 100644 --- a/swift/StringProtocol+StringZilla.swift +++ b/swift/StringProtocol+StringZilla.swift @@ -111,7 +111,7 @@ public protocol StringZillaViewable: Collection { /// /// - Parameters: /// - bytePointer: A pointer to the byte for which the offset is calculated. - /// - startPointer: The starting pointer for the calculation, previously obtained from `szScope`. + /// - startPointer: The starting pointer for the calculation, previously obtained from `withStringZillaScope`. /// - Returns: The calculated index offset. func stringZillaByteOffset(forByte bytePointer: sz_cptr_t, after startPointer: sz_cptr_t) -> Index } @@ -157,7 +157,7 @@ extension Substring.UTF8View: StringZillaViewable { /// Calculates the offset index for a given byte pointer relative to a start pointer. /// - Parameters: /// - bytePointer: A pointer to the byte for which the offset is calculated. - /// - startPointer: The starting pointer for the calculation, previously obtained from `szScope`. + /// - startPointer: The starting pointer for the calculation, previously obtained from `withStringZillaScope`. /// - Returns: The calculated index offset. @_transparent public func stringZillaByteOffset(forByte bytePointer: sz_cptr_t, after startPointer: sz_cptr_t) @@ -188,7 +188,7 @@ extension String.UTF8View: StringZillaViewable { /// Calculates the offset index for a given byte pointer relative to a start pointer. /// - Parameters: /// - bytePointer: A pointer to the byte for which the offset is calculated. - /// - startPointer: The starting pointer for the calculation, previously obtained from `szScope`. + /// - startPointer: The starting pointer for the calculation, previously obtained from `withStringZillaScope`. /// - Returns: The calculated index offset. public func stringZillaByteOffset(forByte bytePointer: sz_cptr_t, after startPointer: sz_cptr_t) -> Index @@ -682,8 +682,8 @@ public class StringZillaHasher { /// Alias for `finalize()`. public func digest() -> UInt64 { return finalize() } - /// Resets the hasher to its initial state with the same seed. - /// - Parameter seed: Optional new seed value (if nil, uses the original seed). + /// Resets the hasher to its initial state. + /// - Parameter seed: New seed value; the original seed is not retained, so omitting this re-seeds with 0. public func reset(seed: UInt64? = nil) { let newSeed = seed ?? 0 // Default to 0 if no seed provided sz_hash_state_init(&state, newSeed) diff --git a/test/README.md b/test/README.md index a7ca3781..bf8c4948 100644 --- a/test/README.md +++ b/test/README.md @@ -15,8 +15,8 @@ Each C++ translation unit exercises one kernel family, and the Python suite mirr The Python modules mirror the C++ translation units one-for-one and run under pytest. -- `find.py`, `hash.py`, `sort.py`, `string.py`, `uncased.py`, `cipher.py`, `utf8_*.py`, `doctests.py`, `stringzilla.py`, `stringzillas.py` — per-family tests. -- `helpers.py` and `utf8_helpers.py` are shared helpers; `conftest.py` holds the pytest configuration. +- `find.py`, `hash.py`, `sort.py`, `string.py`, `uncased.py`, `cipher.py`, `similarities.py`, `fingerprints.py`, `substrings.py`, `utf8_*.py`, `doctests.py`, `stringzillas.py` — per-family tests. +- `sz_helpers.py`, `szs_helpers.py`, and `utf8_helpers.py` are shared helpers; `conftest.py` holds the pytest configuration. - This directory is a Python package via `__init__.py`, so the prefix-less modules namespace as `test.*` and never shadow stdlib names. - Run the suite with `pytest test/`. diff --git a/test/cipher.cpp b/test/cipher.cpp index a4e3a5ae..0e73ecec 100644 --- a/test/cipher.cpp +++ b/test/cipher.cpp @@ -151,12 +151,12 @@ struct gcm_backend_t { }; /** - * @brief Every counter-mode backend compiled into this translation unit, serial first. + * @brief Every counter-mode backend compiled into this translation unit, dispatched first. * * The dispatched entry points lead, because a published vector has to reach whatever the dispatcher * picks as well as each kernel named outright. */ -static ctr_backend_t const ctr_backends_[] = { +static ctr_backend_t const ctr_backends[] = { {"dispatched", sz_aes256_key_init, sz_aes256_ctr_xor}, {"serial", sz_aes256_key_init_serial, sz_aes256_ctr_xor_serial}, #if SZ_USE_WESTMERE @@ -185,8 +185,8 @@ static ctr_backend_t const ctr_backends_[] = { #endif }; -/** @brief Every authenticated backend compiled into this translation unit, serial first. */ -static gcm_backend_t const gcm_backends_[] = { +/** @brief Every authenticated backend compiled into this translation unit, dispatched first. */ +static gcm_backend_t const gcm_backends[] = { {"dispatched", sz_aes256_gcm_key_init, sz_aes256_gcm_encrypt, sz_aes256_gcm_decrypt, sz_aes256_gcm_encryptor_init, sz_aes256_gcm_encryptor_associate, sz_aes256_gcm_encryptor_update, sz_aes256_gcm_encryptor_digest, sz_aes256_gcm_decryptor_init, sz_aes256_gcm_decryptor_associate, sz_aes256_gcm_decryptor_update_unverified, @@ -335,10 +335,10 @@ static void check_gcm_unit_(gcm_backend_t const &backend, known_gcm_t const &vec */ void test_cipher_unit() { - for (ctr_backend_t const &backend : ctr_backends_) + for (ctr_backend_t const &backend : ctr_backends) for (known_ctr_t const &vector : known_ctr_vectors_) check_ctr_unit_(backend, vector); - for (gcm_backend_t const &backend : gcm_backends_) + for (gcm_backend_t const &backend : gcm_backends) for (known_gcm_t const &vector : known_gcm_vectors_) check_gcm_unit_(backend, vector); } @@ -352,7 +352,7 @@ void test_cipher_unit() { * Seeking is the whole reason counter mode is exposed separately, so every offset is compared against * the same bytes taken from a from-zero encryption rather than only against the reference backend. */ -void test_ctr_equivalence(ctr_backend_t const &reference, ctr_backend_t const &candidate, sz_size_t inputs) { +void check_ctr_equivalence_(ctr_backend_t const &reference, ctr_backend_t const &candidate, sz_size_t inputs) { sz_u8_t secret[32], nonce[12]; for (std::size_t index = 0; index != 32; ++index) secret[index] = (sz_u8_t)(index * 7 + 1); for (std::size_t index = 0; index != 12; ++index) nonce[index] = (sz_u8_t)(index * 5 + 2); @@ -402,7 +402,7 @@ void test_ctr_equivalence(ctr_backend_t const &reference, ctr_backend_t const &c * passing one pointer for both sides reaches the bytes and the tag two pointers would, and a rejected * tag still clears the buffer it was handed. */ -void test_gcm_equivalence(gcm_backend_t const &reference, gcm_backend_t const &candidate, sz_size_t inputs) { +void check_gcm_equivalence_(gcm_backend_t const &reference, gcm_backend_t const &candidate, sz_size_t inputs) { sz_u8_t secret[32], nonce[12], reference_tag[16], candidate_tag[16]; for (std::size_t index = 0; index != 32; ++index) secret[index] = (sz_u8_t)(index * 3 + 5); for (std::size_t index = 0; index != 12; ++index) nonce[index] = (sz_u8_t)(index + 9); @@ -582,18 +582,27 @@ void test_cipher_safety() { } } +/** @brief The row named @p name, so a reordering cannot silently hand the differential a new reference. */ +template <typename backend_type_, std::size_t count_> +backend_type_ const &backend_named_(backend_type_ const (&backends)[count_], char const *name) { + for (std::size_t index = 0; index != count_; ++index) + if (std::strcmp(backends[index].name, name) == 0) return backends[index]; + verify(false && "The backend table must carry the named reference"); + return backends[0]; +} + /** @brief Drives the serial-versus-SIMD differential across every cipher backend compiled here. */ void test_cipher_all() { // Each length sweeps a fresh buffer, so the work grows with the square of the count. sz_size_t const cipher_inputs = (sz_size_t)scale_iterations_quadratic(160); - // The serial backend leads the table and is the reference for everything after it. Running it - // against itself first catches a streaming path that disagrees with its own one-shot kernel. - ctr_backend_t const &ctr_reference = ctr_backends_[1]; - gcm_backend_t const &gcm_reference = gcm_backends_[1]; - for (ctr_backend_t const &candidate : ctr_backends_) test_ctr_equivalence(ctr_reference, candidate, cipher_inputs); - for (gcm_backend_t const &candidate : gcm_backends_) test_gcm_equivalence(gcm_reference, candidate, cipher_inputs); + // Serial is the reference for everything, itself included: running it against itself catches a streaming + // path that disagrees with its own one-shot kernel. + ctr_backend_t const &ctr_reference = backend_named_(ctr_backends, "serial"); + gcm_backend_t const &gcm_reference = backend_named_(gcm_backends, "serial"); + for (ctr_backend_t const &candidate : ctr_backends) check_ctr_equivalence_(ctr_reference, candidate, cipher_inputs); + for (gcm_backend_t const &candidate : gcm_backends) check_gcm_equivalence_(gcm_reference, candidate, cipher_inputs); } #pragma endregion // Drivers diff --git a/test/conftest.py b/test/conftest.py index 0bd155e4..d0816e03 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,8 +2,8 @@ Shared pytest configuration for the StringZilla per-family test modules. Hosts the session-wide environment banner and the QEMU capability mask so every split test file -(test_string.py, test_find.py, test_utf8_wordbreaks.py, â€Ļ) inherits them without importing anything. The -seeded-RNG helpers and `SEED_VALUES` live in `test_helpers` and are imported by each module directly. +(string.py, find.py, utf8_wordbreaks.py, â€Ļ) inherits them without importing anything. The +seeded-RNG helpers and `SEED_VALUES` live in `test.sz_helpers` and are imported by each module directly. """ import os diff --git a/test/doctests.py b/test/doctests.py index 266078b4..548782ef 100644 --- a/test/doctests.py +++ b/test/doctests.py @@ -5,7 +5,7 @@ Run with:: - python -m pytest scripts/test_doctests.py -v + python -m pytest test/doctests.py -v """ import doctest diff --git a/test/find.cpp b/test/find.cpp index b5eec031..8f3baeb5 100644 --- a/test/find.cpp +++ b/test/find.cpp @@ -1,6 +1,6 @@ /** * @brief Comparisons, search/find_all/split, misaligned-repetition search, and replacement tests. - * @file scripts/test_find.cpp + * @file test/find.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -49,19 +49,12 @@ #include <cstdio> // `std::printf` #include <cstring> // `std::memcpy` -#include <algorithm> // `std::transform` -#include <iterator> // `std::distance` -#include <map> // `std::map` -#include <memory> // `std::allocator` -#include <numeric> // `std::accumulate` -#include <random> // `std::random_device` -#include <set> // `std::set` -#include <sstream> // `std::ostringstream` -#include <string> // Baseline -#include <string_view> // Baseline -#include <unordered_map> // `std::unordered_map` -#include <unordered_set> // `std::unordered_set` -#include <vector> // `std::vector` +#include <algorithm> // `std::transform` +#include <iterator> // `std::distance` +#include <random> // `std::uniform_int_distribution` +#include <string> // Baseline +#include <string_view> // Baseline +#include <vector> // `std::vector` #if !SZ_IS_CPP11_ #error "This test requires C++11 or later." @@ -134,9 +127,94 @@ static void check_find_unit_( // // The C++ `sz::string_view` wrapper resolves to the same offsets. sz::string_view const haystack_view(haystack, haystack_length); sz::string_view const needle_view(needle, needle_length); - verify(haystack_view.find(needle_view) == (forward_offset == SZ_SIZE_MAX ? sz::string_view::npos : forward_offset)); + verify(haystack_view.find(needle_view) == + (forward_offset == SZ_SIZE_MAX ? sz::string_view::npos : forward_offset) && + "sz::string_view::find must agree with the C API's forward offset"); verify(haystack_view.rfind(needle_view) == - (backward_offset == SZ_SIZE_MAX ? sz::string_view::npos : backward_offset)); + (backward_offset == SZ_SIZE_MAX ? sz::string_view::npos : backward_offset) && + "sz::string_view::rfind must agree with the C API's backward offset"); +} + +/** @brief Lengths every comparison and byte-scan size ladder switches at, plus one either side of each. */ +static sz_size_t const backend_ladder_lengths_[] = {1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255}; + +/** + * @brief Drives one comparison backend across the lengths its size ladder switches at. + * + * Every backend delegates to serial below its vector width, so the three-byte literals above never reach the + * vectorized body at all. The overlapping-load boundaries at 8, 16, 32 and 64 are where these ladders break, + * and a mismatch is placed at the front, the middle and the last byte of each length to catch a tail that + * reads one byte too few or one too many. + */ +static void check_compare_unit_(char const *name, sz_equal_t equal, sz_order_t order) { + std::string left, right; + for (sz_size_t const length : span_over(backend_ladder_lengths_)) { + left.assign((std::size_t)length, 'a'); + + right = left; + if (equal(left.data(), right.data(), length) != sz_true_k) { + std::fprintf(stderr, "%s: equal() denied identical %zu-byte inputs\n", name, (std::size_t)length); + verify(false && "Comparison backend must accept identical inputs at every ladder length"); + } + verify(order(left.data(), length, right.data(), length) == sz_equal_k && + "Comparison backend must order two identical inputs as equal"); + + sz_size_t const positions[] = {0, length / 2, length - 1}; + for (sz_size_t const position : span_over(positions)) { + right = left, right[(std::size_t)position] = 'b'; // ? 'b' sorts after 'a', so `left` is the lesser + if (equal(left.data(), right.data(), length) != sz_false_k) { + std::fprintf(stderr, "%s: equal() missed a difference at byte %zu of %zu\n", name, + (std::size_t)position, (std::size_t)length); + verify(false && "Comparison backend must see a single differing byte at every ladder length"); + } + verify(order(left.data(), length, right.data(), length) == sz_less_k && + "Comparison backend must order the byte-decreased input as lesser"); + verify(order(right.data(), length, left.data(), length) == sz_greater_k && + "Comparison backend must order the byte-increased input as greater"); + } + } +} + +/** + * @brief Drives one byte-scanner pair across the same ladder, with the match at both ends and absent. + * + * The reverse scanners are the risky half: each backend reaches "last match" differently - predicate + * reversal, gather-reversal, or a leading-zero count with a hand-written offset correction - and all of it + * is index arithmetic that an eleven-byte literal never exercises. + */ +static void check_find_byte_unit_(char const *name, sz_find_byte_t find_byte, sz_find_byte_t rfind_byte) { + std::string haystack; + for (sz_size_t const length : span_over(backend_ladder_lengths_)) { + haystack.assign((std::size_t)length, 'a'); + verify(find_byte(haystack.data(), length, "z") == SZ_NULL_CHAR && "Forward byte scan must miss an absent byte"); + verify(rfind_byte(haystack.data(), length, "z") == SZ_NULL_CHAR && + "Reverse byte scan must miss an absent byte"); + + sz_size_t const positions[] = {0, length / 2, length - 1}; + for (sz_size_t const position : span_over(positions)) { + haystack.assign((std::size_t)length, 'a'); + haystack[(std::size_t)position] = 'z'; + sz_cptr_t const expected = haystack.data() + position; + if (find_byte(haystack.data(), length, "z") != expected) { + std::fprintf(stderr, "%s: find_byte missed the byte at %zu of %zu\n", name, (std::size_t)position, + (std::size_t)length); + verify(false && "Forward byte scan must find a lone match at every ladder length"); + } + if (rfind_byte(haystack.data(), length, "z") != expected) { + std::fprintf(stderr, "%s: rfind_byte missed the byte at %zu of %zu\n", name, (std::size_t)position, + (std::size_t)length); + verify(false && "Reverse byte scan must find a lone match at every ladder length"); + } + } + + // Both ends occupied, so forward and reverse must disagree about which one they report. + haystack.assign((std::size_t)length, 'a'); + haystack[0] = 'z', haystack[(std::size_t)length - 1] = 'z'; + verify(find_byte(haystack.data(), length, "z") == haystack.data() && + "Forward byte scan must report the first of two matches"); + verify(rfind_byte(haystack.data(), length, "z") == haystack.data() + length - 1 && + "Reverse byte scan must report the last of two matches"); + } } #pragma endregion // Helpers @@ -172,24 +250,48 @@ void test_find_unit() { check_find_unit_(hello, hello_length, "xyz", 3, SZ_SIZE_MAX, SZ_SIZE_MAX); // Missing needle // `sz_find_byte` / `sz_rfind_byte` in isolation: the byte 'l' occurs at offsets 2, 3, and 9. - verify(sz_find_byte(hello, hello_length, "l") == hello + 2); // Dispatched (automatic kernel) - verify(sz_rfind_byte(hello, hello_length, "l") == hello + 9); // Dispatched (automatic kernel) - verify(sz_find_byte_serial(hello, hello_length, "l") == hello + 2); // Manual propagation to the serial kernel - verify(sz_rfind_byte_serial(hello, hello_length, "l") == hello + 9); // Manual propagation to the serial kernel + // Dispatched (automatic kernel resolution). + verify(sz_find_byte(hello, hello_length, "l") == hello + 2); + verify(sz_rfind_byte(hello, hello_length, "l") == hello + 9); + // Manual propagation to the serial kernel. + verify(sz_find_byte_serial(hello, hello_length, "l") == hello + 2); + verify(sz_rfind_byte_serial(hello, hello_length, "l") == hello + 9); verify(sz_find_byte(hello, hello_length, "z") == SZ_NULL_CHAR); // Missing byte verify(sz_rfind_byte(hello, hello_length, "z") == SZ_NULL_CHAR); // Missing byte - verify(sz_find_byte_serial(hello, hello_length, "z") == SZ_NULL_CHAR); // Missing byte, serial kernel + verify(sz_find_byte_serial(hello, hello_length, "z") == SZ_NULL_CHAR); // Missing byte + // Every compiled tier, swept across the lengths its ladder switches at - "hello world" is eleven bytes, + // so the literals above only ever reach each backend's scalar tail. + check_find_byte_unit_("dispatched", sz_find_byte, sz_rfind_byte); + check_find_byte_unit_("serial", sz_find_byte_serial, sz_rfind_byte_serial); #if SZ_USE_WESTMERE - verify(sz_find_byte_westmere(hello, hello_length, "l") == hello + 2); - verify(sz_rfind_byte_westmere(hello, hello_length, "l") == hello + 9); + check_find_byte_unit_("westmere", sz_find_byte_westmere, sz_rfind_byte_westmere); #endif #if SZ_USE_HASWELL - verify(sz_find_byte_haswell(hello, hello_length, "l") == hello + 2); - verify(sz_rfind_byte_haswell(hello, hello_length, "l") == hello + 9); + check_find_byte_unit_("haswell", sz_find_byte_haswell, sz_rfind_byte_haswell); #endif #if SZ_USE_SKYLAKE - verify(sz_find_byte_skylake(hello, hello_length, "l") == hello + 2); - verify(sz_rfind_byte_skylake(hello, hello_length, "l") == hello + 9); + check_find_byte_unit_("skylake", sz_find_byte_skylake, sz_rfind_byte_skylake); +#endif +#if SZ_USE_NEON + check_find_byte_unit_("neon", sz_find_byte_neon, sz_rfind_byte_neon); +#endif +#if SZ_USE_SVE + check_find_byte_unit_("sve", sz_find_byte_sve, sz_rfind_byte_sve); +#endif +#if SZ_USE_V128 + check_find_byte_unit_("v128", sz_find_byte_v128, sz_rfind_byte_v128); +#endif +#if SZ_USE_V128RELAXED + check_find_byte_unit_("v128relaxed", sz_find_byte_v128relaxed, sz_rfind_byte_v128relaxed); +#endif +#if SZ_USE_RVV + check_find_byte_unit_("rvv", sz_find_byte_rvv, sz_rfind_byte_rvv); +#endif +#if SZ_USE_LASX + check_find_byte_unit_("lasx", sz_find_byte_lasx, sz_rfind_byte_lasx); +#endif +#if SZ_USE_POWERVSX + check_find_byte_unit_("powervsx", sz_find_byte_powervsx, sz_rfind_byte_powervsx); #endif // `sz_find_byteset` / `sz_rfind_byteset`: a set of vowels {a, e, i, o, u} first hits 'e' at offset 1 @@ -201,199 +303,85 @@ void test_find_unit() { sz_byteset_add(&vowels, 'i'); sz_byteset_add(&vowels, 'o'); sz_byteset_add(&vowels, 'u'); - verify(sz_find_byteset(hello, hello_length, &vowels) == hello + 1); // Dispatched (automatic kernel) - verify(sz_rfind_byteset(hello, hello_length, &vowels) == hello + 7); // Dispatched (automatic kernel) - verify(sz_find_byteset_serial(hello, hello_length, &vowels) == - hello + 1); // Manual propagation to the serial kernel - verify(sz_rfind_byteset_serial(hello, hello_length, &vowels) == - hello + 7); // Manual propagation to the serial kernel + // Dispatched (automatic kernel resolution). + verify(sz_find_byteset(hello, hello_length, &vowels) == hello + 1); + verify(sz_rfind_byteset(hello, hello_length, &vowels) == hello + 7); + // Manual propagation to the serial kernel. + verify(sz_find_byteset_serial(hello, hello_length, &vowels) == hello + 1); + verify(sz_rfind_byteset_serial(hello, hello_length, &vowels) == hello + 7); // A set with none of the present bytes returns `SZ_NULL_CHAR`. sz_byteset_t digits; sz_byteset_init(&digits); sz_byteset_add(&digits, '0'); sz_byteset_add(&digits, '9'); verify(sz_find_byteset(hello, hello_length, &digits) == SZ_NULL_CHAR); // No digit present - verify(sz_find_byteset_serial(hello, hello_length, &digits) == SZ_NULL_CHAR); // No digit present, serial kernel + verify(sz_find_byteset_serial(hello, hello_length, &digits) == SZ_NULL_CHAR); // No digit present + + // `sz_find_byte_from` / `sz_find_byte_not_from` / `sz_rfind_byte_from` / `sz_rfind_byte_not_from`: + // the `_from` family takes the needle bytes as the accepted set (the byteset built out of them), and + // the `_not_from` family inverts that set, so they are the byteset family above spelled with a needle + // string in place of an `sz_byteset_t`. Against "hello world" and the needle "helo" (accepts h, e, l, o): + // the first accepted byte is 'h' at offset 0 and the last is 'l' at offset 9; the first byte NOT in the + // set is ' ' at offset 5, and the last byte not in the set is 'd' at offset 10. + verify(sz_find_byte_from(hello, hello_length, "helo", 4) == hello + 0); // First accepted: 'h' + verify(sz_rfind_byte_from(hello, hello_length, "helo", 4) == hello + 9); // Last accepted: 'l' + verify(sz_find_byte_not_from(hello, hello_length, "helo", 4) == hello + 5); // First rejected: ' ' + verify(sz_rfind_byte_not_from(hello, hello_length, "helo", 4) == hello + 10); // Last rejected: 'd' + // An empty needle accepts nothing, so `_from` must miss everywhere and `_not_from` must hit immediately. + verify(sz_find_byte_from(hello, hello_length, "", 0) == SZ_NULL_CHAR); + verify(sz_rfind_byte_from(hello, hello_length, "", 0) == SZ_NULL_CHAR); + verify(sz_find_byte_not_from(hello, hello_length, "", 0) == hello + 0); + verify(sz_rfind_byte_not_from(hello, hello_length, "", 0) == hello + hello_length - 1); + // A needle covering the whole alphabet present in the haystack leaves `_not_from` with nothing to reject. + verify(sz_find_byte_not_from(hello, hello_length, hello, hello_length) == SZ_NULL_CHAR); + verify(sz_rfind_byte_not_from(hello, hello_length, hello, hello_length) == SZ_NULL_CHAR); // `sz_order` / `sz_equal`: lexicographic ordering and byte-equality on hand-verifiable pairs. - verify(sz_order("abc", 3, "abc", 3) == sz_equal_k); // Equal strings - verify(sz_order("abc", 3, "abd", 3) == sz_less_k); // Differ in the last byte - verify(sz_order("abd", 3, "abc", 3) == sz_greater_k); // Differ in the last byte - verify(sz_order("ab", 2, "abc", 3) == sz_less_k); // Prefix orders before the longer string - verify(sz_order("abc", 3, "ab", 2) == sz_greater_k); // Longer string orders after its prefix - verify(sz_equal("abc", "abc", 3) == sz_true_k); // Identical bytes - verify(sz_equal("abc", "abd", 3) == sz_false_k); // Differing bytes - verify(sz_order_serial("abc", 3, "abd", 3) == sz_less_k); // Manual propagation to the serial kernel - verify(sz_equal_serial("abc", "abc", 3) == sz_true_k); // Manual propagation to the serial kernel - verify(sz_equal_serial("abc", "abd", 3) == sz_false_k); // Manual propagation to the serial kernel + verify(sz_order("abc", 3, "abc", 3) == sz_equal_k); // Equal strings + verify(sz_order("abc", 3, "abd", 3) == sz_less_k); // Differ in the last byte + verify(sz_order("abd", 3, "abc", 3) == sz_greater_k); // Differ in the last byte + verify(sz_order("ab", 2, "abc", 3) == sz_less_k); // Prefix orders before the longer string + verify(sz_order("abc", 3, "ab", 2) == sz_greater_k); // Longer string orders after its prefix + verify(sz_equal("abc", "abc", 3) == sz_true_k); // Identical bytes + verify(sz_equal("abc", "abd", 3) == sz_false_k); // Differing bytes + // Manual propagation to the serial kernel. + verify(sz_order_serial("abc", 3, "abd", 3) == sz_less_k); + verify(sz_equal_serial("abc", "abc", 3) == sz_true_k); + verify(sz_equal_serial("abc", "abd", 3) == sz_false_k); + // As above: the three-byte literals never reach a vectorized body, so every tier is swept across the + // lengths its ladder switches at. + check_compare_unit_("dispatched", sz_equal, sz_order); + check_compare_unit_("serial", sz_equal_serial, sz_order_serial); +#if SZ_USE_WESTMERE + check_compare_unit_("westmere", sz_equal_westmere, sz_order_westmere); +#endif #if SZ_USE_HASWELL - verify(sz_order_haswell("abc", 3, "abd", 3) == sz_less_k); - verify(sz_equal_haswell("abc", "abc", 3) == sz_true_k); - verify(sz_equal_haswell("abc", "abd", 3) == sz_false_k); + check_compare_unit_("haswell", sz_equal_haswell, sz_order_haswell); #endif #if SZ_USE_SKYLAKE - verify(sz_order_skylake("abc", 3, "abd", 3) == sz_less_k); - verify(sz_equal_skylake("abc", "abc", 3) == sz_true_k); - verify(sz_equal_skylake("abc", "abd", 3) == sz_false_k); + check_compare_unit_("skylake", sz_equal_skylake, sz_order_skylake); +#endif +#if SZ_USE_NEON + check_compare_unit_("neon", sz_equal_neon, sz_order_neon); +#endif +#if SZ_USE_SVE + check_compare_unit_("sve", sz_equal_sve, sz_order_sve); +#endif +#if SZ_USE_V128 + check_compare_unit_("v128", sz_equal_v128, sz_order_v128); +#endif +#if SZ_USE_V128RELAXED + check_compare_unit_("v128relaxed", sz_equal_v128relaxed, sz_order_v128relaxed); +#endif +#if SZ_USE_RVV + check_compare_unit_("rvv", sz_equal_rvv, sz_order_rvv); +#endif +#if SZ_USE_LASX + check_compare_unit_("lasx", sz_equal_lasx, sz_order_lasx); +#endif +#if SZ_USE_POWERVSX + check_compare_unit_("powervsx", sz_equal_powervsx, sz_order_powervsx); #endif - // And the same orderings through the C++ `sz::string_view` comparison operators. - verify("abc"_sv == "abc"_sv); // Equality operator - verify("abc"_sv != "abd"_sv); // Inequality operator - verify("abc"_sv < "abd"_sv); // Strictly-less operator - verify("abd"_sv > "abc"_sv); // Strictly-greater operator - verify("ab"_sv < "abc"_sv); // Prefix orders before the longer string - - // Searching for a set of characters - verify(sz::string_view("a").find_first_of("az") == 0); - verify(sz::string_view("a").find_last_of("az") == 0); - verify(sz::string_view("a").find_first_of("xz") == sz::string_view::npos); - verify(sz::string_view("a").find_last_of("xz") == sz::string_view::npos); - - verify(sz::string_view("a").find_first_not_of("xz") == 0); - verify(sz::string_view("a").find_last_not_of("xz") == 0); - verify(sz::string_view("a").find_first_not_of("az") == sz::string_view::npos); - verify(sz::string_view("a").find_last_not_of("az") == sz::string_view::npos); - - verify(sz::string_view("aXbYaXbY").find_first_of("XY") == 1); - verify(sz::string_view("axbYaxbY").find_first_of("Y") == 3); - verify(sz::string_view("YbXaYbXa").find_last_of("XY") == 6); - verify(sz::string_view("YbxaYbxa").find_last_of("Y") == 4); - verify(sz::string_view(sz::base64(), sizeof(sz::base64())).find_first_of("_") == sz::string_view::npos); - verify(sz::string_view(sz::base64(), sizeof(sz::base64())).find_first_of("+") == 62); - verify(sz::string_view(sz::ascii_printables(), sizeof(sz::ascii_printables())).find_first_of("~") != - sz::string_view::npos); - - verify("aabaa"_sv.remove_prefix("a") == "abaa"); - verify("aabaa"_sv.remove_suffix("a") == "aaba"); - verify("aabaa"_sv.lstrip("a"_bs) == "baa"); - verify("aabaa"_sv.rstrip("a"_bs) == "aab"); - verify("aabaa"_sv.strip("a"_bs) == "b"); - - // Check more advanced composite operations - verify("abbccc"_sv.partition('b').before.size() == 1); - verify("abbccc"_sv.partition("bb").before.size() == 1); - verify("abbccc"_sv.partition("bb").match.size() == 2); - verify("abbccc"_sv.partition("bb").after.size() == 3); - verify("abbccc"_sv.partition("bb").before == "a"); - verify("abbccc"_sv.partition("bb").match == "bb"); - verify("abbccc"_sv.partition("bb").after == "ccc"); - verify("abb ccc"_sv.partition(sz::whitespaces_set()).after == "ccc"); - - // Check ranges of search matches - verify("hello"_sv.find_all("l").size() == 2); - verify("hello"_sv.rfind_all("l").size() == 2); - - verify(""_sv.find_all(".", sz::include_overlaps_type {}).size() == 0); - verify(""_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 0); - verify("."_sv.find_all(".", sz::include_overlaps_type {}).size() == 1); - verify("."_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 1); - verify(".."_sv.find_all(".", sz::include_overlaps_type {}).size() == 2); - verify(".."_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 2); - verify(""_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 0); - verify(""_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 0); - verify("."_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 1); - verify("."_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 1); - verify(".."_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 2); - verify(".."_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 2); - - verify("a.b.c.d"_sv.find_all(".").size() == 3); - verify("a.,b.,c.,d"_sv.find_all(".,").size() == 3); - verify("a.,b.,c.,d"_sv.rfind_all(".,").size() == 3); - verify("a.b,c.d"_sv.find_all(".,"_bs).size() == 3); - verify("a...b...c"_sv.rfind_all("..").size() == 4); - verify("a...b...c"_sv.rfind_all("..", sz::include_overlaps_type {}).size() == 4); - verify("a...b...c"_sv.rfind_all("..", sz::exclude_overlaps_type {}).size() == 2); - - let_verify(auto finds = "a.b.c"_sv.find_all("abcd"_bs).template to<std::vector<std::string>>(), - finds.size() == 3 && finds[0] == "a"); - let_verify(auto rfinds = "a.b.c"_sv.rfind_all("abcd"_bs).template to<std::vector<std::string>>(), - rfinds.size() == 3 && rfinds[0] == "c"); - - // Test propagating strings and their non-owning views into temporary ranges and iterators - verify(sz::find_all("abc"_sv, "b"_sv).size() == 1); - verify(sz::find_all("hello"_sv, "l"_sv).size() == 2); - verify(sz::rfind_all("abc"_sv, "b"_sv).size() == 1); - - { - sz::string h("abc"), n("b"); - verify(sz::find_all(h, n).size() == 1); - } - { - sz::string h("hello"), n("l"); - verify(sz::find_all(h, n).size() == 2); - } - { - sz::string h("abc"), n("b"); - verify(sz::rfind_all(h, n).size() == 1); - } - - verify(sz::find_all(sz::string("abc"), sz::string("b")).size() == 1); - verify(sz::find_all(sz::string("hello"), sz::string("l")).size() == 2); - verify(sz::rfind_all(sz::string("abc"), sz::string("b")).size() == 1); - - // Lvalue haystacks are borrowed, so slices land inside the caller's own buffer. A copied - // haystack would offset into a private copy - and under SSO those offsets look plausible. - { - sz::string haystack("hello world, hello cpp"); - sz::string sso("a b a"); - let_verify(auto matches = sz::find_all(haystack, "hello").template to<std::vector<sz::string_view>>(), - matches.size() == 2 && // - matches[0].data() - haystack.data() == 0 && // - matches[1].data() - haystack.data() == 13); - let_verify(auto in_sso = sz::find_all(sso, "a").template to<std::vector<sz::string_view>>(), - in_sso.size() == 2 && // - in_sso[0].data() - sso.data() == 0 && // - in_sso[1].data() - sso.data() == 4); - } - - // Needles are copied into the matcher, so a temporary one outlives the expression that built it. - verify(sz::find_all(sz::string("hello world, hello cpp"), sz::string("hello")).size() == 2); - - // Haystack and needle need not share a type - literals, views, and owning strings mix. - { - sz::string owning("a-b-c"); - sz::string_view view("a-b-c"); - sz::string needle("-"); - verify(sz::find_all(view, "-").size() == 2); - verify(sz::find_all(owning, "-").size() == 2); - verify(sz::find_all(owning, view.substr(1, 1)).size() == 2); - verify(sz::find_all(view, needle).size() == 2); - verify(sz::split(owning, "-").size() == 3); - verify(sz::rsplit(view, needle).size() == 3); - verify(sz::split_characters(owning, "-").size() == 3); - } - - // Check splitting - the inverse of `find_all` ranges - let_verify(auto splits = ".a..c."_sv.split("."_bs).template to<std::vector<std::string>>(), - splits.size() == 5 && splits[0] == "" && splits[1] == "a" && splits[4] == ""); - let_verify(auto line_splits = "line1\nline2\nline3"_sv.split("line3").template to<std::vector<std::string>>(), - line_splits.size() == 2 && line_splits[0] == "line1\nline2\n" && line_splits[1] == ""); - - verify(""_sv.split(".").size() == 1); - verify(""_sv.rsplit(".").size() == 1); - - verify("hello"_sv.split("l").size() == 3); - verify("hello"_sv.rsplit("l").size() == 3); - verify(*advanced("hello"_sv.split("l").begin(), 0) == "he"); - verify(*advanced("hello"_sv.rsplit("l").begin(), 0) == "o"); - verify(*advanced("hello"_sv.split("l").begin(), 1) == ""); - verify(*advanced("hello"_sv.rsplit("l").begin(), 1) == ""); - verify(*advanced("hello"_sv.split("l").begin(), 2) == "o"); - verify(*advanced("hello"_sv.rsplit("l").begin(), 2) == "he"); - - verify("a.b.c.d"_sv.split(".").size() == 4); - verify("a.b.c.d"_sv.rsplit(".").size() == 4); - verify(*("a.b.c.d"_sv.split(".").begin()) == "a"); - verify(*("a.b.c.d"_sv.rsplit(".").begin()) == "d"); - verify(*advanced("a.b.c.d"_sv.split(".").begin(), 1) == "b"); - verify(*advanced("a.b.c.d"_sv.rsplit(".").begin(), 1) == "c"); - verify(*advanced("a.b.c.d"_sv.split(".").begin(), 3) == "d"); - verify(*advanced("a.b.c.d"_sv.rsplit(".").begin(), 3) == "a"); - verify("a.b.,c,d"_sv.split(".,").size() == 2); - verify("a.b,c.d"_sv.split(".,"_bs).size() == 4); - - let_verify(auto rsplits = ".a..c."_sv.rsplit("."_bs).template to<std::vector<std::string>>(), - rsplits.size() == 5 && rsplits[0] == "" && rsplits[1] == "c" && rsplits[4] == ""); } /** @@ -410,6 +398,13 @@ void test_compare_unit() { verify("a"_sv == "a"_sv); verify("a"_sv != "a\0"_sv); verify("a\0"_sv == "a\0"_sv); + + // The relational operators over the same orderings. + verify("abc"_sv == "abc"_sv); // Equality operator + verify("abc"_sv != "abd"_sv); // Inequality operator + verify("abc"_sv < "abd"_sv); // Strictly-less operator + verify("abd"_sv > "abc"_sv); // Strictly-greater operator + verify("ab"_sv < "abc"_sv); // Prefix orders before the longer string } #pragma endregion // Unit @@ -420,7 +415,7 @@ void test_compare_unit() { * @brief One substring-search backend (find or rfind), stored by pointer so the differential driver can iterate a * table. `reference(haystack, hlen, needle, nlen)` invokes the kernel via `operator()`. */ -struct search_backend_t { +struct find_backend_t { char const *name; sz_find_t kernel; sz_cptr_t operator()(sz_cptr_t haystack, sz_size_t haystack_length, // @@ -450,7 +445,7 @@ struct byteset_backend_t { * `for_each_cacheline_offset_`, so a needle straddling a 64-byte boundary is always exercised. */ template <typename reference_, typename candidate_> -void test_search_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_find_search_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { // Replays one haystack/needle pair at every intra-cacheline alignment and compares the backends. auto compare_on = [&](std::string const &haystack_pattern, std::string const &needle) { @@ -462,7 +457,12 @@ void test_search_equivalence(reference_ reference, candidate_ candidate, sz_size sz_cptr_t const result_reference = reference(haystack, haystack_length, needle.data(), needle_length); sz_cptr_t const result_candidate = candidate(haystack, haystack_length, needle.data(), needle_length); - verify(result_reference == result_candidate); + if (result_reference != result_candidate) { + std::fprintf( + stderr, "%s vs %s: substring search disagreed on a %zu-byte needle in a %zu-byte haystack\n", + reference.name, candidate.name, (std::size_t)needle_length, (std::size_t)haystack_length); + verify(false && "Candidate backend must resolve every needle to the same offset as the reference"); + } }); }; @@ -500,7 +500,7 @@ void test_search_equivalence(reference_ reference, candidate_ candidate, sz_size * `for_each_cacheline_offset_`, so a match straddling a 64-byte boundary is always exercised. */ template <typename reference_, typename candidate_> -void test_byteset_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_byteset_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { // The byteset of ASCII vowels, used for the hand-picked structured cases. sz_byteset_t vowels; @@ -520,7 +520,11 @@ void test_byteset_equivalence(reference_ reference, candidate_ candidate, sz_siz sz_cptr_t const result_reference = reference(haystack, haystack_length, &byteset); sz_cptr_t const result_candidate = candidate(haystack, haystack_length, &byteset); - verify(result_reference == result_candidate); + if (result_reference != result_candidate) { + std::fprintf(stderr, "%s vs %s: byteset search disagreed on a %zu-byte haystack\n", reference.name, + candidate.name, (std::size_t)haystack_length); + verify(false && "Candidate backend must resolve every byteset to the same offset as the reference"); + } }); }; @@ -627,15 +631,16 @@ void check_find_misaligned_(std::string_view haystack_pattern, std::string_view std::printf("Mismatch at index #%zu: %zu != %zu\n", match_idx, match_stl.data() - haystack_stl.data(), match_sz.data() - haystack_sz.data()); print_all_matches(); - verify(false); + verify(false && "StringZilla must land on the same match offset as the STL reference matcher"); } } if (count_stl != count_sz) { print_all_matches(); - verify(false); + verify(false && "StringZilla must report the same match count as the STL reference matcher"); } - verify(begin_stl == end_stl && begin_sz == end_sz); + verify(begin_stl == end_stl && begin_sz == end_sz && + "Both matchers must exhaust their ranges together, not leave one with unconsumed matches"); offsets_stl.clear(); offsets_sz.clear(); @@ -700,7 +705,7 @@ void check_find_misaligned_(std::string_view haystack_pattern, std::string_view * @brief Extensively tests the correctness of the string class search methods, such as `find` and `find_first_of`. * Covers different alignment cases within a cache line, repetitive patterns, and overlapping matches. */ -void test_find_misaligned_all() { +void test_find_misaligned_equivalence() { // When haystack is only formed of needles: check_find_misaligned_("a", "a"); check_find_misaligned_("ab", "ab"); @@ -752,7 +757,7 @@ void test_find_misaligned_all() { #endif /** @brief Evaluates the correctness of look-up table transforms using random lookup tables. */ -void test_lookup_all(std::size_t lookup_tables_to_try, std::size_t slices_per_table) { +void test_lookup_equivalence(std::size_t lookup_tables_to_try, std::size_t slices_per_table) { std::size_t const body_length = 1024 * 1024; std::string body(body_length, '\0'), transformed(body_length, '\0'); @@ -779,12 +784,103 @@ void test_lookup_all(std::size_t lookup_tables_to_try, std::size_t slices_per_ta } } +/** + * @brief Degenerate and boundary shapes for the search family, asserting survival and in-bounds results. + * + * Answers are not the subject here - a needle longer than its haystack, a zero length, or an empty byteset + * each has one defensible reply, and what matters is that every backend gives it without reading a byte it + * was not handed. The scanners run at every sub-cache-line alignment, since a misaligned tail is where an + * overlapping load reaches past the end. + */ +void test_find_safety() { + std::printf(" - testing degenerate and boundary inputs of the search kernels...\n"); + + char const *body = "the quick brown fox"; + sz_size_t const body_length = (sz_size_t)std::strlen(body); + + // A zero-length haystack, and a needle longer than what it is searched in, must both miss rather than + // read - and every compiled backend has to say so, not merely whichever one the dispatcher picks here. + auto check_degenerate_ = [&](char const *name, sz_find_byte_t find_byte, sz_find_byte_t rfind_byte) { + if (find_byte(body, 0, "a") != SZ_NULL_CHAR) { + std::fprintf(stderr, "%s: find_byte reported a match in a zero-length haystack\n", name); + verify(false && "A zero-length haystack holds no byte to find"); + } + if (rfind_byte(body, 0, "a") != SZ_NULL_CHAR) { + std::fprintf(stderr, "%s: rfind_byte reported a match in a zero-length haystack\n", name); + verify(false && "A zero-length haystack holds no byte to find"); + } + }; + check_degenerate_("dispatched", sz_find_byte, sz_rfind_byte); + check_degenerate_("serial", sz_find_byte_serial, sz_rfind_byte_serial); +#if SZ_USE_WESTMERE + check_degenerate_("westmere", sz_find_byte_westmere, sz_rfind_byte_westmere); +#endif +#if SZ_USE_HASWELL + check_degenerate_("haswell", sz_find_byte_haswell, sz_rfind_byte_haswell); +#endif +#if SZ_USE_SKYLAKE + check_degenerate_("skylake", sz_find_byte_skylake, sz_rfind_byte_skylake); +#endif +#if SZ_USE_NEON + check_degenerate_("neon", sz_find_byte_neon, sz_rfind_byte_neon); +#endif +#if SZ_USE_SVE + check_degenerate_("sve", sz_find_byte_sve, sz_rfind_byte_sve); +#endif +#if SZ_USE_V128 + check_degenerate_("v128", sz_find_byte_v128, sz_rfind_byte_v128); +#endif +#if SZ_USE_V128RELAXED + check_degenerate_("v128relaxed", sz_find_byte_v128relaxed, sz_rfind_byte_v128relaxed); +#endif +#if SZ_USE_RVV + check_degenerate_("rvv", sz_find_byte_rvv, sz_rfind_byte_rvv); +#endif +#if SZ_USE_LASX + check_degenerate_("lasx", sz_find_byte_lasx, sz_rfind_byte_lasx); +#endif +#if SZ_USE_POWERVSX + check_degenerate_("powervsx", sz_find_byte_powervsx, sz_rfind_byte_powervsx); +#endif + + verify(sz_find(body, 0, "a", 1) == SZ_NULL_CHAR); + verify(sz_rfind(body, 0, "a", 1) == SZ_NULL_CHAR); + verify(sz_find(body, 3, body, body_length) == SZ_NULL_CHAR); + verify(sz_rfind(body, 3, body, body_length) == SZ_NULL_CHAR); + + // An empty byteset matches nothing; a full one matches the first and last byte. + sz_byteset_t empty_set, full_set; + sz_byteset_init(&empty_set); + sz_byteset_init(&full_set); + for (int byte_value = 0; byte_value != 256; ++byte_value) sz_byteset_add_u8(&full_set, (sz_u8_t)byte_value); + verify(sz_find_byteset(body, body_length, &empty_set) == SZ_NULL_CHAR); + verify(sz_rfind_byteset(body, body_length, &empty_set) == SZ_NULL_CHAR); + verify(sz_find_byteset(body, body_length, &full_set) == body); + verify(sz_rfind_byteset(body, body_length, &full_set) == body + body_length - 1); + verify(sz_find_byteset(body, 0, &full_set) == SZ_NULL_CHAR); + + // Every scanner, at every sub-cache-line alignment, over a buffer with no match and then one at the end. + for (sz_size_t length : span_over(backend_ladder_lengths_)) { + for_each_cacheline_offset_((std::size_t)length, [&](sz_ptr_t buffer, std::size_t) { + std::memset(buffer, 'a', (std::size_t)length); + verify(sz_find_byte(buffer, length, "z") == SZ_NULL_CHAR); + verify(sz_rfind_byte(buffer, length, "z") == SZ_NULL_CHAR); + verify(sz_find(buffer, length, "zz", 2) == SZ_NULL_CHAR); + buffer[length - 1] = 'z'; + verify(sz_find_byte(buffer, length, "z") == buffer + length - 1); + verify(sz_rfind_byte(buffer, length, "z") == buffer + length - 1); + }); + } + + std::printf(" boundary-input safety passed!\n"); +} + #pragma endregion // Safety #pragma region Drivers /** @brief Forward substring-search (`sz_find`) backends; `dispatched` first keeps the table non-empty on baseline. */ -static search_backend_t const find_backends[] = { +static find_backend_t const find_backends[] = { {"dispatched", sz_find}, #if SZ_USE_WESTMERE {"westmere", sz_find_westmere}, @@ -819,7 +915,7 @@ static search_backend_t const find_backends[] = { }; /** @brief Backward substring-search (`sz_rfind`) backends; same tiers as forward. */ -static search_backend_t const rfind_backends[] = { +static find_backend_t const rfind_backends[] = { {"dispatched", sz_rfind}, #if SZ_USE_SVE {"sve", sz_rfind_sve}, @@ -923,19 +1019,19 @@ static byteset_backend_t const rfind_byteset_backends[] = { * back to back — forward/backward substring search and forward/backward byteset search. */ void test_find_all() { - search_backend_t const find_serial {"serial", sz_find_serial}; - for (search_backend_t const &backend : find_backends) test_search_equivalence(find_serial, backend, 200); + find_backend_t const find_serial {"serial", sz_find_serial}; + for (find_backend_t const &backend : find_backends) check_find_search_equivalence_(find_serial, backend, 200); - search_backend_t const rfind_serial {"serial", sz_rfind_serial}; - for (search_backend_t const &backend : rfind_backends) test_search_equivalence(rfind_serial, backend, 200); + find_backend_t const rfind_serial {"serial", sz_rfind_serial}; + for (find_backend_t const &backend : rfind_backends) check_find_search_equivalence_(rfind_serial, backend, 200); byteset_backend_t const find_byteset_serial {"serial", sz_find_byteset_serial}; for (byteset_backend_t const &backend : find_byteset_backends) - test_byteset_equivalence(find_byteset_serial, backend, 200); + check_byteset_equivalence_(find_byteset_serial, backend, 200); byteset_backend_t const rfind_byteset_serial {"serial", sz_rfind_byteset_serial}; for (byteset_backend_t const &backend : rfind_byteset_backends) - test_byteset_equivalence(rfind_byteset_serial, backend, 200); + check_byteset_equivalence_(rfind_byteset_serial, backend, 200); } #pragma endregion // Drivers diff --git a/test/fingerprints.cuh b/test/fingerprints.cuh index 621932b6..3839afe8 100644 --- a/test/fingerprints.cuh +++ b/test/fingerprints.cuh @@ -1,8 +1,8 @@ /** * @brief Extensive @b stress-testing suite for StringZillas parallel operations, written in CUDA C++. - * @see Stress-tests on real-world and synthetic data are integrated into the @b `scripts/bench*.cpp` benchmarks. + * @see Stress-tests on real-world and synthetic data are integrated into the benchmarks under @b `bench/`. * - * @file scripts/test_fingerprints.cuh + * @file test/fingerprints.cuh * @author Ash Vardanian * @date June 16, 2026 */ @@ -72,14 +72,14 @@ static void check_rolling_hasher_unit_(hasher_type_ &&hasher, std::vector<std::s state_t rolling_state = 0; for (std::size_t j = 0; j < window_width; ++j) rolling_state = hasher.push(rolling_state, str[j]); hash_t rolling_hash = hasher.digest(rolling_state); - sz_assert_(rolling_hash == hashes[0]); + verify(rolling_hash == hashes[0]); // Now compute the rolling hash and compare it to the slice hashes (bounded to the verified positions). std::size_t const rolling_end = window_width + count_hashes - 1; for (std::size_t j = window_width; j < rolling_end; ++j) { rolling_state = hasher.roll(rolling_state, str[j - window_width], str[j]); rolling_hash = hasher.digest(rolling_state); - sz_assert_(rolling_hash == hashes[j - window_width + 1]); + verify(rolling_hash == hashes[j - window_width + 1]); } } } @@ -117,7 +117,7 @@ static void check_rolling_hasher_unit_(hasher_type_ &&hasher, baseline_hasher_ty } hashes[j] = hasher.digest(slice_state); baseline_hashes[j] = baseline_hasher.digest(baseline_slice_state); - sz_assert_(hashes[j] == baseline_hashes[j] && "Slice hashes do not match baseline hashes"); + verify(hashes[j] == baseline_hashes[j] && "Slice hashes do not match baseline hashes"); } // Pre-populate the rolling-hash state until the first window ends @@ -129,7 +129,7 @@ static void check_rolling_hasher_unit_(hasher_type_ &&hasher, baseline_hasher_ty } hash_t rolling_hash = hasher.digest(rolling_state); baseline_hash_t baseline_rolling_hash = baseline_hasher.digest(baseline_rolling_state); - sz_assert_(rolling_hash == baseline_rolling_hash && "Rolling hashes do not match baseline hashes"); + verify(rolling_hash == baseline_rolling_hash && "Rolling hashes do not match baseline hashes"); // Now compute the rolling hash and compare it to the slice hashes (bounded to the verified positions). std::size_t const rolling_end = window_width + count_hashes - 1; @@ -140,9 +140,9 @@ static void check_rolling_hasher_unit_(hasher_type_ &&hasher, baseline_hasher_ty baseline_rolling_state = baseline_hasher.roll(baseline_rolling_state, str[j - window_width], str[j]); baseline_rolling_hash = baseline_hasher.digest(baseline_rolling_state); - sz_assert_(rolling_hash == baseline_rolling_hash && "Rolling hashes do not match baseline rolling hashes"); - sz_assert_(rolling_hash == hashes[j - window_width + 1]); - sz_assert_(baseline_rolling_hash == baseline_hashes[j - window_width + 1]); + verify(rolling_hash == baseline_rolling_hash && "Rolling hashes do not match baseline rolling hashes"); + verify(rolling_hash == hashes[j - window_width + 1]); + verify(baseline_rolling_hash == baseline_hashes[j - window_width + 1]); } } } @@ -230,19 +230,16 @@ std::vector<std::string> rolling_hasher_inconvenient_inputs(std::size_t max_len * from-scratch slice digest at every window position, and (for Rabin-Karp) matches an integer baseline. * * Exercises Rabin-Karp, multiplying, BuzHash, and floating hashers across a ladder of window - * widths - including super-wide windows - over hand-picked, DNA-like, and edge-byte inputs. + * widths - including super-wide windows - over hand-picked inputs. */ void test_fingerprints_unit() { // Some very basic variants: auto unit_strings = rolling_hasher_basic_inputs(); - auto dna_like_strings = rolling_hasher_dna_like_inputs(); - auto inconvenient_strings = rolling_hasher_inconvenient_inputs(); using u16u32_hasher_t = rabin_karp_rolling_hasher<u16_t, u32_t>; using u32u64_hasher_t = rabin_karp_rolling_hasher<u32_t, u64_t>; using u32mul_hasher_t = multiplying_rolling_hasher<u32_t>; - using i32mul_hasher_t = multiplying_rolling_hasher<i32_t>; using u64mul_hasher_t = multiplying_rolling_hasher<u64_t>; using u32buz_hasher_t = buz_rolling_hasher<u32_t>; using u64buz_hasher_t = buz_rolling_hasher<u64_t>; @@ -250,25 +247,19 @@ void test_fingerprints_unit() { using f64u64_hasher_t = floating_rolling_hasher<f64_t>; check_rolling_hasher_unit_(f64u64_hasher_t(4, 257, 65521), u32u64_hasher_t(4, 257, 65521), unit_strings); - check_rolling_hasher_unit_(f64u64_hasher_t(4, 257, 65521), u32u64_hasher_t(4, 257, 65521), dna_like_strings); - check_rolling_hasher_unit_(f64u64_hasher_t(4, 257, 65521), u32u64_hasher_t(4, 257, 65521), inconvenient_strings); std::vector<u16u32_hasher_t> u16u32_hashers; u16u32_hashers.emplace_back(3, 31, 65521); u16u32_hashers.emplace_back(5, 31, 65521); u16u32_hashers.emplace_back(7, 31, 65521); - for (auto hasher : u16u32_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u16u32_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<u32u64_hasher_t> u32u64_hashers; u32u64_hashers.emplace_back(3, 31, 65521); u32u64_hashers.emplace_back(5, 31, 65521); u32u64_hashers.emplace_back(4, 257, SZ_U32_MAX_PRIME); u32u64_hashers.emplace_back(7, 257, SZ_U32_MAX_PRIME); - for (auto hasher : u32u64_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u32u64_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<u32mul_hasher_t> u32mul_hashers; u32mul_hashers.emplace_back(3); @@ -279,22 +270,7 @@ void test_fingerprints_unit() { u32mul_hashers.emplace_back(5, 65521); u32mul_hashers.emplace_back(4, 257); u32mul_hashers.emplace_back(7, SZ_U32_MAX_PRIME); - for (auto hasher : u32mul_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); - - std::vector<i32mul_hasher_t> i32mul_hashers; - i32mul_hashers.emplace_back(3); - i32mul_hashers.emplace_back(5); - i32mul_hashers.emplace_back(4); - i32mul_hashers.emplace_back(7); - i32mul_hashers.emplace_back(3, 31); - i32mul_hashers.emplace_back(5, 65521); - i32mul_hashers.emplace_back(4, 257); - i32mul_hashers.emplace_back(7, SZ_U32_MAX_PRIME); - for (auto hasher : i32mul_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u32mul_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<u64mul_hasher_t> u64mul_hashers; u64mul_hashers.emplace_back(3, 31); @@ -306,9 +282,7 @@ void test_fingerprints_unit() { u64mul_hashers.emplace_back(4, 257); u64mul_hashers.emplace_back(7, SZ_U64_MAX_PRIME); u64mul_hashers.emplace_back(32, 257); - for (auto hasher : u64mul_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u64mul_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<u32buz_hasher_t> u32buz_hashers; u32buz_hashers.emplace_back(3); @@ -319,9 +293,7 @@ void test_fingerprints_unit() { u32buz_hashers.emplace_back(5, 65521); u32buz_hashers.emplace_back(4, 257); u32buz_hashers.emplace_back(7, SZ_U32_MAX_PRIME); - for (auto hasher : u32buz_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u32buz_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<u64buz_hasher_t> u64buz_hashers; u64buz_hashers.emplace_back(3, 31); @@ -333,9 +305,7 @@ void test_fingerprints_unit() { u64buz_hashers.emplace_back(4, 257); u64buz_hashers.emplace_back(7, SZ_U64_MAX_PRIME); u64buz_hashers.emplace_back(32, 257); - for (auto hasher : u64buz_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : u64buz_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<f32u32_hasher_t> f32u32_hashers; f32u32_hashers.emplace_back(3, 31); @@ -351,9 +321,7 @@ void test_fingerprints_unit() { f32u32_hashers.emplace_back(257); // Super-wide window f32u32_hashers.emplace_back(1000); // Super-wide window f32u32_hashers.emplace_back(30000); // Super-wide window - for (auto hasher : f32u32_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : f32u32_hashers) check_rolling_hasher_unit_(hasher, unit_strings); std::vector<f64u64_hasher_t> f64u64_hashers; f64u64_hashers.emplace_back(3, 31); @@ -369,9 +337,7 @@ void test_fingerprints_unit() { f64u64_hashers.emplace_back(257); // Super-wide window f64u64_hashers.emplace_back(1000); // Super-wide window f64u64_hashers.emplace_back(30000); // Super-wide window - for (auto hasher : f64u64_hashers) - check_rolling_hasher_unit_(hasher, unit_strings), check_rolling_hasher_unit_(hasher, dna_like_strings), - check_rolling_hasher_unit_(hasher, inconvenient_strings); + for (auto hasher : f64u64_hashers) check_rolling_hasher_unit_(hasher, unit_strings); } #pragma endregion // Unit @@ -380,7 +346,7 @@ void test_fingerprints_unit() { /** @brief Asserts a baseline and an accelerated min-hash fingerprinter produce identical hashes and counts. */ template <std::size_t dims_, typename texts_type_, typename baseline_hasher_type_, typename accelerated_hasher_type_> -void test_rolling_hashers_equivalence_against_baseline( // +void check_rolling_hashers_against_baseline_( // texts_type_ const &texts, baseline_hasher_type_ &baseline_hasher, accelerated_hasher_type_ &accelerated_hasher) { constexpr std::size_t dims_k = dims_; @@ -391,7 +357,8 @@ void test_rolling_hashers_equivalence_against_baseline( // unified_vector<min_hashes_t> serial_hashes_per_text, accelerated_hashes_per_text; unified_vector<min_counts_t> serial_counts_per_text, accelerated_counts_per_text; - sz_assert_(texts_tape.try_assign(texts.begin(), texts.end()) == status_t::success_k); + let_verify(status_t const assign_status = texts_tape.try_assign(texts.begin(), texts.end()), + assign_status == status_t::success_k); serial_hashes_per_text.resize(texts.size()); accelerated_hashes_per_text.resize(texts.size()); serial_counts_per_text.resize(texts.size()); @@ -404,9 +371,10 @@ void test_rolling_hashers_equivalence_against_baseline( // min_counts_t &serial_counts = serial_counts_per_text[text_index]; min_hashes_t &accelerated_hashes = accelerated_hashes_per_text[text_index]; min_counts_t &accelerated_counts = accelerated_counts_per_text[text_index]; - baseline_hasher.template try_fingerprint<dims_k>(text.template cast<byte_t const>(), serial_hashes, - serial_counts); - accelerated_hasher.try_fingerprint(text.template cast<byte_t const>(), accelerated_hashes, accelerated_counts); + verify(baseline_hasher.template try_fingerprint<dims_k>(text.template cast<byte_t const>(), serial_hashes, + serial_counts) == status_t::success_k); + verify(accelerated_hasher.try_fingerprint(text.template cast<byte_t const>(), accelerated_hashes, + accelerated_counts) == status_t::success_k); // Compare the results std::size_t const first_mismatch_index = @@ -424,17 +392,17 @@ void test_rolling_hashers_equivalence_against_baseline( // std::printf(" [%zu] serial=%u accelerated=%u\n", i, serial_hashes[i], accelerated_hashes[i]); } } - sz_assert_(first_mismatch_index == serial_hashes.size() && "Fingerprints do not match"); + verify(first_mismatch_index == serial_hashes.size() && "Fingerprints do not match"); // Counters can't be zero, if the input string is at least the size of a window for (std::size_t i = 0; i < serial_counts.size(); ++i) { if (text.size() >= baseline_hasher.window_width(i)) { - sz_assert_(serial_counts[i] > 0 && "Serial fingerprint count is zero"); - sz_assert_(accelerated_counts[i] > 0 && "Accelerated fingerprint count is zero"); + verify(serial_counts[i] > 0 && "Serial fingerprint count is zero"); + verify(accelerated_counts[i] > 0 && "Accelerated fingerprint count is zero"); } else { - sz_assert_(serial_counts[i] == 0 && "Serial fingerprint should be zero"); - sz_assert_(accelerated_counts[i] == 0 && "Accelerated fingerprint should be zero"); + verify(serial_counts[i] == 0 && "Serial fingerprint should be zero"); + verify(accelerated_counts[i] == 0 && "Accelerated fingerprint should be zero"); } } @@ -451,7 +419,7 @@ void test_rolling_hashers_equivalence_against_baseline( // std::printf(" [%zu] serial=%u accelerated=%u\n", i, serial_counts[i], accelerated_counts[i]); } } - sz_assert_(first_counts_mismatch_index == serial_counts.size() && "Fingerprint counts do not match"); + verify(first_counts_mismatch_index == serial_counts.size() && "Fingerprint counts do not match"); } } @@ -466,7 +434,8 @@ void test_rolling_hashers_equivalence_against_baseline( // * can end without ever completing one. */ template <std::size_t dims_, typename hasher_type_> -void test_rolling_hashers_batched_against_per_text_(hasher_type_ &hasher, std::size_t window_width, cpu_specs_t specs) { +void check_rolling_hashers_batched_against_per_text_(hasher_type_ &hasher, std::size_t window_width, + cpu_specs_t specs) { constexpr std::size_t dims_k = dims_; using min_hashes_t = safe_array<u32_t, dims_k>; @@ -480,17 +449,21 @@ void test_rolling_hashers_batched_against_per_text_(hasher_type_ &hasher, std::s for (std::size_t repeat = 0; repeat < 3; ++repeat) texts.emplace_back(window_width * 4 + repeat, 'a' + repeat); arrow_strings_tape_t texts_tape; - sz_assert_(texts_tape.try_assign(texts.begin(), texts.end()) == status_t::success_k); + let_verify(status_t const assign_status = texts_tape.try_assign(texts.begin(), texts.end()), + assign_status == status_t::success_k); unified_vector<min_hashes_t> per_text_hashes(texts.size()), batched_hashes(texts.size()); unified_vector<min_counts_t> per_text_counts(texts.size()), batched_counts(texts.size()); for (std::size_t text_index = 0; text_index != texts.size(); ++text_index) - sz_assert_(hasher.try_fingerprint(texts_tape[text_index].template cast<byte_t const>(), - per_text_hashes[text_index], - per_text_counts[text_index]) == status_t::success_k); + let_verify(status_t const fingerprint_status = hasher.try_fingerprint( + texts_tape[text_index].template cast<byte_t const>(), per_text_hashes[text_index], + per_text_counts[text_index]), + fingerprint_status == status_t::success_k); - sz_assert_(hasher(texts_tape, batched_hashes, batched_counts, dummy_executor_t {}, specs) == status_t::success_k); + let_verify( + status_t const batched_status = hasher(texts_tape, batched_hashes, batched_counts, dummy_executor_t {}, specs), + batched_status == status_t::success_k); for (std::size_t text_index = 0; text_index != texts.size(); ++text_index) { min_hashes_t const &expected_hashes = per_text_hashes[text_index]; @@ -504,15 +477,15 @@ void test_rolling_hashers_batched_against_per_text_(hasher_type_ &hasher, std::s "per-text (%u, %u) vs batched (%u, %u)\n", // text_index, texts[text_index].size(), dimension, expected_hashes[dimension], expected_counts[dimension], produced_hashes[dimension], produced_counts[dimension]); - sz_assert_(expected_hashes[dimension] == produced_hashes[dimension]); - sz_assert_(expected_counts[dimension] == produced_counts[dimension]); + verify(expected_hashes[dimension] == produced_hashes[dimension]); + verify(expected_counts[dimension] == produced_counts[dimension]); } } } /** @brief Compares every compiled SIMD/CUDA `floating_rolling_hashers` backend to the serial and scalar baselines. */ template <std::size_t window_width_, std::size_t dims_> -void test_rolling_hashers_equivalence_for_width( // +void check_rolling_hashers_for_width_( // std::vector<std::string> const &unit_strings, // std::vector<std::string> const &dna_like_strings, // std::vector<std::string> const &inconvenient_strings) { @@ -523,14 +496,16 @@ void test_rolling_hashers_equivalence_for_width( // // Define hasher classes using rolling_f64_t = basic_rolling_hashers<floating_rolling_hasher<f64_t>, u32_t>; rolling_f64_t rolling_f64; - sz_assert_(rolling_f64.try_extend(window_width_k, dims_k) == status_t::success_k); + let_verify(status_t const extend_status = rolling_f64.try_extend(window_width_k, dims_k), + extend_status == status_t::success_k); using rolling_serial_t = floating_rolling_hashers<sz_cap_serial_k, dims_k>; rolling_serial_t rolling_serial; - sz_assert_(rolling_serial.try_seed(window_width_k) == status_t::success_k); - test_rolling_hashers_equivalence_against_baseline<dims_k>(unit_strings, rolling_f64, rolling_serial); - test_rolling_hashers_equivalence_against_baseline<dims_k>(dna_like_strings, rolling_f64, rolling_serial); - test_rolling_hashers_equivalence_against_baseline<dims_k>(inconvenient_strings, rolling_f64, rolling_serial); + let_verify(status_t const seed_status = rolling_serial.try_seed(window_width_k), + seed_status == status_t::success_k); + check_rolling_hashers_against_baseline_<dims_k>(unit_strings, rolling_f64, rolling_serial); + check_rolling_hashers_against_baseline_<dims_k>(dna_like_strings, rolling_f64, rolling_serial); + check_rolling_hashers_against_baseline_<dims_k>(inconvenient_strings, rolling_f64, rolling_serial); // The batched entry point on both sides of its large-text threshold. Default specs keep every text below it; // a one-byte `l2_bytes` puts every text above it, which is the only way to reach the chunk-and-merge branch. @@ -538,38 +513,40 @@ void test_rolling_hashers_equivalence_for_width( // cpu_specs_t whole_text_specs; cpu_specs_t chunked_specs; chunked_specs.l2_bytes = 1; - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_serial, window_width_k, whole_text_specs); - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_serial, window_width_k, chunked_specs); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_serial, window_width_k, whole_text_specs); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_serial, window_width_k, chunked_specs); #if SZ_USE_HASWELL using rolling_haswell_t = floating_rolling_hashers<sz_cap_haswell_k, dims_k>; rolling_haswell_t rolling_haswell; - sz_assert_(rolling_haswell.try_seed(window_width_k) == status_t::success_k); - test_rolling_hashers_equivalence_against_baseline<dims_k>(unit_strings, rolling_f64, rolling_haswell); - test_rolling_hashers_equivalence_against_baseline<dims_k>(dna_like_strings, rolling_f64, rolling_haswell); - test_rolling_hashers_equivalence_against_baseline<dims_k>(inconvenient_strings, rolling_f64, rolling_haswell); - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_haswell, window_width_k, whole_text_specs); - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_haswell, window_width_k, chunked_specs); + let_verify(status_t const seed_status = rolling_haswell.try_seed(window_width_k), + seed_status == status_t::success_k); + check_rolling_hashers_against_baseline_<dims_k>(unit_strings, rolling_f64, rolling_haswell); + check_rolling_hashers_against_baseline_<dims_k>(dna_like_strings, rolling_f64, rolling_haswell); + check_rolling_hashers_against_baseline_<dims_k>(inconvenient_strings, rolling_f64, rolling_haswell); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_haswell, window_width_k, whole_text_specs); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_haswell, window_width_k, chunked_specs); #endif #if SZ_USE_SKYLAKE using rolling_skylake_t = floating_rolling_hashers<sz_cap_skylake_k, dims_k>; rolling_skylake_t rolling_skylake; - sz_assert_(rolling_skylake.try_seed(window_width_k) == status_t::success_k); - test_rolling_hashers_equivalence_against_baseline<dims_k>(unit_strings, rolling_f64, rolling_skylake); - test_rolling_hashers_equivalence_against_baseline<dims_k>(dna_like_strings, rolling_f64, rolling_skylake); - test_rolling_hashers_equivalence_against_baseline<dims_k>(inconvenient_strings, rolling_f64, rolling_skylake); - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_skylake, window_width_k, whole_text_specs); - test_rolling_hashers_batched_against_per_text_<dims_k>(rolling_skylake, window_width_k, chunked_specs); + let_verify(status_t const seed_status = rolling_skylake.try_seed(window_width_k), + seed_status == status_t::success_k); + check_rolling_hashers_against_baseline_<dims_k>(unit_strings, rolling_f64, rolling_skylake); + check_rolling_hashers_against_baseline_<dims_k>(dna_like_strings, rolling_f64, rolling_skylake); + check_rolling_hashers_against_baseline_<dims_k>(inconvenient_strings, rolling_f64, rolling_skylake); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_skylake, window_width_k, whole_text_specs); + check_rolling_hashers_batched_against_per_text_<dims_k>(rolling_skylake, window_width_k, chunked_specs); #endif #if SZ_USE_CUDA using rolling_cuda_t = floating_rolling_hashers<sz_cap_cuda_k, dims_k>; rolling_cuda_t rolling_cuda; - sz_assert_(rolling_cuda.try_seed(window_width_k) == status_t::success_k); - test_rolling_hashers_equivalence_against_baseline<dims_k>(unit_strings, rolling_f64, rolling_cuda); - test_rolling_hashers_equivalence_against_baseline<dims_k>(dna_like_strings, rolling_f64, rolling_cuda); - test_rolling_hashers_equivalence_against_baseline<dims_k>(inconvenient_strings, rolling_f64, rolling_cuda); + let_verify(status_t const seed_status = rolling_cuda.try_seed(window_width_k), seed_status == status_t::success_k); + check_rolling_hashers_against_baseline_<dims_k>(unit_strings, rolling_f64, rolling_cuda); + check_rolling_hashers_against_baseline_<dims_k>(dna_like_strings, rolling_f64, rolling_cuda); + check_rolling_hashers_against_baseline_<dims_k>(inconvenient_strings, rolling_f64, rolling_cuda); #endif } @@ -578,29 +555,157 @@ void test_rolling_hashers_equivalence_for_width( // * a ladder of window widths and dimensionalities, over hand-picked, DNA-like, and edge-byte inputs. */ void test_fingerprints_equivalence() { + std::printf(" - testing rolling hashers against DNA-like and edge-byte random corpora...\n"); + + // Every rolling-hasher family, rolling digest against from-scratch slice digest, over randomized DNA-like + // and edge-byte corpora - the same families `test_fingerprints_unit` drives over its hand-picked inputs. + { + auto dna_like_strings = rolling_hasher_dna_like_inputs(); + auto inconvenient_strings = rolling_hasher_inconvenient_inputs(); + + using u16u32_hasher_t = rabin_karp_rolling_hasher<u16_t, u32_t>; + using u32u64_hasher_t = rabin_karp_rolling_hasher<u32_t, u64_t>; + using u32mul_hasher_t = multiplying_rolling_hasher<u32_t>; + using u64mul_hasher_t = multiplying_rolling_hasher<u64_t>; + using u32buz_hasher_t = buz_rolling_hasher<u32_t>; + using u64buz_hasher_t = buz_rolling_hasher<u64_t>; + using f32u32_hasher_t = floating_rolling_hasher<float>; + using f64u64_hasher_t = floating_rolling_hasher<f64_t>; + + check_rolling_hasher_unit_(f64u64_hasher_t(4, 257, 65521), u32u64_hasher_t(4, 257, 65521), dna_like_strings); + check_rolling_hasher_unit_(f64u64_hasher_t(4, 257, 65521), u32u64_hasher_t(4, 257, 65521), + inconvenient_strings); + + std::vector<u16u32_hasher_t> u16u32_hashers; + u16u32_hashers.emplace_back(3, 31, 65521); + u16u32_hashers.emplace_back(5, 31, 65521); + u16u32_hashers.emplace_back(7, 31, 65521); + for (auto hasher : u16u32_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<u32u64_hasher_t> u32u64_hashers; + u32u64_hashers.emplace_back(3, 31, 65521); + u32u64_hashers.emplace_back(5, 31, 65521); + u32u64_hashers.emplace_back(4, 257, SZ_U32_MAX_PRIME); + u32u64_hashers.emplace_back(7, 257, SZ_U32_MAX_PRIME); + for (auto hasher : u32u64_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<u32mul_hasher_t> u32mul_hashers; + u32mul_hashers.emplace_back(3); + u32mul_hashers.emplace_back(5); + u32mul_hashers.emplace_back(4); + u32mul_hashers.emplace_back(7); + u32mul_hashers.emplace_back(3, 31); + u32mul_hashers.emplace_back(5, 65521); + u32mul_hashers.emplace_back(4, 257); + u32mul_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + for (auto hasher : u32mul_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<u64mul_hasher_t> u64mul_hashers; + u64mul_hashers.emplace_back(3, 31); + u64mul_hashers.emplace_back(5, 65521); + u64mul_hashers.emplace_back(4, 257); + u64mul_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + u64mul_hashers.emplace_back(4, 257); + u64mul_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + u64mul_hashers.emplace_back(4, 257); + u64mul_hashers.emplace_back(7, SZ_U64_MAX_PRIME); + u64mul_hashers.emplace_back(32, 257); + for (auto hasher : u64mul_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<u32buz_hasher_t> u32buz_hashers; + u32buz_hashers.emplace_back(3); + u32buz_hashers.emplace_back(5); + u32buz_hashers.emplace_back(4); + u32buz_hashers.emplace_back(7); + u32buz_hashers.emplace_back(3, 31); + u32buz_hashers.emplace_back(5, 65521); + u32buz_hashers.emplace_back(4, 257); + u32buz_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + for (auto hasher : u32buz_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<u64buz_hasher_t> u64buz_hashers; + u64buz_hashers.emplace_back(3, 31); + u64buz_hashers.emplace_back(5, 65521); + u64buz_hashers.emplace_back(4, 257); + u64buz_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + u64buz_hashers.emplace_back(4, 257); + u64buz_hashers.emplace_back(7, SZ_U32_MAX_PRIME); + u64buz_hashers.emplace_back(4, 257); + u64buz_hashers.emplace_back(7, SZ_U64_MAX_PRIME); + u64buz_hashers.emplace_back(32, 257); + for (auto hasher : u64buz_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<f32u32_hasher_t> f32u32_hashers; + f32u32_hashers.emplace_back(3, 31); + f32u32_hashers.emplace_back(4, 257); + f32u32_hashers.emplace_back(4, 257); + f32u32_hashers.emplace_back(4, 257); + f32u32_hashers.emplace_back(32, 257); + f32u32_hashers.emplace_back(5, 257, 7001); + f32u32_hashers.emplace_back(32, 71, 7001); + f32u32_hashers.emplace_back(3); + f32u32_hashers.emplace_back(32); + f32u32_hashers.emplace_back(65); + f32u32_hashers.emplace_back(257); // Super-wide window + f32u32_hashers.emplace_back(1000); // Super-wide window + f32u32_hashers.emplace_back(30000); // Super-wide window + for (auto hasher : f32u32_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + + std::vector<f64u64_hasher_t> f64u64_hashers; + f64u64_hashers.emplace_back(3, 31); + f64u64_hashers.emplace_back(5, 31, 65521); + f64u64_hashers.emplace_back(4, 257); + f64u64_hashers.emplace_back(4, 257); + f64u64_hashers.emplace_back(4, 257); + f64u64_hashers.emplace_back(32, 257); + f64u64_hashers.emplace_back(32, 257, 65521); + f64u64_hashers.emplace_back(3); + f64u64_hashers.emplace_back(32); + f64u64_hashers.emplace_back(65); + f64u64_hashers.emplace_back(257); // Super-wide window + f64u64_hashers.emplace_back(1000); // Super-wide window + f64u64_hashers.emplace_back(30000); // Super-wide window + for (auto hasher : f64u64_hashers) + check_rolling_hasher_unit_(hasher, dna_like_strings), + check_rolling_hasher_unit_(hasher, inconvenient_strings); + } + // AoS-vs-SoA agreement is deterministic per character, so a few KB per string already exercises every // window width tested here (<= 64) across the unrolled paths. Generate the fuzz inputs once with a small - // cap and reuse them across all widths - the 100 KB strings are reserved for `test_fingerprints_unit`, - // which needs them for its super-wide windows. + // cap and reuse them across all widths. auto const unit = rolling_hasher_basic_inputs(); auto const dna = rolling_hasher_dna_like_inputs(4 * 1024); auto const bad = rolling_hasher_inconvenient_inputs(4 * 1024); // Just 2 hashes per input - // test_rolling_hashers_equivalence_for_width<3, 2>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<7, 2>(unit, dna, bad); + // check_rolling_hashers_for_width_<3, 2>(unit, dna, bad); + check_rolling_hashers_for_width_<7, 2>(unit, dna, bad); // 32 hashes per input - test_rolling_hashers_equivalence_for_width<3, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<7, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<33, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<64, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<3, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<7, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<33, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<64, 32>(unit, dna, bad); // 32 hashes per input with windows divisible by 4 - test_rolling_hashers_equivalence_for_width<4, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<8, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<12, 32>(unit, dna, bad); - test_rolling_hashers_equivalence_for_width<16, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<4, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<8, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<12, 32>(unit, dna, bad); + check_rolling_hashers_for_width_<16, 32>(unit, dna, bad); } #pragma endregion // Equivalence @@ -661,44 +766,109 @@ void test_fingerprints_safety() { // device-accessible (unified) memory rather than on the host stack - otherwise the kernel's output writes land // out of bounds. Stage the degenerate inputs into a tape and reuse one unified output slot, as `equivalence` does. arrow_strings_tape_t degenerate_tape; - sz_assert_(degenerate_tape.try_assign(degenerate.begin(), degenerate.end()) == status_t::success_k); + let_verify(status_t const assign_status = degenerate_tape.try_assign(degenerate.begin(), degenerate.end()), + assign_status == status_t::success_k); unified_vector<min_hashes_t> hashes_buffer(1); unified_vector<min_counts_t> counts_buffer(1); - auto check_fingerprinter = [&](auto &fingerprinter) { + auto check_fingerprinter_ = [&](auto &fingerprinter) { for (std::size_t text_index = 0; text_index < degenerate.size(); ++text_index) { auto text = degenerate_tape[text_index]; min_hashes_t &hashes = hashes_buffer[0]; min_counts_t &counts = counts_buffer[0]; - sz_assert_(fingerprinter.try_fingerprint(text.template cast<byte_t const>(), hashes, counts) == - status_t::success_k); + let_verify(status_t const fingerprint_status = fingerprinter.try_fingerprint( + text.template cast<byte_t const>(), hashes, counts), + fingerprint_status == status_t::success_k); // A degenerate input shorter than the window must yield zero counts on every dimension. for (std::size_t dimension = 0; dimension < dims_k; ++dimension) - if (text.size() < fingerprinter.window_width(dimension)) sz_assert_(counts[dimension] == 0); + if (text.size() < fingerprinter.window_width(dimension)) verify(counts[dimension] == 0); } }; floating_rolling_hashers<sz_cap_serial_k, dims_k> rolling_serial; - sz_assert_(rolling_serial.try_seed(window_width_k) == status_t::success_k); - check_fingerprinter(rolling_serial); + let_verify(status_t const seed_status = rolling_serial.try_seed(window_width_k), + seed_status == status_t::success_k); + check_fingerprinter_(rolling_serial); #if SZ_USE_HASWELL floating_rolling_hashers<sz_cap_haswell_k, dims_k> rolling_haswell; - sz_assert_(rolling_haswell.try_seed(window_width_k) == status_t::success_k); - check_fingerprinter(rolling_haswell); + let_verify(status_t const seed_status = rolling_haswell.try_seed(window_width_k), + seed_status == status_t::success_k); + check_fingerprinter_(rolling_haswell); #endif #if SZ_USE_SKYLAKE floating_rolling_hashers<sz_cap_skylake_k, dims_k> rolling_skylake; - sz_assert_(rolling_skylake.try_seed(window_width_k) == status_t::success_k); - check_fingerprinter(rolling_skylake); + let_verify(status_t const seed_status = rolling_skylake.try_seed(window_width_k), + seed_status == status_t::success_k); + check_fingerprinter_(rolling_skylake); #endif #if SZ_USE_CUDA floating_rolling_hashers<sz_cap_cuda_k, dims_k> rolling_cuda; - sz_assert_(rolling_cuda.try_seed(window_width_k) == status_t::success_k); - check_fingerprinter(rolling_cuda); + let_verify(status_t const seed_status = rolling_cuda.try_seed(window_width_k), seed_status == status_t::success_k); + check_fingerprinter_(rolling_cuda); #endif } +/** + * @brief Pins the device-memory contract for both CUDA fingerprinters: unified and plain device outputs are + * accepted, host and page-locked ones refused. + * + * Both engines are driven, because the C ABI routes between them on whether the dimension count divides + * evenly by the slice width - and only one of them used to validate anything. + */ +void test_fingerprints_cuda_memory_safety() { + std::printf(" - testing unified, host, pinned and device memory against the contract...\n"); +#if SZ_USE_CUDA + + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + constexpr std::size_t dims_k = 64; + constexpr std::size_t window_width_k = 5; + using min_hashes_t = safe_array<u32_t, dims_k>; + using min_counts_t = safe_array<u32_t, dims_k>; + + std::vector<std::string> const texts {"the quick brown fox", "jumps over the lazy dog"}; + arrow_strings_tape_t staged; + verify(staged.try_assign(texts.begin(), texts.end()) == status_t::success_k); + + auto check_engine_ = [&](auto &engine) { + // Unified outputs are accepted, which is the baseline every other row is measured against. + unified_vector<min_hashes_t> unified_hashes(texts.size()); + unified_vector<min_counts_t> unified_counts(texts.size()); + verify(engine(staged.view(), unified_hashes, unified_counts, executor, gpu_specs) == status_t::success_k && + "Unified outputs must be accepted"); + + // Host outputs are refused rather than written through, which is the hole this contract closed. + std::vector<min_hashes_t> host_hashes(texts.size()); + std::vector<min_counts_t> host_counts(texts.size()); + verify(engine(staged.view(), host_hashes, host_counts, executor, gpu_specs) == + status_t::device_memory_mismatch_k && + "Host outputs must be refused, not written from the device"); + + // Page-locked host memory is still host memory to the driver. + pinned_vector<min_hashes_t> pinned_hashes(texts.size()); + pinned_vector<min_counts_t> pinned_counts(texts.size()); + verify(engine(staged.view(), pinned_hashes, pinned_counts, executor, gpu_specs) == + status_t::device_memory_mismatch_k && + "Page-locked host outputs must be refused"); + }; + + // The sliced engine, which the C ABI picks whenever the dimensions divide evenly - and which validated + // nothing at all before this contract landed. + floating_rolling_hashers<sz_cap_cuda_k, dims_k> sliced_engine; + verify(sliced_engine.try_seed(window_width_k) == status_t::success_k); + check_engine_(sliced_engine); + + // The per-dimension fallback, which the C ABI picks for a dimension count it cannot slice. + basic_rolling_hashers<floating_rolling_hasher<f64_t>, u32_t, u32_t, unified_alloc<char>, sz_cap_cuda_k> + fallback_engine; + verify(fallback_engine.try_extend(window_width_k, dims_k) == status_t::success_k); + check_engine_(fallback_engine); +#endif // SZ_USE_CUDA +} + #pragma endregion // Safety } // namespace scripts diff --git a/test/hash.cpp b/test/hash.cpp index 6684d019..a00a64e4 100644 --- a/test/hash.cpp +++ b/test/hash.cpp @@ -1,6 +1,6 @@ /** * @brief Hashing, multi-seed hashing, random-generator, and SHA256 equivalence tests. - * @file scripts/test_hash.cpp + * @file test/hash.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -49,19 +49,11 @@ #include <cstdio> // `std::printf` #include <cstring> // `std::memcpy` -#include <algorithm> // `std::transform` -#include <iterator> // `std::distance` -#include <map> // `std::map` -#include <memory> // `std::allocator` -#include <numeric> // `std::accumulate` -#include <random> // `std::random_device` -#include <set> // `std::set` -#include <sstream> // `std::ostringstream` -#include <string> // `std::string` baseline -#include <string_view> // `std::string_view` baseline -#include <unordered_map> // `std::unordered_map` -#include <unordered_set> // `std::unordered_set` -#include <vector> // `std::vector` +#include <limits> // `std::numeric_limits` +#include <random> // `std::uniform_int_distribution` +#include <string> // `std::string` baseline +#include <string_view> // `std::string_view` baseline +#include <vector> // `std::vector` #if !SZ_IS_CPP11_ #error "This test requires C++11 or later." @@ -128,7 +120,8 @@ static void check_sha256_multistate_unit_( digest(states.data(), (sz_size_t)vectors_count, produced.data()); for (std::size_t lane_index = 0; lane_index != vectors_count; ++lane_index) { sha256_digest_from_hex_(vectors[lane_index].digest_hex, expected); - verify(std::memcmp(&produced[lane_index * SZ_SHA256_DIGEST_LENGTH], expected, SZ_SHA256_DIGEST_LENGTH) == 0); + verify(std::memcmp(&produced[lane_index * SZ_SHA256_DIGEST_LENGTH], expected, SZ_SHA256_DIGEST_LENGTH) == 0 && + "Multi-state digest disagreed with the known-answer digest for this lane"); } } @@ -154,39 +147,47 @@ void test_hash_unit() { {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", // "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"}, }; - for (known_sha256_t const &vector : sha256_vectors) { - check_sha256_unit_(sz_sha256_state_init, sz_sha256_state_update, // Dispatched (automatic kernel) + for (known_sha256_t const &vector : span_over(sha256_vectors)) { + // Dispatched (automatic kernel resolution). + check_sha256_unit_(sz_sha256_state_init, sz_sha256_state_update, // sz_sha256_state_digest, vector.message, vector.digest_hex); - check_sha256_unit_(sz_sha256_state_init_serial, sz_sha256_state_update_serial, // Manual: serial kernel + // Manual propagation to each natively-compiled backend kernel. + check_sha256_unit_(sz_sha256_state_init_serial, sz_sha256_state_update_serial, // sz_sha256_state_digest_serial, vector.message, vector.digest_hex); #if SZ_USE_GOLDMONT - check_sha256_unit_(sz_sha256_state_init_goldmont, sz_sha256_state_update_goldmont, // Manual: goldmont kernel + check_sha256_unit_(sz_sha256_state_init_goldmont, sz_sha256_state_update_goldmont, // sz_sha256_state_digest_goldmont, vector.message, vector.digest_hex); #endif } // The same vectors as one batch, so the multi-state entry point is pinned to known digests rather than // only to another implementation of itself - std::size_t const sha256_vectors_count = sizeof(sha256_vectors) / sizeof(sha256_vectors[0]); - check_sha256_multistate_unit_(sz_sha256_multistate_update, // Dispatched (automatic kernel) + std::size_t const sha256_vectors_count = span_over(sha256_vectors).size(); + // Dispatched (automatic kernel resolution). + check_sha256_multistate_unit_(sz_sha256_multistate_update, // sz_sha256_multistate_digest, sha256_vectors, sha256_vectors_count); - check_sha256_multistate_unit_(sz_sha256_multistate_update_serial, // Manual: serial kernel + // Manual propagation to the serial kernel. + check_sha256_multistate_unit_(sz_sha256_multistate_update_serial, // sz_sha256_multistate_digest_serial, sha256_vectors, sha256_vectors_count); // An embedded-NUL message must hash past the NUL: `"abc\x00def"` is 7 bytes, not 3. Construct the // `std::string` with an explicit length so the interior NUL is retained. std::string const embedded_nul("abc\x00" "def", 7); verify(embedded_nul.size() == 7); - check_sha256_unit_(sz_sha256_state_init, sz_sha256_state_update, // Dispatched (automatic kernel) + // Dispatched (automatic kernel resolution). + check_sha256_unit_(sz_sha256_state_init, sz_sha256_state_update, // sz_sha256_state_digest, embedded_nul, "516a5e926ce20c5f4d80f00e1a01abdf14986def6588d6abeed9fce090bc660c"); - check_sha256_unit_(sz_sha256_state_init_serial, sz_sha256_state_update_serial, // Manual: serial kernel + // Manual propagation to the serial kernel. + check_sha256_unit_(sz_sha256_state_init_serial, sz_sha256_state_update_serial, // sz_sha256_state_digest_serial, embedded_nul, "516a5e926ce20c5f4d80f00e1a01abdf14986def6588d6abeed9fce090bc660c"); // `sz_bytesum` is an order-independent byte sum, so "abc" sums to 0x61 + 0x62 + 0x63 = 0x126. - let_verify(auto bytesum_abc = sz_bytesum("abc", 3), bytesum_abc == 0x126u); // Dispatched (automatic kernel) - verify(sz_bytesum_serial("abc", 3) == 0x126u); // Manual propagation to the serial kernel + // Dispatched (automatic kernel resolution). + let_verify(auto bytesum_abc = sz_bytesum("abc", 3), bytesum_abc == 0x126u); + // Manual propagation to the serial kernel. + verify(sz_bytesum_serial("abc", 3) == 0x126u); #if SZ_USE_ICELAKE verify(sz_bytesum_icelake("abc", 3) == 0x126u); #endif @@ -204,6 +205,10 @@ void test_hash_unit() { verify(sz_hash(fox, fox_length, 0u) != sz_hash(fox, fox_length, 1u)); // Seed changes output verify(sz::string_view(fox, fox_length).hash() == sz_hash(fox, fox_length, 0u)); // C++ wrapper + // The seed reaches the serial kernel too, at both a short and a multi-word length. + verify(sz_hash_serial("abc", 3, 100) != sz_hash_serial("abc", 3, 200)); + verify(sz_hash_serial("abcdefgh", 8, 0) != sz_hash_serial("abcdefgh", 8, 7)); + // The hash must also read past an interior NUL, so truncating at the NUL changes the digest. let_verify(auto hash_nul = sz_hash(embedded_nul.data(), embedded_nul.size(), 0u), hash_nul == sz_hash_serial(embedded_nul.data(), embedded_nul.size(), 0u)); // Dispatch == serial @@ -279,25 +284,26 @@ struct bytesum_from_sz_ { * input reaches. `inputs` arrives already scaled by the caller. */ template <typename reference_, typename candidate_> -void test_bytesum_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_bytesum_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { // A sum of bytes is order-independent, so a run of one repeated byte must total `length * byte` on // any backend. This invariant holds without consulting the reference at all. std::vector<std::size_t> const uniform_lengths = {0, 1, 63, 64, 65, 4096}; for (auto length : uniform_lengths) { std::string const uniform(length, static_cast<char>(0xA5)); - verify(candidate(uniform.data(), static_cast<sz_size_t>(length)) == static_cast<sz_u64_t>(length) * 0xA5ull); + verify(candidate(uniform.data(), static_cast<sz_size_t>(length)) == static_cast<sz_u64_t>(length) * 0xA5ull && + "Byte sum of a uniform buffer must equal length times the repeated byte"); } // The fixed ladder covers the sub-register, register, cache-line, and multi-block tiers; each length // is walked across cache-line offsets so the head and tail paths see every misalignment. std::vector<std::size_t> const lengths = {1, 11, 23, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000}; for (auto length : lengths) - for_each_cacheline_offset_(length, [&](sz_ptr_t pointer, std::size_t offset) { - sz_unused_(offset); + for_each_cacheline_offset_(length, [&](sz_ptr_t pointer, [[maybe_unused]] std::size_t offset) { randomize_string(pointer, length); verify(reference(pointer, static_cast<sz_size_t>(length)) == - candidate(pointer, static_cast<sz_size_t>(length))); + candidate(pointer, static_cast<sz_size_t>(length)) && + "Byte sum backend disagreed with the reference at this length and cache-line offset"); }); // Beyond the ladder, fuzz a contiguous run of random lengths at a single alignment. @@ -305,14 +311,16 @@ void test_bytesum_equivalence(reference_ reference, candidate_ candidate, sz_siz for (sz_size_t length = 0; length != inputs; ++length) { text.resize(length); randomize_string(&text[0], length); - verify(reference(text.data(), length) == candidate(text.data(), length)); + verify(reference(text.data(), length) == candidate(text.data(), length) && + "Byte sum backend disagreed with the reference at this fuzzed length"); } // One oversized input, since the Skylake and Ice Lake kernels take a different branch past a megabyte. // The trailing bytes keep the buffer off a page boundary so the head and tail still have work to do. std::string huge(1024ull * 1024ull + 129ull, '\0'); randomize_string(&huge[0], huge.size()); - verify(reference(huge.data(), (sz_size_t)huge.size()) == candidate(huge.data(), (sz_size_t)huge.size())); + verify(reference(huge.data(), (sz_size_t)huge.size()) == candidate(huge.data(), (sz_size_t)huge.size()) && + "Byte sum backend disagreed with the reference on the oversized, past-a-megabyte input"); } /** @@ -322,13 +330,13 @@ void test_bytesum_equivalence(reference_ reference, candidate_ candidate, sz_siz * progressing towards corner cases like empty strings, all-zero inputs, zero seeds, and so on. */ template <typename reference_, typename candidate_> -void test_hash_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_hash_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { auto test_on_seed = [&](std::string const &text, sz_u64_t seed) { // Compute the entire hash at once, expecting the same output sz_u64_t result_base = reference(text.data(), text.size(), seed); sz_u64_t result_simd = candidate(text.data(), text.size(), seed); - verify(result_base == result_simd); + verify(result_base == result_simd && "Hash backend disagreed with the reference on the one-shot digest"); // Compare incremental hashing across platforms sz_hash_state_t state_base, state_simd; @@ -357,9 +365,9 @@ void test_hash_equivalence(reference_ reference, candidate_ candidate, sz_size_t result_base = reference.digest(&state_base); result_simd = candidate.digest(&state_simd); - verify(result_base == result_simd); + verify(result_base == result_simd && "Hash backend disagreed with the reference after a streamed slice"); sz_u64_t result_misaligned = candidate.digest(&state_misaligned); - verify(result_base == result_misaligned); + verify(result_base == result_misaligned && "Misaligned hash state disagreed with the aligned digest"); }); }; @@ -378,8 +386,7 @@ void test_hash_equivalence(reference_ reference, candidate_ candidate, sz_size_t // Let's try truly random inputs of different lengths, placing each input at every sub-cache-line // offset so serial-vs-ISA agreement is checked across all alignments the SIMD kernels may hit. for (sz_size_t length = 0; length != inputs; ++length) { - for_each_cacheline_offset_(length, [&](sz_ptr_t pointer, std::size_t offset) { - sz_unused_(offset); + for_each_cacheline_offset_(length, [&](sz_ptr_t pointer, [[maybe_unused]] std::size_t offset) { randomize_string(pointer, length); std::string text(pointer, length); for (auto seed : seeds) test_on_seed(text, seed); @@ -396,7 +403,7 @@ void test_hash_equivalence(reference_ reference, candidate_ candidate, sz_size_t * per-seed reduction rather than a sibling backend. */ template <typename candidate_> -void test_hash_multiseed_equivalence(candidate_ candidate, sz_size_t inputs) { +void check_hash_multiseed_equivalence_(candidate_ candidate, sz_size_t inputs) { // Enough seeds to exercise full 4-wide groups plus every 1..3-seed tail remainder. std::vector<sz_u64_t> seeds = {0u, 1u, @@ -422,7 +429,8 @@ void test_hash_multiseed_equivalence(candidate_ candidate, sz_size_t inputs) { std::vector<sz_u64_t> output(seed_count + 1, 0xDEADBEEFDEADBEEFull); candidate.multiseed(text.data(), text.size(), seeds.data(), seed_count, output.data()); for (std::size_t index = 0; index < seed_count; ++index) - verify(output[index] == candidate.hash_one(text.data(), text.size(), seeds[index])); + verify(output[index] == candidate.hash_one(text.data(), text.size(), seeds[index]) && + "Multi-seed output disagreed with the single-seed hash at this seed index"); verify(output[seed_count] == 0xDEADBEEFDEADBEEFull); // No overwrite past `seed_count` } }; @@ -441,14 +449,14 @@ void test_hash_multiseed_equivalence(candidate_ candidate, sz_size_t inputs) { * produces exactly the same output across a reference and a candidate implementation. */ template <typename reference_, typename candidate_> -void test_random_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_random_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { auto test_on_nonce = [&](std::size_t length, sz_u64_t nonce) { std::string text_base(length, '\0'); std::string text_simd(length, '\0'); reference(&text_base[0], static_cast<sz_size_t>(length), nonce); candidate(&text_simd[0], static_cast<sz_size_t>(length), nonce); - verify(text_base == text_simd); + verify(text_base == text_simd && "PRNG backend disagreed with the reference for this nonce and length"); }; // Boundary nonces are always exercised, including the 0 and max extremes: @@ -479,7 +487,7 @@ void test_random_equivalence(reference_ reference, candidate_ candidate, sz_size * `inputs` is the maximum length fuzzed, inclusive. */ template <typename reference_, typename candidate_> -void test_sha256_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_sha256_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { // Test random inputs of various lengths for (sz_size_t length = 0; length <= inputs; ++length) { @@ -496,7 +504,8 @@ void test_sha256_equivalence(reference_ reference, candidate_ candidate, sz_size candidate.update(&state_simd, random_text.data(), length); reference.digest(&state_base, digest_base_result); candidate.digest(&state_simd, digest_simd_result); - verify(std::memcmp(digest_base_result, digest_simd_result, SZ_SHA256_DIGEST_LENGTH) == 0); + verify(std::memcmp(digest_base_result, digest_simd_result, SZ_SHA256_DIGEST_LENGTH) == 0 && + "SHA256 backend disagreed with the reference on the one-shot digest at this length"); // Incremental hashing with random chunks reference.init(&state_base); @@ -507,7 +516,8 @@ void test_sha256_equivalence(reference_ reference, candidate_ candidate, sz_size }); reference.digest(&state_base, digest_base_result); candidate.digest(&state_simd, digest_simd_result); - verify(std::memcmp(digest_base_result, digest_simd_result, SZ_SHA256_DIGEST_LENGTH) == 0); + verify(std::memcmp(digest_base_result, digest_simd_result, SZ_SHA256_DIGEST_LENGTH) == 0 && + "SHA256 backend disagreed with the reference on the incrementally streamed digest"); } } @@ -529,7 +539,7 @@ void test_sha256_equivalence(reference_ reference, candidate_ candidate, sz_size * retirement order, the countdown, or the rule that parks a finished lane's cursor on its last full block. */ template <typename reference_, typename candidate_> -void test_sha256_multistate_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_sha256_multistate_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { for (sz_size_t lanes_count = 0; lanes_count <= inputs; ++lanes_count) { std::vector<std::string> messages; @@ -553,7 +563,8 @@ void test_sha256_multistate_equivalence(reference_ reference, candidate_ candida reference.digest(reference_states.data(), lanes_count, reference_digests.data()); candidate.digest(candidate_states.data(), lanes_count, candidate_digests.data()); verify(std::memcmp(reference_digests.data(), candidate_digests.data(), lanes_count * SZ_SHA256_DIGEST_LENGTH) == - 0); + 0 && + "Multi-state backend disagreed with the reference on the one-shot batch digest"); // Incremental: the same messages, cut into random per-lane slices across several calls for (std::size_t lane_index = 0; lane_index != messages.size(); ++lane_index) @@ -582,7 +593,8 @@ void test_sha256_multistate_equivalence(reference_ reference, candidate_ candida reference.digest(reference_states.data(), lanes_count, reference_digests.data()); candidate.digest(candidate_states.data(), lanes_count, candidate_digests.data()); verify(std::memcmp(reference_digests.data(), candidate_digests.data(), lanes_count * SZ_SHA256_DIGEST_LENGTH) == - 0); + 0 && + "Multi-state backend disagreed with the reference on the incrementally sliced batch digest"); for (std::size_t guard_index = 0; guard_index != SZ_SHA256_DIGEST_LENGTH; ++guard_index) // No overwrite past the last lane @@ -610,12 +622,68 @@ void test_sha256_multistate_equivalence(reference_ reference, candidate_ candida reference.digest(reference_states.data(), lanes_count, reference_digests.data()); candidate.digest(candidate_states.data(), lanes_count, candidate_digests.data()); verify(std::memcmp(reference_digests.data(), candidate_digests.data(), lanes_count * SZ_SHA256_DIGEST_LENGTH) == - 0); + 0 && + "Multi-state backend disagreed with the reference on the buffered-head batch digest"); } } #pragma endregion // Equivalence +#pragma region Safety + +/** + * @brief Degenerate lengths and alignments for the hashing family, asserting bounds rather than digests. + * + * A hash of nothing still has to be a hash: the empty input, the single byte and the streaming state fed in + * one-byte pieces all have to agree with the one-shot call over the same bytes, and none may write past the + * digest it was handed. The canary-guarded buffer is what catches the last of those, since a digest that + * overruns by one byte produces a perfectly plausible value. + */ +void test_hash_safety() { + std::printf(" - testing degenerate lengths and alignments of the hashing kernels...\n"); + + // The empty input is hashable, and its digest is stable across calls. + verify(sz_hash("", 0, 0) == sz_hash("", 0, 0)); + verify(sz_hash_serial("", 0, 0) == sz_hash_serial("", 0, 0)); + verify(sz_bytesum("", 0) == 0); + verify(sz_bytesum_serial("", 0) == 0); + + // Streaming in one-byte pieces must reach the same digest as one shot over the whole message. + char const *message = "the quick brown fox jumps over the lazy dog"; + sz_size_t const message_length = (sz_size_t)std::strlen(message); + { + sz_hash_state_t streamed; + sz_hash_state_init(&streamed, 0); + for (sz_size_t index = 0; index != message_length; ++index) sz_hash_state_update(&streamed, message + index, 1); + verify(sz_hash_state_digest(&streamed) == sz_hash(message, message_length, 0)); + } + + // SHA256 into a canary-guarded destination, so a digest that writes one byte too many is caught. + with_guarded_buffer_(SZ_SHA256_DIGEST_LENGTH, [&](sz_ptr_t destination, std::size_t) { + sz_sha256_state_t state; + sz_sha256_state_init(&state); + sz_sha256_state_update(&state, message, message_length); + sz_sha256_state_digest(&state, (sz_u8_t *)destination); + }); + + // Every length across the ladder, at every sub-cache-line alignment: the dispatched and serial kernels + // must agree byte for byte, whatever the buffer's offset. + sz_size_t const lengths[] = {0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128}; + for (sz_size_t length : span_over(lengths)) { + for_each_cacheline_offset_((std::size_t)length, [&](sz_ptr_t buffer, std::size_t) { + for (sz_size_t index = 0; index != length; ++index) buffer[index] = (char)('a' + (index & 15)); + verify(sz_hash(buffer, length, 0) == sz_hash_serial(buffer, length, 0) && + "Dispatched hash disagreed with the serial kernel at this length and alignment"); + verify(sz_bytesum(buffer, length) == sz_bytesum_serial(buffer, length) && + "Dispatched byte sum disagreed with the serial kernel at this length and alignment"); + }); + } + + std::printf(" degenerate-input safety passed!\n"); +} + +#pragma endregion // Safety + #pragma region Drivers /** @@ -627,182 +695,201 @@ void test_hash_all() { using hash_serial_t = hash_from_sz_<sz_hash_serial, sz_hash_state_init_serial, // sz_hash_state_update_serial, sz_hash_state_digest_serial>; hash_serial_t const hash_serial; - sz_unused_(hash_serial); // Used only by the SIMD differential blocks below; unreferenced on no-SIMD-tier targets. // Number of random-length inputs to fuzz per differential test. Each sweeps lengths `0..N` and hashes a buffer // of that length, so the work is quadratic in the count and the baseline is scaled accordingly. sz_size_t const hash_inputs = (sz_size_t)scale_iterations_quadratic(200); sz_size_t const random_inputs = (sz_size_t)scale_iterations_quadratic(200); sz_size_t const sha256_inputs = (sz_size_t)scale_iterations_quadratic(256); - sz_unused_(hash_inputs), sz_unused_(random_inputs), sz_unused_(sha256_inputs); - // Ensure the seed affects hash results - verify(sz_hash_serial("abc", 3, 100) != sz_hash_serial("abc", 3, 200)); - verify(sz_hash_serial("abcdefgh", 8, 0) != sz_hash_serial("abcdefgh", 8, 7)); + // The dispatched path, unguarded. Every sibling family leads its table with a `dispatched` row so a + // baseline build still measures something; hash has no table, so this call is what plays that part - + // without it a target with no SIMD tier runs the two assertions above and nothing else. + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash, sz_hash_state_init, sz_hash_state_update, sz_hash_state_digest> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, fill_random_from_sz_<sz_fill_random> {}, + random_inputs); // Byte sums carry their own backend set - Haswell, NEON, SVE and the WASM tiers all provide one where // the AES-based hash does not - so they need a differential sweep of their own. using bytesum_serial_t = bytesum_from_sz_<sz_bytesum_serial>; bytesum_serial_t const bytesum_serial; sz_size_t const bytesum_inputs = (sz_size_t)scale_iterations_quadratic(200); - sz_unused_(bytesum_serial), sz_unused_(bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum> {}, bytesum_inputs); #if SZ_USE_HASWELL - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_haswell> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_haswell> {}, bytesum_inputs); #endif #if SZ_USE_SKYLAKE - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_skylake> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_skylake> {}, bytesum_inputs); #endif #if SZ_USE_ICELAKE - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_icelake> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_icelake> {}, bytesum_inputs); #endif #if SZ_USE_NEON - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_neon> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_neon> {}, bytesum_inputs); #endif #if SZ_USE_SVE - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_sve> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_sve> {}, bytesum_inputs); #endif #if SZ_USE_SVE2 - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_sve2> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_sve2> {}, bytesum_inputs); #endif #if SZ_USE_V128 - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_v128> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_v128> {}, bytesum_inputs); #endif #if SZ_USE_V128RELAXED - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_v128relaxed> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_v128relaxed> {}, bytesum_inputs); #endif #if SZ_USE_RVV - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_rvv> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_rvv> {}, bytesum_inputs); #endif #if SZ_USE_LASX - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_lasx> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_lasx> {}, bytesum_inputs); #endif #if SZ_USE_POWERVSX - test_bytesum_equivalence(bytesum_serial, bytesum_from_sz_<sz_bytesum_powervsx> {}, bytesum_inputs); + check_bytesum_equivalence_(bytesum_serial, bytesum_from_sz_<sz_bytesum_powervsx> {}, bytesum_inputs); #endif #if SZ_USE_WESTMERE - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_westmere, sz_hash_state_init_westmere, // - sz_hash_state_update_westmere, sz_hash_state_digest_westmere> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_westmere> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_westmere, sz_hash_state_init_westmere, // + sz_hash_state_update_westmere, sz_hash_state_digest_westmere> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_westmere> {}, random_inputs); #endif #if SZ_USE_SKYLAKE - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_skylake, sz_hash_state_init_skylake, // - sz_hash_state_update_skylake, sz_hash_state_digest_skylake> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_skylake> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_skylake, sz_hash_state_init_skylake, // + sz_hash_state_update_skylake, sz_hash_state_digest_skylake> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_skylake> {}, random_inputs); #endif #if SZ_USE_ICELAKE - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_icelake, sz_hash_state_init_icelake, // - sz_hash_state_update_icelake, sz_hash_state_digest_icelake> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_icelake> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_icelake, sz_hash_state_init_icelake, // + sz_hash_state_update_icelake, sz_hash_state_digest_icelake> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_icelake> {}, random_inputs); #endif #if SZ_USE_NEONAES - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_neonaes, sz_hash_state_init_neonaes, // - sz_hash_state_update_neonaes, sz_hash_state_digest_neonaes> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_neonaes> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_neonaes, sz_hash_state_init_neonaes, // + sz_hash_state_update_neonaes, sz_hash_state_digest_neonaes> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_neonaes> {}, random_inputs); #endif #if SZ_USE_SVE2AES - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_sve2aes, sz_hash_state_init_sve2aes, // - sz_hash_state_update_sve2aes, sz_hash_state_digest_sve2aes> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_sve2aes> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_sve2aes, sz_hash_state_init_sve2aes, // + sz_hash_state_update_sve2aes, sz_hash_state_digest_sve2aes> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_sve2aes> {}, random_inputs); #endif #if SZ_USE_V128 - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_v128, sz_hash_state_init_v128, // - sz_hash_state_update_v128, sz_hash_state_digest_v128> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_v128> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_v128, sz_hash_state_init_v128, // + sz_hash_state_update_v128, sz_hash_state_digest_v128> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_v128> {}, random_inputs); #endif #if SZ_USE_V128RELAXED - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_v128relaxed, sz_hash_state_init_v128relaxed, // - sz_hash_state_update_v128relaxed, sz_hash_state_digest_v128relaxed> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_v128relaxed> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_v128relaxed, sz_hash_state_init_v128relaxed, // + sz_hash_state_update_v128relaxed, sz_hash_state_digest_v128relaxed> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_v128relaxed> {}, random_inputs); #endif #if SZ_USE_RVV - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_rvv, sz_hash_state_init_rvv, // - sz_hash_state_update_rvv, sz_hash_state_digest_rvv> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, fill_random_from_sz_<sz_fill_random_rvv> {}, - random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_rvv, sz_hash_state_init_rvv, // + sz_hash_state_update_rvv, sz_hash_state_digest_rvv> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_rvv> {}, random_inputs); +#endif +#if SZ_USE_RVVCRYPTO + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_rvvcrypto, sz_hash_state_init_rvvcrypto, // + sz_hash_state_update_rvvcrypto, sz_hash_state_digest_rvvcrypto> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_rvvcrypto> {}, random_inputs); #endif #if SZ_USE_LASX - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_lasx, sz_hash_state_init_lasx, // - sz_hash_state_update_lasx, sz_hash_state_digest_lasx> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_lasx> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_lasx, sz_hash_state_init_lasx, // + sz_hash_state_update_lasx, sz_hash_state_digest_lasx> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_lasx> {}, random_inputs); #endif #if SZ_USE_POWERVSX - test_hash_equivalence(hash_serial, - hash_from_sz_<sz_hash_powervsx, sz_hash_state_init_powervsx, // - sz_hash_state_update_powervsx, sz_hash_state_digest_powervsx> {}, - hash_inputs); - test_random_equivalence(fill_random_from_sz_<sz_fill_random_serial> {}, - fill_random_from_sz_<sz_fill_random_powervsx> {}, random_inputs); + check_hash_equivalence_(hash_serial, + hash_from_sz_<sz_hash_powervsx, sz_hash_state_init_powervsx, // + sz_hash_state_update_powervsx, sz_hash_state_digest_powervsx> {}, + hash_inputs); + check_random_equivalence_(fill_random_from_sz_<sz_fill_random_serial> {}, + fill_random_from_sz_<sz_fill_random_powervsx> {}, random_inputs); #endif // Test SHA256 implementations using sha256_serial_t = sha256_from_sz_<sz_sha256_state_init_serial, sz_sha256_state_update_serial, sz_sha256_state_digest_serial>; sha256_serial_t const sha256_serial; - sz_unused_(sha256_serial); + check_sha256_equivalence_(sha256_serial, + sha256_from_sz_<sz_sha256_state_init, sz_sha256_state_update, sz_sha256_state_digest> {}, + sha256_inputs); #if SZ_USE_GOLDMONT - test_sha256_equivalence(sha256_serial, - sha256_from_sz_<sz_sha256_state_init_goldmont, sz_sha256_state_update_goldmont, - sz_sha256_state_digest_goldmont> {}, - sha256_inputs); + check_sha256_equivalence_(sha256_serial, + sha256_from_sz_<sz_sha256_state_init_goldmont, sz_sha256_state_update_goldmont, + sz_sha256_state_digest_goldmont> {}, + sha256_inputs); #endif #if SZ_USE_NEONSHA - test_sha256_equivalence(sha256_serial, - sha256_from_sz_<sz_sha256_state_init_neonsha, sz_sha256_state_update_neonsha, - sz_sha256_state_digest_neonsha> {}, - sha256_inputs); + check_sha256_equivalence_(sha256_serial, + sha256_from_sz_<sz_sha256_state_init_neonsha, sz_sha256_state_update_neonsha, + sz_sha256_state_digest_neonsha> {}, + sha256_inputs); #endif #if SZ_USE_V128 - test_sha256_equivalence( + check_sha256_equivalence_( sha256_serial, sha256_from_sz_<sz_sha256_state_init_v128, sz_sha256_state_update_v128, sz_sha256_state_digest_v128> {}, sha256_inputs); #endif #if SZ_USE_RVV - test_sha256_equivalence( + check_sha256_equivalence_( sha256_serial, sha256_from_sz_<sz_sha256_state_init_rvv, sz_sha256_state_update_rvv, sz_sha256_state_digest_rvv> {}, sha256_inputs); #endif +#if SZ_USE_RVVCRYPTO + check_sha256_equivalence_(sha256_serial, + sha256_from_sz_<sz_sha256_state_init_rvvcrypto, sz_sha256_state_update_rvvcrypto, + sz_sha256_state_digest_rvvcrypto> {}, + sha256_inputs); +#endif #if SZ_USE_LASX - test_sha256_equivalence( + check_sha256_equivalence_( sha256_serial, sha256_from_sz_<sz_sha256_state_init_lasx, sz_sha256_state_update_lasx, sz_sha256_state_digest_lasx> {}, sha256_inputs); #endif #if SZ_USE_POWERVSX - test_sha256_equivalence(sha256_serial, - sha256_from_sz_<sz_sha256_state_init_powervsx, sz_sha256_state_update_powervsx, - sz_sha256_state_digest_powervsx> {}, - sha256_inputs); + check_sha256_equivalence_(sha256_serial, + sha256_from_sz_<sz_sha256_state_init_powervsx, sz_sha256_state_update_powervsx, + sz_sha256_state_digest_powervsx> {}, + sha256_inputs); #endif // The multi-state kernels sweep every lane count up to the bound, and each batch carries one message per @@ -811,53 +898,56 @@ void test_hash_all() { using multistate_serial_t = sha256_multistate_from_sz_<sz_sha256_multistate_update_serial, sz_sha256_multistate_digest_serial>; multistate_serial_t const multistate_serial; - sz_unused_(multistate_serial), sz_unused_(multistate_inputs); + check_sha256_multistate_equivalence_( + multistate_serial, sha256_multistate_from_sz_<sz_sha256_multistate_update, sz_sha256_multistate_digest> {}, + multistate_inputs); #if SZ_USE_GOLDMONT - test_sha256_multistate_equivalence( + check_sha256_multistate_equivalence_( multistate_serial, sha256_multistate_from_sz_<sz_sha256_multistate_update_goldmont, sz_sha256_multistate_digest_goldmont> {}, multistate_inputs); #endif #if SZ_USE_HASWELL - test_sha256_multistate_equivalence( + check_sha256_multistate_equivalence_( multistate_serial, sha256_multistate_from_sz_<sz_sha256_multistate_update_haswell, sz_sha256_multistate_digest_haswell> {}, multistate_inputs); #endif #if SZ_USE_SKYLAKE - test_sha256_multistate_equivalence( + check_sha256_multistate_equivalence_( multistate_serial, sha256_multistate_from_sz_<sz_sha256_multistate_update_skylake, sz_sha256_multistate_digest_skylake> {}, multistate_inputs); #endif } -/** @brief Drives `test_hash_multiseed_equivalence` across every hashing backend compiled on this target. */ +/** @brief Drives `check_hash_multiseed_equivalence_` across every hashing backend compiled on this target. */ void test_hash_multiseed_all() { // Cover the <= 64 byte ladder, the 64-byte boundary, and into the wide path. Every length is hashed by // every seeded kernel on every backend, so this count is the family's whole budget. sz_size_t const lengths = (sz_size_t)scale_iterations_quadratic(512); - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed, sz_hash> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed, sz_hash> {}, lengths); // And every backend that ships a specialized multi-seed kernel must match its own single-shot. - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_serial, sz_hash_serial> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_serial, sz_hash_serial> {}, lengths); #if SZ_USE_WESTMERE - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_westmere, sz_hash_westmere> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_westmere, sz_hash_westmere> {}, + lengths); #endif #if SZ_USE_ICELAKE - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_icelake, sz_hash_icelake> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_icelake, sz_hash_icelake> {}, lengths); #endif #if SZ_USE_NEONAES - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_neonaes, sz_hash_neonaes> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_neonaes, sz_hash_neonaes> {}, lengths); #endif #if SZ_USE_V128 - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_v128, sz_hash_v128> {}, lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_v128, sz_hash_v128> {}, lengths); #endif #if SZ_USE_V128RELAXED - test_hash_multiseed_equivalence(hash_multiseed_from_sz_<sz_hash_multiseed_v128relaxed, sz_hash_v128relaxed> {}, - lengths); + check_hash_multiseed_equivalence_(hash_multiseed_from_sz_<sz_hash_multiseed_v128relaxed, sz_hash_v128relaxed> {}, + lengths); #endif } diff --git a/test/hash.py b/test/hash.py index f2034dc1..1607833e 100644 --- a/test/hash.py +++ b/test/hash.py @@ -50,7 +50,6 @@ @pytest.mark.parametrize("seed_value", SEED_VALUES) def test_hash_basic_equivalence(body: str, seed_value: int): """The standalone `sz.hash` and the `Str.hash` method return the same seeded digest for the same body.""" - # TODO: Add streaming hashers and compare slices vs overall hash_seeded = sz.hash(body, seed=seed_value) hash_member = sz.Str(body).hash(seed=seed_value) assert hash_seeded == hash_member diff --git a/test/similarities.cuh b/test/similarities.cuh index 2a412a67..8dd22bcd 100644 --- a/test/similarities.cuh +++ b/test/similarities.cuh @@ -1,8 +1,9 @@ /** * @brief Extensive @b stress-testing suite for StringZillas parallel operations, written in CUDA C++. - * @see Stress-tests on real-world and synthetic data are integrated into the @b `scripts/bench*.cpp` benchmarks. + * @see Stress-tests on real-world and synthetic data are integrated into the @b `bench/similarities.cpp` and + * @b `bench/similarities.cu` benchmarks. * - * @file scripts/test_similarities.cuh + * @file test/similarities.cuh * @author Ash Vardanian * @date June 16, 2026 */ @@ -43,6 +44,12 @@ using ashvardanian::stringzillas::ualloc_t; #pragma region Helpers +/** @brief One input pair for a similarity scorer under test. */ +struct similarity_case_t { + std::string first; + std::string second; +}; + /** @brief Dual-row O(n)-memory reference Levenshtein distance, keeping only the previous and current rows. */ inline std::size_t levenshtein_baseline( // char const *s1, std::size_t len1, char const *s2, std::size_t len2, // @@ -293,7 +300,7 @@ struct levenshtein_baselines_t { template <typename results_type_> status_t operator()(arrow_strings_view_t first, arrow_strings_view_t second, results_type_ *results) const { - sz_assert_(first.size() == second.size()); + verify(first.size() == second.size()); #pragma omp parallel for for (std::size_t i = 0; i != first.size(); ++i) results[i] = gap_opening_cost == gap_extension_cost @@ -322,7 +329,7 @@ struct needleman_wunsch_baselines_t { : substitution_costs(subs), gap_opening_cost(gap.open), gap_extension_cost(gap.extend) {} status_t operator()(arrow_strings_view_t first, arrow_strings_view_t second, sz_ssize_t *results) const { - sz_assert_(first.size() == second.size()); + verify(first.size() == second.size()); #pragma omp parallel for for (std::size_t i = 0; i != first.size(); ++i) @@ -351,7 +358,7 @@ struct smith_waterman_baselines_t { : substitution_costs(subs), gap_opening_cost(gap.open), gap_extension_cost(gap.extend) {} status_t operator()(arrow_strings_view_t first, arrow_strings_view_t second, sz_ssize_t *results) const { - sz_assert_(first.size() == second.size()); + verify(first.size() == second.size()); #pragma omp parallel for for (std::size_t i = 0; i != first.size(); ++i) @@ -396,7 +403,7 @@ struct pairwise_via_cross_t { template <typename score_type_, typename... extra_args_> status_t operator()(arrow_strings_view_t first, arrow_strings_view_t second, score_type_ *results, extra_args_ &&...extra_args) { - sz_assert_(first.size() == second.size()); + verify(first.size() == second.size()); std::size_t const pairs_count = first.size(); for (std::size_t pair_index = 0; pair_index != pairs_count; ++pair_index) { arrow_strings_view_t const first_cell {first.buffer_, first.offsets_.subspan(pair_index, 2)}; @@ -424,9 +431,9 @@ template <typename score_type_, typename base_operator_, typename simd_operator_ static void check_similarities_fixed_(base_operator_ &&base_operator, simd_operator_ &&simd_operator, std::string_view allowed_chars = {}, simd_extra_args_ &&...simd_extra_args) { - std::vector<std::pair<std::string, std::string>> test_cases; + std::vector<similarity_case_t> test_cases; auto append = [&test_cases](std::string const &first, std::string const &second) { - test_cases.emplace_back(first, second); + test_cases.push_back({first, second}); }; // Some vary basic variants: @@ -528,8 +535,8 @@ static void check_similarities_fixed_(base_operator_ &&base_operator, simd_opera // Reset the tapes and results results_base[0] = signaling_score, results_simd[0] = signaling_score; - first_tape.try_assign(&first, &first + 1); - second_tape.try_assign(&second, &second + 1); + verify(first_tape.try_assign(&first, &first + 1) == status_t::success_k); + verify(second_tape.try_assign(&second, &second + 1) == status_t::success_k); // Compute with both backends arrow_strings_view_t first_view = first_tape.view(); @@ -538,11 +545,11 @@ static void check_similarities_fixed_(base_operator_ &&base_operator, simd_opera score_t *results_simd_ptr = results_simd.data(); status_t status_base = base_operator(first_view, second_view, results_base_ptr); status_t status_simd = simd_operator(first_view, second_view, results_simd_ptr, simd_extra_args...); - sz_assert_(status_base == status_t::success_k); - sz_assert_(status_simd == status_t::success_k); + verify(status_base == status_t::success_k && "Base engine failed on a fixed single-pair batch"); + verify(status_simd == status_t::success_k && "SIMD engine failed on a fixed single-pair batch"); if (results_base[0] != results_simd[0]) edit_distance_log_mismatch(first, second, results_base[0], results_simd[0]); - sz_assert_(results_base[0] == results_simd[0]); + verify(results_base[0] == results_simd[0] && "Base and SIMD engines disagree on this fixed test-case pair"); } // Unzip the test cases into two separate tapes and perform batch processing @@ -552,22 +559,25 @@ static void check_similarities_fixed_(base_operator_ &&base_operator, simd_opera first_tape.reset(); second_tape.reset(); for (auto [first, second] : test_cases) { - sz_assert_(first_tape.try_append({first.data(), first.size()}) == status_t::success_k); - sz_assert_(second_tape.try_append({second.data(), second.size()}) == status_t::success_k); + let_verify(status_t const first_append_status = first_tape.try_append({first.data(), first.size()}), + first_append_status == status_t::success_k); + let_verify(status_t const second_append_status = second_tape.try_append({second.data(), second.size()}), + second_append_status == status_t::success_k); } // Compute with both backends status_t status_base = base_operator(first_tape.view(), second_tape.view(), results_base.data()); status_t status_simd = simd_operator(first_tape.view(), second_tape.view(), results_simd.data(), simd_extra_args...); - sz_assert_(status_base == status_t::success_k); - sz_assert_(status_simd == status_t::success_k); + verify(status_base == status_t::success_k && "Base engine failed on the batched fixed test cases"); + verify(status_simd == status_t::success_k && "SIMD engine failed on the batched fixed test cases"); // Individually log the failed results for (std::size_t i = 0; i != test_cases.size(); ++i) { if (results_base[i] == results_simd[i]) continue; edit_distance_log_mismatch(test_cases[i].first, test_cases[i].second, results_base[i], results_simd[i]); - sz_assert_(results_base[i] == results_simd[i]); + verify(results_base[i] == results_simd[i] && + "Base and SIMD engines disagree on a batched fixed test-case pair"); } } } @@ -578,15 +588,17 @@ static void check_similarities_known_(operator_type_ &&similarity_operator, std: std::string const &second, score_type_ expected, extra_args_ &&...extra_args) { arrow_strings_tape_t first_tape, second_tape; - sz_assert_(first_tape.try_append({first.data(), first.size()}) == status_t::success_k); - sz_assert_(second_tape.try_append({second.data(), second.size()}) == status_t::success_k); + let_verify(status_t const first_append_status = first_tape.try_append({first.data(), first.size()}), + first_append_status == status_t::success_k); + let_verify(status_t const second_append_status = second_tape.try_append({second.data(), second.size()}), + second_append_status == status_t::success_k); unified_vector<score_type_> result(1); result[0] = std::numeric_limits<score_type_>::max(); status_t status = similarity_operator(first_tape.view(), second_tape.view(), result.data(), extra_args...); - sz_assert_(status == status_t::success_k); + verify(status == status_t::success_k); if (result[0] != expected) edit_distance_log_mismatch(first, second, expected, result[0]); - sz_assert_(result[0] == expected); + verify(result[0] == expected); } /** @@ -669,14 +681,14 @@ static void check_similarities_fuzzy_(base_operator_ &&base_operator, simd_opera status_t status_base = base_operator(first_tape.view(), second_tape.view(), results_base.data()); status_t status_simd = simd_operator(first_tape.view(), second_tape.view(), results_simd.data(), simd_extra_args...); - sz_assert_(status_base == status_t::success_k); - sz_assert_(status_simd == status_t::success_k); + verify(status_base == status_t::success_k && "Base engine failed on a fuzzy-generated batch"); + verify(status_simd == status_t::success_k && "SIMD engine failed on a fuzzy-generated batch"); // Individually log the failed results for (std::size_t i = 0; i != config.batch_size; ++i) { if (results_base[i] == results_simd[i]) continue; edit_distance_log_mismatch(first_array[i], second_array[i], results_base[i], results_simd[i]); - sz_assert_(results_base[i] == results_simd[i]); + verify(results_base[i] == results_simd[i] && "Base and SIMD engines disagree on a fuzzy-generated pair"); } } } @@ -713,9 +725,9 @@ void test_similarities_equivalence() { // Distance can be computed from the similarity, by inverting the sign around the length of the longest string: auto distance_nw = std::max(7, 7) - similarity_nw; auto distance_sw = std::max(7, 7) - similarity_sw; - sz_assert_(distance_l == 1); - sz_assert_(distance_nw == 1); - sz_assert_(distance_sw == 1); + verify(distance_l == 1); + verify(distance_nw == 1); + verify(distance_sw == 1); } // Let's define some weird scoring schemes for Levenshtein-like distance, that are not unary: @@ -897,7 +909,7 @@ void test_similarities_equivalence() { #if SZ_USE_CUDA gpu_specs_t first_gpu_specs; - sz_assert_(gpu_specs_fetch(first_gpu_specs) == status_t::success_k); + let_verify(status_t const specs_status = gpu_specs_fetch(first_gpu_specs), specs_status == status_t::success_k); #endif #if SZ_USE_CUDA @@ -1009,7 +1021,7 @@ static void check_similarities_degenerate_(base_operator_ &&base_operator, simd_ // The degenerate corpus: empty/empty, empty/non-empty, single-char, identical, and a near-identical // one-edit pair. They live in one batch so the engines also face a mixed-length, mostly-tiny input. - std::vector<std::pair<std::string, std::string>> degenerate_cases { + std::vector<similarity_case_t> degenerate_cases { {"", ""}, // both empty; distance 0 {"", "ABC"}, // empty vs non-empty; pure insertion {"ABC", ""}, // non-empty vs empty; pure deletion @@ -1023,20 +1035,24 @@ static void check_similarities_degenerate_(base_operator_ &&base_operator, simd_ unified_vector<score_type_> results_simd(degenerate_cases.size()); arrow_strings_tape_t first_tape, second_tape; for (auto const &pair : degenerate_cases) { - sz_assert_(first_tape.try_append({pair.first.data(), pair.first.size()}) == status_t::success_k); - sz_assert_(second_tape.try_append({pair.second.data(), pair.second.size()}) == status_t::success_k); + let_verify(status_t const first_append_status = first_tape.try_append({pair.first.data(), pair.first.size()}), + first_append_status == status_t::success_k); + let_verify( + status_t const second_append_status = second_tape.try_append({pair.second.data(), pair.second.size()}), + second_append_status == status_t::success_k); } status_t status_base = base_operator(first_tape.view(), second_tape.view(), results_base.data()); status_t status_simd = simd_operator(first_tape.view(), second_tape.view(), results_simd.data(), simd_extra_args...); - sz_assert_(status_base == status_t::success_k); - sz_assert_(status_simd == status_t::success_k); + verify(status_base == status_t::success_k && "Base engine failed on the degenerate-case batch"); + verify(status_simd == status_t::success_k && "SIMD engine failed on the degenerate-case batch"); for (std::size_t pair_index = 0; pair_index != degenerate_cases.size(); ++pair_index) { if (results_base[pair_index] == results_simd[pair_index]) continue; edit_distance_log_mismatch(degenerate_cases[pair_index].first, degenerate_cases[pair_index].second, results_base[pair_index], results_simd[pair_index]); - sz_assert_(results_base[pair_index] == results_simd[pair_index]); + verify(results_base[pair_index] == results_simd[pair_index] && + "Base and SIMD engines disagree on a degenerate-case pair"); } // Closed-form identities, independent of any O(n²) reference. With a single uniform-cost string the only @@ -1044,18 +1060,23 @@ static void check_similarities_degenerate_(base_operator_ &&base_operator, simd_ std::string const sample = "ADVERSARIAL"; std::string const empty; arrow_strings_tape_t sample_tape, empty_tape, copy_tape; - sz_assert_(sample_tape.try_append({sample.data(), sample.size()}) == status_t::success_k); - sz_assert_(empty_tape.try_append({empty.data(), empty.size()}) == status_t::success_k); - sz_assert_(copy_tape.try_append({sample.data(), sample.size()}) == status_t::success_k); + let_verify(status_t const sample_append_status = sample_tape.try_append({sample.data(), sample.size()}), + sample_append_status == status_t::success_k); + let_verify(status_t const empty_append_status = empty_tape.try_append({empty.data(), empty.size()}), + empty_append_status == status_t::success_k); + let_verify(status_t const copy_append_status = copy_tape.try_append({sample.data(), sample.size()}), + copy_append_status == status_t::success_k); unified_vector<score_type_> closed_form(1); score_type_ const expected_all_gaps = static_cast<score_type_>(sample.size() * static_cast<std::size_t>(gap_cost)); - sz_assert_(simd_operator(sample_tape.view(), empty_tape.view(), closed_form.data(), simd_extra_args...) == - status_t::success_k); - sz_assert_(closed_form[0] == expected_all_gaps); - sz_assert_(simd_operator(sample_tape.view(), copy_tape.view(), closed_form.data(), simd_extra_args...) == - status_t::success_k); - sz_assert_(closed_form[0] == static_cast<score_type_>(0)); + let_verify(status_t const against_empty_status = simd_operator(sample_tape.view(), empty_tape.view(), + closed_form.data(), simd_extra_args...), + against_empty_status == status_t::success_k); + verify(closed_form[0] == expected_all_gaps); + let_verify(status_t const against_copy_status = simd_operator(sample_tape.view(), copy_tape.view(), + closed_form.data(), simd_extra_args...), + against_copy_status == status_t::success_k); + verify(closed_form[0] == static_cast<score_type_>(0)); } /** @@ -1082,7 +1103,7 @@ void test_similarities_safety() { #if SZ_USE_CUDA gpu_specs_t first_gpu_specs; - sz_assert_(gpu_specs_fetch(first_gpu_specs) == status_t::success_k); + let_verify(status_t const specs_status = gpu_specs_fetch(first_gpu_specs), specs_status == status_t::success_k); #endif // Serial Levenshtein distance against the dual-row baseline on degenerate inputs. @@ -1174,7 +1195,8 @@ static void fill_reference_matrix_(baseline_operator_ const &baseline, arrow_str arrow_strings_view_t const candidate_cell {candidates_view.buffer_, candidates_view.offsets_.subspan(candidate_index, 2)}; score_type_ cell_score = 0; - sz_assert_(baseline(query_cell, candidate_cell, &cell_score) == status_t::success_k); + let_verify(status_t const cell_status = baseline(query_cell, candidate_cell, &cell_score), + cell_status == status_t::success_k); reference_matrix[query_index * row_stride + candidate_index] = cell_score; } } @@ -1193,8 +1215,9 @@ static void check_cross_product_cell_exact_(engine_type_ &&engine, baseline_oper // empty sub-view sliced (one offset, zero strings) off a one-string fallback tape. std::string const empty_fallback_string; arrow_strings_tape_t empty_fallback_tape; - sz_assert_(empty_fallback_tape.try_append({empty_fallback_string.data(), empty_fallback_string.size()}) == - status_t::success_k); + let_verify(status_t const fallback_append_status = empty_fallback_tape.try_append( + {empty_fallback_string.data(), empty_fallback_string.size()}), + fallback_append_status == status_t::success_k); auto build_view = [&](fuzzy_config_t config, std::vector<std::string> &array, arrow_strings_tape_t &tape) -> arrow_strings_view_t { if (config.batch_size == 0) @@ -1217,7 +1240,7 @@ static void check_cross_product_cell_exact_(engine_type_ &&engine, baseline_oper strided_rows<score_type_> const results {engine_matrix.data(), queries_count, candidates_count, row_stride}; status_t const status = engine(queries_view, candidates_view, results, trailing_args...); - sz_assert_(status == status_t::success_k); + verify(status == status_t::success_k && "Cross-product engine failed to fill the Q x C score matrix"); // The empty shape is fully validated by the success status above - there are no cells to compare. if (queries_count == 0 || candidates_count == 0) return; @@ -1229,7 +1252,8 @@ static void check_cross_product_cell_exact_(engine_type_ &&engine, baseline_oper if (engine_matrix[cell_offset] == reference_matrix[cell_offset]) continue; edit_distance_log_mismatch(queries_array[query_index], candidates_array[candidate_index], reference_matrix[cell_offset], engine_matrix[cell_offset]); - sz_assert_(engine_matrix[cell_offset] == reference_matrix[cell_offset]); + verify(engine_matrix[cell_offset] == reference_matrix[cell_offset] && + "Cross-product engine and dual-row baseline disagree on this Q x C cell"); } } @@ -1252,15 +1276,17 @@ static void check_symmetric_cell_exact_(engine_type_ &&engine, fuzzy_config_t se unified_vector<sz_size_t> symmetric_matrix(sequences_count * sequences_count); strided_rows<sz_size_t> const results {symmetric_matrix.data(), sequences_count, sequences_count, row_stride}; status_t const status = engine(sequences_view, results, trailing_args...); - sz_assert_(status == status_t::success_k); + verify(status == status_t::success_k && "Symmetric engine failed to fill the self-similarity matrix"); // The diagonal-is-zero identity holds only for a zero match cost (a string aligned to itself pays nothing). // The symmetry identity holds for any cost scheme, so it is always asserted. for (std::size_t row_index = 0; row_index != sequences_count; ++row_index) { - sz_assert_(symmetric_matrix[row_index * row_stride + row_index] == static_cast<sz_size_t>(0)); + verify(symmetric_matrix[row_index * row_stride + row_index] == static_cast<sz_size_t>(0) && + "Self-similarity matrix has a non-zero diagonal cell"); for (std::size_t column_index = 0; column_index != sequences_count; ++column_index) - sz_assert_(symmetric_matrix[row_index * row_stride + column_index] == - symmetric_matrix[column_index * row_stride + row_index]); + verify(symmetric_matrix[row_index * row_stride + column_index] == + symmetric_matrix[column_index * row_stride + row_index] && + "Self-similarity matrix is not symmetric at this cell"); } } @@ -1270,8 +1296,8 @@ static void check_symmetric_cell_exact_(engine_type_ &&engine, fuzzy_config_t se * These carry no kernel geometry: a `1 x N` row, an `N x 1` column, a lone cell, a ragged square set containing * empty strings, a rectangular set, the three degenerate matrices, and the symmetric one-set matrix with its zero * diagonal. Every backend owes the same answers here regardless of its lane width, so wiring a backend in is one - * call rather than a dozen copied lines - which is how `empty_set` came to be tested on the serial engine alone - * while a CUDA defect in that exact shape went unnoticed. + * call rather than a dozen copied lines - `empty_set` is exercised on every wired backend here, including CUDA, + * pinning that exact shape against a CUDA-specific defect too. * * Per-ISA tuning shapes deliberately stay in their own blocks: their batch sizes and lengths encode one kernel's * lane count or tier edge, and generalizing them would erase the reason each number was chosen. @@ -1332,7 +1358,7 @@ void check_cross_product_universals_(trailing_arguments_ &&...trailing_arguments * `Q x C` matrix and the symmetric one-set matrix directly produced by the new API, asserting each cell against * the dual-row baseline (cross) or the symmetry/zero-diagonal identities (symmetric). */ -void test_similarities_cross_product() { +void test_similarities_cross_product_equivalence() { std::printf(" - testing cross-product and symmetric similarity matrices...\n"); [[maybe_unused]] constexpr uniform_substitution_costs_t unit_uniform {0, 1}; // used only in SIMD #if blocks @@ -1373,8 +1399,8 @@ void test_similarities_cross_product() { blosum62_matrix, blosum62_linear_cost}, smith_waterman_baselines_t {blosum62_matrix, blosum62_linear_cost}, many_queries, many_candidates); - // Straddle the per-pair `i16` → `i32` cell-width step, near a combined 327 at magnitude 100. Sizing the cell - // from `longer + 1` instead of the walkers' reach used to underflow the affine seed and flip the score's sign. + // Straddle the per-pair `i16` → `i32` cell-width step, near a combined 327 at magnitude 100. Pins the cell + // sized from `longer + 1`, wide enough that the affine seed cannot underflow and flip the score's sign. { error_costs_32x32_t wide_matrix {}; for (std::size_t first = 0; first != 32; ++first) @@ -1960,16 +1986,16 @@ void test_similarities_cross_product() { #if SZ_USE_CUDA gpu_specs_t first_gpu_specs; - sz_assert_(gpu_specs_fetch(first_gpu_specs) == status_t::success_k); + let_verify(status_t const specs_status = gpu_specs_fetch(first_gpu_specs), specs_status == status_t::success_k); // CUDA cross-product and symmetric matrices stay small so device memory remains bounded. The empty shapes here - // are the ones that used to reach `cuLaunchKernelEx` with a zero grid. + // pin that a zero-sized grid never reaches `cuLaunchKernelEx`. check_cross_product_universals_<sz_cap_cuda_k, ualloc_t>(cuda_executor_t {}, first_gpu_specs); #endif #if SZ_USE_KEPLER - // Kepler had no cross-product coverage at all: its `tile_scorer` specializations were only ever driven through - // the pairwise suites, so the matrix overloads went unexercised on that capability. + // This pins Kepler's cross-product coverage: its `tile_scorer` specializations are otherwise driven only + // through the pairwise suites, leaving the matrix overloads untested on that capability without this block. check_cross_product_universals_<sz_caps_ck_k, ualloc_t>(cuda_executor_t {}, first_gpu_specs); #endif @@ -2022,7 +2048,7 @@ void test_similarities_cross_product() { // High-cost Levenshtein at 700 chars: past the register tier, below the tiled promotion, and wide enough that // the reach needs 4-byte cells - the only shape that reaches the warp tier's `u32` kernel. Mirrors the CPU - // `wide_lev_*` checks above, which were the sole coverage of that band. + // `wide_lev_*` checks above, which are the only CPU-side coverage of that band. fuzzy_config_t const wide_lev_queries {"ABC", /* batch */ 2, /* min */ 700, /* max */ 700}; fuzzy_config_t const wide_lev_candidates {"ABC", /* batch */ 8, /* min */ 700, /* max */ 700}; check_cross_product_cell_exact_<sz_size_t>( @@ -2059,10 +2085,10 @@ void test_similarities_cross_product() { smith_waterman_baselines_t {blosum62_matrix, blosum62_linear_cost}, weighted_mid_200, empty_set, cuda_executor_t {}, first_gpu_specs); - // CUDA UTF-8 rune scoring. The CPU blocks above are the only UTF-8 coverage in this file, so the GPU engine's - // whole-batch tier routing - which, unlike the byte engine, never consults `task.density` - was unexercised. - // Linear costs only: `similarities.cuh` extern-templates just that one for the GPU, and affine UTF-8 is - // documented as staying on the CPU. + // CUDA UTF-8 rune scoring. This is the only GPU UTF-8 coverage in this file: it pins the GPU engine's + // whole-batch tier routing, which, unlike the byte engine, never consults `task.density`. + // Linear costs only: the library header `include/stringzillas/similarities.cuh` extern-templates just that + // one for the GPU, and affine UTF-8 is documented as staying on the CPU. { levenshtein_distances_utf8<linear_gap_costs_t, malloc_t, sz_cap_serial_k> utf8_cuda_oracle {}; auto const utf8_cuda_baseline = [&utf8_cuda_oracle](arrow_strings_view_t queries, @@ -2122,7 +2148,7 @@ void test_similarities_cross_product() { * the whole sweep, so the experiment list is ordered cheapest-first and truncated by `SZ_TESTS_MULTIPLIER`, * letting the emulated CI legs keep the cheap prefix without dropping the shape coverage entirely. */ -void test_similarities_memory_usage() { +void test_similarities_memory_usage_equivalence() { // Cheapest-first, so a reduced `SZ_TESTS_MULTIPLIER` keeps the cheap prefix. Cost grows as `batch * length²`, // so the long rows dominate and stay few; repeating a length at several batch sizes buys nothing. @@ -2141,11 +2167,14 @@ void test_similarities_memory_usage() { {"ABC", /* batch_size */ 4, /* min_string_length */ 4096, /* max_string_length */ 4096}, {"ABC", /* batch_size */ 1, /* min_string_length */ 8192, /* max_string_length */ 8192}, }; - correctness_experiments.resize(scale_iterations(correctness_experiments.size())); + // Clamped to the table: resizing past it appends default-constructed experiments, not harder ones. + static constexpr std::size_t default_experiments_k = 7; + correctness_experiments.resize( + sz_min_of_two(scale_iterations(default_experiments_k), correctness_experiments.size())); #if SZ_USE_CUDA gpu_specs_t first_gpu_specs; - sz_assert_(gpu_specs_fetch(first_gpu_specs) == status_t::success_k); + let_verify(status_t const specs_status = gpu_specs_fetch(first_gpu_specs), specs_status == status_t::success_k); #endif // Let's define some weird scoring schemes for Levenshtein-like distance, that are not unary: @@ -2218,6 +2247,63 @@ void test_similarities_memory_usage() { } } +/** + * @brief Pins the device-memory contract for the CUDA cross-product engines: a result matrix must be + * reachable from the device, whether it is unified or plain device memory. + * + * Host inputs were already refused; this covers the results, which used to be staged and copied back. + */ +void test_similarities_cuda_memory_safety() { + std::printf(" - testing unified, host, pinned and device result matrices against the contract...\n"); +#if SZ_USE_CUDA + + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + std::vector<std::string> const texts {"kitten", "sitting", "flaw"}; + arrow_strings_tape_t tape; + verify(tape.try_assign(texts.begin(), texts.end()) == status_t::success_k); + arrow_strings_view_t const view = tape.view(); + std::size_t const count = view.size(); + + constexpr uniform_substitution_costs_t unit_uniform {0, 1}; + constexpr linear_gap_costs_t unit_linear {1}; + levenshtein_distances<linear_gap_costs_t, ualloc_t, sz_cap_cuda_k> engine {unit_uniform, unit_linear}; + + // Unified results are the baseline, and give the answers the other rows are compared against. + unified_vector<sz_size_t> unified_matrix(count * count); + strided_rows<sz_size_t> const unified_results {unified_matrix.data(), count, count, count}; + verify(engine(view, view, unified_results, executor, gpu_specs) == status_t::success_k && + "A unified result matrix must be accepted"); + + // Host results are refused rather than staged and drained, which is what this contract removed. + std::vector<sz_size_t> host_matrix(count * count); + strided_rows<sz_size_t> const host_results {host_matrix.data(), count, count, count}; + verify(engine(view, view, host_results, executor, gpu_specs) == status_t::device_memory_mismatch_k && + "A host result matrix must be refused"); + + // Page-locked host memory is host memory to the driver, and refused with it. + pinned_vector<sz_size_t> pinned_matrix(count * count); + strided_rows<sz_size_t> const pinned_results {pinned_matrix.data(), count, count, count}; + verify(engine(view, view, pinned_results, executor, gpu_specs) == status_t::device_memory_mismatch_k && + "A page-locked result matrix must be refused"); + + // Plain device memory is accepted, and agrees with the unified answers once drained. + device_vector<sz_size_t> device_matrix; + verify(device_matrix.try_resize_uninitialized(count * count) == status_t::success_k); + strided_rows<sz_size_t> const device_results {device_matrix.data(), count, count, count}; + verify(engine(view, view, device_results, executor, gpu_specs) == status_t::success_k && + "A plain device result matrix must be accepted"); + + std::vector<sz_size_t> drained(count * count); + verify(copy_device_to_host(device_matrix, span<sz_size_t>(drained.data(), drained.size())) == CUDA_SUCCESS && + "Draining the device matrix must succeed"); + for (std::size_t cell = 0; cell < count * count; ++cell) + verify(drained[cell] == unified_matrix[cell] && "Device and unified matrices must agree cell for cell"); +#endif // SZ_USE_CUDA +} + #pragma endregion // Drivers } // namespace scripts diff --git a/test/similarities.py b/test/similarities.py index def5eb3f..32a93400 100644 --- a/test/similarities.py +++ b/test/similarities.py @@ -926,6 +926,7 @@ def make_engine(capabilities): # region Interop +@pytest.mark.parametrize("device_name", DEVICE_NAMES) @pytest.mark.parametrize( "engine_cls, oracle_fn", [ @@ -934,24 +935,18 @@ def make_engine(capabilities): ], ids=["needleman_wunsch", "smith_waterman"], ) -def test_alignment_out_buffer_matches_returned_matrix(engine_cls, oracle_fn): - """The `out=` output-buffer argument must be filled in place AND be the exact object returned, - matching both a fresh call with no `out=` given and the `affine_gaps` oracle.""" +def test_alignment_out_buffer_matches_returned_matrix(engine_cls, oracle_fn, device_name: DeviceName): + """On a CPU scope `out=` must be filled in place AND be the exact object returned; on a GPU scope a + host `out=` is refused. Either way the scores match the `affine_gaps` oracle.""" + device_scope, base_caps = device_scope_and_capabilities(device_name) gap_opening, gap_extension = ag.default_gap_opening, ag.default_gap_extension alphabet, byte_to_class, class_costs = protein_substitution_tables() - engine = engine_cls(byte_to_class, class_costs, open=gap_opening, extend=gap_extension) + engine = engine_cls(byte_to_class, class_costs, open=gap_opening, extend=gap_extension, capabilities=base_caps) queries = Strs(DEGENERATE_PROTEIN_STRINGS) candidates = Strs(DEGENERATE_PROTEIN_STRINGS) - direct_matrix = engine(queries, candidates) - - out_buffer = np.full(direct_matrix.shape, -123456, dtype=np.int64) - returned_matrix = engine(queries, candidates, out=out_buffer) - - assert returned_matrix is out_buffer, "out= must be returned as-is, not copied into a new array" - assert np.array_equal(out_buffer, direct_matrix), "out= buffer must be filled with the same scores as a fresh call" - + direct_matrix = engine(queries, candidates, device=device_scope) oracle_matrix = np.array( [ [ @@ -969,7 +964,17 @@ def test_alignment_out_buffer_matches_returned_matrix(engine_cls, oracle_fn): ], dtype=np.int64, ) - assert np.array_equal(out_buffer, oracle_matrix) + assert np.array_equal(direct_matrix, oracle_matrix) + + out_buffer = np.full(direct_matrix.shape, -123456, dtype=np.int64) + if device_name == "gpu_device": + with pytest.raises(BufferError): + engine(queries, candidates, device=device_scope, out=out_buffer) + return + + returned_matrix = engine(queries, candidates, device=device_scope, out=out_buffer) + assert returned_matrix is out_buffer, "out= must be returned as-is, not copied into a new array" + assert np.array_equal(out_buffer, direct_matrix), "out= must be filled with the same scores as a fresh call" def test_levenshtein_distances_pyarrow_input_matches_list_input(): diff --git a/test/sort.cpp b/test/sort.cpp index b3a0c830..d5d11392 100644 --- a/test/sort.cpp +++ b/test/sort.cpp @@ -1,6 +1,6 @@ /** * @brief Sequence sort equivalence/backends/algorithms and intersection tests. - * @file scripts/test_sort.cpp + * @file test/sort.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -85,23 +85,37 @@ using namespace std::literals; // for ""sv static void check_sort_unit_(sz_sequence_argsort_t argsort, sz_sequence_t const *sequence, std::vector<sz_sorted_idx_t> const &expected) { std::vector<sz_sorted_idx_t> order(expected.size()); - verify(argsort(sequence, nullptr, order.data(), 0, sz_false_k) == sz_success_k); + verify(argsort(sequence, nullptr, order.data(), 0, sz_false_k) == sz_success_k && "Kernel call failed"); verify(order == expected); } +/** @brief One matched pair from an intersection: `first_index` into the first sequence, `second_index` into the second. */ +struct intersect_match_t { + std::size_t first_index; + std::size_t second_index; + + bool operator<(intersect_match_t const &other) const noexcept { + return first_index != other.first_index ? first_index < other.first_index : second_index < other.second_index; + } + bool operator==(intersect_match_t const &other) const noexcept { + return first_index == other.first_index && second_index == other.second_index; + } +}; + /** @brief Runs one sequence intersect backend over both inputs and asserts the matched (first, second) pairs. */ static void check_intersect_unit_(sz_sequence_intersect_t intersect, sz_sequence_t const *first_sequence, sz_sequence_t const *second_sequence, - std::set<std::pair<std::size_t, std::size_t>> const &expected_pairs) { + std::set<intersect_match_t> const &expected_pairs) { sz_size_t const capacity = first_sequence->count < second_sequence->count ? // first_sequence->count : second_sequence->count; std::vector<sz_sorted_idx_t> first_positions(capacity), second_positions(capacity); sz_size_t intersection_size = 0; verify(intersect(first_sequence, second_sequence, nullptr, 0u, &intersection_size, // - first_positions.data(), second_positions.data()) == sz_success_k); - verify(intersection_size == expected_pairs.size()); - std::set<std::pair<std::size_t, std::size_t>> produced; + first_positions.data(), second_positions.data()) == sz_success_k && + "Kernel call failed"); + verify(intersection_size == expected_pairs.size() && "Kernel reported the wrong intersection size"); + std::set<intersect_match_t> produced; for (sz_size_t index = 0; index != intersection_size; ++index) produced.insert({(std::size_t)first_positions[index], (std::size_t)second_positions[index]}); verify(produced == expected_pairs); @@ -118,12 +132,8 @@ static void check_intersect_unit_(sz_sequence_intersect_t intersect, sz_sequence * natively-compiled backend kernels directly (manual propagation to a specific kernel), and through the * C++ `sz::argsort` / `sz::intersect` wrappers, so a regression that the serial-vs-SIMD agreement tests * would miss - because both share the same wrong ordering - is still caught against an external ground truth. - * Then sorts incrementally complex inputs against `std::stable_sort` references: - * 1. Basic tests with predetermined orders. - * 2. Test on long strings of identical length. - * 3. Test on random very small strings of varying lengths, likely with many equal inputs. - * 4. Test on random strings of varying lengths. - * 5. Test on random strings of varying lengths with zero characters. + * The randomized sweeps against `std::stable_sort` live in `test_sort_reference_equivalence`, since this + * tier has to cost the same at every multiplier. */ void test_sort_unit() { using strs_t = std::vector<std::string>; @@ -195,17 +205,20 @@ void test_sort_unit() { // The matched pairs by (first index, second index): banana=(1,2) and cherry=(2,0). Output order is // unspecified, so the helper collects pairs into a set before comparing against the known intersection. - std::set<std::pair<std::size_t, std::size_t>> const expected_pairs = {{1u, 2u}, {2u, 0u}}; + std::set<intersect_match_t> const expected_pairs = {{1u, 2u}, {2u, 0u}}; check_intersect_unit_(sz_sequence_intersect, &first_sequence, &second_sequence, expected_pairs); check_intersect_unit_(sz_sequence_intersect_serial, &first_sequence, &second_sequence, expected_pairs); #if SZ_USE_ICELAKE check_intersect_unit_(sz_sequence_intersect_icelake, &first_sequence, &second_sequence, expected_pairs); #endif +#if SZ_USE_SVE + check_intersect_unit_(sz_sequence_intersect_sve, &first_sequence, &second_sequence, expected_pairs); +#endif sz::intersect_result_t const result = sz::intersect(first, second); verify(result.first_offsets.size() == 2u && result.second_offsets.size() == 2u); - std::set<std::pair<std::size_t, std::size_t>> wrapper_pairs; + std::set<intersect_match_t> wrapper_pairs; for (std::size_t index = 0; index != result.first_offsets.size(); ++index) wrapper_pairs.insert({result.first_offsets[index], result.second_offsets[index]}); verify(wrapper_pairs == expected_pairs); @@ -231,10 +244,10 @@ void test_sort_unit() { {first_positions.data(), first_positions.size()}, {second_positions.data(), second_positions.size()}); verify(matched.status == sz::status_t::success_k); verify(matched.value == 2u); - std::set<std::pair<std::size_t, std::size_t>> low_level_pairs; + std::set<intersect_match_t> low_level_pairs; for (std::size_t index = 0; index != matched.value; ++index) low_level_pairs.insert({first_positions[index], second_positions[index]}); - std::set<std::pair<std::size_t, std::size_t>> const expected_low_level = {{1u, 2u}, {2u, 0u}}; + std::set<intersect_match_t> const expected_low_level = {{1u, 2u}, {2u, 0u}}; verify(low_level_pairs == expected_low_level); } @@ -263,11 +276,139 @@ void test_sort_unit() { let_verify(auto result = sz::argsort( strs_t({"Anna", "Andrew", "Alex", "Bob", "Bobby", "Charlie", "Chris", "David", "Dan"})), result == order_t({2u, 1u, 0u, 3u, 4u, 5u, 6u, 8u, 7u})); +} - // Known-answer sizes stay fixed: this is the `_unit` tier, so it must cost the same at every multiplier. - // The largest crosses 100k strings to exercise the large-input partitioning path. +/** @brief Known-answer intersection pairs through the dispatched API, every native kernel, and the C++ wrapper. */ +void test_intersect_unit() { + using strs_t = std::vector<std::string>; + using result_t = sz::intersect_result_t; + + // The mapping aren't guaranteed to be in any specific order, so we will sort them for comparisons. + using idx_pairs_t = std::set<intersect_match_t>; + auto to_pairs = [](result_t const &result) -> idx_pairs_t { + idx_pairs_t pairs; + for (std::size_t i = 0; i < result.first_offsets.size(); ++i) + pairs.insert({result.first_offsets[i], result.second_offsets[i]}); + return pairs; + }; + + // Predetermined simple cases + { + strs_t abcd({"a", "b", "c", "d"}); + strs_t dcba({"d", "c", "b", "a"}); + strs_t abs({"a", "b", "s"}); + strs_t empty; + result_t result; + // Empty sets + { + result = sz::intersect(empty, empty); + verify(result.first_offsets.size() == 0 && result.second_offsets.size() == 0); + result = sz::intersect(abcd, empty); + verify(result.first_offsets.size() == 0 && result.second_offsets.size() == 0); + } + // Each predetermined non-empty case is verified through the C++ wrapper and through the dispatched API plus + // every natively-compiled kernel, so a serial/SIMD consensus that disagrees with the known answer is caught. + using kernel_pairs_t = std::set<intersect_match_t>; + auto check_all_intersect_kernels_ = [](sz_sequence_t const *first_sequence, + sz_sequence_t const *second_sequence, + kernel_pairs_t const &expected_pairs) { + // Dispatched (automatic kernel resolution). + check_intersect_unit_(sz_sequence_intersect, first_sequence, second_sequence, expected_pairs); + // Manual propagation to each natively-compiled backend kernel. + check_intersect_unit_(sz_sequence_intersect_serial, first_sequence, second_sequence, expected_pairs); +#if SZ_USE_ICELAKE + check_intersect_unit_(sz_sequence_intersect_icelake, first_sequence, second_sequence, expected_pairs); +#endif +#if SZ_USE_SVE + check_intersect_unit_(sz_sequence_intersect_sve, first_sequence, second_sequence, expected_pairs); +#endif + }; + sz_sequence_t const abcd_sequence = sequence_from_(abcd); + sz_sequence_t const dcba_sequence = sequence_from_(dcba); + sz_sequence_t const abs_sequence = sequence_from_(abs); + + // Identity check + { + result = sz::intersect(abcd, abcd); + verify(result.first_offsets.size() == 4 && result.second_offsets.size() == 4); + verify(to_pairs(result) == idx_pairs_t({{0u, 0u}, {1u, 1u}, {2u, 2u}, {3u, 3u}})); + check_all_intersect_kernels_(&abcd_sequence, &abcd_sequence, {{0u, 0u}, {1u, 1u}, {2u, 2u}, {3u, 3u}}); + } + // Identical size, different order + { + result = sz::intersect(abcd, dcba); + verify(result.first_offsets.size() == 4 && result.second_offsets.size() == 4); + verify(to_pairs(result) == idx_pairs_t({{0u, 3u}, {1u, 2u}, {2u, 1u}, {3u, 0u}})); + check_all_intersect_kernels_(&abcd_sequence, &dcba_sequence, {{0u, 3u}, {1u, 2u}, {2u, 1u}, {3u, 0u}}); + } + // Different sets + { + result = sz::intersect(abcd, abs); + verify(result.first_offsets.size() == 2 && result.second_offsets.size() == 2); + verify(to_pairs(result) == idx_pairs_t({{0u, 0u}, {1u, 1u}})); + check_all_intersect_kernels_(&abcd_sequence, &abs_sequence, {{0u, 0u}, {1u, 1u}}); + } + } +} + +/** + * @brief Randomized intersection sizes against `sz::intersect`, across dataset sizes and shapes. + * + * Lives here rather than in `test_intersect_unit` because it draws fresh corpora every run: the `_unit` tier + * has to cost the same at every multiplier, and these sweeps are exactly what does not. + */ +void test_intersect_equivalence() { + std::printf(" - testing intersection sizes against random string sets...\n"); + + using strs_t = std::vector<std::string>; + using result_t = sz::intersect_result_t; + + struct { + std::size_t min_length; + std::size_t max_length; + std::size_t count_strings; + } experiments[] = { + {10, 10, 100}, + {15, 15, 1000}, + {5, 30, 2000}, + }; + auto &generator = global_random_generator(); + for (auto experiment : experiments) { + std::unordered_set<std::string> random_strings; + while (random_strings.size() < experiment.count_strings) + random_strings.insert(sz::scripts::random_string( + experiment.min_length + generator() % (experiment.max_length - experiment.min_length + 1), // + "ab", 2)); + + strs_t all_strings(random_strings.begin(), random_strings.end()); + strs_t first_half(all_strings.begin(), all_strings.begin() + all_strings.size() / 2); + + // Try different joins + result_t result; + result = sz::intersect(all_strings, first_half); + verify(result.first_offsets.size() == first_half.size() && result.second_offsets.size() == first_half.size() && + "A subset intersected with its superset must recover the whole subset"); + } +} + +#pragma endregion // Unit + +/** + * @brief Randomized sorting against a `std::stable_sort` reference, across dataset sizes and shapes. + * + * Lives here rather than in `test_sort_unit` because it draws fresh corpora every run: the `_unit` tier + * has to cost the same at every multiplier, and these sweeps are exactly what does not. + */ +void test_sort_reference_equivalence() { + std::printf(" - testing sorting against a std::stable_sort reference...\n"); + + using strs_t = std::vector<std::string>; + using order_t = std::vector<sz::sorted_idx_t>; + + // Sizes scale with the multiplier, as this tier's contract requires. The largest crosses 100k strings + // to exercise the large-input partitioning path. std::size_t const dataset_sizes[] = {10u, 100u, 1000u, 10000u, 100000u}; - std::size_t const experiment_count = 10u; + std::size_t const experiment_count = scale_iterations(10); // Test on long strings of identical length. for (std::size_t string_length : {5u, 25u}) { @@ -280,7 +421,8 @@ void test_sort_unit() { for (std::size_t experiment_idx = 0; experiment_idx < experiment_count; ++experiment_idx) { std::shuffle(dataset.begin(), dataset.end(), global_random_generator()); auto order = sz::argsort(dataset); - for (std::size_t i = 1; i < dataset.size(); ++i) verify(dataset[order[i - 1]] <= dataset[order[i]]); + for (std::size_t i = 1; i < dataset.size(); ++i) + verify(dataset[order[i - 1]] <= dataset[order[i]] && "argsort output is not sorted"); } } } @@ -294,7 +436,9 @@ void test_sort_unit() { for (std::size_t experiment_idx = 0; experiment_idx < experiment_count; ++experiment_idx) { std::shuffle(dataset.begin(), dataset.end(), global_random_generator()); auto order = sz::argsort(dataset); - for (std::size_t i = 1; i < dataset_size; ++i) { verify(dataset[order[i - 1]] <= dataset[order[i]]); } + for (std::size_t i = 1; i < dataset_size; ++i) { + verify(dataset[order[i - 1]] <= dataset[order[i]] && "argsort output is not sorted"); + } } } @@ -309,7 +453,9 @@ void test_sort_unit() { for (std::size_t experiment_idx = 0; experiment_idx < experiment_count; ++experiment_idx) { std::shuffle(dataset.begin(), dataset.end(), global_random_generator()); auto order = sz::argsort(dataset); - for (std::size_t i = 1; i < dataset_size; ++i) { verify(dataset[order[i - 1]] <= dataset[order[i]]); } + for (std::size_t i = 1; i < dataset_size; ++i) { + verify(dataset[order[i - 1]] <= dataset[order[i]] && "argsort output is not sorted"); + } } } @@ -322,7 +468,9 @@ void test_sort_unit() { for (std::size_t experiment_idx = 0; experiment_idx < experiment_count; ++experiment_idx) { std::shuffle(dataset.begin(), dataset.end(), global_random_generator()); auto order = sz::argsort(dataset); - for (std::size_t i = 1; i < dataset_size; ++i) { verify(dataset[order[i - 1]] <= dataset[order[i]]); } + for (std::size_t i = 1; i < dataset_size; ++i) { + verify(dataset[order[i - 1]] <= dataset[order[i]] && "argsort output is not sorted"); + } } } @@ -360,142 +508,54 @@ void test_sort_unit() { } return order.size() == mixed_count; }; - auto reference_order = [&](std::vector<std::string> const &keys, bool reverse) { + enum class sort_direction_t : bool { ascending_k, descending_k }; + auto reference_order = [&](std::vector<std::string> const &keys, sort_direction_t direction) { order_t reference(mixed_count); std::iota(reference.begin(), reference.end(), 0u); std::stable_sort(reference.begin(), reference.end(), [&](sz::sorted_idx_t a, sz::sorted_idx_t b) { int const ordering = compare_bytes(keys[a], keys[b]); - if (ordering != 0) return reverse ? ordering > 0 : ordering < 0; + if (ordering != 0) return direction == sort_direction_t::descending_k ? ordering > 0 : ordering < 0; return a < b; // Equal keys stay ascending by original index in both directions. }); return reference; }; // Ascending and descending must match a byte-key stable sort exactly. - verify(is_permutation(sz::argsort(mixed)) && sz::argsort(mixed) == reference_order(mixed, false)); - verify(sz::argsort(mixed, 0, true) == reference_order(mixed, true)); + verify(is_permutation(sz::argsort(mixed)) && "argsort output is not a permutation"); + verify(sz::argsort(mixed) == reference_order(mixed, sort_direction_t::ascending_k) && + "Ascending argsort disagrees with the stable-sort reference"); + verify(sz::argsort(mixed, 0, true) == reference_order(mixed, sort_direction_t::descending_k) && + "Descending argsort disagrees with the stable-sort reference"); // Top-K must reproduce the value-prefix of the full sort and stay a permutation. for (std::size_t top_count : {std::size_t(1), std::size_t(50), std::size_t(777), mixed_count}) { - for (bool reverse : {false, true}) { + for (sort_direction_t direction : {sort_direction_t::ascending_k, sort_direction_t::descending_k}) { + bool const reverse = direction == sort_direction_t::descending_k; order_t const got = sz::argsort(mixed, top_count, reverse); - order_t const reference = reference_order(mixed, reverse); - verify(is_permutation(got)); + order_t const reference = reference_order(mixed, direction); + verify(is_permutation(got) && "Top-K argsort output is not a permutation"); std::size_t const head = top_count < mixed_count ? top_count : mixed_count; - for (std::size_t i = 0; i < head; ++i) verify(mixed[got[i]] == mixed[reference[i]]); + for (std::size_t i = 0; i < head; ++i) + verify(mixed[got[i]] == mixed[reference[i]] && "Top-K prefix disagrees with the full sort"); } } // Uncased sort must match folding every string then byte-stable-sorting. std::vector<std::string> folded(mixed_count); for (std::size_t i = 0; i < mixed_count; ++i) folded[i] = fold_string(mixed[i]); - verify(sz::argsort_utf8_uncased(mixed) == reference_order(folded, false)); - verify(sz::argsort_utf8_uncased(mixed, 0, true) == reference_order(folded, true)); + verify(sz::argsort_utf8_uncased(mixed) == reference_order(folded, sort_direction_t::ascending_k) && + "Ascending uncased argsort disagrees with the folded stable-sort reference"); + verify(sz::argsort_utf8_uncased(mixed, 0, true) == reference_order(folded, sort_direction_t::descending_k) && + "Descending uncased argsort disagrees with the folded stable-sort reference"); } -/** @brief Known-answer intersection pairs through the dispatched API, every native kernel, and the C++ wrapper. */ -void test_intersect_unit() { - using strs_t = std::vector<std::string>; - using result_t = sz::intersect_result_t; - - // The mapping aren't guaranteed to be in any specific order, so we will sort them for comparisons. - using idx_pair_t = std::pair<std::size_t, std::size_t>; - using idx_pairs_t = std::set<idx_pair_t>; - auto to_pairs = [](result_t const &result) -> idx_pairs_t { - idx_pairs_t pairs; - for (std::size_t i = 0; i < result.first_offsets.size(); ++i) - pairs.insert({result.first_offsets[i], result.second_offsets[i]}); - return pairs; - }; - - // Predetermined simple cases - { - strs_t abcd({"a", "b", "c", "d"}); - strs_t dcba({"d", "c", "b", "a"}); - strs_t abs({"a", "b", "s"}); - strs_t empty; - result_t result; - // Empty sets - { - result = sz::intersect(empty, empty); - verify(result.first_offsets.size() == 0 && result.second_offsets.size() == 0); - result = sz::intersect(abcd, empty); - verify(result.first_offsets.size() == 0 && result.second_offsets.size() == 0); - } - // Each predetermined non-empty case is verified through the C++ wrapper and through the dispatched API plus - // every natively-compiled kernel, so a serial/SIMD consensus that disagrees with the known answer is caught. - using kernel_pairs_t = std::set<std::pair<std::size_t, std::size_t>>; - auto check_all_intersect_kernels = [](sz_sequence_t const *first_sequence, sz_sequence_t const *second_sequence, - kernel_pairs_t const &expected_pairs) { - check_intersect_unit_(sz_sequence_intersect, first_sequence, second_sequence, expected_pairs); // Dispatched - check_intersect_unit_(sz_sequence_intersect_serial, first_sequence, second_sequence, expected_pairs); -#if SZ_USE_ICELAKE - check_intersect_unit_(sz_sequence_intersect_icelake, first_sequence, second_sequence, expected_pairs); -#endif - }; - sz_sequence_t const abcd_sequence = sequence_from_(abcd); - sz_sequence_t const dcba_sequence = sequence_from_(dcba); - sz_sequence_t const abs_sequence = sequence_from_(abs); - - // Identity check - { - result = sz::intersect(abcd, abcd); - verify(result.first_offsets.size() == 4 && result.second_offsets.size() == 4); - verify(to_pairs(result) == idx_pairs_t({{0u, 0u}, {1u, 1u}, {2u, 2u}, {3u, 3u}})); - check_all_intersect_kernels(&abcd_sequence, &abcd_sequence, {{0u, 0u}, {1u, 1u}, {2u, 2u}, {3u, 3u}}); - } - // Identical size, different order - { - result = sz::intersect(abcd, dcba); - verify(result.first_offsets.size() == 4 && result.second_offsets.size() == 4); - verify(to_pairs(result) == idx_pairs_t({{0u, 3u}, {1u, 2u}, {2u, 1u}, {3u, 0u}})); - check_all_intersect_kernels(&abcd_sequence, &dcba_sequence, {{0u, 3u}, {1u, 2u}, {2u, 1u}, {3u, 0u}}); - } - // Different sets - { - result = sz::intersect(abcd, abs); - verify(result.first_offsets.size() == 2 && result.second_offsets.size() == 2); - verify(to_pairs(result) == idx_pairs_t({{0u, 0u}, {1u, 1u}})); - check_all_intersect_kernels(&abcd_sequence, &abs_sequence, {{0u, 0u}, {1u, 1u}}); - } - } - - // Generate random strings - struct { - std::size_t min_length; - std::size_t max_length; - std::size_t count_strings; - } experiments[] = { - {10, 10, 100}, - {15, 15, 1000}, - {5, 30, 2000}, - }; - for (auto experiment : experiments) { - std::unordered_set<std::string> random_strings; - while (random_strings.size() < experiment.count_strings) - random_strings.insert(sz::scripts::random_string( - experiment.min_length + std::rand() % (experiment.max_length - experiment.min_length + 1), // - "ab", 2)); - - strs_t all_strings(random_strings.begin(), random_strings.end()); - strs_t first_half(all_strings.begin(), all_strings.begin() + all_strings.size() / 2); - - // Try different joins - result_t result; - result = sz::intersect(all_strings, first_half); - verify(result.first_offsets.size() == first_half.size() && result.second_offsets.size() == first_half.size()); - } -} - -#pragma endregion // Unit - #pragma region Equivalence /** * @brief One backend's byte + uncased sequence arg-sort kernels, stored by pointer so the differential driver can * iterate a table. */ -struct sequence_sort_backend_t { +struct sort_backend_t { char const *name; sz_sequence_argsort_t argsort; sz_sequence_argsort_t argsort_uncased; @@ -509,7 +569,7 @@ struct sequence_sort_backend_t { * and reference `order` arrays must match exactly across ascending, descending, and top-K modes. */ template <typename reference_, typename candidate_> -void test_sort_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_sort_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { std::size_t const repetition_count = scale_iterations(inputs); using strs_t = std::vector<std::string>; @@ -574,13 +634,89 @@ void test_sort_equivalence(reference_ reference, candidate_ candidate, sz_size_t #pragma endregion // Equivalence +#pragma region Safety + +/** + * @brief Degenerate sequences for the sorting family, asserting the permutation stays a permutation. + * + * An empty sequence, a single element, and one where every string is identical each have a defensible + * answer, and what is asserted here is the shape of the reply rather than its order: the output must be a + * permutation of the input indices, every index present exactly once. An all-equal input is the one that + * catches a comparator returning a strict order where it should report a tie, since any ordering of it + * looks sorted and only the permutation property fails. + */ +void test_sort_safety() { + std::printf(" - testing degenerate sequences of the sorting kernels...\n"); + + using strs_t = std::vector<std::string>; + + // Every compiled kernel is asked directly: going through `sz::argsort` would only ever reach whichever + // one the dispatcher picks on this machine, leaving the rest of the table unexercised. + auto check_is_permutation_ = [](char const *name, sz_sequence_argsort_t argsort, strs_t const &input) { + sz_sequence_t const sequence = sequence_from_(input); + std::vector<sz_sorted_idx_t> order(input.size()); + verify(argsort(&sequence, nullptr, order.data(), 0, sz_false_k) == sz_success_k && "Kernel call failed"); + std::vector<bool> seen(input.size(), false); + for (sz_sorted_idx_t const index : order) { + if ((std::size_t)index >= input.size() || seen[(std::size_t)index]) { + std::fprintf(stderr, "%s: argsort produced %s for a %zu-element input\n", name, + (std::size_t)index >= input.size() ? "an out-of-range index" : "a repeated index", + input.size()); + verify(false && "A sort's output must be a permutation of the input indices"); + } + seen[(std::size_t)index] = true; + } + }; + + strs_t degenerate_inputs[] = { + strs_t {}, // Empty sequence + strs_t {"only"}, // One element + strs_t(17, "same"), // All equal - any order looks sorted, so only the shape can fail + strs_t(129, "same"), // Past the insertion-sort cutover, still all equal + strs_t {"", "a", "", "aa", "a", "", "aaa"}, // Empty strings and prefixes, where a length tiebreak decides + }; + // Differing only past an embedded NUL, which a length-truncating comparison would call equal. + strs_t embedded; + embedded.push_back(std::string("a\0b", 3)); + embedded.push_back(std::string("a\0a", 3)); + + auto sweep = [&](char const *name, sz_sequence_argsort_t argsort) { + for (strs_t const &input : span_over(degenerate_inputs)) check_is_permutation_(name, argsort, input); + check_is_permutation_(name, argsort, embedded); + }; + + sweep("dispatched", sz_sequence_argsort); + sweep("serial", sz_sequence_argsort_serial); + sweep("dispatched uncased", sz_sequence_argsort_uncased); + sweep("serial uncased", sz_sequence_argsort_uncased_serial); +#if SZ_USE_HASWELL + sweep("haswell", sz_sequence_argsort_haswell); +#endif +#if SZ_USE_SKYLAKE + sweep("skylake", sz_sequence_argsort_skylake); +#endif +#if SZ_USE_SVE + sweep("sve", sz_sequence_argsort_sve); +#endif +#if SZ_USE_NEON + sweep("neon", sz_sequence_argsort_neon); +#endif +#if SZ_USE_RVV + sweep("rvv", sz_sequence_argsort_rvv); +#endif + + std::printf(" degenerate-sequence safety passed!\n"); +} + +#pragma endregion // Safety + #pragma region Drivers /** * @brief The sequence arg-sort backends compiled on this target. The always-present `dispatched` entry keeps the * table non-empty on a baseline build. */ -static sequence_sort_backend_t const sequence_sort_backends[] = { +static sort_backend_t const sequence_sort_backends[] = { {"dispatched", sz_sequence_argsort, sz_sequence_argsort_uncased}, #if SZ_USE_HASWELL {"haswell", sz_sequence_argsort_haswell, sz_sequence_argsort_uncased_haswell}, @@ -599,13 +735,12 @@ static sequence_sort_backend_t const sequence_sort_backends[] = { #endif }; -/** @brief Runs `test_sort_equivalence` (serial reference vs every compiled backend, dispatched first). */ +/** @brief Runs `check_sort_equivalence_` (serial reference vs every compiled backend, dispatched first). */ void test_sort_all() { - sequence_sort_backend_t const serial {"serial", sz_sequence_argsort_serial, sz_sequence_argsort_uncased_serial}; + sort_backend_t const serial {"serial", sz_sequence_argsort_serial, sz_sequence_argsort_uncased_serial}; // Four repetitions at multiplier 1.0, one per top-K mode; `SZ_TESTS_MULTIPLIER` dials both ways from here. constexpr sz_size_t repetitions = 4; - for (sequence_sort_backend_t const &backend : sequence_sort_backends) - test_sort_equivalence(serial, backend, repetitions); + for (sort_backend_t const &backend : sequence_sort_backends) check_sort_equivalence_(serial, backend, repetitions); } #pragma endregion // Drivers diff --git a/test/string.cpp b/test/string.cpp index bc839f0c..dbf99567 100644 --- a/test/string.cpp +++ b/test/string.cpp @@ -1,6 +1,6 @@ /** * @brief Arithmetic/struct plumbing, ASCII utilities, memory, STL-compat, conversions, extensions, and the string class. - * @file scripts/test_string.cpp + * @file test/string.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -130,7 +130,7 @@ struct accounting_allocator : public std::allocator<char> { } void deallocate(char *val, std::size_t n) { - verify(n <= counter_ref()); + verify(n <= counter_ref() && "Deallocated more bytes than were tracked as allocated"); counter_ref() -= n; print_if_verbose("dealloc: %zd -> %zd\n", n, counter_ref()); std::allocator<char>::deallocate(val, n); @@ -151,13 +151,13 @@ struct accounting_allocator : public std::allocator<char> { template <typename callback_type> void assert_balanced_memory(callback_type callback) { auto bytes = accounting_allocator::account_block(callback); - verify(bytes == 0); + verify(bytes == 0 && "Callback leaked or double-freed tracked allocator bytes"); } /** * @brief Runs one movement backend (copy/move/fill) through hand-verifiable known-answer vectors. * - * Mirrors the SHA256 known-answer helper in `test_hash.cpp`: each ISA tier feeds its kernel pointers here, + * Mirrors the SHA256 known-answer helper in `hash.cpp`: each ISA tier feeds its kernel pointers here, * so the dispatched C API and every natively-compiled backend share a single ground-truth check. Guard bytes * past `length` catch stray writes. */ @@ -171,8 +171,8 @@ static void check_memory_unit_(sz_copy_t copy, sz_move_t move, sz_fill_t fill) { char target[sizeof(source) + 1]; std::memset(target, '#', sizeof(target)); copy(target, source, length); - verify(std::memcmp(target, source, length) == 0); - verify(target[length] == '#'); // No overwrite past `length` + verify(std::memcmp(target, source, length) == 0 && "Copy backend diverged from the known-answer source"); + verify(target[length] == '#' && "Copy backend wrote past the requested length"); } // `move` handles overlapping regions. Shifting "abcdef" left-into-itself by two yields "cdef" at the front. @@ -180,7 +180,7 @@ static void check_memory_unit_(sz_copy_t copy, sz_move_t move, sz_fill_t fill) { char const expected[] = "cdef"; // After moving "cdef" (offset 2, 4 bytes) to offset 0 char buffer[] = "abcdef"; move(buffer, buffer + 2, 4); - verify(std::memcmp(buffer, expected, 4) == 0); + verify(std::memcmp(buffer, expected, 4) == 0 && "Move backend produced wrong bytes for overlapping shift"); } // `fill` writes a known byte across a known span, leaving a guard byte untouched. @@ -189,8 +189,8 @@ static void check_memory_unit_(sz_copy_t copy, sz_move_t move, sz_fill_t fill) { char target[5 + 1]; std::memset(target, '#', sizeof(target)); fill(target, 5, (sz_u8_t)'*'); - verify(std::memcmp(target, expected, 5) == 0); - verify(target[5] == '#'); // No overwrite past `length` + verify(std::memcmp(target, expected, 5) == 0 && "Fill backend produced wrong bytes for the known pattern"); + verify(target[5] == '#' && "Fill backend wrote past the requested length"); } } @@ -213,8 +213,8 @@ static void check_lookup_unit_(sz_lookup_t lookup) { char target[sizeof(source) + 1]; std::memset(target, '#', sizeof(target)); lookup(target, length, source, upper_table); - verify(std::memcmp(target, expected, length) == 0); - verify(target[length] == '#'); // No overwrite past `length` + verify(std::memcmp(target, expected, length) == 0 && "Lookup backend diverged from the known-answer upper-casing"); + verify(target[length] == '#' && "Lookup backend wrote past the requested length"); } #pragma endregion // Helpers @@ -410,7 +410,7 @@ void test_allocator_unit() { sz_memory_allocator_t alloc; sz_memory_allocator_init_default(&alloc); void *byte = alloc.allocate(1, alloc.handle); - verify(byte != nullptr); + verify(byte != nullptr && "Default allocator returned NULL for a non-zero-length allocation"); alloc.free(byte, 1, alloc.handle); } @@ -420,7 +420,7 @@ void test_allocator_unit() { sz_memory_allocator_t alloc; sz_memory_allocator_init_fixed(&alloc, buffer, sizeof(buffer)); void *byte = alloc.allocate(1, alloc.handle); - verify(byte != nullptr); + verify(byte != nullptr && "Fixed-buffer allocator returned NULL for an allocation that should fit"); alloc.free(byte, 1, alloc.handle); } } @@ -450,6 +450,7 @@ void test_byteset_unit() { * @brief Tests various ASCII-based methods (e.g., `is_alpha`, `is_digit`) * provided by `sz::string` and `sz::string_view`. */ +/** @brief Known-answer coverage for ASCII classification methods (`is_alpha`, `is_digit`, `contains_only`, ...). */ template <typename string_type> void test_ascii_unit() { @@ -521,8 +522,8 @@ void test_memory_unit(std::size_t max_l2_size) { std::printf(" - testing memory primitive known-answer vectors...\n"); // Movement known-answers, through the dispatched C API and every natively-compiled backend. - check_memory_unit_(sz_copy, sz_move, sz_fill); // Dispatched (automatic kernel) - check_memory_unit_(sz_copy_serial, sz_move_serial, sz_fill_serial); // Manual: serial kernel + check_memory_unit_(sz_copy, sz_move, sz_fill); + check_memory_unit_(sz_copy_serial, sz_move_serial, sz_fill_serial); #if SZ_USE_HASWELL check_memory_unit_(sz_copy_haswell, sz_move_haswell, sz_fill_haswell); #endif @@ -552,8 +553,8 @@ void test_memory_unit(std::size_t max_l2_size) { #endif // Lookup known-answers, through the dispatched C API and every natively-compiled backend. - check_lookup_unit_(sz_lookup); // Dispatched (automatic kernel) - check_lookup_unit_(sz_lookup_serial); // Manual: serial kernel + check_lookup_unit_(sz_lookup); + check_lookup_unit_(sz_lookup_serial); #if SZ_USE_HASWELL check_lookup_unit_(sz_lookup_haswell); #endif @@ -866,15 +867,20 @@ void test_stl_reads_unit() { // Cover every SWAR case for unique string sequences. auto lowercase_alphabet = str("abcdefghijklmnopqrstuvwxyz"); for (std::size_t one_byte_offset = 0; one_byte_offset + 1 <= lowercase_alphabet.size(); ++one_byte_offset) - verify(lowercase_alphabet.find(lowercase_alphabet.substr(one_byte_offset, 1)) == one_byte_offset); + verify(lowercase_alphabet.find(lowercase_alphabet.substr(one_byte_offset, 1)) == one_byte_offset && + "1-byte SWAR needle matched at the wrong offset"); for (std::size_t two_byte_offset = 0; two_byte_offset + 2 <= lowercase_alphabet.size(); ++two_byte_offset) - verify(lowercase_alphabet.find(lowercase_alphabet.substr(two_byte_offset, 2)) == two_byte_offset); + verify(lowercase_alphabet.find(lowercase_alphabet.substr(two_byte_offset, 2)) == two_byte_offset && + "2-byte SWAR needle matched at the wrong offset"); for (std::size_t four_byte_offset = 0; four_byte_offset + 4 <= lowercase_alphabet.size(); ++four_byte_offset) - verify(lowercase_alphabet.find(lowercase_alphabet.substr(four_byte_offset, 4)) == four_byte_offset); + verify(lowercase_alphabet.find(lowercase_alphabet.substr(four_byte_offset, 4)) == four_byte_offset && + "4-byte SWAR needle matched at the wrong offset"); for (std::size_t three_byte_offset = 0; three_byte_offset + 3 <= lowercase_alphabet.size(); ++three_byte_offset) - verify(lowercase_alphabet.find(lowercase_alphabet.substr(three_byte_offset, 3)) == three_byte_offset); + verify(lowercase_alphabet.find(lowercase_alphabet.substr(three_byte_offset, 3)) == three_byte_offset && + "3-byte SWAR needle matched at the wrong offset"); for (std::size_t five_byte_offset = 0; five_byte_offset + 5 <= lowercase_alphabet.size(); ++five_byte_offset) - verify(lowercase_alphabet.find(lowercase_alphabet.substr(five_byte_offset, 5)) == five_byte_offset); + verify(lowercase_alphabet.find(lowercase_alphabet.substr(five_byte_offset, 5)) == five_byte_offset && + "5-byte SWAR needle matched at the wrong offset"); // Simple repeating patterns - with one "almost match" before an actual match in each direction. verify(str("_ab_abc_").find("abc") == 4); @@ -1052,9 +1058,10 @@ void test_stl_reads_unit() { str first_copy = first; str second_copy = second; first_copy.swap(second_copy); - verify(first_copy == second && second_copy == first); + verify(first_copy == second && second_copy == first && + "swap(other) did not exchange contents for this first/second pair"); first_copy.swap(first_copy); - verify(first_copy == second); + verify(first_copy == second && "Self-swap mutated the string for this first/second pair"); } } @@ -1160,8 +1167,6 @@ void test_stl_updates_unit() { // Concatenation. // Following are missing in strings, but are present in vectors. - // scope_verify(str s = "!?", s.push_front('a'), s == "a!?"); - // scope_verify(str s = "!?", s.pop_front(), s == "?"); verify(str().append("test") == "test"); verify(str("test") + "ing" == "testing"); verify(str("test") + str("ing") == "testing"); @@ -1299,16 +1304,19 @@ void test_stl_containers_unit() { std::size_t rank_sz = 0; for (auto const &entry : sorted_words_sz) { - verify(entry.first == ascending_keys[rank_sz]); - verify(entry.second == static_cast<int>(rank_sz)); + verify(entry.first == ascending_keys[rank_sz] && "sz::string map produced the wrong key at sorted rank_sz"); + verify(entry.second == static_cast<int>(rank_sz) && + "sz::string map produced the wrong value at sorted rank_sz"); ++rank_sz; } verify(rank_sz == 5); std::size_t rank_stl = 0; for (auto const &entry : sorted_words_stl) { - verify(entry.first == ascending_keys[rank_stl]); - verify(entry.second == static_cast<int>(rank_stl)); + verify(entry.first == ascending_keys[rank_stl] && + "std::string map via sz::less produced the wrong key at sorted rank_stl"); + verify(entry.second == static_cast<int>(rank_stl) && + "std::string map via sz::less produced the wrong value at sorted rank_stl"); ++rank_stl; } verify(rank_stl == 5); @@ -1331,7 +1339,8 @@ void test_stl_containers_unit() { words_sz.emplace("banana", 7); sz::string grown_sz = "bana"; grown_sz.append("na"); - verify(std::hash<sz::string> {}(grown_sz) == std::hash<sz::string> {}(sz::string("banana"))); + verify(std::hash<sz::string> {}(grown_sz) == std::hash<sz::string> {}(sz::string("banana")) && + "std::hash disagreed for equal-content strings built via different construction paths"); verify(words_sz.find(grown_sz) != words_sz.end()); verify(words_sz.emplace(grown_sz, 9).second == false); verify(words_sz.at(grown_sz) == 7); @@ -1344,7 +1353,8 @@ void test_stl_containers_unit() { std::string grown_stl; for (int repetition = 0; repetition < 200; ++repetition) grown_stl.push_back('x'); words_stl.emplace(heap_key, 7); - verify(sz::hash {}(heap_key) == sz::hash {}(grown_stl)); + verify(sz::hash {}(heap_key) == sz::hash {}(grown_stl) && + "sz::hash disagreed for equal-content strings built via different construction paths"); verify(sz::equal_to {}(heap_key, grown_stl)); verify(sz::equal_to {}(heap_key, "x") == false); verify(words_stl.find(grown_stl) != words_stl.end()); @@ -1520,20 +1530,24 @@ void test_extensions_updates_unit() { str text = "hello brave new world"; for (auto segment : text.utf8_wordbreaks()) { std::ptrdiff_t const offset = segment.data() - text.data(); - verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size())); + verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size()) && + "utf8_wordbreaks segment landed outside the caller's own buffer"); } for (auto token : text.utf8_split_whitespaces()) { std::ptrdiff_t const offset = token.data() - text.data(); - verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size())); + verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size()) && + "utf8_split_whitespaces token landed outside the caller's own buffer"); } for (auto field : text.utf8_split_delimiters()) { std::ptrdiff_t const offset = field.data() - text.data(); - verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size())); + verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(text.size()) && + "utf8_split_delimiters field landed outside the caller's own buffer"); } str sso = "a b c"; for (auto token : sso.utf8_split_whitespaces()) { std::ptrdiff_t const offset = token.data() - sso.data(); - verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(sso.size())); + verify(offset >= 0 && offset <= static_cast<std::ptrdiff_t>(sso.size()) && + "utf8_split_whitespaces token landed outside the small-string-optimized buffer"); } } @@ -1545,6 +1559,173 @@ void test_extensions_updates_unit() { #pragma endregion // Extensions +/** + * @brief The lazy search ranges and their inverses - `find_all`, `rfind_all`, `split`, `rsplit`, `partition`. + * + * Not a template over the string type, unlike its neighbours: these cases deliberately mix owning strings, + * borrowed views and literals in one expression, because what they pin is how the range holds its operands. + * A haystack passed as an lvalue is borrowed, so a match's `data()` must land inside the caller's own buffer + * and not inside a private copy - which under the small-string optimization would still produce plausible + * offsets. A needle, by contrast, is copied into the matcher, so a temporary one may outlive the expression. + */ +void test_extensions_ranges_unit() { + std::printf(" - testing lazy search ranges and splitting...\n"); + + // Searching for a set of characters + verify(sz::string_view("a").find_first_of("az") == 0); + verify(sz::string_view("a").find_last_of("az") == 0); + verify(sz::string_view("a").find_first_of("xz") == sz::string_view::npos); + verify(sz::string_view("a").find_last_of("xz") == sz::string_view::npos); + + verify(sz::string_view("a").find_first_not_of("xz") == 0); + verify(sz::string_view("a").find_last_not_of("xz") == 0); + verify(sz::string_view("a").find_first_not_of("az") == sz::string_view::npos); + verify(sz::string_view("a").find_last_not_of("az") == sz::string_view::npos); + + verify(sz::string_view("aXbYaXbY").find_first_of("XY") == 1); + verify(sz::string_view("axbYaxbY").find_first_of("Y") == 3); + verify(sz::string_view("YbXaYbXa").find_last_of("XY") == 6); + verify(sz::string_view("YbxaYbxa").find_last_of("Y") == 4); + verify(sz::string_view(sz::base64(), sizeof(sz::base64())).find_first_of("_") == sz::string_view::npos); + verify(sz::string_view(sz::base64(), sizeof(sz::base64())).find_first_of("+") == 62); + verify(sz::string_view(sz::ascii_printables(), sizeof(sz::ascii_printables())).find_first_of("~") != + sz::string_view::npos); + + verify("aabaa"_sv.remove_prefix("a") == "abaa"); + verify("aabaa"_sv.remove_suffix("a") == "aaba"); + verify("aabaa"_sv.lstrip("a"_bs) == "baa"); + verify("aabaa"_sv.rstrip("a"_bs) == "aab"); + verify("aabaa"_sv.strip("a"_bs) == "b"); + + // Check more advanced composite operations + verify("abbccc"_sv.partition('b').before.size() == 1); + verify("abbccc"_sv.partition("bb").before.size() == 1); + verify("abbccc"_sv.partition("bb").match.size() == 2); + verify("abbccc"_sv.partition("bb").after.size() == 3); + verify("abbccc"_sv.partition("bb").before == "a"); + verify("abbccc"_sv.partition("bb").match == "bb"); + verify("abbccc"_sv.partition("bb").after == "ccc"); + verify("abb ccc"_sv.partition(sz::whitespaces_set()).after == "ccc"); + + // Check ranges of search matches + verify("hello"_sv.find_all("l").size() == 2); + verify("hello"_sv.rfind_all("l").size() == 2); + + verify(""_sv.find_all(".", sz::include_overlaps_type {}).size() == 0); + verify(""_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 0); + verify("."_sv.find_all(".", sz::include_overlaps_type {}).size() == 1); + verify("."_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 1); + verify(".."_sv.find_all(".", sz::include_overlaps_type {}).size() == 2); + verify(".."_sv.find_all(".", sz::exclude_overlaps_type {}).size() == 2); + verify(""_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 0); + verify(""_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 0); + verify("."_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 1); + verify("."_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 1); + verify(".."_sv.rfind_all(".", sz::include_overlaps_type {}).size() == 2); + verify(".."_sv.rfind_all(".", sz::exclude_overlaps_type {}).size() == 2); + + verify("a.b.c.d"_sv.find_all(".").size() == 3); + verify("a.,b.,c.,d"_sv.find_all(".,").size() == 3); + verify("a.,b.,c.,d"_sv.rfind_all(".,").size() == 3); + verify("a.b,c.d"_sv.find_all(".,"_bs).size() == 3); + verify("a...b...c"_sv.rfind_all("..").size() == 4); + verify("a...b...c"_sv.rfind_all("..", sz::include_overlaps_type {}).size() == 4); + verify("a...b...c"_sv.rfind_all("..", sz::exclude_overlaps_type {}).size() == 2); + + let_verify(auto finds = "a.b.c"_sv.find_all("abcd"_bs).template to<std::vector<std::string>>(), + finds.size() == 3 && finds[0] == "a"); + let_verify(auto rfinds = "a.b.c"_sv.rfind_all("abcd"_bs).template to<std::vector<std::string>>(), + rfinds.size() == 3 && rfinds[0] == "c"); + + // Test propagating strings and their non-owning views into temporary ranges and iterators + verify(sz::find_all("abc"_sv, "b"_sv).size() == 1); + verify(sz::find_all("hello"_sv, "l"_sv).size() == 2); + verify(sz::rfind_all("abc"_sv, "b"_sv).size() == 1); + + { + sz::string h("abc"), n("b"); + verify(sz::find_all(h, n).size() == 1); + } + { + sz::string h("hello"), n("l"); + verify(sz::find_all(h, n).size() == 2); + } + { + sz::string h("abc"), n("b"); + verify(sz::rfind_all(h, n).size() == 1); + } + + verify(sz::find_all(sz::string("abc"), sz::string("b")).size() == 1); + verify(sz::find_all(sz::string("hello"), sz::string("l")).size() == 2); + verify(sz::rfind_all(sz::string("abc"), sz::string("b")).size() == 1); + + // Lvalue haystacks are borrowed, so slices land inside the caller's own buffer. A copied + // haystack would offset into a private copy - and under SSO those offsets look plausible. + { + sz::string haystack("hello world, hello cpp"); + sz::string sso("a b a"); + let_verify(auto matches = sz::find_all(haystack, "hello").template to<std::vector<sz::string_view>>(), + matches.size() == 2 && // + matches[0].data() - haystack.data() == 0 && // + matches[1].data() - haystack.data() == 13 && + "Match offsets did not land inside the borrowed lvalue haystack's own buffer"); + let_verify(auto in_sso = sz::find_all(sso, "a").template to<std::vector<sz::string_view>>(), + in_sso.size() == 2 && // + in_sso[0].data() - sso.data() == 0 && // + in_sso[1].data() - sso.data() == 4 && + "Match offsets did not land inside the small-string-optimized haystack's own buffer"); + } + + // Needles are copied into the matcher, so a temporary one outlives the expression that built it. + verify(sz::find_all(sz::string("hello world, hello cpp"), sz::string("hello")).size() == 2); + + // Haystack and needle need not share a type - literals, views, and owning strings mix. + { + sz::string owning("a-b-c"); + sz::string_view view("a-b-c"); + sz::string needle("-"); + verify(sz::find_all(view, "-").size() == 2); + verify(sz::find_all(owning, "-").size() == 2); + verify(sz::find_all(owning, view.substr(1, 1)).size() == 2); + verify(sz::find_all(view, needle).size() == 2); + verify(sz::split(owning, "-").size() == 3); + verify(sz::rsplit(view, needle).size() == 3); + verify(sz::split_characters(owning, "-").size() == 3); + } + + // Check splitting - the inverse of `find_all` ranges + let_verify(auto splits = ".a..c."_sv.split("."_bs).template to<std::vector<std::string>>(), + splits.size() == 5 && splits[0] == "" && splits[1] == "a" && splits[4] == ""); + let_verify(auto line_splits = "line1\nline2\nline3"_sv.split("line3").template to<std::vector<std::string>>(), + line_splits.size() == 2 && line_splits[0] == "line1\nline2\n" && line_splits[1] == ""); + + verify(""_sv.split(".").size() == 1); + verify(""_sv.rsplit(".").size() == 1); + + verify("hello"_sv.split("l").size() == 3); + verify("hello"_sv.rsplit("l").size() == 3); + verify(*advanced("hello"_sv.split("l").begin(), 0) == "he"); + verify(*advanced("hello"_sv.rsplit("l").begin(), 0) == "o"); + verify(*advanced("hello"_sv.split("l").begin(), 1) == ""); + verify(*advanced("hello"_sv.rsplit("l").begin(), 1) == ""); + verify(*advanced("hello"_sv.split("l").begin(), 2) == "o"); + verify(*advanced("hello"_sv.rsplit("l").begin(), 2) == "he"); + + verify("a.b.c.d"_sv.split(".").size() == 4); + verify("a.b.c.d"_sv.rsplit(".").size() == 4); + verify(*("a.b.c.d"_sv.split(".").begin()) == "a"); + verify(*("a.b.c.d"_sv.rsplit(".").begin()) == "d"); + verify(*advanced("a.b.c.d"_sv.split(".").begin(), 1) == "b"); + verify(*advanced("a.b.c.d"_sv.rsplit(".").begin(), 1) == "c"); + verify(*advanced("a.b.c.d"_sv.split(".").begin(), 3) == "d"); + verify(*advanced("a.b.c.d"_sv.rsplit(".").begin(), 3) == "a"); + verify("a.b.,c,d"_sv.split(".,").size() == 2); + verify("a.b,c.d"_sv.split(".,"_bs).size() == 4); + + let_verify(auto rsplits = ".a..c."_sv.rsplit("."_bs).template to<std::vector<std::string>>(), + rsplits.size() == 5 && rsplits[0] == "" && rsplits[1] == "c" && rsplits[4] == ""); +} + #pragma region String Class /** @brief Tests copy constructor and copy-assignment constructor of `sz::string`. */ @@ -1556,15 +1737,17 @@ void test_string_constructors_unit() { std::vector<sz::string> copies {strings}; verify(copies.size() == strings.size()); for (size_t i = 0; i < copies.size(); ++i) { - verify(copies[i].size() == strings[i].size()); - verify(copies[i] == strings[i]); - for (size_t j = 0; j < strings[i].size(); j++) { verify(copies[i][j] == strings[i][j]); } + verify(copies[i].size() == strings[i].size() && "Copy-constructed string has the wrong length at index i"); + verify(copies[i] == strings[i] && "Copy-constructed string diverged from its source at index i"); + for (size_t j = 0; j < strings[i].size(); j++) + verify(copies[i][j] == strings[i][j] && "Copy-constructed string mismatched a byte at index i, j"); } std::vector<sz::string> assignments = strings; for (size_t i = 0; i < assignments.size(); ++i) { - verify(assignments[i].size() == strings[i].size()); - verify(assignments[i] == strings[i]); - for (size_t j = 0; j < strings[i].size(); j++) { verify(assignments[i][j] == strings[i][j]); } + verify(assignments[i].size() == strings[i].size() && "Copy-assigned string has the wrong length at index i"); + verify(assignments[i] == strings[i] && "Copy-assigned string diverged from its source at index i"); + for (size_t j = 0; j < strings[i].size(); j++) + verify(assignments[i][j] == strings[i][j] && "Copy-assigned string mismatched a byte at index i, j"); } verify(std::equal(strings.begin(), strings.end(), copies.begin())); verify(std::equal(strings.begin(), strings.end(), assignments.begin())); @@ -1609,21 +1792,21 @@ void test_string_reserve_unit() { } /** @brief Checks for memory leaks in the string class using the `accounting_allocator`. */ -void test_memory_stability_unit(std::size_t length, std::size_t iterations) { +void test_memory_stability_equivalence(std::size_t length, std::size_t iterations) { - verify(accounting_allocator::counter_ref() == 0); + verify(accounting_allocator::counter_ref() == 0 && "Allocator counter was not zero before the stability run"); using string = sz::basic_string<char, accounting_allocator>; string base; for (std::size_t i = 0; i < length; ++i) base.push_back('c'); - verify(base.length() == length); + verify(base.length() == length && "Base string has the wrong length after `push_back` construction"); // Do copies leak? assert_balanced_memory([&]() { for (std::size_t i = 0; i < iterations; ++i) { string copy(base); - verify(copy.length() == length); - verify(copy == base); + verify(copy.length() == length && "Copy-constructed string has the wrong length at iteration i"); + verify(copy == base && "Copy-constructed string diverged from `base` at iteration i"); } }); @@ -1632,8 +1815,8 @@ void test_memory_stability_unit(std::size_t length, std::size_t iterations) { for (std::size_t i = 0; i < iterations; ++i) { string copy; copy = base; - verify(copy.length() == length); - verify(copy == base); + verify(copy.length() == length && "Copy-assigned string has the wrong length at iteration i"); + verify(copy == base && "Copy-assigned string diverged from `base` at iteration i"); } }); @@ -1641,11 +1824,11 @@ void test_memory_stability_unit(std::size_t length, std::size_t iterations) { assert_balanced_memory([&]() { for (std::size_t i = 0; i < iterations; ++i) { string unique_item(base); - verify(unique_item.length() == length); - verify(unique_item == base); + verify(unique_item.length() == length && "Pre-move string has the wrong length at iteration i"); + verify(unique_item == base && "Pre-move string diverged from `base` at iteration i"); string copy(std::move(unique_item)); - verify(copy.length() == length); - verify(copy == base); + verify(copy.length() == length && "Move-constructed string has the wrong length at iteration i"); + verify(copy == base && "Move-constructed string diverged from `base` at iteration i"); } }); @@ -1655,8 +1838,9 @@ void test_memory_stability_unit(std::size_t length, std::size_t iterations) { string unique_item(base); string copy; copy = std::move(unique_item); - verify(copy.length() == length); - verify(copy == base); + verify(copy.length() == length && + "Move-assigned (empty target) string has the wrong length at iteration i"); + verify(copy == base && "Move-assigned (empty target) string diverged from `base` at iteration i"); } }); @@ -1667,37 +1851,41 @@ void test_memory_stability_unit(std::size_t length, std::size_t iterations) { string copy; for (std::size_t j = 0; j < 317; j++) copy.push_back('q'); copy = std::move(unique_item); - verify(copy.length() == length); - verify(copy == base); + verify(copy.length() == length && + "Move-assigned (occupied target) string has the wrong length at iteration i"); + verify(copy == base && "Move-assigned (occupied target) string diverged from `base` at iteration i"); } }); // Now let's clear the base and check that we're back to zero base = string(); - verify(accounting_allocator::counter_ref() == 0); + verify(accounting_allocator::counter_ref() == 0 && "Allocator counter did not return to zero after clearing"); } /** @brief Tests the correctness of the string class update methods, such as `push_back` and `erase`. */ -void test_string_updates_unit(std::size_t repetitions) { +void test_string_updates_equivalence(std::size_t repetitions) { // Compare STL and StringZilla strings append functionality. char const alphabet_chars[] = "abcdefghijklmnopqrstuvwxyz"; + auto &generator = global_random_generator(); for (std::size_t repetition = 0; repetition != repetitions; ++repetition) { std::string stl_string; sz::string sz_string; for (std::size_t length = 1; length != 200; ++length) { - char c = alphabet_chars[std::rand() % 26]; + char c = alphabet_chars[generator() % 26]; stl_string.push_back(c); sz_string.push_back(c); - verify(sz::string_view(stl_string) == sz::string_view(sz_string)); + verify(sz::string_view(stl_string) == sz::string_view(sz_string) && + "sz::string diverged from std::string after `push_back`"); } // Compare STL and StringZilla strings erase functionality. while (stl_string.length()) { - std::size_t offset_to_erase = std::rand() % stl_string.length(); - std::size_t chars_to_erase = std::rand() % (stl_string.length() - offset_to_erase) + 1; + std::size_t offset_to_erase = generator() % stl_string.length(); + std::size_t chars_to_erase = generator() % (stl_string.length() - offset_to_erase) + 1; stl_string.erase(offset_to_erase, chars_to_erase); sz_string.erase(offset_to_erase, chars_to_erase); - verify(sz::string_view(stl_string) == sz::string_view(sz_string)); + verify(sz::string_view(stl_string) == sz::string_view(sz_string) && + "sz::string diverged from std::string after `erase`"); } } } @@ -1741,7 +1929,7 @@ inline std::vector<sz_size_t> memory_equivalence_lengths() noexcept { * `inputs` is the number of random source patterns fuzzed at each length. */ template <typename reference_, typename candidate_> -void test_memory_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { +void check_memory_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { std::vector<sz_size_t> const lengths = memory_equivalence_lengths(); sz_size_t const max_length = lengths.back(); @@ -1762,11 +1950,15 @@ void test_memory_equivalence(reference_ reference, candidate_ candidate, sz_size std::vector<char> reference_output(length, '\0'); reference.copy(reference_output.data(), source, length); candidate.copy(target, source, length); - if (length) verify(std::memcmp(reference_output.data(), target, length) == 0); + if (length) + verify(std::memcmp(reference_output.data(), target, length) == 0 && + "Candidate copy backend diverged from the serial reference"); reference.fill(reference_output.data(), length, fill_value); candidate.fill(target, length, fill_value); - if (length) verify(std::memcmp(reference_output.data(), target, length) == 0); + if (length) + verify(std::memcmp(reference_output.data(), target, length) == 0 && + "Candidate fill backend diverged from the serial reference"); }); // `move` with overlapping regions: shift the source pattern within one buffer by a small offset, @@ -1782,14 +1974,16 @@ void test_memory_equivalence(reference_ reference, candidate_ candidate, sz_size std::memcpy(reference_buffer.data(), source, length); candidate.move(buffer + shift, buffer, moved); reference.move(reference_buffer.data() + shift, reference_buffer.data(), moved); - verify(std::memcmp(buffer, reference_buffer.data(), length) == 0); + verify(std::memcmp(buffer, reference_buffer.data(), length) == 0 && + "Candidate move backend diverged from reference on forward overlap"); // Backward overlap: destination behind the source. std::memcpy(buffer, source, length); std::memcpy(reference_buffer.data(), source, length); candidate.move(buffer, buffer + shift, moved); reference.move(reference_buffer.data(), reference_buffer.data() + shift, moved); - verify(std::memcmp(buffer, reference_buffer.data(), length) == 0); + verify(std::memcmp(buffer, reference_buffer.data(), length) == 0 && + "Candidate move backend diverged from reference on backward overlap"); }); } } @@ -1804,29 +1998,43 @@ void test_memory_equivalence(reference_ reference, candidate_ candidate, sz_size * `inputs` is the number of random source patterns fuzzed at each length. */ template <typename reference_, typename candidate_> -void test_lookup_equivalence(reference_ reference, candidate_ candidate, sz_size_t inputs) { - - char lookup_table[256]; - sz_lookup_init_upper(lookup_table); +void check_lookup_equivalence_(reference_ reference, candidate_ candidate, sz_size_t inputs) { + + char upper_table[256], lower_table[256], ascii_table[256]; + sz_lookup_init_upper(upper_table); + sz_lookup_init_lower(lower_table); + sz_lookup_init_ascii(ascii_table); + struct named_table_t { + char const *name; + char const *table; + }; + named_table_t const named_tables[] = { + {"upper", upper_table}, + {"lower", lower_table}, + {"ascii", ascii_table}, + }; std::vector<sz_size_t> const lengths = memory_equivalence_lengths(); sz_size_t const max_length = lengths.back(); - for (sz_size_t length : lengths) { - for (sz_size_t input = 0; input != inputs; ++input) { - - std::vector<char> source_storage(length + SZ_CACHE_LINE_WIDTH, '\0'); - sz_cptr_t const source = source_storage.data() + (input % SZ_CACHE_LINE_WIDTH); - if (length) randomize_string(const_cast<char *>(source), length); - - for_each_cacheline_offset_(max_length, [&](sz_ptr_t target, std::size_t) { - std::vector<char> reference_output(length, '\0'); - reference.lookup(reference_output.data(), length, source, lookup_table); - candidate.lookup(target, length, source, lookup_table); - if (length) verify(std::memcmp(reference_output.data(), target, length) == 0); - }); + for (named_table_t const &named_table : named_tables) + for (sz_size_t length : lengths) { + for (sz_size_t input = 0; input != inputs; ++input) { + + std::vector<char> source_storage(length + SZ_CACHE_LINE_WIDTH, '\0'); + sz_cptr_t const source = source_storage.data() + (input % SZ_CACHE_LINE_WIDTH); + if (length) randomize_string(const_cast<char *>(source), length); + + for_each_cacheline_offset_(max_length, [&](sz_ptr_t target, std::size_t) { + std::vector<char> reference_output(length, '\0'); + reference.lookup(reference_output.data(), length, source, named_table.table); + candidate.lookup(target, length, source, named_table.table); + if (length) + verify(std::memcmp(reference_output.data(), target, length) == 0 && + "Candidate lookup output diverged from reference for this lookup table"); + }); + } } - } } #pragma endregion // Equivalence @@ -1872,18 +2080,23 @@ static void check_memory_safety_(sz_copy_t copy, sz_move_t move, sz_fill_t fill) */ static void check_lookup_safety_(sz_lookup_t lookup) { - char lookup_table[256]; - sz_lookup_init_upper(lookup_table); - - lookup(nullptr, 0, nullptr, lookup_table); // Zero-length must touch nothing - - for (std::size_t length : {(std::size_t)1, (std::size_t)8, (std::size_t)64, (std::size_t)257}) - with_guarded_buffer_(length, [&](sz_ptr_t destination, std::size_t usable_length) { - std::vector<char> source(usable_length, '\0'); - for (std::size_t byte = 0; byte != usable_length; ++byte) - source[byte] = (char)((byte % 3) ? 'a' + (byte % 26) : 0); - lookup(destination, usable_length, source.data(), lookup_table); - }); + char upper_table[256], lower_table[256], ascii_table[256]; + sz_lookup_init_upper(upper_table); + sz_lookup_init_lower(lower_table); + sz_lookup_init_ascii(ascii_table); + char const *const lookup_tables[] = {upper_table, lower_table, ascii_table}; + + for (char const *lookup_table : lookup_tables) { + lookup(nullptr, 0, nullptr, lookup_table); // Zero-length must touch nothing + + for (std::size_t length : {(std::size_t)1, (std::size_t)8, (std::size_t)64, (std::size_t)257}) + with_guarded_buffer_(length, [&](sz_ptr_t destination, std::size_t usable_length) { + std::vector<char> source(usable_length, '\0'); + for (std::size_t byte = 0; byte != usable_length; ++byte) + source[byte] = (char)((byte % 3) ? 'a' + (byte % 26) : 0); + lookup(destination, usable_length, source.data(), lookup_table); + }); + } } /** @@ -1893,8 +2106,11 @@ static void check_lookup_safety_(sz_lookup_t lookup) { */ void test_memory_safety() { - check_memory_safety_(sz_copy, sz_move, sz_fill); // Dispatched (automatic kernel) - check_memory_safety_(sz_copy_serial, sz_move_serial, sz_fill_serial); // Manual: serial kernel + // Dispatched (automatic kernel resolution). + check_memory_safety_(sz_copy, sz_move, sz_fill); + + // Manual propagation to each natively-compiled backend kernel. + check_memory_safety_(sz_copy_serial, sz_move_serial, sz_fill_serial); #if SZ_USE_HASWELL check_memory_safety_(sz_copy_haswell, sz_move_haswell, sz_fill_haswell); #endif @@ -1923,8 +2139,11 @@ void test_memory_safety() { check_memory_safety_(sz_copy_powervsx, sz_move_powervsx, sz_fill_powervsx); #endif - check_lookup_safety_(sz_lookup); // Dispatched (automatic kernel) - check_lookup_safety_(sz_lookup_serial); // Manual: serial kernel + // Dispatched (automatic kernel resolution). + check_lookup_safety_(sz_lookup); + + // Manual propagation to each natively-compiled backend kernel. + check_lookup_safety_(sz_lookup_serial); #if SZ_USE_HASWELL check_lookup_safety_(sz_lookup_haswell); #endif @@ -2036,15 +2255,15 @@ void test_memory_all() { sz_size_t const inputs = (sz_size_t)scale_iterations(2); memory_backend_t const memory_serial {"serial", sz_copy_serial, sz_move_serial, sz_fill_serial}; - for (memory_backend_t const &backend : memory_backends) test_memory_equivalence(memory_serial, backend, inputs); + for (memory_backend_t const &backend : memory_backends) check_memory_equivalence_(memory_serial, backend, inputs); lookup_backend_t const lookup_serial {"serial", sz_lookup_serial}; - for (lookup_backend_t const &backend : lookup_backends) test_lookup_equivalence(lookup_serial, backend, inputs); + for (lookup_backend_t const &backend : lookup_backends) check_lookup_equivalence_(lookup_serial, backend, inputs); } #pragma endregion // Drivers -// Explicit template instantiations for the entry points invoked from `main()` (see `test_stringzilla.cpp`). +// Explicit template instantiations for the entry points invoked from `main()` (see `stringzilla.cpp`). template void test_ascii_unit<sz::string>(); template void test_ascii_unit<sz::string_view>(); #if SZ_IS_CPP17_ && defined(__cpp_lib_string_view) diff --git a/test/string.py b/test/string.py index 47db30c9..ac82241a 100644 --- a/test/string.py +++ b/test/string.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Str and Strs containers: construction, indexing, slicing, comparisons, and the buffer protocol. -Mirrors the C++ scripts/test_string.cpp translation unit. +Mirrors the C++ test/string.cpp translation unit. Covers: Str/Strs construction from native str, bytes, and bytearray, indexing, slicing, and rich comparisons, the read-only buffer protocol, split, rsplit, strip, and splitlines in both eager and @@ -1159,10 +1159,10 @@ def test_strs_from_python_basic(container_class: type, view: bool): def test_strs_from_4gb_list(): """Growing a `Strs` past 4 GB of total content forces its offset table from u32 to u64, and indexing still returns the correct first and last strings afterward. - This will require over 8 GB of memory. To stress-test the behavior, limit memory per process. For 5 and 13 GB: + This will require over 8 GB of memory. To stress-test the behavior, limit memory per process. For 9 and 13 GB: - ulimit -v 9437184 && uv run --no-project python -m pytest scripts/test_stringzilla.py -s -x -k 4gb_list - ulimit -v 13631488 && uv run --no-project python -m pytest scripts/test_stringzilla.py -s -x -k 4gb_list + ulimit -v 9437184 && uv run --no-project python -m pytest test/string.py -s -x -k 4gb_list + ulimit -v 13631488 && uv run --no-project python -m pytest test/string.py -s -x -k 4gb_list """ try: @@ -1201,8 +1201,8 @@ def test_strs_from_4gb_generator(): u64, and indexing still returns the correct first and last strings afterward. This will require over 8 GB of memory. To stress-test the behavior, limit memory per process. For 5 and 13 GB: - ulimit -v 5242880 && uv run --no-project python -m pytest scripts/test_stringzilla.py -s -x -k 4gb_generator - ulimit -v 13631488 && uv run --no-project python -m pytest scripts/test_stringzilla.py -s -x -k 4gb_generator + ulimit -v 5242880 && uv run --no-project python -m pytest test/string.py -s -x -k 4gb_generator + ulimit -v 13631488 && uv run --no-project python -m pytest test/string.py -s -x -k 4gb_generator """ try: diff --git a/test/stringzilla.cpp b/test/stringzilla.cpp index c479a42e..bc9ced15 100644 --- a/test/stringzilla.cpp +++ b/test/stringzilla.cpp @@ -1,6 +1,6 @@ /** * @brief Test entry point and template instantiations; registers every per-domain unit and driver. - * @file scripts/test_stringzilla.cpp + * @file test/stringzilla.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -104,24 +104,7 @@ int main(int argc, char const **argv) { sz_unused_(argc && argv); install_test_signal_handlers(); // Backtrace on SIGSEGV/SIGABRT + line-buffered stdout for crash localization. std::printf("Hi, dear tester! You look nice today!\n"); - std::printf("- Uses Westmere: %s \n", SZ_USE_WESTMERE ? "yes" : "no"); - std::printf("- Uses Goldmont: %s \n", SZ_USE_GOLDMONT ? "yes" : "no"); - std::printf("- Uses Haswell: %s \n", SZ_USE_HASWELL ? "yes" : "no"); - std::printf("- Uses Goldmont: %s \n", SZ_USE_GOLDMONT ? "yes" : "no"); - std::printf("- Uses Skylake: %s \n", SZ_USE_SKYLAKE ? "yes" : "no"); - std::printf("- Uses Ice Lake: %s \n", SZ_USE_ICELAKE ? "yes" : "no"); - std::printf("- Uses NEON: %s \n", SZ_USE_NEON ? "yes" : "no"); - std::printf("- Uses NEON AES: %s \n", SZ_USE_NEONAES ? "yes" : "no"); - std::printf("- Uses NEON SHA: %s \n", SZ_USE_NEONSHA ? "yes" : "no"); - std::printf("- Uses SVE: %s \n", SZ_USE_SVE ? "yes" : "no"); - std::printf("- Uses SVE2: %s \n", SZ_USE_SVE2 ? "yes" : "no"); - std::printf("- Uses SVE2 AES: %s \n", SZ_USE_SVE2AES ? "yes" : "no"); - std::printf("- Uses WASM SIMD128: %s \n", SZ_USE_V128 ? "yes" : "no"); - std::printf("- Uses WASM relaxed SIMD: %s \n", SZ_USE_V128RELAXED ? "yes" : "no"); - std::printf("- Uses RISC-V RVV: %s \n", SZ_USE_RVV ? "yes" : "no"); - std::printf("- Uses LoongArch LASX: %s \n", SZ_USE_LASX ? "yes" : "no"); - std::printf("- Uses Power VSX: %s \n", SZ_USE_POWERVSX ? "yes" : "no"); - std::printf("- Uses CUDA: %s \n", SZ_USE_CUDA ? "yes" : "no"); + log_environment(); print_test_environment(); std::size_t failures = 0; @@ -136,14 +119,18 @@ int main(int argc, char const **argv) { failures += run_test("test_hash_unit", test_hash_unit); failures += run_test("test_hash_all", test_hash_all); failures += run_test("test_hash_multiseed_all", test_hash_multiseed_all); + failures += run_test("test_hash_safety", test_hash_safety); failures += run_test("test_cipher_unit", test_cipher_unit); failures += run_test("test_cipher_safety", test_cipher_safety); failures += run_test("test_cipher_all", test_cipher_all); failures += run_test("test_sort_unit", test_sort_unit); + failures += run_test("test_sort_reference_equivalence", test_sort_reference_equivalence); failures += run_test("test_sort_all", test_sort_all); + failures += run_test("test_sort_safety", test_sort_safety); failures += run_test("test_intersect_unit", test_intersect_unit); + failures += run_test("test_intersect_equivalence", test_intersect_equivalence); failures += run_test("test_ascii_unit<sz::string>", test_ascii_unit<sz::string>); failures += run_test("test_ascii_unit<sz::string_view>", test_ascii_unit<sz::string_view>); @@ -166,25 +153,29 @@ int main(int argc, char const **argv) { failures += run_test("test_extensions_reads_unit<sz::string_view>", test_extensions_reads_unit<sz::string_view>); failures += run_test("test_extensions_reads_unit<sz::string>", test_extensions_reads_unit<sz::string>); failures += run_test("test_extensions_updates_unit", test_extensions_updates_unit); + failures += run_test("test_extensions_ranges_unit", test_extensions_ranges_unit); failures += run_test("test_string_constructors_unit", test_string_constructors_unit); failures += run_test("test_string_reserve_unit", test_string_reserve_unit); - failures += run_test("test_memory_stability_unit(1024)", [] { test_memory_stability_unit(1024); }); - failures += run_test("test_memory_stability_unit(14)", [] { test_memory_stability_unit(14); }); - failures += run_test("test_string_updates_unit", [] { test_string_updates_unit(); }); // ! Defaulted arg + failures += run_test("test_memory_stability_equivalence(1024)", [] { test_memory_stability_equivalence(1024); }); + failures += run_test("test_memory_stability_equivalence(14)", [] { test_memory_stability_equivalence(14); }); + failures += run_test("test_string_updates_equivalence", [] { test_string_updates_equivalence(); }); // ! Defaulted failures += run_test("test_compare_unit", test_compare_unit); failures += run_test("test_find_unit", test_find_unit); failures += run_test("test_find_all", test_find_all); - failures += run_test("test_lookup_all", [] { test_lookup_all(); }); // ! Defaulted args + failures += run_test("test_find_safety", test_find_safety); + failures += run_test("test_lookup_equivalence", [] { test_lookup_equivalence(); }); // ! Defaulted args #if SZ_IS_CPP17_ && defined(__cpp_lib_string_view) - failures += run_test("test_find_misaligned_all", test_find_misaligned_all); + failures += run_test("test_find_misaligned_equivalence", test_find_misaligned_equivalence); #endif failures += run_test("test_utf8_runes_unit", test_utf8_runes_unit); + failures += run_test("test_utf8_runes_scripts_unit", test_utf8_runes_scripts_unit); failures += run_test("test_utf8_runes_safety", test_utf8_runes_safety); failures += run_test("test_utf8_runes_all", test_utf8_runes_all); failures += run_test("test_utf8_tokens_unit", test_utf8_tokens_unit); + failures += run_test("test_utf8_tokens_scripts_unit", test_utf8_tokens_scripts_unit); failures += run_test("test_utf8_tokens_safety", test_utf8_tokens_safety); failures += run_test("test_utf8_tokens_all", test_utf8_tokens_all); failures += run_test("test_utf8_wordbreaks_unit", test_utf8_wordbreaks_unit); @@ -211,6 +202,8 @@ int main(int argc, char const **argv) { failures += run_test("test_utf8_delimiters_all", test_utf8_delimiters_all); failures += run_test("test_uncased_unit", test_uncased_unit); + failures += run_test("test_uncased_scripts_unit", test_uncased_scripts_unit); + failures += run_test("test_uncased_regressions_unit", test_uncased_regressions_unit); failures += run_test("test_uncased_all", test_uncased_all); failures += run_test("test_uncased_safety", test_uncased_safety); diff --git a/test/stringzilla.hpp b/test/stringzilla.hpp index add7b819..c41b063a 100644 --- a/test/stringzilla.hpp +++ b/test/stringzilla.hpp @@ -1,10 +1,10 @@ /** * @brief Helper structures and functions for C++ unit- and stress-tests. - * @file scripts/test_stringzilla.hpp + * @file test/stringzilla.hpp * @author Ash Vardanian * @date June 16, 2026 * - * @section Environment Variables + * @section test_environment_variables Environment Variables * * The test infrastructure supports the following environment variables for reproducible * stress testing and fuzzing: @@ -20,7 +20,26 @@ * - `SZ_TESTS_FILTER` : ECMAScript regex matched against test names; only matching tests run * (e.g. `SZ_TESTS_FILTER=utf8`). Unset or empty runs everything. Honored by `run_test`. * - * @section Example Usage + * @section test_driver_tiers Driver Tiers + * + * A driver's suffix states what it costs and what it may assume, so the name answers both without + * reading the body. A family names its drivers `test_<family>_<tier>`, or `test_<family>_<operation>_<tier>` + * where one family covers several operations - `substrings` counts, finds, rewrites and scores, and each + * wants its own tiers. Helpers that are not drivers take a `check_` prefix and a trailing underscore, and + * are never registered in a `main`. + * + * - `_unit` Known-answer vectors against an external ground truth. Fixed cost: it must run + * identically at every `SZ_TESTS_MULTIPLIER`, so no randomness and no sweeps. + * - `_equivalence` A reference against a candidate over generated corpora - serial against each compiled + * backend, or the library against `std::`. This tier owns randomness. + * - `_safety` Malformed, adversarial and boundary inputs. Asserts survival, bounds and stated + * refusals - never answers, since a wrong answer is not what is under test here. + * Scales with `SZ_TESTS_MULTIPLIER` alongside `_equivalence`; only `_unit` is pinned. + * - `_all` Walks the family's backend table and drives the tiers above. Holds no assertions + * of its own; a literal here belongs in `_unit`. + * - `_rules` Annex rule coverage, where a family transcribes a published spec (UAX-29, UAX-14). + * + * @section test_example_usage Example Usage * * @code{.sh} * # Run with a specific seed for reproducibility @@ -49,7 +68,6 @@ #include <algorithm> // `std::copy`, `std::generate` #include <chrono> // `std::chrono::steady_clock` for per-test timing #include <exception> // `std::exception` -#include <fstream> // `std::ifstream` #include <random> // `std::random_device` #include <regex> // `std::regex_search` for `SZ_TESTS_FILTER` #include <string> // `std::string` @@ -79,12 +97,20 @@ } \ } while (0) +/** + * @brief One case whose subject has to be named before it can be asserted on, scoped to the case. + * + * Prefer it wherever a bare `verify` would need a preceding declaration that outlives its one use: + * a run of these reads as a table of cases, where the same run written longhand reads as prose. + */ #define let_verify(init, condition) \ do { \ init; \ verify(condition); \ } while (0) +/** @brief As `let_verify`, when the subject must also be acted on before the assertion holds - a mutation + * whose result is the subject itself, so there is nothing for the condition to bind. */ #define scope_verify(init, operation, condition) \ do { \ init; \ @@ -92,6 +118,8 @@ verify(condition); \ } while (0) +/** @brief That @p expression throws @p exception_type. The only assertion whose subject is the failure, + * so a passing call - or one that throws something else - is the defect it reports. */ #define throws_verify(expression, exception_type) \ do { \ bool threw = false; \ @@ -122,17 +150,74 @@ template <typename value_type_> using unified_vector = std::vector<value_type_, stringzillas::unified_alloc<value_type_>>; #endif -inline std::string read_file(std::string path) noexcept(false) { - std::ifstream stream(path); - if (!stream.is_open()) throw std::runtime_error("Failed to open file: " + path); - return std::string((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>()); +#if SZ_USE_CUDA +/** + * @brief Page-locked host memory, which the driver reports as host and every engine therefore refuses. + * + * A third memory kind beside unified and device, and the one a caller is most likely to expect to work. + */ +template <typename value_type_> +using pinned_vector = std::vector<value_type_, stringzillas::pinned_alloc<value_type_>>; + +/** + * @brief Plain device memory a kernel can write and the host cannot touch. + * + * `safe_vector` is what the engines already store device-resident scratch in, and its + * `try_resize_uninitialized` is the only growth a non-host-accessible allocator admits. + */ +template <typename value_type_> +using device_vector = stringzillas::safe_vector<value_type_, stringzillas::device_alloc<value_type_>>; + +/** + * @brief Drains a device-resident buffer into @p destination, forwarding whatever the driver reported. + * @param[out] destination At least as many elements as @p source holds; only that prefix is written. + */ +template <typename value_type_> +inline CUresult copy_device_to_host(device_vector<value_type_> const &source, span<value_type_> destination) { + if (source.size() == 0) return CUDA_SUCCESS; + if (destination.size() < source.size()) return CUDA_ERROR_INVALID_VALUE; + return cuMemcpyDtoH(destination.data(), (CUdeviceptr)source.data(), source.size() * sizeof(value_type_)); } +#endif // SZ_USE_CUDA + +/** + * @brief Copies @p texts into unified memory a CUDA kernel can reach, as one span per string. + * + * Owns the bytes the spans point into, so it has to outlive every call that reads `view()`. + */ +struct unified_texts_t { + std::vector<unified_vector<char>> storage; + unified_vector<span<char const>> spans; + + explicit unified_texts_t(std::vector<std::string> const &texts) : storage(texts.size()), spans(texts.size()) { + for (std::size_t index = 0; index != texts.size(); ++index) { + storage[index].assign(texts[index].begin(), texts[index].end()); + spans[index] = {storage[index].data(), storage[index].size()}; + } + } -inline void write_file(std::string path, std::string content) noexcept(false) { - std::ofstream stream(path); - if (!stream.is_open()) throw std::runtime_error("Failed to open file: " + path); - stream << content; - stream.close(); + span<span<char const> const> view() const noexcept { return {spans.data(), spans.size()}; } +}; + +/** + * @brief Reads a file into a string via LibC `<cstdio>`. A non-zero @p max_bytes stops the read after + * that many bytes, so the file tail is never touched. + */ +inline std::string read_file(std::string path, std::size_t max_bytes = 0) noexcept(false) { + std::FILE *file = std::fopen(path.c_str(), "rb"); + if (!file) throw std::runtime_error("Failed to open file: " + path); + std::size_t capacity = max_bytes; + if (capacity == 0) { + std::fseek(file, 0, SEEK_END); + long const size = std::ftell(file); + std::fseek(file, 0, SEEK_SET); + capacity = size > 0 ? static_cast<std::size_t>(size) : 0; + } + std::string content(capacity, '\0'); + std::size_t const read_bytes = std::fread(&content[0], 1, capacity, file); + std::fclose(file); + content.resize(read_bytes); + return content; } /** @@ -391,7 +476,6 @@ inline sz_sequence_t sequence_from_(std::vector<std::string> const &strings) { return sequence; } - struct fuzzy_config_t { std::string alphabet = "ABC"; // ? Drawn one UTF-8 character at a time, so `"ιβÎŗ"` yields valid multi-byte text. std::size_t batch_size = 16; @@ -399,29 +483,22 @@ struct fuzzy_config_t { std::size_t max_string_length = 200; }; -inline void randomize_strings(fuzzy_config_t config, std::vector<std::string> &array, bool unique = false) { +inline void randomize_strings(fuzzy_config_t config, std::vector<std::string> &array) { array.resize(config.batch_size); std::vector<std::string> const characters = alphabet_characters(config.alphabet); std::uniform_int_distribution<std::size_t> length_distribution(config.min_string_length, config.max_string_length); - for (std::size_t i = 0; i != config.batch_size; ++i) - array[i] = random_string(length_distribution(global_random_generator()), characters); - - if (unique) { - std::sort(array.begin(), array.end()); - auto last = std::unique(array.begin(), array.end()); - array.erase(last, array.end()); - } + for (std::size_t index = 0; index != config.batch_size; ++index) + array[index] = random_string(length_distribution(global_random_generator()), characters); } -inline void randomize_strings(fuzzy_config_t config, std::vector<std::string> &array, arrow_strings_tape_t &tape, - bool unique = false) { +inline void randomize_strings(fuzzy_config_t config, std::vector<std::string> &array, arrow_strings_tape_t &tape) { - randomize_strings(config, array, unique); + randomize_strings(config, array); // Convert to a GPU-friendly layout - status_t status = tape.try_assign(array.data(), array.data() + array.size()); - sz_assert_(status == status_t::success_k); + status_t const status = tape.try_assign(array.data(), array.data() + array.size()); + verify(status == status_t::success_k); } inline char const *status_name(status_t s) noexcept { @@ -648,6 +725,7 @@ void test_byteset_unit(); #pragma region Hashing void test_hash_unit(); +void test_hash_safety(); void test_hash_all(); void test_hash_multiseed_all(); @@ -664,9 +742,11 @@ void test_cipher_all(); #pragma region UTF-8 void test_utf8_runes_unit(); +void test_utf8_runes_scripts_unit(); void test_utf8_runes_safety(); void test_utf8_runes_all(); void test_utf8_tokens_unit(); +void test_utf8_tokens_scripts_unit(); void test_utf8_tokens_safety(); void test_utf8_tokens_all(); void test_utf8_wordbreaks_unit(); @@ -697,6 +777,8 @@ void test_utf8_delimiters_all(); #pragma region Uncased UTF-8 void test_uncased_unit(); +void test_uncased_scripts_unit(); +void test_uncased_regressions_unit(); void test_uncased_all(); void test_uncased_safety(); @@ -727,18 +809,20 @@ void test_extensions_reads_unit(); void test_extensions_updates_unit(); void test_string_constructors_unit(); void test_string_reserve_unit(); -void test_memory_stability_unit(std::size_t length = 1ull << 10, std::size_t iterations = scale_iterations(100)); -void test_string_updates_unit(std::size_t repetitions = 1024); +void test_memory_stability_equivalence(std::size_t length = 1ull << 10, std::size_t iterations = scale_iterations(100)); +void test_string_updates_equivalence(std::size_t repetitions = 1024); #pragma endregion // String Class and STL Compatibility #pragma region Search and Comparison void test_compare_unit(); +void test_extensions_ranges_unit(); void test_find_unit(); +void test_find_safety(); void test_find_all(); -void test_find_misaligned_all(); -void test_lookup_all(std::size_t lookup_tables_to_try = 32, std::size_t slices_per_table = 16); +void test_find_misaligned_equivalence(); +void test_lookup_equivalence(std::size_t lookup_tables_to_try = 32, std::size_t slices_per_table = 16); #pragma endregion // Search and Comparison @@ -746,6 +830,9 @@ void test_lookup_all(std::size_t lookup_tables_to_try = 32, std::size_t slices_p void test_sort_all(); void test_sort_unit(); +void test_sort_safety(); +void test_sort_reference_equivalence(); void test_intersect_unit(); +void test_intersect_equivalence(); #pragma endregion // Sequence Algorithms diff --git a/test/stringzillas.cpp b/test/stringzillas.cpp index c928f9a6..d3094204 100644 --- a/test/stringzillas.cpp +++ b/test/stringzillas.cpp @@ -1,8 +1,8 @@ /** * @brief Extensive @b stress-testing suite for StringZillas parallel operations, written in CUDA C++. - * @see Stress-tests on real-world and synthetic data are integrated into the @b `scripts/bench*.cpp` benchmarks. + * @see Stress-tests on real-world and synthetic data are integrated into the benchmarks under @b `bench/`. * - * @file scripts/test_stringzillas.cpp + * @file test/stringzillas.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -31,6 +31,7 @@ #include "stringzilla.hpp" +#include "substrings.cuh" #include "fingerprints.cuh" #include "similarities.cuh" @@ -49,12 +50,29 @@ int main(int argc, char const **argv) { failures += run_test("test_fingerprints_unit", test_fingerprints_unit); failures += run_test("test_fingerprints_equivalence", test_fingerprints_equivalence); failures += run_test("test_fingerprints_safety", test_fingerprints_safety); + failures += run_test("test_fingerprints_cuda_memory_safety", test_fingerprints_cuda_memory_safety); failures += run_test("test_similarities_unit", test_similarities_unit); failures += run_test("test_similarities_equivalence", test_similarities_equivalence); - failures += run_test("test_similarities_cross_product", test_similarities_cross_product); + failures += run_test("test_similarities_cross_product_equivalence", test_similarities_cross_product_equivalence); failures += run_test("test_similarities_safety", test_similarities_safety); - failures += run_test("test_similarities_memory_usage", test_similarities_memory_usage); + failures += run_test("test_similarities_cuda_memory_safety", test_similarities_cuda_memory_safety); + failures += run_test("test_similarities_memory_usage_equivalence", test_similarities_memory_usage_equivalence); + + failures += run_test("test_substrings_unit", test_substrings_unit); + failures += run_test("test_substrings_uncased_unit", test_substrings_uncased_unit); + failures += run_test("test_substrings_uncased_equivalence", test_substrings_uncased_equivalence); + failures += run_test("test_substrings_construction_equivalence", test_substrings_construction_equivalence); + failures += run_test("test_substrings_adversarial_equivalence", test_substrings_adversarial_equivalence); + failures += run_test("test_substrings_large_haystacks_equivalence", test_substrings_large_haystacks_equivalence); + failures += run_test("test_substrings_cover_equivalence", test_substrings_cover_equivalence); + failures += run_test("test_substrings_rewriting_equivalence", test_substrings_rewriting_equivalence); + failures += run_test("test_substrings_scoring_unit", test_substrings_scoring_unit); + failures += run_test("test_substrings_scoring_wide_equivalence", test_substrings_scoring_wide_equivalence); + failures += run_test("test_substrings_cuda_memory_safety", test_substrings_cuda_memory_safety); + failures += run_test("test_substrings_cuda_equivalence", test_substrings_cuda_equivalence); + failures += run_test("test_substrings_safety", test_substrings_safety); + failures += run_test("test_substrings_buffer_safety", test_substrings_buffer_safety); if (failures != 0) { std::fprintf(stderr, "\n%zu test(s) failed.\n", failures); diff --git a/test/stringzillas.cu b/test/stringzillas.cu index a0026048..c2362804 100644 --- a/test/stringzillas.cu +++ b/test/stringzillas.cu @@ -1,8 +1,8 @@ /** * @brief Extensive @b stress-testing suite for StringZillas parallel operations, written in CUDA C++. - * @see Stress-tests on real-world and synthetic data are integrated into the @b `scripts/bench*.cpp` benchmarks. + * @see Stress-tests on real-world and synthetic data are integrated into the benchmarks under @b `bench/`. * - * @file scripts/test_stringzillas.cu + * @file test/stringzillas.cu * @author Ash Vardanian * @date June 16, 2026 */ @@ -31,6 +31,7 @@ #include "stringzilla.hpp" +#include "substrings.cuh" #include "fingerprints.cuh" #include "similarities.cuh" @@ -44,22 +45,40 @@ int main(int argc, char const **argv) { if (auto code = log_environment(); code != 0) return code; print_test_environment(); - int failures = 0; + std::size_t failures = 0; - std::printf("\n=== Fingerprints ===\n"); + std::printf("\nTesting fingerprints\n"); failures += run_test("test_fingerprints_unit", test_fingerprints_unit); failures += run_test("test_fingerprints_equivalence", test_fingerprints_equivalence); failures += run_test("test_fingerprints_safety", test_fingerprints_safety); + failures += run_test("test_fingerprints_cuda_memory_safety", test_fingerprints_cuda_memory_safety); - std::printf("\n=== Similarities ===\n"); + std::printf("\nTesting similarities\n"); failures += run_test("test_similarities_unit", test_similarities_unit); failures += run_test("test_similarities_equivalence", test_similarities_equivalence); - failures += run_test("test_similarities_cross_product", test_similarities_cross_product); + failures += run_test("test_similarities_cross_product_equivalence", test_similarities_cross_product_equivalence); failures += run_test("test_similarities_safety", test_similarities_safety); - failures += run_test("test_similarities_memory_usage", test_similarities_memory_usage); + failures += run_test("test_similarities_cuda_memory_safety", test_similarities_cuda_memory_safety); + failures += run_test("test_similarities_memory_usage_equivalence", test_similarities_memory_usage_equivalence); + + std::printf("\nTesting substrings\n"); + failures += run_test("test_substrings_unit", test_substrings_unit); + failures += run_test("test_substrings_uncased_unit", test_substrings_uncased_unit); + failures += run_test("test_substrings_uncased_equivalence", test_substrings_uncased_equivalence); + failures += run_test("test_substrings_construction_equivalence", test_substrings_construction_equivalence); + failures += run_test("test_substrings_adversarial_equivalence", test_substrings_adversarial_equivalence); + failures += run_test("test_substrings_large_haystacks_equivalence", test_substrings_large_haystacks_equivalence); + failures += run_test("test_substrings_cover_equivalence", test_substrings_cover_equivalence); + failures += run_test("test_substrings_rewriting_equivalence", test_substrings_rewriting_equivalence); + failures += run_test("test_substrings_scoring_unit", test_substrings_scoring_unit); + failures += run_test("test_substrings_scoring_wide_equivalence", test_substrings_scoring_wide_equivalence); + failures += run_test("test_substrings_cuda_memory_safety", test_substrings_cuda_memory_safety); + failures += run_test("test_substrings_cuda_equivalence", test_substrings_cuda_equivalence); + failures += run_test("test_substrings_safety", test_substrings_safety); + failures += run_test("test_substrings_buffer_safety", test_substrings_buffer_safety); if (failures != 0) { - std::fprintf(stderr, "\n%d test(s) failed.\n", failures); + std::fprintf(stderr, "\n%zu test(s) failed.\n", failures); return 1; } std::printf("All tests passed... Unbelievable!\n"); diff --git a/test/stringzillas.py b/test/stringzillas.py index 8b6a9852..14d2f5f4 100644 --- a/test/stringzillas.py +++ b/test/stringzillas.py @@ -285,8 +285,7 @@ def test_parameter_validation(): # Test computation input validation engine = szs.LevenshteinDistances() - # A None query raises `TypeError` or `RuntimeError`, the latter from GPU memory issues. - # `candidates=None` is valid and requests symmetric self-similarity, so it is not an error. + # A None query is a type error; `candidates=None` is valid and requests symmetric self-similarity. with pytest.raises((TypeError, RuntimeError)): engine(None, Strs(["test"])) diff --git a/test/substrings.cuh b/test/substrings.cuh new file mode 100644 index 00000000..97d46da9 --- /dev/null +++ b/test/substrings.cuh @@ -0,0 +1,2775 @@ +/** + * @brief Extensive @b stress-testing suite for the StringZillas multi-pattern search engine (Aho-Corasick). + * @see Stress-tests on real-world and synthetic data are integrated into the @b `bench/substrings.cpp` and + * @b `bench/substrings.cu` benchmarks. + * + * @file test/substrings.cuh + * @author Ash Vardanian + * @date June 16, 2026 + */ +#include "stringzilla/utf8_uncased.h" // `sz_utf8_uncased_search`, the independent single-needle oracle + +#include "stringzillas/substrings.hpp" + +#if SZ_USE_CUDA +#include "stringzillas/substrings.cuh" +#endif + +#if !SZ_IS_CPP17_ +#error "This test requires C++17 or later." +#endif + +#include <cmath> // `std::fabs` +#include <cstdio> // `std::printf`, `std::fprintf` +#include <cstring> // `std::memcmp`, `std::memcpy` + +#include <algorithm> // `std::sort`, `std::unique` +#include <limits> // `std::numeric_limits` +#include <map> // `std::map` +#include <random> // `std::mt19937`, `std::uniform_int_distribution` +#include <set> // `std::set` +#include <string> // `std::string` +#include <utility> // `std::move` +#include <vector> // `std::vector` + +#include <stringzilla/stringzilla.h> // Primary C API + +#include "stringzilla.hpp" // `verify`, `run_test`, `global_random_generator`, `fuzzy_config_t` +#include "utf8.hpp" // `malformed_classes_`, `utf8_random_segmentation_corpus_` + +namespace ashvardanian { +namespace stringzilla { +namespace scripts { + +using ashvardanian::stringzillas::dummy_executor_t; +using ashvardanian::stringzillas::substrings_cased_k; +using ashvardanian::stringzillas::forkunion_executor_t; +using ashvardanian::stringzillas::substrings_bm25_t; +using ashvardanian::stringzillas::substrings_leftmost_first_k; +using ashvardanian::stringzillas::substrings_leftmost_longest_k; +using ashvardanian::stringzillas::substrings_match_t; +using ashvardanian::stringzillas::substrings_overlap_policy_t; +using ashvardanian::stringzillas::substrings_overlapping_k; +using ashvardanian::stringzillas::substrings_case_sensitivity_t; +using ashvardanian::stringzillas::substrings_parallel_t; +using ashvardanian::stringzillas::substrings_serial_t; +using ashvardanian::stringzillas::substrings_state_width_t; +using ashvardanian::stringzillas::substrings_u16_dictionary_t; +using ashvardanian::stringzillas::substrings_u32_dictionary_t; +using ashvardanian::stringzillas::substrings_uncased_k; + +#if SZ_USE_CUDA +using ashvardanian::stringzillas::cuda_executor_t; +using ashvardanian::stringzillas::substrings_cuda_t; +using ashvardanian::stringzillas::gpu_specs_fetch; +using ashvardanian::stringzillas::gpu_specs_t; +#endif + +#pragma region Helpers + +/** @brief Field-by-field ordering for `finalize`; the public match deliberately carries no comparators. */ +inline bool substrings_match_less_(substrings_match_t const &left, substrings_match_t const &right) noexcept { + if (left.haystack_index != right.haystack_index) return left.haystack_index < right.haystack_index; + if (left.needle_index != right.needle_index) return left.needle_index < right.needle_index; + if (left.byte_offset != right.byte_offset) return left.byte_offset < right.byte_offset; + return left.byte_length < right.byte_length; +} +/** @brief Position-only ordering for the cover's touch check; ignores needle index and length. */ +inline bool substrings_match_by_position_less_(substrings_match_t const &left, + substrings_match_t const &right) noexcept { + if (left.haystack_index != right.haystack_index) return left.haystack_index < right.haystack_index; + return left.byte_offset < right.byte_offset; +} +inline bool substrings_match_equal_(substrings_match_t const &left, substrings_match_t const &right) noexcept { + return left.haystack_index == right.haystack_index && left.needle_index == right.needle_index && + left.byte_offset == right.byte_offset && left.byte_length == right.byte_length; +} + +/** + * @brief Sorted, duplicate-checked set of every match one count-then-find pass reported, owning the pass's + * intermediate buffers so a sweep reuses one allocation instead of sizing a fresh pair per cell. + * + * The intermediate `counts` and `matches` buffers are unified, since the CUDA engine writes them; the + * sorted set itself is host-only, and `clear` keeps its backing allocation. + */ +struct substrings_match_set_t { + std::vector<substrings_match_t> matches; + unified_vector<std::size_t> collected_counts; + unified_vector<substrings_match_t> collected_matches; + + substrings_match_set_t() = default; + + /** @brief Builds an already-ordered expectation from a hand-written table, which can therefore spell its + * matches in whatever order reads best. */ + substrings_match_set_t(std::initializer_list<substrings_match_t> initial) { + for (substrings_match_t const &match : initial) matches.push_back(match); + finalize(); + } + + void clear() noexcept { matches.clear(); } + void append(substrings_match_t match) { matches.push_back(match); } + std::size_t size() const noexcept { return matches.size(); } + bool empty() const noexcept { return matches.empty(); } + auto begin() const noexcept { return matches.begin(); } + auto end() const noexcept { return matches.end(); } + + /** @brief Orders the collected matches and fails on a repeat - the four fields identify a match uniquely, + * so a duplicate is always a bug. */ + void finalize() { + std::sort(matches.begin(), matches.end(), substrings_match_less_); + bool const has_duplicate = std::adjacent_find(matches.begin(), matches.end(), substrings_match_equal_) != + matches.end(); + verify(!has_duplicate && "Duplicate (haystack, needle, offset, length) reported twice"); + } + + /** @note Both sides must have been finalized since their last mutation. */ + bool operator==(substrings_match_set_t const &other) const noexcept { + return matches.size() == other.matches.size() && + std::equal(matches.begin(), matches.end(), other.matches.begin(), substrings_match_equal_); + } + bool operator!=(substrings_match_set_t const &other) const noexcept { return !(*this == other); } +}; + +/** + * @brief Runs a full count-then-find pass through @p engine under @p overlap_policy and reduces every match + * to its identity in @p out, whose own buffers hold the intermediates. Every differential check in + * this file is driven through this one public-API shape. + * @note `trailing_args_` forwards an executor (and optionally specs) to backends that want a specific one; + * an empty pack leaves each engine on its own defaults. + */ +template <typename engine_type_, typename haystacks_type_, typename... trailing_args_> +void collect_matches_under_(engine_type_ &engine, haystacks_type_ const &haystacks, + substrings_overlap_policy_t overlap_policy, substrings_match_set_t &out, + trailing_args_ &&...trailing_args) { + out.collected_counts.assign(haystacks.size(), 0); + std::size_t matches_total = 0; + span<std::size_t> const counts {out.collected_counts.data(), out.collected_counts.size()}; + + verify(engine.try_count(haystacks, overlap_policy, counts, matches_total, trailing_args...) == status_t::success_k); + + // Sized to exactly what counting promised, so a `try_find` overrun trips instead of writing into slack. + out.collected_matches.assign(matches_total, substrings_match_t {}); + std::size_t matches_found = 0; + span<substrings_match_t> const matches {out.collected_matches.data(), out.collected_matches.size()}; + verify(engine.try_find(haystacks, overlap_policy, matches, matches_found, trailing_args...) == status_t::success_k); + verify(matches_found == matches_total && "try_count and try_find disagree on the match count"); + + out.clear(); + for (substrings_match_t const &match : out.collected_matches) out.append(match); + out.finalize(); +} + +/** @brief `collect_matches_under_` for the overlapping policy, which is what most checks compare against. */ +template <typename engine_type_, typename haystacks_type_, typename... trailing_args_> +void collect_overlapping_matches_into_(engine_type_ &engine, haystacks_type_ const &haystacks, + substrings_match_set_t &out, trailing_args_ &&...trailing_args) { + collect_matches_under_(engine, haystacks, substrings_overlapping_k, out, + std::forward<trailing_args_>(trailing_args)...); +} + +/** @brief A vocabulary of short random needles/haystack fragments over a ten-letter alphabet, so the + * resulting automaton is small enough to reason about by hand yet large enough to grow a real cold + * tier. Wraps the shared `randomize_strings` in the by-value shape these fixtures read better with. */ +inline std::vector<std::string> random_short_strings_(std::size_t count, std::size_t minimum_length, + std::size_t maximum_length) { + std::vector<std::string> result; + randomize_strings({"abcdefghij", count, minimum_length, maximum_length}, result); + return result; +} + +/** @brief Random haystacks that each carry one of @p needles, so the corpus still reaches the match paths + * once `SZ_TESTS_MULTIPLIER` shrinks it - ten letters spell the whole vocabulary, which leaves a + * chance hit vanishingly rare at small counts, and a fixture comparing engines over no matches at + * all compares nothing. Both rotations advance a phase each turn, so the needles spread across the + * haystacks and land inside them, where covers and neighbouring matches mean something, rather than + * all at the tail. @p needles must not be empty. */ +inline std::vector<std::string> random_haystacks_with_needles_(std::vector<std::string> const &needles, + std::size_t count, std::size_t minimum_length, + std::size_t maximum_length) { + std::vector<std::string> result = random_short_strings_(count, minimum_length, maximum_length); + for (std::size_t index = 0; index < result.size(); ++index) { + std::string const &needle = needles[rotating_index(index, needles.size())]; + result[index].insert(rotating_index(index, result[index].size() + 1), needle); + } + return result; +} + +/** @brief One de-duplicated match span in `independent_uncased_matches_`, keyed by source byte offsets. */ +struct uncased_match_span_t { + std::size_t byte_begin = 0; + std::size_t byte_end = 0; + + bool operator<(uncased_match_span_t const &other) const { + return byte_begin != other.byte_begin ? byte_begin < other.byte_begin : byte_end < other.byte_end; + } +}; + +/** + * @brief Independent case-folded substring oracle: folds haystack and needle codepoint by codepoint via + * `sz_unicode_fold_codepoint_`, then slides the folded needle over the folded haystack, reporting + * every distinct source span whose folded run matches - overlaps included. + * + * Shares no machinery with the Aho-Corasick engine under test beyond the one folding table every backend + * reads. @sa `reference_uncased_find_` in `test/uncased.cpp`, which reports only the first match. + */ +inline std::vector<span<char const>> independent_uncased_matches_(span<char const> haystack, span<char const> needle) { + std::vector<sz_rune_t> needle_folded; + for (char const *cursor = needle.begin(), *end = needle.end(); cursor != end;) { + sz_rune_t rune; + sz_rune_length_t const consumed = sz_rune_decode(cursor, end, &rune); + verify(consumed != sz_rune_invalid_k && "Independent oracle needles must be well-formed UTF-8"); + sz_rune_t folded[3]; + std::size_t const folded_count = sz_unicode_fold_codepoint_(rune, folded); + for (std::size_t index = 0; index < folded_count; ++index) needle_folded.push_back(folded[index]); + cursor += consumed; + } + + std::vector<sz_rune_t> haystack_folded; + std::vector<std::size_t> source_begin, source_end; + for (char const *cursor = haystack.begin(), *end = haystack.end(); cursor != end;) { + sz_rune_t rune; + sz_rune_length_t const consumed = sz_rune_decode(cursor, end, &rune); + verify(consumed != sz_rune_invalid_k && "Independent oracle haystacks must be well-formed UTF-8"); + sz_rune_t folded[3]; + std::size_t const folded_count = sz_unicode_fold_codepoint_(rune, folded); + std::size_t const codepoint_begin = (std::size_t)(cursor - haystack.begin()); + std::size_t const codepoint_end = codepoint_begin + (std::size_t)consumed; + for (std::size_t index = 0; index < folded_count; ++index) { + haystack_folded.push_back(folded[index]); + source_begin.push_back(codepoint_begin); + source_end.push_back(codepoint_end); + } + cursor += consumed; + } + + // A match is any contiguous run of the folded haystack, including one that begins or ends part-way + // through an expansion - "s" does match inside the sharp S. Neither end has a byte of its own there, so + // both snap outward to the codepoint that produced them, and two runs that snap to one span are one + // match. This is the same rule `sz_utf8_uncased_search` follows. + std::set<uncased_match_span_t> spans; + std::vector<span<char const>> matches; + std::size_t const needle_length = needle_folded.size(); + if (needle_length == 0) return matches; + for (std::size_t start = 0; start + needle_length <= haystack_folded.size(); ++start) { + bool equal = true; + for (std::size_t index = 0; index < needle_length; ++index) + if (haystack_folded[start + index] != needle_folded[index]) { + equal = false; + break; + } + if (!equal) continue; + std::size_t const from = source_begin[start], to = source_end[start + needle_length - 1]; + if (!spans.emplace(uncased_match_span_t {from, to}).second) continue; + matches.emplace_back(haystack.data() + from, to - from); + } + return matches; +} + +/** + * @brief Ground-truth match keys for a whole vocabulary against a whole batch, by brute force per needle. + * + * Shares nothing with the automaton under test: cased matching is a plain byte slide, and uncased defers to + * `independent_uncased_matches_`. A constant both the serial and the accelerated walk got wrong is still + * caught here, which comparing two backends against each other cannot do. + */ +inline void collect_independent_matches_(substrings_case_sensitivity_t sensitivity, arrow_strings_view_t needles, + arrow_strings_view_t haystacks, substrings_match_set_t &out) { + out.clear(); + for (std::size_t haystack_index = 0; haystack_index < haystacks.size(); ++haystack_index) { + span<char const> const haystack = haystacks[haystack_index]; + for (std::size_t needle_index = 0; needle_index < needles.size(); ++needle_index) { + span<char const> const needle = needles[needle_index]; + // No byte-length pre-filter for uncased matching: a needle longer than the haystack can still + // match, because folding expands haystack codepoints - the 3-byte U+1FC7 folds to 6 bytes, so + // 4 of them legitimately carry a 24-byte needle. The cased loop's own bound handles oversize. + if (needle.size() == 0) continue; + + if (sensitivity == substrings_uncased_k) { + for (auto const &match : independent_uncased_matches_(haystack, needle)) + out.append( + {haystack_index, needle_index, (std::size_t)(match.data() - haystack.data()), match.size()}); + continue; + } + for (std::size_t offset = 0; offset + needle.size() <= haystack.size(); ++offset) + if (std::memcmp(haystack.data() + offset, needle.data(), needle.size()) == 0) + out.append({haystack_index, needle_index, offset, needle.size()}); + } + } + out.finalize(); +} + +/** @brief Selects one closed-set adversarial needle vocabulary. */ +enum class substrings_needle_generator_t { + /** "a", "aa", "aaa", â€Ļ - every needle a suffix of the next, quadratic outputs. */ + self_overlapping_k, + /** One needle's suffix is another's prefix - the textbook failure-link case. */ + mutual_overlap_k, + /** One long common prefix, diverging only at the final byte. */ + shared_prefix_fan_k, + /** Only bytes 0x01 and 0xFF - worst case for the double-array packer. */ + sparse_wide_alphabet_k, + /** Wide fold images sampled from the shipped tables - multi-rune expansions under folding. */ + fold_expanding_k, + /** A fixed core plus deduplicated random needles - the control. */ + random_short_k, + /** Needles of 1-3 runes sampled from the fold-preimage tables' image side. */ + preimage_sampled_k, + /** One rune per UTF-8 width class 1-4, so a single needle spans every decode path. */ + mixed_width_k, + /** One past the last generator, so the sweep can never drift from the list. */ + count_k, +}; + +/** @brief Selects one byte-agnostic haystack skeleton; the planted bytes come from the transform. */ +enum class substrings_placement_t { + /** Noise, one transformed needle, noise. */ + planted_in_noise_k, + /** Up to 4 consecutive needles' variants back to back - forces overlapping matches. */ + concatenated_k, + /** The variant twice back to back, flush with the first byte and flush with the last. */ + boundary_flush_k, + /** One ~16 KB haystack with variants planted every ~1 KB - pairs with the sliced-specs cells. */ + straddling_k, + /** The variant interrupted by one byte no fold can bridge - the planting must never match. */ + torn_codepoint_k, + /** One past the last placement, so the sweep can never drift from the list. */ + count_k, +}; + +/** @brief Selects how a planted needle is re-spelled, decoupled from where it lands. */ +enum class substrings_transform_t { + /** The needle verbatim - preserved under both modes. */ + identity_k, + /** Table-driven inverse folding - preserved uncased, defeated cased when any byte changed. */ + fold_preimage_k, + /** `^0x20` on every ASCII letter - preserved uncased, defeated cased when any letter existed. */ + case_flip_ascii_k, + /** `^0x01` on the last ASCII byte - defeated under BOTH modes, the true negative control. */ + byte_perturb_k, + /** The final codepoint dropped - defeated under both modes. */ + truncated_tail_k, + /** One past the last transform, so the sweep can never drift from the list. */ + count_k, +}; + +/** @brief What a planting demands of the engine: the exact match record present, or absent. */ +enum class substrings_planting_effect_t { + preserved_k, + defeated_k, +}; + +/** @brief One planted variant's ground truth: the exact match record and what the engine owes it. */ +struct substrings_planting_t { + substrings_match_t match; + substrings_planting_effect_t effect; +}; + +/** @brief Whether a generator's needles are well-formed UTF-8, and so can be case-folded at all. Only the + * double-array pressure vocabulary reaches for `0xFF`, which no UTF-8 sequence ever contains. */ +constexpr bool substrings_needles_are_utf8_(substrings_needle_generator_t kind) noexcept { + return kind != substrings_needle_generator_t::sparse_wide_alphabet_k; +} + +/** @brief Whether a generator's automaton grows with the square of its needle count, so the driver can pick + * `scale_iterations_quadratic` over `scale_iterations` and keep the suite balanced. */ +constexpr bool substrings_needles_grow_quadratically_(substrings_needle_generator_t kind) noexcept { + return kind == substrings_needle_generator_t::self_overlapping_k; +} + +constexpr char const *substrings_needle_generator_name(substrings_needle_generator_t kind) noexcept { + switch (kind) { + case substrings_needle_generator_t::self_overlapping_k: return "self_overlapping"; + case substrings_needle_generator_t::mutual_overlap_k: return "mutual_overlap"; + case substrings_needle_generator_t::shared_prefix_fan_k: return "shared_prefix_fan"; + case substrings_needle_generator_t::sparse_wide_alphabet_k: return "sparse_wide_alphabet"; + case substrings_needle_generator_t::fold_expanding_k: return "fold_expanding"; + case substrings_needle_generator_t::random_short_k: return "random_short"; + case substrings_needle_generator_t::preimage_sampled_k: return "preimage_sampled"; + case substrings_needle_generator_t::mixed_width_k: return "mixed_width"; + case substrings_needle_generator_t::count_k: break; // ? Never a real generator, only the sweep's bound + } + return "unknown"; +} + +constexpr char const *substrings_placement_name(substrings_placement_t placement) noexcept { + switch (placement) { + case substrings_placement_t::planted_in_noise_k: return "planted_in_noise"; + case substrings_placement_t::concatenated_k: return "concatenated"; + case substrings_placement_t::boundary_flush_k: return "boundary_flush"; + case substrings_placement_t::straddling_k: return "straddling"; + case substrings_placement_t::torn_codepoint_k: return "torn_codepoint"; + case substrings_placement_t::count_k: break; // ? Never a real placement, only the sweep's bound + } + return "unknown"; +} + +constexpr char const *substrings_transform_name(substrings_transform_t transform) noexcept { + switch (transform) { + case substrings_transform_t::identity_k: return "identity"; + case substrings_transform_t::fold_preimage_k: return "fold_preimage"; + case substrings_transform_t::case_flip_ascii_k: return "case_flip_ascii"; + case substrings_transform_t::byte_perturb_k: return "byte_perturb"; + case substrings_transform_t::truncated_tail_k: return "truncated_tail"; + case substrings_transform_t::count_k: break; // ? Never a real transform, only the sweep's bound + } + return "unknown"; +} + +/** @brief Sorts and deduplicates a locally built vocabulary pool, then assigns it into the tape. */ +inline void assign_deduplicated_(std::vector<std::string> &pool, arrow_strings_tape_t &needles) { + std::sort(pool.begin(), pool.end()); + pool.erase(std::unique(pool.begin(), pool.end()), pool.end()); + verify(needles.try_assign(pool.data(), pool.data() + pool.size()) == status_t::success_k); +} + +/** @brief Appends @p rune to @p out as 1-4 UTF-8 bytes; sampled fold runes must always re-encode. */ +inline void append_rune_utf8_(sz_rune_t rune, std::string &out) { + sz_u8_t encoded[4]; + sz_rune_length_t const encoded_length = sz_rune_encode(rune, encoded); + verify(encoded_length != sz_rune_invalid_k && "Sampled fold rune must re-encode"); + out.append((char const *)encoded, (std::size_t)encoded_length); +} + +/** + * @brief Which source codepoints fold onto each image, built once by folding every codepoint forward. + * + * The engine has no use for this direction - it folds needles and haystacks the same way and compares - so + * the index is the test's own, derived from `sz_unicode_fold_codepoint_` rather than from a shipped table. + * Deriving it here is also what keeps the fixtures honest: they cannot drift from the fold under test. + */ +struct fold_preimage_index_t { + /** @brief Single-rune images, ascending, so a range of them can be binary-searched. */ + std::vector<sz_rune_t> narrow_images; + /** @brief Multi-rune images, each zero-padded to three runes. */ + std::vector<std::array<sz_rune_t, 3>> wide_images; + /** @brief Sources per image, keyed by the image's runes. */ + std::map<std::vector<sz_rune_t>, std::vector<sz_rune_t>> sources_of_image; +}; + +/** @brief The one index, folded on first use and shared by every generator that needs a preimage. */ +inline fold_preimage_index_t const &fold_preimage_index_() { + static fold_preimage_index_t const index = [] { + fold_preimage_index_t built; + for (sz_rune_t rune = 0; rune <= 0x10FFFF; ++rune) { + if (rune >= 0xD800 && rune <= 0xDFFF) continue; // ? Surrogates are not codepoints on their own + sz_rune_t images[3]; + std::size_t const runes = sz_unicode_fold_codepoint_(rune, images); + if (runes == 1 && images[0] == rune) continue; // ? Folds to itself, so it is nobody's preimage + built.sources_of_image[std::vector<sz_rune_t>(images, images + runes)].push_back(rune); + } + for (auto const &[image, sources] : built.sources_of_image) { + if (image.size() == 1) { + built.narrow_images.push_back(image[0]); + continue; + } + std::array<sz_rune_t, 3> padded {}; + for (std::size_t index = 0; index < image.size(); ++index) padded[index] = image[index]; + built.wide_images.push_back(padded); + } + std::sort(built.narrow_images.begin(), built.narrow_images.end()); + return built; + }(); + return index; +} + +/** @brief Source codepoints whose full fold is exactly @p image, or an empty span when it folds to itself. */ +inline span<sz_rune_t const> fold_preimage_of_runes_(span<sz_rune_t const> image) { + auto const &sources = fold_preimage_index_().sources_of_image; + auto const found = sources.find(std::vector<sz_rune_t>(image.begin(), image.end())); + if (found == sources.end()) return {}; + return {found->second.data(), found->second.size()}; +} + +/** @brief Source codepoints whose full fold is the single rune @p image. */ +inline span<sz_rune_t const> fold_preimage_of_rune_(sz_rune_t image) { return fold_preimage_of_runes_({&image, 1}); } + +/** @brief One fold-space rune drawn uniformly from the narrow preimage table's images within + * `[first_rune, last_rune)`; the image array is sorted, so the range is a binary-searched slab. */ +inline sz_rune_t sample_fold_image_rune_(sz_rune_t first_rune, sz_rune_t last_rune) { + std::vector<sz_rune_t> const &images = fold_preimage_index_().narrow_images; + auto const slab_begin = std::lower_bound(images.begin(), images.end(), first_rune); + auto const slab_end = std::lower_bound(images.begin(), images.end(), last_rune); + verify(slab_begin != slab_end && "The requested rune range holds no fold images"); + std::uniform_int_distribution<std::size_t> slab_position(0, (std::size_t)(slab_end - slab_begin) - 1); + std::size_t const chosen_position = slab_position(global_random_generator()); + return slab_begin[chosen_position]; +} + +/** + * @brief Appends @p needle re-encoded through fold space with every image swapped for one genuine + * preimage - wide multi-rune images first - so only case folding can reconcile the result. + * Returns the number of swapped images; runes without preimages pass through verbatim. + */ +inline std::size_t append_fold_preimage_variant_(span<char const> needle, std::string &out) { + std::vector<sz_rune_t> folded; + for (char const *cursor = needle.begin(), *end = needle.end(); cursor != end;) { + sz_rune_t rune; + sz_rune_length_t const consumed = sz_rune_decode(cursor, end, &rune); + verify(consumed != sz_rune_invalid_k && "Fold-preimage variants need well-formed UTF-8 needles"); + sz_rune_t folded_runes[3]; + std::size_t const folded_count = sz_unicode_fold_codepoint_(rune, folded_runes); + for (std::size_t index = 0; index < folded_count; ++index) folded.push_back(folded_runes[index]); + cursor += consumed; + } + + auto &generator = global_random_generator(); + std::size_t swapped_images = 0; + for (std::size_t position = 0; position < folded.size();) { + // Wide images first, longest window first, so multi-rune expansions like the sharp S appear. + std::size_t const remaining_runes = folded.size() - position; + span<sz_rune_t const> preimages; + std::size_t consumed_runes = 1; + if (remaining_runes >= 3) // + preimages = fold_preimage_of_runes_({folded.data() + position, 3}), consumed_runes = 3; + if (preimages.size() == 0 && remaining_runes >= 2) + preimages = fold_preimage_of_runes_({folded.data() + position, 2}), consumed_runes = 2; + if (preimages.size() == 0) // + preimages = fold_preimage_of_rune_(folded[position]), consumed_runes = 1; + if (preimages.size() == 0) { + append_rune_utf8_(folded[position], out); + position += 1; + continue; + } + std::uniform_int_distribution<std::size_t> preimage_position(0, preimages.size() - 1); + std::size_t const chosen_position = preimage_position(generator); + append_rune_utf8_(preimages[chosen_position], out); + ++swapped_images; + position += consumed_runes; + } + return swapped_images; +} + +/** @brief Perturbs the last ASCII byte with `^0x01`, appending the variant; a needle without a single + * ASCII byte is left un-appended. @return whether the perturbation applied. */ +inline bool try_perturb_last_ascii_byte_(span<char const> needle, std::string &out) { + std::size_t ascii_position = needle.size(); + for (std::size_t index = needle.size(); index > 0; --index) + if ((unsigned char)needle[index - 1] < 0x80) { + ascii_position = index - 1; + break; + } + if (ascii_position == needle.size()) return false; + std::size_t const variant_begin = out.size(); + out.append(needle.data(), needle.size()); + out[variant_begin + ascii_position] = (char)(out[variant_begin + ascii_position] ^ 0x01); + return true; +} + +/** @brief Drops the needle's final codepoint by walking back over continuation bytes - on a non-UTF-8 + * vocabulary the walk stops immediately, dropping one byte; a single-codepoint needle refuses. */ +inline bool try_truncate_needle_tail_(span<char const> needle, std::string &out) { + if (needle.size() < 2) return false; + std::size_t truncated_size = needle.size() - 1; + while (truncated_size > 0 && ((unsigned char)needle[truncated_size] & 0xC0) == 0x80) --truncated_size; + if (truncated_size == 0) return false; + out.append(needle.data(), truncated_size); + return true; +} + +/** + * @brief Appends one transform's variant of @p needle to @p out, answering what the planting demands of + * the engine under @p sensitivity. Transforms that cannot apply fall through a terminating ladder: + * a needle with no ASCII byte sends the perturbation to tail truncation, a single-codepoint needle + * sends truncation to the perturbation, and the identity ends every path. + */ +inline substrings_planting_effect_t append_transformed_needle_( // + substrings_transform_t transform, substrings_case_sensitivity_t sensitivity, + substrings_needle_generator_t needle_kind, span<char const> needle, std::string &out) { + + if (transform == substrings_transform_t::fold_preimage_k && !substrings_needles_are_utf8_(needle_kind)) + transform = substrings_transform_t::case_flip_ascii_k; // ? Folding needs decodable needles. + + std::size_t const variant_begin = out.size(); + switch (transform) { + case substrings_transform_t::identity_k: break; + + case substrings_transform_t::fold_preimage_k: { + [[maybe_unused]] std::size_t const swapped_images = append_fold_preimage_variant_(needle, out); + // A swap may reproduce the original spelling - the sharp S is its own preimage - so the cased + // verdict compares bytes, not swap counts. + bool const changed = out.size() - variant_begin != needle.size() || + std::memcmp(out.data() + variant_begin, needle.data(), needle.size()) != 0; + if (sensitivity == substrings_uncased_k || !changed) return substrings_planting_effect_t::preserved_k; + return substrings_planting_effect_t::defeated_k; + } + + case substrings_transform_t::case_flip_ascii_k: { + bool any_letter_flipped = false; + for (std::size_t index = 0; index < needle.size(); ++index) { + char const letter = needle[index]; + bool const is_ascii_letter = (letter >= 'a' && letter <= 'z') || (letter >= 'A' && letter <= 'Z'); + any_letter_flipped |= is_ascii_letter; + out.push_back(is_ascii_letter ? (char)(letter ^ 0x20) : letter); + } + if (sensitivity == substrings_uncased_k || !any_letter_flipped) + return substrings_planting_effect_t::preserved_k; + return substrings_planting_effect_t::defeated_k; + } + + case substrings_transform_t::byte_perturb_k: { + if (try_perturb_last_ascii_byte_(needle, out)) return substrings_planting_effect_t::defeated_k; + if (try_truncate_needle_tail_(needle, out)) return substrings_planting_effect_t::defeated_k; + break; + } + + case substrings_transform_t::truncated_tail_k: { + if (try_truncate_needle_tail_(needle, out)) return substrings_planting_effect_t::defeated_k; + if (try_perturb_last_ascii_byte_(needle, out)) return substrings_planting_effect_t::defeated_k; + break; + } + + case substrings_transform_t::count_k: break; // ? Never a real transform, only the sweep's bound + } + out.append(needle.data(), needle.size()); + return substrings_planting_effect_t::preserved_k; +} + +/** + * @brief Fills @p needles with one closed-set adversarial vocabulary, @p count deep where the kind scales. + * + * Every case either appends straight into the tape or, for the one kind needing a whole-collection sort and + * deduplication, fills a local pool first. + */ +inline void generate_substrings_needles_(substrings_needle_generator_t kind, std::size_t count, + arrow_strings_tape_t &needles) { + needles.reset(); + std::string scratch; + switch (kind) { + + case substrings_needle_generator_t::self_overlapping_k: + // Every needle is a suffix of the next, so the deepest state carries `count` merged outputs and the + // output pool grows with the square of the depth - the worst case for anything sizing that pool. + for (std::size_t depth = 1; depth <= count; ++depth) { + scratch.push_back('a'); + verify(needles.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + } + return; + + case substrings_needle_generator_t::mutual_overlap_k: + // One needle's suffix is another's prefix, so failure links traverse rather than collapse to the root. + for (std::size_t index = 0; index < count; ++index) { + char const first = (char)('a' + index % 6), second = (char)('a' + (index + 1) % 6); + scratch.assign({first, second, first}); + verify(needles.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + scratch.assign({second, first}); + verify(needles.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + } + return; + + case substrings_needle_generator_t::shared_prefix_fan_k: + // A single deep trunk with `count` leaves, so almost every state has out-degree one and the + // frequency ordering has nothing to separate. + for (std::size_t index = 0; index < count; ++index) { + scratch.assign(12, 'q'); + scratch.push_back((char)('a' + index % 26)); + scratch.push_back((char)('a' + (index / 26) % 26)); + verify(needles.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + } + return; + + case substrings_needle_generator_t::sparse_wide_alphabet_k: + // Two bytes at opposite ends of the alphabet, so every double-array base must reserve a 256-wide + // window to hold two edges - maximum collision pressure for the packer. + for (std::size_t pattern = 0; pattern < count; ++pattern) { + scratch.clear(); + for (std::size_t depth = 0; depth < 8; ++depth) scratch.push_back((pattern >> depth) & 1 ? '\xFF' : '\x01'); + verify(needles.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + } + return; + + case substrings_needle_generator_t::fold_expanding_k: { + // Needles whose fold preimages span more bytes than the needle itself - "k" also matches the 3-byte + // Kelvin sign, so `max_source_match_bytes` cannot be read off needle lengths. The Kelvin anchor stays + // pinned; every following needle is a wide image repeated 1-4 times, so 2:1 and 3:1 byte expansions + // from every script appear organically rather than from a hand-picked seed list. + std::vector<std::string> pool {"k"}; + auto &generator = global_random_generator(); + std::vector<std::array<sz_rune_t, 3>> const &wide_images = fold_preimage_index_().wide_images; + std::uniform_int_distribution<std::size_t> row_position(0, wide_images.size() - 1); + for (std::size_t index = 0; index < count; ++index) { + std::array<sz_rune_t, 3> const &image = wide_images[row_position(generator)]; + scratch.clear(); + for (std::size_t repeat = 0; repeat <= index % 4; ++repeat) + for (std::size_t rune_index = 0; rune_index < 3 && image[rune_index] != 0; ++rune_index) + append_rune_utf8_(image[rune_index], scratch); + pool.push_back(scratch); + } + assign_deduplicated_(pool, needles); + return; + } + + case substrings_needle_generator_t::random_short_k: { + // A fixed textbook core plus random needles, deduplicated as one pool. + std::vector<std::string> pool {"he", "she", "his", "hers"}; + for (std::string &value : random_short_strings_(count, 3, 6)) pool.push_back(std::move(value)); + assign_deduplicated_(pool, needles); + return; + } + + case substrings_needle_generator_t::preimage_sampled_k: { + // 1-3 fold-space runes per needle, sampled from the narrow table's image side, so Greek, Cyrillic, + // Cherokee, and astral fold pairs appear by construction rather than from hand-picked literals. + std::vector<std::string> pool; + auto &generator = global_random_generator(); + std::uniform_int_distribution<std::size_t> rune_count_distribution(1, 3); + for (std::size_t index = 0; index < count; ++index) { + scratch.clear(); + std::size_t const rune_count = rune_count_distribution(generator); + for (std::size_t rune_index = 0; rune_index < rune_count; ++rune_index) + append_rune_utf8_(sample_fold_image_rune_(0, 0x110000), scratch); + pool.push_back(scratch); + } + assign_deduplicated_(pool, needles); + return; + } + + case substrings_needle_generator_t::mixed_width_k: { + // One rune per UTF-8 width class - ASCII, 2-byte, 3-byte, 4-byte - so every needle drags the walk + // through every decode width, and match spans never equal codepoint counts. + std::vector<std::string> pool; + auto &generator = global_random_generator(); + std::uniform_int_distribution<int> ascii_letter('a', 'z'); + for (std::size_t index = 0; index < count; ++index) { + scratch.clear(); + scratch.push_back((char)ascii_letter(generator)); + append_rune_utf8_(sample_fold_image_rune_(0x80, 0x800), scratch); + append_rune_utf8_(sample_fold_image_rune_(0x800, 0x10000), scratch); + append_rune_utf8_(sample_fold_image_rune_(0x10000, 0x110000), scratch); + pool.push_back(scratch); + } + assign_deduplicated_(pool, needles); + return; + } + + case substrings_needle_generator_t::count_k: break; // ? Never a real generator, only the sweep's bound + } +} + +/** + * @brief Fills @p haystacks with @p haystack_count skeletons of @p placement, planting each needle's + * @p transform variant and recording every planting's ground truth into @p plantings. + */ +inline void generate_substrings_placements_(substrings_placement_t placement, substrings_transform_t transform, + substrings_case_sensitivity_t sensitivity, + substrings_needle_generator_t needle_kind, arrow_strings_view_t needles, + std::size_t haystack_count, arrow_strings_tape_t &haystacks, + std::vector<substrings_planting_t> &plantings) { + haystacks.reset(); + plantings.clear(); + if (needles.size() == 0) return; + + auto &generator = global_random_generator(); + std::uniform_int_distribution<int> noise_length(4, 24); + std::uniform_int_distribution<int> noise_byte('m', 'z'); // ? Disjoint from every needle alphabet above. + // Cased cells sprinkle malformed UTF-8 into the noise - lone continuation bytes - since the byte-exact + // oracle is total over arbitrary bytes, while the uncased oracle demands well-formed haystacks. + std::uniform_int_distribution<int> malformed_byte(0x80, 0xBF); + std::uniform_int_distribution<int> malformed_gate(0, 7); + std::string scratch; + + auto append_noise = [&] { + for (int index = 0, length = noise_length(generator); index < length; ++index) + if (sensitivity == substrings_cased_k && malformed_gate(generator) == 0) + scratch.push_back((char)malformed_byte(generator)); + else scratch.push_back((char)noise_byte(generator)); + }; + auto plant = [&](std::size_t haystack_index, std::size_t needle_index) { + std::size_t const offset = scratch.size(); + substrings_planting_effect_t const effect = append_transformed_needle_(transform, sensitivity, needle_kind, + needles[needle_index], scratch); + plantings.push_back({{haystack_index, needle_index, offset, scratch.size() - offset}, effect}); + }; + auto tear_last_planting = [&] { + // One byte no fold can bridge - '0' appears in no vocabulary and in no letter's fold - lands inside + // the variant, so the planting must never match. The uncased oracle demands well-formed haystacks, + // so there the tear retreats to a codepoint boundary; single-codepoint variants stay whole. + substrings_planting_t &planting = plantings.back(); + if (planting.match.byte_length < 2) return; + std::size_t tear_offset = planting.match.byte_offset + planting.match.byte_length / 2; + if (sensitivity == substrings_uncased_k) + while (tear_offset > planting.match.byte_offset && ((unsigned char)scratch[tear_offset] & 0xC0) == 0x80) + --tear_offset; + if (tear_offset == planting.match.byte_offset) return; + scratch.insert(scratch.begin() + (std::ptrdiff_t)tear_offset, '0'); + planting.match.byte_length += 1; + planting.effect = substrings_planting_effect_t::defeated_k; + }; + auto append_haystack = [&] { + verify(haystacks.try_append({scratch.data(), scratch.size()}) == status_t::success_k); + scratch.clear(); + }; + + switch (placement) { + + case substrings_placement_t::planted_in_noise_k: + for (std::size_t haystack_index = 0; haystack_index < haystack_count; ++haystack_index) { + append_noise(); + plant(haystack_index, haystack_index % needles.size()); + append_noise(); + append_haystack(); + } + return; + + case substrings_placement_t::concatenated_k: + // No separator, so matches of different needles' variants overlap and nest at the seams. + for (std::size_t haystack_index = 0; haystack_index < haystack_count; ++haystack_index) { + for (std::size_t offset = 0; offset < 4 && offset < needles.size(); ++offset) + plant(haystack_index, (haystack_index + offset) % needles.size()); + append_haystack(); + } + return; + + case substrings_placement_t::boundary_flush_k: + // Flush with the first byte and flush with the last, plus a self-repeat in between, so a match + // lands on every edge a chunked or windowed walk could mishandle. + for (std::size_t haystack_index = 0; haystack_index < haystack_count; ++haystack_index) { + plant(haystack_index, haystack_index % needles.size()); + plant(haystack_index, haystack_index % needles.size()); + append_haystack(); + } + return; + + case substrings_placement_t::straddling_k: { + // One large haystack for the all-cores-on-one-haystack path: variants planted at a fixed stride, + // so under the sliced specs the matches keep crossing slice seams. + std::size_t const haystack_bytes = 16 * 1024; + std::size_t const stride_bytes = 1024; + std::size_t planted_count = 0; + while (scratch.size() < haystack_bytes) { + std::size_t const next_plant_offset = (std::min)(scratch.size() + stride_bytes, haystack_bytes); + while (scratch.size() < next_plant_offset) append_noise(); + plant(0, planted_count % needles.size()); + ++planted_count; + } + append_haystack(); + return; + } + + case substrings_placement_t::torn_codepoint_k: + for (std::size_t haystack_index = 0; haystack_index < haystack_count; ++haystack_index) { + append_noise(); + plant(haystack_index, haystack_index % needles.size()); + tear_last_planting(); + append_noise(); + append_haystack(); + } + return; + + case substrings_placement_t::count_k: break; // ? Never a real placement, only the sweep's bound + } +} + +#pragma endregion // Helpers + +#pragma region Unit + +/** + * @brief Known-answer vectors for `substrings`, pinned by hand rather than against a second backend. + * + * Covers the textbook Aho-Corasick overlap example - "he" and "she" both completing at the same end offset + * over "ushers" - attribution of matches to the right haystack in a batch, and a self-overlapping + * vocabulary of {"a", "ab", "abc"} over "abcabc". Every match is reported, nested ones included; this is + * not leftmost-first matching. + */ +void test_substrings_unit() { + std::printf(" - testing substrings known-answer vectors...\n"); + + // "she" and "he" both complete the instant the scan reaches the shared 'e', so both are reported at the + // same end offset; "his" never occurs in "ushers" and contributes nothing. + { + std::vector<std::string> const needle_strings {"he", "she", "his", "hers"}; + std::vector<std::string> const haystack_strings {"ushers"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + substrings_match_set_t const expected { + {0, 0, 2, 2}, // "he" at byte [2, 4) + {0, 1, 1, 3}, // "she" at byte [1, 4) - same end offset as "he" + {0, 3, 2, 4}, // "hers" at byte [2, 6) + }; + verify(matches == expected && "Classic he/she/his/hers overlap example mismatched"); + } + + // Matches attribute to the right haystack in a batch, including a haystack with none at all. + { + std::vector<std::string> const needle_strings {"cat", "dog"}; + std::vector<std::string> const haystack_strings {"cats and dogs", "no pets here", "dogcat"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + substrings_match_set_t const expected { + {0, 0, 0, 3}, // "cat" in haystack 0 at byte [0, 3) + {0, 1, 9, 3}, // "dog" in haystack 0 at byte [9, 12) + {2, 1, 0, 3}, // "dog" in haystack 2 at byte [0, 3) + {2, 0, 3, 3}, // "cat" in haystack 2 at byte [3, 6) + }; + verify(matches == expected && "Batch haystack attribution mismatched"); + } + + // A vocabulary where every needle is a prefix of the next only makes sense once every overlapping and + // nested match is reported - never leftmost-first, never longest-only. + { + std::vector<std::string> const needle_strings {"a", "ab", "abc"}; + std::vector<std::string> const haystack_strings {"abcabc"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + substrings_match_set_t const expected { + {0, 0, 0, 1}, {0, 1, 0, 2}, {0, 2, 0, 3}, // "a", "ab", "abc" at the first occurrence + {0, 0, 3, 1}, {0, 1, 3, 2}, {0, 2, 3, 3}, // and again at the second + }; + verify(matches == expected && "Nested-prefix overlap example mismatched"); + } + + // A match may begin part-way through an expansion, and then reports the whole codepoint it began inside. + // Folding "\xC3\x9Fsoft" gives "sssoft", so "ss" matches both the sharp S alone and the run crossing from + // its second "s" into the literal one - the second snapping outward to byte 0 and running to byte 3. + { + std::vector<std::string> const needle_strings {"ss", "\xC3\x9F"}; + std::vector<std::string> const haystack_strings {"\xC3\x9Fsoft"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_uncased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + substrings_match_set_t const expected { + {0, 0, 0, 2}, // "ss" matches the sharp-S codepoint at byte [0, 2) + {0, 1, 0, 2}, // "\xC3\x9F" matches itself at byte [0, 2) + {0, 0, 0, 3}, // "ss" again, from the sharp S's second "s" into the literal one + {0, 1, 0, 3}, // and "\xC3\x9F" folds to the same "ss", so it matches that run too + }; + verify(matches == expected && "Mid-expansion matches must snap outward to the codepoint they begin in"); + } + + // Needles whose fold self-overlaps across a mixed-width fold image: the folded stream lines up in more + // ways than the source bytes do, so each case pins both which windows match and which source span each + // one snaps to. `U+212A` is the Kelvin sign (folds to `k`), `U+017F` the long s (folds to `s`). + struct uncased_case_t { + std::vector<std::string> needles; + std::string haystack; + substrings_match_set_t expected; + char const *label; + }; + uncased_case_t const uncased_cases[] = { + // The ASCII runs join their escapes directly, which is safe only because `k` and `s` are not hex + // digits. A case whose codepoint is followed by one of `a`-`f` must break the literal, or `\x9F` + // before an `a` would parse as the single overlong escape `\x9Fa`. + {{"kk"}, "\xE2\x84\xAAkk", {{0, 0, 0, 4}, {0, 0, 3, 2}}, "kk over Kelvin-k-k"}, + {{"ss"}, "\xC5\xBFss", {{0, 0, 0, 3}, {0, 0, 2, 2}}, "ss over long-s-s"}, + // Both windows of four folded s-runes. Over "ss\xC3\x9Fs" each one starts on a codepoint; over + // "\xC3\x9Fsss" the second starts on the sharp S's *second* "s", so it snaps back to byte 0 and runs + // to the end. + {{"ssss"}, "ss\xC3\x9Fs", {{0, 0, 0, 4}, {0, 0, 1, 4}}, "ssss over s-s-sharp-s"}, + {{"ssss"}, "\xC3\x9Fsss", {{0, 0, 0, 4}, {0, 0, 0, 5}}, "ssss over sharp-s-s-s"}, + // Nested needles: a wrong failure chain on one needle corrupts the other's reported matches too. + {{"k", "kk"}, "\xE2\x84\xAAk", {{0, 0, 0, 3}, {0, 1, 0, 4}, {0, 0, 3, 1}}, "nested k/kk over Kelvin-k"}, + }; + for (uncased_case_t const &probe : uncased_cases) { + arrow_strings_tape_t needles, haystacks; + std::vector<std::string> const haystack_strings {probe.haystack}; + verify(needles.try_assign(probe.needles.data(), probe.needles.data() + probe.needles.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + 1) == status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_uncased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + if (matches != probe.expected) std::fprintf(stderr, "Reconvergence hazard: %s\n", probe.label); + verify(matches == probe.expected && "Reconvergence hazard produced the wrong match set"); + } +} + +#pragma endregion // Unit + +#pragma region Uncased Conformance + +/** @brief Builds a single-needle uncased dictionary and asserts every `must_match` haystack matches at least + * once, and every `must_not_match` haystack matches zero times. */ +static void check_uncased_needle_matches_(char const *needle, std::vector<std::string> const &must_match, + std::vector<std::string> const &must_not_match) { + span<char const> const needle_span(needle, std::strlen(needle)); + std::vector<span<char const>> const needles {needle_span}; + + substrings_serial_t engine; + verify(engine.try_index(needles, substrings_uncased_k) == status_t::success_k); + + for (std::string const &haystack : must_match) { + std::vector<span<char const>> const haystacks {span<char const>(haystack.data(), haystack.size())}; + std::vector<std::size_t> counts(1, 0); + std::size_t matches_total = 0; + verify(engine.try_count(haystacks, substrings_overlapping_k, span<std::size_t>(counts.data(), counts.size()), + matches_total) == status_t::success_k); + verify(counts[0] >= 1 && "Needle must match this haystack under full case folding"); + } + for (std::string const &haystack : must_not_match) { + std::vector<span<char const>> const haystacks {span<char const>(haystack.data(), haystack.size())}; + std::vector<std::size_t> counts(1, 0); + std::size_t matches_total = 0; + verify(engine.try_count(haystacks, substrings_overlapping_k, span<std::size_t>(counts.data(), counts.size()), + matches_total) == status_t::success_k); + verify(counts[0] == 0 && "Needle must not match this haystack"); + } +} + +/** + * @brief Full Unicode case-folding conformance table for `substrings_uncased_k`, then a differential against + * `independent_uncased_matches_`. + * + * Every non-ASCII fixture is spelled as `\xHH` byte escapes, never as a raw literal or a `\u` universal + * character name - both have been silently re-encoded by tooling here before. A byte-level check against a + * numeric expected-bytes array guards the trickiest fixtures. + */ +void test_substrings_uncased_unit() { + std::printf(" - testing full Unicode case-folding conformance...\n"); + + // Corruption guard: two load-bearing fixtures re-spelled as integer byte arrays, so a tool that silently + // renormalizes the string literals below is caught here. + { + unsigned char const sharp_s_expected[] = {0xC3, 0x9F}; + verify(std::memcmp("\xC3\x9F", sharp_s_expected, 2) == 0 && "Sharp S literal corrupted"); + unsigned char const kelvin_sign_expected[] = {0xE2, 0x84, 0xAA}; + verify(std::memcmp("\xE2\x84\xAA", kelvin_sign_expected, 3) == 0 && "Kelvin sign literal corrupted"); + } + + // | Needle | Must match | Must NOT match | + check_uncased_needle_matches_("ss", {"ss", "SS", "sS", "Ss", "\xC3\x9F", "\xE1\xBA\x9E"}, {"s"}); + check_uncased_needle_matches_("\xC3\x9F", {"\xC3\x9F", "\xE1\xBA\x9E", "ss", "SS", "sS", "Ss"}, {"s"}); + check_uncased_needle_matches_("K", {"K", "k", "temp\xE2\x84\xAAvalue"}, {}); + check_uncased_needle_matches_("\xC3\x85", {"\xC3\x85", "\xC3\xA5", "temp\xE2\x84\xABvalue"}, {"A\xCC\x8A"}); + check_uncased_needle_matches_("\xC4\xB0", {"i\xCC\x87"}, {"i", "I"}); + check_uncased_needle_matches_("\xC3\xA9", {"\xC3\xA9", "\xC3\x89"}, {"e\xCC\x81"}); + + // A length-changing fold in the MIDDLE of a needle, not only at its end, so the byte-delta state keying + // that reconverges variable-length preimages is actually exercised mid-walk, not just at acceptance. + check_uncased_needle_matches_("wei\xC3\x9Frd", {"weissrd", "weiSSrd", "wei\xE1\xBA\x9Erd"}, {"weisrd", "weird"}); +} + +#pragma endregion // Uncased Conformance + +#pragma region Agreement + +/** + * @brief A ONE-needle dictionary must agree exactly with the shipped `sz_utf8_uncased_search` over a fuzz + * corpus - that agreement is the entire semantic claim of `substrings_uncased_k`. + * + * `substrings` reports every overlapping match, while `sz_utf8_uncased_search` reports only the first by + * start offset, so the comparison reduces `substrings`'s set to its own earliest-starting match. + */ +void test_substrings_uncased_equivalence() { + std::printf(" - testing one-needle agreement with sz_utf8_uncased_search...\n"); + + // Differential: for a spread of needles and randomized valid UTF-8 haystacks, the full match set found by + // `substrings` must equal the full match set found by the independent fold-and-scan oracle. + { + std::printf(" - differential against the independent fold-and-scan oracle...\n"); + std::vector<std::string> const needle_pool { + "ss", "\xC3\x9F", "K", "\xC3\x85", "\xC4\xB0", "\xC3\xA9", "wei\xC3\x9Frd", "the", + }; + span<string_view const> const empty_motifs; + auto &generator = global_random_generator(); + for (std::size_t iteration = 0; iteration < scale_iterations(60); ++iteration) { + std::string const &needle = needle_pool[iteration % needle_pool.size()]; + std::string haystack; + utf8_random_segmentation_corpus_(haystack, 96, utf8_corpus_flavor_t::valid_k, utf8_default_alphabet, + empty_motifs, generator); + haystack.append(needle); // ? Guarantees at least one hit most iterations, without excluding zero-hit ones. + + std::vector<span<char const>> const needles {span<char const>(needle.data(), needle.size())}; + substrings_serial_t engine; + verify(engine.try_index(needles, substrings_uncased_k) == status_t::success_k); + std::vector<std::string> const haystack_strings {haystack}; + arrow_strings_tape_t haystacks; + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + substrings_match_set_t engine_matches; + collect_overlapping_matches_into_(engine, haystacks.view(), engine_matches); + + span<char const> const haystack_view = haystacks[0]; + auto const oracle_matches = independent_uncased_matches_(haystack_view, needles[0]); + substrings_match_set_t expected_matches; + for (auto const &oracle_match : oracle_matches) + expected_matches.append( + {0, 0, (std::size_t)(oracle_match.data() - haystack_view.data()), oracle_match.size()}); + expected_matches.finalize(); + + if (engine_matches != expected_matches) { + std::fprintf(stderr, "Uncased differential mismatch for needle \"%s\": %zu vs %zu matches\n", + needle.c_str(), engine_matches.size(), expected_matches.size()); + for (substrings_match_t const &key : engine_matches) + std::fprintf(stderr, " engine offset=%zu length=%zu\n", key.byte_offset, key.byte_length); + for (substrings_match_t const &key : expected_matches) + std::fprintf(stderr, " oracle offset=%zu length=%zu\n", key.byte_offset, key.byte_length); + verify(false && "substrings disagrees with the independent fold-and-scan oracle"); + } + } + } + + std::vector<std::string> needle_pool { + "the", + "quick", + "STRASSE", + "stra\xC3\x9F" "e", // ? Split so "\x9Fe" cannot parse as one escape. + "ss", + "K", + "caf\xC3\xA9", + "\xC3\x85ngstrom", + }; + // Beyond the pinned anchors, needles sampled from the fold tables, so the agreement claim covers every + // script with fold pairs rather than the hand-picked list: preimage re-spellings of each anchor, then + // fold-space runes drawn straight from the narrow table's image side. + for (std::size_t anchor_index = 0, anchors = needle_pool.size(); anchor_index < anchors; ++anchor_index) { + std::string variant; + span<char const> const anchor {needle_pool[anchor_index].data(), needle_pool[anchor_index].size()}; + [[maybe_unused]] std::size_t const swapped_images = append_fold_preimage_variant_(anchor, variant); + needle_pool.push_back(std::move(variant)); + } + for (std::size_t sample_index = 0; sample_index < 8; ++sample_index) { + std::string sampled; + append_rune_utf8_(sample_fold_image_rune_(0, 0x110000), sampled); + append_rune_utf8_(sample_fold_image_rune_(0, 0x110000), sampled); + needle_pool.push_back(std::move(sampled)); + } + span<string_view const> const empty_motifs; + auto &generator = global_random_generator(); + + for (std::size_t iteration = 0; iteration < scale_iterations(200); ++iteration) { + std::string const &needle = needle_pool[iteration % needle_pool.size()]; + + std::string haystack; + utf8_random_segmentation_corpus_(haystack, 80, utf8_corpus_flavor_t::valid_k, utf8_default_alphabet, + empty_motifs, generator); + if ((iteration & 1) == 0) haystack.append(needle); // ? Half the iterations guarantee a hit. + + std::vector<span<char const>> const needles {span<char const>(needle.data(), needle.size())}; + substrings_serial_t engine; + verify(engine.try_index(needles, substrings_uncased_k) == status_t::success_k); + std::vector<std::string> const haystack_strings {haystack}; + arrow_strings_tape_t haystacks; + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + bool engine_found = false; + std::size_t engine_offset = 0, engine_length = 0; + for (substrings_match_t const &match : matches) + if (!engine_found || match.byte_offset < engine_offset) { + engine_found = true; + engine_offset = match.byte_offset; + engine_length = match.byte_length; + } + + sz_utf8_uncased_needle_metadata_t metadata = {}; + sz_size_t reference_length = 0; + // The serial variant rather than the dispatched one: `SZ_API_COMPTIME`, so it needs no stringzilla + // core library on this binary's link line, and it is the reference every backend is validated against. + sz_cptr_t const reference_result = sz_utf8_uncased_search_serial( + haystack.data(), haystack.size(), needle.data(), needle.size(), &metadata, &reference_length); + bool const reference_found = reference_result != SZ_NULL_CHAR; + std::size_t const reference_offset = reference_found ? (std::size_t)(reference_result - haystack.data()) : 0; + + bool const agrees = engine_found == reference_found && + (!engine_found || (engine_offset == reference_offset && engine_length == reference_length)); + if (!agrees) { + std::fprintf( // + stderr, // + "Agreement mismatch for needle \"%s\": substrings found=%d offset=%zu length=%zu | " // + "sz_utf8_uncased_search found=%d offset=%zu length=%zu\n", // + needle.c_str(), engine_found, engine_offset, engine_length, reference_found, reference_offset, + reference_length); + verify(false && "substrings and sz_utf8_uncased_search disagree"); + } + } +} + +#pragma endregion // Agreement + +#pragma region Adversarial + +/** + * @brief Everything one adversarial cell needs, constructed once by the sweep and refilled per cell. + * + * No member is ever rebuilt: the generators `reset` the tapes they fill and every collector `clear`s the + * buffers it owns, so one backing allocation survives the whole sweep. + */ +struct substrings_adversarial_scratch_t { + /** @brief Which axis triple produced the current cell, so a failing check can name its configuration + * instead of leaving it to be reconstructed from a step index. */ + substrings_needle_generator_t needle_kind = substrings_needle_generator_t::self_overlapping_k; + substrings_placement_t placement = substrings_placement_t::planted_in_noise_k; + substrings_transform_t transform = substrings_transform_t::identity_k; + + /** @brief Default on most cells; every third cell shrinks `l2_bytes` so straddling haystacks slice. */ + cpu_specs_t specs; + + arrow_strings_tape_t needles; + arrow_strings_tape_t haystacks; + std::vector<substrings_planting_t> plantings; + substrings_match_set_t engine_keys, oracle_keys, variant_keys; +#if SZ_USE_CUDA + substrings_match_set_t cuda_keys; +#endif +}; + +/** @brief Reports @p what went wrong and which axis triple produced the cell, mirroring how + * `edit_distance_log_mismatch` prefixes a similarity failure with the pair that caused it. */ +void log_substrings_cell_mismatch_(substrings_adversarial_scratch_t const &scratch, char const *what) { + std::fprintf(stderr, "%s on needles=%s placement=%s transform=%s\n", what, + substrings_needle_generator_name(scratch.needle_kind), substrings_placement_name(scratch.placement), + substrings_transform_name(scratch.transform)); +} + +/** + * @brief The indexing check: every match the serial engine reports must agree with brute force, needle, + * offset, length and haystack all at once. Runs on every cell. + */ +void check_substrings_against_oracle_(substrings_case_sensitivity_t sensitivity, + substrings_adversarial_scratch_t &scratch) { + arrow_strings_view_t const needles_view = scratch.needles.view(); + arrow_strings_view_t const haystacks_view = scratch.haystacks.view(); + + substrings_serial_t engine; + verify(engine.try_index(needles_view, sensitivity) == status_t::success_k); + collect_overlapping_matches_into_(engine, haystacks_view, scratch.engine_keys); + collect_independent_matches_(sensitivity, needles_view, haystacks_view, scratch.oracle_keys); + + if (scratch.engine_keys != scratch.oracle_keys) { + log_substrings_cell_mismatch_(scratch, "Oracle disagreement"); + verify(false && "substrings disagrees with the brute-force vocabulary oracle"); + } +} + +/** + * @brief Construction-time truth: every preserved planting must appear among the engine's keys as its + * exact record, every defeated one must not. A third witness beside the oracle - the generator + * knows where it planted what, so a misconception the engine and oracle share cannot survive it. + * Runs on every cell, right after the oracle check fills `engine_keys`. + */ +void check_substrings_declared_effects_(substrings_adversarial_scratch_t &scratch) { + for (substrings_planting_t const &planting : scratch.plantings) { + bool const present = std::binary_search(scratch.engine_keys.matches.begin(), scratch.engine_keys.matches.end(), + planting.match, substrings_match_less_); + bool const expected = planting.effect == substrings_planting_effect_t::preserved_k; + if (present == expected) continue; + log_substrings_cell_mismatch_(scratch, expected ? "Preserved planting missing" : "Defeated planting matched"); + std::fprintf(stderr, " haystack=%zu needle=%zu offset=%zu length=%zu\n", planting.match.haystack_index, + planting.match.needle_index, planting.match.byte_offset, planting.match.byte_length); + verify(false && "The engine's matches contradict a planting's declared effect"); + } +} + +/** @brief Serial and fork-union-parallel must report the identical set on the same cell; the parallel + * engine takes the real pool and the cell's specs, so sliced-L2 cells genuinely slice. */ +void check_substrings_backends_agree_(substrings_case_sensitivity_t sensitivity, + substrings_adversarial_scratch_t &scratch, forkunion_executor_t &pool) { + arrow_strings_view_t const needles_view = scratch.needles.view(); + arrow_strings_view_t const haystacks_view = scratch.haystacks.view(); + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles_view, sensitivity) == status_t::success_k); + collect_overlapping_matches_into_(serial_engine, haystacks_view, scratch.engine_keys); + + substrings_parallel_t parallel_engine; + verify(parallel_engine.try_index(needles_view, sensitivity) == status_t::success_k); + collect_overlapping_matches_into_(parallel_engine, haystacks_view, scratch.variant_keys, pool, scratch.specs); + if (scratch.engine_keys != scratch.variant_keys) { + log_substrings_cell_mismatch_(scratch, "Serial-vs-parallel divergence"); + verify(false && "Serial and parallel backends disagree"); + } +} + +/** + * @brief A `u16` automaton over the same cell either agrees exactly or declines with `overflow_risk_k`. + * + * Declining is a legitimate outcome - the narrower id space caps at 65534 states, and an adversarial + * vocabulary is built to reach ceilings - so the property is that it never silently truncates instead. + * Reports whether it got as far as comparing, so the sweep can refuse to pass on declines alone. + */ +bool check_substrings_narrow_width_(substrings_case_sensitivity_t sensitivity, + substrings_adversarial_scratch_t &scratch) { + arrow_strings_view_t const needles_view = scratch.needles.view(); + arrow_strings_view_t const haystacks_view = scratch.haystacks.view(); + + substrings_u16_dictionary_t narrow; + narrow.case_sensitivity(sensitivity); + for (std::size_t index = 0; index < needles_view.size(); ++index) { + status_t const status = narrow.try_insert(needles_view[index]); + if (status == status_t::overflow_risk_k) return false; + verify(status == status_t::success_k); + } + status_t const build_status = narrow.try_build(); + if (build_status == status_t::overflow_risk_k) return false; + verify(build_status == status_t::success_k); + + scratch.variant_keys.clear(); + for (std::size_t haystack_index = 0; haystack_index < haystacks_view.size(); ++haystack_index) + narrow.find(haystacks_view[haystack_index], + [&](std::size_t needle_index, std::size_t match_offset, std::size_t match_length) noexcept { + scratch.variant_keys.append({haystack_index, needle_index, match_offset, match_length}); + return true; + }); + scratch.variant_keys.finalize(); + + collect_independent_matches_(sensitivity, needles_view, haystacks_view, scratch.oracle_keys); + if (scratch.variant_keys != scratch.oracle_keys) { + log_substrings_cell_mismatch_(scratch, "Narrow-width divergence"); + verify(false && "u16 automaton disagrees with the oracle"); + } + return true; +} + +/** + * @brief The same dictionary with `hot_count` forced from fully cold to fully hot must report an identical + * set every time. + * + * A small dictionary is entirely hot by default and never enters the cold double array at all, so a + * tier-boundary bug is otherwise invisible. Driven through `aho_corasick_dictionary::hot_count` directly, + * the engine wrapper having no hook to override the split before `try_build`. + */ +void check_substrings_tier_invariant_(substrings_case_sensitivity_t sensitivity, + substrings_adversarial_scratch_t &scratch) { + arrow_strings_view_t const needles_view = scratch.needles.view(); + arrow_strings_view_t const haystacks_view = scratch.haystacks.view(); + + std::size_t raw_state_count = 0; + { + substrings_u32_dictionary_t probe; + probe.case_sensitivity(sensitivity); + for (std::size_t index = 0; index < needles_view.size(); ++index) + verify(probe.try_insert(needles_view[index]) == status_t::success_k); + // Build it: uncased reconvergence splits states during the build, so the published state count - + // the true "all states hot" target for the sweep below - is only known after `try_build`. + verify(probe.try_build() == status_t::success_k); + raw_state_count = probe.count_states(); + } + verify(raw_state_count > 0); + + std::size_t const hot_counts[] = {0, 1, raw_state_count / 2, raw_state_count}; + bool saw_cold_tier = false, saw_all_hot = false; + for (std::size_t variant_index = 0; variant_index < 4; ++variant_index) { + substrings_u32_dictionary_t dictionary; + dictionary.case_sensitivity(sensitivity); + for (std::size_t index = 0; index < needles_view.size(); ++index) + verify(dictionary.try_insert(needles_view[index]) == status_t::success_k); + dictionary.hot_count(hot_counts[variant_index]); + verify(dictionary.try_build() == status_t::success_k); + + auto const automaton = dictionary.view(); + saw_cold_tier |= automaton.hot_count < automaton.state_count; + saw_all_hot |= automaton.all_hot(); + + scratch.variant_keys.clear(); + for (std::size_t haystack_index = 0; haystack_index < haystacks_view.size(); ++haystack_index) + dictionary.find(haystacks_view[haystack_index], + [&](std::size_t needle_index, std::size_t match_offset, std::size_t match_length) noexcept { + scratch.variant_keys.append({haystack_index, needle_index, match_offset, match_length}); + return true; + }); + scratch.variant_keys.finalize(); + + if (variant_index == 0) { scratch.engine_keys = scratch.variant_keys; } + else if (scratch.variant_keys != scratch.engine_keys) { + log_substrings_cell_mismatch_(scratch, "Tier mismatch"); + std::fprintf(stderr, " at hot_count=%zu (hot=%u, states=%u): %zu vs %zu matches\n", + hot_counts[variant_index], automaton.hot_count, automaton.state_count, + scratch.variant_keys.size(), scratch.engine_keys.size()); + verify(false && "Match set changed across the hot/cold tier boundary"); + } + } + verify(saw_cold_tier && "Tier sweep never exercised the cold tier - grow the vocabulary"); + verify(saw_all_hot && "Tier sweep never exercised the fully-hot path"); +} + +#if SZ_USE_CUDA +/** + * @brief Reaches the scalar rune helpers from device code, so a helper that stops being reachable stops + * this file from compiling. + * + * Uncased matching walks a folded cursor and that cursor decodes a rune per step. The helpers it calls + * name no execution space of their own: `SZ_HELPER_AUTO` marks them `constexpr`, and + * `--expt-relaxed-constexpr` is what lets a kernel reach a host `constexpr` function at all. Downgrade one + * to `SZ_HELPER_INLINE` and it becomes host-only, yet `nvcc` still resolves the call from inside the + * cursor without a diagnostic - every multi-byte codepoint then decodes as a run of malformed single + * bytes, so an uncased search matches nothing outside ASCII on the device while every host backend agrees + * with the oracle and the whole suite passes. + * + * A direct call from a `__global__` function is what turns that silence into a build error, and a build + * error is the only form this can be caught in without a GPU: the device backend is compiled on every + * CUDA runner and executed on none of them, so `check_substrings_cuda_agrees_` below never gets to run. + */ +__global__ void reach_rune_helpers_on_device_(sz_cptr_t utf8, sz_size_t length, sz_rune_t *rune_out, + sz_rune_length_t *length_out, sz_u8_t *encoded_out) { + sz_rune_t rune = 0; + *length_out = sz_rune_decode(utf8, utf8 + length, &rune); + *rune_out = rune; + sz_rune_encode(rune, encoded_out); +} + +/** @brief The device backend must report the identical set for the same cell, through the very same call + * shape every host backend takes - only the memory the haystacks live in differs. */ +void check_substrings_cuda_agrees_(substrings_case_sensitivity_t sensitivity, + substrings_adversarial_scratch_t &scratch) { + arrow_strings_view_t const needles_view = scratch.needles.view(); + arrow_strings_view_t const haystacks_view = scratch.haystacks.view(); + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles_view, sensitivity) == status_t::success_k); + collect_overlapping_matches_into_(serial_engine, haystacks_view, scratch.engine_keys); + + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + substrings_cuda_t cuda_engine; + verify(cuda_engine.try_index(needles_view, sensitivity, executor, gpu_specs) == status_t::success_k); + + // One match type and one signature across backends: the device runs the very same collection body as + // every CPU engine. The fixture tape is `unified_alloc`-backed under CUDA, so no copy exists anywhere. + collect_overlapping_matches_into_(cuda_engine, haystacks_view, scratch.cuda_keys, executor, gpu_specs); + if (scratch.cuda_keys != scratch.engine_keys) { + log_substrings_cell_mismatch_(scratch, "CUDA-vs-serial divergence"); + verify(false && "CUDA backend disagrees with the serial reference"); + } + + // The leftmost covers are a sequential greedy over each haystack, which the device resolves after the + // walk, over the matches it emitted, in segments no match reaches across. Both policies are checked, + // since they differ only in which match wins a start and that choice propagates. + substrings_overlap_policy_t const covers[] = {substrings_leftmost_first_k, substrings_leftmost_longest_k}; + for (substrings_overlap_policy_t const policy : covers) { + collect_matches_under_(serial_engine, haystacks_view, policy, scratch.engine_keys); + collect_matches_under_(cuda_engine, haystacks_view, policy, scratch.cuda_keys, executor, gpu_specs); + if (scratch.cuda_keys != scratch.engine_keys) { + log_substrings_cell_mismatch_(scratch, policy == substrings_leftmost_first_k + ? "CUDA-vs-serial divergence under leftmost-first" + : "CUDA-vs-serial divergence under leftmost-longest"); + verify(false && "CUDA leftmost cover disagrees with the serial reference"); + } + } +} + +#endif // SZ_USE_CUDA + +/** + * @brief Pins the two sides of the device memory contract: scattered device-resident haystacks - several + * separate unified allocations rather than one packed tape - are searched correctly, and host-backed + * haystacks are refused with `device_memory_mismatch_k` rather than silently copied. + */ +void test_substrings_cuda_memory_safety() { + std::printf(" - testing unified, host, pinned and device memory against the contract...\n"); +#if SZ_USE_CUDA + + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + std::vector<std::string> const needle_strings {"he", "she", "his", "hers"}; + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_cuda_t cuda_engine; + verify(cuda_engine.try_index(needles.view(), substrings_cased_k, executor, gpu_specs) == status_t::success_k); + + // Three haystacks in three separate unified allocations: the descriptors point wherever the caller's + // memory happens to live, which is the whole point of not assuming a tape. + std::vector<std::string> const texts {"ushers", "hishers", "she"}; + unified_texts_t const scattered {texts}; + span<span<char const> const> const scattered_view = scattered.view(); + + arrow_strings_tape_t reference_haystacks; + verify(reference_haystacks.try_assign(texts.data(), texts.data() + texts.size()) == status_t::success_k); + substrings_match_set_t serial_keys, scattered_keys; + collect_overlapping_matches_into_(serial_engine, reference_haystacks.view(), serial_keys); + collect_overlapping_matches_into_(cuda_engine, scattered_view, scattered_keys, executor, gpu_specs); + verify(!serial_keys.empty() && "The fixture must produce matches"); + verify(scattered_keys == serial_keys && "Scattered unified allocations must match the packed reference"); + + // Host memory is refused, never copied: materialization is the caller's explicit choice, as in every + // other domain. The strings above live on the host, so their spans are exactly the illegal input. + std::vector<span<char const>> host_spans; + for (std::string const &text : texts) host_spans.push_back({text.data(), text.size()}); + unified_vector<std::size_t> counts(host_spans.size()); + std::size_t matches_total = 0; + auto const host_status = cuda_engine.try_count(host_spans, substrings_overlapping_k, + span<std::size_t>(counts.data(), counts.size()), matches_total, + executor, gpu_specs); + verify(host_status == status_t::device_memory_mismatch_k && "Host input must be refused, not copied"); + + // Host OUTPUTS are refused just as firmly, and every verb probes its own. The haystacks below are the + // unified ones, so the only illegal thing in each call is where the results were asked to land. + { + std::vector<std::size_t> host_counts(scattered_view.size()); + std::size_t total = 0; + verify(cuda_engine.try_count(scattered_view, substrings_overlapping_k, + span<std::size_t>(host_counts.data(), host_counts.size()), total, executor, + gpu_specs) == status_t::device_memory_mismatch_k && + "A host counts array must be refused"); + + std::vector<substrings_match_t> host_matches(16); + std::size_t found = 0; + verify(cuda_engine.try_find(scattered_view, substrings_overlapping_k, + span<substrings_match_t>(host_matches.data(), host_matches.size()), found, executor, + gpu_specs) == status_t::device_memory_mismatch_k && + "A host matches array must be refused"); + + unified_vector<float> weights(needle_strings.size(), 1.0f); + std::vector<float> host_scores(scattered_view.size()); + substrings_bm25_t parameters; + parameters.average_document_length = 8.0f; + verify(cuda_engine.try_score_bm25(scattered_view, span<float const>(), parameters, + {weights.data(), weights.size()}, + span<float>(host_scores.data(), host_scores.size()), executor, + gpu_specs) == status_t::device_memory_mismatch_k && + "A host scores array must be refused"); + } + + // Page-locked host memory is a third kind, and the driver reports it as host - so it is refused too, + // which is the refusal a caller is least likely to predict. + { + pinned_vector<std::size_t> pinned_counts(scattered_view.size()); + std::size_t total = 0; + verify(cuda_engine.try_count(scattered_view, substrings_overlapping_k, + span<std::size_t>(pinned_counts.data(), pinned_counts.size()), total, executor, + gpu_specs) == status_t::device_memory_mismatch_k && + "Page-locked host memory must be refused, like any other host memory"); + } + + // Plain device memory is accepted, which unified memory alone would not prove: every host loop that once + // filled these arrays would have faulted on a pointer the host cannot touch. + { + device_vector<std::size_t> device_counts; + verify(device_counts.try_resize_uninitialized(scattered_view.size()) == status_t::success_k); + std::size_t total = 0; + verify(cuda_engine.try_count(scattered_view, substrings_overlapping_k, + span<std::size_t>(device_counts.data(), device_counts.size()), total, executor, + gpu_specs) == status_t::success_k && + "Plain device memory must be accepted, not just unified"); + + std::vector<std::size_t> drained(device_counts.size()); + verify(copy_device_to_host(device_counts, span<std::size_t>(drained.data(), drained.size())) == CUDA_SUCCESS && + "Draining the device counts must succeed"); + std::size_t drained_total = 0; + for (std::size_t const count : drained) drained_total += count; + verify(drained_total == total && "Counts written to device memory must sum to the reported total"); + } +#endif // SZ_USE_CUDA +} + +/** + * @brief Crosses every needle vocabulary with every placement skeleton and every needle transform. + * + * Walked by rotation rather than nested loops: `rotating_index` advances a phase each full turn, so + * crossing the rotations still reaches every combination, and `scale_iterations` decides how far into that + * product a run gets. Ground truth and the declared planting effects run on every cell; the costlier + * properties rotate, keeping the per-cell price flat while the sweep as a whole still exercises each of + * them against each axis triple. + */ +void test_substrings_adversarial_equivalence() { + std::printf(" - testing adversarial needle x placement x transform cross-product...\n"); + std::size_t const needle_generators = (std::size_t)substrings_needle_generator_t::count_k; + std::size_t const placements = (std::size_t)substrings_placement_t::count_k; + std::size_t const transforms = (std::size_t)substrings_transform_t::count_k; + std::size_t const cells = scale_iterations(needle_generators * placements * transforms); + substrings_adversarial_scratch_t scratch; // ? Constructed once, refilled by every cell below. + bool saw_narrow_build = false; // ? A sweep of nothing but declines would prove nothing. + + // One real pool, so sliced-specs cells exercise the all-cores-on-one-haystack path; a dummy executor + // would run `for_slices` as a single slice and never slice at all. + forkunion_executor_t pool; + verify(pool.try_spawn(4) == status_t::success_k); + + for (std::size_t step = 0; step < cells; ++step) { + auto const needle_kind = (substrings_needle_generator_t)rotating_index(step, needle_generators); + auto const placement = (substrings_placement_t)rotating_index(step, placements); + auto const transform = (substrings_transform_t)rotating_index(step, transforms); + // A vocabulary of arbitrary bytes can only be matched byte-exactly: folding rejects malformed UTF-8 + // by contract, and the brute-force oracle decodes its inputs too. + auto const sensitivity = step % 2 && substrings_needles_are_utf8_(needle_kind) ? substrings_uncased_k + : substrings_cased_k; + + scratch.needle_kind = needle_kind; + scratch.placement = placement; + scratch.transform = transform; + scratch.specs = cpu_specs_t {}; + if (step % 3 == 0) scratch.specs.l2_bytes = 1024; // ? Far below the fixtures, forcing haystack slicing. + + // A vocabulary whose output pool grows with the square of its depth takes the quadratic knob, so + // doubling the multiplier doubles the work rather than quadrupling it. + std::size_t const depth = substrings_needles_grow_quadratically_(needle_kind) ? scale_iterations_quadratic(10) + : scale_iterations(10); + generate_substrings_needles_(needle_kind, depth, scratch.needles); + generate_substrings_placements_(placement, transform, sensitivity, needle_kind, scratch.needles.view(), 6, + scratch.haystacks, scratch.plantings); + + check_substrings_against_oracle_(sensitivity, scratch); + check_substrings_declared_effects_(scratch); + switch (rotating_index(step, 3)) { + case 0: check_substrings_backends_agree_(sensitivity, scratch, pool); break; + case 1: saw_narrow_build |= check_substrings_narrow_width_(sensitivity, scratch); break; + case 2: check_substrings_tier_invariant_(sensitivity, scratch); break; + } +#if SZ_USE_CUDA + check_substrings_cuda_agrees_(sensitivity, scratch); +#endif + } + + verify(saw_narrow_build && "Narrow-width sweep never built a u16 automaton - shrink the vocabulary"); +} + +/** + * @brief The parallel engine's large-haystack path - every core slicing one haystack - must agree with the + * serial engine even when matches start on nearly every byte and straddle every slice boundary. + * + * The default `cpu_specs_t` L2 threshold keeps the adversarial fixtures above on the one-core-per-haystack + * path, so this one forces the slicing with a threshold far below its own size and a real pool. + */ +void test_substrings_large_haystacks_equivalence() { + std::printf(" - testing the all-cores-on-one-large-haystack path...\n"); + + forkunion_executor_t pool; + verify(pool.try_spawn(4) == status_t::success_k); + cpu_specs_t sliced_specs; + sliced_specs.l2_bytes = 1024; // ? Far below the fixture, so `is_large_` takes the all-cores path + + // Overlapping needles over a periodic haystack: "ab" completes at every second byte, "abababab" spans + // whole slice overlaps, so every boundary sees matches that begin before it and end after it. + std::vector<std::string> const needle_strings {"ab", "aba", "bab", "abababab"}; + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + std::string periodic(scale_iterations(8) * 1024, '\0'); + for (std::size_t index = 0; index < periodic.size(); ++index) periodic[index] = index % 2 ? 'b' : 'a'; + std::vector<std::string> const haystack_strings {periodic}; + arrow_strings_tape_t haystacks; + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t serial_keys, parallel_keys; + collect_overlapping_matches_into_(serial_engine, haystacks.view(), serial_keys); + + substrings_parallel_t parallel_engine; + verify(parallel_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + collect_overlapping_matches_into_(parallel_engine, haystacks.view(), parallel_keys, pool, sliced_specs); + + verify(!serial_keys.empty() && "The fixture must produce matches"); + verify(parallel_keys == serial_keys && "The sliced parallel path disagrees with the serial reference"); + + // The same split, folded. Several of the four cuts land mid-codepoint on this fixture, which a folded + // walk cannot restart on, so the slices snap back to a codepoint start before folding. Attribution by + // start position makes an unsnapped cut safe on its own - no match can begin inside the codepoint a cut + // lands in - so this pins parallel and serial agreement over multi-byte content rather than the snap. + { + std::vector<std::string> const folded_needles {"ss", "\xC3\x9F", "k", "\xE2\x84\xAA"}; + arrow_strings_tape_t uncased_needles; + verify(uncased_needles.try_assign(folded_needles.data(), folded_needles.data() + folded_needles.size()) == + status_t::success_k); + + std::string multibyte; + char const *const cycle[] = {"\xC3\x9F", "\xC5\xBF", "\xE2\x84\xAA", "s", "k"}; + for (std::size_t index = 0; index < scale_iterations(4) * 1024; ++index) multibyte += cycle[index % 5]; + std::vector<std::string> const uncased_haystack_strings {multibyte}; + arrow_strings_tape_t uncased_haystacks; + verify(uncased_haystacks.try_assign(uncased_haystack_strings.data(), uncased_haystack_strings.data() + 1) == + status_t::success_k); + + substrings_serial_t uncased_serial; + verify(uncased_serial.try_index(uncased_needles.view(), substrings_uncased_k) == status_t::success_k); + substrings_match_set_t uncased_serial_keys, uncased_parallel_keys; + collect_overlapping_matches_into_(uncased_serial, uncased_haystacks.view(), uncased_serial_keys); + + substrings_parallel_t uncased_parallel; + verify(uncased_parallel.try_index(uncased_needles.view(), substrings_uncased_k) == status_t::success_k); + collect_overlapping_matches_into_(uncased_parallel, uncased_haystacks.view(), uncased_parallel_keys, pool, + sliced_specs); + + verify(!uncased_serial_keys.empty() && "The folded fixture must produce matches"); + verify(uncased_parallel_keys == uncased_serial_keys && + "The sliced parallel path disagrees with the serial reference on folded multi-byte content"); + } +} + +#pragma endregion // Adversarial + +#pragma region Construction + +/** + * @brief Structural invariants of the compiled automaton, checked directly against the published + * `aho_corasick_view` rather than against another backend's output. + */ +void test_substrings_construction_equivalence() { + std::printf(" - testing structural invariants of the compiled automaton...\n"); + + // Re-indexing replaces the needle set outright. An engine is tiered for the machine that indexed it, so + // being able to index again is what lets one engine be re-pointed at a new vocabulary - or re-tiered for + // another device - instead of being destroyed and rebuilt. + { + std::vector<std::string> const first_strings {"alpha", "beta"}; + std::vector<std::string> const second_strings {"gamma", "delta", "epsilon"}; + std::vector<std::string> const haystack_strings {"alpha beta gamma delta epsilon"}; + arrow_strings_tape_t first, second, haystacks; + verify(first.try_assign(first_strings.data(), first_strings.data() + first_strings.size()) == + status_t::success_k); + verify(second.try_assign(second_strings.data(), second_strings.data() + second_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t reused; + verify(reused.try_index(first.view(), substrings_cased_k) == status_t::success_k); + verify(reused.count_needles() == first_strings.size()); + substrings_match_set_t first_matches; + collect_overlapping_matches_into_(reused, haystacks.view(), first_matches); + + verify(reused.try_index(second.view(), substrings_cased_k) == status_t::success_k); + verify(reused.count_needles() == second_strings.size() && + "Re-indexing must replace the needle set, not extend it"); + substrings_match_set_t second_matches; + collect_overlapping_matches_into_(reused, haystacks.view(), second_matches); + + // A freshly built engine over the same vocabulary is the oracle: re-indexing must leave nothing of + // the first automaton behind, in the dictionary or in the scratch sized against it. + substrings_serial_t fresh; + verify(fresh.try_index(second.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t fresh_matches; + collect_overlapping_matches_into_(fresh, haystacks.view(), fresh_matches); + verify(second_matches == fresh_matches && "A re-indexed engine must match a freshly indexed one"); + verify(second_matches != first_matches && "The fixture must distinguish the two vocabularies"); + } + + // Overlap policy, pinned as a count rather than left implicit: every occurrence of every needle is + // reported, including the ones nested inside or overlapping a longer match. Over "abcabc", the + // vocabulary {"a", "ab", "abc"} completes each of its three needles at each of two repetitions. + { + std::vector<std::string> const needle_strings {"a", "ab", "abc"}; + std::vector<std::string> const haystack_strings {"abcabc"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + verify(matches.size() == 6 && "All matches are reported - this is not leftmost-longest matching"); + } + + // An empty needle is rejected outright. Skipping it silently would consume no needle index, so every + // later needle's reported `needle_index` would disagree with the caller's own array. + { + substrings_u32_dictionary_t dictionary; + verify(dictionary.try_insert(span<char const> {}) == status_t::unexpected_dimensions_k); + verify(dictionary.count_needles() == 0 && "A rejected needle must not consume an index"); + verify(dictionary.try_insert(span<char const> {"ab", 2}) == status_t::success_k); + verify(dictionary.count_needles() == 1); + } + + // The doubling family: a run of `s` folds ambiguously, since each `s` is also half a sharp-S. Folding the + // stream leaves one path per needle, so the run is a plain chain of its own folded length and every id + // width has room for it, whatever `state_id_t` is. + { + std::string const long_run(11, 's'); + substrings_u16_dictionary_t narrow; + narrow.case_sensitivity(substrings_uncased_k); + verify(narrow.try_insert({long_run.data(), long_run.size()}) == status_t::success_k); + verify(narrow.try_build() == status_t::success_k && "The doubling family is linear once the stream folds"); + verify(narrow.count_states() == long_run.size() + 1 && "One state per folded byte, plus the root"); + + substrings_u32_dictionary_t wide; + wide.case_sensitivity(substrings_uncased_k); + verify(wide.try_insert({long_run.data(), long_run.size()}) == status_t::success_k); + verify(wide.try_build() == status_t::success_k); + verify(wide.count_states() == narrow.count_states() && "The id width cannot change the automaton's shape"); + + // Both widths still find the sharp S spelling of the same run, which is what makes it the hard case. + std::string const spelled = "\xC3\x9F\xC3\x9F\xC3\x9F\xC3\x9F\xC3\x9Fs"; + std::size_t found = 0; + wide.find({spelled.data(), spelled.size()}, + [&](std::size_t, std::size_t, std::size_t) { return ++found, true; }); + verify(found != 0 && "Eleven folded s-runes are spelled by five sharp-S codepoints and one s"); + } + + // Narrowing an automaton that does fit. The engine derives once at the wider id and keeps the narrower + // dictionary when its ceilings hold, so what is checked here is that it actually does: halving the row + // width is the point - a `u16` row is 512 bytes where `u32`'s is 1024. That the narrowed arrays answer + // identically to the wide ones is `check_substrings_narrow_width_`'s job, against the brute-force oracle. + { + std::vector<std::string> const needle_strings = random_short_strings_(scale_iterations(200), 3, 7); + std::vector<std::string> const haystack_strings = random_haystacks_with_needles_(needle_strings, + scale_iterations(64), 16, 96); + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t narrowed; + verify(narrowed.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + verify(narrowed.state_width() == substrings_state_width_t::u16_k && + "A few hundred short needles fit the narrow id, so the engine must have kept it"); + + // The wide derivation the narrowing copied from, rebuilt here, must describe the same automaton. + substrings_u32_dictionary_t wide; + wide.case_sensitivity(substrings_cased_k); + for (span<char const> const &needle : needles.view()) verify(wide.try_insert(needle) == status_t::success_k); + verify(wide.try_build() == status_t::success_k); + verify(narrowed.count_states() == wide.count_states()); + verify(narrowed.count_needles() == wide.count_needles()); + + substrings_match_set_t narrowed_matches; + collect_overlapping_matches_into_(narrowed, haystacks.view(), narrowed_matches); + verify(narrowed_matches.collected_matches.size() != 0 && "The fixture must produce matches to compare"); + } + + // Cold-tier double-array invariant: every slot a state claims sits within 256 of that state's own base, + // and the failure chain from any cold state reaches the root. `hot_count` is forced to zero, so the cold + // tier is genuinely exercised whatever the default hot tier would have been. + { + std::vector<std::string> const needle_strings = random_short_strings_(scale_iterations(300), 3, 7); + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + substrings_u32_dictionary_t dictionary; + dictionary.case_sensitivity(substrings_cased_k); + for (span<char const> const &needle : needles.view()) + verify(dictionary.try_insert(needle) == status_t::success_k); + dictionary.hot_count(0); + verify(dictionary.try_build() == status_t::success_k); + + auto const automaton = dictionary.view(); + constexpr u32_t invalid_state_k = std::numeric_limits<u32_t>::max(); + std::size_t const cold_capacity = (std::size_t)automaton.state_count + 255; + bool exercised_cold_tier = false; + for (std::size_t slot = automaton.hot_count; slot < cold_capacity; ++slot) { + u32_t const owner = automaton.check[slot]; + if (owner == invalid_state_k) continue; + verify(owner < automaton.state_count && "Cold slot owned by an out-of-range state"); + std::size_t const base_of_owner = automaton.base[owner]; + verify(slot >= base_of_owner && slot - base_of_owner < 256 && + "Cold slot's offset from its owner's base is not a valid byte"); + exercised_cold_tier = true; + } + verify(exercised_cold_tier && "Construction sweep never exercised the cold tier - grow the vocabulary"); + + for (u32_t state = automaton.hot_count; state < automaton.state_count; ++state) { + u32_t cursor = state; + std::size_t hops = 0; + while (cursor != automaton.root && hops <= automaton.state_count) { + cursor = automaton.fail[cursor]; + ++hops; + } + verify(cursor == automaton.root && "Failure chain never reaches the root"); + } + } + + // Uncased match spans land on UTF-8 codepoint boundaries at both ends. + { + std::vector<std::string> const needle_strings {"ss", "\xC3\x9F", "caf\xC3\xA9", "K"}; + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_uncased_k) == status_t::success_k); + + span<string_view const> const empty_motifs; + auto &generator = global_random_generator(); + std::vector<std::string> haystack_storage; + for (std::size_t index = 0; index < scale_iterations(20); ++index) { + std::string corpus; + utf8_random_segmentation_corpus_(corpus, 200, utf8_corpus_flavor_t::valid_k, utf8_default_alphabet, + empty_motifs, generator); + haystack_storage.push_back(std::move(corpus)); + } + arrow_strings_tape_t haystacks; + verify(haystacks.try_assign(haystack_storage.data(), haystack_storage.data() + haystack_storage.size()) == + status_t::success_k); + substrings_match_set_t matches; + collect_overlapping_matches_into_(engine, haystacks.view(), matches); + + std::vector<std::size_t> counts(haystacks.view().size(), 0); + std::size_t matches_total = 0; + verify(engine.try_count(haystacks.view(), substrings_overlapping_k, + span<std::size_t>(counts.data(), counts.size()), matches_total) == status_t::success_k); + std::vector<substrings_match_t> raw_matches(matches_total); + std::size_t matches_found = 0; + verify(engine.try_find(haystacks.view(), substrings_overlapping_k, + span<substrings_match_t>(raw_matches.data(), raw_matches.size()), + matches_found) == status_t::success_k); + + for (substrings_match_t const &match : raw_matches) { + // The match carries offsets only, so the bytes come from the haystack it names - the exact + // recipe the match struct's own docstring prescribes. + span<char const> const haystack = haystacks[match.haystack_index]; + std::size_t const end = match.byte_offset + match.byte_length; + bool const start_is_boundary = (haystack[match.byte_offset] & 0xC0) != 0x80; + bool const end_is_boundary = end == haystack.size() || (haystack[end] & 0xC0) != 0x80; + if (!start_is_boundary || !end_is_boundary) + std::fprintf(stderr, "Boundary violation: needle_index=%zu byte_offset=%zu byte_length=%zu\n", + match.needle_index, match.byte_offset, match.byte_length); + verify(start_is_boundary && "Uncased match starts mid-codepoint"); + verify(end_is_boundary && "Uncased match ends mid-codepoint"); + } + } +} + +#pragma endregion // Construction + +#pragma region Matching + +/** + * @brief The leftmost covers are a subset of the overlapping matches, and they share no bytes. + * + * Stated against the overlapping walk rather than a second implementation: every match a cover reports must + * be a match the exhaustive walk also found, and no two of them may touch. + */ +void test_substrings_cover_equivalence() { + std::printf(" - testing that a leftmost cover is a non-overlapping subset of every match...\n"); + + std::vector<std::string> const needle_strings = random_short_strings_(scale_iterations(150), 2, 5); + std::vector<std::string> const haystack_strings = random_haystacks_with_needles_(needle_strings, + scale_iterations(40), 16, 64); + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_match_set_t overlapping; + collect_matches_under_(engine, haystacks.view(), substrings_overlapping_k, overlapping); + + substrings_overlap_policy_t const leftmost_policies[2] = {substrings_leftmost_longest_k, + substrings_leftmost_first_k}; + for (substrings_overlap_policy_t const policy : leftmost_policies) { + substrings_match_set_t cover; + collect_matches_under_(engine, haystacks.view(), policy, cover); + + for (substrings_match_t const &match : cover) + verify(std::binary_search(overlapping.matches.begin(), overlapping.matches.end(), match, + substrings_match_less_) && + "A cover may only report matches the exhaustive walk also found"); + + // The set orders by needle before offset, so it has to be re-sorted by position before neighbours + // mean anything - only then does "no two neighbours touch" imply "no two matches touch". + std::vector<substrings_match_t> by_position = cover.matches; + std::sort(by_position.begin(), by_position.end(), substrings_match_by_position_less_); + for (std::size_t index = 1; index < by_position.size(); ++index) { + substrings_match_t const &earlier = by_position[index - 1]; + substrings_match_t const &later = by_position[index]; + if (earlier.haystack_index != later.haystack_index) continue; + verify(earlier.byte_offset + earlier.byte_length <= later.byte_offset && + "Two matches of one cover shared a byte"); + } + } +} + +#pragma endregion // Matching + +#pragma region Rewriting + +/** + * @brief The flat tape and per-haystack offsets a single rewrite pass produced. + * + * Both members are unified, because a CUDA engine writes them from the device and refuses host memory. + */ +struct substrings_rewrite_tape_t { + unified_vector<char> text; + unified_vector<std::size_t> offsets; + + span<char const> operator[](std::size_t haystack_index) const noexcept { + return {text.data() + offsets[haystack_index], offsets[haystack_index + 1] - offsets[haystack_index]}; + } +}; + +/** @brief Whether @p rewritten 's whole tape holds @p expected, compared as bytes. */ +inline bool tape_equals_(substrings_rewrite_tape_t const &rewritten, std::string const &expected) noexcept { + return rewritten.text.size() == expected.size() && + std::memcmp(rewritten.text.data(), expected.data(), expected.size()) == 0; +} + +/** @brief Whether @p rewritten holds @p expected at @p haystack_index, compared as bytes. */ +inline bool rewritten_equals_(substrings_rewrite_tape_t const &rewritten, std::size_t haystack_index, + std::string const &expected) noexcept { + span<char const> const produced = rewritten[haystack_index]; + return produced.size() == expected.size() && std::memcmp(produced.data(), expected.data(), expected.size()) == 0; +} + +/** + * @brief Rewrites @p haystacks through @p engine under @p overlap_policy, returning the flat result tape. + * + * Sizes the tape with a zero-capacity call first, which is the same size query `try_find` offers, so the + * rewrite itself always runs against a buffer that is exactly big enough. + */ +template <typename engine_type_, typename haystacks_type_, typename... trailing_args_> +substrings_rewrite_tape_t rewrite_all_(engine_type_ &engine, haystacks_type_ const &haystacks, + substrings_overlap_policy_t overlap_policy, + arrow_strings_tape_t const &replacements, trailing_args_ &&...trailing) { + + substrings_rewrite_tape_t rewritten; + rewritten.offsets.assign(haystacks.size() + 1, 0); + span<std::size_t> const offsets_view(rewritten.offsets.data(), rewritten.offsets.size()); + std::size_t needed = 0; + status_t const sized = engine.try_replace(haystacks, overlap_policy, replacements.view(), span<char>(), + offsets_view, needed, trailing...); + verify((needed == 0 ? sized == status_t::success_k : sized == status_t::unexpected_dimensions_k) && + "A zero-capacity rewrite must refuse and name the size it wanted"); + + rewritten.text.assign(needed, '\0'); + std::size_t written = 0; + verify(engine.try_replace(haystacks, overlap_policy, replacements.view(), span<char>(rewritten.text.data(), needed), + offsets_view, written, trailing...) == status_t::success_k); + verify(written == needed && "The sizing pass and the writing pass must agree"); + verify(rewritten.offsets[haystacks.size()] == written && "The terminator must close the last haystack"); + return rewritten; +} + +/** + * @brief Rewriting under both leftmost policies, against the oracle a rewrite carries for free. + * + * Replacing every needle with itself must reproduce the input byte for byte, whatever the vocabulary and + * whichever cover resolved it - a property that needs no second implementation to check against. + */ +void test_substrings_rewriting_equivalence() { + std::printf(" - testing rewriting against the self-replacement oracle...\n"); + + substrings_overlap_policy_t const leftmost_policies[2] = {substrings_leftmost_longest_k, + substrings_leftmost_first_k}; + + // Replacing each needle with itself is the identity, so any deviation is the rewrite's own bug. + { + std::vector<std::string> const needle_strings = random_short_strings_(scale_iterations(200), 2, 6); + std::vector<std::string> const haystack_strings = random_haystacks_with_needles_(needle_strings, + scale_iterations(50), 16, 64); + arrow_strings_tape_t needles, haystacks, replacements; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + for (substrings_overlap_policy_t const policy : leftmost_policies) { + substrings_rewrite_tape_t const rewritten = rewrite_all_(engine, haystacks.view(), policy, replacements); + verify(rewritten.offsets.size() == haystack_strings.size() + 1 && "One boundary per haystack, plus one"); + for (std::size_t index = 0; index < haystack_strings.size(); ++index) + verify(rewritten_equals_(rewritten, index, haystack_strings[index]) && + "Replacing every needle with itself must reproduce the input"); + } + } + + // An empty replacement deletes, and a replacement shorter than its needle shrinks the output - the sizing + // pass accumulates removals and insertions apart, so neither can wrap the unsigned total. + { + std::vector<std::string> const needle_strings {"cat", "dog"}; + std::vector<std::string> const haystack_strings {"cats and dogs", "dogcat", "no pets here"}; + std::vector<std::string> const replacement_strings {"", "z"}; + arrow_strings_tape_t needles, haystacks, replacements; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(replacement_strings.data(), + replacement_strings.data() + replacement_strings.size()) == status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_rewrite_tape_t const rewritten = rewrite_all_(engine, haystacks.view(), + substrings_leftmost_longest_k, replacements); + verify(rewritten_equals_(rewritten, 0, "s and zs") && "An empty replacement deletes and a shorter one shrinks"); + verify(rewritten_equals_(rewritten, 1, "z") && "Both needles rewrite in one haystack"); + verify(rewritten_equals_(rewritten, 2, haystack_strings[2]) && + "A haystack with no match passes through untouched"); + } + + // A longer needle shadows a shorter one under `leftmost_longest`, while `leftmost_first` takes whichever + // needle was listed first - the same distinction the matching policies draw, carried into the rewrite. + { + std::vector<std::string> const needle_strings {"cat", "catalog"}; + std::vector<std::string> const haystack_strings {"catalog"}; + std::vector<std::string> const replacement_strings {"feline", "directory"}; + arrow_strings_tape_t needles, haystacks, replacements; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(replacement_strings.data(), + replacement_strings.data() + replacement_strings.size()) == status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + verify(rewritten_equals_(rewrite_all_(engine, haystacks.view(), substrings_leftmost_longest_k, replacements), 0, + "directory") && + "The longest needle shadows the shorter one it contains"); + verify(rewritten_equals_(rewrite_all_(engine, haystacks.view(), substrings_leftmost_first_k, replacements), 0, + "felinealog") && + "The lower needle index wins however long its rival"); + } + + // An overlapping rewrite is not a function - two matches sharing a byte have no single answer - so the + // policy is refused rather than resolved to some arbitrary winner. + { + std::vector<std::string> const needle_strings {"ab"}; + std::vector<std::string> const haystack_strings {"abab"}; + arrow_strings_tape_t needles, haystacks, replacements; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + std::vector<std::size_t> offsets(haystacks.view().size() + 1, 0); + std::size_t written = 0; + verify(engine.try_replace(haystacks.view(), substrings_overlapping_k, replacements.view(), span<char>(), + span<std::size_t>(offsets.data(), offsets.size()), written) != status_t::success_k); + } + + // A haystack no core's cache holds is split, each core resolving its own stretch of the cover from a + // restart no match spans. The identity oracle holds across the cuts, and so does the serial engine's tape. + { + std::vector<std::string> const needle_strings {"cat", "at", "concat", "the"}; + std::vector<std::string> const replacement_strings {"[CAT]", "@", "<<CONCAT>>", ""}; + std::vector<std::string> long_strings; + for (std::size_t index = 0; index < 5; ++index) { + std::string text; + while (text.size() < 96u * 1024u) text += "the cat sat on a mat concatenating cats at "; + long_strings.push_back(std::move(text)); + } + + arrow_strings_tape_t needles, haystacks, replacements, identity; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(long_strings.data(), long_strings.data() + long_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(replacement_strings.data(), + replacement_strings.data() + replacement_strings.size()) == status_t::success_k); + verify(identity.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + + substrings_serial_t serial_engine; + substrings_parallel_t parallel_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + verify(parallel_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + forkunion_executor_t pool; + verify(pool.try_spawn(4) == status_t::success_k); + cpu_specs_t split_specs; + split_specs.l2_bytes = 4096; // ? Small enough that every haystack above is split across all cores. + + for (substrings_overlap_policy_t policy : leftmost_policies) { + substrings_rewrite_tape_t const unsplit = rewrite_all_(serial_engine, haystacks.view(), policy, + replacements); + substrings_rewrite_tape_t const split = rewrite_all_(parallel_engine, haystacks.view(), policy, + replacements, pool, split_specs); + verify(split.offsets == unsplit.offsets && "A split rewrite must land on the same boundaries"); + verify(split.text == unsplit.text && "A split rewrite must produce the same bytes"); + + substrings_rewrite_tape_t const unchanged = rewrite_all_(parallel_engine, haystacks.view(), policy, + identity, pool, split_specs); + std::string packed; + for (std::string const &text : long_strings) packed += text; + verify(tape_equals_(unchanged, packed) && "Self-replacement must survive every slice boundary"); + } + } +} + +#pragma endregion // Rewriting + +#pragma region Scoring + +/** + * @brief BM25 against scores worked out by hand, plus the bit-stability the header promises. + * + * Term frequencies are raw overlapping counts, so a needle nested in another still contributes every one of + * its own occurrences - which is what classic BM25 scores and what a leftmost cover would have suppressed. + */ +void test_substrings_scoring_unit() { + std::printf(" - testing BM25 scores against hand-computed values...\n"); + + std::vector<std::string> const needle_strings {"cat", "dog"}; + std::vector<std::string> const haystack_strings {"catcat", "dog", "nothing"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + + substrings_bm25_t const parameters {.term_frequency_saturation = 1.2f, + .length_normalization = 0.75f, + .average_document_length = 6.0f}; + std::vector<float> const weights {1.0f, 2.0f}; + std::vector<float> scores(haystack_strings.size(), 0.0f); + span<float const> const weights_view(weights.data(), weights.size()); + span<float> const scores_view(scores.data(), scores.size()); + + verify(engine.try_score_bm25(haystacks.view(), span<float const>(), parameters, weights_view, scores_view) == + status_t::success_k); + + // "catcat" is 6 bytes against a 6-byte mean, so the length term is exactly one and "cat" twice scores + // 1.0 * 2 * (1.2 + 1) / (2 + 1.2 * 1) = 4.4 / 3.2 + verify(std::fabs(scores[0] - 4.4f / 3.2f) < 1e-5f && "Two occurrences must saturate, not double"); + + // "dog" is 3 bytes against the same mean, so the length term is 1 - 0.75 + 0.75 * 3 / 6 = 0.625, and the + // needle's own weight of two scales the whole term: 2.0 * 1 * 2.2 / (1 + 1.2 * 0.625). + verify(std::fabs(scores[1] - 2.0f * 2.2f / (1.0f + 1.2f * 0.625f)) < 1e-5f && + "A weight scales its needle's whole contribution"); + verify(scores[2] == 0.0f && "A haystack no needle hits scores exactly zero"); + + // Bit-stability: the same call on the same engine must reproduce every score exactly, not merely closely. + std::vector<float> const first_scores = scores; + for (std::size_t repeat = 0; repeat < 4; ++repeat) { + std::fill(scores.begin(), scores.end(), 0.0f); + verify(engine.try_score_bm25(haystacks.view(), span<float const>(), parameters, weights_view, scores_view) == + status_t::success_k); + verify(scores == first_scores && "Scores must be bit-identical across runs of one backend"); + } + + // An empty `document_lengths` means byte lengths, which the caller can also state outright. + { + std::vector<float> byte_lengths; + for (std::string const &haystack : haystack_strings) byte_lengths.push_back((float)haystack.size()); + std::vector<float> explicit_scores(haystack_strings.size(), 0.0f); + verify(engine.try_score_bm25( + haystacks.view(), span<float const>(byte_lengths.data(), byte_lengths.size()), parameters, + weights_view, span<float>(explicit_scores.data(), explicit_scores.size())) == status_t::success_k); + verify(explicit_scores == first_scores && "Byte lengths stated outright must score identically"); + } + + // A zero weight removes its needle from the ranking without removing it from the automaton. + { + std::vector<float> const muted {0.0f, 0.0f}; + std::vector<float> muted_scores(haystack_strings.size(), 1.0f); + verify(engine.try_score_bm25(haystacks.view(), span<float const>(), parameters, + span<float const>(muted.data(), muted.size()), + span<float>(muted_scores.data(), muted_scores.size())) == status_t::success_k); + for (float const score : muted_scores) verify(score == 0.0f && "Zero weights must score zero"); + } + + // The parallel backend splits a haystack no core's cache holds, merging per-core tallies. Frequencies are + // integers and the merged row reduces in the same ascending order, so the scores stay bit-identical. + { + std::vector<std::string> long_strings; + for (std::size_t index = 0; index < 6; ++index) { + std::string text; + while (text.size() < 64u * 1024u) text += "cat dog concatenate at the "; + long_strings.push_back(std::move(text)); + } + arrow_strings_tape_t long_haystacks; + verify(long_haystacks.try_assign(long_strings.data(), long_strings.data() + long_strings.size()) == + status_t::success_k); + + substrings_parallel_t parallel_engine; + verify(parallel_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + forkunion_executor_t pool; + verify(pool.try_spawn(4) == status_t::success_k); + cpu_specs_t split_specs; + split_specs.l2_bytes = 4096; // ? Small enough that every haystack above is split across all cores. + + std::vector<float> serial_long(long_strings.size(), 0.0f), parallel_long(long_strings.size(), 0.0f); + verify(engine.try_score_bm25(long_haystacks.view(), span<float const>(), parameters, weights_view, + span<float>(serial_long.data(), serial_long.size())) == status_t::success_k); + verify(parallel_engine.try_score_bm25(long_haystacks.view(), span<float const>(), parameters, weights_view, + span<float>(parallel_long.data(), parallel_long.size()), pool, + split_specs) == status_t::success_k); + verify(parallel_long == serial_long && "A split haystack must score exactly as an unsplit one"); + } +} + +/** + * @brief BM25 over dictionaries wide enough to change how both backends hold their counters. + * + * The CPU orders the needles a document hit in passes covering the widest needle index, so a two-needle + * fixture drives only one pass. The GPU indexes its table straight by needle while the dictionary fits it, + * hashes once it does not, and spills to a per-block row beyond that. All three fail silently, since a + * needle counted twice scores as two small counts and `substrings_bm25_term` is concave. + * + * Frequencies are known by construction: the needles are fixed width and space separated, so a match can + * only begin where a token does, and the text's own recipe is the oracle. + */ +void test_substrings_scoring_wide_equivalence() { + std::printf(" - testing BM25 across the direct, hashed and overflow regimes...\n"); + + // Fixed-width needles, so none is a substring of another and a haystack's distinct count is exactly the + // number of tokens it was built from - which is what lets the regime be chosen rather than hoped for. + auto const needle_at = [](std::size_t index) { + std::string text = "w00000"; + for (std::size_t digit = 0, value = index; digit < 5; ++digit, value /= 10) + text[5 - digit] = (char)('0' + (value % 10)); + return text; + }; + + // Below one slot per needle the GPU indexes its table directly; above it, hashed. The widths also carry + // the CPU ordering past one radix pass, which a needle index under 256 would never reach. + for (std::size_t needle_count : {std::size_t {4000}, std::size_t {20000}}) { + std::vector<std::string> needle_strings; + for (std::size_t index = 0; index < needle_count; ++index) needle_strings.push_back(needle_at(index)); + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + + // One haystack per distinct-count, the widest touching every needle, so a 20,000-needle dictionary + // overruns an 8,192-slot table while a 4,000-needle one never can. The recipe is kept as the oracle. + std::vector<std::string> texts; + std::vector<std::vector<std::uint32_t>> expected_frequencies; + for (std::size_t distinct : {needle_count, needle_count / 2, std::size_t {17}, std::size_t {0}}) { + std::string text; + std::vector<std::uint32_t> frequencies(needle_count, 0u); + for (std::size_t index = 0; index < distinct; ++index) { + text += needle_at(index), text += ' ', ++frequencies[index]; + if (index % 3 == 0) text += needle_at(index), text += ' ', ++frequencies[index]; + } + texts.push_back(std::move(text)); + expected_frequencies.push_back(std::move(frequencies)); + } + + arrow_strings_tape_t host_haystacks; + verify(host_haystacks.try_assign(texts.data(), texts.data() + texts.size()) == status_t::success_k); + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + + substrings_bm25_t const parameters {.term_frequency_saturation = 1.2f, + .length_normalization = 0.75f, + .average_document_length = 4096.0f}; + // Weights are read by the scoring kernel, so on a CUDA scope they must live where it can reach them. + unified_vector<float> weights(needle_count); + for (std::size_t index = 0; index < needle_count; ++index) weights[index] = 1.0f + (float)(index % 7); + span<float const> const weights_view {weights.data(), weights.size()}; + + std::vector<float> host_scores(texts.size(), 0.0f); + verify(serial_engine.try_score_bm25(host_haystacks.view(), span<float const>(), parameters, weights_view, + span<float>(host_scores.data(), host_scores.size())) == + status_t::success_k); + + // The oracle sums the recipe's own frequencies ascending by needle, in `f32`, which is the order the + // header publishes - so this is an equality rather than a tolerance, and it is what pins the ordering + // the engine reaches by a different route. + for (std::size_t index = 0; index < texts.size(); ++index) { + float const document_length = (float)texts[index].size(); + float expected = 0; + for (std::size_t needle = 0; needle < needle_count; ++needle) { + std::uint32_t const frequency = expected_frequencies[index][needle]; + if (frequency == 0u) continue; + expected += weights[needle] * substrings_bm25_term(parameters, (float)frequency, document_length); + } + verify(host_scores[index] == expected && "Serial scores must match an ascending-order oracle exactly"); + } + +#if SZ_USE_CUDA + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + // A kernel cannot reach the caller's host memory, so the same texts are staged unified for the device. + unified_texts_t const staged {texts}; + span<span<char const> const> const haystacks_view = staged.view(); + + substrings_cuda_t cuda_engine; + verify(cuda_engine.try_index(needles.view(), substrings_cased_k, executor, gpu_specs) == status_t::success_k); + + unified_vector<float> device_scores(texts.size(), 0.0f); + verify(cuda_engine + .try_score_bm25(haystacks_view, span<float const>(), parameters, weights_view, + span<float>(device_scores.data(), device_scores.size()), executor, gpu_specs) + .status == status_t::success_k); + + for (std::size_t index = 0; index < texts.size(); ++index) { + verify(std::isfinite(device_scores[index]) && "A score must be a number in every regime"); + verify(std::fabs(device_scores[index] - host_scores[index]) <= + 1e-4f * std::fabs(host_scores[index]) + 1e-6f && + "Device scores must agree with the serial engine at every dictionary width"); + } + + // One needle weighted alone, so a count split between the table and the overflow row reads as + // `5 * term(1)` against `term(5)` - which the relative comparison above would hide at this width. + { + std::size_t const watched = needle_count - 1; + std::string text; + for (std::size_t index = 0; index < needle_count; ++index) text += needle_at(index) + " "; + for (std::size_t repeat = 0; repeat < 4; ++repeat) text += needle_at(watched) + " "; + + unified_texts_t const watched_staged {{text}}; + + unified_vector<float> lone_weights(needle_count, 0.0f); + lone_weights[watched] = 1.0f; + unified_vector<float> lone_score(1, 0.0f); + verify(cuda_engine + .try_score_bm25(watched_staged.view(), span<float const>(), parameters, + span<float const>(lone_weights.data(), lone_weights.size()), + span<float>(lone_score.data(), lone_score.size()), executor, gpu_specs) + .status == status_t::success_k); + + float const normalized = 1.0f - parameters.length_normalization + + parameters.length_normalization * (float)text.size() / + parameters.average_document_length; + float const expected = 5.0f * (parameters.term_frequency_saturation + 1.0f) / + (5.0f + parameters.term_frequency_saturation * normalized); + verify(std::fabs(lone_score[0] - expected) <= 1e-5f * expected && + "Five occurrences of one needle must sum into one slot, not split across two"); + } + + // A hashed table is filled by racing lanes, so the seating order differs run to run; fixed-point + // accumulation is what makes the total independent of it. + for (std::size_t repeat = 0; repeat < 3; ++repeat) { + unified_vector<float> repeated(texts.size(), 0.0f); + verify(cuda_engine + .try_score_bm25(haystacks_view, span<float const>(), parameters, weights_view, + span<float>(repeated.data(), repeated.size()), executor, gpu_specs) + .status == status_t::success_k); + for (std::size_t index = 0; index < texts.size(); ++index) + verify(repeated[index] == device_scores[index] && "Device scores must repeat bit for bit"); + } +#endif // SZ_USE_CUDA + } + + // Full length normalization against an empty haystack leaves the closed form at `0/0`. Nothing is counted, + // so no term is ever evaluated, and the score is a plain zero rather than a NaN that would poison a rank. + { + std::vector<std::string> const needle_strings {"cat", "dog"}; + std::vector<std::string> const haystack_strings {""}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_bm25_t const degenerate {.term_frequency_saturation = 1.2f, + .length_normalization = 1.0f, + .average_document_length = 6.0f}; + unified_vector<float> const weights(2, 1.0f); + span<float const> const weights_view {weights.data(), weights.size()}; + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + std::vector<float> host_scores(1, 1.0f); + verify(serial_engine.try_score_bm25(haystacks.view(), span<float const>(), degenerate, weights_view, + span<float>(host_scores.data(), host_scores.size())) == + status_t::success_k); + verify(host_scores[0] == 0.0f && "An empty haystack scores zero, not a NaN"); + +#if SZ_USE_CUDA + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + unified_texts_t const empty_staged {{std::string {}}}; + + substrings_cuda_t cuda_engine; + verify(cuda_engine.try_index(needles.view(), substrings_cased_k, executor, gpu_specs) == status_t::success_k); + unified_vector<float> device_scores(1, 1.0f); + verify(cuda_engine + .try_score_bm25(empty_staged.view(), span<float const>(), degenerate, weights_view, + span<float>(device_scores.data(), device_scores.size()), executor, gpu_specs) + .status == status_t::success_k); + verify(device_scores[0] == 0.0f && "An empty haystack scores zero on the device too"); +#endif // SZ_USE_CUDA + } +} + +/** + * @brief Rewriting and scoring on the GPU, against the serial engine and against the identity oracle. + * + * The oracle carries the rewrite's own proof - replacing every needle with itself must reproduce the input + * byte for byte - so a device kernel is checked without a second device implementation to check it against. + * Scores are compared within a tolerance across backends and bit for bit against the device itself, which is + * the pair of promises the header makes - see the reduction note beside the comparison below. + */ +void test_substrings_cuda_equivalence() { + std::printf(" - testing CUDA rewriting and BM25 against the serial engine...\n"); +#if SZ_USE_CUDA + + gpu_specs_t gpu_specs; + verify(gpu_specs_fetch(gpu_specs) == status_t::success_k); + cuda_executor_t executor; + + std::vector<std::string> const needle_strings {"cat", "at", "concat", "dog", "the"}; + std::vector<std::string> const replacement_strings {"[CAT]", "@", "<<CONCAT>>", "", "THE"}; + arrow_strings_tape_t needles, replacements, identity; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(replacements.try_assign(replacement_strings.data(), + replacement_strings.data() + replacement_strings.size()) == status_t::success_k); + verify(identity.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + + std::vector<std::string> texts; + for (std::size_t index = 0; index < scale_iterations(24); ++index) + texts.push_back("the cat sat on a mat, concatenating cats at the dog " + std::to_string(index)); + + // Unified, because a kernel cannot reach the caller's host memory - the contract every CUDA entry keeps. + unified_texts_t const staged {texts}; + span<span<char const> const> const haystacks_view = staged.view(); + arrow_strings_tape_t reference_haystacks; + verify(reference_haystacks.try_assign(texts.data(), texts.data() + texts.size()) == status_t::success_k); + + for (substrings_case_sensitivity_t sensitivity : {substrings_cased_k, substrings_uncased_k}) { + substrings_serial_t serial_engine; + substrings_cuda_t cuda_engine; + verify(serial_engine.try_index(needles.view(), sensitivity) == status_t::success_k); + verify(cuda_engine.try_index(needles.view(), sensitivity, executor, gpu_specs) == status_t::success_k); + + for (substrings_overlap_policy_t policy : {substrings_leftmost_longest_k, substrings_leftmost_first_k}) { + substrings_rewrite_tape_t const on_host = rewrite_all_(serial_engine, reference_haystacks.view(), policy, + replacements); + substrings_rewrite_tape_t const on_device = rewrite_all_(cuda_engine, haystacks_view, policy, replacements, + executor, gpu_specs); + verify(on_device.offsets == on_host.offsets && "Rewritten boundaries must agree with the serial engine"); + verify(on_device.text == on_host.text && "Rewritten bytes must agree with the serial engine"); + + // Replacing every needle with itself is the identity, whatever the cover resolved to. + substrings_rewrite_tape_t const unchanged = rewrite_all_(cuda_engine, haystacks_view, policy, identity, + executor, gpu_specs); + std::string packed; + for (std::string const &text : texts) packed += text; + verify(tape_equals_(unchanged, packed) && "Replacing each needle with itself must reproduce the input"); + } + + unified_vector<float> weights(needle_strings.size(), 1.5f); + unified_vector<float> device_scores(texts.size()); + std::vector<float> host_scores(texts.size()); + substrings_bm25_t parameters; + parameters.average_document_length = 48.0f; + verify(serial_engine.try_score_bm25(reference_haystacks.view(), span<float const>(), parameters, + {weights.data(), weights.size()}, + {host_scores.data(), host_scores.size()}) == status_t::success_k); + verify(cuda_engine.try_score_bm25( + haystacks_view, span<float const>(), parameters, {weights.data(), weights.size()}, + {device_scores.data(), device_scores.size()}, executor, gpu_specs) == status_t::success_k); + // Bit-stability is promised per backend, not across two of them: both reduce in ascending needle + // order, but `nvcc` contracts `score + weight * term` into an FMA where the host compiler need not, + // which moves the last ulp. So the backends are compared numerically, and the device is compared + // against itself for the exact reproducibility the header does promise. + for (std::size_t index = 0; index < texts.size(); ++index) + verify(std::fabs(device_scores[index] - host_scores[index]) <= + 1e-5f * std::fabs(host_scores[index]) + 1e-6f && + "Device scores must agree with the serial engine"); + + unified_vector<float> repeated(texts.size()); + for (std::size_t repeat = 0; repeat < 3; ++repeat) { + std::fill(repeated.begin(), repeated.end(), 0.0f); + verify(cuda_engine.try_score_bm25(haystacks_view, span<float const>(), parameters, + {weights.data(), weights.size()}, {repeated.data(), repeated.size()}, + executor, gpu_specs) == status_t::success_k); + for (std::size_t index = 0; index < texts.size(); ++index) + verify(repeated[index] == device_scores[index] && + "Scores must be bit-identical across runs of one backend"); + } + } +#endif // SZ_USE_CUDA +} + +#pragma endregion // Scoring + +#pragma region Safety + +/** + * @brief The three degenerate output-buffer shapes: an empty batch, an undersized buffer, and an oversized + * one. The cases above always size `matches` to exactly the count `try_count` produced. + */ +void test_substrings_buffer_safety() { + std::printf(" - testing empty, undersized and oversized output buffers...\n"); + + std::vector<std::string> const needle_strings {"he", "she", "his", "hers"}; + std::vector<std::string> const haystack_strings {"ushers", "hishers"}; + arrow_strings_tape_t needles, haystacks; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + verify(haystacks.try_assign(haystack_strings.data(), haystack_strings.data() + haystack_strings.size()) == + status_t::success_k); + + substrings_serial_t serial_engine; + verify(serial_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + substrings_parallel_t parallel_engine; + verify(parallel_engine.try_index(needles.view(), substrings_cased_k) == status_t::success_k); + + std::vector<std::size_t> counts(haystacks.view().size(), 0); + std::size_t required = 0; + verify(serial_engine.try_count(haystacks.view(), substrings_overlapping_k, + span<std::size_t>(counts.data(), counts.size()), required) == status_t::success_k); + verify(required > 1 && "fixtures must produce several matches for the undersized case to mean anything"); + + std::size_t matches_found = 0; + + // An empty batch must be accepted and write nothing, rather than dereferencing an unallocated offsets + // array while seeding its first prefix-sum entry. + { + std::vector<span<char const>> const no_haystacks; + span<substrings_match_t> const no_matches; + verify(parallel_engine.try_find(no_haystacks, substrings_overlapping_k, no_matches, matches_found) == + status_t::success_k); + verify(matches_found == 0); + } + + // An undersized buffer must be refused before anything is written, on both backends, and the reported + // count must name the capacity the call wanted. The status already distinguishes a refusal from a + // success, so reporting the need costs no ambiguity and saves the caller a counting pass. + { + std::vector<substrings_match_t> too_small(required - 1); + span<substrings_match_t> const too_small_view(too_small.data(), too_small.size()); + verify(serial_engine.try_find(haystacks.view(), substrings_overlapping_k, too_small_view, matches_found) == + status_t::unexpected_dimensions_k); + verify(matches_found == required); + verify(parallel_engine.try_find(haystacks.view(), substrings_overlapping_k, too_small_view, matches_found) == + status_t::unexpected_dimensions_k); + verify(matches_found == required); + } + + // The same refusal with no buffer at all is the canonical size query: one call, no allocation, and the + // need comes back in `matches_found`. + { + span<substrings_match_t> const no_capacity; + verify(serial_engine.try_find(haystacks.view(), substrings_overlapping_k, no_capacity, matches_found) == + status_t::unexpected_dimensions_k); + verify(matches_found == required); + verify(parallel_engine.try_find(haystacks.view(), substrings_overlapping_k, no_capacity, matches_found) == + status_t::unexpected_dimensions_k); + verify(matches_found == required); + } + + // An oversized buffer is legal: `matches` supplies capacity, and `matches_found` states how much of it + // was actually used. + { + std::vector<substrings_match_t> too_large(required + 16); + span<substrings_match_t> const too_large_view(too_large.data(), too_large.size()); + verify(serial_engine.try_find(haystacks.view(), substrings_overlapping_k, too_large_view, matches_found) == + status_t::success_k); + verify(matches_found == required); + verify(parallel_engine.try_find(haystacks.view(), substrings_overlapping_k, too_large_view, matches_found) == + status_t::success_k); + verify(matches_found == required); + } +} + +/** + * @brief Malformed-needle rejection over every malformed-input class - exact mode accepts any byte sequence, + * uncased mode accepts a needle exactly when it is structurally well-formed UTF-8 - then a sweep of + * malformed, mutated haystacks against a well-formed dictionary, where `try_count` and `try_find` + * must agree on the match count without crashing or hanging. + */ +void test_substrings_safety() { + std::printf(" - testing malformed-needle rejection and malformed-haystack robustness...\n"); + + auto &generator = global_random_generator(); + + // The malformed pool mixes ill-formed byte sequences with noncharacters, which are reserved codepoints but + // structurally well-formed UTF-8; only the former must be rejected. `sz_rune_decode` decides which is which, + // so this tracks structural well-formedness rather than assuming every sample is malformed. The sweep walks + // every class rather than sampling, so both outcomes are reached at any `SZ_TESTS_MULTIPLIER`. + sz::span<char const *const> const malformed_pool = malformed_classes_(); + bool saw_rejection = false, saw_acceptance = false; + for (std::size_t index = 0; index < malformed_pool.size(); ++index) { + std::string const malformed {malformed_pool[index]}; + std::vector<span<char const>> const needles {span<char const>(malformed.data(), malformed.size())}; + + substrings_serial_t exact_engine; + verify(exact_engine.try_index(needles, substrings_cased_k) == status_t::success_k && + "Exact mode must accept any byte sequence as a needle"); + + bool structurally_valid = true; + for (char const *cursor = malformed.data(), *end = cursor + malformed.size(); cursor != end;) { + sz_rune_t rune; + sz_rune_length_t const consumed = sz_rune_decode(cursor, end, &rune); + if (consumed == sz_rune_invalid_k) { + structurally_valid = false; + break; + } + cursor += consumed; + } + + substrings_serial_t uncased_engine; + status_t const uncased_status = uncased_engine.try_index(needles, substrings_uncased_k); + status_t const expected_status = structurally_valid ? status_t::success_k : status_t::invalid_utf8_k; + verify(uncased_status == expected_status && + "Uncased mode's acceptance must track structural UTF-8 well-formedness exactly"); + saw_rejection |= !structurally_valid; + saw_acceptance |= structurally_valid; + } + verify(saw_rejection && "Safety sweep never produced a genuinely ill-formed needle"); + verify(saw_acceptance && "Safety sweep never produced a structurally well-formed needle"); + + // Positive control: a well-formed UTF-8 needle is accepted under case folding. + { + std::vector<span<char const>> const needles {span<char const>("caf\xC3\xA9", 5)}; + substrings_serial_t engine; + verify(engine.try_index(needles, substrings_uncased_k) == status_t::success_k && + "A well-formed UTF-8 needle must be accepted under case folding"); + } + + // Malformed and mutated haystacks never crash or hang a well-formed uncased dictionary, and `try_count` + // and `try_find` always agree on how many matches there were. + { + std::vector<std::string> const needle_strings {"ss", "\xC3\x9F", "the", "K"}; + arrow_strings_tape_t needles; + verify(needles.try_assign(needle_strings.data(), needle_strings.data() + needle_strings.size()) == + status_t::success_k); + substrings_serial_t engine; + verify(engine.try_index(needles.view(), substrings_uncased_k) == status_t::success_k); + + span<string_view const> const empty_motifs; + for (std::size_t index = 0; index < scale_iterations(80); ++index) { + std::string haystack; + utf8_random_segmentation_corpus_(haystack, 128, utf8_corpus_flavor_t::malformed_k, utf8_default_alphabet, + empty_motifs, generator); + apply_mutation_passes_(haystack, generator); + std::vector<span<char const>> const haystacks {span<char const>(haystack.data(), haystack.size())}; + + std::vector<std::size_t> counts(1, 0); + std::size_t matches_total = 0; + verify(engine.try_count(haystacks, substrings_overlapping_k, + span<std::size_t>(counts.data(), counts.size()), + matches_total) == status_t::success_k); + std::vector<substrings_match_t> matches(matches_total); + std::size_t matches_found = 0; + verify(engine.try_find(haystacks, substrings_overlapping_k, + span<substrings_match_t>(matches.data(), matches.size()), + matches_found) == status_t::success_k); + verify(matches_found == counts[0] && "try_count and try_find disagree on match count"); + } + } +} + +#pragma endregion // Safety + +} // namespace scripts +} // namespace stringzilla +} // namespace ashvardanian diff --git a/test/substrings.py b/test/substrings.py new file mode 100644 index 00000000..d47689ff --- /dev/null +++ b/test/substrings.py @@ -0,0 +1,637 @@ +#!/usr/bin/env python3 +"""Substrings: multi-pattern Aho-Corasick search via `szs.Substrings` construction and its four verbs. + +Mirrors the C++ test/substrings.cuh translation unit. + +Covers: engine construction from every `Strs` layout, hand-pinned counts and match records for the +classic `he`/`she`/`his`/`hers` automaton, the three overlap policies, full Unicode case folding, +BM25 against hand-computed values, rewriting including the self-replacement identity, the arithmetic +`replace_bound` ceiling, and haystacks large enough to cross the per-core slicing path. +Compares against: hand-written expectations, `sz.utf8_uncased_matches` as the single-pattern oracle the +folded walk must agree with, every other device scope as a backend differential, and `pyahocorasick` as +an independent automaton when it imports. + +Run: + uv pip install numpy pytest + uv pip install --group differential # optional, unlocks the third-party differential + SZ_TARGET=stringzillas-cpus uv pip install -e . --force-reinstall --no-build-isolation + uv run --no-project python -m pytest test/substrings.py -q + SZ_TARGET=stringzillas-cuda uv pip install -e . --force-reinstall --no-build-isolation + uv run --no-project python -m pytest test/substrings.py -q +""" + +import random +import sys + +import pytest +import numpy as np + +import stringzilla as sz +import stringzillas as szs +from stringzilla import Strs + +from test.sz_helpers import SEED_VALUES, malformed_utf8_corpus, scale_iterations, seed_random_generators +from test.szs_helpers import DEVICE_NAMES, device_float32_array, device_scope_and_capabilities + +CLASSIC_NEEDLES = ["he", "she", "his", "hers"] +CLASSIC_HAYSTACKS = ["ushers", "nothing", "hishers"] + +#: Every walk the engine offers, so a sweep names them rather than repeating three literals. +POLICIES = ["overlapping", "leftmost-longest", "leftmost-first"] + +#: The two covers, which are the only policies a rewrite accepts. +COVER_POLICIES = ["leftmost-longest", "leftmost-first"] + +#: Every scope the differential region sweeps, each paired with the capabilities its engine is built +#: for. The first entry is the reference every other must reach. +DIFFERENTIAL_DEVICE_SCOPES = [(name, *device_scope_and_capabilities(name)) for name in DEVICE_NAMES] + + +def matches_as_tuples(engine, haystacks, **kwargs): + """Zips the four columns `find` returns into records, so expectations read as literals.""" + haystack_ix, needle_ix, offsets, lengths = engine.find(haystacks, **kwargs) + return sorted((int(h), int(n), int(o), int(l)) for h, n, o, l in zip(haystack_ix, needle_ix, offsets, lengths)) + + +def random_ascii_corpus(rng: random.Random, documents: int, length: int, alphabet: str = "abcdefg ") -> list: + """A corpus drawn from a tiny alphabet, so needles of two or three letters hit often enough to + exercise the walk rather than measuring an empty one.""" + return ["".join(rng.choice(alphabet) for _ in range(rng.randint(1, length))) for _ in range(documents)] + + +# region Unit + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_counts_and_matches(device_name: str): + """The textbook automaton, pinned by hand: `ushers` holds `she`, `he` and `hers`, `hishers` + additionally holds `his`, and `nothing` holds none of them.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + + counts, total = engine.count(Strs(CLASSIC_HAYSTACKS), device=device) + assert list(int(c) for c in counts) == [3, 0, 4] + assert total == 7 + assert counts.dtype == np.uint64 + + found = matches_as_tuples(engine, Strs(CLASSIC_HAYSTACKS), device=device) + assert found == [ + (0, 0, 2, 2), # ushers -> "he" + (0, 1, 1, 3), # ushers -> "she" + (0, 3, 2, 4), # ushers -> "hers" + (2, 0, 3, 2), # hishers -> "he" + (2, 1, 2, 3), # hishers -> "she" + (2, 2, 0, 3), # hishers -> "his" + (2, 3, 3, 4), # hishers -> "hers" + ] + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_leftmost_policies(device_name: str): + """A cover keeps no two matches sharing a byte. Longest prefers the widest match at the earliest + start, first prefers the lowest needle index there, and both are subsets of the overlapping walk.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + haystacks = Strs(CLASSIC_HAYSTACKS) + + overlapping = set(matches_as_tuples(engine, haystacks, device=device, policy="overlapping")) + longest = matches_as_tuples(engine, haystacks, device=device, policy="leftmost-longest") + first = matches_as_tuples(engine, haystacks, device=device, policy="leftmost-first") + + assert set(longest) <= overlapping + assert set(first) <= overlapping + # Longest *at the leftmost start*, not the longest overall: `ushers` commits `she` at 1, which rules + # out the wider `hers` at 2 because the two would share bytes. + assert longest == [(0, 1, 1, 3), (2, 2, 0, 3), (2, 3, 3, 4)] + # First takes the same starts but the lowest needle index there, so `hishers` commits `his` at 0 and + # then `he` at 3 rather than the wider `hers`. + assert first == [(0, 1, 1, 3), (2, 0, 3, 2), (2, 2, 0, 3)] + # Each cover is disjoint within itself; the two covers are different answers, so they are not + # compared against each other. + for cover in (longest, first): + by_haystack = {} + for haystack_index, _, offset, length in cover: + spans = by_haystack.setdefault(haystack_index, []) + for other_offset, other_length in spans: + assert offset + length <= other_offset or other_offset + other_length <= offset + spans.append((offset, length)) + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_uncased_folds_both_sides(device_name: str): + """Folding applies to needle and haystack alike, so a capitalized corpus matches a lowercase + dictionary, and the byte-exact engine over the same inputs does not.""" + device, capabilities = device_scope_and_capabilities(device_name) + haystacks = Strs(["The Hershey Company"]) + + cased = szs.Substrings(Strs(["hershey"]), device=device, capabilities=capabilities) + uncased = szs.Substrings(Strs(["hershey"]), case_sensitivity="uncased", device=device, capabilities=capabilities) + assert cased.count(haystacks, device=device)[1] == 0 + assert uncased.count(haystacks, device=device)[1] == 1 + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_uncased_conformance_table(device_name: str): + """Full `CaseFolding.txt` conformance, ported from the C++ table. Folding is not length-preserving, + so a needle matches spans of a byte length its own does not predict, and the equivalence classes are + exactly the ones status codes `C` and `F` draw.""" + device, capabilities = device_scope_and_capabilities(device_name) + + # Corruption guard: the two load-bearing fixtures re-spelled as bytes, so a tool that silently + # renormalizes the escapes below is caught here rather than in a confusing match count. + assert "ß".encode() == b"\xc3\x9f", "Sharp S literal corrupted" + assert "â„Ē".encode() == b"\xe2\x84\xaa", "Kelvin sign literal corrupted" + + # | needle | must match | must not match | + table = [ + ("ss", ["ss", "SS", "sS", "Ss", "ß", "áēž"], ["s"]), + ("ß", ["ß", "áēž", "ss", "SS", "sS", "Ss"], ["s"]), + ("K", ["K", "k", "tempâ„Ēvalue"], []), + ("Å", ["Å", "ÃĨ", "tempâ„Ģvalue"], ["AĖŠ"]), + ("Ä°", ["iĖ‡"], ["i", "I"]), + ("Ê", ["Ê", "É"], ["eĖ"]), + # A length-changing fold in the MIDDLE of a needle, so the byte-delta state keying that + # reconverges variable-length preimages is exercised mid-walk rather than only at acceptance. + ("weißrd", ["weissrd", "weiSSrd", "weiáēžrd"], ["weisrd", "weird"]), + ] + for needle, matching, missing in table: + engine = szs.Substrings(Strs([needle]), case_sensitivity="uncased", device=device, capabilities=capabilities) + for haystack in matching: + assert engine.count(Strs([haystack]), device=device)[1] >= 1, f"{needle!r} must match {haystack!r}" + for haystack in missing: + assert engine.count(Strs([haystack]), device=device)[1] == 0, f"{needle!r} must miss {haystack!r}" + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_uncased_matches_single_pattern_engine(device_name: str): + """Agreement with `sz.utf8_uncased_matches` is the entire semantic claim of the uncased mode: a + one-needle dictionary must report exactly the spans the shipped single-pattern engine reports.""" + device, capabilities = device_scope_and_capabilities(device_name) + needles = ["ss", "ß", "K", "Å", "Ä°", "Ê", "weißrd", "the"] + haystacks = [ + "the Straße was STRASSE", + "tempâ„Ēvalue and K and k", + "ÅngstrÃļm â„Ģ ÃĨ", + "weißrd weissrd weiSSrd", + "Ä°stanbul iĖ‡ I i", + "cafÊ CAFÉ cafeĖ", + ] + + for needle in needles: + engine = szs.Substrings(Strs([needle]), case_sensitivity="uncased", device=device, capabilities=capabilities) + for haystack in haystacks: + view = sz.Str(haystack) + oracle = sorted( + (match.offset_within(view), match.nbytes) + for match in sz.utf8_uncased_matches(view, needle, include_overlapping=True) + ) + found = [ + (offset, length) for _, _, offset, length in matches_as_tuples(engine, Strs([haystack]), device=device) + ] + assert found == oracle, f"needle {needle!r} over {haystack!r}" + + +# endregion Unit + + +# region Scoring + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_score_bm25_hand_computed(device_name: str): + """The exact values the C++ suite pins, so a constant both implementations share cannot hide.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(["cat", "dog"]), device=device, capabilities=capabilities) + haystacks = Strs(["catcat", "dog", "nothing"]) + weights = device_float32_array([1.0, 2.0], device_name) + + scores = engine.score_bm25(haystacks, weights, 6.0, device=device) + # "catcat" is 6 bytes against a 6-byte mean, so the length term is exactly one and "cat" twice scores + # 1.0 * 2 * (1.2 + 1) / (2 + 1.2 * 1) = 4.4 / 3.2 + assert abs(float(scores[0]) - 4.4 / 3.2) < 1e-5, "Two occurrences must saturate, not double" + # "dog" is 3 bytes against the same mean, so the length term is 1 - 0.75 + 0.75 * 3 / 6 = 0.625, and + # the needle's own weight of two scales the whole term. + assert abs(float(scores[1]) - 2.0 * 2.2 / (1.0 + 1.2 * 0.625)) < 1e-5, "A weight scales its whole term" + assert float(scores[2]) == 0.0, "A haystack no needle hits scores exactly zero" + + # Bit-stability: the same call on the same engine must reproduce every score exactly. + for _ in range(scale_iterations(4)): + repeated = engine.score_bm25(haystacks, weights, 6.0, device=device) + assert np.array_equal(repeated, scores), "Scores must be bit-identical across runs of one backend" + + # Omitted `document_lengths` means byte lengths, which the caller can also state outright. + byte_lengths = device_float32_array([6.0, 3.0, 7.0], device_name) + stated = engine.score_bm25(haystacks, weights, 6.0, device=device, document_lengths=byte_lengths) + assert np.array_equal(stated, scores), "Byte lengths stated outright must score identically" + + # A zero weight removes its needle from the ranking without removing it from the automaton. + muted = engine.score_bm25(haystacks, device_float32_array([0.0, 0.0], device_name), 6.0, device=device) + assert not muted.any(), "Zero weights must score zero" + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_score_bm25_orders_documents(device_name: str): + """A document holding none of the query terms scores exactly zero, one holding more of them scores + higher, and the score rises with a term's weight.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + haystacks = Strs(CLASSIC_HAYSTACKS) + + uniform = device_float32_array([1.0] * len(CLASSIC_NEEDLES), device_name) + scores = engine.score_bm25(haystacks, uniform, 6.0, device=device) + assert scores.dtype == np.float32 and len(scores) == 3 + assert scores[1] == 0.0 # "nothing" holds no needle + assert scores[2] > scores[0] > 0.0 # "hishers" holds four, "ushers" three + + # Scaling has to allocate for the scope too, since `uniform * 2.0` would land back on the host. + doubled = device_float32_array([2.0] * len(CLASSIC_NEEDLES), device_name) + louder = engine.score_bm25(haystacks, doubled, 6.0, device=device) + assert louder[0] > scores[0] and louder[1] == 0.0 + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_score_bm25_checks_the_weight_count(device_name: str): + """One weight per needle, no more and no fewer, since the engine reads the dictionary's width.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + + # Two weights against four needles, on the scope's own memory so the refusal is about the count. + with pytest.raises(Exception): + engine.score_bm25(Strs(CLASSIC_HAYSTACKS), device_float32_array([1.0, 1.0], device_name), 6.0, device=device) + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_score_bm25_refuses_a_missing_mean(device_name: str): + """Normalizing divides by the corpus mean, so a positive `length_normalization` without one is + refused rather than silently dropping both it and `document_lengths`.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + haystacks = Strs(CLASSIC_HAYSTACKS) + uniform = device_float32_array([1.0] * len(CLASSIC_NEEDLES), device_name) + + with pytest.raises(Exception): + engine.score_bm25(haystacks, uniform, 0.0, device=device) + + # Opting out of normalization is how a caller works without a mean, and then zero is accepted. + unnormalized = engine.score_bm25(haystacks, uniform, 0.0, device=device, length_normalization=0.0) + assert unnormalized[1] == 0.0 and unnormalized[0] > 0.0 + + +# endregion Scoring + + +# region Rewriting + + +@pytest.mark.parametrize("policy", COVER_POLICIES) +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_replace(device_name: str, policy: str): + """Each match is substituted by its needle's replacement under a cover, and the offsets delimit one + rewritten haystack each. Both covers commit `she` in `ushers`, so both rewrite it the same way.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + haystacks = Strs(CLASSIC_HAYSTACKS) + replacements = Strs(["[HE]", "[SHE]", "[HIS]", "[HERS]"]) + + data, offsets = engine.replace(haystacks, replacements, device=device, policy=policy) + assert len(offsets) == len(CLASSIC_HAYSTACKS) + 1 + pieces = [data[offsets[i] : offsets[i + 1]].decode() for i in range(len(CLASSIC_HAYSTACKS))] + assert pieces[0] == "u[SHE]rs" + assert pieces[1] == "nothing" + # `hishers` commits `his` at 0 either way, then the widest match at 3 under longest and the lowest + # needle index there under first. + assert pieces[2] == ("[HIS][HERS]" if policy == "leftmost-longest" else "[HIS][HE]rs") + + +@pytest.mark.parametrize("policy", COVER_POLICIES) +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_replace_identity(device_name: str, policy: str): + """Replacing every needle with itself must reproduce the input byte for byte - an oracle that needs + no second implementation to be trustworthy, and one that holds under either cover.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + haystacks = Strs(CLASSIC_HAYSTACKS) + + data, offsets = engine.replace(haystacks, Strs(CLASSIC_NEEDLES), device=device, policy=policy) + pieces = [data[offsets[i] : offsets[i + 1]].decode() for i in range(len(CLASSIC_HAYSTACKS))] + assert pieces == CLASSIC_HAYSTACKS + + +def test_substrings_replace_deletes_on_empty(): + """An empty replacement deletes its match rather than being refused.""" + engine = szs.Substrings(Strs(["cat"])) + data, offsets = engine.replace(Strs(["a cat here"]), Strs([""])) + assert data[offsets[0] : offsets[1]].decode() == "a here" + + +def test_substrings_replace_inserts_verbatim_under_folding(): + """No case adaptation: every fold preimage takes the replacement's exact bytes, so a rewrite over a + 3-byte Kelvin sign shrinks.""" + engine = szs.Substrings(Strs(["k"]), case_sensitivity="uncased") + data, offsets = engine.replace(Strs(["Kâ„Ēk"]), Strs(["x"])) + assert data[offsets[0] : offsets[1]] == b"xxx" + + +def test_substrings_replace_bound_covers_the_widest_expansion(): + """The bound is arithmetic over the needle set, so sizing a tape to it makes the rewrite a single + call that cannot be refused.""" + engine = szs.Substrings(Strs(CLASSIC_NEEDLES)) + haystacks = Strs(CLASSIC_HAYSTACKS) + replacements = Strs(["[HE]", "[SHE]", "[HIS]", "[HERS]"]) + + input_bytes = sum(len(h) for h in CLASSIC_HAYSTACKS) + bound = engine.replace_bound(replacements, input_bytes) + data, _ = engine.replace(haystacks, replacements) + assert bound >= len(data) and bound >= input_bytes + + # "a" expands 1 byte into 4 and "bb" expands 2 into 4, so the widest ratio is 4 and a fully tiled + # 8-byte haystack cannot exceed 32; a dictionary that only shrinks still bounds at the input length. + assert szs.Substrings(Strs(["a", "bb"])).replace_bound(Strs(["wxyz", "wxyz"]), 8) == 32 + assert szs.Substrings(Strs(["aaaa"])).replace_bound(Strs(["z"]), 8) == 8 + + +# endregion Rewriting + + +# region Corner cases + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_empty_and_missing(device_name: str): + """A corpus with no match scores zero everywhere, and an empty needle is refused outright.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + + counts, total = engine.count(Strs(["xyz", "abc"]), device=device) + assert total == 0 and list(int(c) for c in counts) == [0, 0] + assert len(engine.find(Strs(["xyz"]), device=device)[0]) == 0 + + with pytest.raises(Exception): + szs.Substrings(Strs(["ok", ""]), device=device, capabilities=capabilities) + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_empty_batch(device_name: str): + """A batch of no haystacks is a legal batch: every verb returns an empty result rather than failing.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(CLASSIC_NEEDLES), device=device, capabilities=capabilities) + empty = Strs([]) + + counts, total = engine.count(empty, device=device) + assert len(counts) == 0 and total == 0 + assert all(len(column) == 0 for column in engine.find(empty, device=device)) + weights = device_float32_array([1.0] * len(CLASSIC_NEEDLES), device_name) + assert len(engine.score_bm25(empty, weights, 6.0, device=device)) == 0 + + data, offsets = engine.replace(empty, Strs(CLASSIC_NEEDLES), device=device) + assert len(data) == 0 and list(offsets) == [0] + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_degenerate_shapes(device_name: str): + """One-byte haystacks, an all-same run that tiles, and a needle set of nested prefixes - the shapes + that put a match at every position or none.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(["a", "aa", "aaa"]), device=device, capabilities=capabilities) + + counts, _ = engine.count(Strs(["", "a", "aaaa"]), device=device, policy="overlapping") + # "aaaa" holds "a" four times, "aa" three times and "aaa" twice. + assert list(int(c) for c in counts) == [0, 1, 9] + + # A cover tiles the same haystack with the widest needle it can commit at each start, then resumes + # past it: "aaa" at 0, and the single "a" left over at 3. + cover = matches_as_tuples(engine, Strs(["aaaa"]), device=device, policy="leftmost-longest") + assert cover == [(0, 0, 3, 1), (0, 2, 0, 3)] + + +@pytest.mark.parametrize("device_name", DEVICE_NAMES) +def test_substrings_large_haystacks(device_name: str): + """A haystack far past any per-core slice, so matches that straddle a slice boundary are exercised. + Every backend must reach the count a single-threaded scan of the same text reaches.""" + device, capabilities = device_scope_and_capabilities(device_name) + engine = szs.Substrings(Strs(["cat", "dog", "concatenate"]), device=device, capabilities=capabilities) + + unit = "cat dog concatenate at the " + text = unit * (256 * 1024 // len(unit)) + # "concatenate" contains "cat", and each unit holds one of each plus the "cat" inside "concatenate". + per_unit = 4 + repeats = len(text) // len(unit) + + counts, total = engine.count(Strs([text, text]), device=device, policy="overlapping") + assert list(int(c) for c in counts) == [per_unit * repeats, per_unit * repeats] + assert total == 2 * per_unit * repeats + + # Self-replacement must survive every slice boundary byte for byte. + data, offsets = engine.replace(Strs([text]), Strs(["cat", "dog", "concatenate"]), device=device) + assert data[offsets[0] : offsets[1]].decode() == text + + +# endregion Corner cases + + +# region Backend differential + + +@pytest.mark.parametrize("policy", POLICIES) +@pytest.mark.parametrize("seed_value", SEED_VALUES) +def test_substrings_backend_differential(seed_value: int, policy: str): + """Every scope must reach the answer the default scope reaches, over a randomized corpus. The scopes + differ in how a batch is split across cores and devices, never in what a walk reports.""" + seed_random_generators(seed_value) + rng = random.Random(seed_value) + needles = sorted( + {"".join(rng.choice("abcdefg") for _ in range(rng.randint(1, 4))) for _ in range(scale_iterations(12))} + ) + corpus = random_ascii_corpus(rng, documents=24, length=200) + haystacks = Strs(corpus) + + reference = None + for name, device, capabilities in DIFFERENTIAL_DEVICE_SCOPES: + engine = szs.Substrings(Strs(needles), device=device, capabilities=capabilities) + counts, total = engine.count(haystacks, device=device, policy=policy) + answer = ( + list(int(c) for c in counts), + total, + matches_as_tuples(engine, haystacks, device=device, policy=policy), + ) + if reference is None: + reference, reference_name = answer, name + assert answer[1] == sum(answer[0]), "The total must be the sum of the per-haystack counts" + assert len(answer[2]) == total, "Every counted match must be locatable" + else: + assert answer == reference, f"{name} disagrees with {reference_name}" + + +@pytest.mark.parametrize("seed_value", SEED_VALUES) +def test_substrings_covers_are_subsets_of_the_overlapping_walk(seed_value: int): + """Over a randomized corpus, each cover is a disjoint subset of the overlapping walk - the property + the policies are defined by, checked where hand-pinned fixtures cannot reach.""" + seed_random_generators(seed_value) + rng = random.Random(seed_value) + needles = sorted( + {"".join(rng.choice("abc") for _ in range(rng.randint(1, 4))) for _ in range(scale_iterations(10))} + ) + haystacks = Strs(random_ascii_corpus(rng, documents=16, length=120, alphabet="abc ")) + engine = szs.Substrings(Strs(needles)) + + overlapping = set(matches_as_tuples(engine, haystacks, policy="overlapping")) + for policy in COVER_POLICIES: + cover = matches_as_tuples(engine, haystacks, policy=policy) + assert set(cover) <= overlapping, f"{policy} reported a match the overlapping walk did not" + spans = {} + for haystack_index, _, offset, length in cover: + taken = spans.setdefault(haystack_index, []) + for other_offset, other_length in taken: + assert offset + length <= other_offset or other_offset + other_length <= offset + taken.append((offset, length)) + + +@pytest.mark.parametrize("seed_value", SEED_VALUES) +def test_substrings_replace_identity_over_random_corpora(seed_value: int): + """Self-replacement is a byte-exact identity for any needle set and any corpus, so a randomized sweep + of it catches every off-by-one a fixed fixture would not.""" + seed_random_generators(seed_value) + rng = random.Random(seed_value) + needles = sorted( + {"".join(rng.choice("abcd") for _ in range(rng.randint(1, 3))) for _ in range(scale_iterations(8))} + ) + corpus = random_ascii_corpus(rng, documents=12, length=150, alphabet="abcd ") + engine = szs.Substrings(Strs(needles)) + + for policy in COVER_POLICIES: + data, offsets = engine.replace(Strs(corpus), Strs(needles), policy=policy) + pieces = [data[offsets[i] : offsets[i + 1]].decode() for i in range(len(corpus))] + assert pieces == corpus, f"self-replacement changed the corpus under {policy}" + + +def test_substrings_malformed_utf8(): + """Cased matching is over bytes and accepts anything; uncased matching is over codepoints and refuses + a needle that is not well-formed UTF-8. Neither may crash on a malformed haystack.""" + malformed = malformed_utf8_corpus() + + cased = szs.Substrings(Strs([b"\xff\xfe", b"ab"])) + counts, total = cased.count(Strs(malformed)) + assert total == sum(int(c) for c in counts) + + for needle in malformed: + try: + needle.decode("utf-8") + except UnicodeDecodeError: + with pytest.raises(Exception): + szs.Substrings(Strs([needle]), case_sensitivity="uncased") + else: + # An embedded NUL is a legal codepoint, so a well-formed needle is accepted whatever it holds. + szs.Substrings(Strs([needle]), case_sensitivity="uncased") + + # A well-formed uncased dictionary over malformed haystacks must still answer without crashing. + uncased = szs.Substrings(Strs(["ab"]), case_sensitivity="uncased") + uncased.count(Strs(malformed)) + + +# endregion Backend differential + + +# region Third-party differential + + +@pytest.mark.parametrize("seed_value", SEED_VALUES) +def test_stress_matches_pyahocorasick(seed_value: int): + """An independent C automaton must report the same overlapping match set over a randomized corpus. + + ASCII only, since `pyahocorasick` indexes a `str` by codepoint while this engine reports bytes. Only + `Automaton.iter` is used as an oracle: its `iter_long` drops a match it has already banked when the + input ends mid-way through a longer candidate - with needles `a` and `dace` over `"da"` it reports + nothing where `iter` reports `a` - so it does not witness the leftmost-longest cover. The Rust suite + differentials all three covers against the `aho-corasick` crate instead. + """ + ahocorasick = pytest.importorskip("ahocorasick") + seed_random_generators(seed_value) + rng = random.Random(seed_value) + needles = sorted( + {"".join(rng.choice("abcde") for _ in range(rng.randint(1, 4))) for _ in range(scale_iterations(12))} + ) + corpus = random_ascii_corpus(rng, documents=20, length=180, alphabet="abcde ") + engine = szs.Substrings(Strs(needles)) + + automaton = ahocorasick.Automaton() + for index, needle in enumerate(needles): + automaton.add_word(needle, (index, len(needle))) + automaton.make_automaton() + + oracle = sorted( + (haystack_index, needle_index, end_index - needle_length + 1, needle_length) + for haystack_index, haystack in enumerate(corpus) + for end_index, (needle_index, needle_length) in automaton.iter(haystack) + ) + assert matches_as_tuples(engine, Strs(corpus), policy="overlapping") == oracle + + +# endregion Third-party differential + + +# region Interop + + +def test_substrings_needle_layouts_agree(): + """Needles reach the engine as a callback-addressed sequence whatever layout they arrived in, so a + tape-backed `Strs` and a sliced one must compile to the same automaton.""" + tape = Strs(CLASSIC_NEEDLES) + fragmented = Strs(CLASSIC_NEEDLES + ["unused"])[:4] + haystacks = Strs(CLASSIC_HAYSTACKS) + + from_tape = szs.Substrings(tape) + from_fragmented = szs.Substrings(fragmented) + assert matches_as_tuples(from_tape, haystacks) == matches_as_tuples(from_fragmented, haystacks) + assert from_tape.count(haystacks)[1] == from_fragmented.count(haystacks)[1] + + +def test_substrings_haystack_layouts_agree(): + """Haystacks reach three different C entry points - a 32-bit tape, a 64-bit one, and the callback + addressed sequence a reordered `Strs` becomes - and all three must report one answer.""" + engine = szs.Substrings(Strs(CLASSIC_NEEDLES)) + tape = Strs(CLASSIC_HAYSTACKS) + # A contiguous slice stays a tape view, so only a reordering step makes the fragmented layout; two + # reversals restore the order while leaving the layout behind. + fragmented = Strs(CLASSIC_HAYSTACKS)[::-1][::-1] + + assert matches_as_tuples(engine, tape) == matches_as_tuples(engine, fragmented) + assert engine.count(tape)[1] == engine.count(fragmented)[1] + + # Rewriting is tape in, tape out, so the reordered layout is refused rather than silently reordered. + with pytest.raises(TypeError): + engine.replace(fragmented, Strs(CLASSIC_NEEDLES)) + + +# endregion Interop + + +# region Argument surface + + +def test_substrings_rejects_wrong_types(): + """Every entry point names the conversion a caller needs rather than failing obscurely.""" + engine = szs.Substrings(Strs(CLASSIC_NEEDLES)) + with pytest.raises(TypeError): + engine.count(["not", "a", "Strs"]) + with pytest.raises(ValueError): + engine.count(Strs(CLASSIC_HAYSTACKS), policy="sideways") + with pytest.raises(ValueError): + szs.Substrings(Strs(CLASSIC_NEEDLES), case_sensitivity="sideways") + with pytest.raises(TypeError): + engine.count(Strs(CLASSIC_HAYSTACKS), device="not a scope") + with pytest.raises(ValueError): + engine.replace(Strs(CLASSIC_HAYSTACKS), Strs(CLASSIC_NEEDLES), policy="overlapping") + + +def test_substrings_repr_and_capabilities(): + """`repr` names the dictionary's shape, and the engine reports what it was built for.""" + engine = szs.Substrings(Strs(CLASSIC_NEEDLES)) + assert "4 needles" in repr(engine) and "cased" in repr(engine) + assert isinstance(engine.__capabilities__, tuple) + + +# endregion Argument surface + + +if __name__ == "__main__": + sys.exit(pytest.main(["-x", "-s", __file__])) diff --git a/test/sz_helpers.py b/test/sz_helpers.py index 52271288..c0d36cf6 100644 --- a/test/sz_helpers.py +++ b/test/sz_helpers.py @@ -1299,7 +1299,7 @@ def representatives_by_class( # region General test scaffolding # General fixtures and seeded-RNG helpers shared by every per-family test module (the Python analog of the -# C++ `test_stringzilla.hpp` harness). Kept here so a split test file imports one place for both the Unicode +# C++ `test/stringzilla.hpp` harness). Kept here so a split test file imports one place for both the Unicode # data loaders above and the seeding / random-string utilities below. The NumPy / PyArrow availability flags # and their defensive imports live in the top import block. @@ -1307,7 +1307,7 @@ def representatives_by_class( # `SystemRandom` gives true randomness independent of the seeded RNG state. _random_seed_for_run = int.from_bytes(os.urandom(4), "little") -# Reproducible test seeds for consistent CI runs (kept in sync with test_stringzillas.py). +# Reproducible test seeds for consistent CI runs (kept in sync with test/stringzillas.py). SEED_VALUES = [ 42, # Classic test seed 0, # Edge case: zero seed diff --git a/test/szs_helpers.py b/test/szs_helpers.py index 43f10952..775f5aef 100644 --- a/test/szs_helpers.py +++ b/test/szs_helpers.py @@ -24,6 +24,15 @@ def device_scope_and_capabilities(device: DeviceName): raise ValueError(f"Unknown device type: {device}") +def device_float32_array(values, device: DeviceName): + """One float32 array per scope: unified memory on a GPU scope, which refuses host buffers.""" + if device != "gpu_device": + return np.asarray(values, dtype=np.float32) + array = szs.unified_array(len(values), dtype=np.float32) + array[:] = values + return array + + InputSizeConfig = Literal["one-large", "few-big", "many-small"] INPUT_SIZE_CONFIGS = ["one-large", "few-big", "many-small"] diff --git a/test/uncased.cpp b/test/uncased.cpp index 4998c863..72ea25f3 100644 --- a/test/uncased.cpp +++ b/test/uncased.cpp @@ -1,6 +1,6 @@ /** * @brief Uncased UTF-8 case-folding equivalence/fuzzing and uncased substring search tests. - * @file scripts/test_uncased.cpp + * @file test/uncased.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -49,26 +49,17 @@ #include <cstdio> // `std::printf` #include <cstring> // `std::memcpy` -#include <algorithm> // `std::transform` -#include <array> // `std::array` -#include <iterator> // `std::distance` -#include <map> // `std::map` -#include <memory> // `std::allocator` -#include <numeric> // `std::accumulate` -#include <random> // `std::random_device` -#include <set> // `std::set` -#include <sstream> // `std::ostringstream` -#include <string> // Baseline -#include <string_view> // Baseline -#include <unordered_map> // `std::unordered_map` -#include <unordered_set> // `std::unordered_set` -#include <vector> // `std::vector` +#include <array> // `std::array` +#include <random> // `std::uniform_int_distribution` +#include <string> // Baseline +#include <string_view> // Baseline +#include <vector> // `std::vector` #if !SZ_IS_CPP11_ #error "This test requires C++11 or later." #endif -#include "stringzilla.hpp" // `global_random_generator`, `random_string` +#include "utf8.hpp" // `print_utf8_test_bytes_`, `encoded_rune_` namespace sz = ashvardanian::stringzilla; using namespace sz::scripts; @@ -104,13 +95,6 @@ static void check_uncased_fold_unit_( // verify(std::memcmp(produced, expected, expected_length) == 0); } -/** @brief Prints one labeled hex dump line to `stderr`; used by the adversarial UTF-8 case tests below. */ -static void print_uncased_test_bytes_(char const *label, char const *bytes, std::size_t length) { - std::fprintf(stderr, " %s (%zu bytes): ", label, length); - for (std::size_t i = 0; i < length; ++i) std::fprintf(stderr, "%02X ", (unsigned char)bytes[i]); - std::fprintf(stderr, "\n"); -} - /** * @brief Independent ground-truth uncased search: `fold(needle)` as a contiguous run of `fold(haystack)`. * @@ -229,8 +213,8 @@ static void check_uncased_find_three_way_( // stderr, "%s FAIL: base offset=%ld len=%zu | simd offset=%ld len=%zu kernel=%u | reference offset=%ld len=%zu\n", test_name, base_offset, (std::size_t)base_matched, simd_offset, (std::size_t)simd_matched, simd_metadata.kernel_id, reference_offset, (std::size_t)reference_matched); - print_uncased_test_bytes_("needle ", needle, needle_length); - print_uncased_test_bytes_("haystack", haystack, haystack_length); + print_utf8_test_bytes_("needle ", needle, needle_length); + print_utf8_test_bytes_("haystack", haystack, haystack_length); verify(base_matches_reference && "Uncased find base backend disagrees with the reference"); verify(simd_matches_reference && "Uncased find SIMD backend disagrees with the reference"); verify(base_matches_simd && "Uncased find backends disagree with each other"); @@ -254,15 +238,15 @@ static void check_uncased_find_three_way_( // * @param max_needles_per_haystack 0 = exhaustive, >0 = sample this many per haystack * @param total_queries Total needle searches to perform across all haystacks */ -static void test_uncased_find_fuzz(sz_utf8_uncased_search_t find_serial, sz_utf8_uncased_search_t find_simd, - sz_utf8_uncased_fold_t uncased_fold, sz_utf8_seek_t utf8_seek, - sz_utf8_count_t utf8_count, std::size_t haystack_length, - std::size_t max_needles_per_haystack, std::size_t total_queries) { +static void check_uncased_find_fuzz_(sz_utf8_uncased_search_t find_serial, sz_utf8_uncased_search_t find_simd, + sz_utf8_uncased_fold_t uncased_fold, sz_utf8_seek_t utf8_seek, + sz_utf8_count_t utf8_count, std::size_t haystack_length, + std::size_t max_needles_per_haystack, std::size_t total_queries) { char const *mode = max_needles_per_haystack == 0 ? "exhaustive" : "sampled"; std::printf(" - fuzz testing (%s, haystack_len=%zu, queries=%zu)...\n", mode, haystack_length, total_queries); - auto &rng = global_random_generator(); + auto &generator = global_random_generator(); // Character pool with normal + weird Unicode characters from safety profiles char const *char_pool[] = { @@ -448,7 +432,7 @@ static void test_uncased_find_fuzz(sz_utf8_uncased_search_t find_serial, sz_utf8 while (queries_remaining > 0) { // 1. Generate random haystack of ~haystack_length bytes haystack.clear(); - while (haystack.size() < haystack_length) haystack += char_pool[pool_dist(rng)]; + while (haystack.size() < haystack_length) haystack += char_pool[pool_dist(generator)]; // 2. Case-fold the haystack - expands up to 3x haystack_folded.resize(haystack.size() * 3); @@ -533,9 +517,9 @@ static void test_uncased_find_fuzz(sz_utf8_uncased_search_t find_serial, sz_utf8 // Sampled mode: random (start, length) pairs std::uniform_int_distribution<sz_size_t> start_dist(0, runes_in_folded_haystack - 1); for (std::size_t i = 0; i < needles_in_this_haystack && queries_remaining > 0; ++i) { - sz_size_t start = start_dist(rng); + sz_size_t start = start_dist(generator); std::uniform_int_distribution<sz_size_t> rune_count_dist(1, runes_in_folded_haystack - start); - if (test_needle(start, rune_count_dist(rng))) { + if (test_needle(start, rune_count_dist(generator))) { ++total_passed; --queries_remaining; } @@ -806,8 +790,8 @@ static void check_uncased_find_crossing_(sz_utf8_uncased_search_t find_base, sz_ * short helpers - swept across the 64-byte chunk boundary, each result checked against the independent * fold-subset reference via the three-way `check_uncased_find_three_way_`. */ -static void test_uncased_find_long_crossing_fuzz(sz_utf8_uncased_search_t find_base, - sz_utf8_uncased_search_t find_simd) { +static void check_uncased_find_long_crossing_fuzz_(sz_utf8_uncased_search_t find_base, + sz_utf8_uncased_search_t find_simd) { std::printf(" - testing uncased find across long (Rabin-Karp) expansion runs...\n"); struct expander_t { @@ -867,21 +851,21 @@ static void test_uncased_find_long_crossing_fuzz(sz_utf8_uncased_search_t find_b * cross-expansion needles) of `find_simd` against the serial baseline and the independent reference. * Called once per backend so coverage stays uniform and a new backend cannot silently skip a test. */ -static void run_uncased_find_battery_(sz_utf8_uncased_search_t find_simd) { +static void check_uncased_find_battery_(sz_utf8_uncased_search_t find_simd) { sz_utf8_uncased_search_t const find_serial = sz_utf8_uncased_search_serial; std::size_t const queries = scale_iterations(8000); - test_uncased_find_fuzz(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, - sz_utf8_count_serial, 16, 0, queries); - test_uncased_find_fuzz(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, - sz_utf8_count_serial, 32, 0, queries); - test_uncased_find_fuzz(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, - sz_utf8_count_serial, 100, 100, queries); - test_uncased_find_fuzz(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, - sz_utf8_count_serial, 200, 100, queries); + check_uncased_find_fuzz_(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, + sz_utf8_count_serial, 16, 0, queries); + check_uncased_find_fuzz_(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, + sz_utf8_count_serial, 32, 0, queries); + check_uncased_find_fuzz_(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, + sz_utf8_count_serial, 100, 100, queries); + check_uncased_find_fuzz_(find_serial, find_simd, sz_utf8_uncased_fold_serial, sz_utf8_seek_serial, + sz_utf8_count_serial, 200, 100, queries); check_uncased_find_preimages_(find_serial, find_simd); check_uncased_find_tails_(find_serial, find_simd); check_uncased_find_crossing_(find_serial, find_simd); - test_uncased_find_long_crossing_fuzz(find_serial, find_simd); + check_uncased_find_long_crossing_fuzz_(find_serial, find_simd); // A long ASCII needle (well past the 32-rune ring buffer and the 3-rune short helpers) drives the // pure Rabin-Karp path inside a large random haystack, both where the needle was spliced in (so a @@ -917,18 +901,11 @@ static void run_uncased_find_battery_(sz_utf8_uncased_search_t find_simd) { #pragma region Unit /** - * @brief Known-answer + C++ API coverage for the uncased UTF-8 family on simple, hand-verifiable inputs. + * @brief Known-answer battery for a single `sz_utf8_uncased_order` backend: case-insensitive equality, + * ASCII less/greater, length-prefix ordering, and 2-byte accented folds (Ãļ = C3 B6, Ê = C3 A9). * - * First exercises each function through the dispatched C API (automatic kernel resolution), through the - * natively-compiled backend kernels directly (manual propagation to a specific kernel), and through the - * C++ wrappers, so a regression that the serial-vs-SIMD agreement tests would miss - because both share - * a wrong constant - is still caught against an external, hand-derived ground truth. It then sweeps a - * broad battery of cross-script C++ wrapper cases: ordering, finding, ligatures, expansions, and - * SIMD-boundary regressions discovered by earlier fuzzing. + * Each compiled per-ISA kernel is run through it directly, mirroring the per-ISA `_search` coverage. */ -// Known-answer battery for a single `sz_utf8_uncased_order` backend: case-insensitive equality, ASCII -// less/greater, length-prefix ordering, and 2-byte accented folds (Ãļ = C3 B6, Ê = C3 A9). Each compiled -// per-ISA kernel is run through it directly, mirroring the per-ISA `_search` coverage. static void check_uncased_order_(sz_utf8_uncased_order_t order) { verify(order("Hello", 5, "HELLO", 5) == sz_equal_k); verify(order("abc", 3, "abd", 3) == sz_less_k); @@ -938,6 +915,14 @@ static void check_uncased_order_(sz_utf8_uncased_order_t order) { verify(order("caf\xC3\xA9", 5, "CAF\xC3\x89", 5) == sz_equal_k); // 'Ê' fold (2-byte) } +/** + * @brief Known-answer + C++ API coverage for the uncased UTF-8 family on simple, hand-verifiable inputs. + * + * First exercises each function through the dispatched C API (automatic kernel resolution), through the + * natively-compiled backend kernels directly (manual propagation to a specific kernel), and through the + * C++ wrappers, so a regression that the serial-vs-SIMD agreement tests would miss - because both share + * a wrong constant - is still caught against an external, hand-derived ground truth. + */ void test_uncased_unit() { using str = sz::string_view; @@ -947,33 +932,29 @@ void test_uncased_unit() { // `sz_utf8_uncased_search`: "world" matches case-insensitively in "Hello World" at byte offset 6, length 5. char const *greeting = "Hello World"; sz_size_t const greeting_length = (sz_size_t)std::strlen(greeting); - check_uncased_find_unit_(sz_utf8_uncased_search, greeting, greeting_length, // Dispatched (automatic kernel) - "world", 5, 6, 5); - check_uncased_find_unit_(sz_utf8_uncased_search_serial, greeting, greeting_length, // Manual: serial kernel - "world", 5, 6, 5); + // Dispatched (automatic kernel resolution). + check_uncased_find_unit_(sz_utf8_uncased_search, greeting, greeting_length, "world", 5, 6, 5); + // Manual propagation to each natively-compiled backend kernel. + check_uncased_find_unit_(sz_utf8_uncased_search_serial, greeting, greeting_length, "world", 5, 6, 5); #if SZ_USE_HASWELL - check_uncased_find_unit_(sz_utf8_uncased_search_haswell, greeting, greeting_length, // Manual: haswell kernel - "world", 5, 6, 5); + check_uncased_find_unit_(sz_utf8_uncased_search_haswell, greeting, greeting_length, "world", 5, 6, 5); #endif #if SZ_USE_ICELAKE - check_uncased_find_unit_(sz_utf8_uncased_search_icelake, greeting, greeting_length, // Manual: icelake kernel - "world", 5, 6, 5); + check_uncased_find_unit_(sz_utf8_uncased_search_icelake, greeting, greeting_length, "world", 5, 6, 5); #endif // `sz_utf8_uncased_search`: 'ß' (U+00DF, C3 9F) folds to "ss", so needle "SS" matches the whole 2-byte 'ß'. char const *sharp_s = "\xC3\x9F" "fox"; // 'ß' followed by "fox" → folds to "ssfox" sz_size_t const sharp_s_length = (sz_size_t)std::strlen(sharp_s); - check_uncased_find_unit_(sz_utf8_uncased_search, sharp_s, sharp_s_length, // Dispatched (automatic kernel) - "SS", 2, 0, 2); - check_uncased_find_unit_(sz_utf8_uncased_search_serial, sharp_s, sharp_s_length, // Manual: serial kernel - "SS", 2, 0, 2); + // Dispatched (automatic kernel resolution). + check_uncased_find_unit_(sz_utf8_uncased_search, sharp_s, sharp_s_length, "SS", 2, 0, 2); + // Manual propagation to each natively-compiled backend kernel. + check_uncased_find_unit_(sz_utf8_uncased_search_serial, sharp_s, sharp_s_length, "SS", 2, 0, 2); #if SZ_USE_HASWELL - check_uncased_find_unit_(sz_utf8_uncased_search_haswell, sharp_s, sharp_s_length, // Manual: haswell kernel - "SS", 2, 0, 2); + check_uncased_find_unit_(sz_utf8_uncased_search_haswell, sharp_s, sharp_s_length, "SS", 2, 0, 2); #endif #if SZ_USE_ICELAKE - check_uncased_find_unit_(sz_utf8_uncased_search_icelake, sharp_s, sharp_s_length, // Manual: icelake kernel - "SS", 2, 0, 2); + check_uncased_find_unit_(sz_utf8_uncased_search_icelake, sharp_s, sharp_s_length, "SS", 2, 0, 2); #endif // C++ wrapper on `sz::string_view`: same two cases through `utf8_uncased_search`. @@ -997,13 +978,15 @@ void test_uncased_unit() { } // `sz_utf8_uncased_fold`: "HeLLo" → "hello", and 'ß' (U+00DF) → "ss". - check_uncased_fold_unit_(sz_utf8_uncased_fold, "HeLLo", 5, "hello"); // Dispatched (automatic kernel) - check_uncased_fold_unit_(sz_utf8_uncased_fold_serial, "HeLLo", 5, "hello"); // Manual: serial kernel - check_uncased_fold_unit_(sz_utf8_uncased_fold, "\xC3\x9F", 2, "ss"); // Dispatched (automatic kernel) - check_uncased_fold_unit_(sz_utf8_uncased_fold_serial, "\xC3\x9F", 2, "ss"); // Manual: serial kernel + // Dispatched (automatic kernel resolution). + check_uncased_fold_unit_(sz_utf8_uncased_fold, "HeLLo", 5, "hello"); + // Manual propagation to each natively-compiled backend kernel. + check_uncased_fold_unit_(sz_utf8_uncased_fold_serial, "HeLLo", 5, "hello"); + check_uncased_fold_unit_(sz_utf8_uncased_fold, "\xC3\x9F", 2, "ss"); + check_uncased_fold_unit_(sz_utf8_uncased_fold_serial, "\xC3\x9F", 2, "ss"); #if SZ_USE_ICELAKE - check_uncased_fold_unit_(sz_utf8_uncased_fold_icelake, "HeLLo", 5, "hello"); // Manual: icelake kernel - check_uncased_fold_unit_(sz_utf8_uncased_fold_icelake, "\xC3\x9F", 2, "ss"); // Manual: icelake kernel + check_uncased_fold_unit_(sz_utf8_uncased_fold_icelake, "HeLLo", 5, "hello"); + check_uncased_fold_unit_(sz_utf8_uncased_fold_icelake, "\xC3\x9F", 2, "ss"); #endif // C++ wrapper: in-place fold on a mutable `sz::string`. @@ -1019,29 +1002,34 @@ void test_uncased_unit() { } // `sz_utf8_uncased_order`: "Hello" and "HELLO" compare equal ignoring case. - verify(sz_utf8_uncased_order("Hello", 5, "HELLO", 5) == sz_equal_k); // Dispatched (automatic kernel) - verify(sz_utf8_uncased_order_serial("Hello", 5, "HELLO", 5) == sz_equal_k); // Manual: serial kernel - check_uncased_order_(sz_utf8_uncased_order_serial); // serial battery + // Dispatched (automatic kernel resolution). + verify(sz_utf8_uncased_order("Hello", 5, "HELLO", 5) == sz_equal_k); + // Manual propagation to each natively-compiled backend kernel. + verify(sz_utf8_uncased_order_serial("Hello", 5, "HELLO", 5) == sz_equal_k); + check_uncased_order_(sz_utf8_uncased_order_serial); // serial battery #if SZ_USE_HASWELL - check_uncased_order_(sz_utf8_uncased_order_haswell); // Manual: haswell kernel + check_uncased_order_(sz_utf8_uncased_order_haswell); #endif #if SZ_USE_ICELAKE - check_uncased_order_(sz_utf8_uncased_order_icelake); // Manual: icelake kernel + check_uncased_order_(sz_utf8_uncased_order_icelake); #endif #if SZ_USE_NEON - check_uncased_order_(sz_utf8_uncased_order_neon); // Manual: neon kernel + check_uncased_order_(sz_utf8_uncased_order_neon); +#endif +#if SZ_USE_SVE2 + check_uncased_order_(sz_utf8_uncased_order_sve2); #endif #if SZ_USE_V128 - check_uncased_order_(sz_utf8_uncased_order_v128); // Manual: v128 kernel + check_uncased_order_(sz_utf8_uncased_order_v128); #endif #if SZ_USE_RVV - check_uncased_order_(sz_utf8_uncased_order_rvv); // Manual: rvv kernel + check_uncased_order_(sz_utf8_uncased_order_rvv); #endif #if SZ_USE_LASX - check_uncased_order_(sz_utf8_uncased_order_lasx); // Manual: lasx kernel + check_uncased_order_(sz_utf8_uncased_order_lasx); #endif #if SZ_USE_POWERVSX - check_uncased_order_(sz_utf8_uncased_order_powervsx); // Manual: powervsx kernel + check_uncased_order_(sz_utf8_uncased_order_powervsx); #endif verify(str("Hello").utf8_uncased_order("HELLO") == sz_equal_k); // C++ wrapper @@ -1049,19 +1037,23 @@ void test_uncased_unit() { // "äģˇæ ŧ 123" is caseless (CJK + digits + space), so no rune participates in case → NULL. char const *caseless = "\xE4\xBB\xB7\xE6\xA0\xBC 123"; // "äģˇæ ŧ 123" sz_size_t const caseless_length = (sz_size_t)std::strlen(caseless); - verify(sz_utf8_find_cased(caseless, caseless_length) == SZ_NULL_CHAR); // Dispatched (automatic kernel) - verify(sz_utf8_find_cased_serial(caseless, caseless_length) == SZ_NULL_CHAR); // Manual: serial kernel + // Dispatched (automatic kernel resolution). + verify(sz_utf8_find_cased(caseless, caseless_length) == SZ_NULL_CHAR); + // Manual propagation to each natively-compiled backend kernel. + verify(sz_utf8_find_cased_serial(caseless, caseless_length) == SZ_NULL_CHAR); #if SZ_USE_ICELAKE - verify(sz_utf8_find_cased_icelake(caseless, caseless_length) == SZ_NULL_CHAR); // Manual: icelake kernel + verify(sz_utf8_find_cased_icelake(caseless, caseless_length) == SZ_NULL_CHAR); #endif // "123Abc" has its first cased codepoint 'A' at byte offset 3. char const *mixed = "123Abc"; sz_size_t const mixed_length = (sz_size_t)std::strlen(mixed); - verify(sz_utf8_find_cased(mixed, mixed_length) == mixed + 3); // Dispatched (automatic kernel) - verify(sz_utf8_find_cased_serial(mixed, mixed_length) == mixed + 3); // Manual: serial kernel + // Dispatched (automatic kernel resolution). + verify(sz_utf8_find_cased(mixed, mixed_length) == mixed + 3); + // Manual propagation to each natively-compiled backend kernel. + verify(sz_utf8_find_cased_serial(mixed, mixed_length) == mixed + 3); #if SZ_USE_ICELAKE - verify(sz_utf8_find_cased_icelake(mixed, mixed_length) == mixed + 3); // Manual: icelake kernel + verify(sz_utf8_find_cased_icelake(mixed, mixed_length) == mixed + 3); #endif // A cased rune hiding behind a caseless prefix longer than any SIMD front's block: sixteen @@ -1088,8 +1080,31 @@ void test_uncased_unit() { #endif #if SZ_USE_V128 verify(sz_utf8_find_cased_v128(deep, deep_length) == deep + 64); +#endif +#if SZ_USE_RVV + verify(sz_utf8_find_cased_rvv(deep, deep_length) == deep + 64); +#endif +#if SZ_USE_LASX + verify(sz_utf8_find_cased_lasx(deep, deep_length) == deep + 64); +#endif +#if SZ_USE_POWERVSX + verify(sz_utf8_find_cased_powervsx(deep, deep_length) == deep + 64); #endif } +} + +/** + * @brief Known-answer sweep of the uncased C++ wrappers across the world's scripts. + * + * Ordering, finding, ligatures and expansions over Latin-1, Central European, German Eszett, math + * symbols, Greek, Cyrillic, Turkish, Armenian, Vietnamese, Georgian, Cherokee, Coptic, Glagolitic and + * the caseless scripts - CJK, Arabic, Hebrew and emoji - each with a hand-derived expected offset and + * byte length, including the runs that straddle a 64-byte SIMD block. + */ +void test_uncased_scripts_unit() { + std::printf(" - testing uncased search and order across Unicode scripts...\n"); + + using str = sz::string_view; // Equal strings (ASCII) verify(str("hello").utf8_uncased_order("HELLO") == sz_equal_k); @@ -1545,7 +1560,6 @@ void test_uncased_unit() { // Emoji Context let_verify(auto m = str("smile 😀😁😂").utf8_uncased_search("😁"), m.offset == 10 && m.length == 4); - // Regressions & Complex Cases // "Fuzz Regression": Needle "nÔąÔ˛ÕÔĩÕˇ" (Mixed case Armenian + ASCII) let_verify(auto m = str("nÔąÔ˛ÕÔĩÕˇ").utf8_uncased_search("nÕĄÕĸրÕĨÕˇ"), m.offset == 0 && m.length == 11); @@ -1668,12 +1682,23 @@ void test_uncased_unit() { let_verify(auto m = str("HELLO").utf8_uncased_search("hello"), m.offset == 0 && m.length == 5); let_verify(auto m = str("Hello").utf8_uncased_search("xyz"), m.offset == str::npos); let_verify(auto m = str("Hello").utf8_uncased_search(""), m.offset == 0 && m.length == 0); +} + +/** + * @brief Minimized known-answer vectors pinning serial-vs-SIMD mismatches found by the find fuzzers. + * + * Each numbered pattern pins the smallest input reproducing a serial-vs-SIMD disagreement - ligature and + * Eszett expansions, one-to-many folds with combining marks, ring-buffer-length needles, and runs + * landing on a 64-byte block edge - so the fix stays nailed down at a fixed cost. + */ +void test_uncased_regressions_unit() { + std::printf(" - testing uncased fuzz-discovered regressions...\n"); + + using str = sz::string_view; // Fuzz-Discovered Regressions (Serial vs SIMD mismatches) - // These patterns were discovered by the find fuzzers and expose - // disagreements between serial and SIMD implementations. - // Pattern 0: Ligature tail-match in mixed-case context (historical verify crash). + // Pattern 0: Ligature tail-match in mixed-case context; pins a verify crash on this input. // Haystack: C3 96 45 47 76 C3 91 2C 50 EF AC 84 ... EF AC 82 70 // Needle: 67 76 C3 B1 2C 70 66 { @@ -1848,10 +1873,10 @@ void test_uncased_unit() { } // Minimal Divergence Cases (Ice Lake vs Serial) - // These were discovered by multi-seed fuzzing and represent minimal inputs - // that previously caused Serial/SIMD disagreement. + // Minimal inputs pinning Serial/SIMD agreement on expansion boundaries that generic + // fuzzing rarely hits: a fold that grows the needle mid-match, straddling a kernel's block edge. - // Pattern 7: "sss" prefix matching "Sß" (seed 5678, Kernel 2) + // Pattern 7: "sss" prefix matching "Sß" - a triple-s expansion from the Eszett fold. // Haystack: "brown Sßà jumps" - bytes at [6]: 53 C3 9F C3 A0 ("Sßà") = 5 bytes // Needle: "sssà" - bytes: 73 73 73 C3 A0 = 5 bytes // 'S' → 's', 'ß' → "ss", 'à' → 'à', so "Sßà" → "sssà" (should match!) @@ -1868,7 +1893,7 @@ void test_uncased_unit() { let_verify(auto m = str("brown \xE1\xBA\x9E\xC3\xA0 jumps").utf8_uncased_search("ss\xC3\xA0"), m.offset == 6 && m.length == 5); - // Triple-s with space (seed 1234): "sß " should match "sss " + // Triple-s with space: "sß " should match "sss " // Match starts at byte 7 where 's' is (byte 6 is space before 's') let_verify(auto m = str("\xC7\xB0" "bee3 s\xC3\x9F ee\xC3\xA9 nc").utf8_uncased_search("sss ee\xC3\xA9"), m.offset == 7 && m.length == 8); @@ -1880,10 +1905,10 @@ void test_uncased_unit() { let_verify(auto m = str("s\xC3\x9F" "abc").utf8_uncased_search("sssabc"), m.offset == 0 && m.length == 6); } - // Pattern 8: Greek Mu UTF-8 boundary (seed 300, 1000, 1700, Kernel 5) + // Pattern 8: Greek Mu UTF-8 boundary. // Needle: CE BC (Greek Îŧ - U+03BC) - // Bug: SIMD was incorrectly matching mid-byte BC as standalone - // Fix: Ensure proper UTF-8 character boundary validation + // Pins that a match only lands on a valid UTF-8 character boundary, never on the mid-byte + // BC that also appears as the trailing byte of an unrelated codepoint such as Âŧ. { // Simple Greek mu search let_verify(auto m = str("hello \xCE\xBC world").utf8_uncased_search("\xCE\xBC"), @@ -1900,7 +1925,7 @@ void test_uncased_unit() { m.offset == 2 && m.length == 2); } - // Pattern 9: Cyrillic Moscow case folding (seed 9999, 44444, 55555, Kernel 4) + // Pattern 9: Cyrillic Moscow case folding. // Haystack: "ŅĐĩ МоŅĐēва" (uppercase М - D0 9C) // Needle: "ŅĐĩ ĐŧĐžŅĐēва" (lowercase Đŧ - D0 BC) // Should match uncasedly @@ -1926,7 +1951,7 @@ void test_uncased_unit() { m.offset == 0 && m.length == 12); } - // Pattern 10: Ligature fi expansion (seed 500, Kernel 2) + // Pattern 10: Ligature fi expansion. // Haystack contains īŦ (EF AC 81 - U+FB01) // Needle has "fi" (66 69) // īŦ should case-fold to "fi" @@ -1950,7 +1975,7 @@ void test_uncased_unit() { let_verify(auto m = str("wa\xEF\xAC\x84" "e").utf8_uncased_search("waffle"), m.offset == 0 && m.length == 6); } - // Pattern 11: Combining marks vs precomposed (seed 123, 42, Kernel 2) + // Pattern 11: Combining marks vs precomposed. // j + combining caron (6A CC 8C) vs Į° (C7 B0 - U+01F0) // These are canonically equivalent in Unicode // Note: StringZilla may or may not perform normalization - document behavior @@ -1965,7 +1990,7 @@ void test_uncased_unit() { let_verify(auto m = str("\xC3\xA9" "lan").utf8_uncased_search("e\xCC\x81" "lan"), m.offset == str::npos); } - // Pattern 12: Mixed script verification (seeds 456, 789, 22222, Kernels 3, 5, 6) + // Pattern 12: Mixed script verification. // These test that case folding works correctly when multiple scripts are mixed { // Greek ÎēĪŒĪƒÎŧ mixed with Latin @@ -1997,8 +2022,8 @@ void test_uncased_unit() { * sweep of every valid Unicode codepoint, both in order and shuffled. */ template <typename reference_, typename candidate_> -void test_fold_equivalence(reference_ reference, candidate_ candidate, sz_size_t min_text_length, - sz_size_t min_iterations) { +void check_uncased_fold_equivalence_(reference_ reference, candidate_ candidate, sz_size_t min_text_length, + sz_size_t min_iterations) { // Output buffers (3x input for worst-case expansion) std::vector<char> output_base(min_text_length * 3 + 256); @@ -2104,7 +2129,7 @@ void test_fold_equivalence(reference_ reference, candidate_ candidate, sz_size_t "Hello \xF0\x9F\x8C\x8D World", // Hello 🌍 World }; - auto &rng = global_random_generator(); + auto &generator = global_random_generator(); std::size_t const content_count = span_over(utf8_content).size(); std::uniform_int_distribution<std::size_t> content_dist(0, content_count - 1); @@ -2117,7 +2142,7 @@ void test_fold_equivalence(reference_ reference, candidate_ candidate, sz_size_t // Build up a random string of at least `min_text_length` bytes while (text.size() < min_text_length) { - std::size_t content_index = content_dist(rng); + std::size_t content_index = content_dist(generator); text.append(utf8_content[content_index]); } check(text); @@ -2137,7 +2162,7 @@ void test_fold_equivalence(reference_ reference, candidate_ candidate, sz_size_t std::vector<char> input_buffer(all_runes.size() * 4); // Max UTF-8 size is 4 bytes per rune std::size_t const sweep_iterations = scale_iterations(6); for (std::size_t iteration = 0; iteration < sweep_iterations; ++iteration) { - if (iteration > 0) std::shuffle(all_runes.begin(), all_runes.end(), rng); + if (iteration > 0) std::shuffle(all_runes.begin(), all_runes.end(), generator); char *write_cursor = input_buffer.data(); for (sz_rune_t codepoint : all_runes) write_cursor += sz_rune_encode(codepoint, (sz_u8_t *)write_cursor); @@ -2156,7 +2181,7 @@ void test_fold_equivalence(reference_ reference, candidate_ candidate, sz_size_t * inside a folded expansion (like 'Ęž' U+02BE inside 'áēš' → "aĘž"), so neither may be invariant. * Fully generative: a Unicode table update re-derives the expected set automatically. */ -void test_uncased_invariant_reference() { +void check_uncased_invariant_reference_() { std::printf(" - testing case-invariant closure over the fold table...\n"); std::size_t preimages_checked = 0, outputs_checked = 0; @@ -2208,7 +2233,6 @@ static void check_uncased_safety_(sz::span<uncased_safety_backend_t const> backe std::printf(" - testing invalid-input safety of case kernels (%zu random buffers)...\n", random_inputs); - std::size_t const max_input_length = 70; char const *needle = "st"; // Short valid needle: the folds of 'īŦ…' and 'īŦ†' collapse onto it auto check = [&](char const *input, std::size_t input_length) { @@ -2221,7 +2245,7 @@ static void check_uncased_safety_(sz::span<uncased_safety_backend_t const> backe if (folded_length > length) { std::fprintf(stderr, "%s fold of invalid input returned %zu bytes for %zu input bytes\n", candidate.name, (std::size_t)folded_length, input_length); - print_uncased_test_bytes_("input", input, input_length); + print_utf8_test_bytes_("input", input, input_length); verify(false && "Fold output must stay within 3x the input length plus one mis-decoded rune"); } }); @@ -2234,34 +2258,9 @@ static void check_uncased_safety_(sz::span<uncased_safety_backend_t const> backe } }; - char input[max_input_length]; - - // All 256 single bytes: truncated leads, stray continuations, 0xFE/0xFF - for (std::size_t byte = 0; byte < 256; ++byte) { - input[0] = (char)byte; - check(input, 1); - } - - // All 65,536 byte pairs: every lead × continuation interaction, including overlong shapes - for (std::size_t first_byte = 0; first_byte < 256; ++first_byte) - for (std::size_t second_byte = 0; second_byte < 256; ++second_byte) { - input[0] = (char)first_byte; - input[1] = (char)second_byte; - check(input, 2); - } - - // Random garbage buffers spanning whole SIMD chunks - auto &rng = global_random_generator(); - std::uniform_int_distribution<std::size_t> length_distribution(1, max_input_length); - std::uniform_int_distribution<int> byte_distribution(0, 255); - for (std::size_t iteration = 0; iteration < random_inputs; ++iteration) { - std::size_t input_length = length_distribution(rng); - for (std::size_t index = 0; index < input_length; ++index) input[index] = (char)byte_distribution(rng); - check(input, input_length); - } + for_each_adversarial_utf8_input_(global_random_generator(), random_inputs, check); - std::printf(" passed %zu cases (256 singles + 65536 pairs + %zu random)\n", // - 256 + 65536 + random_inputs, random_inputs); + std::printf(" invalid-input safety passed!\n"); } /** @@ -2284,6 +2283,18 @@ static uncased_safety_backend_t const uncased_safety_backends[] = { #if SZ_USE_SVE2 {"sve2", sz_utf8_uncased_fold_sve2, sz_utf8_uncased_search_sve2, sz_utf8_find_cased_sve2}, #endif +#if SZ_USE_V128 + {"v128", sz_utf8_uncased_fold_v128, sz_utf8_uncased_search_v128, sz_utf8_find_cased_v128}, +#endif +#if SZ_USE_RVV + {"rvv", sz_utf8_uncased_fold_rvv, sz_utf8_uncased_search_rvv, sz_utf8_find_cased_rvv}, +#endif +#if SZ_USE_LASX + {"lasx", sz_utf8_uncased_fold_lasx, sz_utf8_uncased_search_lasx, sz_utf8_find_cased_lasx}, +#endif +#if SZ_USE_POWERVSX + {"powervsx", sz_utf8_uncased_fold_powervsx, sz_utf8_uncased_search_powervsx, sz_utf8_find_cased_powervsx}, +#endif }; /** @brief Adversarial invalid-input safety driver across every backend compiled on this target. */ @@ -2295,8 +2306,8 @@ void test_uncased_safety() { check_uncased_safety_(span_over(uncased_safety_back /** * @brief One UTF-8 case-folding + case-insensitive search backend compiled on this target. The struct doubles as - * the fold functor for `test_fold_equivalence` (via `operator()`), so the differential and the find battery - * iterate one table; the always-present `dispatched` entry keeps it non-empty on a baseline build. + * the fold functor for `check_uncased_fold_equivalence_` (via `operator()`), so the differential and the find + * battery iterate one table; the always-present `dispatched` entry keeps it non-empty on a baseline build. */ struct uncased_backend_t { char const *name; @@ -2347,13 +2358,13 @@ void test_uncased_all() { uncased_backend_t const serial {"serial", sz_utf8_uncased_fold_serial, sz_utf8_uncased_search_serial}; // Backend-independent: the fold table and the case-invariant classifier must stay closed. - test_uncased_invariant_reference(); + check_uncased_invariant_reference_(); // Serial reference vs every compiled backend (dispatched first): the case-fold differential and the full find // battery, paired per backend so their ISA coverage stays in lockstep. for (uncased_backend_t const &backend : uncased_backends) { - test_fold_equivalence(serial, backend, 4000, scale_iterations(1200)); - run_uncased_find_battery_(backend.search); + check_uncased_fold_equivalence_(serial, backend, 4000, scale_iterations(1200)); + check_uncased_find_battery_(backend.search); } } diff --git a/test/utf8.hpp b/test/utf8.hpp index 3f2e30e5..c031fd36 100644 --- a/test/utf8.hpp +++ b/test/utf8.hpp @@ -1,14 +1,16 @@ /** * @brief Shared harness for the UTF-8 segmentation family tests (words / graphemes / sentences / linebreaks). - * @file scripts/test_utf8.hpp + * @file test/utf8.hpp * @author Ash Vardanian * - * Each segmentation family lives in its own translation unit (`test_utf8_<family>.cpp`) and pulls its substrate - * from here. The three layers are: + * Each segmentation family lives in its own translation unit (`utf8_<family>.cpp`) and pulls its substrate + * from here. The four layers are: * - known-answer goldens (`check_utf8_segment_unit_`), compared lazily against expected literals; * - the malformed-input safety sweep (`check_utf8_segment_safety_`); - * - the serial-vs-ISA differential (`test_utf8_segment_equivalence_`), a short orchestrator over named - * deterministic and randomized stressors. + * - the serial-vs-ISA differential (`check_utf8_segment_equivalence_`), a short orchestrator over named + * deterministic and randomized stressors; + * - the rule-coverage sweep (`check_utf8_rule_coverage_`), which the `_rules` tier drivers use to confirm + * every named UAX boundary rule fires at least once. * * Two segmentation backends are compared by STREAMING them in lockstep through @ref utf8_segment_cursor_t (a * fixed-capacity batch pull with `bytes_consumed` resume) and asserting each emitted segment agrees — no @@ -341,9 +343,9 @@ struct utf8_segment_corpora_t { char const *family_name; /**< human label printed by the driver (e.g. "word") */ sz::span<sz::string_view const> motifs; /**< the family's own corner-case motifs */ /** Streams the family's high-density homogeneous runs (each spans several 64-byte windows) to @p sink. */ - void (*dense_runs)(std::mt19937 &rng, utf8_run_sink_t sink, void *context); + void (*dense_runs)(std::mt19937 &generator, utf8_run_sink_t sink, void *context); /** Streams the family's long-range straddling constructions for a given @p gap to @p sink. */ - void (*straddles)(std::mt19937 &rng, std::size_t gap, utf8_run_sink_t sink, void *context); + void (*straddles)(std::mt19937 &generator, std::size_t gap, utf8_run_sink_t sink, void *context); sz::span<sz::string_view const> regressions; /**< optional fixed hand-found regression inputs */ utf8_corpus_alphabet_t const *alphabet; /**< per-family random-corpus alphabet (null -> the shared default) */ }; @@ -357,6 +359,10 @@ struct utf8_repro_t { utf8_corpus_flavor_t flavor; }; +/** @brief A per-position boundary rule, as `sz_utf8_is_word_boundary_serial` and its grapheme twin spell it. + * Declared here rather than beside `sz_utf8_segmenter_t` because only the tests call one. */ +typedef sz_bool_t (*utf8_boundary_oracle_t)(sz_cptr_t, sz_size_t, sz_size_t); + #pragma endregion // Shared constants and types #pragma region Shared helpers @@ -368,12 +374,18 @@ inline void print_utf8_test_bytes_(char const *label, char const *bytes, std::si std::fprintf(stderr, "\n"); } -/** @brief Append one codepoint to @p text as UTF-8 via `sz_rune_encode` (silently skips invalid runes). */ -inline void append_codepoint_(std::string &text, sz_rune_t codepoint) { +/** + * @brief One codepoint encoded as UTF-8 via `sz_rune_encode`; empty when the rune is unencodable. + * + * Short-string optimization keeps the four bytes in the returned object, so the corpus builders that call this + * once per codepoint never reach the allocator. Returning rather than appending is what lets every arm of a + * corpus `switch` read as one `out.append(...)`. + */ +inline std::string encoded_rune_(sz_rune_t codepoint) { sz_u8_t bytes[4]; sz_rune_length_t const length = sz_rune_encode(codepoint, bytes); - if (length == sz_rune_invalid_k) return; - text.append((char const *)bytes, (std::size_t)length); + if (length == sz_rune_invalid_k) return std::string(); + return std::string((char const *)bytes, (std::size_t)length); } /** @@ -392,17 +404,58 @@ static sz::string_view const utf8_astral_fixtures[] = { }; /** @brief Append @p link_count Regional-Indicator codepoints to @p out (cleared first); cross-family run builder. */ -inline void utf8_dense_regional_indicators_(std::string &out, std::mt19937 &rng, std::size_t link_count) { +inline void utf8_dense_regional_indicators_(std::string &out, std::mt19937 &generator, std::size_t link_count) { out.clear(); std::uniform_int_distribution<sz_rune_t> indicator(0x1F1E6, 0x1F1FF); // U+1F1E6..U+1F1FF - for (std::size_t index = 0; index != link_count; ++index) append_codepoint_(out, indicator(rng)); + for (std::size_t index = 0; index != link_count; ++index) out.append(encoded_rune_(indicator(generator))); +} + +/** + * @brief Random well-formed UTF-8 of @p target_codepoints codepoints, spanning all four byte-widths and every + * 1->2->3->4 transition, so the count / find-nth / unpack / scan kernels hit their mixed-width paths. + */ +inline std::string random_valid_utf8_(std::size_t target_codepoints, std::mt19937 &generator) { + // Disjoint ranges, one per byte-width, chosen to avoid surrogates and noncharacters. + static struct { + sz_rune_t low, high; + } const ranges[] = { + {0x000020u, 0x00007Eu}, // 1-byte ASCII printable + {0x0000A1u, 0x0007FFu}, // 2-byte Latin-1 symbols, Greek and Cyrillic letters + {0x000800u, 0x00CFFFu}, // 3-byte punctuation and CJK; stays below the U+D800 surrogate block + {0x010000u, 0x0EFFFFu}, // 4-byte symbols and emoji; avoids the plane-ender noncharacters + }; + std::uniform_int_distribution<int> width_pick(0, 3); + std::string text; + text.reserve(target_codepoints * 4); + for (std::size_t index = 0; index != target_codepoints; ++index) { + int const width = width_pick(generator); + std::uniform_int_distribution<sz_rune_t> codepoint(ranges[width].low, ranges[width].high); + // Retry until one valid codepoint is actually appended, so the emitted count is exact. + std::size_t const before = text.size(); + while (text.size() == before) text.append(encoded_rune_(codepoint(generator))); + } + return text; +} + +/** @brief Well-formed UTF-8 of exactly @p target_bytes bytes, ASCII-padded to land on the mark. */ +inline std::string random_valid_utf8_bytes_(std::size_t target_bytes, std::mt19937 &generator) { + std::string text; + text.reserve(target_bytes); + while (text.size() != target_bytes) { + std::size_t const remaining = target_bytes - text.size(); + std::string const one_rune = random_valid_utf8_(1, generator); + if (one_rune.size() <= remaining) text += one_rune; + else text.append(remaining, 'x'); + } + return text; } /** - * @brief Append one randomly chosen malformed-UTF-8 class to @p text: overlong encodings, surrogates, lone - * continuations, invalid leads, truncated tails, out-of-range leads, and noncharacters. Drawn from @p rng. + * @brief The malformed-UTF-8 sample pool: overlong encodings, surrogates, lone continuations, invalid leads, + * truncated tails, out-of-range leads, and noncharacters. Exposed so a test can sweep every class + * deterministically instead of hoping a scaled-down random draw reaches all of them. */ -inline void append_malformed_class_(std::string &text, std::mt19937 &rng) { +inline sz::span<char const *const> malformed_classes_() { static char const *const malformed[] = { "\xC0\x80", // overlong 2-byte encoding of NUL "\xC1\xBF", // overlong 2-byte encoding of U+007F @@ -425,32 +478,38 @@ inline void append_malformed_class_(std::string &text, std::mt19937 &rng) { "\xEF\xBF\xBE", // plane-ender noncharacter U+FFFE "\xEF\xBF\xBF", // plane-ender noncharacter U+FFFF }; - std::size_t const count = span_over(malformed).size(); - std::uniform_int_distribution<std::size_t> pick(0, count - 1); - text.append(malformed[pick(rng)]); + return span_over(malformed); +} + +/** @brief One entry of `malformed_classes_()`, drawn from @p generator. */ +inline char const *random_malformed_class_(std::mt19937 &generator) { + sz::span<char const *const> const pool = malformed_classes_(); + std::uniform_int_distribution<std::size_t> pick(0, pool.size() - 1); + return pool[pick(generator)]; } /** * @brief Apply structural mutation passes to @p text: NUL injection (10%), random byte-swap (10%), a - * truncate-last-codepoint pass, and a stray-continuation insertion. All randomness flows through @p rng. + * truncate-last-codepoint pass, and a stray-continuation insertion. Randomness flows through @p generator. */ -inline void apply_mutation_passes_(std::string &text, std::mt19937 &rng) { +inline void apply_mutation_passes_(std::string &text, std::mt19937 &generator) { if (text.empty()) return; std::uniform_int_distribution<std::size_t> byte_index(0, text.size() - 1); std::size_t const to_corrupt = text.size() / 10; - for (std::size_t index = 0; index != to_corrupt; ++index) text[byte_index(rng)] = '\0'; - for (std::size_t index = 0; index != to_corrupt; ++index) std::swap(text[byte_index(rng)], text[byte_index(rng)]); + for (std::size_t index = 0; index != to_corrupt; ++index) text[byte_index(generator)] = '\0'; + for (std::size_t index = 0; index != to_corrupt; ++index) + std::swap(text[byte_index(generator)], text[byte_index(generator)]); // Truncate the last codepoint: drop a 1-3 byte trailing run so a multi-byte sequence loses its tail. std::uniform_int_distribution<std::size_t> truncate_distribution(1, 3); - std::size_t const truncate_by = std::min((std::size_t)truncate_distribution(rng), text.size()); + std::size_t const truncate_by = std::min((std::size_t)truncate_distribution(generator), text.size()); text.resize(text.size() - truncate_by); // Insert a stray continuation byte at a random position, breaking codepoint alignment. if (!text.empty()) { std::uniform_int_distribution<std::size_t> insert_at(0, text.size()); std::uniform_int_distribution<int> continuation(0x80, 0xBF); - text.insert(text.begin() + insert_at(rng), (char)continuation(rng)); + text.insert(text.begin() + insert_at(generator), (char)continuation(generator)); } } @@ -482,7 +541,7 @@ static char const *const utf8_default_snippets[] = { "\xE2\x80\xA9", }; -/** @brief The shared default alphabet (weights mirror the legacy snippet/boundary/astral/motif/malformed mix). */ +/** @brief The shared default alphabet, weighting the snippet, boundary, astral, motif and malformed mix. */ static utf8_corpus_alphabet_t const utf8_default_alphabet = { span_over(utf8_default_snippets), span_over(utf8_default_boundary_codepoints), @@ -496,7 +555,7 @@ static utf8_corpus_alphabet_t const utf8_default_alphabet = { */ inline void utf8_random_segmentation_corpus_(std::string &out, std::size_t min_length, utf8_corpus_flavor_t flavor, utf8_corpus_alphabet_t const &alphabet, - sz::span<sz::string_view const> motifs, std::mt19937 &rng) { + sz::span<sz::string_view const> motifs, std::mt19937 &generator) { out.clear(); std::array<double, utf8_corpus_category_count_k> weights; for (int category = 0; category != utf8_corpus_category_count_k; ++category) @@ -515,19 +574,21 @@ inline void utf8_random_segmentation_corpus_(std::string &out, std::size_t min_l std::uniform_int_distribution<std::size_t> motif_pick(0, motifs.empty() ? 0 : motifs.size() - 1); while (out.size() < min_length) { - switch ((utf8_corpus_category_t)category(rng)) { - case utf8_corpus_malformed_k: append_malformed_class_(out, rng); break; - case utf8_corpus_boundary_k: append_codepoint_(out, alphabet.boundary_codepoints[boundary_pick(rng)]); break; + switch ((utf8_corpus_category_t)category(generator)) { + case utf8_corpus_malformed_k: out.append(random_malformed_class_(generator)); break; + case utf8_corpus_boundary_k: + out.append(encoded_rune_(alphabet.boundary_codepoints[boundary_pick(generator)])); + break; case utf8_corpus_astral_k: { - sz::string_view const fixture = utf8_astral_fixtures[astral_pick(rng)]; + sz::string_view const fixture = utf8_astral_fixtures[astral_pick(generator)]; out.append(fixture.data(), fixture.size()); } break; case utf8_corpus_motif_k: { - sz::string_view const motif = motifs[motif_pick(rng)]; + sz::string_view const motif = motifs[motif_pick(generator)]; out.append(motif.data(), motif.size()); } break; case utf8_corpus_snippet_k: - default: out.append(alphabet.snippets[snippet_pick(rng)]); break; + default: out.append(alphabet.snippets[snippet_pick(generator)]); break; } } } @@ -745,7 +806,7 @@ inline void check_utf8_rule_coverage_(char const *family, sz_utf8_segmenter_t re * Factored so each family's `_safety` runs one shared sweep instead of three copies of the 65 536-pair loop. */ template <typename callback_type_> -inline void for_each_adversarial_utf8_input_(std::mt19937 &rng, std::size_t random_input_count, +inline void for_each_adversarial_utf8_input_(std::mt19937 &generator, std::size_t random_input_count, callback_type_ &&callback) { char input[utf8_unit_capacity_k]; @@ -773,8 +834,8 @@ inline void for_each_adversarial_utf8_input_(std::mt19937 &rng, std::size_t rand std::uniform_int_distribution<std::size_t> length_distribution(1, utf8_unit_capacity_k); std::uniform_int_distribution<int> byte_distribution(0, 255); for (std::size_t iteration = 0; iteration != random_input_count; ++iteration) { - std::size_t const input_length = length_distribution(rng); - for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(rng); + std::size_t const input_length = length_distribution(generator); + for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(generator); for_each_cacheline_offset_(input_length, [&](sz_ptr_t buffer, std::size_t /*offset*/) { std::memcpy(buffer, input, input_length); callback((char const *)buffer, input_length); @@ -807,6 +868,38 @@ inline void check_utf8_segment_safety_(char const *family, sz::span<utf8_segment for_each_adversarial_utf8_input_(global_random_generator(), random_inputs, probe); } +/** + * @brief Holds a streaming segmenter to the per-position rule oracle over one text. + * + * The segmenters carry left context forward in a state machine; the oracles re-walk it at every position. + * Both transcribe the same annex, and the headers only ever claimed they agree - this is where that is + * checked. Inputs the segmenter could not drain in one call are skipped, since the oracle reads the whole + * text and a truncated prefix would ask the two a different question. + */ +inline void check_utf8_segment_against_oracle_(char const *family, sz_utf8_segmenter_t segmenter, + utf8_boundary_oracle_t oracle, char const *text, std::size_t length) { + sz_size_t offsets[utf8_unit_capacity_k + 1], lengths[utf8_unit_capacity_k + 1]; + sz_size_t bytes_consumed = 0; + sz_size_t const found = segmenter(text, (sz_size_t)length, offsets, lengths, (sz_size_t)(utf8_unit_capacity_k + 1), + &bytes_consumed); + if (bytes_consumed != (sz_size_t)length || found > utf8_unit_capacity_k) return; + + // Segments tile the input, so a position is a boundary exactly when one starts there. + bool starts_a_segment[utf8_unit_capacity_k + 1] = {}; + starts_a_segment[0] = true, starts_a_segment[length] = true; + for (sz_size_t index = 0; index != found; ++index) starts_a_segment[offsets[index]] = true; + + for (sz_size_t position = 0; position <= (sz_size_t)length; ++position) { + bool const oracle_says = oracle(text, (sz_size_t)length, position) == sz_true_k; + if (oracle_says == starts_a_segment[position]) continue; + std::fprintf(stderr, "%s: segmenter and rule oracle disagree at position %zu of %zu (%s vs %s)\n", family, + (std::size_t)position, length, starts_a_segment[position] ? "boundary" : "interior", + oracle_says ? "boundary" : "interior"); + print_utf8_test_bytes_("input", text, length); + verify(false && "The streaming segmenter must agree with the per-position rule oracle"); + } +} + #pragma endregion // Safety sweep #pragma region Differential stressors @@ -874,7 +967,7 @@ struct utf8_differential_context_t { sz::span<utf8_segment_backend_t const> candidates; std::vector<std::string> labels; // "<family>:<candidate>" per candidate, named in the divergence record utf8_segment_corpora_t const *corpora; - std::mt19937 *rng; + std::mt19937 *generator; std::string scratch; std::size_t input_index; // rotates the capacity sweep when the multiplier samples instead of exhausts }; @@ -944,26 +1037,26 @@ inline void utf8_differential_fuzz_corpus_(utf8_differential_context_t &context, std::string mutated; for (std::size_t iteration = 0; iteration != iterations; ++iteration) { utf8_random_segmentation_corpus_(context.scratch, 400, utf8_corpus_flavor_t::valid_k, alphabet, motifs, - *context.rng); + *context.generator); utf8_differential_input_(context, "fuzz-corpus", iteration, context.scratch.data(), context.scratch.size(), utf8_corpus_flavor_t::valid_k); if ((iteration & 0x7u) == 0) { utf8_random_segmentation_corpus_(context.scratch, 4096, utf8_corpus_flavor_t::valid_k, alphabet, motifs, - *context.rng); + *context.generator); utf8_differential_input_(context, "fuzz-corpus-wide", iteration, context.scratch.data(), context.scratch.size(), utf8_corpus_flavor_t::valid_k); } utf8_random_segmentation_corpus_(context.scratch, 400, utf8_corpus_flavor_t::valid_k, alphabet, motifs, - *context.rng); + *context.generator); mutated.assign(context.scratch.data(), context.scratch.size()); - apply_mutation_passes_(mutated, *context.rng); + apply_mutation_passes_(mutated, *context.generator); utf8_differential_input_(context, "fuzz-mutated", iteration, mutated.data(), mutated.size(), utf8_corpus_flavor_t::malformed_k); utf8_random_segmentation_corpus_(context.scratch, 400, utf8_corpus_flavor_t::malformed_k, alphabet, motifs, - *context.rng); + *context.generator); utf8_differential_input_(context, "fuzz-malformed", iteration, context.scratch.data(), context.scratch.size(), utf8_corpus_flavor_t::malformed_k); } @@ -976,7 +1069,7 @@ inline void utf8_differential_fuzz_dense_runs_(utf8_differential_context_t &cont for (std::size_t iteration = 0; iteration != iterations; ++iteration) { utf8_sink_context_t sink; sink.context = &context, sink.stressor = "dense-run", sink.iteration = iteration, sink.filler = 0; - context.corpora->dense_runs(*context.rng, utf8_sink_run_, &sink); + context.corpora->dense_runs(*context.generator, utf8_sink_run_, &sink); } } @@ -989,8 +1082,8 @@ inline void utf8_differential_fuzz_straddles_(utf8_differential_context_t &conte for (std::size_t gap : utf8_straddle_gaps) { utf8_sink_context_t sink; sink.context = &context, sink.stressor = "straddle", sink.iteration = iteration; - sink.filler = filler_length(*context.rng); - context.corpora->straddles(*context.rng, gap, utf8_sink_run_, &sink); + sink.filler = filler_length(*context.generator); + context.corpora->straddles(*context.generator, gap, utf8_sink_run_, &sink); } } @@ -1001,7 +1094,7 @@ inline void utf8_differential_alignment_sweep_(utf8_differential_context_t &cont utf8_corpus_alphabet_t const &alphabet = utf8_context_alphabet_(context); for (std::size_t round = 0; round != span_over(utf8_sweep_capacities).size(); ++round) { utf8_random_segmentation_corpus_(context.scratch, 256, utf8_corpus_flavor_t::valid_k, alphabet, - context.corpora->motifs, *context.rng); + context.corpora->motifs, *context.generator); std::string const probe = context.scratch; // stable source copied into each aligned buffer for_each_cacheline_offset_(probe.size(), [&](sz_ptr_t buffer, std::size_t /*offset*/) { std::memcpy(buffer, probe.data(), probe.size()); @@ -1114,14 +1207,14 @@ inline void utf8_differential_byte_edge_exhaustive_(utf8_differential_context_t * `utf8_differential_input_` for all @p candidates, asserting serial≡ISA, capacity-independence, and the * reference's own tiling/alignment invariants, and aborts with a full repro record at the first divergence. */ -inline void test_utf8_segment_equivalence_(sz_utf8_segmenter_t reference, - sz::span<utf8_segment_backend_t const> candidates, - utf8_segment_corpora_t const &corpora, - std::size_t iterations = scale_iterations(5000)) { +inline void check_utf8_segment_equivalence_(sz_utf8_segmenter_t reference, + sz::span<utf8_segment_backend_t const> candidates, + utf8_segment_corpora_t const &corpora, + std::size_t iterations = scale_iterations(5000)) { std::printf(" - testing %s serial-vs-ISA differential...\n", corpora.family_name); utf8_differential_context_t context; context.reference = reference, context.candidates = candidates, context.corpora = &corpora, - context.rng = &global_random_generator(), context.input_index = 0; + context.generator = &global_random_generator(), context.input_index = 0; for (utf8_segment_backend_t const &candidate : candidates) context.labels.push_back(std::string(corpora.family_name) + ":" + candidate.name); diff --git a/test/utf8_codepoints.py b/test/utf8_codepoints.py index 90f9dda1..4f2f0abd 100644 --- a/test/utf8_codepoints.py +++ b/test/utf8_codepoints.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """UTF-8 codepoint counting and iteration: sz.utf8_count and sz.utf8_codepoints. -Mirrors the C++ scripts/test_utf8_codepoints.cpp translation unit. +Mirrors the C++ test/utf8_runes.cpp translation unit. Covers: utf8_count and utf8_codepoints on ASCII, multi-byte, and mixed-script text, str/Str/bytes input parity, round-trip fidelity against random valid UTF-8 corpora, malformed-byte safety diff --git a/test/utf8_graphemes.cpp b/test/utf8_graphemes.cpp index 3e367946..9131d25d 100644 --- a/test/utf8_graphemes.cpp +++ b/test/utf8_graphemes.cpp @@ -1,7 +1,7 @@ /** * @brief UAX-29 grapheme-cluster (Grapheme_Cluster_Break) tests: known-answer goldens, malformed-input safety, * and the serial-vs-ISA differential over hardened corpora. - * @file scripts/test_utf8_graphemes.cpp + * @file test/utf8_graphemes.cpp * @author Ash Vardanian */ #undef NDEBUG // ! Enable all assertions for testing @@ -23,7 +23,7 @@ #include <string> // `std::string` #include <vector> // `std::vector` -#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `test_stringzilla.hpp`) +#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `stringzilla.hpp`) #pragma region Unit @@ -142,11 +142,11 @@ static sz::string_view const utf8_graphemes_motifs[] = { static void utf8_graphemes_dense_zwj_pictograph_chain_(std::string &out, std::size_t link_count) { out.clear(); static sz_rune_t const pictographs[] = {0x1F468, 0x1F469, 0x1F467, 0x1F466}; // man, woman, girl, boy - append_codepoint_(out, pictographs[0]); + out.append(encoded_rune_(pictographs[0])); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x200D); // ZWJ - append_codepoint_(out, pictographs[(index + 1) & 0x3u]); // next pictograph - if (index & 1u) append_codepoint_(out, 0xFE0F); // VS16 on alternating links + out.append(encoded_rune_(0x200D)); // ZWJ + out.append(encoded_rune_(pictographs[(index + 1) & 0x3u])); // next pictograph + if (index & 1u) out.append(encoded_rune_(0xFE0F)); // VS16 on alternating links } } @@ -155,27 +155,27 @@ static void utf8_graphemes_dense_combining_marks_(std::string &out, std::size_t out.clear(); static sz_rune_t const marks[] = {0x0301, 0x0300, 0x0308, 0x0327, 0x0323, // acute, grave, diaeresis, cedilla, dot 0x0651, 0x093C, 0x0E48, 0x1D16E}; // shadda, nukta, Thai mai ek, astral flag - append_codepoint_(out, 0x0061); // base 'a' - for (std::size_t index = 0; index != link_count; ++index) append_codepoint_(out, marks[index % 9u]); + out.append(encoded_rune_(0x0061)); // base 'a' + for (std::size_t index = 0; index != link_count; ++index) out.append(encoded_rune_(marks[index % 9u])); } -/** @brief @p link_count emoji each followed by a skin-tone modifier drawn from @p rng (GB9 Extend), into @p out. */ -static void utf8_graphemes_dense_skin_tone_run_(std::string &out, std::mt19937 &rng, std::size_t link_count) { +/** @brief @p link_count emoji each followed by a skin-tone modifier from @p generator (GB9 Extend), into @p out. */ +static void utf8_graphemes_dense_skin_tone_run_(std::string &out, std::mt19937 &generator, std::size_t link_count) { out.clear(); std::uniform_int_distribution<sz_rune_t> modifier(0x1F3FB, 0x1F3FF); // U+1F3FB..U+1F3FF for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x1F44D); // thumbs up - append_codepoint_(out, modifier(rng)); + out.append(encoded_rune_(0x1F44D)); // thumbs up + out.append(encoded_rune_(modifier(generator))); } } /** @brief @p link_count Indic consonant+virama conjunct links (GB9c InCB Consonant Linker chains), into @p out. */ static void utf8_graphemes_dense_indic_conjunct_(std::string &out, std::size_t link_count) { out.clear(); - append_codepoint_(out, 0x0915); // Devanagari KA + out.append(encoded_rune_(0x0915)); // Devanagari KA for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x094D); // virama (Linker) - append_codepoint_(out, 0x0915); // KA + out.append(encoded_rune_(0x094D)); // virama (Linker) + out.append(encoded_rune_(0x0915)); // KA } } @@ -183,46 +183,46 @@ static void utf8_graphemes_dense_indic_conjunct_(std::string &out, std::size_t l static void utf8_graphemes_dense_hangul_jamo_(std::string &out, std::size_t link_count) { out.clear(); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x1100); // L (Choseong Kiyeok) - append_codepoint_(out, 0x1161); // V (Jungseong A) - append_codepoint_(out, 0x11A8); // T (Jongseong Kiyeok) + out.append(encoded_rune_(0x1100)); // L (Choseong Kiyeok) + out.append(encoded_rune_(0x1161)); // V (Jungseong A) + out.append(encoded_rune_(0x11A8)); // T (Jongseong Kiyeok) } } /** @brief Stream the grapheme family's high-density homogeneous runs (each spans several 64-byte windows) to @p sink. */ -static void utf8_graphemes_dense_runs_(std::mt19937 &rng, utf8_run_sink_t sink, void *context) { +static void utf8_graphemes_dense_runs_(std::mt19937 &generator, utf8_run_sink_t sink, void *context) { std::string scratch; std::uniform_int_distribution<std::size_t> wide(60, 220); std::uniform_int_distribution<std::size_t> chain(20, 80); - std::size_t const wide_count = wide(rng); - std::size_t const chain_count = chain(rng); - utf8_dense_regional_indicators_(scratch, rng, wide_count), sink(context, scratch.data(), scratch.size()); + std::size_t const wide_count = wide(generator); + std::size_t const chain_count = chain(generator); + utf8_dense_regional_indicators_(scratch, generator, wide_count), sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_zwj_pictograph_chain_(scratch, chain_count), sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_combining_marks_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); - utf8_graphemes_dense_skin_tone_run_(scratch, rng, chain_count), sink(context, scratch.data(), scratch.size()); + utf8_graphemes_dense_skin_tone_run_(scratch, generator, chain_count), sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_indic_conjunct_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_hangul_jamo_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); } /** @brief Stream the grapheme family's long-range straddling constructions for a given @p gap to @p sink. */ -static void utf8_graphemes_straddles_(std::mt19937 &rng, std::size_t gap, utf8_run_sink_t sink, void *context) { +static void utf8_graphemes_straddles_(std::mt19937 &generator, std::size_t gap, utf8_run_sink_t sink, void *context) { std::string scratch; - utf8_dense_regional_indicators_(scratch, rng, gap); + utf8_dense_regional_indicators_(scratch, generator, gap); scratch.append("a"); // ASCII tail forces the GB12/13 parity decision after the long run sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_zwj_pictograph_chain_(scratch, gap); - append_codepoint_(scratch, 0x0061); // ASCII break after the chain + scratch.append(encoded_rune_(0x0061)); // ASCII break after the chain sink(context, scratch.data(), scratch.size()); utf8_graphemes_dense_indic_conjunct_(scratch, gap); - append_codepoint_(scratch, 0x0061); // ASCII break after the conjunct + scratch.append(encoded_rune_(0x0061)); // ASCII break after the conjunct sink(context, scratch.data(), scratch.size()); // A long RI run, then a ZWJ before a final RI: RI...RI ZWJ RI. The ZWJ does NOT bridge two RIs (GB11 bridges only // Extended_Pictographic), so this must break after the ZWJ - and the ZWJ resets the GB12/13 RI parity. Straddles // the 64-byte window so the parity carry and the post-ZWJ break are exercised across the edge. - utf8_dense_regional_indicators_(scratch, rng, gap); - append_codepoint_(scratch, 0x200D); // ZWJ - append_codepoint_(scratch, 0x1F1E6); // Regional_Indicator after the ZWJ - append_codepoint_(scratch, 0x0061); // ASCII tail + utf8_dense_regional_indicators_(scratch, generator, gap); + scratch.append(encoded_rune_(0x200D)); // ZWJ + scratch.append(encoded_rune_(0x1F1E6)); // Regional_Indicator after the ZWJ + scratch.append(encoded_rune_(0x0061)); // ASCII tail sink(context, scratch.data(), scratch.size()); } @@ -322,8 +322,13 @@ void test_utf8_graphemes_safety() { /** @brief Serial-vs-ISA grapheme differential over the hardened corpora (high-density + long-range). */ void test_utf8_graphemes_all() { utf8_segment_corpora_t const corpora = utf8_graphemes_corpora_(); - test_utf8_segment_equivalence_(sz_utf8_graphemes_serial, span_over(utf8_graphemes_backends), corpora, - scale_iterations(8)); // This family's share of the suite budget + check_utf8_segment_equivalence_(sz_utf8_graphemes_serial, span_over(utf8_graphemes_backends), corpora, + scale_iterations(8)); // This family's share of the suite budget + + // The streaming segmenter against the per-position GB1-GB999 transcription, which nothing else calls. + for (sz::string_view const motif : span_over(utf8_graphemes_motifs)) + check_utf8_segment_against_oracle_("grapheme", sz_utf8_graphemes_serial, sz_utf8_is_grapheme_boundary_serial, + motif.data(), motif.size()); } #pragma endregion // Drivers diff --git a/test/utf8_helpers.py b/test/utf8_helpers.py index ef784ef6..65b4c767 100644 --- a/test/utf8_helpers.py +++ b/test/utf8_helpers.py @@ -1,13 +1,13 @@ """ Shared UTF-8 segmentation test driver for the per-family test modules. -The Python analog of the C++ `scripts/test_utf8.hpp`: a single place that owns the boundary-relevant +The Python analog of the C++ `test/utf8.hpp`: a single place that owns the boundary-relevant palettes, the SMP/astral fixtures, the malformed-UTF-8 corpus generators, the window-seam length sweep, the adversarial-byte battery, and the metamorphic tiling invariant, so every family TU -(test_utf8_wordbreaks.py, test_utf8_graphemes.py, â€Ļ) shares one driver instead of copying corpora. +(utf8_wordbreaks.py, utf8_graphemes.py, â€Ļ) shares one driver instead of copying corpora. Differential oracles live here too: `icu_segmenter` / `icu_normalizer` wrap PyICU (skipped when absent), -generalizing the sentence-only ICU idiom that previously lived inline in the monolith. +covering word, grapheme, sentence, and line boundaries through one shared idiom. Palette members are built from explicit codepoints (the source stays pure ASCII) so an editor or a delegated agent cannot silently NFC-normalize a raw multi-byte literal. @@ -288,7 +288,7 @@ def icu_segmenter(kind: str) -> Callable[[str], List[str]]: """Return an ICU `BreakIterator`-backed segmenter `text -> list[str]` for the given boundary `kind`. `kind` is one of ``"word"`` / ``"grapheme"`` / ``"sentence"`` / ``"line"``. Skips the test when PyICU is - absent. Generalizes the sentence-only idiom that previously lived inline in the monolith. + absent. """ icu = pytest.importorskip("icu", reason="PyICU not installed") factories = { @@ -347,7 +347,7 @@ def icu_normalizer(form: str) -> Callable[[str], str]: # region Rule-derived generators # Generators that turn the UCD break-property / combining-class / decomposition tables (extracted in -# test_helpers.py) into hard synthetic corner cases, rather than relying on a hand-picked palette. +# sz_helpers.py) into hard synthetic corner cases, rather than relying on a hand-picked palette. def class_adjacency_strings( diff --git a/test/utf8_linebreaks.cpp b/test/utf8_linebreaks.cpp index ad280141..ac2bef9a 100644 --- a/test/utf8_linebreaks.cpp +++ b/test/utf8_linebreaks.cpp @@ -1,7 +1,7 @@ /** * @brief UAX-14 line-break (linewrap) tests: known-answer goldens, malformed-input safety, and the * serial-vs-ISA differential over hardened corpora. - * @file scripts/test_utf8_linebreaks.cpp + * @file test/utf8_linebreaks.cpp * @author Ash Vardanian */ #undef NDEBUG // ! Enable all assertions for testing @@ -23,7 +23,7 @@ #include <string> // `std::string` #include <vector> // `std::vector` -#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `test_stringzilla.hpp`) +#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `stringzilla.hpp`) #pragma region Unit @@ -116,12 +116,12 @@ static sz::string_view const utf8_linebreaks_motifs[] = { static void utf8_linebreaks_dense_mandatory_breaks_(std::string &out, std::size_t link_count) { out.clear(); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x0061); // 'a' + out.append(encoded_rune_(0x0061)); // 'a' switch (index & 0x3u) { - case 0: out.append("\r\n"); break; // CRLF - case 1: append_codepoint_(out, 0x2028); break; // LINE SEPARATOR - case 2: append_codepoint_(out, 0x2029); break; // PARAGRAPH SEPARATOR - default: append_codepoint_(out, 0x000B); break; // vertical tab (BK) + case 0: out.append("\r\n"); break; // CRLF + case 1: out.append(encoded_rune_(0x2028)); break; // LINE SEPARATOR + case 2: out.append(encoded_rune_(0x2029)); break; // PARAGRAPH SEPARATOR + default: out.append(encoded_rune_(0x000B)); break; // vertical tab (BK) } } } @@ -140,16 +140,17 @@ static void utf8_linebreaks_dense_numeric_(std::string &out, std::size_t link_co } /** @brief Stream the linewrap family's high-density homogeneous runs (each spans several 64-byte windows) to @p sink. */ -static void utf8_linebreaks_dense_runs_(std::mt19937 &rng, utf8_run_sink_t sink, void *context) { +static void utf8_linebreaks_dense_runs_(std::mt19937 &generator, utf8_run_sink_t sink, void *context) { std::string scratch; - std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(rng); + std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(generator); utf8_linebreaks_dense_mandatory_breaks_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_linebreaks_dense_nesting_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_linebreaks_dense_numeric_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); } /** @brief Stream the linewrap family's long-range straddling constructions for a given @p gap to @p sink. */ -static void utf8_linebreaks_straddles_(std::mt19937 & /*rng*/, std::size_t gap, utf8_run_sink_t sink, void *context) { +static void utf8_linebreaks_straddles_(std::mt19937 & /*generator*/, std::size_t gap, utf8_run_sink_t sink, + void *context) { std::string scratch; utf8_linebreaks_dense_mandatory_breaks_(scratch, gap), sink(context, scratch.data(), scratch.size()); utf8_linebreaks_dense_nesting_(scratch, gap), sink(context, scratch.data(), scratch.size()); @@ -272,8 +273,8 @@ void test_utf8_linebreaks_safety() { /** @brief Serial-vs-ISA line differential over the hardened corpora (high-density + long-range). */ void test_utf8_linebreaks_all() { utf8_segment_corpora_t const corpora = utf8_linebreaks_corpora_(); - test_utf8_segment_equivalence_(sz_utf8_linebreaks_serial, span_over(utf8_linebreaks_backends), corpora, - scale_iterations(25)); // This family's share of the suite budget + check_utf8_segment_equivalence_(sz_utf8_linebreaks_serial, span_over(utf8_linebreaks_backends), corpora, + scale_iterations(25)); // This family's share of the suite budget } #pragma endregion // Drivers diff --git a/test/utf8_linebreaks.py b/test/utf8_linebreaks.py index b989bd2a..4bd6d59b 100644 --- a/test/utf8_linebreaks.py +++ b/test/utf8_linebreaks.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """UAX-14 line-break-opportunity segmentation: sz.utf8_linebreaks and the Str.utf8_linebreaks method. -Mirrors the C++ scripts/test_utf8_linebreaks.cpp translation unit. +Mirrors the C++ test/utf8_linebreaks.cpp translation unit. Covers: basic and Unicode line iteration including CRLF as a single break opportunity, the tiling invariant across empty, single-line, and blank-line inputs, and Str-method parity with the module diff --git a/test/utf8_norm.cpp b/test/utf8_norm.cpp index fdfdad72..acfa6ca8 100644 --- a/test/utf8_norm.cpp +++ b/test/utf8_norm.cpp @@ -1,6 +1,6 @@ /** * @brief UTF-8 normalization (NFC/NFD/NFKC/NFKD) known-answer, serial-vs-ISA equivalence, and safety. - * @file scripts/test_utf8_norm.cpp + * @file test/utf8_norm.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -60,23 +60,12 @@ #error "This test requires C++11 or later." #endif -#include "stringzilla.hpp" // `global_random_generator`, `random_string` +#include "utf8.hpp" // `print_utf8_test_bytes_`, `encoded_rune_` namespace sz = ashvardanian::stringzilla; using namespace sz::scripts; using sz::literals::operator""_sv; // for `sz::string_view` -#pragma region Helpers - -/** @brief Prints one labeled hex dump line to `stderr`; used by the malformed-input safety test below. */ -static void print_utf8_test_bytes_(char const *label, char const *bytes, std::size_t length) { - std::fprintf(stderr, " %s (%zu bytes): ", label, length); - for (std::size_t index = 0; index < length; ++index) std::fprintf(stderr, "%02X ", (unsigned char)bytes[index]); - std::fprintf(stderr, "\n"); -} - -#pragma endregion // Helpers - #pragma region Unit /** @@ -99,15 +88,11 @@ void test_utf8_norm_unit() { SZ_NULL_CHAR); // Dispatched: already NFC verify(sz_utf8_find_denormalized(cafe_nfc, cafe_length, sz_normal_form_nfd_k) != SZ_NULL_CHAR); // Dispatched: not NFD - verify(sz_utf8_find_denormalized_serial(cafe_nfc, cafe_length, sz_normal_form_nfc_k) == - SZ_NULL_CHAR); // Manual: serial - verify(sz_utf8_find_denormalized_serial(cafe_nfc, cafe_length, sz_normal_form_nfd_k) != - SZ_NULL_CHAR); // Manual: serial + verify(sz_utf8_find_denormalized_serial(cafe_nfc, cafe_length, sz_normal_form_nfc_k) == SZ_NULL_CHAR); + verify(sz_utf8_find_denormalized_serial(cafe_nfc, cafe_length, sz_normal_form_nfd_k) != SZ_NULL_CHAR); #if SZ_USE_ICELAKE - verify(sz_utf8_find_denormalized_icelake(cafe_nfc, cafe_length, sz_normal_form_nfc_k) == - SZ_NULL_CHAR); // Manual: icelake - verify(sz_utf8_find_denormalized_icelake(cafe_nfc, cafe_length, sz_normal_form_nfd_k) != - SZ_NULL_CHAR); // Manual: icelake + verify(sz_utf8_find_denormalized_icelake(cafe_nfc, cafe_length, sz_normal_form_nfc_k) == SZ_NULL_CHAR); + verify(sz_utf8_find_denormalized_icelake(cafe_nfc, cafe_length, sz_normal_form_nfd_k) != SZ_NULL_CHAR); #endif { char norm_buffer[64]; @@ -116,11 +101,11 @@ void test_utf8_norm_unit() { sz_size_t const nfd_length = sz_utf8_norm(cafe_nfc, cafe_length, sz_normal_form_nfd_k, norm_buffer); verify(nfd_length == 6u); // "caf" + 'e' + U+0301 (2-byte combining acute) sz_size_t const nfd_length_serial = sz_utf8_norm_serial(cafe_nfc, cafe_length, sz_normal_form_nfd_k, - norm_buffer); // Manual: serial + norm_buffer); verify(nfd_length_serial == 6u); #if SZ_USE_ICELAKE sz_size_t const nfd_length_icelake = sz_utf8_norm_icelake(cafe_nfc, cafe_length, sz_normal_form_nfd_k, - norm_buffer); // Manual: icelake + norm_buffer); verify(nfd_length_icelake == 6u); #endif } @@ -179,7 +164,7 @@ struct utf8_norm_backend_t { * astral planes stay reachable on a cheap run. */ template <typename reference_, typename candidate_> -void test_norm_equivalence(reference_ reference, candidate_ candidate, std::size_t iterations) { +void check_utf8_norm_equivalence_(reference_ reference, candidate_ candidate, std::size_t iterations) { std::size_t const codepoint_stride = sweep_stride(0x110000); std::vector<sz_rune_t> all_runes; all_runes.reserve(0x110000 / codepoint_stride); @@ -193,12 +178,12 @@ void test_norm_equivalence(reference_ reference, candidate_ candidate, std::size std::vector<char> input_buffer(all_runes.size() * 4); std::vector<char> output_reference(input_buffer.size() * 4 + 64); // decomposition can expand std::vector<char> output_candidate(input_buffer.size() * 4 + 64); - auto &rng = global_random_generator(); + auto &generator = global_random_generator(); static sz_normal_form_t const norm_forms[4] = {sz_normal_form_nfd_k, sz_normal_form_nfc_k, sz_normal_form_nfkd_k, sz_normal_form_nfkc_k}; for (std::size_t iteration = 0; iteration != iterations; ++iteration) { - if (iteration > 0) std::shuffle(all_runes.begin(), all_runes.end(), rng); + if (iteration > 0) std::shuffle(all_runes.begin(), all_runes.end(), generator); char *write_cursor = input_buffer.data(); for (sz_rune_t codepoint : all_runes) write_cursor += sz_rune_encode(codepoint, (sz_u8_t *)write_cursor); sz_size_t input_length = (sz_size_t)(write_cursor - input_buffer.data()); @@ -244,7 +229,7 @@ void test_norm_equivalence(reference_ reference, candidate_ candidate, std::size static void check_utf8_norm_safety_(sz_utf8_norm_t norm, sz_utf8_find_denormalized_t violation, std::size_t random_inputs = scale_iterations(10000)) { - std::size_t const max_input_length = 70; + std::size_t const max_input_length = utf8_unit_capacity_k; // The normalizer's documented worst case is 18x the input for a single-codepoint compatibility // decomposition (see `utf8_norm.h`); a truncated trailing sequence may mis-decode one extra rune. std::size_t const norm_bound = max_input_length * 18 + 18; @@ -282,40 +267,7 @@ static void check_utf8_norm_safety_(sz_utf8_norm_t norm, sz_utf8_find_denormaliz } }; - char input[max_input_length]; - - // The named adversarial shapes, exercised directly. - check("\x80", 1); // Lone continuation byte - check("\xC0\x80", 2); // Overlong encoding of NUL - check("\xED\xA0\x80", 3); // Surrogate-encoded codepoint (U+D800) - check("hello\xF0\x9F\x98", 8); // Truncated 4-byte sequence at the very end - - // All 256 single bytes: truncated leads, stray continuations, 0xFE/0xFF. - for (std::size_t byte = 0; byte < 256; byte += sweep_stride(256)) { - input[0] = (char)byte; - check(input, 1); - } - - // All 65,536 byte pairs: every lead x continuation interaction, including overlong and surrogate shapes. - // The pair index is walked flat, so a strided run samples both bytes rather than a prefix of the leads. - for (std::size_t pair = 0; pair < 65536; pair += sweep_stride(65536)) { - input[0] = (char)(pair >> 8); - input[1] = (char)(pair & 0xFF); - check(input, 2); - } - - // Random garbage buffers spanning whole SIMD chunks, at every sub-cache-line alignment. - auto &rng = global_random_generator(); - std::uniform_int_distribution<std::size_t> length_distribution(1, max_input_length); - std::uniform_int_distribution<int> byte_distribution(0, 255); - for (std::size_t iteration = 0; iteration != random_inputs; ++iteration) { - std::size_t const input_length = length_distribution(rng); - for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(rng); - for_each_cacheline_offset_(input_length, [&](sz_ptr_t buffer, std::size_t /*offset*/) { - std::memcpy(buffer, input, input_length); - check(buffer, input_length); - }); - } + for_each_adversarial_utf8_input_(global_random_generator(), random_inputs, check); } /** @brief Drive the malformed-input normalization safety probe through serial, dispatched, and every backend. */ @@ -414,7 +366,7 @@ void test_utf8_norm_all() { // One iteration pushes every assigned codepoint through 4 forms x 4 kernel calls, once per compiled backend. // Three passes - one in codepoint order plus two shuffles - is this family's share of the suite budget. for (utf8_norm_backend_t const &backend : utf8_norm_backends) - test_norm_equivalence(serial, backend, scale_iterations(3)); + check_utf8_norm_equivalence_(serial, backend, scale_iterations(3)); } #pragma endregion // Drivers diff --git a/test/utf8_runes.cpp b/test/utf8_runes.cpp index 5da4d40f..89583bdd 100644 --- a/test/utf8_runes.cpp +++ b/test/utf8_runes.cpp @@ -1,6 +1,6 @@ /** * @brief UTF-8 codepoint counting, nth-character finding, and streaming rune-unpacking tests. - * @file scripts/test_utf8_runes.cpp + * @file test/utf8_runes.cpp * @author Ash Vardanian * @date June 20, 2026 */ @@ -60,7 +60,7 @@ #error "This test requires C++11 or later." #endif -#include "stringzilla.hpp" // `global_random_generator`, `random_string` +#include "utf8.hpp" // `encoded_rune_`, `random_valid_utf8_`, `print_utf8_test_bytes_` namespace sz = ashvardanian::stringzilla; using namespace sz::scripts; @@ -68,54 +68,6 @@ using sz::literals::operator""_sv; // for `sz::string_view` #pragma region Helpers -/** @brief Append one codepoint to @p text as UTF-8 via `sz_rune_encode` (silently skips invalid runes). */ -static void append_codepoint_(std::string &text, sz_rune_t codepoint) { - sz_u8_t bytes[4]; - sz_rune_length_t const length = sz_rune_encode(codepoint, bytes); - if (length == sz_rune_invalid_k) return; - text.append((char const *)bytes, (std::size_t)length); -} - -/** - * @brief Builds a random, well-formed UTF-8 string whose codepoints span all four byte-widths and - * every 1->2->3->4 transition, so the count/find-nth/unpack kernels hit their mixed-width paths. - */ -static std::string random_valid_utf8_(std::size_t target_codepoints, std::mt19937 &rng) { - // Disjoint ranges, one per byte-width, chosen to avoid surrogates and noncharacters. - static struct { - sz_rune_t low, high; - } const ranges[] = { - {0x000020u, 0x00007Eu}, // 1-byte ASCII printable - {0x0000A1u, 0x0007FFu}, // 2-byte - {0x000800u, 0x00CFFFu}, // 3-byte (stays below the U+D800 surrogate block) - {0x010000u, 0x0EFFFFu}, // 4-byte (avoids the U+FFFE/U+FFFF plane enders below 0x10000) - }; - std::uniform_int_distribution<int> width_pick(0, 3); - std::string text; - text.reserve(target_codepoints * 4); - for (std::size_t index = 0; index != target_codepoints; ++index) { - int const width = width_pick(rng); - std::uniform_int_distribution<sz_rune_t> codepoint(ranges[width].low, ranges[width].high); - std::size_t const before = text.size(); - // Retry until one valid codepoint is actually appended, so the emitted count is exact. - while (text.size() == before) append_codepoint_(text, codepoint(rng)); - } - return text; -} - -/** @brief Well-formed UTF-8 of exactly @p target_bytes bytes, ASCII-padded to land on the mark. */ -static std::string random_valid_utf8_bytes_(std::size_t target_bytes, std::mt19937 &rng) { - std::string text; - text.reserve(target_bytes); - while (text.size() != target_bytes) { - std::size_t const remaining = target_bytes - text.size(); - std::string const one_rune = random_valid_utf8_(1, rng); - if (one_rune.size() <= remaining) text += one_rune; - else text.append(remaining, 'x'); - } - return text; -} - /** @brief Repeats one UTF-8 encoded codepoint @p repeats times, giving a run of a single byte-width. */ static std::string uniform_utf8_run_(char const *encoded_rune, std::size_t repeats) { std::string text; @@ -157,7 +109,7 @@ static void collect_unpacked_runes_(sz_utf8_decode_t unpack, sz_cptr_t text, sz_ * @brief Runs one UTF-8 codepoint backend (count + nth-finder + chunk-unpacker) over the known-answer * anchor and asserts the produced count, byte offsets, and decoded runes match the expectations. * - * Mirrors `check_sha256_unit_` in `test_hash.cpp`: the caller drives it once per backend (dispatched, + * Mirrors `check_sha256_unit_` in `hash.cpp`: the caller drives it once per backend (dispatched, * serial, and each natively-compiled kernel), so a wrong constant shared by the serial-vs-SIMD agreement * tests is still caught against an external ground truth. * @@ -269,8 +221,7 @@ void test_utf8_runes_unit() { std::vector<sz_rune_t> const mixed_runes = {0x61u, 0xDFu, 0x4E2Du}; // Drive the count + find-nth + unpack known-answer through the dispatched, serial, and native kernels. - check_utf8_runes_unit_(sz_utf8_count, sz_utf8_seek, sz_utf8_decode, // Dispatched - mixed, mixed_length, 3u, mixed_runes); + check_utf8_runes_unit_(sz_utf8_count, sz_utf8_seek, sz_utf8_decode, mixed, mixed_length, 3u, mixed_runes); check_utf8_runes_unit_(sz_utf8_count_serial, sz_utf8_seek_serial, sz_utf8_decode_serial, // serial mixed, mixed_length, 3u, mixed_runes); #if SZ_USE_HASWELL @@ -354,6 +305,77 @@ void test_utf8_runes_unit() { verify(text.utf8_seek(3) == sz::string_view::npos); } + // 64-byte chunk boundaries and batch limits, materialized via the vector wrapper. + { + // Critical 63, 64, 65 byte boundaries + let_verify(std::string s63(63, 'x'), sz::string_view(s63).utf8_runes().size() == 63); + let_verify(std::string s64(64, 'x'), sz::string_view(s64).utf8_runes().size() == 64); + let_verify(std::string s65(65, 'x'), sz::string_view(s65).utf8_runes().size() == 65); + + // ASCII batch limit: 16 characters max per Ice Lake iteration + let_verify(std::string s17(17, 'x'), sz::string_view(s17).utf8_runes().size() == 17); + let_verify(std::string s20(20, 'x'), sz::string_view(s20).utf8_runes().size() == 20); + + // 2-byte batch limit: 32 characters (64 bytes) max per iteration + scope_verify(std::string cyr32, for (int i = 0; i < 32; ++i) cyr32 += "\xD0\x9F", + sz::string_view(cyr32).utf8_count() == 32); + scope_verify(std::string cyr33, for (int i = 0; i < 33; ++i) cyr33 += "\xD0\x9F", + sz::string_view(cyr33).utf8_count() == 33); + + // 3-byte batch limit: 16 characters (48 bytes) max per iteration + scope_verify(std::string cjk16, for (int i = 0; i < 16; ++i) cjk16 += "\xE4\xB8\x96", + sz::string_view(cjk16).utf8_count() == 16); + scope_verify(std::string cjk17, for (int i = 0; i < 17; ++i) cjk17 += "\xE4\xB8\x96", + sz::string_view(cjk17).utf8_count() == 17); + + // 4-byte batch limit: 16 characters (64 bytes) max per iteration + scope_verify(std::string emoji16, for (int i = 0; i < 16; ++i) emoji16 += "\xF0\x9F\x98\x80", + sz::string_view(emoji16).utf8_count() == 16); + scope_verify(std::string emoji17, for (int i = 0; i < 17; ++i) emoji17 += "\xF0\x9F\x98\x80", + sz::string_view(emoji17).utf8_count() == 17); + + // Asymmetric at chunk boundary: 60 ASCII + "ПП世" = 63 chars, 67 bytes + scope_verify(std::string boundary_asym(60, 'x'), boundary_asym += "\xD0\x9F\xD0\x9F\xE4\xB8\x96", + sz::string_view(boundary_asym).utf8_count() == 63); + + // Sequences exceeding batch limits + scope_verify(std::string cyr100, for (int i = 0; i < 100; ++i) cyr100 += "\xD0\x9F", + sz::string_view(cyr100).utf8_runes().size() == 100); + scope_verify(std::string cjk50, for (int i = 0; i < 50; ++i) cjk50 += "\xE4\xB8\x96", + sz::string_view(cjk50).utf8_runes().size() == 50); + scope_verify(std::string emoji50, for (int i = 0; i < 50; ++i) emoji50 += "\xF0\x9F\x98\x80", + sz::string_view(emoji50).utf8_runes().size() == 50); + + // Asymmetric overflow: 20x (2 ASCII + 3 Cyrillic) = 100 chars, 140 bytes + scope_verify(std::string overflow_asym, + for (int i = 0; i < 20; ++i) overflow_asym += "aa\xD0\x9F\xD0\xA0\xD0\xA1", + sz::string_view(overflow_asym).utf8_count() == 100); + + // Transitions at chunk boundaries + scope_verify(std::string boundary_test(63, 'x'), boundary_test += "\xD0\x9F", + sz::string_view(boundary_test).utf8_runes().size() == 64); + scope_verify( + std::string span_asym, + { + for (int i = 0; i < 30; ++i) span_asym += "aa"; + for (int i = 0; i < 8; ++i) span_asym += "\xD0\x9F\xD0\xA0\xD0\xA1"; + }, + sz::string_view(span_asym).utf8_count() == 84); + scope_verify(std::string exact_boundary(64, 'x'), exact_boundary += "\xD0\x9F\xE4\xB8\x96\xF0\x9F\x98\x80", + sz::string_view(exact_boundary).utf8_count() == 67); + } +} + +/** + * @brief Known-answer rune-iteration vectors spanning the Unicode script range and every byte-width transition. + * + * Decodes hand-written samples of ASCII, CJK, Cyrillic, Arabic, Hebrew, Thai, Devanagari, emoji, the maximum + * codepoint U+10FFFF, Deseret, zero-width and combining marks through the C++ `utf8_runes` wrapper, then walks + * every 1/2/3/4-byte neighbor pair, so a kernel that assumes a homogeneous byte-width run is caught here. + */ +void test_utf8_runes_scripts_unit() { + std::printf(" - testing UTF-8 codepoints across Unicode scripts...\n"); + // C++ API: codepoint (rune) iteration materialized as a vector - never a range-for over the view range, // whose sentinel comparison is a C++17 extension that errors at C++11. { @@ -475,66 +497,6 @@ void test_utf8_runes_unit() { scope_verify(std::string asym_long, for (int i = 0; i < 30; ++i) asym_long += "xx\xD0\x9F\xD0\x9F\xD0\x9F", sz::string_view(asym_long).utf8_count() == 150); } - - // 64-byte chunk boundaries and batch limits, materialized via the vector wrapper. - { - // Critical 63, 64, 65 byte boundaries - let_verify(std::string s63(63, 'x'), sz::string_view(s63).utf8_runes().size() == 63); - let_verify(std::string s64(64, 'x'), sz::string_view(s64).utf8_runes().size() == 64); - let_verify(std::string s65(65, 'x'), sz::string_view(s65).utf8_runes().size() == 65); - - // ASCII batch limit: 16 characters max per Ice Lake iteration - let_verify(std::string s17(17, 'x'), sz::string_view(s17).utf8_runes().size() == 17); - let_verify(std::string s20(20, 'x'), sz::string_view(s20).utf8_runes().size() == 20); - - // 2-byte batch limit: 32 characters (64 bytes) max per iteration - scope_verify(std::string cyr32, for (int i = 0; i < 32; ++i) cyr32 += "\xD0\x9F", - sz::string_view(cyr32).utf8_count() == 32); - scope_verify(std::string cyr33, for (int i = 0; i < 33; ++i) cyr33 += "\xD0\x9F", - sz::string_view(cyr33).utf8_count() == 33); - - // 3-byte batch limit: 16 characters (48 bytes) max per iteration - scope_verify(std::string cjk16, for (int i = 0; i < 16; ++i) cjk16 += "\xE4\xB8\x96", - sz::string_view(cjk16).utf8_count() == 16); - scope_verify(std::string cjk17, for (int i = 0; i < 17; ++i) cjk17 += "\xE4\xB8\x96", - sz::string_view(cjk17).utf8_count() == 17); - - // 4-byte batch limit: 16 characters (64 bytes) max per iteration - scope_verify(std::string emoji16, for (int i = 0; i < 16; ++i) emoji16 += "\xF0\x9F\x98\x80", - sz::string_view(emoji16).utf8_count() == 16); - scope_verify(std::string emoji17, for (int i = 0; i < 17; ++i) emoji17 += "\xF0\x9F\x98\x80", - sz::string_view(emoji17).utf8_count() == 17); - - // Asymmetric at chunk boundary: 60 ASCII + "ПП世" = 63 chars, 67 bytes - scope_verify(std::string boundary_asym(60, 'x'), boundary_asym += "\xD0\x9F\xD0\x9F\xE4\xB8\x96", - sz::string_view(boundary_asym).utf8_count() == 63); - - // Sequences exceeding batch limits - scope_verify(std::string cyr100, for (int i = 0; i < 100; ++i) cyr100 += "\xD0\x9F", - sz::string_view(cyr100).utf8_runes().size() == 100); - scope_verify(std::string cjk50, for (int i = 0; i < 50; ++i) cjk50 += "\xE4\xB8\x96", - sz::string_view(cjk50).utf8_runes().size() == 50); - scope_verify(std::string emoji50, for (int i = 0; i < 50; ++i) emoji50 += "\xF0\x9F\x98\x80", - sz::string_view(emoji50).utf8_runes().size() == 50); - - // Asymmetric overflow: 20x (2 ASCII + 3 Cyrillic) = 100 chars, 140 bytes - scope_verify(std::string overflow_asym, - for (int i = 0; i < 20; ++i) overflow_asym += "aa\xD0\x9F\xD0\xA0\xD0\xA1", - sz::string_view(overflow_asym).utf8_count() == 100); - - // Transitions at chunk boundaries - scope_verify(std::string boundary_test(63, 'x'), boundary_test += "\xD0\x9F", - sz::string_view(boundary_test).utf8_runes().size() == 64); - scope_verify( - std::string span_asym, - { - for (int i = 0; i < 30; ++i) span_asym += "aa"; - for (int i = 0; i < 8; ++i) span_asym += "\xD0\x9F\xD0\xA0\xD0\xA1"; - }, - sz::string_view(span_asym).utf8_count() == 84); - scope_verify(std::string exact_boundary(64, 'x'), exact_boundary += "\xD0\x9F\xE4\xB8\x96\xF0\x9F\x98\x80", - sz::string_view(exact_boundary).utf8_count() == 67); - } } #pragma endregion // Unit @@ -550,11 +512,11 @@ void test_utf8_runes_unit() { * generated once and driven through all @p candidates, so every backend sees byte-identical bytes and a * divergence reproduces on the next ladder entry. */ -static inline void test_utf8_runes_equivalence( // +static inline void check_utf8_runes_equivalence_( // sz_utf8_count_t count_serial, sz_utf8_seek_t find_nth_serial, sz_utf8_decode_t unpack_serial, // sz::span<utf8_runes_backend_t const> candidates, sz_size_t inputs) { - auto &rng = global_random_generator(); + auto &generator = global_random_generator(); std::vector<sz_rune_t> runes_serial, runes_candidate; std::vector<sz_cptr_t> offsets_serial; @@ -584,29 +546,29 @@ static inline void test_utf8_runes_equivalence( } } }; - auto check_text = [&](std::string const &text) { check(text.data(), (sz_size_t)text.size()); }; + auto check_text_ = [&](std::string const &text) { check(text.data(), (sz_size_t)text.size()); }; // Structured length ladder around the SIMD window boundaries, every codepoint count exercised. sz_size_t const ladder[] = {0u, 1u, 2u, 15u, 16u, 17u, 31u, 32u, 33u, 63u, 64u, 65u, 100u, 200u}; - for (sz_size_t codepoints : ladder) check_text(random_valid_utf8_(codepoints, rng)); + for (sz_size_t codepoints : ladder) check_text_(random_valid_utf8_(codepoints, generator)); // Byte-exact ladder: the codepoint ladder above lands on arbitrary byte lengths, so the 16/32/64-byte // vector widths and their neighbours are otherwise only hit by chance. sz_size_t const byte_ladder[] = {15u, 16u, 17u, 31u, 32u, 33u, 47u, 48u, 63u, 64u, 65u, 127u, 128u, 129u}; - for (sz_size_t bytes : byte_ladder) check_text(random_valid_utf8_bytes_(bytes, rng)); + for (sz_size_t bytes : byte_ladder) check_text_(random_valid_utf8_bytes_(bytes, generator)); // Homogeneous runs spanning several windows: the mixed generator never emits a long single-width stretch, // where a width-specialized fast path runs unbroken. char const *const uniform_runes[] = {"x", "\xD0\x9F", "\xE4\xB8\x96", "\xF0\x9F\x98\x80"}; sz_size_t const uniform_repeats[] = {17u, 65u, 200u}; for (char const *encoded_rune : uniform_runes) - for (sz_size_t repeats : uniform_repeats) check_text(uniform_utf8_run_(encoded_rune, repeats)); + for (sz_size_t repeats : uniform_repeats) check_text_(uniform_utf8_run_(encoded_rune, repeats)); // Fuzzed inputs of random codepoint counts, each placed at every sub-cache-line offset so serial-vs-ISA // agreement is checked across all alignments the SIMD kernels may hit. std::uniform_int_distribution<std::size_t> codepoint_distribution(0, 96); for (sz_size_t iteration = 0; iteration != inputs; ++iteration) { - std::string const text = random_valid_utf8_(codepoint_distribution(rng), rng); + std::string const text = random_valid_utf8_(codepoint_distribution(generator), generator); for_each_cacheline_offset_(text.size(), [&](sz_ptr_t buffer, std::size_t /*offset*/) { std::memcpy(buffer, text.data(), text.size()); check(buffer, (sz_size_t)text.size()); @@ -618,7 +580,7 @@ static inline void test_utf8_runes_equivalence( * @brief Large-buffer count agreement: a few hundred KB of mixed-width codepoints where the dispatched * and C++ counts must equal the serial reference and the exact known total. */ -static void test_utf8_runes_large_count() { +static void check_utf8_runes_large_count_() { // Every repeat contributes one ASCII 'x', one 2-byte, one 3-byte, and one 4-byte codepoint - 4 // codepoints in 10 bytes - so the total is exactly `repeats * 4`. char const unit[] = "x\xD0\x9F\xE4\xB8\xAD\xF0\x9F\x98\x80"; // 'x' + U+041F + U+4E2D + U+1F600, 10 bytes @@ -630,8 +592,8 @@ static void test_utf8_runes_large_count() { sz_size_t const expected_codepoints = (sz_size_t)(repeats * 4); sz_size_t const count_serial = sz_utf8_count_serial(mixed.data(), mixed.size()); verify(count_serial == expected_codepoints); - verify(sz_utf8_count(mixed.data(), mixed.size()) == count_serial); // Dispatched matches serial - verify(sz::string_view(mixed).utf8_count() == count_serial); // C++ wrapper matches serial + verify(sz_utf8_count(mixed.data(), mixed.size()) == count_serial); + verify(sz::string_view(mixed).utf8_count() == count_serial); // C++ wrapper matches serial } #pragma endregion // Equivalence @@ -655,13 +617,13 @@ static void test_utf8_runes_large_count() { static void check_utf8_runes_safety_(sz_utf8_count_t count, sz_utf8_decode_t unpack, std::size_t random_inputs = scale_iterations(4000)) { - std::size_t const max_input_length = 70; + std::size_t const max_input_length = utf8_unit_capacity_k; std::vector<sz_rune_t> rune_destination; auto check = [&](char const *input, std::size_t input_length) { - // Counting just has to survive and return some value on arbitrary bytes. + // Counting must survive arbitrary bytes and never report more runes than the input holds bytes. sz_size_t const counted = count(input, (sz_size_t)input_length); - sz_unused_(counted); + verify(counted <= input_length && "Count reported more runes than the input holds bytes"); // Streaming unpack is total under the unified contract: it runs on arbitrary bytes, substituting U+FFFD // for ill-formed input. Each call must report no more runes than the destination holds, a cursor that @@ -694,54 +656,21 @@ static void check_utf8_runes_safety_(sz_utf8_count_t count, sz_utf8_decode_t unp } }; - char input[max_input_length]; + auto &generator = global_random_generator(); + for_each_adversarial_utf8_input_(generator, random_inputs, check); - // The named adversarial shapes, exercised directly. - check("\x80", 1); // Lone continuation byte - check("\xC0\x80", 2); // Overlong encoding of NUL - check("\xED\xA0\x80", 3); // Surrogate-encoded codepoint (U+D800) - check("hello\xF0\x9F\x98", 8); // Truncated 4-byte sequence at the very end - - // All 256 single bytes: truncated leads, stray continuations, 0xFE/0xFF. Strided so a low multiplier - // samples the whole byte space instead of truncating to a prefix of it. - std::size_t const byte_stride = sweep_stride(256); - for (std::size_t byte = 0; byte < 256; byte += byte_stride) { - input[0] = (char)byte; - check(input, 1); - } - - // All 65,536 byte pairs: every lead x continuation interaction, including overlong and surrogate shapes. - // Walked flat so a strided run samples both bytes evenly, and its cost stays proportional to the multiplier. - for (std::size_t pair = 0; pair < 65536; pair += sweep_stride(65536)) { - input[0] = (char)(pair >> 8); - input[1] = (char)(pair & 0xFF); - check(input, 2); - } - - auto &rng = global_random_generator(); + // Valid text with a few bytes overwritten: damage surrounded by long well-formed runs, which neither the + // battery's uniform garbage nor the well-formed equivalence corpus produces. std::uniform_int_distribution<std::size_t> length_distribution(1, max_input_length); std::uniform_int_distribution<int> byte_distribution(0, 255); - - // Valid text with a few bytes overwritten: damage surrounded by long well-formed runs, which uniform - // garbage never produces and the equivalence pass - fed only well-formed text - never reaches. std::uniform_int_distribution<std::size_t> codepoint_distribution(1, max_input_length / 4); for (std::size_t iteration = 0; iteration != random_inputs / 8 + 1; ++iteration) { - std::string text = random_valid_utf8_(codepoint_distribution(rng), rng); + std::string text = random_valid_utf8_(codepoint_distribution(generator), generator); if (text.size() > max_input_length) text.resize(max_input_length); for (std::size_t corruption = 0; corruption != 3; ++corruption) - text[length_distribution(rng) % text.size()] = (char)byte_distribution(rng); + text[length_distribution(generator) % text.size()] = (char)byte_distribution(generator); check(text.data(), text.size()); } - - // Random garbage buffers spanning whole SIMD chunks, at every sub-cache-line alignment. - for (std::size_t iteration = 0; iteration != random_inputs; ++iteration) { - std::size_t const input_length = length_distribution(rng); - for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(rng); - for_each_cacheline_offset_(input_length, [&](sz_ptr_t buffer, std::size_t /*offset*/) { - std::memcpy(buffer, input, input_length); - check(buffer, input_length); - }); - } } /** @brief Drive the malformed-input safety probe through serial, dispatched, and every native backend. */ @@ -771,11 +700,11 @@ void test_utf8_runes_all() { // Serial is the reference; the dispatched entry and every native backend are differenced against it. A // `SZ_NULL` decoder (e.g. `v128relaxed`) simply skips the streaming-decode leg inside the helper. - test_utf8_runes_equivalence(sz_utf8_count_serial, sz_utf8_seek_serial, sz_utf8_decode_serial, // - span_over(utf8_runes_backends), inputs); + check_utf8_runes_equivalence_(sz_utf8_count_serial, sz_utf8_seek_serial, sz_utf8_decode_serial, // + span_over(utf8_runes_backends), inputs); // Large-buffer count agreement: serial == dispatched == C++ wrapper == known total. - test_utf8_runes_large_count(); + check_utf8_runes_large_count_(); } #pragma endregion // Drivers diff --git a/test/utf8_sentences.cpp b/test/utf8_sentences.cpp index 9b3e25a1..3b4b4c6a 100644 --- a/test/utf8_sentences.cpp +++ b/test/utf8_sentences.cpp @@ -1,7 +1,7 @@ /** * @brief UAX-29 sentence-boundary (Sentence_Break) tests: known-answer goldens, malformed-input safety, and the * serial-vs-ISA differential over hardened corpora. - * @file scripts/test_utf8_sentences.cpp + * @file test/utf8_sentences.cpp * @author Ash Vardanian */ #undef NDEBUG // ! Enable all assertions for testing @@ -23,7 +23,7 @@ #include <string> // `std::string` #include <vector> // `std::vector` -#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `test_stringzilla.hpp`) +#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `stringzilla.hpp`) #pragma region Unit @@ -119,10 +119,10 @@ static sz::string_view const utf8_sentences_motifs[] = { /** @brief ATerm + Close* + Sp* run @p link_count wide, then a Lower (SB8 continuation, no break), into @p out. */ static void utf8_sentences_dense_aterm_sp_lower_(std::string &out, std::size_t link_count) { out.clear(); - append_codepoint_(out, 0x0055); // 'U' Upper - append_codepoint_(out, 0x002E); // '.' ATerm - for (std::size_t index = 0; index != link_count; ++index) append_codepoint_(out, 0x0020); // Sp run - append_codepoint_(out, 0x0061); // 'a' Lower + out.append(encoded_rune_(0x0055)); // 'U' Upper + out.append(encoded_rune_(0x002E)); // '.' ATerm + for (std::size_t index = 0; index != link_count; ++index) out.append(encoded_rune_(0x0020)); // Sp run + out.append(encoded_rune_(0x0061)); // 'a' Lower } /** @brief Terminator-dense `A. A. A. ...` repeated @p link_count times (one break per terminator), into @p out. */ @@ -135,44 +135,45 @@ static void utf8_sentences_dense_terminators_(std::string &out, std::size_t link static void utf8_sentences_dense_cjk_term_(std::string &out, std::size_t link_count) { out.clear(); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x4E2D); // 中 - append_codepoint_(out, 0x3002); // 。 ideographic full stop (STerm) - append_codepoint_(out, 0x0020); // Sp + out.append(encoded_rune_(0x4E2D)); // 中 + out.append(encoded_rune_(0x3002)); // 。 ideographic full stop (STerm) + out.append(encoded_rune_(0x0020)); // Sp } } /** @brief Stream the sentence family's high-density homogeneous runs (each spans several 64-byte windows) to @p sink. */ -static void utf8_sentences_dense_runs_(std::mt19937 &rng, utf8_run_sink_t sink, void *context) { +static void utf8_sentences_dense_runs_(std::mt19937 &generator, utf8_run_sink_t sink, void *context) { std::string scratch; - std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(rng); + std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(generator); utf8_sentences_dense_aterm_sp_lower_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_sentences_dense_terminators_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_sentences_dense_cjk_term_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); } /** @brief SB8: `Upper ATerm Sp{gap} Lower` — long Sp run before a lowercase keeps one sentence, into @p out. */ -static void utf8_straddle_sb8_lower_(std::string &out, std::size_t gap) { +static void utf8_sentences_straddle_sb8_lower_(std::string &out, std::size_t gap) { out.clear(); - append_codepoint_(out, 0x0055); // 'U' - append_codepoint_(out, 0x002E); // '.' - for (std::size_t index = 0; index != gap; ++index) append_codepoint_(out, 0x0020); // Sp run across windows - append_codepoint_(out, 0x0061); // 'a' Lower + out.append(encoded_rune_(0x0055)); // 'U' + out.append(encoded_rune_(0x002E)); // '.' + for (std::size_t index = 0; index != gap; ++index) out.append(encoded_rune_(0x0020)); // Sp run across windows + out.append(encoded_rune_(0x0061)); // 'a' Lower } /** @brief SB11: `Upper ATerm Sp{gap} Upper` — same run before an uppercase must break, into @p out. */ -static void utf8_straddle_sb11_upper_(std::string &out, std::size_t gap) { +static void utf8_sentences_straddle_sb11_upper_(std::string &out, std::size_t gap) { out.clear(); - append_codepoint_(out, 0x0055); // 'U' - append_codepoint_(out, 0x002E); // '.' - for (std::size_t index = 0; index != gap; ++index) append_codepoint_(out, 0x0020); // Sp run across windows - append_codepoint_(out, 0x0042); // 'B' Upper + out.append(encoded_rune_(0x0055)); // 'U' + out.append(encoded_rune_(0x002E)); // '.' + for (std::size_t index = 0; index != gap; ++index) out.append(encoded_rune_(0x0020)); // Sp run across windows + out.append(encoded_rune_(0x0042)); // 'B' Upper } /** @brief Stream the sentence family's long-range straddling constructions for a given @p gap to @p sink. */ -static void utf8_sentences_straddles_(std::mt19937 & /*rng*/, std::size_t gap, utf8_run_sink_t sink, void *context) { +static void utf8_sentences_straddles_(std::mt19937 & /*generator*/, std::size_t gap, utf8_run_sink_t sink, + void *context) { std::string scratch; - utf8_straddle_sb8_lower_(scratch, gap), sink(context, scratch.data(), scratch.size()); - utf8_straddle_sb11_upper_(scratch, gap), sink(context, scratch.data(), scratch.size()); + utf8_sentences_straddle_sb8_lower_(scratch, gap), sink(context, scratch.data(), scratch.size()); + utf8_sentences_straddle_sb11_upper_(scratch, gap), sink(context, scratch.data(), scratch.size()); } /** @brief Sentence-biased snippets: ATerm/STerm + space + case across scripts, numeric, CJK stop, ParaSep. */ @@ -245,8 +246,8 @@ void test_utf8_sentences_rules() { /** @brief Malformed-input safety of the UTF-8 sentence kernels (serial / dispatched / icelake). */ void test_utf8_sentences_safety() { std::printf(" - testing malformed-input safety of UTF-8 sentence kernels...\n"); - utf8_segment_backend_t const serial_reference[] = {{"serial", sz_utf8_sentences_serial}}; - check_utf8_segment_safety_("sentence", span_over(serial_reference)); + utf8_segment_backend_t const serial_only[] = {{"serial", sz_utf8_sentences_serial}}; + check_utf8_segment_safety_("sentence", span_over(serial_only)); check_utf8_segment_safety_("sentence", span_over(utf8_sentences_backends)); std::printf(" sentence safety passed!\n"); } @@ -258,8 +259,8 @@ void test_utf8_sentences_safety() { /** @brief Serial-vs-ISA sentence differential over the hardened corpora (high-density + long-range). */ void test_utf8_sentences_all() { utf8_segment_corpora_t const corpora = utf8_sentences_corpora_(); - test_utf8_segment_equivalence_(sz_utf8_sentences_serial, span_over(utf8_sentences_backends), corpora, - scale_iterations(90)); // This family's share of the suite budget + check_utf8_segment_equivalence_(sz_utf8_sentences_serial, span_over(utf8_sentences_backends), corpora, + scale_iterations(90)); // This family's share of the suite budget } #pragma endregion // Drivers diff --git a/test/utf8_sentences.py b/test/utf8_sentences.py index f89b24e6..7ac68840 100644 --- a/test/utf8_sentences.py +++ b/test/utf8_sentences.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """UAX-29 sentence-boundary segmentation: sz.utf8_sentences and Str.utf8_sentences. -Mirrors the C++ scripts/test_utf8_sentences.cpp translation unit. +Mirrors the C++ test/utf8_sentences.cpp translation unit. Covers: sentence iteration and the Str method mirror across ASCII, Cyrillic, and ideographic terminators, malformed-byte and window-seam tiling safety, full UAX-29 SentenceBreakTest.txt diff --git a/test/utf8_tokens.cpp b/test/utf8_tokens.cpp index 5c787153..7682ff62 100644 --- a/test/utf8_tokens.cpp +++ b/test/utf8_tokens.cpp @@ -1,6 +1,6 @@ /** * @brief UTF-8 newline/whitespace boundary equivalence and C++ line/token splitting semantics. - * @file scripts/test_utf8_tokens.cpp + * @file test/utf8_tokens.cpp * @author Ash Vardanian * @date June 16, 2026 */ @@ -60,7 +60,7 @@ #error "This test requires C++11 or later." #endif -#include "stringzilla.hpp" // `global_random_generator`, `random_string` +#include "utf8.hpp" // `encoded_rune_`, `random_valid_utf8_`, `print_utf8_test_bytes_` namespace sz = ashvardanian::stringzilla; using namespace sz::scripts; @@ -68,19 +68,18 @@ using sz::literals::operator""_sv; // for `sz::string_view` #pragma region Helpers -/** @brief Prints one labeled hex dump line to `stderr`; used by the malformed-input safety test below. */ -static void print_utf8_test_bytes_(char const *label, char const *bytes, std::size_t length) { - std::fprintf(stderr, " %s (%zu bytes): ", label, length); - for (std::size_t index = 0; index < length; ++index) std::fprintf(stderr, "%02X ", (unsigned char)bytes[index]); - std::fprintf(stderr, "\n"); -} +/** @brief One expected boundary match: the byte offset where it starts and its byte length. */ +struct boundary_span_t { + sz_size_t offset; + sz_size_t length; +}; /** * @brief Runs one UTF-8 backend's counting and boundary-finding kernels over the known-answer anchors * and asserts the produced codepoint count and the emitted newline/whitespace (offset, length) * spans match the expected lists exactly. * - * Mirrors `check_sha256_unit_` in `test_hash.cpp`: the caller drives it once per backend (dispatched, + * Mirrors `check_sha256_unit_` in `hash.cpp`: the caller drives it once per backend (dispatched, * serial, and each natively-compiled kernel), so a wrong constant shared by the serial-vs-SIMD agreement * tests is still caught against these external ground-truth vectors. * @@ -101,24 +100,24 @@ static void check_utf8_unit_( sz_utf8_count_t count, sz_utf8_segmenter_t newlines, sz_utf8_segmenter_t whitespaces, // sz_cptr_t count_text, sz_size_t count_length, sz_size_t expected_count, // sz_cptr_t newline_text, sz_size_t newline_length, // - std::vector<std::pair<sz_size_t, sz_size_t>> const &expected_newlines, // + std::vector<boundary_span_t> const &expected_newlines, // sz_cptr_t whitespace_text, sz_size_t whitespace_length, // - std::vector<std::pair<sz_size_t, sz_size_t>> const &expected_whitespaces) { + std::vector<boundary_span_t> const &expected_whitespaces) { verify(count(count_text, count_length) == expected_count); - auto check_boundaries = [](sz_utf8_segmenter_t finder, sz_cptr_t text, sz_size_t length, - std::vector<std::pair<sz_size_t, sz_size_t>> const &expected) { + auto check_boundaries_ = [](sz_utf8_segmenter_t finder, sz_cptr_t text, sz_size_t length, + std::vector<boundary_span_t> const &expected) { sz_size_t found_offsets[16], found_lengths[16], consumed = 0; sz_size_t const found = finder(text, length, found_offsets, found_lengths, 16u, &consumed); verify(found == expected.size()); for (sz_size_t index = 0; index != found; ++index) { - verify(found_offsets[index] == expected[index].first); - verify(found_lengths[index] == expected[index].second); + verify(found_offsets[index] == expected[index].offset); + verify(found_lengths[index] == expected[index].length); } }; - check_boundaries(newlines, newline_text, newline_length, expected_newlines); - check_boundaries(whitespaces, whitespace_text, whitespace_length, expected_whitespaces); + check_boundaries_(newlines, newline_text, newline_length, expected_newlines); + check_boundaries_(whitespaces, whitespace_text, whitespace_length, expected_whitespaces); } /** @@ -220,7 +219,7 @@ void test_utf8_tokens_unit() { // single length-2 newline at byte 3 (CRLF merges into one match). char const newline_text[] = "a\nb\r\nc"; sz_size_t const newline_length = (sz_size_t)(sizeof(newline_text) - 1); - std::vector<std::pair<sz_size_t, sz_size_t>> const newline_spans = {{1u, 1u}, {3u, 2u}}; + std::vector<boundary_span_t> const newline_spans = {{1u, 1u}, {3u, 2u}}; // `sz_utf8_whitespaces`: the space is a length-1 match at byte 1, the tab at byte 3, and U+200A HAIR SPACE // (E2 80 8A) a length-3 match at byte 5 (there is no CRLF merging in the whitespace set - each codepoint is its @@ -231,13 +230,12 @@ void test_utf8_tokens_unit() { "d" "\xE2\x80\x8B\xE2\x80\x8C\xE2\x80\x8D" // U+200B/200C/200D (NOT whitespace) "e"; sz_size_t const whitespace_length = (sz_size_t)(sizeof(whitespace_text) - 1); - std::vector<std::pair<sz_size_t, sz_size_t>> const whitespace_spans = {{1u, 1u}, {3u, 1u}, {5u, 3u}}; + std::vector<boundary_span_t> const whitespace_spans = {{1u, 1u}, {3u, 1u}, {5u, 3u}}; // `sz_utf8_count` (6 bytes, 3 codepoints) plus the newline/whitespace boundary anchors, driven through // the dispatched (automatic kernel), serial, and each natively-compiled backend. - check_utf8_unit_(sz_utf8_count, sz_utf8_newlines, sz_utf8_whitespaces, // Dispatched - mixed, mixed_length, 3u, newline_text, newline_length, newline_spans, whitespace_text, - whitespace_length, whitespace_spans); + check_utf8_unit_(sz_utf8_count, sz_utf8_newlines, sz_utf8_whitespaces, mixed, mixed_length, 3u, newline_text, + newline_length, newline_spans, whitespace_text, whitespace_length, whitespace_spans); check_utf8_unit_(sz_utf8_count_serial, sz_utf8_newlines_serial, sz_utf8_whitespaces_serial, // serial mixed, mixed_length, 3u, newline_text, newline_length, newline_spans, whitespace_text, whitespace_length, whitespace_spans); @@ -324,6 +322,58 @@ void test_utf8_tokens_unit() { let_verify(auto l = lines("\n\x00"_sv), l.size() == 2); // Newline before NUL - split correctly } + // Test with `sz::string` - not just `sz::string_view` + { + sz::string multiline = "a\nb\nc"; + let_verify(auto l = multiline.utf8_split_newlines().template to<std::vector<std::string>>(), + l.size() == 3 && l[1] == "b"); + + sz::string words_str = "foo bar baz"; + let_verify(auto w = words_str.utf8_split_whitespaces().template to<std::vector<std::string>>(), + w.size() == 3 && w[2] == "baz"); + } + + // The kernel-named accessors yield the DELIMITER runs themselves (not the segments between). + { + // `utf8_newlines` on "a\nb\r\nc": the "\n" and "\r\n". + let_verify(auto n = sz::string_view("a\nb\r\nc").utf8_newlines().template to<std::vector<std::string>>(), + n.size() == 2 && n[0] == "\n" && n[1] == "\r\n"); + // `utf8_whitespaces` on "a b c": each whitespace codepoint is its own delimiter (runs are not coalesced). + let_verify(auto w = sz::string_view("a b c").utf8_whitespaces().template to<std::vector<std::string>>(), + w.size() == 3 && w[0] == " " && w[1] == " " && w[2] == " "); + } + + // `.with_separators()` interleaves segments and delimiters losslessly: concatenation reconstructs the input. + { + for (sz::string_view input : {sz::string_view("Hi, world"), sz::string_view("a\nb\nc"), + sz::string_view(" x "), sz::string_view(""), sz::string_view("plain")}) { + std::string rejoined; + for (auto piece : input.utf8_split_whitespaces().with_separators()) + rejoined.append(piece.data(), piece.size()); + let_verify(std::string round = rejoined, round == std::string(input.data(), input.size())); + } + } + + // `.skip_empty()`: a compile-time, branchless variant that drops empty segments, matching Rust/Python. + { + // Whitespace tokens across a double space: "a b" -> "a", "b" (the empty middle dropped). + let_verify( + auto t = + sz::string_view("a b").utf8_split_whitespaces().skip_empty().template to<std::vector<std::string>>(), + t.size() == 2 && t[0] == "a" && t[1] == "b"); + } +} + +/** + * @brief Known-answer whitespace-splitting vectors covering all 25 Unicode White_Space characters by byte length. + * + * Walks the 1-byte ASCII set, the 2-byte NEL/NBSP pair and the 17 three-byte space forms through the C++ + * `utf8_split_whitespaces` wrapper, and pins the Format characters U+200B/200C/200D as NOT whitespace, so a + * backend that widens the E2 80 [80-8A] block would shatter ZWJ emoji and Arabic/Indic words is caught here. + */ +void test_utf8_tokens_scripts_unit() { + std::printf(" - testing UTF-8 whitespace codepoints across Unicode scripts...\n"); + // Split by Unicode whitespace (25 total Unicode White_Space characters) { auto words = [](sz::string_view t) { @@ -434,47 +484,6 @@ void test_utf8_tokens_unit() { sz::string_view(long_mixed).utf8_split_whitespaces().template to<std::vector<std::string>>().size() == 50); // 50 words } - - // Test with `sz::string` - not just `sz::string_view` - { - sz::string multiline = "a\nb\nc"; - let_verify(auto l = multiline.utf8_split_newlines().template to<std::vector<std::string>>(), - l.size() == 3 && l[1] == "b"); - - sz::string words_str = "foo bar baz"; - let_verify(auto w = words_str.utf8_split_whitespaces().template to<std::vector<std::string>>(), - w.size() == 3 && w[2] == "baz"); - } - - // The kernel-named accessors yield the DELIMITER runs themselves (not the segments between). - { - // `utf8_newlines` on "a\nb\r\nc": the "\n" and "\r\n". - let_verify(auto n = sz::string_view("a\nb\r\nc").utf8_newlines().template to<std::vector<std::string>>(), - n.size() == 2 && n[0] == "\n" && n[1] == "\r\n"); - // `utf8_whitespaces` on "a b c": each whitespace codepoint is its own delimiter (runs are not coalesced). - let_verify(auto w = sz::string_view("a b c").utf8_whitespaces().template to<std::vector<std::string>>(), - w.size() == 3 && w[0] == " " && w[1] == " " && w[2] == " "); - } - - // `.with_separators()` interleaves segments and delimiters losslessly: concatenation reconstructs the input. - { - for (sz::string_view input : {sz::string_view("Hi, world"), sz::string_view("a\nb\nc"), - sz::string_view(" x "), sz::string_view(""), sz::string_view("plain")}) { - std::string rejoined; - for (auto piece : input.utf8_split_whitespaces().with_separators()) - rejoined.append(piece.data(), piece.size()); - let_verify(std::string round = rejoined, round == std::string(input.data(), input.size())); - } - } - - // `.skip_empty()`: a compile-time, branchless variant that drops empty segments, matching Rust/Python. - { - // Whitespace tokens across a double space: "a b" -> "a", "b" (the empty middle dropped). - let_verify( - auto t = - sz::string_view("a b").utf8_split_whitespaces().skip_empty().template to<std::vector<std::string>>(), - t.size() == 2 && t[0] == "a" && t[1] == "b"); - } } #pragma endregion // Unit @@ -503,8 +512,8 @@ struct utf8_tokens_backend_t { * * For each generated string, compares: * - sz_utf8_count: character counting - * - sz_utf8_find_newline: newline detection (position and matched length) - * - sz_utf8_find_whitespace: whitespace detection (position and matched length) + * - sz_utf8_newlines: newline detection (position and matched length) + * - sz_utf8_whitespaces: whitespace detection (position and matched length) * * @param reference Serial reference backend bundle (counting + newline/whitespace boundaries). * @param candidate ISA-specific backend bundle under test (counting + newline/whitespace boundaries). @@ -512,8 +521,8 @@ struct utf8_tokens_backend_t { * @param min_iterations Number of random strings to generate and check. */ template <typename reference_, typename candidate_> -void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // - std::size_t min_text_length, std::size_t min_iterations) { +void check_utf8_tokens_equivalence_(reference_ reference, candidate_ candidate, // + std::size_t min_text_length, std::size_t min_iterations) { // Adapt the bundle methods to the plain boundary-finder signature `drain_matches_`/`reconstruct_segments_` expect. auto reference_newlines = [&](sz_cptr_t data, sz_size_t length, sz_size_t *offsets, sz_size_t *lengths, @@ -612,7 +621,7 @@ void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // "\xE2\x81\x9F", "\xE3\x80\x80", // 3-byte }; - auto &rng = global_random_generator(); + auto &generator = global_random_generator(); std::size_t const utf8_content_count = span_over(utf8_content).size(); std::size_t const special_delimiter_count = span_over(special_chars).size(); std::size_t const total_strings_to_sample = utf8_content_count + special_delimiter_count; @@ -645,7 +654,7 @@ void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // // Build up a random string of at least `min_text_length` bytes while (text.size() < min_text_length) { - std::size_t random_content_index = content_dist(rng); + std::size_t random_content_index = content_dist(generator); if (random_content_index < utf8_content_count) { text.append(utf8_content[random_content_index]); } else { text.append(special_chars[random_content_index - utf8_content_count]); } } @@ -655,15 +664,15 @@ void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // std::size_t num_bytes_to_corrupt = text.size() / 10; std::uniform_int_distribution<std::size_t> byte_index_dist(0, text.size() - 1); for (std::size_t i = 0; i < num_bytes_to_corrupt; ++i) { - std::size_t byte_index = byte_index_dist(rng); + std::size_t byte_index = byte_index_dist(generator); text[byte_index] = '\0'; } check(text.data(), text.size()); // Swap 10% of bytes at random positions, creating malformed UTF-8 sequences for (std::size_t i = 0; i < num_bytes_to_corrupt; ++i) { - std::size_t byte_index_1 = byte_index_dist(rng); - std::size_t byte_index_2 = byte_index_dist(rng); + std::size_t byte_index_1 = byte_index_dist(generator); + std::size_t byte_index_2 = byte_index_dist(generator); std::swap(text[byte_index_1], text[byte_index_2]); } check(text.data(), text.size()); @@ -673,7 +682,7 @@ void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // // so the SIMD load alignment - not just the content - is swept against the serial reference. std::string alignment_probe; while (alignment_probe.size() < 256) { - std::size_t random_content_index = content_dist(rng); + std::size_t random_content_index = content_dist(generator); if (random_content_index < utf8_content_count) { alignment_probe.append(utf8_content[random_content_index]); } else { alignment_probe.append(special_chars[random_content_index - utf8_content_count]); } } @@ -696,12 +705,12 @@ void test_utf8_tokens_equivalence(reference_ reference, candidate_ candidate, // void test_utf8_tokens_safety() { std::printf(" - testing malformed-input safety of UTF-8 newline/whitespace kernels...\n"); - static constexpr std::size_t max_input_length = 70; + static constexpr std::size_t max_input_length = utf8_unit_capacity_k; // Drive every newline/whitespace boundary finder shipped on this target over one malformed input. auto check = [&](char const *input, std::size_t input_length) { sz_size_t boundary_offsets[max_input_length + 1], boundary_lengths[max_input_length + 1]; - auto check_boundaries = [&](sz_utf8_segmenter_t finder, char const *finder_name) { + auto check_boundaries_ = [&](sz_utf8_segmenter_t finder, char const *finder_name) { sz_size_t bytes_consumed = 0; sz_size_t const found = finder(input, (sz_size_t)input_length, boundary_offsets, boundary_lengths, (sz_size_t)(max_input_length + 1), &bytes_consumed); @@ -716,81 +725,45 @@ void test_utf8_tokens_safety() { }; // Serial baseline and the dispatched (automatic kernel resolution) entry points face the same contract. - check_boundaries(sz_utf8_newlines_serial, "serial newline finder"); - check_boundaries(sz_utf8_whitespaces_serial, "serial whitespace finder"); - check_boundaries(sz_utf8_newlines, "dispatched newline finder"); - check_boundaries(sz_utf8_whitespaces, "dispatched whitespace finder"); + check_boundaries_(sz_utf8_newlines_serial, "serial newline finder"); + check_boundaries_(sz_utf8_whitespaces_serial, "serial whitespace finder"); + check_boundaries_(sz_utf8_newlines, "dispatched newline finder"); + check_boundaries_(sz_utf8_whitespaces, "dispatched whitespace finder"); #if SZ_USE_HASWELL - check_boundaries(sz_utf8_newlines_haswell, "haswell newline finder"); - check_boundaries(sz_utf8_whitespaces_haswell, "haswell whitespace finder"); + check_boundaries_(sz_utf8_newlines_haswell, "haswell newline finder"); + check_boundaries_(sz_utf8_whitespaces_haswell, "haswell whitespace finder"); #endif #if SZ_USE_ICELAKE - check_boundaries(sz_utf8_newlines_icelake, "icelake newline finder"); - check_boundaries(sz_utf8_whitespaces_icelake, "icelake whitespace finder"); + check_boundaries_(sz_utf8_newlines_icelake, "icelake newline finder"); + check_boundaries_(sz_utf8_whitespaces_icelake, "icelake whitespace finder"); #endif #if SZ_USE_NEON - check_boundaries(sz_utf8_newlines_neon, "neon newline finder"); - check_boundaries(sz_utf8_whitespaces_neon, "neon whitespace finder"); + check_boundaries_(sz_utf8_newlines_neon, "neon newline finder"); + check_boundaries_(sz_utf8_whitespaces_neon, "neon whitespace finder"); #endif #if SZ_USE_SVE2 - check_boundaries(sz_utf8_newlines_sve2, "sve2 newline finder"); - check_boundaries(sz_utf8_whitespaces_sve2, "sve2 whitespace finder"); + check_boundaries_(sz_utf8_newlines_sve2, "sve2 newline finder"); + check_boundaries_(sz_utf8_whitespaces_sve2, "sve2 whitespace finder"); #endif #if SZ_USE_V128 - check_boundaries(sz_utf8_newlines_v128, "v128 newline finder"); - check_boundaries(sz_utf8_whitespaces_v128, "v128 whitespace finder"); + check_boundaries_(sz_utf8_newlines_v128, "v128 newline finder"); + check_boundaries_(sz_utf8_whitespaces_v128, "v128 whitespace finder"); #endif #if SZ_USE_RVV - check_boundaries(sz_utf8_newlines_rvv, "rvv newline finder"); - check_boundaries(sz_utf8_whitespaces_rvv, "rvv whitespace finder"); + check_boundaries_(sz_utf8_newlines_rvv, "rvv newline finder"); + check_boundaries_(sz_utf8_whitespaces_rvv, "rvv whitespace finder"); #endif #if SZ_USE_LASX - check_boundaries(sz_utf8_newlines_lasx, "lasx newline finder"); - check_boundaries(sz_utf8_whitespaces_lasx, "lasx whitespace finder"); + check_boundaries_(sz_utf8_newlines_lasx, "lasx newline finder"); + check_boundaries_(sz_utf8_whitespaces_lasx, "lasx whitespace finder"); #endif #if SZ_USE_POWERVSX - check_boundaries(sz_utf8_newlines_powervsx, "powervsx newline finder"); - check_boundaries(sz_utf8_whitespaces_powervsx, "powervsx whitespace finder"); + check_boundaries_(sz_utf8_newlines_powervsx, "powervsx newline finder"); + check_boundaries_(sz_utf8_whitespaces_powervsx, "powervsx whitespace finder"); #endif }; - char input[max_input_length]; - - // The named adversarial shapes the task calls out, exercised directly. - check("\x80", 1); // Lone continuation byte - check("\xC0\x80", 2); // Overlong encoding of NUL - check("\xED\xA0\x80", 3); // Surrogate-encoded codepoint (U+D800) - check("hello\xF0\x9F\x98", 8); // Truncated 4-byte sequence at the very end - - // All 256 single bytes: truncated leads, stray continuations, 0xFE/0xFF. Both the single-byte sweep and the - // pair sweep below stride each dimension, so a low multiplier samples the whole space instead of a prefix. - std::size_t const byte_step = sweep_stride(256); - for (std::size_t byte = 0; byte < 256; byte += byte_step) { - input[0] = (char)byte; - check(input, 1); - } - - // All 65,536 byte pairs: every lead x continuation interaction, including overlong and surrogate shapes. - // Walked flat so a strided run samples both bytes evenly, and its cost stays proportional to the multiplier. - for (std::size_t pair = 0; pair < 65536; pair += sweep_stride(65536)) { - input[0] = (char)(pair >> 8); - input[1] = (char)(pair & 0xFF); - check(input, 2); - } - - // Random garbage buffers spanning whole SIMD chunks, at every sub-cache-line alignment. - std::size_t const random_inputs = scale_iterations(10000); - auto &rng = global_random_generator(); - std::uniform_int_distribution<std::size_t> length_distribution(1, max_input_length); - std::uniform_int_distribution<int> byte_distribution(0, 255); - for (std::size_t iteration = 0; iteration != random_inputs; ++iteration) { - std::size_t const input_length = length_distribution(rng); - for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(rng); - for_each_cacheline_offset_(input_length, [&](sz_ptr_t buffer, std::size_t /*offset*/) { - std::memcpy(buffer, input, input_length); - check(buffer, input_length); - }); - } + for_each_adversarial_utf8_input_(global_random_generator(), scale_iterations(10000), check); std::printf(" malformed-input safety passed!\n"); } @@ -842,59 +815,20 @@ void test_utf8_tokens_all() { // Each iteration drains a 4 KB input through six capacities down to 1, re-entering the kernel once per match. // The input count is this family's share of the suite budget, sized against its siblings. for (utf8_tokens_backend_t const &backend : utf8_tokens_backends) - test_utf8_tokens_equivalence(serial, backend, 4000, scale_iterations(250)); + check_utf8_tokens_equivalence_(serial, backend, 4000, scale_iterations(250)); } #pragma endregion // Drivers #pragma region Delimiter Helpers -/** @brief Append one codepoint to @p text as UTF-8 via `sz_rune_encode` (silently skips invalid runes). */ -static void append_codepoint_(std::string &text, sz_rune_t codepoint) { - sz_u8_t bytes[4]; - sz_rune_length_t const length = sz_rune_encode(codepoint, bytes); - if (length == sz_rune_invalid_k) return; - text.append((char const *)bytes, (std::size_t)length); -} - -/** - * @brief Builds a random, well-formed UTF-8 string whose codepoints span all four byte-widths and every - * 1->2->3->4 transition, mixing delimiter and non-delimiter codepoints so the scan kernels hit their - * mixed-width paths and resolve a match at every alignment. - */ -static std::string random_valid_utf8_(std::size_t target_codepoints, std::mt19937 &rng) { - static struct { - sz_rune_t low, high; - } const ranges[] = { - {0x000020u, 0x00007Eu}, // 1-byte ASCII printable (space + punctuation + letters + digits) - {0x0000A1u, 0x0007FFu}, // 2-byte (Latin-1 symbols, Greek/Cyrillic letters) - {0x000800u, 0x00CFFFu}, // 3-byte (general punctuation, CJK; stays below U+D800 surrogates) - {0x010000u, 0x0EFFFFu}, // 4-byte (symbols, emoji; avoids the plane-ender noncharacters) - }; - std::uniform_int_distribution<int> width_pick(0, 3); - std::string text; - text.reserve(target_codepoints * 4); - for (std::size_t index = 0; index != target_codepoints; ++index) { - int const width = width_pick(rng); - std::uniform_int_distribution<sz_rune_t> codepoint(ranges[width].low, ranges[width].high); - std::size_t const before = text.size(); - while (text.size() == before) append_codepoint_(text, codepoint(rng)); - } - return text; -} - /** * @brief The UTF-8 delimiter segmenters compiled on this target. The always-present `dispatched` entry keeps the * table non-empty on a baseline build with no SIMD tier, and the single ladder is shared by the unit, * safety and equivalence drivers so their ISA coverage cannot diverge. Only serial, Haswell, Ice Lake, * NEON and SVE2 implement `sz_utf8_delimiters`; every other target dispatches to serial. */ -struct utf8_delimiters_backend_t { - char const *name; - sz_utf8_segmenter_t finder; -}; - -static utf8_delimiters_backend_t const utf8_delimiters_backends[] = { +static utf8_segment_backend_t const utf8_delimiters_backends[] = { {"dispatched", sz_utf8_delimiters}, #if SZ_USE_HASWELL {"haswell", sz_utf8_delimiters_haswell}, @@ -965,7 +899,7 @@ void test_utf8_delimiters_unit() { } std::vector<sz_size_t> offsets, lengths; - auto check_backend = [&](sz_utf8_segmenter_t finder) { + auto check_backend_ = [&](sz_utf8_segmenter_t finder) { for (auto const &one : cases) { drain_matches_(finder, one.text, one.length, one.length + 1, offsets, lengths); verify(offsets.size() == one.expected_count && "Delimiter count mismatch"); @@ -986,8 +920,8 @@ void test_utf8_delimiters_unit() { "Resume offset must be the end of the last emitted delimiter, not the vector window's edge"); } }; - check_backend(sz_utf8_delimiters_serial); - for (utf8_delimiters_backend_t const &backend : utf8_delimiters_backends) check_backend(backend.finder); + check_backend_(sz_utf8_delimiters_serial); + for (utf8_segment_backend_t const &backend : utf8_delimiters_backends) check_backend_(backend.finder); // The C++ range wrappers over the same kernel, on the same hand-verifiable inputs. { @@ -1022,9 +956,9 @@ void test_utf8_delimiters_unit() { * well-formed inputs: the full (offset, length) match list must agree, both in one shot and when the * candidate is drained through a tiny capacity so its `bytes_consumed` resume path is exercised. */ -static void test_utf8_delimiters_equivalence(sz_utf8_segmenter_t finder_serial, sz_utf8_segmenter_t finder_candidate, - sz_size_t inputs) { - auto &rng = global_random_generator(); +static void check_utf8_delimiters_equivalence_(sz_utf8_segmenter_t finder_serial, sz_utf8_segmenter_t finder_candidate, + sz_size_t inputs) { + auto &generator = global_random_generator(); std::vector<sz_size_t> serial_offsets, serial_lengths, candidate_offsets, candidate_lengths, resumed_offsets, resumed_lengths; @@ -1052,7 +986,7 @@ static void test_utf8_delimiters_equivalence(sz_utf8_segmenter_t finder_serial, sz_size_t const ladder[] = {0u, 1u, 2u, 15u, 16u, 17u, 31u, 32u, 33u, 63u, 64u, 65u, 100u, 200u}; for (sz_size_t codepoints : ladder) { - std::string const text = random_valid_utf8_(codepoints, rng); + std::string const text = random_valid_utf8_(codepoints, generator); check(text.data(), (sz_size_t)text.size()); } @@ -1074,7 +1008,7 @@ static void test_utf8_delimiters_equivalence(sz_utf8_segmenter_t finder_serial, std::uniform_int_distribution<std::size_t> codepoint_distribution(0, 300); for (sz_size_t iteration = 0; iteration != inputs; ++iteration) { - std::string const text = random_valid_utf8_(codepoint_distribution(rng), rng); + std::string const text = random_valid_utf8_(codepoint_distribution(generator), generator); // The probe buffer itself is handed to the kernels: copying it back into a fresh `std::string` would // hand them whatever alignment the allocator picked, and the sweep would test one alignment repeatedly. for_each_cacheline_offset_(text.size(), [&](sz_ptr_t buffer, std::size_t /*offset*/) { @@ -1091,7 +1025,6 @@ static void test_utf8_delimiters_equivalence(sz_utf8_segmenter_t finder_serial, /** @brief Feeds malformed / invalid UTF-8 through one backend, asserting in-bounds, ascending, well-formed output. */ static void check_utf8_delimiters_safety_(sz_utf8_segmenter_t finder, std::size_t random_inputs = scale_iterations(2500)) { - static constexpr std::size_t max_input_length = 70; std::vector<sz_size_t> offsets, lengths; // Malformed bytes meet a capacity too small to hold the batch, so the resume path - not just the one-shot @@ -1110,32 +1043,7 @@ static void check_utf8_delimiters_safety_(sz_utf8_segmenter_t finder, } }; - char input[max_input_length]; - check("\x80", 1); // Lone continuation byte - check("\xC0\x80", 2); // Overlong encoding of NUL - check("\xED\xA0\x80", 3); // Surrogate-encoded codepoint (U+D800) - check("hello\xF0\x9F\x98", 8); // Truncated 4-byte sequence at the very end - - // Both sweeps stride every dimension, so a low multiplier samples the whole space instead of a prefix. - std::size_t const byte_step = sweep_stride(256); - for (std::size_t byte = 0; byte < 256; byte += byte_step) { input[0] = (char)byte, check(input, 1); } - // Walked flat so a strided run samples both bytes evenly, and its cost stays proportional to the multiplier. - for (std::size_t pair = 0; pair < 65536; pair += sweep_stride(65536)) { - input[0] = (char)(pair >> 8), input[1] = (char)(pair & 0xFF); - check(input, 2); - } - - auto &rng = global_random_generator(); - std::uniform_int_distribution<std::size_t> length_distribution(1, max_input_length); - std::uniform_int_distribution<int> byte_distribution(0, 255); - for (std::size_t iteration = 0; iteration != random_inputs; ++iteration) { - std::size_t const input_length = length_distribution(rng); - for (std::size_t index = 0; index != input_length; ++index) input[index] = (char)byte_distribution(rng); - for_each_cacheline_offset_(input_length, [&](sz_ptr_t buffer, std::size_t /*offset*/) { - std::memcpy(buffer, input, input_length); - check(buffer, input_length); - }); - } + for_each_adversarial_utf8_input_(global_random_generator(), random_inputs, check); } #pragma endregion // Safety @@ -1146,7 +1054,7 @@ static void check_utf8_delimiters_safety_(sz_utf8_segmenter_t finder, void test_utf8_delimiters_safety() { std::printf(" - testing malformed-input safety of UTF-8 delimiter kernels...\n"); check_utf8_delimiters_safety_(sz_utf8_delimiters_serial); - for (utf8_delimiters_backend_t const &backend : utf8_delimiters_backends) + for (utf8_segment_backend_t const &backend : utf8_delimiters_backends) check_utf8_delimiters_safety_(backend.finder); std::printf(" malformed-input safety passed!\n"); } @@ -1154,8 +1062,8 @@ void test_utf8_delimiters_safety() { /** @brief Drive the serial-vs-SIMD UTF-8 delimiter differential across every backend compiled on this target. */ void test_utf8_delimiters_all() { sz_size_t const inputs = (sz_size_t)scale_iterations(700); - for (utf8_delimiters_backend_t const &backend : utf8_delimiters_backends) - test_utf8_delimiters_equivalence(sz_utf8_delimiters_serial, backend.finder, inputs); + for (utf8_segment_backend_t const &backend : utf8_delimiters_backends) + check_utf8_delimiters_equivalence_(sz_utf8_delimiters_serial, backend.finder, inputs); } #pragma endregion // Drivers diff --git a/test/utf8_wordbreaks.cpp b/test/utf8_wordbreaks.cpp index 673eba32..05b6bf7b 100644 --- a/test/utf8_wordbreaks.cpp +++ b/test/utf8_wordbreaks.cpp @@ -1,7 +1,7 @@ /** * @brief UAX-29 word-boundary (Word_Break) tests: known-answer goldens, malformed-input safety, and the * serial-vs-ISA differential over hardened corpora. - * @file scripts/test_utf8_wordbreaks.cpp + * @file test/utf8_wordbreaks.cpp * @author Ash Vardanian */ #undef NDEBUG // ! Enable all assertions for testing @@ -23,7 +23,7 @@ #include <string> // `std::string` #include <vector> // `std::vector` -#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `test_stringzilla.hpp`) +#include "utf8.hpp" // shared segmentation harness (pulls in StringZilla + `stringzilla.hpp`) #pragma region Unit @@ -41,7 +41,7 @@ static utf8_unit_case_t const utf8_wordbreaks_unit_cases[] = { }; /** @brief Known-answer property table for `sz_rune_is_word_char` (UAX-29 word-character classification). */ -static void test_utf8_wordbreaks_classification_() { +static void check_utf8_wordbreaks_classification_() { // ASCII letters, digits, underscore, and the mid-word apostrophe are word characters. verify(sz_rune_is_word_char('A') == sz_true_k); verify(sz_rune_is_word_char('z') == sz_true_k); @@ -134,7 +134,7 @@ static void check_utf8_wordbreaks_lengths_(char const *label, sz_utf8_segmenter_ * Regression corpus for the cross-window bridge-shadow carry: a failed bridge must still emit the break * at the mid, keep WB7a's Hebrew x Single_Quote join, and leave the mid as the effective left context. */ -static void test_utf8_wordbreaks_deferred_mid_() { +static void check_utf8_wordbreaks_deferred_mid_() { static sz_size_t const run_lengths[] = {3, 8, 15, 31, 32, 33, 100}; static char const combining_grave[] = "\xCC\x80"; // U+0300, Word_Break=Extend static char const hebrew_he[] = "\xD7\x94"; // U+05D4, Hebrew_Letter @@ -179,8 +179,8 @@ static void test_utf8_wordbreaks_deferred_mid_() { void test_utf8_wordbreaks_unit() { std::printf(" - testing UTF-8 word-break known-answer vectors...\n"); - test_utf8_wordbreaks_classification_(); - test_utf8_wordbreaks_deferred_mid_(); + check_utf8_wordbreaks_classification_(); + check_utf8_wordbreaks_deferred_mid_(); check_utf8_segment_unit_("word", sz_utf8_wordbreaks_serial, span_over(utf8_wordbreaks_unit_cases)); for (utf8_segment_backend_t const &backend : utf8_wordbreaks_backends) @@ -234,8 +234,8 @@ static sz::string_view const utf8_wordbreaks_motifs[] = { }; /** - * @brief Multi-window seam regressions (each > 64 bytes): WB15/16 Regional_Indicator parity and WB6/7/11/12 - * Mid-bridge carry once miscounted across the 64-byte window boundary. Stored as raw bytes so the + * @brief Multi-window seam regressions (each > 64 bytes): WB15/16 Regional_Indicator parity and the WB6/7/11/12 + * Mid-bridge carry state, each pinned across the 64-byte window boundary. Stored as raw bytes so the * differential driver feeds them to serial-vs-ISA directly (no inline agreement asserts). */ static sz::string_view const utf8_wordbreaks_seam_regressions[] = { @@ -258,7 +258,7 @@ static sz::string_view const utf8_wordbreaks_seam_regressions[] = { /** @brief Katakana run @p link_count codepoints long (WB13 Katakana x Katakana), into @p out (cleared first). */ static void utf8_wordbreaks_dense_katakana_(std::string &out, std::size_t link_count) { out.clear(); - for (std::size_t index = 0; index != link_count; ++index) append_codepoint_(out, 0x30AB); // ã‚Ģ + for (std::size_t index = 0; index != link_count; ++index) out.append(encoded_rune_(0x30AB)); // ã‚Ģ } /** @brief Numeric run with MidNum and Extend marks, @p link_count groups (WB11/12 + Extend), into @p out. */ @@ -266,8 +266,8 @@ static void utf8_wordbreaks_dense_numeric_(std::string &out, std::size_t link_co out.clear(); for (std::size_t index = 0; index != link_count; ++index) { out.append("12"); - append_codepoint_(out, 0x0301); // Extend combining mark inside a number - out.append(",34 "); // MidNum comma + out.append(encoded_rune_(0x0301)); // Extend combining mark inside a number + out.append(",34 "); // MidNum comma } } @@ -275,9 +275,9 @@ static void utf8_wordbreaks_dense_numeric_(std::string &out, std::size_t link_co static void utf8_wordbreaks_dense_midletter_(std::string &out, std::size_t link_count) { out.clear(); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x0061); // 'a' - append_codepoint_(out, (index & 1u) ? 0x00B7 : 0x0027); // MIDDLE DOT or apostrophe - append_codepoint_(out, 0x0062); // 'b' + out.append(encoded_rune_(0x0061)); // 'a' + out.append(encoded_rune_((index & 1u) ? 0x00B7 : 0x0027)); // MIDDLE DOT or apostrophe + out.append(encoded_rune_(0x0062)); // 'b' } } @@ -285,17 +285,17 @@ static void utf8_wordbreaks_dense_midletter_(std::string &out, std::size_t link_ static void utf8_wordbreaks_dense_hebrew_quote_(std::string &out, std::size_t link_count) { out.clear(); for (std::size_t index = 0; index != link_count; ++index) { - append_codepoint_(out, 0x05D0); // א - append_codepoint_(out, 0x0027); // single quote - append_codepoint_(out, 0x05D1); // ב + out.append(encoded_rune_(0x05D0)); // א + out.append(encoded_rune_(0x0027)); // single quote + out.append(encoded_rune_(0x05D1)); // ב } } /** @brief Stream the word family's high-density homogeneous runs (each spans several 64-byte windows) to @p sink. */ -static void utf8_wordbreaks_dense_runs_(std::mt19937 &rng, utf8_run_sink_t sink, void *context) { +static void utf8_wordbreaks_dense_runs_(std::mt19937 &generator, utf8_run_sink_t sink, void *context) { std::string scratch; - std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(rng); - utf8_dense_regional_indicators_(scratch, rng, wide_count), sink(context, scratch.data(), scratch.size()); + std::size_t const wide_count = std::uniform_int_distribution<std::size_t>(60, 220)(generator); + utf8_dense_regional_indicators_(scratch, generator, wide_count), sink(context, scratch.data(), scratch.size()); utf8_wordbreaks_dense_katakana_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_wordbreaks_dense_numeric_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); utf8_wordbreaks_dense_midletter_(scratch, wide_count), sink(context, scratch.data(), scratch.size()); @@ -303,9 +303,9 @@ static void utf8_wordbreaks_dense_runs_(std::mt19937 &rng, utf8_run_sink_t sink, } /** @brief Stream the word family's long-range straddling constructions for a given @p gap to @p sink. */ -static void utf8_wordbreaks_straddles_(std::mt19937 &rng, std::size_t gap, utf8_run_sink_t sink, void *context) { +static void utf8_wordbreaks_straddles_(std::mt19937 &generator, std::size_t gap, utf8_run_sink_t sink, void *context) { std::string scratch; - utf8_dense_regional_indicators_(scratch, rng, gap); + utf8_dense_regional_indicators_(scratch, generator, gap); scratch.append("a"); // ASCII tail forces the WB15/16 parity decision after the long run sink(context, scratch.data(), scratch.size()); utf8_wordbreaks_dense_midletter_(scratch, gap), sink(context, scratch.data(), scratch.size()); @@ -422,8 +422,13 @@ void test_utf8_wordbreaks_safety() { /** @brief Serial-vs-ISA word differential over the hardened corpora (high-density + long-range + seam regressions). */ void test_utf8_wordbreaks_all() { // The iteration count is this family's share of the suite budget, sized against its siblings. - test_utf8_segment_equivalence_(sz_utf8_wordbreaks_serial, span_over(utf8_wordbreaks_backends), - utf8_wordbreaks_corpora_(), scale_iterations(20)); + check_utf8_segment_equivalence_(sz_utf8_wordbreaks_serial, span_over(utf8_wordbreaks_backends), + utf8_wordbreaks_corpora_(), scale_iterations(20)); + + // The streaming segmenter against the per-position WB1-WB16 transcription, which nothing else calls. + for (sz::string_view const motif : span_over(utf8_wordbreaks_motifs)) + check_utf8_segment_against_oracle_("word", sz_utf8_wordbreaks_serial, sz_utf8_is_word_boundary_serial, + motif.data(), motif.size()); } #pragma endregion // Drivers