Skip to content

Commit 86b45f7

Browse files
authored
Merge branch 'main' into codex/pr-1814-standalone-build
2 parents de7e247 + 8343f82 commit 86b45f7

112 files changed

Lines changed: 19874 additions & 59 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/build.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,32 @@ jobs:
219219
artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}"
220220
file_to_upload: "java/cuvs-java/target/"
221221
sha: ${{ inputs.sha }}
222+
lucene-build:
223+
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against.
224+
needs: [java-build-matrix, java-build]
225+
permissions:
226+
actions: read
227+
contents: read
228+
id-token: write
229+
packages: read
230+
pull-requests: read
231+
secrets: inherit # zizmor: ignore[secrets-inherit]
232+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
233+
# Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions.
234+
# If these jobs start producing artifacts, the names will have to differentiate between CUDA versions.
235+
strategy:
236+
fail-fast: false
237+
matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }}
238+
with:
239+
build_type: ${{ inputs.build_type || 'branch' }}
240+
branch: ${{ inputs.branch }}
241+
arch: "amd64"
242+
date: ${{ inputs.date }}
243+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
244+
script: "ci/build_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}"
245+
artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
246+
file_to_upload: "java/cuvs-lucene/target/"
247+
sha: ${{ inputs.sha }}
222248
python-build:
223249
needs: [build-details, cpp-build]
224250
permissions:

.github/workflows/pr.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ jobs:
2626
- rocky8-clib-tests
2727
- conda-java-build-and-tests-matrix
2828
- conda-java-build-and-tests
29+
- conda-lucene-build-and-tests
2930
- rust-build-matrix
3031
- rust-build
3132
- go-build-matrix
@@ -143,12 +144,17 @@ jobs:
143144
- '!README.md'
144145
- '!SECURITY.md'
145146
- '!ci/build_docs.sh'
147+
- '!ci/build_go.sh'
148+
- '!ci/build_java.sh'
149+
- '!ci/build_lucene.sh'
146150
- '!ci/build_python.sh'
151+
- '!ci/build_rust.sh'
147152
- '!ci/build_wheel*.sh'
148153
- '!ci/check_style.sh'
149154
- '!ci/release/update-version.sh'
150155
- '!ci/run_cuvs_pytests.sh'
151156
- '!ci/test_java.sh'
157+
- '!ci/test_lucene.sh'
152158
- '!ci/test_python.sh'
153159
- '!ci/test_wheel_cuvs.sh'
154160
- '!ci/validate_wheel.sh'
@@ -238,12 +244,14 @@ jobs:
238244
- '!.yamllint.yaml'
239245
- '!ci/build_go.sh'
240246
- '!ci/build_java.sh'
247+
- '!ci/build_lucene.sh'
241248
- '!ci/build_rust.sh'
242249
- '!ci/check_style.sh'
243250
- '!ci/release/update-version.sh'
244251
- '!ci/run_ctests.sh'
245252
- '!ci/test_cpp.sh'
246253
- '!ci/test_java.sh'
254+
- '!ci/test_lucene.sh'
247255
- '!ci/test_standalone_c.sh'
248256
- '!ci/test_wheel_cuvs.sh'
249257
- '!ci/validate_wheel.sh'
@@ -289,6 +297,7 @@ jobs:
289297
- '!ci/build_docs.sh'
290298
- '!ci/build_go.sh'
291299
- '!ci/build_java.sh'
300+
- '!ci/build_lucene.sh'
292301
- '!ci/build_python.sh'
293302
- '!ci/build_rust.sh'
294303
- '!ci/build_standalone_c.sh'
@@ -297,6 +306,7 @@ jobs:
297306
- '!ci/run_ctests.sh'
298307
- '!ci/test_cpp.sh'
299308
- '!ci/test_java.sh'
309+
- '!ci/test_lucene.sh'
300310
- '!ci/test_python.sh'
301311
- '!ci/test_standalone_c.sh'
302312
- '!ci/validate_wheel.sh'
@@ -346,6 +356,7 @@ jobs:
346356
- '!ci/build_docs.sh'
347357
- '!ci/build_go.sh'
348358
- '!ci/build_java.sh'
359+
- '!ci/build_lucene.sh'
349360
- '!ci/build_python.sh'
350361
- '!ci/build_wheel*.sh'
351362
- '!ci/check_style.sh'
@@ -354,6 +365,7 @@ jobs:
354365
- '!ci/run_cuvs_pytests.sh'
355366
- '!ci/test_cpp.sh'
356367
- '!ci/test_java.sh'
368+
- '!ci/test_lucene.sh'
357369
- '!ci/test_python.sh'
358370
- '!ci/test_wheel_cuvs.sh'
359371
- '!ci/validate_wheel.sh'
@@ -394,6 +406,7 @@ jobs:
394406
- '!SECURITY.md'
395407
- '!ci/build_docs.sh'
396408
- '!ci/build_java.sh'
409+
- '!ci/build_lucene.sh'
397410
- '!ci/build_python.sh'
398411
- '!ci/build_rust.sh'
399412
- '!ci/build_wheel*.sh'
@@ -403,6 +416,7 @@ jobs:
403416
- '!ci/run_cuvs_pytests.sh'
404417
- '!ci/test_cpp.sh'
405418
- '!ci/test_java.sh'
419+
- '!ci/test_lucene.sh'
406420
- '!ci/test_python.sh'
407421
- '!ci/test_standalone_c.sh'
408422
- '!ci/test_wheel_cuvs.sh'
@@ -623,6 +637,31 @@ jobs:
623637
script: "ci/test_java.sh"
624638
artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}"
625639
file_to_upload: "java/cuvs-java/target/"
640+
conda-lucene-build-and-tests:
641+
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and
642+
# reuses its matrix so the two always agree on the set of CUDA versions.
643+
needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests]
644+
permissions:
645+
actions: read
646+
contents: read
647+
id-token: write
648+
packages: read
649+
pull-requests: read
650+
secrets: inherit # zizmor: ignore[secrets-inherit]
651+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
652+
# Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions.
653+
# If these jobs start producing artifacts, the names will have to differentiate between CUDA versions.
654+
strategy:
655+
fail-fast: false
656+
matrix: ${{ fromJSON(needs.conda-java-build-and-tests-matrix.outputs.matrix) }}
657+
with:
658+
build_type: pull-request
659+
node_type: "gpu-l4-latest-1"
660+
arch: "amd64"
661+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
662+
script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}"
663+
artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
664+
file_to_upload: "java/cuvs-lucene/target/"
626665
rust-build-matrix:
627666
needs: [conda-cpp-build, changed-files]
628667
permissions:

.github/workflows/test.yaml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,34 @@ jobs:
104104
script: "ci/test_java.sh"
105105
artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}"
106106
file_to_upload: "java/cuvs-java/target/"
107+
conda-lucene-build-and-tests:
108+
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and
109+
# reuses its matrix so the two always agree on the set of CUDA versions.
110+
needs: [conda-java-build-and-tests-matrix, conda-java-build-and-tests]
111+
permissions:
112+
actions: read
113+
contents: read
114+
id-token: write
115+
packages: read
116+
pull-requests: read
117+
secrets: inherit # zizmor: ignore[secrets-inherit]
118+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
119+
# Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions.
120+
# If these jobs start producing artifacts, the names will have to differentiate between CUDA versions.
121+
strategy:
122+
fail-fast: false
123+
matrix: ${{ fromJSON(needs.conda-java-build-and-tests-matrix.outputs.matrix) }}
124+
with:
125+
build_type: ${{ inputs.build_type }}
126+
branch: ${{ inputs.branch }}
127+
date: ${{ inputs.date }}
128+
sha: ${{ inputs.sha }}
129+
node_type: "gpu-l4-latest-1"
130+
arch: "amd64"
131+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
132+
script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}"
133+
artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
134+
file_to_upload: "java/cuvs-lucene/target/"
107135
wheel-tests-cuvs:
108136
permissions:
109137
actions: read

.pre-commit-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ repos:
9797
name: spotless-fmt
9898
entry: ci/checks/run_spotless.sh
9999
pass_filenames: false
100-
files: ^java/cuvs-java/src/.*\.java$
100+
files: ^java/(cuvs-java|cuvs-lucene)/([^/]+/)?src/.*\.java$
101101
exclude: .*/panama/.*
102102
language: script
103103
- id: clang-format-with-cmake-placeholders

build.sh

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ ARGS=$*
1919
# scripts, and that this script resides in the repo dir!
2020
REPODIR=$(cd "$(dirname "$0")"; pwd)
2121

22-
VALIDARGS="clean libcuvs python rust go java docs tests bench-ann examples tarball --tarball-build-tests --uninstall -v -g -n --allgpuarch --no-mg --mnmg-tests --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h --run-java-tests"
22+
VALIDARGS="clean libcuvs python rust go java lucene docs tests bench-ann examples tarball --tarball-build-tests --uninstall -v -g -n --allgpuarch --no-mg --mnmg-tests --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h --run-java-tests"
2323
HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>] [--limit-bench-ann=<targets>] [--build-metrics=<filename>]
2424
where <target> is:
2525
clean - remove all existing build artifacts and configuration (start over)
@@ -29,6 +29,7 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
2929
rust - build the cuvs Rust bindings
3030
go - build the cuvs Go bindings
3131
java - build the cuvs Java bindings
32+
lucene - build the cuvs-lucene Apache Lucene codecs (requires the Java bindings)
3233
docs - build the documentation
3334
tests - build the tests
3435
bench-ann - build end-to-end ann benchmarks
@@ -76,7 +77,8 @@ FERN_DOCS_DIR=${REPODIR}/fern
7677
PYTHON_BUILD_DIR=${REPODIR}/python/cuvs/_skbuild
7778
RUST_BUILD_DIR=${REPODIR}/rust/target
7879
JAVA_BUILD_DIR=${REPODIR}/java/cuvs-java/target
79-
BUILD_DIRS="${LIBCUVS_BUILD_DIR} ${PYTHON_BUILD_DIR} ${RUST_BUILD_DIR} ${JAVA_BUILD_DIR}"
80+
LUCENE_BUILD_DIR=${REPODIR}/java/cuvs-lucene/target
81+
BUILD_DIRS="${LIBCUVS_BUILD_DIR} ${PYTHON_BUILD_DIR} ${RUST_BUILD_DIR} ${JAVA_BUILD_DIR} ${LUCENE_BUILD_DIR}"
8082

8183
# Set defaults for vars modified by flags to this script
8284
CMAKE_LOG_LEVEL=""
@@ -557,6 +559,19 @@ if (( NUMARGS == 0 )) || hasArg java; then
557559
fi
558560
fi
559561

562+
# Build the cuvs-lucene codecs
563+
if (( NUMARGS == 0 )) || hasArg lucene; then
564+
if ! hasArg java; then
565+
echo "Please add 'java' to this script's arguments (ex. './build.sh libcuvs java lucene') if the cuvs Java bindings are not already built"
566+
fi
567+
cd "${REPODIR}"/java/cuvs-lucene
568+
if hasArg --run-java-tests; then
569+
./build.sh --run-java-tests
570+
else
571+
./build.sh
572+
fi
573+
fi
574+
560575
RAPIDS_VERSION="$(sed -E -e 's/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2.\3/' "${REPODIR}/VERSION")"
561576
export RAPIDS_VERSION
562577
RAPIDS_VERSION_MAJOR_MINOR="$(sed -E -e 's/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2/' "${REPODIR}/VERSION")"

c/include/cuvs/neighbors/cagra.h

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,27 @@ struct cuvsCagraIndexParams {
223223

224224
typedef struct cuvsCagraIndexParams* cuvsCagraIndexParams_t;
225225

226+
/** Algorithm used to merge physical CAGRA indices. */
227+
enum cuvsCagraMergeAlgo {
228+
CUVS_CAGRA_MERGE_AUTO = 0,
229+
CUVS_CAGRA_MERGE_FASTENER = 1,
230+
CUVS_CAGRA_MERGE_REBUILD = 2
231+
};
232+
233+
/** Parameters controlling how physical CAGRA indices are merged. */
234+
struct cuvsCagraMergeParams {
235+
enum cuvsCagraMergeAlgo algo;
236+
uint32_t levels;
237+
uint32_t root_fanout;
238+
uint32_t lower_fanout;
239+
double leader_fraction;
240+
uint32_t max_leaders;
241+
uint32_t leaf_size;
242+
uint32_t leaf_degree;
243+
};
244+
245+
typedef struct cuvsCagraMergeParams* cuvsCagraMergeParams_t;
246+
226247
/**
227248
* @brief Allocate CAGRA Index params, and populate with default values
228249
*
@@ -239,6 +260,12 @@ CUVS_EXPORT cuvsError_t cuvsCagraIndexParamsCreate(cuvsCagraIndexParams_t* param
239260
*/
240261
CUVS_EXPORT cuvsError_t cuvsCagraIndexParamsDestroy(cuvsCagraIndexParams_t params);
241262

263+
/** Allocate CAGRA merge params and populate them with AUTO defaults. */
264+
CUVS_EXPORT cuvsError_t cuvsCagraMergeParamsCreate(cuvsCagraMergeParams_t* params);
265+
266+
/** De-allocate CAGRA merge params. */
267+
CUVS_EXPORT cuvsError_t cuvsCagraMergeParamsDestroy(cuvsCagraMergeParams_t params);
268+
242269
/**
243270
* @brief Allocate CAGRA Compression params, and populate with default values
244271
*
@@ -967,7 +994,7 @@ CUVS_EXPORT cuvsError_t cuvsCagraIndexFromArgs(cuvsResources_t res,
967994
*
968995
* All input indices must have been built with the same data type (`index.dtype`) and
969996
* have the same dimensionality (`index.dims`). The merged index uses the output
970-
* parameters specified in `cuvsCagraIndexParams`.
997+
* parameters specified in `cuvsCagraIndexParams`. The merge algorithm is selected automatically.
971998
*
972999
* Input indices must have:
9731000
* - `index.dtype.code` and `index.dtype.bits` matching across all indices.
@@ -1013,7 +1040,7 @@ CUVS_EXPORT cuvsError_t cuvsCagraIndexFromArgs(cuvsResources_t res,
10131040
* @endcode
10141041
*
10151042
* @param[in] res cuvsResources_t opaque C handle
1016-
* @param[in] params cuvsCagraIndexParams_t parameters controlling merge behavior
1043+
* @param[in] params cuvsCagraIndexParams_t parameters for the output index
10171044
* @param[in] indices Array of input cuvsCagraIndex_t handles to merge
10181045
* @param[in] num_indices Number of input indices
10191046
* @param[in] filter Filter that can be used to filter out vectors from the merged index
@@ -1034,6 +1061,34 @@ CUVS_EXPORT cuvsError_t cuvsCagraMerge(cuvsResources_t res,
10341061
cuvsDataset_t merged_dataset,
10351062
cuvsCagraIndex_t output_index);
10361063

1064+
/**
1065+
* @brief Merge multiple CAGRA indices with explicit merge parameters.
1066+
*
1067+
* @param[in] res cuvsResources_t opaque C handle
1068+
* @param[in] params cuvsCagraIndexParams_t parameters for the output index
1069+
* @param[in] merge_params cuvsCagraMergeParams_t parameters controlling the merge algorithm, or
1070+
* NULL to use AUTO defaults
1071+
* @param[in] indices Array of input cuvsCagraIndex_t handles to merge
1072+
* @param[in] num_indices Number of input indices
1073+
* @param[in] filter Filter that can be used to filter out vectors from the merged index
1074+
* @param[out] merged_dataset Empty owning dataset handle. Merge first attempts to allocate and
1075+
* populate device storage with the same layout as the input indices. For
1076+
* an unfiltered merge, AUTO and REBUILD can fall back to host storage if
1077+
* device allocation fails; explicit FASTENER reports the allocation
1078+
* failure instead. Keep this dataset alive while using `output_index`.
1079+
* A host-backed output index must be updated with
1080+
* `cuvsCagraUpdateDataset` before device search.
1081+
* @param[out] output_index Output handle initialized with `cuvsCagraIndexCreate`
1082+
*/
1083+
CUVS_EXPORT cuvsError_t cuvsCagraMergeWithParams(cuvsResources_t res,
1084+
cuvsCagraIndexParams_t params,
1085+
cuvsCagraMergeParams_t merge_params,
1086+
cuvsCagraIndex_t* indices,
1087+
size_t num_indices,
1088+
cuvsFilter filter,
1089+
cuvsDataset_t merged_dataset,
1090+
cuvsCagraIndex_t output_index);
1091+
10371092
/**
10381093
* @}
10391094
*/

0 commit comments

Comments
 (0)