diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000000..67f39679b95 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,10 @@ +FROM gcr.io/oss-fuzz-base/base-builder-rust +# The distro's protobuf-compiler (3.6.x on Ubuntu 20.04) predates +# --experimental_allow_proto3_optional. Install a release that supports it. +RUN apt-get update && apt-get install -y --no-install-recommends unzip && \ + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/protoc-3.20.3-linux-x86_64.zip && \ + unzip protoc-3.20.3-linux-x86_64.zip -d /usr/local && \ + rm protoc-3.20.3-linux-x86_64.zip +COPY . $SRC/quickwit +WORKDIR $SRC/quickwit +COPY .clusterfuzzlite/build.sh $SRC/build.sh diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 00000000000..f76bddbbbb6 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash -eu +cd "$SRC/quickwit/quickwit" +RUSTFLAGS="--cfg tokio_unstable" cargo +nightly fuzz build --fuzz-dir quickwit-fuzz +for fuzz_target in fuzz_query_dsl fuzz_query_string fuzz_datetime fuzz_doc_mapper fuzz_doc_mapper_config fuzz_java_datetime_format fuzz_otlp_logs fuzz_otlp_spans; do + cp "target/x86_64-unknown-linux-gnu/release/$fuzz_target" "$OUT/" +done diff --git a/.dockerignore b/.dockerignore index f5090e60254..17619394139 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ **/target docs examples +!.clusterfuzzlite/ !.git/ !quickwit-ui/build/.gitignore !quickwit-ui/.gitignore_for_build_directory diff --git a/.github/workflows/clusterfuzzlite-batch.yml b/.github/workflows/clusterfuzzlite-batch.yml new file mode 100644 index 00000000000..20fb725ac62 --- /dev/null +++ b/.github/workflows/clusterfuzzlite-batch.yml @@ -0,0 +1,31 @@ +name: ClusterFuzzLite Batch +on: + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +permissions: {} + +jobs: + batch-fuzzing: + runs-on: ubuntu-latest + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address] + steps: + - uses: actions/checkout@v4 + - name: Build Fuzzers (${{ matrix.sanitizer }}) + uses: google/clusterfuzzlite/actions/build_fuzzers@v1 + with: + language: rust + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 3600 + mode: 'batch' + sanitizer: ${{ matrix.sanitizer }} diff --git a/.github/workflows/clusterfuzzlite.yml b/.github/workflows/clusterfuzzlite.yml new file mode 100644 index 00000000000..6d76e9990da --- /dev/null +++ b/.github/workflows/clusterfuzzlite.yml @@ -0,0 +1,46 @@ +name: ClusterFuzzLite +on: + push: + branches: [main] + paths: + - "quickwit/**" + - "!quickwit/quickwit-ui/**" + - ".clusterfuzzlite/**" + - ".github/workflows/clusterfuzzlite*.yml" + pull_request: + branches: [main] + paths: + - "quickwit/**" + - "!quickwit/quickwit-ui/**" + - ".clusterfuzzlite/**" + - ".github/workflows/clusterfuzzlite*.yml" + workflow_dispatch: + +permissions: {} + +jobs: + fuzzing: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }} + cancel-in-progress: true + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address] + steps: + - uses: actions/checkout@v4 + - name: Build Fuzzers (${{ matrix.sanitizer }}) + uses: google/clusterfuzzlite/actions/build_fuzzers@v1 + with: + language: rust + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 30 + mode: 'code-change' + sanitizer: ${{ matrix.sanitizer }} diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index c16aecbbda0..874538b3ca9 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -23,6 +23,7 @@ anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle- anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay +arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald , Manish Goregaokar , Simonas Kazlauskas , Brian L. Troutwine , Corey Farwell " arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner arrayref,https://github.com/droundy/arrayref,BSD-2-Clause,David Roundy arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss @@ -448,6 +449,7 @@ libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Develo libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas liblzma,https://github.com/portable-network-archive/liblzma-rs,MIT OR Apache-2.0,"Alex Crichton , Portable-Network-Archive Developers" liblzma-sys,https://github.com/portable-network-archive/liblzma-rs,MIT OR Apache-2.0,"Alex Crichton , Portable-Network-Archive Developers" +libfuzzer-sys,https://github.com/rust-fuzz/libfuzzer,(MIT OR Apache-2.0) AND NCSA,The rust-fuzz Project Developers libm,https://github.com/rust-lang/compiler-builtins,MIT,"Alex Crichton , Amanieu d'Antras , Jorge Aparicio , Trevor Gross " libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index c9f0fc98f0d..568de70fef9 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -42,6 +42,7 @@ members = [ "quickwit-serve", "quickwit-storage", "quickwit-transport", + "quickwit-fuzz", ] # The following list excludes `quickwit-metastore-utils` diff --git a/quickwit/deny.toml b/quickwit/deny.toml index 6b3c5b8560c..11ed9ea3d2d 100644 --- a/quickwit/deny.toml +++ b/quickwit/deny.toml @@ -77,7 +77,11 @@ allow = [ confidence-threshold = 0.8 # Allow 1 or more licenses on a per-crate basis, so that particular licenses # aren't accepted for every possible crate as with the normal allow list -exceptions = [] +exceptions = [ + # libfuzzer-sys is only used in the quickwit-fuzz crate (never shipped); + # NCSA is OSI-approved and FSF-Free but not in the general allowlist. + { allow = ["NCSA"], name = "libfuzzer-sys" }, +] # Some crates don't have (easily) machine readable licensing information, # adding a clarification entry for it allows you to manually specify the diff --git a/quickwit/quickwit-datetime/src/date_time_parsing.rs b/quickwit/quickwit-datetime/src/date_time_parsing.rs index ad8132e2c13..5b76718c81e 100644 --- a/quickwit/quickwit-datetime/src/date_time_parsing.rs +++ b/quickwit/quickwit-datetime/src/date_time_parsing.rs @@ -123,7 +123,12 @@ pub fn parse_timestamp_str(timestamp_str: &str) -> Option { if let Ok(timestamp_secs @ MIN_TIMESTAMP_SECONDS..=MAX_TIMESTAMP_SECONDS) = timestamp_secs_str.parse::() { - let num_subsecond_digits = subsecond_digits_str.len().min(9); + let num_subsecond_digits = subsecond_digits_str + .as_bytes() + .iter() + .position(|b| !b.is_ascii_digit()) + .unwrap_or(subsecond_digits_str.len()) + .min(9); if let Ok(subsecond_digits) = subsecond_digits_str[..num_subsecond_digits].parse::() diff --git a/quickwit/quickwit-fuzz/.gitignore b/quickwit/quickwit-fuzz/.gitignore new file mode 100644 index 00000000000..ff75c48623e --- /dev/null +++ b/quickwit/quickwit-fuzz/.gitignore @@ -0,0 +1,4 @@ +target/ +artifacts/ +corpus/ +Cargo.lock diff --git a/quickwit/quickwit-fuzz/Cargo.toml b/quickwit/quickwit-fuzz/Cargo.toml new file mode 100644 index 00000000000..0c19c3a211e --- /dev/null +++ b/quickwit/quickwit-fuzz/Cargo.toml @@ -0,0 +1,68 @@ +[package] +name = "quickwit-fuzz" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[package.metadata] +cargo-fuzz = true + +[profile.release] +debug = true + +[dependencies] +libfuzzer-sys = "0.4" +serde_json = { workspace = true } +quickwit-datetime = { workspace = true } +quickwit-doc-mapper = { workspace = true } +quickwit-opentelemetry = { workspace = true } +quickwit-query = { workspace = true } + +[[bin]] +name = "fuzz_query_dsl" +path = "fuzz_targets/fuzz_query_dsl.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_query_string" +path = "fuzz_targets/fuzz_query_string.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_datetime" +path = "fuzz_targets/fuzz_datetime.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_doc_mapper" +path = "fuzz_targets/fuzz_doc_mapper.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_doc_mapper_config" +path = "fuzz_targets/fuzz_doc_mapper_config.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_java_datetime_format" +path = "fuzz_targets/fuzz_java_datetime_format.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_otlp_logs" +path = "fuzz_targets/fuzz_otlp_logs.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_otlp_spans" +path = "fuzz_targets/fuzz_otlp_spans.rs" +test = false +doc = false diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_datetime.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_datetime.rs new file mode 100644 index 00000000000..e67b1b1d95f --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_datetime.rs @@ -0,0 +1,29 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] +use libfuzzer_sys::fuzz_target; +use quickwit_datetime::{DateTimeInputFormat, parse_date_time_str}; + +fuzz_target!(|data: &[u8]| { + if let Ok(datetime_str) = std::str::from_utf8(data) { + let formats = &[ + DateTimeInputFormat::Iso8601, + DateTimeInputFormat::Rfc3339, + DateTimeInputFormat::Rfc2822, + DateTimeInputFormat::Timestamp, + ]; + let _ = parse_date_time_str(datetime_str, formats); + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper.rs new file mode 100644 index 00000000000..4dca55269c5 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper.rs @@ -0,0 +1,29 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] +use libfuzzer_sys::fuzz_target; + +// Constructed once to avoid expensive re-initialisation on every iteration. +static DOC_MAPPER: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + let builder: quickwit_doc_mapper::DocMapperBuilder = serde_json::from_str("{}").unwrap(); + builder.try_build().unwrap() + }); + +fuzz_target!(|data: &[u8]| { + if let Ok(json_str) = std::str::from_utf8(data) { + let _ = DOC_MAPPER.doc_from_json_str(json_str); + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper_config.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper_config.rs new file mode 100644 index 00000000000..b5a1f6f9673 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper_config.rs @@ -0,0 +1,31 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Fuzz the doc mapper *configuration* (field mappings, tokenizer definitions, +// index settings) as supplied via the index-creation REST API. +// +// The existing `fuzz_doc_mapper` target exercises document *ingestion* against +// a fixed empty schema. This target covers the orthogonal path: parsing and +// validating the schema definition itself, including regex tokenizer patterns +// that could cause catastrophic backtracking. +#![no_main] +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(json_str) = std::str::from_utf8(data) + && let Ok(builder) = serde_json::from_str::(json_str) + { + let _ = builder.try_build(); + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_java_datetime_format.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_java_datetime_format.rs new file mode 100644 index 00000000000..867e8dadbb4 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_java_datetime_format.rs @@ -0,0 +1,34 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Fuzz Java SimpleDateFormat and strptime *format-string* parsing. +// +// The existing `fuzz_datetime` target tests parsing date strings with fixed +// known formats. This target covers the orthogonal path: parsing the format +// specification itself, which uses a recursive grammar for optional groups +// `[...]` and resolves named aliases (e.g. `date_optional_time`). +// +// Elasticsearch range queries accept an untrusted `"format"` field that flows +// through `StrptimeParser::from_java_datetime_format`, making this an external +// input surface beyond just configuration. +#![no_main] +use libfuzzer_sys::fuzz_target; +use quickwit_datetime::StrptimeParser; + +fuzz_target!(|data: &[u8]| { + if let Ok(format_str) = std::str::from_utf8(data) { + let _ = StrptimeParser::from_java_datetime_format(format_str); + let _ = StrptimeParser::from_strptime(format_str); + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_logs.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_logs.rs new file mode 100644 index 00000000000..d760bbd5ba8 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_logs.rs @@ -0,0 +1,31 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Fuzz OTLP log parsing (protobuf and JSON paths). +// +// Log records from external collectors (Fluentd, Vector, etc.) arrive at the +// gRPC boundary as raw bytes. `parse_otlp_logs_protobuf/json` deserialise the +// payload and process each LogRecord's body/attributes, where recursive +// `AnyValue` nesting has no depth guard. +#![no_main] +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_logs_protobuf(data) { + for _ in iter {} + } + if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_logs_json(data) { + for _ in iter {} + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_spans.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_spans.rs new file mode 100644 index 00000000000..58f283e52fe --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_spans.rs @@ -0,0 +1,31 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Fuzz OTLP trace parsing (protobuf and JSON paths). +// +// The internal `otlp_value_to_json_value` helper processes OTLP `AnyValue` +// recursively (arrays and key-value lists of itself) with no depth limit. +// A crafted payload with deep nesting can cause a stack overflow — this target +// exercises that code path from both wire formats. +#![no_main] +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_spans_protobuf(data) { + for _ in iter {} + } + if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_spans_json(data) { + for _ in iter {} + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_dsl.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_dsl.rs new file mode 100644 index 00000000000..8cbe6ffb334 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_dsl.rs @@ -0,0 +1,22 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(json_str) = std::str::from_utf8(data) { + let _ = serde_json::from_str::(json_str); + } +}); diff --git a/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_string.rs b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_string.rs new file mode 100644 index 00000000000..0b29166d2b0 --- /dev/null +++ b/quickwit/quickwit-fuzz/fuzz_targets/fuzz_query_string.rs @@ -0,0 +1,40 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] +use libfuzzer_sys::fuzz_target; +use quickwit_query::BooleanOperand; +use quickwit_query::query_ast::UserInputQuery; + +fuzz_target!(|data: &[u8]| { + if let Ok(query_str) = std::str::from_utf8(data) { + let query = UserInputQuery { + user_text: query_str.to_string(), + default_fields: Some(vec!["body".to_string()]), + default_operator: BooleanOperand::Or, + lenient: true, + }; + // libfuzzer-sys installs a panic hook that calls abort() before stack + // unwinding, so catch_unwind alone is not enough to suppress a panic. + // Temporarily swap in a no-op hook so the known upstream tantivy bug + // ("Exist query without a field isn't allowed") is caught instead of + // aborting the fuzzer process. + let abort_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + query.parse_user_query(&["body".to_string()]) + })); + std::panic::set_hook(abort_hook); + } +});