add fuzzing infrastructure, expand fuzz coverage, fix CI integration

.clusterfuzzlite/Dockerfile

@@ -0,0 +1,10 @@
+FROM gcr.io/oss-fuzz-base/base-builder-rust
+# The distro's protobuf-compiler (3.6.x on Ubuntu 20.04) predates
+# --experimental_allow_proto3_optional.  Install a release that supports it.
+RUN apt-get update && apt-get install -y --no-install-recommends unzip && \
+    curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/protoc-3.20.3-linux-x86_64.zip && \
+    unzip protoc-3.20.3-linux-x86_64.zip -d /usr/local && \
+    rm protoc-3.20.3-linux-x86_64.zip
+COPY . $SRC/quickwit
+WORKDIR $SRC/quickwit
+COPY .clusterfuzzlite/build.sh $SRC/build.sh

.clusterfuzzlite/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -eu
+cd "$SRC/quickwit/quickwit"
+RUSTFLAGS="--cfg tokio_unstable" cargo +nightly fuzz build --fuzz-dir quickwit-fuzz
+for fuzz_target in fuzz_query_dsl fuzz_query_string fuzz_datetime fuzz_doc_mapper fuzz_doc_mapper_config fuzz_java_datetime_format fuzz_otlp_logs fuzz_otlp_spans; do
+    cp "target/x86_64-unknown-linux-gnu/release/$fuzz_target" "$OUT/"
+done

.dockerignore

@@ -8,6 +8,7 @@
 **/target
 docs
 examples
+!.clusterfuzzlite/
 !.git/
 !quickwit-ui/build/.gitignore
 !quickwit-ui/.gitignore_for_build_directory

.github/workflows/clusterfuzzlite-batch.yml

@@ -0,0 +1,31 @@
+name: ClusterFuzzLite Batch
+on:
+  schedule:
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  batch-fuzzing:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Fuzzers (${{ matrix.sanitizer }})
+        uses: google/clusterfuzzlite/actions/build_fuzzers@v1
+        with:
+          language: rust
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Run Fuzzers (${{ matrix.sanitizer }})
+        uses: google/clusterfuzzlite/actions/run_fuzzers@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fuzz-seconds: 3600
+          mode: 'batch'
+          sanitizer: ${{ matrix.sanitizer }}

.github/workflows/clusterfuzzlite.yml

@@ -0,0 +1,46 @@
+name: ClusterFuzzLite
+on:
+  push:
+    branches: [main]
+    paths:
+      - "quickwit/**"
+      - "!quickwit/quickwit-ui/**"
+      - ".clusterfuzzlite/**"
+      - ".github/workflows/clusterfuzzlite*.yml"
+  pull_request:
+    branches: [main]
+    paths:
+      - "quickwit/**"
+      - "!quickwit/quickwit-ui/**"
+      - ".clusterfuzzlite/**"
+      - ".github/workflows/clusterfuzzlite*.yml"
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  fuzzing:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }}
+      cancel-in-progress: true
+    permissions:
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Fuzzers (${{ matrix.sanitizer }})
+        uses: google/clusterfuzzlite/actions/build_fuzzers@v1
+        with:
+          language: rust
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Run Fuzzers (${{ matrix.sanitizer }})
+        uses: google/clusterfuzzlite/actions/run_fuzzers@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fuzz-seconds: 30
+          mode: 'code-change'
+          sanitizer: ${{ matrix.sanitizer }}

LICENSE-3rdparty.csv

@@ -23,6 +23,7 @@ anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-
 anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors
 anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors
 anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
+arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald <fitzgen@gmail.com>, Manish Goregaokar <manishsmail@gmail.com>, Simonas Kazlauskas <arbitrary@kazlauskas.me>, Brian L. Troutwine <brian@troutwine.us>, Corey Farwell <coreyf@rwell.org>"
 arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner <vorner@vorner.cz>
 arrayref,https://github.com/droundy/arrayref,BSD-2-Clause,David Roundy <roundyd@physics.oregonstate.edu>
 arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss
@@ -448,6 +449,7 @@ libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Develo
 libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas <libloading@kazlauskas.me>
 liblzma,https://github.com/portable-network-archive/liblzma-rs,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Portable-Network-Archive Developers"
 liblzma-sys,https://github.com/portable-network-archive/liblzma-rs,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Portable-Network-Archive Developers"
+libfuzzer-sys,https://github.com/rust-fuzz/libfuzzer,(MIT OR Apache-2.0) AND NCSA,The rust-fuzz Project Developers
 libm,https://github.com/rust-lang/compiler-builtins,MIT,"Alex Crichton <alex@alexcrichton.com>, Amanieu d'Antras <amanieu@gmail.com>, Jorge Aparicio <japaricious@gmail.com>, Trevor Gross <tg@trevorgross.com>"
 libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com>
 libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers

quickwit/Cargo.toml

@@ -42,6 +42,7 @@ members = [
   "quickwit-serve",
   "quickwit-storage",
   "quickwit-transport",
+  "quickwit-fuzz",
 ]
 
 # The following list excludes `quickwit-metastore-utils`

quickwit/deny.toml

@@ -77,7 +77,11 @@ allow = [
 confidence-threshold = 0.8
 # Allow 1 or more licenses on a per-crate basis, so that particular licenses
 # aren't accepted for every possible crate as with the normal allow list
-exceptions = []
+exceptions = [
+    # libfuzzer-sys is only used in the quickwit-fuzz crate (never shipped);
+    # NCSA is OSI-approved and FSF-Free but not in the general allowlist.
+    { allow = ["NCSA"], name = "libfuzzer-sys" },
+]
 
 # Some crates don't have (easily) machine readable licensing information,
 # adding a clarification entry for it allows you to manually specify the

quickwit/quickwit-datetime/src/date_time_parsing.rs

@@ -123,7 +123,12 @@ pub fn parse_timestamp_str(timestamp_str: &str) -> Option<TantivyDateTime> {
         if let Ok(timestamp_secs @ MIN_TIMESTAMP_SECONDS..=MAX_TIMESTAMP_SECONDS) =
             timestamp_secs_str.parse::<i64>()
         {
-            let num_subsecond_digits = subsecond_digits_str.len().min(9);
+            let num_subsecond_digits = subsecond_digits_str
+                .as_bytes()
+                .iter()
+                .position(|b| !b.is_ascii_digit())
+                .unwrap_or(subsecond_digits_str.len())
+                .min(9);
 
             if let Ok(subsecond_digits) =
                 subsecond_digits_str[..num_subsecond_digits].parse::<i64>()

quickwit/quickwit-fuzz/.gitignore

@@ -0,0 +1,4 @@
+target/
+artifacts/
+corpus/
+Cargo.lock

quickwit/quickwit-fuzz/Cargo.toml

@@ -0,0 +1,68 @@
+[package]
+name = "quickwit-fuzz"
+version = "0.0.0"
+publish = false
+edition.workspace = true
+license.workspace = true
+
+[package.metadata]
+cargo-fuzz = true
+
+[profile.release]
+debug = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+serde_json = { workspace = true }
+quickwit-datetime = { workspace = true }
+quickwit-doc-mapper = { workspace = true }
+quickwit-opentelemetry = { workspace = true }
+quickwit-query = { workspace = true }
+
+[[bin]]
+name = "fuzz_query_dsl"
+path = "fuzz_targets/fuzz_query_dsl.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_query_string"
+path = "fuzz_targets/fuzz_query_string.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_datetime"
+path = "fuzz_targets/fuzz_datetime.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_doc_mapper"
+path = "fuzz_targets/fuzz_doc_mapper.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_doc_mapper_config"
+path = "fuzz_targets/fuzz_doc_mapper_config.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_java_datetime_format"
+path = "fuzz_targets/fuzz_java_datetime_format.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_otlp_logs"
+path = "fuzz_targets/fuzz_otlp_logs.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_otlp_spans"
+path = "fuzz_targets/fuzz_otlp_spans.rs"
+test = false
+doc = false

quickwit/quickwit-fuzz/fuzz_targets/fuzz_datetime.rs

@@ -0,0 +1,29 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use quickwit_datetime::{DateTimeInputFormat, parse_date_time_str};
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(datetime_str) = std::str::from_utf8(data) {
+        let formats = &[
+            DateTimeInputFormat::Iso8601,
+            DateTimeInputFormat::Rfc3339,
+            DateTimeInputFormat::Rfc2822,
+            DateTimeInputFormat::Timestamp,
+        ];
+        let _ = parse_date_time_str(datetime_str, formats);
+    }
+});

quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper.rs

@@ -0,0 +1,29 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+// Constructed once to avoid expensive re-initialisation on every iteration.
+static DOC_MAPPER: std::sync::LazyLock<quickwit_doc_mapper::DocMapper> =
+    std::sync::LazyLock::new(|| {
+        let builder: quickwit_doc_mapper::DocMapperBuilder = serde_json::from_str("{}").unwrap();
+        builder.try_build().unwrap()
+    });
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(json_str) = std::str::from_utf8(data) {
+        let _ = DOC_MAPPER.doc_from_json_str(json_str);
+    }
+});

quickwit/quickwit-fuzz/fuzz_targets/fuzz_doc_mapper_config.rs

@@ -0,0 +1,31 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Fuzz the doc mapper *configuration* (field mappings, tokenizer definitions,
+// index settings) as supplied via the index-creation REST API.
+//
+// The existing `fuzz_doc_mapper` target exercises document *ingestion* against
+// a fixed empty schema.  This target covers the orthogonal path: parsing and
+// validating the schema definition itself, including regex tokenizer patterns
+// that could cause catastrophic backtracking.
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(json_str) = std::str::from_utf8(data)
+        && let Ok(builder) = serde_json::from_str::<quickwit_doc_mapper::DocMapperBuilder>(json_str)
+    {
+        let _ = builder.try_build();
+    }
+});

quickwit/quickwit-fuzz/fuzz_targets/fuzz_java_datetime_format.rs

@@ -0,0 +1,34 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Fuzz Java SimpleDateFormat and strptime *format-string* parsing.
+//
+// The existing `fuzz_datetime` target tests parsing date strings with fixed
+// known formats.  This target covers the orthogonal path: parsing the format
+// specification itself, which uses a recursive grammar for optional groups
+// `[...]` and resolves named aliases (e.g. `date_optional_time`).
+//
+// Elasticsearch range queries accept an untrusted `"format"` field that flows
+// through `StrptimeParser::from_java_datetime_format`, making this an external
+// input surface beyond just configuration.
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use quickwit_datetime::StrptimeParser;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(format_str) = std::str::from_utf8(data) {
+        let _ = StrptimeParser::from_java_datetime_format(format_str);
+        let _ = StrptimeParser::from_strptime(format_str);
+    }
+});

quickwit/quickwit-fuzz/fuzz_targets/fuzz_otlp_logs.rs

@@ -0,0 +1,31 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Fuzz OTLP log parsing (protobuf and JSON paths).
+//
+// Log records from external collectors (Fluentd, Vector, etc.) arrive at the
+// gRPC boundary as raw bytes.  `parse_otlp_logs_protobuf/json` deserialise the
+// payload and process each LogRecord's body/attributes, where recursive
+// `AnyValue` nesting has no depth guard.
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_logs_protobuf(data) {
+        for _ in iter {}
+    }
+    if let Ok(iter) = quickwit_opentelemetry::otlp::parse_otlp_logs_json(data) {
+        for _ in iter {}
+    }
+});