Hot-reload TLS cert periodically or on SIGHUP

config/quickwit.yaml

@@ -40,9 +40,32 @@ version: 0.8
 #   extra_headers:
 #     x-header-1: header-value-1
 #     x-header-2: header-value-2
+#   tls:
+#     cert_path: /path/to/server.crt
+#     key_path: /path/to/server.key
+#     # mTLS: require clients to present a certificate signed by this CA.
+#     ca_path: /path/to/ca.crt
+#     verify_client_cert: true
+#     # How often cert_path/key_path are polled for changes and hot-reloaded; an
+#     # immediate reload can also be triggered with SIGHUP. Defaults to 5m.
+#     cert_reload_interval: 5m
 #
 # grpc:
 #   max_message_size: 10 MiB
+#   tls:
+#     cert_path: /path/to/server.crt
+#     key_path: /path/to/server.key
+#     # Trust root for peers this node *connects to* (every node is also a gRPC client to
+#     # its peers). With verify_client_cert, also validates peers connecting to this node.
+#     ca_path: /path/to/ca.crt
+#     # mTLS: require peers to present a certificate signed by ca_path. Each node reuses
+#     # cert_path/key_path above as its own client identity toward peers.
+#     verify_client_cert: true
+#     # Hostname checked against the peer certificate's SAN when connecting to peers.
+#     expected_name: quickwit.local
+#     # How often cert_path/key_path are polled for changes and hot-reloaded; an
+#     # immediate reload can also be triggered with SIGHUP. Defaults to 5m.
+#     cert_reload_interval: 5m
 #
 # IP address advertised by the node, i.e. the IP address that peer nodes should use to connect to the node for RPCs.
 # The environment variable `QW_ADVERTISE_ADDRESS` can also be used to override this value.

docs/configuration/node-config.md

@@ -42,6 +42,7 @@ This section contains the REST API configuration options.
 | `listen_port` | The port on which the REST API listens for HTTP traffic. | `QW_REST_LISTEN_PORT` | `7280` |
 | `cors_allow_origins` | Configure the CORS origins which are allowed to access the API. [Read more](#configuring-cors-cross-origin-resource-sharing) | |
 | `extra_headers` | List of header names and values | | |
+| `tls` | Enables HTTPS for the REST API. [Read more](#tls-configuration) | | |
 
 ### Configuring CORS (Cross-origin resource sharing)
 
@@ -74,6 +75,7 @@ This section contains the configuration options for gRPC services and clients us
 | Property | Description | Env variable | Default value |
 | --- | --- | --- | --- |
 | `max_message_size` | The maximum size (in bytes) of messages exchanged by internal gRPC clients and services. | | `20 MiB` |
+| `tls` | Enables TLS for gRPC services and clients. [Read more](#tls-configuration) | | |
 
 Example of a gRPC configuration:
 
@@ -87,6 +89,58 @@ We advise changing the default value of 20 MiB only if you encounter the followi
 `Error, message length too large: found 24732228 bytes, the limit is: 20971520 bytes.` In that case, increase `max_message_size` by increments of 10 MiB until the issue disappears. This is a temporary fix: the next version of Quickwit will rely exclusively on gRPC streaming endpoints and handle messages of any length.
 :::
 
+## TLS configuration
+
+Both the REST API (`rest.tls`) and the internal gRPC services (`grpc.tls`) can be secured with TLS, optionally with mutual TLS (mTLS). The two sections share the same properties:
+
+| Property | Description | Default value |
+| --- | --- | --- |
+| `cert_path` | Path to the PEM-encoded X.509 certificate (or chain) presented by the server. Setting this enables TLS. | |
+| `key_path` | Path to the PEM-encoded private key matching `cert_path`. | |
+| `ca_path` | Path to a PEM file holding the trusted CA certificate(s). Used by the server to validate client certificates when `verify_client_cert` is enabled, and by the gRPC client to validate peer certificates. Multiple CA certificates may be concatenated in the same file: all of them are trusted (see [CA rotation](#ca-rotation)). | |
+| `verify_client_cert` | If `true`, require clients (REST) or peers (gRPC) to present a certificate signed by `ca_path`, i.e. enforce mutual TLS. | `false` |
+| `expected_name` | gRPC only. The hostname the gRPC client checks against the peer certificate's Subject Alternative Name (SAN). Defaults to the peer's address. | |
+| `cert_reload_interval` | How often `cert_path` and `key_path` are polled for on-disk changes and hot-reloaded, without restarting the process. An immediate reload can also be triggered by sending `SIGHUP` to the process. | `5m` |
+
+Certificates are hot-reloaded: when `cert_path`/`key_path` change on disk, new connections pick up the new certificate within `cert_reload_interval` (or immediately on `SIGHUP`), while in-flight connections keep the certificate they negotiated. A new certificate is only applied if it parses and matches its key; otherwise the previous certificate is kept. Note that the CA trust roots (`ca_path`) are **not** hot-reloaded — rotating them still requires a restart.
+
+### CA rotation
+
+Because `ca_path` accepts multiple CA certificates concatenated in a single PEM file, you can rotate the CA without downtime by temporarily trusting both the old and the new CA:
+
+1. Append the **new** CA certificate to the `ca_path` file, so it contains both the old and the new CA.
+2. Roll-restart every node. Each node now trusts certificates signed by either CA, while peers may still present certificates signed by the old CA.
+3. Re-issue every node's `cert_path`/`key_path` with certificates signed by the new CA (these are hot-reloaded, no restart needed).
+4. Remove the **old** CA certificate from the `ca_path` file, leaving only the new CA.
+5. Roll-restart every node again to drop trust in the old CA.
+
+Since the CA file is read once at startup, both restarts are required to pick up the changes to `ca_path`.
+
+Example of a REST configuration with mTLS:
+
+```yaml
+rest:
+  tls:
+    cert_path: /path/to/server.crt
+    key_path: /path/to/server.key
+    ca_path: /path/to/ca.crt
+    verify_client_cert: true
+    cert_reload_interval: 5m
+```
+
+Example of a gRPC configuration with mTLS:
+
+```yaml
+grpc:
+  tls:
+    cert_path: /path/to/server.crt
+    key_path: /path/to/server.key
+    ca_path: /path/to/ca.crt
+    expected_name: quickwit.local
+    verify_client_cert: true
+    cert_reload_interval: 5m
+```
+
 ## Storage configuration
 
 Please refer to the dedicated [storage configuration](storage-config) page to learn more about configuring Quickwit for various storage providers.

quickwit/Cargo.toml

@@ -41,6 +41,7 @@ members = [
   "quickwit-search",
   "quickwit-serve",
   "quickwit-storage",
+  "quickwit-transport",
 ]
 
 # The following list excludes `quickwit-metastore-utils`
@@ -82,6 +83,7 @@ default-members = [
   "quickwit-search",
   "quickwit-serve",
   "quickwit-storage",
+  "quickwit-transport",
 ]
 
 [workspace.package]
@@ -401,6 +403,7 @@ quickwit-rest-client = { path = "quickwit-rest-client" }
 quickwit-search = { path = "quickwit-search" }
 quickwit-serve = { path = "quickwit-serve" }
 quickwit-storage = { path = "quickwit-storage" }
+quickwit-transport = { path = "quickwit-transport" }
 
 tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "6270552", default-features = false, features = [
   "lz4-compression",

quickwit/quickwit-cli/Cargo.toml

@@ -65,6 +65,7 @@ quickwit-rest-client = { workspace = true }
 quickwit-search = { workspace = true }
 quickwit-serve = { workspace = true }
 quickwit-storage = { workspace = true }
+quickwit-transport = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { workspace = true }

quickwit/quickwit-cli/src/service.rs

@@ -24,7 +24,7 @@ use quickwit_common::runtimes::RuntimesConfig;
 use quickwit_common::uri::Uri;
 use quickwit_config::service::QuickwitService;
 use quickwit_serve::tcp_listener::DefaultTcpListenerResolver;
-use quickwit_serve::{BuildInfo, EnvFilterReloadFn, serve_quickwit};
+use quickwit_serve::{BuildInfo, EnvFilterReloadFn, reload_tls_cert, serve_quickwit};
 use tokio::signal;
 use tracing::{debug, info};
 
@@ -81,6 +81,16 @@ async fn listen_sigterm() {
     info!("SIGTERM received");
 }
 
+async fn listen_sighup() {
+    let mut sighup = signal::unix::signal(signal::unix::SignalKind::hangup())
+        .expect("registering a signal handler for SIGHUP should not fail");
+
+    while sighup.recv().await.is_some() {
+        info!("SIGHUP received");
+        reload_tls_cert();
+    }
+}
+
 impl RunCliCommand {
     pub fn parse_cli_args(mut matches: ArgMatches) -> anyhow::Result<Self> {
         let config_uri = matches
@@ -122,6 +132,8 @@ impl RunCliCommand {
         let shutdown_signal = Box::pin(async {
             select(pin!(listen_interrupt()), pin!(listen_sigterm())).await;
         });
+        // Reload TLS certificates on SIGHUP for the lifetime of the process.
+        tokio::spawn(listen_sighup());
         serve_quickwit(
             node_config,
             runtimes_config,

quickwit/quickwit-cli/src/tool.rs

@@ -25,9 +25,7 @@ use clap::{ArgMatches, Command, arg};
 use colored::{ColoredString, Colorize};
 use humantime::format_duration;
 use quickwit_actors::{ActorExitStatus, ActorHandle, Mailbox, Universe};
-use quickwit_cluster::{
-    ChannelTransport, Cluster, ClusterMember, FailureDetectorConfig, make_client_grpc_config,
-};
+use quickwit_cluster::{ChitchatTransport, Cluster, ClusterMember, FailureDetectorConfig};
 use quickwit_common::pubsub::EventBroker;
 use quickwit_common::runtimes::RuntimesConfig;
 use quickwit_common::uri::Uri;
@@ -54,6 +52,7 @@ use quickwit_serve::{
     BodyFormat, SearchRequestQueryString, SortBy, search_request_from_api_request,
 };
 use quickwit_storage::{BundleStorage, Storage};
+use quickwit_transport::ChannelFactory;
 use thousands::Separable;
 use tracing::{debug, info};
 
@@ -1007,16 +1006,16 @@ async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result<Cluster> {
         ingester_status: IngesterStatus::default(),
         availability_zone: None,
     };
-    let client_grpc_config = make_client_grpc_config(&config.grpc_config)?;
+    let channel_factory = ChannelFactory::for_grpc(&config.grpc_config)?;
     let cluster = Cluster::join(
         config.cluster_id.clone(),
         self_node,
         config.gossip_advertise_addr,
         Vec::new(),
         config.gossip_interval,
         FailureDetectorConfig::default(),
-        &ChannelTransport::default(),
-        client_grpc_config,
+        &ChitchatTransport::default(),
+        channel_factory,
     )
     .await?;
 

quickwit/quickwit-cluster/Cargo.toml

@@ -32,6 +32,7 @@ quickwit-common = { workspace = true }
 quickwit-metrics = { workspace = true }
 quickwit-config = { workspace = true }
 quickwit-proto = { workspace = true }
+quickwit-transport = { workspace = true }
 
 [features]
 testsuite = []