diff --git a/go.mod b/go.mod index 0acf8c4b..27c7b451 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/BlueMonday/go-scryfall v0.10.0 github.com/SevereCloud/vksdk/v3 v3.3.1 github.com/cockroachdb/errors v1.14.0 - github.com/redis/go-redis/v9 v9.21.0 + github.com/redis/go-redis/v9 v9.22.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/tdewolff/minify/v2 v2.24.14 diff --git a/go.sum b/go.sum index 416cd5a8..21e58e43 100644 --- a/go.sum +++ b/go.sum @@ -377,8 +377,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E= -github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= diff --git a/vendor/github.com/redis/go-redis/v9/.gitignore b/vendor/github.com/redis/go-redis/v9/.gitignore index 93affec7..4134a1f4 100644 --- a/vendor/github.com/redis/go-redis/v9/.gitignore +++ b/vendor/github.com/redis/go-redis/v9/.gitignore @@ -17,3 +17,12 @@ maintenanceNotifications/docs/ # Docker-generated files (TLS certificates, cluster data, etc.) dockers/*/tls/ dockers/osscluster-tls/ + +# Per-user Claude Code settings (machine/personal overrides, not shared policy) +.claude/settings.local.json + +# example build artifacts +example/autopipeline/autopipeline + +# Claude Code worktrees (ephemeral, local) +.claude/worktrees/ diff --git a/vendor/github.com/redis/go-redis/v9/AGENTS.md b/vendor/github.com/redis/go-redis/v9/AGENTS.md new file mode 100644 index 00000000..ca0a3116 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/AGENTS.md @@ -0,0 +1,228 @@ +# AGENTS.md + +Guidance for AI coding agents (and humans) working in this repository. This is +the shared, tool-agnostic source of truth: Claude Code loads it through +`CLAUDE.md` (which imports this file), and other agents (Codex, Cursor, Aider, +Zed, …) read `AGENTS.md` directly. Edit repository guidance here, not in +`CLAUDE.md`. + +## Repository + +go-redis is the official Redis client for Go. Module path: +`github.com/redis/go-redis/v9` (Go 1.24+). The repo is a multi-module workspace +— every directory containing a `go.mod` is built and tested independently: + +- root (`github.com/redis/go-redis/v9`) — the client library. +- `extra/redisotel`, `extra/redisotel-native`, `extra/redisprometheus`, + `extra/rediscensus`, `extra/rediscmd` — instrumentation adapters with their + own module paths (so they can pin large telemetry deps without forcing them on + root consumers). +- `internal/customvet` — custom `go vet` analyzers (also its own module). +- `maintnotifications/e2e`, `doctests`, `fuzz`, examples under `example/` — + separate modules. + +The Makefile iterates over every `go.mod` (`GO_MOD_DIRS`) when running +`test.ci`, `go_mod_tidy`, etc. When you add a dependency in one module, you +almost never need to update the others. + +## Common commands + +Tests run against a Redis stack started via Docker Compose. Profiles in +`docker-compose.yml` control which services come up (`standalone`, `cluster`, +`sentinel`, `all`, `e2e`). + +```sh +make docker.start # bring up the full test stack (profile: all) +make docker.stop +make test # docker.start -> test.ci -> docker.stop +make test.ci # run tests assuming containers are already up +make test.ci.skip-vectorsets # when REDIS_VERSION < 8 +make bench # go test -bench=. (root module only) +make fmt # gofumpt + goimports -local github.com/redis/go-redis +make build +make go_mod_tidy # go mod tidy across every module +``` + +E2E (maintenance notifications) needs the extra `cae-resp-proxy` service: + +```sh +make test.e2e # starts e2e profile, runs ./maintnotifications/e2e/, tears down +make test.e2e.docker # subset that runs inside docker +make test.e2e.logic # logic-only tests, no proxy required +``` + +Run a single test. The root suite is Ginkgo-based (`bsm/ginkgo` + `bsm/gomega` +forks), so `go test -run` matches the Go-level wrapper and you focus a spec with +the Ginkgo flag: + +```sh +go test -run TestGinkgoSuite . -ginkgo.focus="ZAdd" +go test -run TestGinkgoSuite . -ginkgo.focus="cluster" +``` + +Plain `go test` tests (most files outside the Ginkgo suite, e.g. `internal/...`, +`maintnotifications/...`) work the usual way: + +```sh +go test -run TestConnStateMachine ./internal/pool/... +go test -race -run TestCircuitBreaker ./maintnotifications/... +``` + +Env knobs (passed through the Makefile): + +- `REDIS_VERSION` — e.g. `8.8`. Drives both the test image tag and + `main_test.go` version-gating (`SkipBeforeRedisVersion` / + `SkipAfterRedisVersion`). +- `CLIENT_LIBS_TEST_IMAGE` — full image ref, e.g. + `redislabs/client-libs-test:8.8-m03`. +- `RE_CLUSTER=true` — run against a Redis Enterprise cluster instead of the + docker-compose stack (the suite then skips ring/sentinel/TLS-cluster setup). +- `RCE_DOCKER=true` — Redis CE in docker (default for `make test`). +- `REDIS_PORT` — override the default standalone port (`6380`). + +CI also runs the custom vet tool: +`go vet -vettool ./internal/customvet/customvet ./...`. The `setval` analyzer +requires every `Cmder` with a `Result()` to also have a `SetVal()`. + +## Architecture + +### Client types (root package) + +All clients are in the root package and share most plumbing: + +- `Client` (`redis.go`) — single-node client. +- `ClusterClient` (`osscluster.go`) — Redis Cluster aware. `osscluster_router.go` + routes commands to the right shard; `internal/routing/` handles cluster-wide + aggregation policies (e.g. fan-out for `KEYS`, `DBSIZE`). +- `Ring` (`ring.go`) — client-side sharding across independent Redis nodes + (consistent hashing, no cluster protocol). +- Failover client (`sentinel.go`) — Sentinel-managed failover. +- `UniversalClient` (`universal.go`) — wrapper that picks one of the above based + on options. + +Command surface lives in topical files: `string_commands.go`, `hash_commands.go`, +`stream_commands.go`, `search_commands.go`, `vectorset_commands.go`, etc. Each +file defines methods on the shared `Cmdable` interface so every client type gets +the same API. + +### Hooks (`redis.go` `hooksMixin`) + +Three hook chains run around every operation: `DialHook`, `ProcessHook`, +`ProcessPipelineHook`. Hooks are registered via `client.AddHook(...)` and chain +in FIFO order; each hook must call `next` to continue. When a hook wraps an +error, it must call `cmd.SetErr(wrappedErr)` so the typed-error helpers +(`redis.IsLoadingError`, `IsMovedError`, etc. in `error.go`) keep working through +`errors.As`. The README has a longer pipeline-hook example. + +### Connection pool (`internal/pool`) + +Owns dialing, idle/active connection bookkeeping, conn state (`conn_state.go`), +pubsub-conn lifecycle (`pubsub.go`), and the dial-retry/backoff logic that powers +`DialerRetries` / `DialerRetryBackoff` (also exposed at `dial_retry_backoff.go` +in the root). `OnConnect`, `MinIdleConns`, and the buffer-size options +(`ReadBufferSize`/`WriteBufferSize`, default 32 KiB since v9.12) flow through +here. + +### Protocol (`internal/proto`) + +RESP2/RESP3 reader and writer. Push notifications (RESP3 `>`-prefixed frames) are +peeked here and dispatched via the `push/` package. The `push.Registry` lets +callers register handlers for specific notification names; +`maintnotifications/push_notification_handler.go` is how `maintnotifications` +plugs in. + +### Maintenance notifications (`maintnotifications/`) + +This is a non-trivial subsystem worth understanding before touching +cluster/handoff code. It listens for RESP3 push notifications about cluster +maintenance (`MOVING`, `MIGRATING`, `MIGRATED`, `FAILING_OVER`, `FAILED_OVER` +for standalone; `SMIGRATING`, `SMIGRATED` for cluster) and performs seamless +connection handoff to new endpoints. Key pieces: + +- `manager.go` — coordinates state transitions. +- `handoff_worker.go` — moves in-flight ops to new connections. +- `pool_hook.go` — integrates with `internal/pool` to mark/replace connections. +- `circuit_breaker.go` — backs off when the upstream is unhealthy. +- `state.go` — per-connection state machine. +- E2E coverage lives in `maintnotifications/e2e/` and drives a fault-injector / + RESP proxy (`cae-resp-proxy`). + +Configuration is via `redis.Options.MaintNotificationsConfig`; modes are +`ModeAuto` (default), `ModeEnabled` (require server support), `ModeDisabled`. +RESP3 (`Protocol: 3`) is required. + +### Authentication (`auth/`, `internal/auth/streaming`) + +Four credential sources, in priority order: streaming provider (e.g. Entra ID +via `go-redis-entraid`), context-based provider, function provider, static +`Username`/`Password`. The streaming provider is what enables token rotation +without reconnecting — the listener in `auth/reauth_credentials_listener.go` +issues `AUTH` on each refresh. + +### Internal helpers + +- `internal/hscan` — struct scanning for `HGETALL` results (`Scan` interface + re-exported as `redis.Scanner`). +- `internal/hashtag` — extracts `{tag}` segments for cluster slot routing. +- `internal/routing` — aggregator policies and shard pickers used by + `ClusterClient` for multi-shard commands. +- `internal/otel` — small OpenTelemetry shim used to keep root free of telemetry + deps; full instrumentation lives in `extra/redisotel-native`. + +## Architectural specs + +Read the relevant design doc **before** changing code in that subsystem. They +cover invariants and decisions that aren't obvious from the code, and are plain +markdown any tool or editor can open: + +- `.claude/specs/pool.md` — connection pool: `wantConn` queue and FIFO + discipline, `ConnState` machine, dial retry/backoff, hook integration, the + re-auth/handoff coexistence contract. +- `.claude/specs/cluster-routing.md` — slot computation, MOVED/ASK redirection, + request/response policies, aggregators, replica routing, topology reload, + cross-slot rules. +- `.claude/specs/maintnotifications.md` — RESP3 push notification protocol, mode + handshake, per-conn state, handoff worker pool, circuit breaker, endpoint-type + resolution, cluster vs. standalone differences. + +## Conventions + +- New `Cmder` type → also implement `SetVal` (the custom vet `setval` check + enforces this; `SetErr` is on the embedded `baseCmd`). +- Wrap errors with custom error types that implement `Unwrap`, or use + `fmt.Errorf("...: %w", err)`. Always call `cmd.SetErr(...)` after wrapping so + typed-error checks still pass. +- `gofumpt` + `goimports -local github.com/redis/go-redis` is the formatter + (`make fmt`); CI runs both. +- Don't log directly — use `internal.Logger` (set via `redis.SetLogger`); + `logging.Disable()` is called in tests. +- Version-gate Redis-version-specific tests with `SkipBeforeRedisVersion` / + `SkipAfterRedisVersion` rather than skipping at the suite level. + +### Commits and PRs + +Conventional Commits, short and exact — `(): `. +Subject ≤50 chars (hard cap 72), imperative ("add", not "added"), no trailing +period. Body only when the *why* isn't obvious from the diff; wrap at 72. + +- Types: `feat`, `fix`, `refactor`, `perf`, `docs`, `test`, `chore` (also + `build`, `ci`, `style`, `revert`). +- Scope = the subsystem touched, lowercase: `pool`, `conn`, `pubsub`, + `sentinel`, `retry`, `command`/`cmd`, `vectorset`, `otel`, `streams`, `push`, + `deps`, `ci`, `tests`, `docs`. Omit only for genuinely cross-cutting changes. +- Breaking change: `feat(scope)!: ...` plus a `BREAKING CHANGE:` body line. + Reference issues/PRs at the end — `Closes #42`, `Refs #17`. +- **No AI-attribution trailer.** Do not add `Co-Authored-By: …`, "Generated with + …", or any AI-attribution line to commits or PR bodies in this repo. + +## Repo-specific tooling + +`.claude/` holds shared AI config: + +- `commands/` — slash commands (e.g. `check-ci`, which summarizes a PR's CI). +- `skills/` — task playbooks: `testing`, `add-command`, `commit-style`, + `update-ci-image`, `prepare-release`. +- `specs/` — the architecture docs listed above. + +For Claude Code, the skills auto-trigger from their descriptions. For other +tools, each `SKILL.md` is plain markdown you can open and follow directly. diff --git a/vendor/github.com/redis/go-redis/v9/CLAUDE.md b/vendor/github.com/redis/go-redis/v9/CLAUDE.md new file mode 100644 index 00000000..7e9eb08c --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/CLAUDE.md @@ -0,0 +1,19 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Repository guidance — layout, commands, architecture, and conventions — is +maintained in [`AGENTS.md`](AGENTS.md) so it stays shared across AI tools. It is +imported below; edit `AGENTS.md`, not this file, for that content. + +@AGENTS.md + +## Claude-specific + +- Repo-local **skills** under `.claude/skills/` auto-trigger from their + descriptions — no need to invoke them manually (the set is listed in + `AGENTS.md`). +- **Slash commands** under `.claude/commands/` (e.g. `/check-ci`) are available + in-session. +- Architectural **specs** under `.claude/specs/` are read on demand; open the + relevant one before changing that subsystem. diff --git a/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md b/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md index 8c68c522..9f175644 100644 --- a/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md +++ b/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md @@ -37,7 +37,7 @@ Here's how to get started with your code contribution: > Note: this clones and builds the docker containers specified in `docker-compose.yml`, to understand more about > the infrastructure that will be started you can check the `docker-compose.yml`. You also have the possiblity > to specify the redis image that will be pulled with the env variable `CLIENT_LIBS_TEST_IMAGE`. -> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.2.1-pre`. +> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.10.0`. > If you want to test with newer Redis version, using a newer version of `redislabs/client-libs-test` should work out of the box. 4. While developing, make sure the tests pass by running `make test` (if you have the docker containers running, `make test.ci` may be sufficient). diff --git a/vendor/github.com/redis/go-redis/v9/Makefile b/vendor/github.com/redis/go-redis/v9/Makefile index 1322bd4f..8881f2d3 100644 --- a/vendor/github.com/redis/go-redis/v9/Makefile +++ b/vendor/github.com/redis/go-redis/v9/Makefile @@ -1,8 +1,8 @@ GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort) -REDIS_VERSION ?= 8.8 +REDIS_VERSION ?= 8.10 RE_CLUSTER ?= false RCE_DOCKER ?= true -CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.8.0 +CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.10.0 docker.start: export RE_CLUSTER=$(RE_CLUSTER) && \ @@ -71,6 +71,35 @@ test.ci.skip-vectorsets: cd internal/customvet && go build . go vet -vettool ./internal/customvet/customvet +# Replay the parametrized command integration suites through the AutoPipeliner +# Cmdable faces (blocking + async) to prove the commands behave the same +# batched as on a plain client. Selected via GOREDIS_TEST_SUBJECT (see +# newUniversalSubject in main_test.go). Requires the docker env +# (make docker.start). +# +# The focus covers every parametrized suite (some via the Commands/BitCount +# substrings — Describe names differ in case). Skipped on purpose: DDL Commands (needs the cluster topology the +# lightweight BeforeSuite doesn't build), HotKeys Commands (same), and +# AutoPipeline Blocking Commands (the AP suite itself — running it through an +# AP subject would nest engines). +test.autopipeline-subjects: + # RE_CLUSTER=true forces the lightweight BeforeSuite (no sentinel/ring/cluster + # setup): the command suite only needs the standalone Redis, and running the + # full stateful BeforeSuite twice (once per subject) against the same server + # corrupts replication state and fails the second run. Both faces then run + # cleanly against the same env. The explicit -timeout keeps a hang from + # eating go test's 10m default per subject. + set -e; for subj in ap-blocking ap-async; do \ + echo "=== command suite via GOREDIS_TEST_SUBJECT=$$subj ==="; \ + (export RE_CLUSTER=true && \ + export RCE_DOCKER=$(RCE_DOCKER) && \ + export REDIS_VERSION=$(REDIS_VERSION) && \ + export GOREDIS_TEST_SUBJECT=$$subj && \ + go test -v . -race -skip Example -run TestGinkgoSuite -timeout 8m \ + -ginkgo.focus='Commands|RediSearch commands|Probabilistic commands|RedisTimeseries commands|Redis VectorSet commands|BitCount|ScanIterator|Advanced JSON' \ + -ginkgo.skip='AutoPipeline Blocking Commands|DDL Commands|HotKeys Commands'); \ + done + bench: export RE_CLUSTER=$(RE_CLUSTER) && \ export RCE_DOCKER=$(RCE_DOCKER) && \ @@ -101,7 +130,7 @@ test.e2e.logic: go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/ @echo "Logic tests completed!" -.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop +.PHONY: all test test.ci test.ci.skip-vectorsets test.autopipeline-subjects bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop build: export RE_CLUSTER=$(RE_CLUSTER) && \ diff --git a/vendor/github.com/redis/go-redis/v9/README.md b/vendor/github.com/redis/go-redis/v9/README.md index ae90d2b7..59a2eee8 100644 --- a/vendor/github.com/redis/go-redis/v9/README.md +++ b/vendor/github.com/redis/go-redis/v9/README.md @@ -21,6 +21,7 @@ In `go-redis` we are aiming to support the last three releases of Redis. Current - [Redis 8.2](https://raw.githubusercontent.com/redis/redis/8.2/00-RELEASENOTES) - using Redis CE 8.2 - [Redis 8.4](https://raw.githubusercontent.com/redis/redis/8.4/00-RELEASENOTES) - using Redis CE 8.4 - [Redis 8.8](https://raw.githubusercontent.com/redis/redis/8.8/00-RELEASENOTES) - using Redis CE 8.8 +- [Redis 8.10](https://raw.githubusercontent.com/redis/redis/8.10/00-RELEASENOTES) - using Redis CE 8.10 Although the `go.mod` states it requires at minimum `go 1.24`, our CI is configured to run the tests against all supported versions of Redis and multiple versions of Go ([1.24](https://go.dev/doc/devel/release#go1.24.0), oldstable, and stable). We observe that some modules related test may not pass with @@ -78,9 +79,13 @@ surface. The API is experimental and may change in a future release. - [StreamingCredentialsProvider (e.g. entra id, oauth)](#1-streaming-credentials-provider-highest-priority) (experimental) - [Pub/Sub](https://redis.uptrace.dev/guide/go-redis-pubsub.html). - [Pipelines and transactions](https://redis.uptrace.dev/guide/go-redis-pipelines.html). +- [Automatic pipelining](#automatic-pipelining) (experimental) — batches concurrent + commands into pipelines for you; meant for high-throughput / high-load / scale + use cases. - [Scripting](https://redis.uptrace.dev/guide/lua-scripting.html). - [Redis Sentinel](https://redis.uptrace.dev/guide/go-redis-sentinel.html). - [Redis Cluster](https://redis.uptrace.dev/guide/go-redis-cluster.html). +- [Client-side caching](#client-side-caching). - [Redis Performance Monitoring](https://redis.uptrace.dev/guide/redis-performance-monitoring.html). - [Redis Probabilistic [RedisStack]](https://redis.io/docs/data-types/probabilistic/) - [Customizable read and write buffers size.](#custom-buffer-sizes) @@ -285,6 +290,50 @@ rdb := redis.NewClient(&redis.Options{ }) ``` +### Client-side caching + +go-redis supports server-assisted client-side caching for standalone clients. +Eligible read replies are stored in the application's memory, so repeated reads +can avoid a Redis round trip. Redis tracks which keys each connection has read +and sends RESP3 invalidation notifications when those keys change. go-redis +uses those notifications to evict affected entries automatically. + +> **Experimental:** The client-side caching API may change in a minor release. + +Enable the built-in bounded cache with `ClientSideCacheConfig`: + +```go +rdb := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Protocol: 3, + DB: 0, + ClientSideCacheConfig: &redis.ClientSideCacheConfig{ + MaxEntries: 10_000, + }, +}) +defer rdb.Close() +``` + +Client-side caching currently requires RESP3, a standalone client, and database +0. Fixed `Username` and `Password` values are supported. It is disabled when a +dynamic credential provider is configured, because cached data must never be +reused after the client's ACL identity changes. Only deterministic read +commands supported by the cache are stored; writes and streaming responses +bypass it. + +While client-side caching is enabled, go-redis rejects `SELECT`, `AUTH`, +`HELLO` with arguments, `RESET`, `CLIENT TRACKING`, and raw `SUBSCRIBE`, +`PSUBSCRIBE`, or `SSUBSCRIBE` commands because they would change connection +state that the cache relies on. A guarded command also fails its whole +pipeline. The typed `Subscribe`, `PSubscribe`, and `SSubscribe` APIs remain +supported because they use dedicated connections. + +Invalidations are processed asynchronously. `DrainInterval` controls how often +idle connections are checked for them, while `MaxStaleness` can provide an +optional upper bound on an entry's lifetime. See the +[client-side caching example](./example/client-side-caching) for a working +demonstration. + ### Connecting via a redis url go-redis also supports connecting via the @@ -340,6 +389,118 @@ rdb := redis.NewClient(&redis.Options{ }) ``` +### Automatic pipelining + +**Experimental** — the API may still change. Reach for autopipelining in +high-throughput / high-load / scale scenarios; at low concurrency a plain +client is simpler and just as fast. A runnable usage tour and throughput +comparison live in [`example/autopipeline`](example/autopipeline). + +> **EXPERIMENTAL:** the autopipelining API is subject to change in a future +> release as we gather feedback — pin your go-redis version if you adopt it. + +When many goroutines issue commands concurrently, autopipelining batches them +into Redis pipelines automatically — without you writing any pipeline code. It +comes in two faces: + +- **`AutoPipeline()` — blocking, drop-in.** Each command call blocks until it + executes and returns its own value/error, exactly like a normal client, so + existing code keeps working unchanged. Under concurrency the engine coalesces + commands from all goroutines into deep, back-to-back pipelines (a single + ordered batch stream by default), reaching several times a plain client's + executed commands per second in the same environment — roughly an order of + magnitude with a parallel-batch config (`MaxConcurrentBatches` > 1 with + `Unordered`). Per-goroutine ordering is preserved. +- **`AsyncAutoPipeline()` — deferred, highest throughput.** Command calls return + immediately; you submit a window of commands and read their results afterward, + which keeps each pipeline deep — tens of times a plain client's throughput. + Ordered by default. Absolute numbers depend heavily on the machine, network + path and server; see `autopipeline_bench_README.md` for the benchmark + methodology and multipliers. + +```go +rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379"}) +defer rdb.Close() +ctx := context.Background() + +// Blocking face: drop-in for a normal client, batched under the hood. +ap, err := rdb.AutoPipeline() +if err != nil { // invalid AutoPipelineOptions, or the client is closed + log.Fatal(err) +} +defer ap.Close() + +var wg sync.WaitGroup +for i := 0; i < 1000; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + key := fmt.Sprintf("key:%d", i) + if err := ap.Set(ctx, key, i, 0).Err(); err != nil { // blocks until executed + log.Printf("set %s: %v", key, err) + } + }(i) +} +wg.Wait() +``` + +For maximum throughput, submit a window on the async face and read later: + +```go +ctx := context.Background() +ap, err := rdb.AsyncAutoPipeline() // ordered by default +if err != nil { + log.Fatal(err) +} +defer ap.Close() + +cmds := make([]*redis.StatusCmd, 0, 200) +for i := 0; i < 200; i++ { + cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("key:%d", i), i, 0)) // returns immediately +} +for _, cmd := range cmds { + if err := cmd.Err(); err != nil { // blocks until executed + log.Printf("set: %v", err) + } +} +``` + +Each face has a no-argument form that uses `Options.AutoPipelineOptions` (or the +built-in default) and a `WithOptions` form that takes an explicit +`*AutoPipelineOptions`; both return `(*AutoPipeliner, error)` — the error is +non-nil for an invalid config or a closed client (e.g. +`ap, err := rdb.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxConcurrentBatches: 8, Unordered: true})`); +a handful of parallel batches saturates the link — more permits only add +overlapping batches without deepening them. +They work on `ClusterClient` too: commands are routed to the correct shard per +key, so a single batch may span many slots; ordering across nodes is per key +(same-key commands stay in order, different nodes' sub-pipelines run +concurrently). Because batches share a few pipeline connections, autopipelining +also needs far fewer connections than a plain client at the same concurrency +(see `PipelinePoolSize`). Autopipelining is only a win under concurrency (or +windowed submission) — a single goroutine issuing one blocking command at a +time sees little benefit, and a hand-written `Pipeline()` is still fastest when +you can batch by hand. + +Caveats: a command's context is not honored once it is queued (batches execute +on the autopipeliner's own context) — use a plain client for per-command +deadlines. Blocking commands (`BLPOP`, `WAIT`, ...) are never batched and run +directly on your context — as are `SHUTDOWN` and `MONITOR`, which would +poison a shared pipeline connection — and `Do` also bypasses batching with plain +`Client.Do` semantics — prefer the typed methods (`ap.Set`, `ap.Get`, ...). On +a dropped connection a batch is retried whole (up to `MaxRetries`), so +non-idempotent commands may execute twice. Both faces return a cached, +client-shared instance: the first call's config wins and `Close` stops it for +all callers. Hooks may read command results (the engine hands a hook running +on the dispatch goroutine the same view a plain pipeline hook gets), but a +hook must never issue a command on the same autopipeliner and wait for it — +the nested command needs the very dispatch slot the hook is holding, and the +engine only recovers by failing that flush after its 30s permit backstops. +`Options.Limiter` is consulted once per batch dispatch (as with a manual +pipeline), not once per command. An autopipeliner created on a +`WithTimeout`/`WithReadTimeout` clone is not stopped by the parent's `Close` — +close it explicitly. + ### Advanced Configuration go-redis supports extending the client identification phase to allow projects to send their own custom client identification. @@ -457,6 +618,24 @@ vals, err := rdb.Eval(ctx, "return {KEYS[1],ARGV[1]}", []string{"key"}, "hello") res, err := rdb.Do(ctx, "set", "key", "value").Result() ``` +### Raw commands and connection state + +`Do` sends the command verbatim on whichever pooled connection happens to be +free. For keyspace commands that is all you need. It is the wrong tool for +any command that alters **connection session state** — `SELECT`, +`CLIENT SETNAME`, `CLIENT TRACKING`, `RESET`, `HIMPORT PREPARE`/`DISCARD`, +and similar: the state lands on (or is wiped from) a single arbitrary +connection, later commands are served by other connections that don't share +it, and the affected connection eventually returns to the pool and serves +unrelated callers. The result is nondeterministic behavior that typed APIs +manage for you — for example, the typed `HImport*` methods keep a +client-side registry and replay fieldsets onto every connection that needs +them, while a raw `Do(ctx, "himport", "prepare", ...)` bypasses that +entirely, with no replay, recovery, or discard propagation. + +For session-scoped work without a typed API, hold a dedicated connection +(`client.Conn()`) for its whole lifetime and close it afterwards. + ## Typed Errors go-redis provides typed error checking functions for common Redis errors: diff --git a/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md b/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md index a125344d..0ebed092 100644 --- a/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md +++ b/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md @@ -1,5 +1,244 @@ # Release Notes +# 9.22.0 (2026-08-03) + +This is a minor release introducing two flagship (experimental) features — **client-side caching** and **automatic pipelining** — alongside support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. It consolidates everything shipped in 9.22.0-beta.1, so the notes below cover the full 9.21.0 → 9.22.0 upgrade. + +⚠️ Two changes to be aware of when upgrading from 9.21.0: + +- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected. +- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update. + +## 🚀 Highlights + +### Client-Side Caching (Experimental) + +The standalone `Client` gains server-assisted client-side caching built on RESP3 `CLIENT TRACKING`. Enable it by setting `ClientSideCacheConfig` in `Options` (or supply your own cache via `ClientSideCache` — e.g. to share one cache across clients). Cacheable read results are served from a local in-process cache and invalidated automatically when the server reports a change, cutting round trips for read-heavy workloads. + +The invalidation architecture is selected by `ClientSideCacheStrategy`; the default (and currently only) strategy is `CSCStrategySharedTracking`: one shared cache, every pool connection runs plain `CLIENT TRACKING ON`, and a background drainer applies buffered invalidations — portable (no BCAST) and consistent with the other Redis client libraries. Requirements and guardrails: RESP3 (`Protocol: 3`), standalone client, DB 0 only; commands that would change the connection identity (`SELECT`, `AUTH`, ...) are rejected while caching is enabled, and CSC is disabled when a credentials provider is set (fixed `Username`/`Password` work and are namespaced). See the README's [client-side caching section](README.md#client-side-caching) and the runnable [example](example/client-side-caching). + +**Experimental:** the API may change in a minor release. + +([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa) + +### Automatic Pipelining (Experimental) + +`AutoPipeliner` is a background batcher that coalesces commands from many concurrent goroutines into Redis pipelines, multiplying throughput without any manual pipeline management. It comes in two faces, available on `Client` and `ClusterClient` (and configurable via `Options.AutoPipelineOptions` / `UniversalOptions.AutoPipelineOptions`): + +- **`AutoPipeline()`** — the blocking face: a drop-in `Cmdable` where each call blocks until executed, exactly like a plain client, while concurrent callers' commands batch together under the hood (measured locally over loopback: ~1M+ SET/sec vs ~100k unpipelined; indicative, not a guarantee). Per-goroutine command order is preserved. +- **`AsyncAutoPipeline()`** — the deferred face: command calls return immediately and every typed result accessor (`Val`/`Result`/`Err`/...) blocks until the command has executed. Submit a window of commands, then read the results, to keep pipelines deep (~2–3M SET/sec locally; indicative). + +`AutoPipelineOptions` controls batching: `MaxBatchSize` (soft target, default 200; the blocking face's preset uses 300), `MaxBatchBytes` (approximate payload cap so huge values flush as several bounded writes), `MaxFlushDelay` with optional `AdaptiveDelay` (delay scales down as the queue fills), and `MaxConcurrentBatches` (default 1 = a single ordered batch stream; raising it requires `Unordered: true`, so ordering is never lost by accident — `Validate()` rejects the combination otherwise). A usage tour and throughput comparison live in [`example/autopipeline`](example/autopipeline). + +**Experimental:** the API may change in a future release — pin your go-redis version if you adopt it. + +([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs) + +### Redis 8.10 Support + +This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)). + +Coverage for the new commands and options that ship with Redis 8.10: + +- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`). +- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call. +- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result. +- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns. +- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface. +- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage. + +### Cross-SDK Aligned Defaults + +Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)): + +| Setting | Old default | New default | +|---|---|---| +| `ReadTimeout` / `WriteTimeout` | 3s | 5s | +| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s | +| Cluster state reload interval | 10s | 60s | +| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) | + +Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones. + +### Data-Race and Parser Hardening Sweep + +A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)). + +### PubSub `Receive` Hang Fix + +`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935). + +([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) + +### Correct Cluster Transaction Retries + +The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs). + +### Credential Redaction in Command Tracing + +`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as ``: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e). + +## ✨ New Features + +- **Client-side caching**: server-assisted caching for the standalone client via `ClientSideCacheConfig` / `ClientSideCache`, with the `CSCStrategySharedTracking` invalidation strategy ([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **Automatic pipelining**: `AutoPipeline()` (blocking) and `AsyncAutoPipeline()` (deferred results) on `Client` and `ClusterClient`, configured via `AutoPipelineOptions` ([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs) +- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov) +- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov) + +## 🐛 Bug Fixes + +- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) +- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs) +- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e) +- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9) +- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel) +- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs) +- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam) +- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev) +- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev) +- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev) +- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e) +- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215) +- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov) +- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov) + +## 🧰 Maintenance + +- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov) +- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940)) +- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs) +- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis) + +## 👥 Contributors + +We'd like to thank all the contributors who worked on this release! + +[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev) + +--- + +**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0 + +# 9.22.0-beta.1 (2026-07-29) + +This is a **beta** release adding support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. The 9.22.0 GA release will follow once client-side caching and auto-pipelining are merged. + +⚠️ Two changes to be aware of when upgrading from 9.21.0: + +- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected. +- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update. + +## 🚀 Highlights + +### Redis 8.10 Support + +This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)). + +Coverage for the new commands and options that ship with Redis 8.10: + +- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`). +- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call. +- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result. +- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns. +- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface. +- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage. + +### Cross-SDK Aligned Defaults + +Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)): + +| Setting | Old default | New default | +|---|---|---| +| `ReadTimeout` / `WriteTimeout` | 3s | 5s | +| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s | +| Cluster state reload interval | 10s | 60s | +| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) | + +Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones. + +### Data-Race and Parser Hardening Sweep + +A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)). + +### PubSub `Receive` Hang Fix + +`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935). + +([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) + +### Correct Cluster Transaction Retries + +The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs). + +### Credential Redaction in Command Tracing + +`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as ``: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e). + +## ✨ New Features + +- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov) +- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov) + +## 🐛 Bug Fixes + +- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) +- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs) +- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e) +- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9) +- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel) +- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs) +- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam) +- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev) +- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev) +- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev) +- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e) +- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215) +- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov) +- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov) + +## 🧰 Maintenance + +- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov) +- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940)) +- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs) +- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis) + +## 👥 Contributors + +We'd like to thank all the contributors who worked on this release! + +[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev) + +--- + +**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0-beta.1 + # 9.21.0 (2026-06-18) This is a minor release adding new features and bug fixes. There are no breaking changes; upgrading from 9.20.x is a drop-in replacement. diff --git a/vendor/github.com/redis/go-redis/v9/autopipeline.go b/vendor/github.com/redis/go-redis/v9/autopipeline.go new file mode 100644 index 00000000..42ac8d50 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/autopipeline.go @@ -0,0 +1,2609 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "io" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sys/cpu" + + "github.com/redis/go-redis/v9/internal" +) + +// AutoPipelineOptions configures the autopipelining behavior. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +type AutoPipelineOptions struct { + // MaxBatchSize is the target batch size: the accumulator stops waiting for + // more commands once the shard queue reaches it, so a batch flushes promptly + // instead of lingering. It is a soft threshold, not a hard cap — under heavy + // concurrent enqueue (or while a flush waits on the concurrency semaphore) the + // queue can grow past it and execute as a single larger pipeline, which is + // safe and simply yields a deeper pipeline. + // Default: 200 (the blocking face's no-options preset, + // DefaultBlockingAutoPipelineOptions, uses 300). + MaxBatchSize int + + // MaxBatchBytes, when > 0, caps a batch by APPROXIMATE payload volume: the + // accumulator stops waiting once the queued commands' argument bytes reach + // it, so many large values flush as several bounded writes instead of one + // huge burst (300 x 64KiB is ~19MB written down one connection before any + // reply is read — enough to stall a constrained link past its write + // deadline). Like MaxBatchSize it is a soft threshold, not a hard cap. + // The estimate counts string/[]byte argument lengths plus a small + // per-argument overhead. Default: 0 (no byte cap). + MaxBatchBytes int + + // MaxConcurrentBatches is the maximum number of pipeline batches that may + // execute concurrently. + // + // Default: 1, which gives a single ordered command stream — batches execute + // serially in submit order, so even a windowed caller (submit many, read + // later) sees strict ordering, while still reaching high throughput via deep + // pipelines (~3M ops/sec locally). + // + // Setting this above 1 runs batches in parallel for maximum throughput, but + // commands then have NO guaranteed execution order. Because that trades away + // ordering, it is only allowed together with Unordered: true — otherwise the + // configuration is rejected (see Validate). This makes the trade-off + // explicit: you cannot accidentally lose ordering by raising concurrency. + MaxConcurrentBatches int + + // Unordered must be set to true to allow MaxConcurrentBatches > 1. It is the + // caller's explicit acknowledgement that parallel batch execution gives up + // command ordering in exchange for throughput. With the default (false), + // MaxConcurrentBatches is forced to 1 (an ordered stream) and any value > 1 + // is a configuration error. + Unordered bool + + // contentSharded is set internally by cluster wiring when commands are + // routed to shards by content (slot), so same-key commands always share a + // shard and per-key order holds even with several shards. It exempts that + // wiring from the NumShards ordering check in newAutoPipeliner. Never set + // by users (unexported). + contentSharded bool + + // NumShards is the number of independent queue+flusher shards the + // autopipeliner runs. 0 (the default) means auto: a single shard, which + // funnels every caller into one queue so batches stay deep — measured + // throughput and latency are best with one shard even under heavy + // goroutine concurrency. Cluster clients default to several slot-routed + // shards instead, so commands for different nodes queue independently + // (per-key order still holds: a key's slot always maps to the same + // shard). Raising NumShards splits the queue: it reduces enqueue-mutex + // contention but fragments batches, which usually costs far more than the + // contention saves. Every shard always has at least one concurrency + // permit, so the effective global batch concurrency is + // max(NumShards, MaxConcurrentBatches) — and because shards flush + // concurrently, NumShards > 1 on the deferred (async) face requires + // Unordered: true (construction fails otherwise). + NumShards int + + // MaxFlushDelay is the maximum delay after flushing before checking for more commands. + // A small delay (e.g., 100μs) can significantly reduce CPU usage by allowing + // more commands to batch together, at the cost of slightly higher latency. + // + // Trade-off: + // - 0 (default): Lowest latency, higher CPU usage + // - 100μs: Balanced (recommended for most workloads) + // - 500μs: Lower CPU usage, higher latency + // + // Based on benchmarks, 100μs can reduce CPU usage by 50% + // while adding only ~100μs average latency per command. + // Default: 0, meaning the flusher applies no coalescing wait — it flushes + // each batch as soon as the queue is ready and lets in-flight backpressure + // coalesce concurrent callers (see accumulateBatch). Set a value here to add + // an explicit accumulation window, trading latency for larger batches / less + // CPU as described above. + MaxFlushDelay time.Duration + + // AdaptiveDelay enables smart delay calculation based on queue fill level. + // When enabled, the delay is automatically adjusted: + // - Queue ≥75% full: No delay (flush immediately to prevent overflow) + // - Queue ≥50% full: 25% of MaxFlushDelay (queue filling up) + // - Queue ≥25% full: 50% of MaxFlushDelay (moderate load) + // - Queue <25% full: 100% of MaxFlushDelay (low load, maximize batching) + // + // This provides automatic adaptation to varying load patterns without + // manual tuning. Uses integer-only arithmetic for optimal performance. + // Default: false (use fixed MaxFlushDelay) + AdaptiveDelay bool +} + +// autoPipelinePermitBackstop bounds how long a flush waits for a concurrency +// permit when all are busy. It is only a safety net against a wedged semaphore: +// every permit holder releases it (via defer) and each batch Exec is itself +// bounded by the connection's read/write timeout, so in normal operation a +// permit frees long before this. It is set well above the default ReadTimeout +// and a maintnotifications relaxed window so a legitimately slow in-flight batch +// never makes waiters fail spuriously. The wait deliberately does NOT end on +// Close: commands taken from the queue were already accepted, and Close's +// contract is to flush them (it waits via wg/batchWg), so permit waits run on +// a background context bounded only by this backstop. +const autoPipelinePermitBackstop = 30 * time.Second + +// autoPipelineCloseBackstop bounds Close's wait for in-flight dispatches. It +// deliberately carries the same value as the permit backstop but its OWN name: +// the two answer different questions, and this one may want tuning on its own. +// +// Why it is generous rather than snappy: the bound is only ever REACHED when a +// dispatch cannot end by itself — a blocking command with no timeout, or a +// stalled read with ReadTimeout disabled. In every other configuration the +// read timeout ends the dispatch and Close returns the moment it does, well +// under this value. A tighter bound would not speed up healthy shutdowns; it +// would instead make Close report failure while legitimate work is still +// finishing (a large final batch, or a maintnotifications relaxed window +// during a failover), turning a correct slow drain into a spurious error. +const autoPipelineCloseBackstop = 30 * time.Second + +// numAutoPipelineShards is the shard-count default used by CLUSTER wiring, +// where commands are routed to shards by slot so different nodes' batches +// queue independently (every shard keeps at least one concurrency permit, so +// several shards can flush to their nodes in parallel regardless of +// MaxConcurrentBatches). It is NOT used for standalone clients: those default +// to one shard (see newAutoPipeliner), because a single deep queue pipelines +// far better than a fragmented one. Deliberately NOT derived from +// MaxConcurrentBatches — coupling shard count to the permit budget silently +// collapsed cluster slot routing to a single shard at the default budget. +func numAutoPipelineShards() int { + n := runtime.GOMAXPROCS(0) + if n < 1 { + n = 1 + } + const maxShards = 16 + if n > maxShards { + n = maxShards + } + return n +} + +// DefaultAutoPipelineOptions returns the default autopipelining configuration. +// +// The default is ordered: MaxConcurrentBatches is 1, so batches execute +// serially in submit order (a single ordered command stream) while still +// reaching high throughput via deep pipelines when callers submit in windows. +// To trade ordering for parallel-batch throughput, set MaxConcurrentBatches > 1 +// together with Unordered: true. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func DefaultAutoPipelineOptions() *AutoPipelineOptions { + return &AutoPipelineOptions{ + MaxBatchSize: 200, + MaxConcurrentBatches: 1, // ordered by default + MaxFlushDelay: 0, // lowest latency; no coalescing wait (batch via in-flight backpressure) + } +} + +// DefaultBlockingAutoPipelineOptions returns the default config for the +// blocking face (Client.AutoPipeline). It uses a single ordered batch stream +// (MaxConcurrentBatches: 1). Counterintuitively this maximizes throughput AND +// minimizes latency for the blocking face: with one batch in flight, callers whose +// commands return while it executes re-enqueue and flush together as the next +// batch, so batches stay deep (a near-continuous, double-buffered pipeline), +// while a lone caller flushes promptly in a single round-trip (no coalescing +// wait — see accumulateBatch). More parallel permits (MaxConcurrentBatches>1) do the +// opposite: each command finds a free permit and flushes on its own before +// others accumulate, collapsing batch size — and throughput — toward one command +// per round-trip while latency rises. For maximum throughput use the async face +// (AsyncAutoPipeline) with a window of in-flight commands (inflight>1); it keeps +// MaxConcurrentBatches: 1 as well. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func DefaultBlockingAutoPipelineOptions() *AutoPipelineOptions { + return &AutoPipelineOptions{ + MaxBatchSize: 300, + MaxConcurrentBatches: 1, + } +} + +// Validate reports whether the configuration is self-consistent. It returns an +// error if MaxConcurrentBatches > 1 without Unordered: true — raising +// concurrency gives up command ordering, so the caller must opt in explicitly. +// +// Validate()==nil does not guarantee construction succeeds: rules that need +// the face (e.g. NumShards>1 requires Unordered on the deferred face) are +// enforced by the AutoPipeline/AsyncAutoPipeline getters. Note also that +// Options.AutoPipelineOptions is validated lazily — on the first getter +// call, not in NewClient. +func (cfg *AutoPipelineOptions) Validate() error { + if cfg.MaxConcurrentBatches > 1 && !cfg.Unordered { + return fmt.Errorf("redis: AutoPipelineOptions.MaxConcurrentBatches=%d requires Unordered:true "+ + "(parallel batches do not preserve command ordering); set Unordered:true to allow it, "+ + "or keep MaxConcurrentBatches=1 for an ordered stream", cfg.MaxConcurrentBatches) + } + // Reject obviously-wrong negatives so a typo surfaces at construction rather + // than being silently coerced to a default. Zero is allowed and means "use + // the default" (MaxBatchSize) or "no delay" (MaxFlushDelay). + if cfg.MaxBatchSize < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxBatchSize=%d must be >= 0", cfg.MaxBatchSize) + } + if cfg.MaxBatchBytes < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxBatchBytes=%d must be >= 0", cfg.MaxBatchBytes) + } + if cfg.MaxConcurrentBatches < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxConcurrentBatches=%d must be >= 0", cfg.MaxConcurrentBatches) + } + if cfg.MaxFlushDelay < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxFlushDelay=%s must be >= 0", cfg.MaxFlushDelay) + } + if cfg.NumShards < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.NumShards=%d must be >= 0", cfg.NumShards) + } + if cfg.AdaptiveDelay && cfg.MaxFlushDelay <= 0 { + return fmt.Errorf("redis: AutoPipelineOptions.AdaptiveDelay requires MaxFlushDelay > 0 " + + "(adaptive delay scales MaxFlushDelay by queue fill; with no MaxFlushDelay it would " + + "silently disable batch accumulation entirely)") + } + return nil +} + +// cmdableClient is an interface for clients that support pipelining. +// Both Client and ClusterClient implement this interface. It embeds +// UniversalClient (Cmdable + Process + Do + AddHook + Watch + Subscribe... + +// Close + PoolStats) so the AutoPipeliner can delegate the non-batched surface +// back to the underlying client and itself satisfy UniversalClient. +type cmdableClient interface { + UniversalClient + // processPipelineHook is the hook-wrapped []Cmder pipeline entry — the same + // method Pipeline.Exec is wired to (see Client.Pipeline). The flusher + // dispatches drained batches through it directly, skipping the per-batch + // Pipeline construction; hooks/OTel see the identical call. + processPipelineHook(ctx context.Context, cmds []Cmder) error + // The async faces additionally dispatch through withProcessPipelineHook / + // withProcessHook with the base processors as the innermost, so the batch + // can be completed UNDER the user hooks (results ready the moment exec + // returns, before hooks unwind). Both *Client and *ClusterClient satisfy + // these via hooksMixin and their base processors. + withProcessPipelineHook(ctx context.Context, cmds []Cmder, hook ProcessPipelineHook) error + hookCount() int + withProcessHook(ctx context.Context, cmd Cmder, hook ProcessHook) error + processPipeline(ctx context.Context, cmds []Cmder) error + process(ctx context.Context, cmd Cmder) error +} + +// apBatch is the completion signal shared by every command flushed together. +// Its done channel is closed exactly once, when the batch's pipeline has +// executed. Closing one channel wakes all waiters in a single operation, +// instead of doing one buffered-channel send per command — under high +// concurrency the per-command sends dominated CPU (channel-lock contention and +// one goroutine wake-up apiece). +type apBatch struct { + done chan struct{} + // closed makes close() idempotent: on the async faces the dispatch closes + // the batch at the innermost exec seam (under the user hooks, so a hook + // reading a result after next() does not block on a channel its own + // goroutine closes — the #3867 deadlock), while the flusher keeps its + // deferred close as a panic backstop. Whichever runs first wins. + closed atomic.Bool + // dispGid is the goroutine id of the dispatcher while the batch is inside + // the hook chain (0 otherwise). await() consults it before blocking so a + // hook on the dispatch goroutine reading a result BEFORE next() gets the + // not-yet-executed view — what a plain pipeline hook sees — instead of a + // self-deadlock. + dispGid atomic.Int64 + // nodeGids registers cluster per-node executor goroutines: the cluster + // pipeline fans a batch out to one goroutine per node, and each runs the + // NODE client's own hook chain (OnNewNode hooks — redisotel's tracing + // lives there), which the single dispGid slot cannot vouch for. A node + // hook reading a result there would block on a batch that completes only + // after its own return — reproduced as a permanent wedge with a + // rediscmd-shaped Err() peek. Guarded by nodeMu; entered/left once per + // node call, consulted only on the guards' slow path (done still open). + nodeMu sync.Mutex + nodeGids []int64 + // nodeCount mirrors len(nodeGids) so isExecutorGoroutine's fast path can + // skip the goroutine-id parse and the mutex entirely when nobody is + // registered — which is every standalone batch, always, and a cluster + // batch outside its node fan-out window. + nodeCount atomic.Int32 +} + +// enterNodeDispatch registers the calling goroutine as an executor of this +// batch for the duration of a cluster node call; the returned func +// unregisters it. Registered goroutines get the same treatment as the +// dispatcher in the accessor guards: result reads return the current view +// instead of self-deadlocking on the batch's own completion signal. +func (b *apBatch) enterNodeDispatch() func() { + gid := curGoroutineID() + b.nodeMu.Lock() + b.nodeGids = append(b.nodeGids, gid) + b.nodeCount.Store(int32(len(b.nodeGids))) + b.nodeMu.Unlock() + return func() { + b.nodeMu.Lock() + for i, g := range b.nodeGids { + if g == gid { + b.nodeGids[i] = b.nodeGids[len(b.nodeGids)-1] + b.nodeGids = b.nodeGids[:len(b.nodeGids)-1] + break + } + } + b.nodeCount.Store(int32(len(b.nodeGids))) + b.nodeMu.Unlock() + } +} + +// isExecutorGoroutine reports whether the CALLING goroutine is currently +// executing this batch: the flusher/dispatch goroutine or a registered +// cluster node executor. The no-executor fast path (dispGid unset and no +// node executors) is two atomic loads — no goroutine-id parse, no lock. That +// laziness is load-bearing: every blocking-face command and every pre-done +// future passes here once per wait, and an earlier revision that parsed the +// goroutine id and took the mutex unconditionally cost the blocking face 6x +// of its throughput (measured 830k -> 138k ops/sec on a loopback bench). +func (b *apBatch) isExecutorGoroutine() bool { + disp := b.dispGid.Load() + if disp == 0 && b.nodeCount.Load() == 0 { + return false + } + gid := curGoroutineID() + if disp != 0 && disp == gid { + return true + } + if b.nodeCount.Load() == 0 { + return false + } + b.nodeMu.Lock() + defer b.nodeMu.Unlock() + for _, g := range b.nodeGids { + if g == gid { + return true + } + } + return false +} + +// noopUnregister is registerBatchExecutors' zero-batch result, shared so the +// plain-pipeline path stays allocation-free. +var noopUnregister = func() {} + +// registerBatchExecutors marks the calling goroutine as an executor of every +// deferred-face batch among cmds (plain pipeline commands carry none) and +// returns the combined unregister. The cluster pipeline calls it around each +// node's hook chain. +func registerBatchExecutors(cmds []Cmder) func() { + var undo []func() + var seenFirst *apBatch + var seenMore map[*apBatch]struct{} + for _, cmd := range cmds { + bc, ok := cmd.(interface{ readyBatch() *apBatch }) + if !ok { + continue + } + b := bc.readyBatch() + if b == nil || b == seenFirst { + continue + } + if seenFirst == nil { + seenFirst = b + } else { + if seenMore == nil { + seenMore = make(map[*apBatch]struct{}, 2) + } + if _, dup := seenMore[b]; dup { + continue + } + seenMore[b] = struct{}{} + } + undo = append(undo, b.enterNodeDispatch()) + } + if len(undo) == 0 { + return noopUnregister + } + return func() { + for _, u := range undo { + u() + } + } +} + +func newAPBatch() *apBatch { return &apBatch{done: make(chan struct{})} } + +// close completes the batch exactly once, waking every waiter. +func (b *apBatch) close() { + if b.closed.CompareAndSwap(false, true) { + close(b.done) + } +} + +// curGoroutineID parses the goroutine id from runtime.Stack's header +// ("goroutine 123 ["). Called only on paths already paying a dispatch or an +// about-to-block round-trip wait — never on await()'s fast path — so the +// microsecond-scale stack read is noise against the batch RTT. +// armSelfDeadlockGuard reports whether async dispatch should stamp the +// dispatcher's goroutine id on the batches (see apBatch.dispGid) — the +// mechanism that lets a hook on the dispatch goroutine read a command +// without deadlocking on a batch only that goroutine completes: before +// next() it sees the not-yet-executed view, after next() the populated +// results (batches complete only when the whole chain has returned). Armed +// when user hooks exist — without hooks nothing can read a command inside +// the chain — and always on cluster clients, whose node clients may carry +// their own hooks (OnNewNode + AddHook, the redisotel pattern) that +// hookCount() cannot see. NOTE: node-level hooks run on node-worker +// goroutines the gid guard cannot identify, so they must not read command +// results on the async face; the same applies to a goroutine a hook spawns +// and joins before returning. A hook added concurrently with an in-flight +// dispatch misses the guard for that one batch. The guard covers result +// READS only: a hook that ISSUES a command on the same AutoPipeliner and +// synchronously waits for it cannot be saved — the nested command needs the +// dispatch slot the hook chain is holding, and the engine recovers only by +// failing the flush after the permit backstops (see +// autoPipelinePermitBackstop) expire. +func (ap *AutoPipeliner) armSelfDeadlockGuard() bool { + return ap.pipeliner.hookCount() > 0 || ap.config.contentSharded +} + +func curGoroutineID() int64 { + var buf [64]byte + n := runtime.Stack(buf[:], false) + const skip = len("goroutine ") + var id int64 + for _, c := range buf[skip:n] { + if c < '0' || c > '9' { + break + } + id = id*10 + int64(c-'0') + } + return id +} + +// The shard queue stores bare Cmders. The batch a command waits on is the +// shard's curBatch at enqueue time — read once to wire the command's ready +// channel and never needed per-command afterward (the flusher closes the one +// shared batch). Storing []Cmder removes a per-command wrapper allocation. + +var queueSlicePool = sync.Pool{ + New: func() interface{} { s := make([]Cmder, 0, 100); return &s }, +} + +func getQueueSlice(capacity int) []Cmder { + slice := (*queueSlicePool.Get().(*[]Cmder))[:0] + if cap(slice) < capacity { + queueSlicePool.Put(&slice) + return make([]Cmder, 0, capacity) + } + return slice +} + +func putQueueSlice(slice []Cmder) { + if cap(slice) <= 1000 { + // Zero only the used prefix: elements beyond len are already nil — + // slices enter the pool fully zeroed (here) and are only appended to + // afterwards, so the tail invariant holds. Zeroing the whole capacity + // memclr'd up to 8 KB per flush for small batches on large recycled + // arrays. + for i := range slice { + slice[i] = nil + } + queueSlicePool.Put(&slice) + } +} + +// AutoPipeliner automatically batches commands and executes them in pipelines. +// It's safe for concurrent use by multiple goroutines. +// +// AutoPipeliner works by collecting commands from multiple goroutines into a +// shared queue and flushing them as one Redis pipeline when the batch reaches +// MaxBatchSize or a configured coalescing window (MaxFlushDelay) elapses. By +// default there is no window: each batch flushes as soon as the queue is ready +// and concurrent callers coalesce via in-flight backpressure, so a lone command +// flushes in a single round-trip while batches stay deep under load. +// +// This provides significant performance improvements for workloads with many +// concurrent small operations, as it reduces the number of network round-trips. +// +// AutoPipeliner implements the Cmdable interface, so you can use it like a +// regular client. Prefer the typed methods (Set, Get, ...); Do runs OUTSIDE +// the pipeline on a normal connection (see Do). +// AutoPipeline / AsyncAutoPipeline return an error for an invalid config, so check it once: +// +// ap, err := client.AutoPipeline() +// if err != nil { +// return err +// } +// ap.Set(ctx, "key", "value", 0) +// ap.Get(ctx, "key") +// ap.Close() +// +// Per-command contexts: a command is batched and executed on the AutoPipeliner's +// own long-lived context, NOT the context passed to the command. A per-command +// deadline or cancellation is therefore not honored once the command is queued +// (this is deliberate — a per-batch timer per command would cost a goroutine +// each). Use a plain client for commands that need their own deadline. +// The one exception is a blocking command (readTimeout() != nil, e.g. BLPOP): +// it is never batched and runs directly on the caller's context, which is +// honored as usual. +// +// Retries: like any pipeline, a batch that fails on a network error is retried +// as a whole (up to Options.MaxRetries). If the connection drops after the +// server executed part of the batch, non-idempotent commands (INCR, LPUSH, ...) +// may execute twice. Run commands that must not be retransmitted on a plain +// client, or set MaxRetries: -1. +// +// Lifetime: AutoPipeline() returns a single, client-owned instance shared by all +// callers. Close()ing it stops the shared pipeliner for everyone; a later +// AutoPipeline() call on the client builds a fresh one. Closing the CLIENT also +// stops it, but permanently: the getters then return ErrClosed. +// +// Formatting: String()/%v on a command issued by the deferred face WAITS for +// execution, exactly like Err()/Val()/Result() — formatting reads the result +// fields, and reading them unsynchronized would race the dispatcher populating +// them. The one exception is a hook formatting a command from the batch's own +// dispatch goroutine: that returns the not-yet-executed view instead of +// self-deadlocking. Use Name()/Args() if you need to log a submission without +// waiting for it. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +type AutoPipeliner struct { + cmdable // Embed cmdable to get all Redis command methods + + pipeliner cmdableClient + config *AutoPipelineOptions + // blocking selects how the typed command surface (Set, Get, ...) behaves: + // when true the command call itself blocks until the command has executed + // (drop-in, synchronous shape); when false the call returns immediately and + // the result accessors (Val/Result/Err) block. See AutoPipeline (blocking) + // vs AsyncAutoPipeline (deferred). + blocking bool + + // Sharded command queues. Each shard has its own queue, mutex and flusher + // goroutine, so enqueues from many goroutines spread across shards instead + // of all contending on a single mutex and being drained by a single + // flusher. Commands are assigned to shards round-robin; per-goroutine + // ordering is still guaranteed because Do blocks for each command's result + // before issuing the next one. + shards []*apShard + next atomic.Uint32 // round-robin shard selector + // shardFn, when set, picks a command's shard from its content (cluster mode + // sets it to route by slot so all commands for one node land in the same + // shard's batch — keeping per-node pipelines deep instead of splitting every + // batch across nodes). When nil, commands are assigned round-robin. + shardFn func(Cmder) int + + // preflight, when set, can reject a command at submit time, before it is + // enqueued or dispatched (cluster mode refuses fan-out-policy commands + // that cannot ride a pipeline, so one caller's command cannot poison a + // merged batch). The returned error is set on the command. + preflight func(ctx context.Context, cmd Cmder) error + + // mustDivert, when set, forces a command off the batching path even though + // it is otherwise batchable — cluster mode uses it for commands whose + // routing is NOT slot-derived (ReqSpecial, e.g. FT.CURSOR READ, which is + // sticky to the node that owns the cursor). Batched, mapCmdsByNode would + // route them by slot and reach the wrong shard; diverted, they go through + // Client/ClusterClient.Process and keep their special routing. + mustDivert func(ctx context.Context, cmd Cmder) bool + + // sharedClosed, when non-nil, is the owning client's pool-set closed flag + // (shared across WithTimeout clones). The getters refuse to build a fresh + // pipeliner once it is set; this reference makes an ALREADY-built + // pipeliner refuse new work too — without it, a clone's Close would leave + // a cached pipeliner accepting enqueues against closed pools, failing + // them one dispatch at a time instead of with ErrClosed at submit. + sharedClosed *atomic.Bool + + // expectedArrivals counts how many commands the engine expects to arrive + // at any moment: a completed batch of N≥2 commands wakes its N waiters + // together, and in a closed loop each immediately submits its next command + // — so completion announces N expected arrivals, and every enqueue accounts + // for one. The default coalescing wait (awaitExpectedArrivals) holds the + // flusher while arrivals are still expected, so the whole wakeup wave + // flushes as one deep pipeline — an exact count, not a smoothed estimate, + // which cannot ratchet into fragmentation. Single-command batches announce + // nothing, so a lone caller and open-loop traffic never wait. May + // transiently go negative (arrivals nobody announced); readers clamp to + // zero. Pipeliner-global, not per-shard: cluster routing may land a + // follow-up on a different shard than the batch that woke its caller. + expectedArrivals atomic.Int64 + + // execEWMA is an exponentially-weighted moving average (alpha 1/8) of + // batch execution time in nanoseconds — the engine's own view of the + // server round-trip. It scales awaitExpectedArrivals's silence fallback so a + // wave staggered by scheduling on a slow link is not split mid-landing. Updates + // are racy read-modify-writes by design: losing an occasional sample is + // harmless for a smoothing heuristic. 0 means "no sample yet". + execEWMA atomic.Int64 + + // Lifecycle + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup // Tracks flusher goroutines + batchWg sync.WaitGroup // Tracks batch execution goroutines + // divertWg tracks the goroutines that execute DIVERTED commands (blocking + // and connection-hostile ones, which never enter a batch). Close waits on + // it exactly like batchWg so a diverted command's pooled connection is not + // left in flight after Close returns — bounded, see Close. + // + // divertMu serializes "observe not-closed, then register" against Close's + // "mark closed, then wait": without it a diverted command could pass the + // closed check, Close could see a zero counter and return, and only then + // would the goroutine register — leaving an accepted command holding a + // pooled connection past Close (and racing WaitGroup Add against Wait). + divertMu sync.Mutex + divertWg sync.WaitGroup + closed atomic.Bool +} + +// apShard is one queue + flusher. Its fields are touched only by enqueuing +// goroutines (under mu) and by its own single flusher goroutine. +// apEnqueueStripes is how many enqueue stripes a shard runs when striping is +// safe (unordered configs, and every blocking-face shard — a blocking caller +// waits for each command, so stripes cannot reorder its stream). The +// enqueue mutex is the hottest lock in the engine (128 concurrent callers on +// one shard spend ~half their CPU in lock slow paths); striping the queue +// spreads that contention while the flusher still drains every stripe into ONE +// merged pipeline, so batches stay deep. Ordered shards always use a single +// stripe: with several stripes a caller's consecutive commands can land in +// stripes on opposite sides of an in-progress drain and execute out of order. +const apEnqueueStripes = 8 + +// apStripe is one striped slice of a shard's enqueue queue. Each stripe has +// its own batch-completion signal so a drain can take stripes one lock at a +// time; every batch taken in one drain completes together after the merged +// pipeline executes. Padded so neighbouring stripes' mutexes do not share a +// cache line. +type apStripe struct { + mu sync.Mutex + queue []Cmder + queueLen atomic.Int32 + // queueBytes approximates the queued commands' payload volume; maintained + // only when MaxBatchBytes is configured (see cmdApproxBytes). + queueBytes atomic.Int64 + curBatch *apBatch // completion signal for currently-queued cmds + // Pad each stripe onto its own cache line(s). Without it, one stripe's hot + // fields (queueLen/curBatch) share a cache line with the NEXT stripe's + // contended mutex, so a lock-free counter bump on stripe i invalidates the + // line a different core is trying to lock stripe i+1 on — false sharing + // that measured ~16x on a contended microbenchmark. cpu.CacheLinePad is + // sized per GOARCH (64 B on x86-64/arm64, 128 B on ppc64, 256 B on s390x), + // so this is correct on every target rather than a hand-tuned constant. + _ cpu.CacheLinePad +} + +type apShard struct { + ap *AutoPipeliner + + next atomic.Uint32 // round-robin stripe pick (unordered mode) + stripes []apStripe // 1 stripe when ordered, apEnqueueStripes when Unordered + notify chan struct{} // buffered (cap 1) enqueue wake-up + sem *internal.FIFOSemaphore // per-shard concurrent-batch budget + + // inFlight counts this shard's dispatched-but-unfinished batches. When it + // is zero and no arrivals are expected, the shard is idle and a + // new command flushes immediately; when batches are in flight, arrivals + // are mid-stream and the flusher holds them briefly to coalesce (see + // awaitExpectedArrivals). + inFlight atomic.Int32 +} + +// stripe picks the enqueue stripe for the next command: the single stripe in +// ordered mode (preserving strict FIFO), round-robin in unordered mode. +func (s *apShard) stripe() *apStripe { + if len(s.stripes) == 1 { + return &s.stripes[0] + } + return &s.stripes[s.next.Add(1)%uint32(len(s.stripes))] +} + +// getOrCreateAutoPipeliner is the shared caching protocol behind the four +// AutoPipeline/AsyncAutoPipeline getters (Client and ClusterClient, each +// face): return the cached live instance, refuse on a closed client, or build +// and cache a new one. The caller supplies its cached-slot pointer, its +// closed flag (both guarded by the mutex), the explicit-config override, the +// fallback config, and a build closure (the cluster one wraps +// clusterAutoPipelineOptions and installs slot sharding). +func getOrCreateAutoPipeliner( + mu *sync.Mutex, + slot **AutoPipeliner, + closed *bool, + sharedClosed *atomic.Bool, + override *AutoPipelineOptions, + fallback func() *AutoPipelineOptions, + build func(*AutoPipelineOptions) (*AutoPipeliner, error), +) (*AutoPipeliner, error) { + mu.Lock() + defer mu.Unlock() + // closed covers THIS wrapper's Close; sharedClosed covers the shared + // pools closing through ANY sharer (e.g. a WithTimeout clone falling + // through to baseClient.Close) — a fresh pipeliner against closed pools + // would leak flushers that error forever. + if *closed || (sharedClosed != nil && sharedClosed.Load()) { + return nil, ErrClosed + } + if *slot != nil && !(*slot).closed.Load() { + return *slot, nil + } + cfg := override + if cfg == nil { + cfg = fallback() + } + ap, err := build(cfg) + if err != nil { + return nil, err + } + // Thread the shared pool-set closed flag into the pipeliner so an + // ALREADY-cached instance also refuses enqueues once any sharer closes + // the pools (the check above only protects fresh builds). + ap.sharedClosed = sharedClosed + *slot = ap + return ap, nil +} + +// newAutoPipeliner builds an autopipeliner in either blocking or deferred mode. +// It is unexported on purpose: the public entry points are +// Client/ClusterClient.AutoPipeline and AsyncAutoPipeline, which also install +// cluster slot-sharding. Constructing one directly would skip that wiring and +// give a *ClusterClient degraded (cross-node) batching. +func newAutoPipeliner(pipeliner cmdableClient, config *AutoPipelineOptions, blocking bool) (*AutoPipeliner, error) { + if config == nil { + config = DefaultAutoPipelineOptions() + } else { + // Copy so default-filling below doesn't mutate the caller's struct — the + // same *AutoPipelineOptions may be shared across clients (e.g. a reused + // Options.AutoPipelineOptions), and callers may inspect it afterward. + cfgCopy := *config + config = &cfgCopy + } + + // Validate BEFORE default-filling: Validate treats zero as "use the + // default" but rejects negatives, and coercing first would silently + // swallow a negative typo the documented contract promises to error on. + if err := config.Validate(); err != nil { + return nil, err + } + + // Apply defaults for zero values + if config.MaxBatchSize <= 0 { + config.MaxBatchSize = 200 + } + + if config.MaxConcurrentBatches <= 0 { + // Default to an ordered single stream. Callers raise this (with + // Unordered:true) to opt into parallel-batch throughput. + config.MaxConcurrentBatches = 1 + } + + // NumShards > 1 on the deferred (async) face distributes commands + // round-robin across shards that flush concurrently, so submit order is + // not preserved — require the explicit Unordered opt-in, exactly like + // MaxConcurrentBatches > 1. The blocking face is exempt (each caller waits + // per command, and Submit is rejected there), as is cluster slot sharding + // (contentSharded: same-key commands always land in the same shard, so + // per-key order holds). + if config.NumShards > 1 && !config.Unordered && !blocking && !config.contentSharded { + return nil, fmt.Errorf( + "redis: AutoPipelineOptions.NumShards=%d requires Unordered:true on the deferred (async) face "+ + "(commands are distributed round-robin across shards, which flush concurrently and do not preserve submit order)", + config.NumShards) + } + + ctx, cancel := context.WithCancel(context.Background()) + + ap := &AutoPipeliner{ + pipeliner: pipeliner, + config: config, + blocking: blocking, + ctx: ctx, + cancel: cancel, + } + + // Route the typed command surface. Blocking: the command call blocks until + // executed (synchronous drop-in shape). Deferred: the call returns at once + // and the result accessors block until the batch executes. + if blocking { + ap.cmdable = ap.processBlocking + } else { + ap.cmdable = ap.processAsync + } + + // Pick the shard count. NumShards=0 (auto) means ONE shard: a single deep + // queue outperforms a sharded one because batches stay large — sharding by + // core count coupled batch fragmentation to MaxConcurrentBatches and + // collapsed pipelining (measured: 16 shards cut async throughput ~4x and + // tripled latency versus one shard at the same permit count). Cluster + // wiring passes an explicit NumShards so slot-routed shards keep each + // batch on one node. + nShards := config.NumShards + if nShards <= 0 { + nShards = 1 + } + // Split the concurrent-batch budget across shards so each shard has its own + // semaphore. A single shared semaphore became a contention point once the + // per-shard queue mutexes were no longer the bottleneck. Integer division + // drops a remainder, so hand the leftover permits to the first shards: the + // per-shard permits then sum to exactly MaxConcurrentBatches. + perShard := config.MaxConcurrentBatches / nShards + remainder := config.MaxConcurrentBatches % nShards + if perShard < 1 { + // Budget smaller than the shard count: give every shard one permit so + // each flusher can still make progress. The sum then exceeds the + // configured budget, which is unavoidable with per-shard semaphores. + perShard = 1 + remainder = 0 + } + ap.shards = make([]*apShard, nShards) + for i := range ap.shards { + permits := perShard + if i < remainder { + permits++ + } + // Stripe when reordering is impossible or waived: a BLOCKING caller + // waits for each command before issuing its next, so its per-goroutine + // order holds no matter which stripe each command lands in; the async + // face may only stripe when the user set Unordered. The remaining case + // (async, ordered) keeps one stripe to preserve strict submit order. + nStripes := 1 + if config.Unordered || blocking { + nStripes = apEnqueueStripes + } + s := &apShard{ + ap: ap, + notify: make(chan struct{}, 1), + stripes: make([]apStripe, nStripes), + sem: internal.NewFIFOSemaphore(int32(permits)), + } + for j := range s.stripes { + s.stripes[j].queue = getQueueSlice(config.MaxBatchSize) + s.stripes[j].curBatch = newAPBatch() + } + ap.shards[i] = s + ap.wg.Add(1) + go s.flusher() + } + + return ap, nil +} + +// Do executes a raw command on a NORMAL connection, outside the pipeline. +// Arbitrary command names can carry connection state (SELECT, MULTI, SUBSCRIBE, +// CLIENT ...) or block the connection (BLPOP ...); batching those onto a shared +// pipeline connection would silently poison it for every later batch, or stall +// unrelated commands. (Submit enforces the same rule for raw Cmders: names in +// the connection-hostile set are diverted off the pipeline automatically.) +// The typed surface (ap.Set, ap.Get, ...) is safe by +// construction and IS batched — prefer it. Do carries the same caveats as +// Client.Do: a stateful command still affects the (normal, non-pipeline) +// pooled connection it runs on. Do keeps each face's call shape: on +// a blocking autopipeliner the call blocks until the command has executed; on a +// deferred (async) one it returns immediately and the command's result +// accessors (Err/Val/Result) block until it completes. +func (ap *AutoPipeliner) Do(ctx context.Context, args ...interface{}) *Cmd { + cmd := NewCmd(ctx, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + + // Both faces go through runOutsidePipeline: it applies the divert + // registration gate, so Close cannot conclude "nothing in flight" while an + // accepted raw command — a blocking one on the blocking face runs inline on + // the caller's goroutine — is still holding a pooled connection. + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// runOutsidePipeline executes an escape-hatch command (Do, DoRaw, +// DoRawWriteTo) on a normal pooled connection, outside the batching engine, +// following the face's call shape. Blocking face: synchronous Process. +// Deferred face: returns-immediately — the command runs on a background +// goroutine and a ready batch makes its result accessors block until it +// completes. The batch completes at the innermost seam (under the user +// hooks) so a ProcessHook reading the result cannot self-deadlock; the +// deferred close is the panic backstop. Tracked by divertWg under divertMu, +// so Close waits for accepted diverted work (bounded — see Close) instead of +// returning while it still holds a pooled connection. +func (ap *AutoPipeliner) runOutsidePipeline(ctx context.Context, cmd Cmder) *apBatch { + if ap.blocking { + // The blocking face runs it inline, so the caller's own goroutine holds + // the connection; still take the gate so Close cannot decide "nothing + // in flight" while this command is executing. + ap.divertMu.Lock() + if ap.isClosed() { + ap.divertMu.Unlock() + cmd.SetErr(ErrClosed) + return completedBatch + } + ap.divertWg.Add(1) + ap.divertMu.Unlock() + defer ap.divertWg.Done() + _ = ap.pipeliner.Process(ctx, cmd) + return completedBatch + } + // Register under divertMu with a closed re-check, so registration and the + // close transition cannot interleave (see the divertMu comment). A command + // that loses the race is rejected here rather than running after Close. + // The gate comes BEFORE setReady: publishing the fresh batch first and then + // rejecting would leave the command gated on a batch nobody ever closes, + // hanging every accessor. + ap.divertMu.Lock() + if ap.isClosed() { + ap.divertMu.Unlock() + cmd.SetErr(ErrClosed) + cmd.setReady(completedBatch) + return completedBatch + } + b := newAPBatch() + cmd.setReady(b) + ap.divertWg.Add(1) + ap.divertMu.Unlock() + go func() { + defer ap.divertWg.Done() + defer b.close() + defer recoverDispatchPanic([]Cmder{cmd}) + if ap.armSelfDeadlockGuard() { + b.dispGid.Store(curGoroutineID()) + } + // A hook that returns nil WITHOUT calling next has short-circuited + // SUCCESSFULLY (it served the command itself); plain Client hooks may do + // that, so nothing here synthesizes an error for it — see dispatchCmds. + err := ap.pipeliner.withProcessHook(ctx, cmd, func(ctx context.Context, cmd Cmder) error { + return ap.pipeliner.process(ctx, cmd) + }) + // The chain's final verdict, exactly like Client.Process — recorded + // before the deferred close wakes the reader, so short-circuits, + // post-next rewrites and suppressions are all honored. + cmd.SetErr(err) + }() + return b +} + +// DoRaw mirrors Do for raw RESP access: AutoPipeliner embeds cmdable, so +// without this override DoRaw would ride the batching engine — but raw +// commands carry Do's caveats and DoRawWriteTo-style streaming must not run +// inside a shared batch's reply loop. Runs outside the pipeline, following +// the face's call shape (see Do). +func (ap *AutoPipeliner) DoRaw(ctx context.Context, args ...interface{}) *RawCmd { + cmd := NewRawCmd(ctx, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// DoRawWriteTo mirrors Do for streamed raw RESP access (see DoRaw). On the +// deferred face the write to w happens when the command executes; use the +// result accessors (Err/Written) to wait before reading w. +func (ap *AutoPipeliner) DoRawWriteTo(ctx context.Context, w io.Writer, args ...interface{}) *RawWriteToCmd { + cmd := NewRawWriteToCmd(ctx, w, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// Process queues a command for autopipelined execution, following the +// autopipeliner's mode like the typed methods and Do: on a blocking +// autopipeliner the call blocks until the command has executed; on a deferred +// (async) one it returns immediately and reading the command's result +// (Val/Result/Err) blocks until its batch is flushed. +func (ap *AutoPipeliner) Process(ctx context.Context, cmd Cmder) error { + return ap.cmdable(ctx, cmd) +} + +// The methods below complete the UniversalClient surface by delegating to the +// underlying client. They are NOT autopipelined — pub/sub, transactions (Watch), +// hooks, Do and pool stats cannot be batched — so an AutoPipeliner used as a +// UniversalClient batches only the typed data commands; everything here runs on +// the underlying client exactly as it would there. +// +// Note on lifecycle: Close() (defined elsewhere) closes the AUTOPIPELINER — +// drains in-flight batches and stops flushers — but does NOT close the +// underlying client, whose lifecycle is owned by whoever created it. + +// AddHook adds a hook to the underlying client. Autopipelined batches are hooked +// too, since dispatch goes through the hook-wrapped pipeline entry. +func (ap *AutoPipeliner) AddHook(hook Hook) { ap.pipeliner.AddHook(hook) } + +// The four commands below have CLUSTER-WIDE overrides on ClusterClient +// (DBSize sums every master, the Script commands fan out to every shard). +// The embedded generic cmdable would route them as ordinary keyless commands +// to one picked shard — partial results, scripts missing on other shards — +// so they delegate to the underlying client instead of batching. On a +// standalone client the delegation is semantically identical to the generic +// path; these are rare admin/script-management commands, not data-path. + +// DBSize delegates to the underlying client (cluster-wide sum on ClusterClient). +func (ap *AutoPipeliner) DBSize(ctx context.Context) *IntCmd { + return ap.pipeliner.DBSize(ctx) +} + +// ScriptLoad delegates to the underlying client (loads every shard on ClusterClient). +func (ap *AutoPipeliner) ScriptLoad(ctx context.Context, script string) *StringCmd { + return ap.pipeliner.ScriptLoad(ctx, script) +} + +// ScriptFlush delegates to the underlying client (flushes every shard on ClusterClient). +func (ap *AutoPipeliner) ScriptFlush(ctx context.Context) *StatusCmd { + return ap.pipeliner.ScriptFlush(ctx) +} + +// ScriptExists delegates to the underlying client (ANDs results across shards +// on ClusterClient). +func (ap *AutoPipeliner) ScriptExists(ctx context.Context, hashes ...string) *BoolSliceCmd { + return ap.pipeliner.ScriptExists(ctx, hashes...) +} + +// HImportPrepare, HImportDiscard and HImportDiscardAll are the remaining +// cluster-wide overrides (see the delegation note above): ClusterClient fans +// them out to every master and updates the shared fieldset registry, so +// running them on a single routed node would let a later HImportSet for a key +// on another master fail with "no such fieldset". TestAPDelegatesClusterWideOverrides +// fails if a future ClusterClient override is added without a delegate here. +func (ap *AutoPipeliner) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + return ap.pipeliner.HImportPrepare(ctx, fieldsetName, fields...) +} + +func (ap *AutoPipeliner) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + return ap.pipeliner.HImportDiscard(ctx, fieldsetName) +} + +func (ap *AutoPipeliner) HImportDiscardAll(ctx context.Context) *IntCmd { + return ap.pipeliner.HImportDiscardAll(ctx) +} + +// Watch runs a transactional function on the underlying client (not batched). +func (ap *AutoPipeliner) Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error { + return ap.pipeliner.Watch(ctx, fn, keys...) +} + +// Subscribe opens a pub/sub on the underlying client (not batched — pub/sub +// needs a dedicated connection). +func (ap *AutoPipeliner) Subscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.Subscribe(ctx, channels...) +} + +// PSubscribe opens a pattern pub/sub on the underlying client (not batched). +func (ap *AutoPipeliner) PSubscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.PSubscribe(ctx, channels...) +} + +// SSubscribe opens a sharded pub/sub on the underlying client (not batched). +func (ap *AutoPipeliner) SSubscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.SSubscribe(ctx, channels...) +} + +// PoolStats returns the underlying client's connection pool statistics. +func (ap *AutoPipeliner) PoolStats() *PoolStats { return ap.pipeliner.PoolStats() } + +// AutoPipeline delegates to the underlying client, which returns its cached +// autopipeliner (typically this same instance). Present to satisfy the +// UniversalClient surface. +func (ap *AutoPipeliner) AutoPipeline() (*AutoPipeliner, error) { + return ap.pipeliner.AutoPipeline() +} + +// AutoPipelineWithOptions delegates to the underlying client. +func (ap *AutoPipeliner) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return ap.pipeliner.AutoPipelineWithOptions(config) +} + +// AsyncAutoPipeline delegates to the underlying client. Present to satisfy the +// UniversalClient surface. +func (ap *AutoPipeliner) AsyncAutoPipeline() (*AutoPipeliner, error) { + return ap.pipeliner.AsyncAutoPipeline() +} + +// AsyncAutoPipelineWithOptions delegates to the underlying client. +func (ap *AutoPipeliner) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return ap.pipeliner.AsyncAutoPipelineWithOptions(config) +} + +// AutoFuture is the handle returned by Submit. Call Wait (or Result on the +// command after Wait) once the result is needed; it blocks only until the +// command's batch has executed. +type AutoFuture struct { + cmd Cmder + batch *apBatch +} + +// Wait blocks until the submitted command has executed, then returns its error. +// The zero AutoFuture (no submitted command) returns an error rather than +// panicking. +func (f AutoFuture) Wait() error { + if f.batch == nil { + if f.cmd != nil { + return f.cmd.Err() + } + return errZeroAutoFuture + } + select { + case <-f.batch.done: + default: + // Same self-deadlock guard as baseCmd.await(): a pipeline hook on + // the batch's own dispatch goroutine waiting a future pre-next() + // would block a channel only its goroutine can close. Give it the + // not-yet-executed view instead. + if f.batch.isExecutorGoroutine() { + return f.cmd.rawErr() + } + <-f.batch.done + } + return f.cmd.Err() +} + +// WaitContext is like Wait but stops waiting when ctx is done. The command +// still executes and its result remains readable once its batch completes — +// ctx abandons only this wait, it does not cancel the command (per-command +// contexts are not honored after enqueue; see the AutoPipeliner doc). +// +// After a ctx error the result may simply not be there YET: the batch is +// still in flight and may populate the command at any moment, so do not read +// Cmd()'s value or error directly — that races the executing batch. Call Wait +// (or WaitContext with a fresh context) again; once it returns a non-context +// error, the command's result is complete and safe to read. +func (f AutoFuture) WaitContext(ctx context.Context) error { + if f.batch == nil { + if f.cmd != nil { + return f.cmd.Err() + } + return errZeroAutoFuture + } + select { + case <-f.batch.done: + return f.cmd.Err() + default: + if f.batch.isExecutorGoroutine() { + return f.cmd.rawErr() // see Wait: dispatch-goroutine self-deadlock guard + } + } + select { + case <-f.batch.done: + return f.cmd.Err() + case <-ctx.Done(): + return ctx.Err() + } +} + +// Cmd returns the underlying command (call Wait first before reading results). +func (f AutoFuture) Cmd() Cmder { return f.cmd } + +// outsidePipelineCommands lists commands that must never ride a SHARED +// pipeline connection. SHUTDOWN terminates the server before replying (its +// batchmates would all fail with EOF and the batch would retry against a +// dead server); MONITOR rebinds the connection into a monitor stream, +// desyncing every reply behind it; the rest change per-connection state +// (database, auth, protocol, transaction, subscription mode) that would +// leak to every unrelated caller sharing the pipeline conn afterwards. The +// typed surface cannot produce most of the stateful ones (they live on +// statefulCmdable) — but ReadOnly/ReadWrite ARE on cmdable, and raw +// Submit/Do accept any Cmder. Diverted commands execute directly on their +// own pooled connection — the same semantics (including the same footguns) +// as plain Client.Do. +var outsidePipelineCommands = map[string]struct{}{ + "shutdown": {}, "monitor": {}, + "select": {}, "auth": {}, "hello": {}, "reset": {}, "quit": {}, + "multi": {}, "exec": {}, "discard": {}, "watch": {}, "unwatch": {}, + "subscribe": {}, "unsubscribe": {}, "psubscribe": {}, "punsubscribe": {}, + "ssubscribe": {}, "sunsubscribe": {}, + "client": {}, + // Connection-scoped cluster state: queued onto a shared pipeline conn + // they would leak replica-reads (or a pending redirect) to every later + // batch on that conn. + "readonly": {}, "readwrite": {}, "asking": {}, +} + +func runsOutsidePipeline(name string) bool { + _, ok := outsidePipelineCommands[name] + return ok +} + +// blockingCommands are commands that park on the server until data arrives or +// their own timeout expires. The TYPED helpers set a per-command read timeout +// (see cmdable.BLPop), which submit already diverts on; a RAW Cmder built by +// hand — NewCmd(ctx, "blpop", key, 0) via Submit/Process/Do — carries no such +// marker, so without this set it would be queued onto a shared pipeline +// connection and hold the whole batch for the block duration. +// Derived from the typed helpers rather than guessed: every cmdable method that +// calls cmd.setReadTimeout parks the connection, so +// +// grep -rn 'setReadTimeout' --include='*.go' . | grep -v _test +// +// enumerates exactly the wire names that belong here (the arg-driven ones are +// handled in isBlockingCmd instead). Re-run that grep when adding a blocking +// command. +var blockingCommands = map[string]struct{}{ + "blpop": {}, "brpop": {}, "brpoplpush": {}, + "blmove": {}, "blmovem": {}, "blmpop": {}, + "bzpopmin": {}, "bzpopmax": {}, "bzmpop": {}, + "wait": {}, "waitaof": {}, + // MIGRATE blocks the source instance for up to its timeout. + "migrate": {}, +} + +// isBlockingCmd reports whether cmd parks the connection. XREAD/XREADGROUP are +// decided by ARGUMENTS, not by name: only the BLOCK form blocks, and +// blanket-diverting the (far more common) non-blocking form would drop it out +// of batching for nothing. +func isBlockingCmd(cmd Cmder) bool { + name := cmd.Name() + if _, ok := blockingCommands[name]; ok { + return true + } + // Arg-driven: these block only in their BLOCK form, and blanket-diverting + // the far more common non-blocking form would drop it out of batching for + // nothing. TS.READ takes BLOCK the same way (see TSReadWithArgs). + if name != "xread" && name != "xreadgroup" && name != "ts.read" { + return false + } + // Match the token the way the encoder does: a raw Cmder may carry RESP + // tokens as []byte or *string (see baseCmd.stringArg), and a type switch on + // string alone would let NewCmd(ctx, "xread", []byte("BLOCK"), 0, ...) be + // batched onto a shared connection. + for _, arg := range cmd.Args() { + if internal.ToLower(blockingArgString(arg)) == "block" { + return true + } + } + return false +} + +// blockingArgString renders a command argument as the string the encoder will +// write for the token comparisons above. Only the forms that can carry a RESP +// keyword are handled; anything else cannot be the BLOCK token. +func blockingArgString(arg interface{}) string { + switch v := arg.(type) { + case string: + return v + case []byte: + return string(v) + case *string: + if v == nil { + return "" + } + return *v + default: + return "" + } +} + +// submit queues a command without blocking and returns its completion future. +func (ap *AutoPipeliner) submit(ctx context.Context, cmd Cmder) AutoFuture { + // finish marks the command ready on the deferred face so its result + // accessors (Val/Result/Err) self-gate through await() — whether the + // caller goes through the typed surface or raw Submit. Reading a + // Submit()-ed command before Wait() was previously a silent data race + // with the dispatch goroutine. The blocking face deliberately never + // carries a batch: its callers only regain control after execution, and + // the dispatcher-gid deadlock guard relies on that. + finish := func(f AutoFuture) AutoFuture { + if !ap.blocking { + cmd.setReady(f.batch) + } + return f + } + // Decide DIVERSION first. The cluster preflight rejects commands whose + // request policy cannot ride a pipeline (ReqAllNodes/ReqAllShards), but a + // diverted command never rides one: it goes through the underlying + // Client/ClusterClient.Process, which performs the normal cluster-wide + // fan-out and aggregation. Running the preflight first therefore rejected + // commands that would have worked — typed WAIT/WAITAOF on a cluster with + // command policies enabled (review finding by codex on #3942). + diverted := cmd.readTimeout() != nil || runsOutsidePipeline(cmd.Name()) || isBlockingCmd(cmd) || + (ap.mustDivert != nil && ap.mustDivert(ctx, cmd)) + if !diverted && ap.preflight != nil { + if err := ap.preflight(ctx, cmd); err != nil { + cmd.SetErr(err) + return finish(AutoFuture{cmd: cmd, batch: completedBatch}) + } + } + if diverted { + // Blocking commands (and the conn-hostile ones above) are executed + // directly, outside the pipeline — via runOutsidePipeline, which + // keeps each face's call shape: the blocking face runs the command + // synchronously, the deferred face runs it on its own goroutine so + // this call returns immediately and the result accessors block (a + // BLPOP submitted on the async face must not stall the submitter, + // exactly like Do). They still must respect a closed AutoPipeliner: + // enqueue() rejects on the batched path, so mirror that here instead + // of running after Close(). + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return finish(AutoFuture{cmd: cmd, batch: completedBatch}) + } + // runOutsidePipeline sets the command ready itself on the deferred + // face; the returned batch completes when the command has executed. + return AutoFuture{cmd: cmd, batch: ap.runOutsidePipeline(ctx, cmd)} + } + // No finish here: enqueue stamps ready under the stripe lock, before the + // command is visible to any drain (the error paths above still go through + // finish for uniform accessor behavior). + return AutoFuture{cmd: cmd, batch: ap.enqueue(cmd)} +} + +// ErrSubmitBlockingFace rejects Submit on the blocking face: Submit does not +// wait, so a windowed caller could have several commands in flight at once — +// but the blocking face stripes its enqueue queue on the strength of every +// caller waiting per command, and a non-waiting window there can be reordered. +// The deferred face (AsyncAutoPipeline) is built for exactly that usage. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +var ErrSubmitBlockingFace = errors.New( + "redis: Submit requires the deferred autopipeliner (AsyncAutoPipeline); on the blocking face use the typed methods or Do") + +// errZeroAutoFuture is returned by Wait/WaitContext on a zero AutoFuture. +var errZeroAutoFuture = errors.New("redis: Wait on a zero AutoFuture") + +// errDoNoArgs is returned by Do when called without a command. +var errDoNoArgs = errors.New("redis: AutoPipeliner.Do requires at least one argument") + +// ErrAutoPipelineTimeout is set on drained commands when a flush could not +// obtain a batch permit within the engine's internal backstop — the engine is +// overloaded or an in-flight batch is wedged (e.g. read timeouts disabled on +// a dead peer). It is deliberately NOT context.DeadlineExceeded: the caller's +// own context did not expire, and errors.Is(err, context.DeadlineExceeded) +// must not fire for an internal engine timeout. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +var ErrAutoPipelineTimeout = errors.New( + "redis: autopipeline: no batch permit within the internal backstop (engine overloaded or a batch is wedged)") + +// Submit queues a command without blocking and returns an AutoFuture; Wait on +// it when the result is needed. This is the explicit form for working with raw +// Cmders on the deferred (async) face, where the typed methods (Set, Get, ...) +// provide the same deferred behaviour returning the usual *XxxCmd. The +// command's own result accessors (Err/Val/Result) are safe to use instead of +// Wait — they block until the command has executed. Connection-hostile +// command names (SHUTDOWN, MONITOR, SELECT, AUTH, MULTI, SUBSCRIBE, CLIENT, +// ...) never ride a shared pipeline connection: they are diverted to a +// normal pooled connection with plain Client.Do semantics. On a BLOCKING +// autopipeliner Submit is rejected (the future's Wait returns an error): the +// blocking face's ordering relies on every caller waiting for each command +// before issuing the next, which Submit by design does not do. +func (ap *AutoPipeliner) Submit(ctx context.Context, cmd Cmder) AutoFuture { + if ap.blocking { + cmd.SetErr(ErrSubmitBlockingFace) + return AutoFuture{cmd: cmd, batch: completedBatch} + } + return ap.submit(ctx, cmd) +} + +// processAsync is the cmdable backing the typed command surface: it queues a +// command without blocking the caller and marks it ready so the command's +// result accessors (Val/Result/Err) block until the batch executes. This gives +// the autopipeliner the full typed surface (ap.Set, ap.Get, ...) with the exact +// same call shape as a normal client — only the wait is deferred to the point a +// result is read. +func (ap *AutoPipeliner) processAsync(ctx context.Context, cmd Cmder) error { + // submit marks the command ready (see the finish closure there): a hook + // that reads the command before that store lands sees a nil ready — the + // non-blocking not-yet-executed view — while the caller always sees its + // own store before any await. + f := ap.submit(ctx, cmd) + // Report SUBMIT-time rejections (a closed pipeliner, a cluster preflight + // refusal): those paths set the error on the command and hand back the + // shared completed batch without queueing anything, so returning nil made + // Process claim success for a command that will never run — and callers + // reaching the engine through UniversalClient.Process see only this return + // value (review finding by codex on #3942). Execution errors are NOT + // reported here: the deferred face's contract is that this call does not + // wait, so those stay on the command for its accessors. rawErr keeps the + // check non-blocking. + if f.batch == completedBatch { + return cmd.rawErr() + } + return nil +} + +// processBlocking is the cmdable backing the blocking face: it queues the +// command and blocks until its batch has executed, so the command call has the +// same synchronous shape as a normal client (the returned *XxxCmd already holds +// its result). The flusher still batches this command with other concurrent +// callers' commands into a pipeline, so throughput is far above a plain client +// even though each caller waits. Per-goroutine ordering holds regardless of +// MaxConcurrentBatches: a caller cannot issue its next command until this one +// returns, so its commands execute in submit order. +func (ap *AutoPipeliner) processBlocking(ctx context.Context, cmd Cmder) error { + return ap.submit(ctx, cmd).Wait() +} + +// completedBatch is a reusable already-completed batch: returned both for +// commands that already executed directly (blocking commands, Submit-time +// rejections) and for error cases like enqueue-after-Close, so Wait returns +// immediately and the command's own error tells the story. +var completedBatch = func() *apBatch { + b := newAPBatch() + b.close() + return b +}() + +// enqueue queues a command and returns the batch whose done channel completes +// when it has executed. On a closed autopipeliner it errors the command and +// returns the already-closed batch. +// isClosed reports whether this pipeliner (or the shared pool set it rides +// on) has been closed. Two atomic loads; no locks. +// +// EVERY closed check that gates accepting new work must go through this, not +// ap.closed directly: a WithTimeout clone's Close sets only the shared flag, +// so a guard reading ap.closed alone would accept commands against pools that +// are already gone and surface pool-closed errors instead of ErrClosed. +// (Close's own CompareAndSwap on ap.closed is the one deliberate direct use: +// it claims the shutdown for this instance.) +func (ap *AutoPipeliner) isClosed() bool { + return ap.closed.Load() || (ap.sharedClosed != nil && ap.sharedClosed.Load()) +} + +func (ap *AutoPipeliner) enqueue(cmd Cmder) *apBatch { + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return completedBatch + } + + // Pick a shard. With shardFn (cluster mode) route by command content so all + // commands for one node collect in the same shard's batch; otherwise spread + // round-robin to keep each shard's mutex lightly contended. + var s *apShard + if ap.shardFn != nil { + // uint conversion instead of negation: -math.MinInt overflows back to + // itself and a negative modulo would panic the index. The unsigned + // modulo is deterministic for every int, including MinInt. + idx := ap.shardFn(cmd) + s = ap.shards[uint(idx)%uint(len(ap.shards))] + } else if len(ap.shards) == 1 { + // Single shard (the standalone default): skip the round-robin counter — + // it is a shared cache line bumped by every enqueue for a pick that is + // constant. Same guard the stripe pick already has. + s = ap.shards[0] + } else { + // Unsigned modulo: converting to int first goes negative after the + // uint32 counter passes 2^31 on 32-bit platforms and panics. + s = ap.shards[int((ap.next.Add(1)-1)%uint32(len(ap.shards)))] + } + + st := s.stripe() + st.mu.Lock() + // Re-check closed under the stripe lock (see Close): either we win the lock + // first and the shutdown drain flushes us, or the drain ran first and we + // reject here — so a late enqueue never hangs on an unclosed done. + if ap.isClosed() { + st.mu.Unlock() + cmd.SetErr(ErrClosed) + return completedBatch + } + batch := st.curBatch + if !ap.blocking { + // Publish the gating batch BEFORE the command becomes visible to a + // drain (the drain takes this same stripe lock): a flush racing the + // submitter's return path must observe ready already set, or the + // cluster node-executor registration would skip this command's batch + // and a node hook reading the command mid-dispatch could block on a + // batch its own call chain completes. The blocking face deliberately + // never carries a batch (see submit). + cmd.setReady(batch) + } + st.queue = append(st.queue, cmd) + st.queueLen.Store(int32(len(st.queue))) + if ap.config.MaxBatchBytes > 0 { + st.queueBytes.Add(cmdApproxBytes(cmd)) + } + st.mu.Unlock() + + // One expected arrival has landed (see expectedArrivals). + ap.expectedArrivals.Add(-1) + + s.wake() + return batch +} + +// wake signals the shard's flusher that work is available without blocking. +func (s *apShard) wake() { + select { + case s.notify <- struct{}{}: + default: + } +} + +// IsBlocking reports which face this autopipeliner is: true for the blocking +// face (Client.AutoPipeline — calls wait for execution), false for the +// deferred face (AsyncAutoPipeline — calls return immediately and result +// accessors block). The two faces reject different usage (Submit is +// blocking-face-rejected), so code handed an *AutoPipeliner can branch on +// this instead of probing with errors. +func (ap *AutoPipeliner) IsBlocking() bool { return ap.blocking } + +// Config returns a copy of the effective configuration (defaults filled in). +func (ap *AutoPipeliner) Config() AutoPipelineOptions { + cfg := *ap.config + // Strip internal-only fields. contentSharded is set by cluster wiring and + // tells Validate that shards are slot-routed, so same-key commands cannot + // be reordered — which exempts the config from the NumShards>1 ordering + // requirement. Handing that bit back to a caller who copies this config + // into a STANDALONE async autopipeliner would silence that check for + // round-robin shards, which really do flush concurrently and really do + // break submit order (review finding by codex on #3942). + cfg.contentSharded = false + return cfg +} + +// IsClosed reports whether the AutoPipeliner has been closed, either by an +// explicit Close or by closing the owning client. A closed AutoPipeliner +// rejects new commands with ErrClosed. +func (ap *AutoPipeliner) IsClosed() bool { + return ap.isClosed() +} + +// numShards reports how many shards this autopipeliner runs. +func (ap *AutoPipeliner) numShards() int { return len(ap.shards) } + +// setShardFn installs a content-based shard selector. In cluster mode it maps +// a command's SLOT to a shard, which is a batch-depth heuristic, not an +// invariant: slot ranges are assigned to shards proportionally, so when a +// node's slots are non-contiguous one shard's batch can still span nodes and +// mapCmdsByNode splits it (correctness is unaffected — that router resolves +// every command's own slot — but those per-node pipelines are shallower). +// What the mapping DOES guarantee is that a given key always lands on the same +// shard, so a caller's relative order for that key is preserved regardless of +// how the shard's batch is split. Must be called before the autopipeliner is +// used. Not safe to change concurrently with enqueues. +func (ap *AutoPipeliner) setShardFn(fn func(Cmder) int) { ap.shardFn = fn } + +// setPreflight installs a submit-time command filter (cluster wiring rejects +// commands whose request policy cannot ride a pipeline). Called once during +// construction, before the AutoPipeliner is published. +func (ap *AutoPipeliner) setPreflight(fn func(ctx context.Context, cmd Cmder) error) { + ap.preflight = fn +} + +// setMustDivert installs a predicate that forces a command off the batching +// path (see the mustDivert field). Called once during construction, before the +// AutoPipeliner is published. +func (ap *AutoPipeliner) setMustDivert(fn func(ctx context.Context, cmd Cmder) bool) { + ap.mustDivert = fn +} + +// Close stops the autopipeliner and flushes any pending commands. Worst +// case it blocks up to the internal permit backstop (~30s) PER SHARD if +// in-flight batches are wedged (e.g. read timeouts disabled against a dead +// peer) — healthy shutdowns take one round trip per shard with commands +// queued, near-zero otherwise. +func (ap *AutoPipeliner) Close() error { + if !ap.closed.CompareAndSwap(false, true) { + return nil // Already closed + } + + // Cancel context to stop flushers + ap.cancel() + + // Wake every shard's flusher so each observes the cancelled context promptly. + for _, s := range ap.shards { + s.wake() + } + + // Pass through the divert gate once: after the CompareAndSwap above, any + // registration either completed before this (so the counter already sees + // it) or will observe closed==true and reject. Without this handshake the + // wait below could read a zero counter while a diverted command was + // between its closed check and its Add. + ap.divertMu.Lock() + ap.divertMu.Unlock() //nolint:staticcheck // handshake, not a critical section + + // Drain everything that remains, BOUNDED AS ONE UNIT: the flusher exit, the + // final shard sweep, and the batch/diverted dispatch waits. + // + // None of it can be cancelled: commands taken from a queue (or accepted for + // diverted execution) were already ACCEPTED, and Close's contract is to + // flush them, so ap.cancel() deliberately does not reach an in-flight + // dispatch. With ReadTimeout disabled — a supported configuration — a + // stalled read against a dead peer, or a diverted BLPOP with a zero + // timeout, has nothing to end it. Bounding only the LAST wait would not + // help: the wedged dispatch can just as easily sit in a flusher that + // ap.wg.Wait() is waiting for, or in the shutdown sweep's own dispatch, so + // Close would hang before ever reaching the bound it documents (review + // finding by codex on #3942). On expiry, report what is still outstanding + // instead of blocking the caller: the engine is already closed to new work, + // and the leaked goroutines end when the server or the OS breaks the + // connection. See autoPipelineCloseBackstop for why the bound is generous. + return ap.drainAll(autoPipelineCloseBackstop) +} + +// drainAll runs Close's whole drain tail under a single bound and returns an +// error naming every stage that was still outstanding when it expired. Split +// out of Close so the bound is testable without a real stalled connection. +// +// The stages are ordered as Close needs them — the shard sweep must not start +// before the flushers are provably gone — but they are waited on +// CONCURRENTLY with the timer, which is the whole point: any stage can be the +// one that never finishes. +func (ap *AutoPipeliner) drainAll(timeout time.Duration) error { + flushers := make(chan struct{}) + go func() { defer close(flushers); ap.wg.Wait() }() + + // swept: after the flushers are gone, drain each shard once more under its + // lock. A command can pass enqueue's under-lock closed-recheck just before + // Close's CompareAndSwap and append to a shard AFTER that shard's flusher + // has already drained and exited — leaving its batch.done unclosed and the + // caller's accessor blocked forever. s.mu serializes the two, so either the + // late enqueue appends first and this sweep flushes it, or the sweep runs + // first and the enqueue then observes closed==true and rejects. + swept := make(chan struct{}) + go func() { + defer close(swept) + <-flushers + for _, s := range ap.shards { + s.flushBatchSliceShutdown() + } + }() + + batches := make(chan struct{}) + go func() { + defer close(batches) + <-swept + ap.batchWg.Wait() + }() + + diverted := make(chan struct{}) + go func() { defer close(diverted); ap.divertWg.Wait() }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + batchesDone, divertedDone := false, false + for !batchesDone || !divertedDone { + select { + case <-batches: + batchesDone = true + batches = nil // a closed channel is always ready; stop selecting it + case <-diverted: + divertedDone = true + diverted = nil + case <-timer.C: + var outstanding []string + if !batchesDone { + // Name the precise stage: a wedged flusher and a wedged batch + // dispatch need different operator responses. + select { + case <-flushers: + select { + case <-swept: + outstanding = append(outstanding, "batch dispatches") + default: + outstanding = append(outstanding, "the shutdown flush") + } + default: + outstanding = append(outstanding, "the flusher drain") + } + } + if !divertedDone { + outstanding = append(outstanding, "diverted (blocking) commands") + } + return fmt.Errorf( + "redis: autopipeline: Close timed out after %s with %s still in flight; "+ + "they hold pooled connections until the server or the OS ends them "+ + "(most often a blocking command with no timeout, or ReadTimeout disabled)", + timeout, strings.Join(outstanding, " and ")) + } + } + return nil +} + +// flusher is the per-shard background goroutine that flushes batches. +func (s *apShard) flusher() { + defer s.ap.wg.Done() + ap := s.ap + + for { + // Wait for a command to arrive (or shutdown). The notify channel is a + // cheap buffered wake-up; no lock is taken on the hot enqueue path. + if s.Len() == 0 { + select { + case <-s.notify: + case <-ap.ctx.Done(): + } + } + + // Check if context is cancelled + if ap.ctx.Err() != nil { + // Final flush before shutdown - use background context to avoid immediate cancellation + s.flushBatchSliceShutdown() + return + } + + // Apply the coalescing window if one is configured (MaxFlushDelay / + // AdaptiveDelay). With the default config this returns at once: batching + // under concurrent load comes from in-flight backpressure, not a wait — + // see accumulateBatch. + s.accumulateBatch() + + // Flush all pending commands + for s.Len() > 0 { + select { + case <-ap.ctx.Done(): + // Final flush before shutdown + s.flushBatchSliceShutdown() + return + default: + } + + s.flushBatchSlice() + + // Between batches, apply the configured window again so the next + // pipeline is also full. A no-op with the default config (see + // accumulateBatch); the next drain picks up whatever has queued. + if s.Len() > 0 && s.Len() < ap.config.MaxBatchSize && !s.bytesFull() { + s.accumulateBatch() + } + } + } +} + +// accumulateBatch lets commands pile up before the flusher drains the queue, +// so pipelines carry many commands instead of one. It returns as soon as any +// of these holds: +// +// - the queue reaches MaxBatchSize (batch is full); +// - a configured MaxFlushDelay / AdaptiveDelay window elapses; or +// - with no configured window (the default), the expected resubmission +// wave of arrivals has landed — see awaitExpectedArrivals. +// +// A configured MaxFlushDelay / AdaptiveDelay is an intentional accumulation +// window and is waited in full (AdaptiveDelay scales it down as the queue fills +// and returns 0 — flush now — once the queue is ≥75% full). +func (s *apShard) accumulateBatch() { + ap := s.ap + batchSize := ap.config.MaxBatchSize + if batchSize <= 0 { + batchSize = 1 + } + if s.Len() >= batchSize || s.bytesFull() { + return + } + + // Pick the accumulation window. calculateDelay returns 0 both when no + // MaxFlushDelay is configured (the default) and when AdaptiveDelay resolves + // the current fill level to "flush immediately". The fill level is this + // shard's own length — each shard flushes independently, so a global count + // would mis-tune a quiet shard while another is busy. + window := ap.calculateDelay(s.Len()) + if window <= 0 { + if ap.config.MaxFlushDelay == 0 && !ap.config.AdaptiveDelay { + // Default: coalesce by expected-arrival count, not by wall-clock. + s.awaitExpectedArrivals(batchSize) + } + return + } + + // Explicit window: wait the whole delay (or until the batch fills). Each + // enqueue sends on notify, so we re-check the queue length on every wake-up + // and return once the batch is full. + deadline := time.NewTimer(window) + defer deadline.Stop() + for { + select { + case <-ap.ctx.Done(): + return + case <-deadline.C: + return + case <-s.notify: + if s.Len() >= batchSize || s.bytesFull() { + return + } + } + } +} + +// silenceGapFloor / silenceGapCeil bound awaitExpectedArrivals's silence fallback. +// The floor covers fast links; the RTT-scaled value (execEWMA/8) takes over on +// slow ones, where a wakeup wave staggered by goroutine scheduling can pause +// longer than the floor mid-landing and a premature flush is expensive (each +// batch fragment occupies a pipeline connection for a full round trip). The +// ceiling bounds how long a stale expectation (callers that left) can delay a +// flush. +const ( + silenceGapFloor = 200 * time.Microsecond + silenceGapCeil = 2 * time.Millisecond +) + +// coalesceMinFlush is the smallest pipeline worth dispatching while other +// batches are still executing. Below it, a gap-fire holds the queued +// stragglers for the next wave instead of burning a connection on a +// near-empty flush; once nothing is in flight, any size flushes immediately. +const coalesceMinFlush = 8 + +// observeBatchExec folds one batch execution duration into execEWMA. +func (ap *AutoPipeliner) observeBatchExec(d time.Duration) { + sample := int64(d) + if sample <= 0 { + return + } + old := ap.execEWMA.Load() + if old == 0 { + ap.execEWMA.Store(sample) + return + } + ap.execEWMA.Store(old + (sample-old)/8) +} + +// silenceGap returns the silence fallback for awaitExpectedArrivals, scaled to the +// observed batch round-trip: clamp(execEWMA/8, floor, ceil). +func (ap *AutoPipeliner) silenceGap() time.Duration { + g := time.Duration(ap.execEWMA.Load() / 8) + if g < silenceGapFloor { + return silenceGapFloor + } + if g > silenceGapCeil { + return silenceGapCeil + } + return g +} + +// awaitExpectedArrivals holds the flusher while related work is in motion, so +// commands flush as deep pipelines instead of fragmenting into small batches +// (each fragment costs a pipeline connection for a full round trip). Two +// signals — both facts the engine already has, not wall-clock guesses — decide +// whether anything is imminent: +// +// - expectedArrivals: a completed batch of N commands wakes its N waiters +// together, and in a closed loop each immediately submits its next +// command. Completion announces the exact count; every enqueue accounts +// for one; the wait ends the moment the count drains — the wave of +// arrivals has fully landed. An exact per-wave count has no failure mode +// where an averaged estimate undershoots the true wave and locks the +// engine into fragmented flushes. +// - inFlight: batches still executing mean their waiters will wake shortly +// and stragglers are mid-stream — worth holding a moment to coalesce with, +// bounded by the silence gap. This also recovers a fragmented state (many +// singles in flight, which announce nothing): their staggered returns land +// within one gap, merge into a real batch, and arrival tracking resumes. +// +// When neither holds, the shard is idle and the flush happens immediately: a +// lone caller pays a single round trip with no timer armed. That is the point +// of the design — the previous fixed ~20µs debounce timer armed on every flush +// fires ~1ms late on an idle or low-core host (wakeup latency dominates the +// requested delay), taxing every low-concurrency command ~5x its round trip. +// Here the gap timer never fires in steady state, closed loop or open; it only +// ends waits for callers that left. +func (s *apShard) awaitExpectedArrivals(batchSize int) { + ap := s.ap + expected := ap.expectedArrivals.Load() + if expected < 0 { + // Arrivals outran what was announced (open-loop traffic); re-zero so + // the deficit does not mask the next wave. CAS: only clear the value + // we saw, never a concurrent announcement. + ap.expectedArrivals.CompareAndSwap(expected, 0) + expected = 0 + } + expectingWave := expected > 0 + if !expectingWave && s.inFlight.Load() == 0 { + // Idle shard: nothing imminent, flush in one round trip. + return + } + + gap := ap.silenceGap() + // Reset is drain-safe on Go 1.23+ (see go.mod: go 1.24). + fallback := time.NewTimer(gap) + defer fallback.Stop() + lastSeenExpected := expected // count as of the most recent timer (re)arm + var holdStart time.Time // set on the first straggler-hold gap fire + for { + select { + case <-ap.ctx.Done(): + return + case <-fallback.C: + if !expectingWave && s.Len() < coalesceMinFlush && s.inFlight.Load() > 0 { + // Only stragglers queued while batches are still executing: + // flushing a near-empty pipeline burns a connection for a full + // round trip (measured at high WAN concurrency: straggler + // flushes of 1-3 commands starved the connection pool and + // doubled p50). Hold them — the next completed batch's wave + // sweeps them along, and the wave path below flushes promptly. + // The hold is bounded like the permit wait: with read timeouts + // disabled a wedged batch could pin inFlight forever, and the + // held stragglers must not hang with it. + if holdStart.IsZero() { + holdStart = time.Now() + } + if time.Since(holdStart) < autoPipelinePermitBackstop { + lastSeenExpected = ap.expectedArrivals.Load() + fallback.Reset(gap) + continue + } + } + if expectingWave { + // A whole gap passed with no arrivals on this shard: the + // expected callers left (workload shrank), so clear the stale + // expectation or future flushes will wait for ghosts. But only + // if it did not GROW during the silent gap — growth means a + // batch elsewhere (another shard, or racing this fire) + // announced a fresh wave, and erasing that would fragment a + // wave that is really coming. CAS, never a blind store, so an + // announcement racing the reset itself also survives. + if d := ap.expectedArrivals.Load(); d > 0 && d <= lastSeenExpected { + ap.expectedArrivals.CompareAndSwap(d, 0) + } + } + return + case <-s.notify: + if s.Len() >= batchSize || s.bytesFull() { + return + } + if d := ap.expectedArrivals.Load(); d > 0 { + // An in-flight batch completed mid-wait: its wave is now the + // thing to wait out, with the exact-count exit below. + expectingWave = true + lastSeenExpected = d + } else if expectingWave { + // The wave has fully landed; flush it as one batch. + return + } else if s.inFlight.Load() == 0 { + // Nothing executing, no wave expected: no completion will + // wake more callers, so flush what we have now. + return + } + fallback.Reset(gap) + } + } +} + +// dispatchCmds executes the drained stripe queues as one pipeline without +// constructing a Pipeline object: the queue slices go straight to the client's +// hook-wrapped pipeline processor (the exact entry Pipeline.Exec is wired to), +// so hooks and OTel behave identically while the per-batch Pipeline allocation, +// its append-growth reallocations and the per-command Process calls disappear. +// A single-stripe drain (every ordered shard, and any drain that found one +// non-empty stripe) passes its queue zero-copy; multi-stripe drains merge into +// one pooled slice. +// The batches stay OPEN throughout: completion happens at the caller's +// deferred closes, after the whole hook chain has returned. Hooks on the +// dispatch goroutine can still read results without deadlocking via the +// dispGid guard in await() (pre-next: the not-yet-executed view; post-next: +// the populated results), and — exactly like a plain pipeline — they may +// even adjust results before any waiter wakes. +// +// The innermost records whether execution actually happened. Two hook +// behaviours the chain's return value can carry are surfaced, both while the +// batches are still open (the callers' deferred closes run after this +// returns, so no waiter is reading yet): +// - short-circuit (hook returned without calling next): nothing set the +// commands' results — the chain's error, if any, is set +// on every command; +// - post-next verdict (exec ran, a hook still returned an error): applied +// to the commands ONLY when every one of them is error-free — the case +// where the hook's verdict would otherwise vanish entirely. A plain +// pipeline hands that verdict to the Exec caller without rewriting +// per-command results; with no Exec caller here, per-command errors +// recorded by the exec always win and are never overwritten. +func (ap *AutoPipeliner) dispatchCmds(ctx context.Context, queues [][]Cmder, total int) { + cmds := queues[0] + if len(queues) > 1 { + cmds = getQueueSlice(total) + for i := range queues { + cmds = append(cmds, queues[i]...) + } + } + // A command that forbids retries (today: the zero-copy reads, whose reply + // decodes into a caller buffer that a retry could not un-write) disables + // retries for the WHOLE slice it is dispatched in — see cmdsContainNoRetry. + // In a shared batch that would silently strip retries from unrelated + // callers' ordinary commands, so a mixed batch is dispatched as several + // pipelines instead of one. + // + // Split into CONTIGUOUS RUNS, in order, never into two policy groups: + // grouping would reorder the stream — a zero-copy read submitted before a + // SET to the same key would execute after it, so the read observes the new + // value on a face that promises submit order. Runs preserve every relative + // position while still keeping each dispatched slice policy-uniform (both + // findings by codex on #3942; the grouping bug was introduced by the first + // fix for the retry leak). + if runs := splitRetryRuns(cmds); runs != nil { + ap.dispatchSequential(ctx, runs) + if len(queues) > 1 { + putQueueSlice(cmds) + } + return + } + executed := false + chainErr := ap.pipeliner.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { + executed = true + return ap.pipeliner.processPipeline(ctx, cmds) + }) + // NOTE: a hook that returns nil WITHOUT calling next has short-circuited + // SUCCESSFULLY — it served the batch itself (a cache, a mock) and set the + // command values. Plain Pipeline/Client hooks are allowed to do exactly + // that, so no error is synthesized for it: doing so made a hook that works + // on a pipeline fail on an autopipelined batch (review finding by codex on + // #3942). Only the hook's own error propagates, below. + if chainErr != nil { + if !executed { + setCmdsErr(cmds, chainErr) + } else if cmdsFirstErr(cmds) == nil { + // Post-next error on an all-clean batch: the exec fully succeeded, + // so the error can only be the hook's own verdict — apply it. + // On a mixed batch it is applied to nothing: hooks conventionally + // return next's error (`err := next(...); return err`), so after a + // partial failure the chain error is presumed to be that echo, and + // stamping it on the commands that DID succeed would overwrite + // valid replies with their batchmates' failure. Exec-recorded + // per-command outcomes always win over a post-next rewrap. + setCmdsErr(cmds, chainErr) + } + } + if len(queues) > 1 { + putQueueSlice(cmds) + } +} + +// dispatchCmdsMaybeChunked dispatches a drained batch, splitting it into +// byte-bounded chunks when MaxBatchBytes is configured: each chunk is its own +// pipeline write+read cycle, so a batch of many large values becomes several +// bounded bursts instead of one huge write that can stall a constrained link +// past its deadline. The commands' batches still complete only after ALL +// chunks executed (the caller's deferred closes), exactly like an unchunked +// dispatch — chunking bounds the wire bursts, it does not change completion +// semantics. Each chunk runs the full hook chain, like consecutive pipelines. +func (ap *AutoPipeliner) dispatchCmdsMaybeChunked(ctx context.Context, queues [][]Cmder, total int) { + limit := int64(ap.config.MaxBatchBytes) + if limit <= 0 { + ap.dispatchCmds(ctx, queues, total) + return + } + + // Merge (borrowed from dispatchCmds's multi-queue path) so chunk + // boundaries can cross stripe queues. + cmds := queues[0] + merged := false + if len(queues) > 1 { + cmds = getQueueSlice(total) + for i := range queues { + cmds = append(cmds, queues[i]...) + } + merged = true + } + + // Cut the byte-bounded chunks, then hand the ordered sequence to the shared + // dispatcher — which stops after a chunk dies on a transport-class failure, + // so later commands cannot overtake a failed prefix (see + // dispatchSequential; the retry-policy runs go through the same helper). + chunks := make([][]Cmder, 0, 4) + start := 0 + var chunkBytes int64 + for i, cmd := range cmds { + chunkBytes += cmdApproxBytes(cmd) + if chunkBytes >= limit && i+1 > start { + chunks = append(chunks, cmds[start:i+1]) + start = i + 1 + chunkBytes = 0 + } + } + if start < len(cmds) { + chunks = append(chunks, cmds[start:]) + } + ap.dispatchSequential(ctx, chunks) + if merged { + putQueueSlice(cmds) + } +} + +// dispatchSequential dispatches an ORDERED sequence of sub-batches, stopping +// once one of them dies on a transport-class failure and failing the rest with +// that error. +// +// The stop is the same contract the unchunked path has: it fails or retries the +// batch as a UNIT, so in an ordered stream later commands must never overtake a +// prefix that died (retries exhausted, hook abort). Per-command redis errors +// (WRONGTYPE, nil) are normal outcomes and do not stop the sequence. +// +// Both callers that break a batch into ordered pieces — the MaxBatchBytes +// chunker and the retry-policy runs — go through here, because the first +// version of each got this wrong independently (review findings by codex on +// #3942). +func (ap *AutoPipeliner) dispatchSequential(ctx context.Context, groups [][]Cmder) { + var abortErr error + for _, group := range groups { + if len(group) == 0 { + continue + } + if abortErr != nil { + setCmdsErr(group, abortErr) + continue + } + ap.dispatchCmds(ctx, [][]Cmder{group}, len(group)) + for _, cmd := range group { + if err := cmd.rawErr(); err != nil && !isRedisError(err) { + abortErr = err + break + } + } + } +} + +// splitRetryRuns slices cmds into maximal CONTIGUOUS runs of one retry policy, +// preserving order: run i's commands all precede run i+1's, exactly as +// submitted. It returns nil when the whole batch is already policy-uniform — +// the overwhelmingly common case — so uniform batches allocate nothing and are +// dispatched as one pipeline. +// +// Runs are sub-slices of cmds, not copies, so they must be dispatched before +// cmds is recycled and must not be returned to the slice pool individually. +func splitRetryRuns(cmds []Cmder) [][]Cmder { + if len(cmds) < 2 { + return nil + } + first := cmds[0].NoRetry() + boundary := -1 + for i := 1; i < len(cmds); i++ { + if cmds[i].NoRetry() != first { + boundary = i + break + } + } + if boundary < 0 { + return nil // uniform: one dispatch, no split + } + runs := make([][]Cmder, 0, 4) + start := 0 + policy := first + for i := 1; i < len(cmds); i++ { + if p := cmds[i].NoRetry(); p != policy { + runs = append(runs, cmds[start:i]) + start = i + policy = p + } + } + return append(runs, cmds[start:]) +} + +// recoverDispatchPanic converts a panic on a dispatch goroutine (a hook or +// command-encoder panic inside Process/Exec) into per-command errors instead +// of crashing the process. On a plain client the same panic unwinds into the +// CALLER, who can recover; the engine's dispatch goroutines have no caller, +// so an unrecovered panic here would kill the whole program on behalf of one +// bad command. Registered LAST at each dispatch site so it runs FIRST on +// unwind (LIFO) — the errors are stamped before the deferred batch closes +// wake the waiters. setCmdsErr fills only commands without an error, so +// exec-recorded outcomes for commands that finished are preserved. +func recoverDispatchPanic(cmds ...[]Cmder) { + r := recover() + if r == nil { + return + } + err := fmt.Errorf("redis: autopipeline: panic during dispatch: %v", r) + for _, batch := range cmds { + setCmdsErr(batch, err) + } + internal.Logger.Printf(context.Background(), "autopipeline: recovered dispatch panic: %v\n%s", r, debug.Stack()) +} + +// flushBatchSlice takes the shard's currently-queued commands as one batch, +// swaps in a fresh batch for subsequent enqueues, and dispatches the taken +// batch. Completion is signalled by closing the batch's done channel once +// (waking every waiter in a single operation) rather than one channel send +// per command. +func (s *apShard) flushBatchSlice() { + ap := s.ap + + // Drain every stripe into one combined batch and roll fresh queues for the + // commands enqueued after this point. Striped enqueue spreads the hot + // mutex; one merged flush keeps the pipeline deep. accumulateBatch already + // bounds the total to roughly MaxBatchSize before we get here. + queues := make([][]Cmder, 0, len(s.stripes)) + batches := make([]*apBatch, 0, len(s.stripes)) + total := 0 + for i := range s.stripes { + st := &s.stripes[i] + // Skip provably-empty stripes without taking their mutex. Safe in + // THIS path only: an enqueue publishes queueLen under the stripe lock + // and wakes the flusher after unlocking, so a command that appears + // concurrently with this unlocked read is re-observed by the + // flusher's Len() loop or the buffered notify — the same protocol the + // flusher already relies on. The shutdown drain must keep locking + // unconditionally (see flushBatchSliceShutdown). + if st.queueLen.Load() == 0 { + continue + } + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + if total == 0 { + return + } + + // Acquire a concurrency permit. The wait runs on a background context with + // a generous backstop deadline against a wedged semaphore: commands taken + // from the queue were already ACCEPTED, so a concurrent Close must not + // cancel them mid-acquire — Close's contract is to flush pending commands + // (it waits for this dispatch via wg/batchWg before tearing anything + // down). The backstop is deliberately well above both the default + // ReadTimeout and a maintnotifications relaxed window, so a legitimately + // slow batch (e.g. during a failover) holding a permit does not cause + // waiters to spuriously fail. + if !s.sem.TryAcquire() { + err := s.sem.Acquire(context.Background(), autoPipelinePermitBackstop, ErrAutoPipelineTimeout) + if err != nil { + // A permit not freeing within the backstop means the in-flight + // batch is wedged well past any configured timeout — leave an + // operator breadcrumb before failing the drained commands. + internal.Logger.Printf(context.Background(), + "redis: autopipeline: no batch permit after %s; failing %d queued commands", + autoPipelinePermitBackstop, total) + batchErr := err + for i := range queues { + for _, qc := range queues[i] { + qc.SetErr(batchErr) + } + batches[i].close() + putQueueSlice(queues[i]) + } + return + } + + // Wave merge. We took the queue and then waited a full batch round + // trip for the permit; callers whose replies landed just after our + // take re-submitted into the FRESH queue during that wait. Executing + // without them splits the group into two alternating waves — each + // observing two round trips, at half throughput — a state that is + // stable once entered (measured: p50 pinned at 2xRTT for entire runs + // at mid worker counts on a 52ms link). On the default window, let the + // wave of follow-ups land and fold it into this batch before + // executing, which merges the waves back into one batch per round + // trip. Explicit-delay configs keep their own timing. + if ap.config.MaxFlushDelay == 0 && !ap.config.AdaptiveDelay { + s.awaitExpectedArrivals(ap.config.MaxBatchSize) + for i := range s.stripes { + st := &s.stripes[i] + if st.queueLen.Load() == 0 { + continue + } + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + } + } + + // Fast path for single command: skip the pipeline and Process directly, in + // its own goroutine. The dispatch MUST NOT run inline in the flusher: a + // synchronous Process blocks the flusher for a full round trip, and on a + // slow link a solo straggler then holds up an entire landed wave for one + // RTT — whose flush then delays the straggler's next command in turn, a + // stable phase-lock where everyone pays 2x RTT (measured: ~25% of runs on + // a 57ms link locked at exactly 2x RTT until perturbed). + // No expectedArrivals announcement: a single waiter waking is the + // lone-caller case, which must keep flushing immediately. + if total == 1 { + ap.batchWg.Add(1) + s.inFlight.Add(1) + go func() { + // Defer order matters: the batch close is registered BEFORE the + // permit release and inFlight decrement so it runs AFTER them + // (LIFO) — a woken lone caller's next command then observes an + // idle shard and takes the immediate-flush path instead of + // arming the silence-gap wait. + defer ap.batchWg.Done() + defer batches[0].close() + defer s.inFlight.Add(-1) + defer s.sem.Release() + defer putQueueSlice(queues[0]) + defer recoverDispatchPanic(queues[0]) + // Background for the same reason as the batch goroutine below: + // accepted commands execute even under a concurrent Close. + execStart := time.Now() + b := batches[0] + if !ap.blocking && ap.armSelfDeadlockGuard() { + b.dispGid.Store(curGoroutineID()) + } + solo := queues[0][0] + // Both faces run the user-hook chain via withProcessHook. The + // command records the CHAIN's final verdict — exactly what + // Client.Process does — before the deferred close wakes the + // waiter, so a hook that short-circuits, rewrites, or suppresses + // the error is honored. Hooks on this goroutine read the command + // deadlock-free via the dispGid guard stamped above. + // A successful short-circuit stays successful (see dispatchCmds). + err := ap.pipeliner.withProcessHook(context.Background(), solo, func(ctx context.Context, cmd Cmder) error { + return ap.pipeliner.process(ctx, cmd) + }) + solo.SetErr(err) + ap.observeBatchExec(time.Since(execStart)) + }() + return + } + + // Track this goroutine in the batchWg so Close() waits for it. + // IMPORTANT: Add to WaitGroup AFTER semaphore is acquired to avoid deadlock. + ap.batchWg.Add(1) + s.inFlight.Add(1) + go func() { + defer ap.batchWg.Done() + defer s.inFlight.Add(-1) + defer s.sem.Release() + // Signal completion with one close per taken stripe. Deferred so a + // panic in Process/Exec (e.g. a malformed command or encoder panic) + // still wakes every waiter in await() instead of hanging them forever; + // the closes run after Exec on the happy path, so results are + // populated first. + defer func() { + for i := range queues { + batches[i].close() + putQueueSlice(queues[i]) + } + }() + defer recoverDispatchPanic(queues...) + + // Execute on a background context: these commands were accepted before + // any concurrent Close, and Close waits for this goroutine (batchWg) + // before the client tears down its pools — cancelling here would + // error already-accepted commands while the shutdown sweep flushes + // later ones, an inverted outcome. The wire timeouts (Read/Write + // Timeout, or maintnotifications relaxed windows) still bound the + // execution; no per-batch timer is allocated. + ctx := context.Background() + + // The batches complete at the deferred closes, AFTER the whole hook + // chain has returned — so a hook's post-next verdict is honored and, + // like a plain pipeline, a hook may adjust results before any waiter + // wakes. Hooks on this goroutine read results deadlock-free via the + // dispGid guard in await() (armed below when hooks can exist). + if !ap.blocking && ap.armSelfDeadlockGuard() { + gid := curGoroutineID() + for i := range batches { + batches[i].dispGid.Store(gid) + } + } + + execStart := time.Now() + ap.dispatchCmdsMaybeChunked(ctx, queues, total) + ap.observeBatchExec(time.Since(execStart)) + + // Announce the expected arrivals BEFORE the deferred closes wake this + // batch's waiters, so the flusher knows the wave size the moment its + // first command lands (see expectedArrivals). + ap.expectedArrivals.Add(int64(total)) + }() +} + +// flushBatchSliceShutdown flushes commands during shutdown. +// Unlike flushBatchSlice, this doesn't use ap.ctx for semaphore acquisition +// because ap.ctx is already cancelled during shutdown. +// Executes synchronously to preserve command order. +func (s *apShard) flushBatchSliceShutdown() { + ap := s.ap + // Flush all remaining commands synchronously to preserve order. + // + // The loop condition is checked UNDER each stripe's lock (not via the + // unlocked s.Len()): a late enqueue appends to a stripe's queue and updates + // its queueLen under that stripe's mutex, so reading queueLen without the + // lock could miss a command that was just appended (seeing 0 and exiting + // while a command sits in the queue). Locking first makes "is the stripe + // empty?" and "take the stripe" atomic against that enqueue — this is what + // closes the lost-command race on Close. + for { + // Take every stripe's queue as one merged batch and roll fresh queues. + queues := make([][]Cmder, 0, len(s.stripes)) + batches := make([]*apBatch, 0, len(s.stripes)) + total := 0 + for i := range s.stripes { + st := &s.stripes[i] + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + if total == 0 { + return + } + + // Serialize with any still-running in-flight batch: the shutdown drain + // used to bypass the per-shard permit, so under MaxConcurrentBatches:1 + // a drained command could execute CONCURRENTLY with the in-flight + // batch during Close and be observed out of order. Acquire the permit + // (bounded by the backstop, on a background context — ap.ctx is + // already cancelled here); if the backstop expires the permit holder + // is wedged and we proceed anyway rather than strand the commands. + acquired := s.sem.TryAcquire() + if !acquired { + acquired = s.sem.Acquire(context.Background(), autoPipelinePermitBackstop, ErrAutoPipelineTimeout) == nil + if !acquired { + internal.Logger.Printf(context.Background(), + "redis: autopipeline: no batch permit after %s during shutdown; flushing unserialized", + autoPipelinePermitBackstop) + } + } + + // Execute each batch in a func so close(batch.done) is deferred: a panic + // in Process/Exec still signals completion (waking await()) before it + // propagates, instead of leaving shutdown waiters hung. + func() { + if acquired { + defer s.sem.Release() + } + defer func() { + for i := range queues { + batches[i].close() + putQueueSlice(queues[i]) + } + }() + defer recoverDispatchPanic(queues...) + + // ap.ctx is already cancelled here (Close cancels it before draining), + // so use a fresh background context with no artificial deadline. The + // wire timeout is then governed by the connection's ReadTimeout / + // WriteTimeout — exactly like the normal flush path and a plain client + // Exec. Crucially this lets a relaxed timeout (set by maintnotifications + // during a failover/migration) take effect; a hardcoded short deadline + // here would cap that relaxed window and time out in-flight commands the + // relaxation was meant to protect. (A user who wants shutdown bounded + // sets ReadTimeout/WriteTimeout on the client, as for any command.) + if !ap.blocking && ap.armSelfDeadlockGuard() { + gid := curGoroutineID() + for i := range batches { + batches[i].dispGid.Store(gid) + } + } + ap.dispatchCmdsMaybeChunked(context.Background(), queues, total) + }() + } +} + +// Len returns the number of queued commands in this shard. +func (s *apShard) Len() int { + n := 0 + for i := range s.stripes { + n += int(s.stripes[i].queueLen.Load()) + } + return n +} + +// bytesFull reports whether the shard's queued payload volume has reached the +// configured MaxBatchBytes (false when the cap is disabled). Like the +// MaxBatchSize trigger it is soft: enqueues racing the check can overshoot. +func (s *apShard) bytesFull() bool { + limit := int64(s.ap.config.MaxBatchBytes) + if limit <= 0 { + return false + } + var n int64 + for i := range s.stripes { + n += s.stripes[i].queueBytes.Load() + if n >= limit { + return true + } + } + return false +} + +// cmdApproxBytes estimates a command's wire payload for MaxBatchBytes +// accounting: string/[]byte argument lengths plus a small fixed overhead per +// argument (type marker, length line, CRLFs). Exactness doesn't matter — the +// cap bounds burst size, it is not a protocol calculation. +func cmdApproxBytes(cmd Cmder) int64 { + const perArgOverhead = 16 + n := int64(0) + for _, a := range cmd.Args() { + switch v := a.(type) { + case string: + n += int64(len(v)) + case []byte: + n += int64(len(v)) + default: + n += 8 + } + n += perArgOverhead + } + return n +} + +// Len returns the current number of queued commands across all shards. +func (ap *AutoPipeliner) Len() int { + total := 0 + for _, s := range ap.shards { + total += s.Len() + } + return total +} + +// calculateDelay calculates the delay based on the given queue length (the +// caller's own shard, not the global total, so each shard tunes independently). +// Uses integer-only arithmetic for optimal performance (no float operations). +// Returns 0 if MaxFlushDelay is 0. +func (ap *AutoPipeliner) calculateDelay(queueLen int) time.Duration { + maxDelay := ap.config.MaxFlushDelay + if maxDelay == 0 { + return 0 + } + + // If adaptive delay is disabled, return fixed delay + if !ap.config.AdaptiveDelay { + return maxDelay + } + + if queueLen == 0 { + return 0 + } + + maxBatch := ap.config.MaxBatchSize + + // Use integer arithmetic to avoid float operations + // Calculate thresholds: 75%, 50%, 25% of maxBatch + // Multiply by 4 to avoid division: queueLen * 4 vs maxBatch * 3 (75%) + // + // Adaptive delay strategy: + // - ≥75% full: No delay (flush immediately to prevent overflow) + // - ≥50% full: 25% of max delay (queue filling up) + // - ≥25% full: 50% of max delay (moderate load) + // - <25% full: 100% of max delay (low load, maximize batching) + switch { + case queueLen*4 >= maxBatch*3: // queueLen >= 75% of maxBatch + return 0 // Flush immediately + case queueLen*2 >= maxBatch: // queueLen >= 50% of maxBatch + return maxDelay >> 2 // Divide by 4 using bit shift (faster) + case queueLen*4 >= maxBatch: // queueLen >= 25% of maxBatch + return maxDelay >> 1 // Divide by 2 using bit shift (faster) + default: + return maxDelay + } +} + +// Pipeline returns a new pipeline that uses the underlying pipeliner. +// This allows you to create a traditional pipeline from an autopipeliner. +func (ap *AutoPipeliner) Pipeline() Pipeliner { + return ap.pipeliner.Pipeline() +} + +// Pipelined executes a function in a pipeline context. +// This is a convenience method that creates a pipeline, executes the function, +// and returns the results. +func (ap *AutoPipeliner) Pipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { + return ap.pipeliner.Pipeline().Pipelined(ctx, fn) +} + +// TxPipelined executes a function in a transaction pipeline context. +// This is a convenience method that creates a transaction pipeline, executes the function, +// and returns the results. It delegates to the underlying client's TxPipeline. +func (ap *AutoPipeliner) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { + return ap.pipeliner.TxPipeline().Pipelined(ctx, fn) +} + +// TxPipeline returns a new transaction pipeline that uses the underlying pipeliner. +// This allows you to create a traditional transaction pipeline from an autopipeliner. +// It delegates to the underlying client's TxPipeline. +func (ap *AutoPipeliner) TxPipeline() Pipeliner { + return ap.pipeliner.TxPipeline() +} + +// validate AutoPipeliner implements Cmdable +var _ Cmdable = (*AutoPipeliner)(nil) diff --git a/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md b/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md new file mode 100644 index 00000000..40febdf9 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md @@ -0,0 +1,128 @@ +# Autopipelining benchmarks + +These benchmarks validate the goal of autopipelining: batching concurrent +commands into pipelines cuts network round-trips and raises throughput, without +callers writing pipeline code. + +Two measurement rules keep every number honest: + +- **Throughput is measured on executed commands** — a command is counted only + after its result has been read (`.Result()` / `.Err()`), never when it is + merely queued. On the deferred face `ap.Set` returns immediately, so counting + calls would measure enqueue speed, not throughput. +- **Rates divide by the timed region, not the nominal window.** The + fixed-duration drivers keep draining whatever was in flight when the deadline + hit; that drain is part of the timed region, so `ops/sec` cannot be inflated + by work that finished after the window closed. + +## Running + +The benchmarks talk to a real Redis on `:6379` (they skip when none answers). +Start one first: + +```sh +make docker.start # or: docker run --rm -p 6379:6379 redis + +# the headline three-way throughput comparison: +go test -run '^$' -bench BenchmarkAutoPipelineThroughput -benchtime=1x . + +# everything: +go test -run '^$' -bench Benchmark -benchmem -benchtime=1x . +``` + +The throughput benchmarks run for a fixed wall-clock duration (~3s each) and +report `ops/sec`, so `-benchtime=1x` (one iteration) is correct for them. +**Do not pass a time-based `-benchtime`** (e.g. `-benchtime=5s`): the +fixed-duration drivers ignore `b.N`, so Go's framework would re-run the full +window geometrically trying to fill the time budget. + +The per-operation benchmarks (`BenchmarkDispatchPath`, +`BenchmarkAutoPipelineZeroCopy`) are the opposite: their ns/op and allocs/op +are only meaningful at the **default** `-benchtime` — at `-benchtime=1x` the +workers still issue a minimum window each, all billed to a single iteration. +(The "everything" command above also sweeps the repo's other root-package +benchmarks; that is harmless, just broader than this file.) + +## The headline benchmark: BenchmarkAutoPipelineThroughput + +Three ways to issue the same workload (2000 goroutines), each counting only +executed commands: + +1. **Normal** — a plain client; each `Set` is a blocking round-trip. Bounded by + Redis's non-pipelined ceiling (like `redis-benchmark` without `-P`). +2. **AutoPipelineBlocking** — the blocking face with a parallel-batch config: + `ap.Set(...)` blocks until executed, the same call shape as a normal client. + Only one command per caller is in flight, but the flusher batches across the + 2000 callers into deep pipelines. +3. **AutoPipelineWindowed** — the deferred face: each caller submits a window of + 200 commands, then reads the results. Keeps pipelines deepest. + +The `WindowedGET` variant repeats (3) with GET instead of SET: SET throughput is +server-bound (Redis's write processing), GET is cheaper on the server, so the +GET number shows the client machinery itself is not the limit. + +**Absolute numbers are machine- and load-dependent and vary a lot** — CPU +count, Redis's own ceiling, network path (loopback vs docker veth vs real +network), and noisy neighbors all move them by integer factors. The signal is +the WITHIN-RUN multiplier against the `Normal` baseline measured in the same +environment, plus `allocs/op` (which is exact and stable everywhere): + +| variant | vs Normal (same run) | +| ----------------------- | ---------------------- | +| Normal | 1x (the baseline) | +| AutoPipelineBlocking | ~10x | +| AutoPipelineWindowed | ~25-30x | +| AutoPipelineWindowedGET | above Windowed (reads) | + +As one concrete example: an Apple Silicon laptop with a loopback Redis puts +`Normal` around 80k ops/sec (so ~800k blocking, ~2.5M windowed); a 4-vCPU CI +runner with dockerized Redis lands near half that on the CPU-bound variants — +different absolutes, same multipliers and ordering. + +The autopipeline variants use an explicit parallel-batch config +(`MaxBatchSize: 300, MaxConcurrentBatches: 80, Unordered: true`) — **not the +ordered default**. The default (`MaxConcurrentBatches: 1`, +`DefaultAutoPipelineOptions` / `DefaultBlockingAutoPipelineOptions`) serializes +batch execution: blocking usage lands at roughly half the parallel-batch +multiplier, while windowed submission stays well into the tens-of-x even +ordered. + +## The other benchmarks + +- **BenchmarkIndividualCommands** — plain-client baseline: one blocking + round-trip per command across GOMAXPROCS workers. Its ns/op is your + environment's RTT floor; every other number is best read against it. +- **BenchmarkManualPipeline** — hand-built 100-deep `Pipeline().Exec()`, + sequential: the per-command cost of explicit pipelining (roughly a tenth + of a round-trip per command). The ceiling autopipelining approaches + without anyone writing pipeline code. +- **BenchmarkDispatchPath** — the engine's per-command dispatch cost with + honest `b.N` accounting: ns/op and allocs/op per executed command + (4 allocs/cmd on the submit path; unordered dispatch roughly halves the + ordered ns/op), plus the lone-command blocking fast path (~1 RTT). +- **BenchmarkFutureFace** — the typed future face on the ordered default + config: per-command reads (`InOrder`) vs windowed reads (`Window200`, + roughly 2x InOrder). +- **BenchmarkAutoPipelineSubmit** — the non-blocking `Submit` entry point, + windowed, on the ordered default; lands in the same band as + `FutureFace/Window200`. +- **BenchmarkAutoPipelineZeroCopy** — `GetToBuffer`/`SetFromBuffer` vs regular + `Get`/`Set` (Set+Get pairs): B/op drops ~10x at 4KiB and ~90x at 64KiB + (payloads decode into the caller's buffer instead of fresh strings), with + throughput at parity or better; allocs/op 10 vs 11. The B/op and allocs/op + ratios are environment-independent. +- **BenchmarkClusterAutoPipelineThroughput** — the same blocking/windowed + drivers against a local 3-master cluster (slot-routed shard batches keep + per-node pipelines deep; scales past the standalone numbers in the same + environment, with a wide run-order-dependent spread). Skips when no + cluster answers on `:16600-16602`. + +## What was deliberately removed + +Earlier revisions carried "tuning sweep" benchmarks (batch sizes, flush +delays, buffer sizes) whose numbers were dominated by the configured +`MaxFlushDelay` timer at low parallelism — every swept value reported the same +timer readout, which could only mislead someone tuning from them. They were +removed rather than fixed: `BenchmarkDispatchPath` and the throughput drivers +cover the engine's real knobs. Tune with your own workload shape; the engine's +defaults need no tuning to hit the numbers above. diff --git a/vendor/github.com/redis/go-redis/v9/command.go b/vendor/github.com/redis/go-redis/v9/command.go index ae0158b3..f0575c0e 100644 --- a/vendor/github.com/redis/go-redis/v9/command.go +++ b/vendor/github.com/redis/go-redis/v9/command.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/redis/go-redis/v9/internal" @@ -155,6 +156,7 @@ const ( CmdTypeFTSearch CmdTypeTSTimestampValue CmdTypeTSTimestampValueSlice + CmdTypeTSNRangePivotRowSlice CmdTypeHotKeys CmdTypeIncrEXInt CmdTypeIncrEXFloat @@ -222,12 +224,24 @@ type Cmder interface { stepCount() int8 SetStepCount(int8) + // cachedSlot/setCachedSlot memoize the cluster slot so it is computed once + // (in the autopipeline shard router) and reused at pipeline-flush routing. + cachedSlot() (int, bool) + setCachedSlot(int) + readTimeout() *time.Duration readReply(rd *proto.Reader) error readRawReply(rd *proto.Reader) error SetErr(error) Err() error + // setReady marks a command as asynchronously pending (autopipeline async + // faces); await blocks the public accessors until it has executed; rawErr + // reads the error without awaiting (internal execution path). + setReady(*apBatch) + await() + rawErr() error + // NoRetry returns true if the command should not be retried on failure. // Commands that write directly to an io.Writer should return true since // partial writes cannot be undone on retry. @@ -239,7 +253,8 @@ type Cmder interface { func setCmdsErr(cmds []Cmder, e error) { for _, cmd := range cmds { - if cmd.Err() == nil { + // rawErr: this runs on the execution path; never await here. + if cmd.rawErr() == nil { cmd.SetErr(e) } } @@ -247,7 +262,8 @@ func setCmdsErr(cmds []Cmder, e error) { func cmdsFirstErr(cmds []Cmder) error { for _, cmd := range cmds { - if err := cmd.Err(); err != nil { + // rawErr: this runs on the execution path; never await here. + if err := cmd.rawErr(); err != nil { return err } } @@ -294,6 +310,14 @@ func cmdFirstKeyPosWithInfo(cmd Cmder, info *CommandInfo) int { return 0 } + // Module commands registered keyless in the static policy table (e.g. + // ft.aliaslist) route as keyless even while the command-info cache is + // cold, so the first calls of a process don't hash a non-key argument + // (such as an index name) into a slot. + if defaultPolicyKeyless(name) { + return 0 + } + switch name { case "eval", "evalsha", "eval_ro", "evalsha_ro": if cmd.stringArg(2) != "0" { @@ -328,7 +352,7 @@ func cmdString(cmd Cmder, val interface{}) string { b = internal.AppendArg(b, arg) } - if err := cmd.Err(); err != nil { + if err := cmd.rawErr(); err != nil { b = append(b, ": "...) b = append(b, err.Error()...) } else if val != nil { @@ -350,6 +374,90 @@ type baseCmd struct { rawVal interface{} _readTimeout *time.Duration cmdType CmdType + // slotCache memoizes the cluster slot once computed, so the cluster + // autopipeline shard router and the pipeline flush router don't each + // recompute it. 0 = not computed; it stores slot+1 so a real slot of 0 is + // distinguishable from unset. A plain field is safe by construction: it + // is written at most once, on the submitting goroutine BEFORE the command + // is published to a stripe queue (the stripe mutex is the happens-before + // edge to the flusher that later reads it). Do not write it from any + // other point in the command's life. + slotCache uint16 + + // ready, when non-nil, is the batch whose done channel closes once the + // command has executed. It is set only by the deferred (async) + // autopipeliner, which hands the command back to the caller before it + // runs. The public result accessors (Err/Val/Result/String) call await + // so they transparently block until execution; internal execution-path + // reads use rawErr to avoid awaiting the very batch they are producing + // (formatting included: cmdString reads rawErr and receives the value + // snapshot from its caller, so String methods await BEFORE reading their + // val field — otherwise formatting an in-flight async command would race + // with reply processing). ready stays + // nil for ordinary synchronous commands, whose accessors never block. + ready atomic.Pointer[apBatch] +} + +// setReady publishes the batch gating this command's result accessors. The +// field is atomic, NOT lock-ordered with the enqueue: a dispatch-side hook +// racing this store simply reads nil and takes the non-blocking path — the +// correct "not executed yet" view — while the setting goroutine always sees +// its own store before it awaits. +func (cmd *baseCmd) setReady(b *apBatch) { cmd.ready.Store(b) } + +// await blocks until an asynchronously-submitted command has executed. It is a +// single nil-pointer load for synchronous commands, so the common path stays +// allocation- and contention-free. +func (cmd *baseCmd) await() { + b := cmd.ready.Load() + if b == nil { + return + } + select { + case <-b.done: + return + default: + } + if b.isExecutorGoroutine() { + // A hook on one of the batch's own executor goroutines (the + // dispatcher, or a cluster per-node executor) is reading this + // command's result BEFORE next() has executed it. Blocking would + // self-deadlock (the batch completes only after that goroutine + // returns); return the not-yet-executed state instead — the same + // view a plain pipeline hook has before next(). + return + } + <-b.done +} + +// rawErr returns the command error WITHOUT awaiting. The internal +// execution/serialization path (setCmdsErr, cmdsFirstErr, and the cmdString +// formatter — public String methods await before calling it) uses it so that +// reading errors while a batch is being executed does not deadlock on the +// batch's own completion signal. +func (cmd *baseCmd) rawErr() error { return cmd.err } + +// readyBatch exposes the deferred-face batch gating this command (nil for +// synchronous commands) to the cluster fan-out, which registers its per-node +// goroutines as executors of every batch they carry. +func (cmd *baseCmd) readyBatch() *apBatch { return cmd.ready.Load() } + +// resultReady reports whether the command's result can be read WITHOUT +// blocking: either it never rode the deferred autopipeline face (no gating +// batch) or that batch has already completed. Post-execution bookkeeping in +// the command wrappers — the OTel metric emissions — consults it so that +// enabling telemetry cannot turn a deferred submission into a blocking call. +func (cmd *baseCmd) resultReady() bool { + b := cmd.ready.Load() + if b == nil { + return true + } + select { + case <-b.done: + return true + default: + return false + } } var _ Cmder = (*Cmd)(nil) @@ -389,6 +497,11 @@ func (cmd *baseCmd) stringArg(pos int) string { switch v := arg.(type) { case string: return v + case *string: + if v == nil { + return "" + } + return *v case []byte: return string(v) default: @@ -405,6 +518,21 @@ func (cmd *baseCmd) SetFirstKeyPos(keyPos int8) { cmd.keyPos = keyPos } +// cachedSlot returns the cached cluster slot and whether one was set. +func (cmd *baseCmd) cachedSlot() (int, bool) { + if cmd.slotCache == 0 { + return 0, false + } + return int(cmd.slotCache - 1), true +} + +// setCachedSlot stores the computed cluster slot (0..16383) for reuse. +func (cmd *baseCmd) setCachedSlot(slot int) { + if slot >= 0 && slot < 16384 { + cmd.slotCache = uint16(slot + 1) + } +} + func (cmd *baseCmd) stepCount() int8 { return cmd._stepCount } @@ -418,6 +546,7 @@ func (cmd *baseCmd) SetErr(e error) { } func (cmd *baseCmd) Err() error { + cmd.await() return cmd.err } @@ -488,6 +617,7 @@ func NewCmd(ctx context.Context, args ...interface{}) *Cmd { } func (cmd *Cmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -496,14 +626,17 @@ func (cmd *Cmd) SetVal(val interface{}) { } func (cmd *Cmd) Val() interface{} { + cmd.await() return cmd.val } func (cmd *Cmd) Result() (interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *Cmd) Text() (string, error) { + cmd.await() if cmd.err != nil { return "", cmd.err } @@ -521,6 +654,7 @@ func toString(val interface{}) (string, error) { } func (cmd *Cmd) Int() (int, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -536,6 +670,7 @@ func (cmd *Cmd) Int() (int, error) { } func (cmd *Cmd) Int64() (int64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -555,6 +690,7 @@ func toInt64(val interface{}) (int64, error) { } func (cmd *Cmd) Uint64() (uint64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -574,6 +710,7 @@ func toUint64(val interface{}) (uint64, error) { } func (cmd *Cmd) Float32() (float32, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -597,6 +734,7 @@ func toFloat32(val interface{}) (float32, error) { } func (cmd *Cmd) Float64() (float64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -616,6 +754,7 @@ func toFloat64(val interface{}) (float64, error) { } func (cmd *Cmd) Bool() (bool, error) { + cmd.await() if cmd.err != nil { return false, cmd.err } @@ -637,6 +776,7 @@ func toBool(val interface{}) (bool, error) { } func (cmd *Cmd) Slice() ([]interface{}, error) { + cmd.await() if cmd.err != nil { return nil, cmd.err } @@ -787,18 +927,22 @@ func (cmd *RawCmd) SetVal(val []byte) { } func (cmd *RawCmd) Val() []byte { + cmd.await() return cmd.val } func (cmd *RawCmd) Result() ([]byte, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RawCmd) Bytes() ([]byte, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RawCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -846,14 +990,17 @@ func (cmd *RawWriteToCmd) SetVal(written int64) { } func (cmd *RawWriteToCmd) Val() int64 { + cmd.await() return cmd.written } func (cmd *RawWriteToCmd) Result() (int64, error) { + cmd.await() return cmd.written, cmd.err } func (cmd *RawWriteToCmd) String() string { + cmd.await() return cmdString(cmd, cmd.written) } @@ -914,20 +1061,24 @@ func (cmd *ZeroCopyStringCmd) SetVal(n int) { } func (cmd *ZeroCopyStringCmd) Val() int { + cmd.await() return cmd.n } // Result returns the number of bytes read and any error. func (cmd *ZeroCopyStringCmd) Result() (int, error) { + cmd.await() return cmd.n, cmd.err } // Bytes returns the slice of the user-provided buffer containing the read data. func (cmd *ZeroCopyStringCmd) Bytes() []byte { + cmd.await() return cmd.buf[:cmd.n] } func (cmd *ZeroCopyStringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.n) } @@ -1012,20 +1163,24 @@ func (cmd *SliceCmd) SetVal(val []interface{}) { } func (cmd *SliceCmd) Val() []interface{} { + cmd.await() return cmd.val } func (cmd *SliceCmd) Result() ([]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *SliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } // Scan scans the results from the map into a destination struct. The map keys // are matched in the Redis struct fields by the `redis:"field"` tag. func (cmd *SliceCmd) Scan(dst interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -1085,18 +1240,22 @@ func (cmd *StatusCmd) SetVal(val string) { } func (cmd *StatusCmd) Val() string { + cmd.await() return cmd.val } func (cmd *StatusCmd) Result() (string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StatusCmd) Bytes() ([]byte, error) { + cmd.await() return util.StringToBytes(cmd.val), cmd.err } func (cmd *StatusCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1137,18 +1296,22 @@ func (cmd *IntCmd) SetVal(val int64) { } func (cmd *IntCmd) Val() int64 { + cmd.await() return cmd.val } func (cmd *IntCmd) Result() (int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *IntCmd) Uint64() (uint64, error) { + cmd.await() return uint64(cmd.val), cmd.err } func (cmd *IntCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1187,14 +1350,17 @@ func (cmd *UintCmd) SetVal(val uint64) { } func (cmd *UintCmd) Val() uint64 { + cmd.await() return cmd.val } func (cmd *UintCmd) Result() (uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *UintCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1247,14 +1413,17 @@ func (cmd *DigestCmd) SetVal(val uint64) { } func (cmd *DigestCmd) Val() uint64 { + cmd.await() return cmd.val } func (cmd *DigestCmd) Result() (uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *DigestCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1304,14 +1473,17 @@ func (cmd *IntSliceCmd) SetVal(val []int64) { } func (cmd *IntSliceCmd) Val() []int64 { + cmd.await() return cmd.val } func (cmd *IntSliceCmd) Result() ([]int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *IntSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1322,8 +1494,13 @@ func (cmd *IntSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]int64, n) for i := 0; i < len(cmd.val); i++ { - if cmd.val[i], err = rd.ReadInt(); err != nil { + switch num, err := rd.ReadInt(); { + case err == Nil: + cmd.val[i] = 0 + case err != nil: return err + default: + cmd.val[i] = num } } return nil @@ -1364,14 +1541,17 @@ func (cmd *UintSliceCmd) SetVal(val []uint64) { } func (cmd *UintSliceCmd) Val() []uint64 { + cmd.await() return cmd.val } func (cmd *UintSliceCmd) Result() ([]uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *UintSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1382,8 +1562,13 @@ func (cmd *UintSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]uint64, n) for i := range cmd.val { - if cmd.val[i], err = rd.ReadUint(); err != nil { + switch num, err := rd.ReadUint(); { + case err == Nil: + cmd.val[i] = 0 + case err != nil: return err + default: + cmd.val[i] = num } } return nil @@ -1428,14 +1613,17 @@ func (cmd *DurationCmd) SetVal(val time.Duration) { } func (cmd *DurationCmd) Val() time.Duration { + cmd.await() return cmd.val } func (cmd *DurationCmd) Result() (time.Duration, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *DurationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1488,14 +1676,17 @@ func (cmd *TimeCmd) SetVal(val time.Time) { } func (cmd *TimeCmd) Val() time.Time { + cmd.await() return cmd.val } func (cmd *TimeCmd) Result() (time.Time, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TimeCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1547,14 +1738,17 @@ func (cmd *BoolCmd) SetVal(val bool) { } func (cmd *BoolCmd) Val() bool { + cmd.await() return cmd.val } func (cmd *BoolCmd) Result() (bool, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *BoolCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1602,18 +1796,22 @@ func (cmd *StringCmd) SetVal(val string) { } func (cmd *StringCmd) Val() string { + cmd.await() return cmd.val } func (cmd *StringCmd) Result() (string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringCmd) Bytes() ([]byte, error) { + cmd.await() return util.StringToBytes(cmd.val), cmd.err } func (cmd *StringCmd) Bool() (bool, error) { + cmd.await() if cmd.err != nil { return false, cmd.err } @@ -1621,6 +1819,7 @@ func (cmd *StringCmd) Bool() (bool, error) { } func (cmd *StringCmd) Int() (int, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1628,6 +1827,7 @@ func (cmd *StringCmd) Int() (int, error) { } func (cmd *StringCmd) Int64() (int64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1635,6 +1835,7 @@ func (cmd *StringCmd) Int64() (int64, error) { } func (cmd *StringCmd) Uint64() (uint64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1642,6 +1843,7 @@ func (cmd *StringCmd) Uint64() (uint64, error) { } func (cmd *StringCmd) Float32() (float32, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1653,6 +1855,7 @@ func (cmd *StringCmd) Float32() (float32, error) { } func (cmd *StringCmd) Float64() (float64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1660,6 +1863,7 @@ func (cmd *StringCmd) Float64() (float64, error) { } func (cmd *StringCmd) Time() (time.Time, error) { + cmd.await() if cmd.err != nil { return time.Time{}, cmd.err } @@ -1667,6 +1871,7 @@ func (cmd *StringCmd) Time() (time.Time, error) { } func (cmd *StringCmd) Scan(val interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -1674,6 +1879,7 @@ func (cmd *StringCmd) Scan(val interface{}) error { } func (cmd *StringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1714,14 +1920,17 @@ func (cmd *FloatCmd) SetVal(val float64) { } func (cmd *FloatCmd) Val() float64 { + cmd.await() return cmd.val } func (cmd *FloatCmd) Result() (float64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FloatCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1762,14 +1971,17 @@ func (cmd *FloatSliceCmd) SetVal(val []float64) { } func (cmd *FloatSliceCmd) Val() []float64 { + cmd.await() return cmd.val } func (cmd *FloatSliceCmd) Result() ([]float64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FloatSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1830,18 +2042,22 @@ func (cmd *StringSliceCmd) SetVal(val []string) { } func (cmd *StringSliceCmd) Val() []string { + cmd.await() return cmd.val } func (cmd *StringSliceCmd) Result() ([]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *StringSliceCmd) ScanSlice(container interface{}) error { + cmd.await() return proto.ScanSlice(cmd.val, container) } @@ -1902,14 +2118,17 @@ func (cmd *StringSliceSliceCmd) SetVal(val [][]string) { } func (cmd *StringSliceSliceCmd) Val() [][]string { + cmd.await() return cmd.val } func (cmd *StringSliceSliceCmd) Result() ([][]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringSliceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1987,14 +2206,17 @@ func (cmd *KeyValueSliceCmd) SetVal(val []KeyValue) { } func (cmd *KeyValueSliceCmd) Val() []KeyValue { + cmd.await() return cmd.val } func (cmd *KeyValueSliceCmd) Result() ([]KeyValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *KeyValueSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2030,6 +2252,9 @@ func (cmd *KeyValueSliceCmd) readReply(rd *proto.Reader) error { // nolint:dupl if array { cmd.val = make([]KeyValue, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the key-value array, wanted a multiple of 2", n) + } cmd.val = make([]KeyValue, n/2) } @@ -2089,14 +2314,17 @@ func (cmd *BoolSliceCmd) SetVal(val []bool) { } func (cmd *BoolSliceCmd) Val() []bool { + cmd.await() return cmd.val } func (cmd *BoolSliceCmd) Result() ([]bool, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *BoolSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2107,8 +2335,13 @@ func (cmd *BoolSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]bool, n) for i := 0; i < len(cmd.val); i++ { - if cmd.val[i], err = rd.ReadBool(); err != nil { + switch b, err := rd.ReadBool(); { + case err == Nil: + cmd.val[i] = false + case err != nil: return err + default: + cmd.val[i] = b } } return nil @@ -2147,6 +2380,7 @@ func NewMapStringStringCmd(ctx context.Context, args ...interface{}) *MapStringS } func (cmd *MapStringStringCmd) Val() map[string]string { + cmd.await() return cmd.val } @@ -2155,16 +2389,19 @@ func (cmd *MapStringStringCmd) SetVal(val map[string]string) { } func (cmd *MapStringStringCmd) Result() (map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringStringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } // Scan scans the results from the map into a destination struct. The map keys // are matched in the Redis struct fields by the `redis:"field"` tag. func (cmd *MapStringStringCmd) Scan(dest interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -2245,14 +2482,17 @@ func (cmd *MapStringIntCmd) SetVal(val map[string]int64) { } func (cmd *MapStringIntCmd) Val() map[string]int64 { + cmd.await() return cmd.val } func (cmd *MapStringIntCmd) Result() (map[string]int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringIntCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2309,6 +2549,7 @@ func NewMapStringSliceInterfaceCmd(ctx context.Context, args ...interface{}) *Ma } func (cmd *MapStringSliceInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2317,10 +2558,12 @@ func (cmd *MapStringSliceInterfaceCmd) SetVal(val map[string][]interface{}) { } func (cmd *MapStringSliceInterfaceCmd) Result() (map[string][]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringSliceInterfaceCmd) Val() map[string][]interface{} { + cmd.await() return cmd.val } @@ -2384,6 +2627,11 @@ func (cmd *MapStringSliceInterfaceCmd) readReply(rd *proto.Reader) (err error) { cmd.val[key] = append(cmd.val[key], data) } } + default: + // Any other reply type leaves the peeked frame unread. Returning nil + // here would put the connection back in the pool with those bytes + // buffered, so the next command reads them as its own reply. + return fmt.Errorf("redis: can't parse map-string-slice-interface reply: unexpected type %c", readType) } return nil @@ -2432,14 +2680,17 @@ func (cmd *StringStructMapCmd) SetVal(val map[string]struct{}) { } func (cmd *StringStructMapCmd) Val() map[string]struct{} { + cmd.await() return cmd.val } func (cmd *StringStructMapCmd) Result() (map[string]struct{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringStructMapCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2507,14 +2758,17 @@ func (cmd *XMessageSliceCmd) SetVal(val []XMessage) { } func (cmd *XMessageSliceCmd) Val() []XMessage { + cmd.await() return cmd.val } func (cmd *XMessageSliceCmd) Result() ([]XMessage, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XMessageSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2653,14 +2907,17 @@ func (cmd *XStreamSliceCmd) SetVal(val []XStream) { } func (cmd *XStreamSliceCmd) Val() []XStream { + cmd.await() return cmd.val } func (cmd *XStreamSliceCmd) Result() ([]XStream, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XStreamSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2757,14 +3014,17 @@ func (cmd *XPendingCmd) SetVal(val *XPending) { } func (cmd *XPendingCmd) Val() *XPending { + cmd.await() return cmd.val } func (cmd *XPendingCmd) Result() (*XPending, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XPendingCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2862,14 +3122,17 @@ func (cmd *XPendingExtCmd) SetVal(val []XPendingExt) { } func (cmd *XPendingExtCmd) Val() []XPendingExt { + cmd.await() return cmd.val } func (cmd *XPendingExtCmd) Result() ([]XPendingExt, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XPendingExtCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2946,14 +3209,17 @@ func (cmd *XAutoClaimCmd) SetVal(val []XMessage, start string) { } func (cmd *XAutoClaimCmd) Val() (messages []XMessage, start string) { + cmd.await() return cmd.val, cmd.start } func (cmd *XAutoClaimCmd) Result() (messages []XMessage, start string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.err } func (cmd *XAutoClaimCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3040,14 +3306,17 @@ func (cmd *XAutoClaimWithDeletedCmd) SetVal(val []XMessage, start string, delete } func (cmd *XAutoClaimWithDeletedCmd) Val() (messages []XMessage, start string, deletedIDs []string) { + cmd.await() return cmd.val, cmd.start, cmd.deletedIDs } func (cmd *XAutoClaimWithDeletedCmd) Result() (messages []XMessage, start string, deletedIDs []string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.deletedIDs, cmd.err } func (cmd *XAutoClaimWithDeletedCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3151,14 +3420,17 @@ func (cmd *XAutoClaimJustIDCmd) SetVal(val []string, start string) { } func (cmd *XAutoClaimJustIDCmd) Val() (ids []string, start string) { + cmd.await() return cmd.val, cmd.start } func (cmd *XAutoClaimJustIDCmd) Result() (ids []string, start string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.err } func (cmd *XAutoClaimJustIDCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3247,14 +3519,17 @@ func (cmd *XInfoConsumersCmd) SetVal(val []XInfoConsumer) { } func (cmd *XInfoConsumersCmd) Val() []XInfoConsumer { + cmd.await() return cmd.val } func (cmd *XInfoConsumersCmd) Result() ([]XInfoConsumer, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoConsumersCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3353,14 +3628,17 @@ func (cmd *XInfoGroupsCmd) SetVal(val []XInfoGroup) { } func (cmd *XInfoGroupsCmd) Val() []XInfoGroup { + cmd.await() return cmd.val } func (cmd *XInfoGroupsCmd) Result() ([]XInfoGroup, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoGroupsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3491,14 +3769,17 @@ func (cmd *XInfoStreamCmd) SetVal(val *XInfoStream) { } func (cmd *XInfoStreamCmd) Val() *XInfoStream { + cmd.await() return cmd.val } func (cmd *XInfoStreamCmd) Result() (*XInfoStream, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoStreamCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3718,14 +3999,17 @@ func (cmd *XInfoStreamFullCmd) SetVal(val *XInfoStreamFull) { } func (cmd *XInfoStreamFullCmd) Val() *XInfoStreamFull { + cmd.await() return cmd.val } func (cmd *XInfoStreamFullCmd) Result() (*XInfoStreamFull, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoStreamFullCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4100,14 +4384,17 @@ func (cmd *ZSliceCmd) SetVal(val []Z) { } func (cmd *ZSliceCmd) Val() []Z { + cmd.await() return cmd.val } func (cmd *ZSliceCmd) Result() ([]Z, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ZSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4132,6 +4419,9 @@ func (cmd *ZSliceCmd) readReply(rd *proto.Reader) error { // nolint:dupl if array { cmd.val = make([]Z, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the sorted set array, wanted a multiple of 2", n) + } cmd.val = make([]Z, n/2) } @@ -4191,14 +4481,17 @@ func (cmd *ZWithKeyCmd) SetVal(val *ZWithKey) { } func (cmd *ZWithKeyCmd) Val() *ZWithKey { + cmd.await() return cmd.val } func (cmd *ZWithKeyCmd) Result() (*ZWithKey, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ZWithKeyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4268,14 +4561,17 @@ func (cmd *ScanCmd) SetVal(page []string, cursor uint64) { } func (cmd *ScanCmd) Val() (keys []string, cursor uint64) { + cmd.await() return cmd.page, cmd.cursor } func (cmd *ScanCmd) Result() (keys []string, cursor uint64, err error) { + cmd.await() return cmd.page, cmd.cursor, cmd.err } func (cmd *ScanCmd) String() string { + cmd.await() return cmdString(cmd, cmd.page) } @@ -4362,14 +4658,17 @@ func (cmd *ClusterSlotsCmd) SetVal(val []ClusterSlot) { } func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { + cmd.await() return cmd.val } func (cmd *ClusterSlotsCmd) Result() ([]ClusterSlot, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterSlotsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4585,14 +4884,17 @@ func (cmd *GeoLocationCmd) SetVal(locations []GeoLocation) { } func (cmd *GeoLocationCmd) Val() []GeoLocation { + cmd.await() return cmd.locations } func (cmd *GeoLocationCmd) Result() ([]GeoLocation, error) { + cmd.await() return cmd.locations, cmd.err } func (cmd *GeoLocationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.locations) } @@ -4794,14 +5096,17 @@ func (cmd *GeoSearchLocationCmd) SetVal(val []GeoLocation) { } func (cmd *GeoSearchLocationCmd) Val() []GeoLocation { + cmd.await() return cmd.val } func (cmd *GeoSearchLocationCmd) Result() ([]GeoLocation, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *GeoSearchLocationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4812,11 +5117,28 @@ func (cmd *GeoSearchLocationCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]GeoLocation, n) + // Each element is an array of [name, ...] whose minimum length is set by + // the requested WITH flags. Entries shorter than that would make the + // parser read into the next reply; extra elements are drained below so a + // longer entry (e.g. from a newer server) can't leave frames on the wire. + withLen := 1 + if cmd.opt.WithDist { + withLen++ + } + if cmd.opt.WithHash { + withLen++ + } + if cmd.opt.WithCoord { + withLen++ + } for i := 0; i < n; i++ { - _, err = rd.ReadArrayLen() + nn, err := rd.ReadArrayLen() if err != nil { return err } + if nn < withLen { + return fmt.Errorf("redis: got %d elements in GEOSEARCH reply, expected at least %d", nn, withLen) + } var loc GeoLocation @@ -4849,6 +5171,11 @@ func (cmd *GeoSearchLocationCmd) readReply(rd *proto.Reader) error { return err } } + for j := withLen; j < nn; j++ { + if err := rd.DiscardNext(); err != nil { + return err + } + } cmd.val[i] = loc } @@ -4919,14 +5246,17 @@ func (cmd *GeoPosCmd) SetVal(val []*GeoPos) { } func (cmd *GeoPosCmd) Val() []*GeoPos { + cmd.await() return cmd.val } func (cmd *GeoPosCmd) Result() ([]*GeoPos, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *GeoPosCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5021,14 +5351,17 @@ func (cmd *CommandsInfoCmd) SetVal(val map[string]*CommandInfo) { } func (cmd *CommandsInfoCmd) Val() map[string]*CommandInfo { + cmd.await() return cmd.val } func (cmd *CommandsInfoCmd) Result() (map[string]*CommandInfo, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *CommandsInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5205,6 +5538,11 @@ type cmdsInfoCache struct { once internal.Once refreshLock sync.RWMutex cmds map[string]*CommandInfo + // cmdsAtomic mirrors cmds for lock-free reads via Peek. cmds is only ever + // replaced wholesale (never mutated in place), so an atomic pointer load is a + // safe, contention-free read — Peek is on the hot per-command cluster routing + // path where the RWMutex.RLock showed up as a bottleneck under heavy load. + cmdsAtomic atomic.Pointer[map[string]*CommandInfo] } func newCmdsInfoCache(fn func(ctx context.Context) (map[string]*CommandInfo, error)) *cmdsInfoCache { @@ -5231,6 +5569,7 @@ func (c *cmdsInfoCache) Get(ctx context.Context) (map[string]*CommandInfo, error } c.cmds = lowerCmds + c.cmdsAtomic.Store(&lowerCmds) return nil }) return c.cmds, err @@ -5245,16 +5584,19 @@ func (c *cmdsInfoCache) Refresh() { // Peek returns the cached CommandInfo map without triggering a Redis round-trip. // Returns nil when the cache is cold; callers should fall back to other heuristics. -// Note: during the very first Get() (initial population) this call will block on -// the writer lock. After that, concurrent Peek() calls do not block each other. +// The read is lock-free (a single atomic load) and never blocks, even while a +// concurrent Get() is populating the cache — it simply returns nil until the +// first population publishes the map. // The returned map and its entries MUST NOT be mutated by the caller. func (c *cmdsInfoCache) Peek() map[string]*CommandInfo { if c == nil { return nil } - c.refreshLock.RLock() - defer c.refreshLock.RUnlock() - return c.cmds + // Lock-free read: cmds is replaced wholesale, never mutated in place. + if p := c.cmdsAtomic.Load(); p != nil { + return *p + } + return nil } // ------------------------------------------------------------------------------ @@ -5301,6 +5643,10 @@ type SlowLog struct { // https://redis.io/commands/slowlog#output-format ClientAddr string ClientName string + // CommandArgc is the command's total argument count (including the command + // name), emitted only by Redis 8.10 or greater. It may exceed len(Args) when + // the slow log truncates the stored arguments (slowlog-max-argc, default 32). + CommandArgc int64 } type SlowLogCmd struct { @@ -5326,14 +5672,17 @@ func (cmd *SlowLogCmd) SetVal(val []SlowLog) { } func (cmd *SlowLogCmd) Val() []SlowLog { + cmd.await() return cmd.val } func (cmd *SlowLogCmd) Result() ([]SlowLog, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *SlowLogCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5396,6 +5745,21 @@ func (cmd *SlowLogCmd) readReply(rd *proto.Reader) error { return err } } + + // Redis 8.10+ appends a 7th field: the command's total argument count. + if nn >= 7 { + if cmd.val[i].CommandArgc, err = rd.ReadInt(); err != nil { + return err + } + } + + // Drain any elements past the 7 this parser knows about so a server + // that declares a longer entry array doesn't leave frames on the wire. + for j := 7; j < nn; j++ { + if err = rd.DiscardNext(); err != nil { + return err + } + } } return nil @@ -5407,11 +5771,12 @@ func (cmd *SlowLogCmd) Clone() Cmder { val = make([]SlowLog, len(cmd.val)) for i, log := range cmd.val { val[i] = SlowLog{ - ID: log.ID, - Time: log.Time, - Duration: log.Duration, - ClientAddr: log.ClientAddr, - ClientName: log.ClientName, + ID: log.ID, + Time: log.Time, + Duration: log.Duration, + ClientAddr: log.ClientAddr, + ClientName: log.ClientName, + CommandArgc: log.CommandArgc, } if log.Args != nil { val[i].Args = make([]string, len(log.Args)) @@ -5455,14 +5820,17 @@ func (cmd *LatencyCmd) SetVal(val []Latency) { } func (cmd *LatencyCmd) Val() []Latency { + cmd.await() return cmd.val } func (cmd *LatencyCmd) Result() ([]Latency, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *LatencyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5477,8 +5845,8 @@ func (cmd *LatencyCmd) readReply(rd *proto.Reader) error { if err != nil { return err } - if nn < 3 { - return fmt.Errorf("redis: got %d elements in latency get, expected at least 3", nn) + if nn < 4 { + return fmt.Errorf("redis: got %d elements in latency get, expected at least 4", nn) } if cmd.val[i].Name, err = rd.ReadString(); err != nil { return err @@ -5498,6 +5866,13 @@ func (cmd *LatencyCmd) readReply(rd *proto.Reader) error { return err } cmd.val[i].Max = time.Duration(maximum) * time.Millisecond + // Drain any elements beyond the 4 this parser reads so a server that + // declares a longer entry array can't leave frames on the wire. + for j := 4; j < nn; j++ { + if err = rd.DiscardNext(); err != nil { + return err + } + } } return nil } @@ -5570,14 +5945,17 @@ func (cmd *HotKeysCmd) SetVal(val *HotKeysResult) { } func (cmd *HotKeysCmd) Val() *HotKeysResult { + cmd.await() return cmd.val } func (cmd *HotKeysCmd) Result() (*HotKeysResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *HotKeysCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5690,6 +6068,14 @@ func (cmd *HotKeysCmd) readReply(rd *proto.Reader) error { result.ByNetBytes = parseHotKeysKeyEntries(v) } + // Only the first element of the outer array is parsed; drain the rest so a + // server that wraps more than one element doesn't leave frames on the wire. + for i := 1; i < arrayLen; i++ { + if err := rd.DiscardNext(); err != nil { + return err + } + } + cmd.val = result return nil } @@ -5776,14 +6162,17 @@ func (cmd *MapStringInterfaceCmd) SetVal(val map[string]interface{}) { } func (cmd *MapStringInterfaceCmd) Val() map[string]interface{} { + cmd.await() return cmd.val } func (cmd *MapStringInterfaceCmd) Result() (map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5855,14 +6244,17 @@ func (cmd *MapStringStringSliceCmd) SetVal(val []map[string]string) { } func (cmd *MapStringStringSliceCmd) Val() []map[string]string { + cmd.await() return cmd.val } func (cmd *MapStringStringSliceCmd) Result() ([]map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringStringSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5933,6 +6325,7 @@ func NewMapMapStringInterfaceCmd(ctx context.Context, args ...interface{}) *MapM } func (cmd *MapMapStringInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5941,10 +6334,12 @@ func (cmd *MapMapStringInterfaceCmd) SetVal(val map[string]interface{}) { } func (cmd *MapMapStringInterfaceCmd) Result() (map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapMapStringInterfaceCmd) Val() map[string]interface{} { + cmd.await() return cmd.val } @@ -6032,14 +6427,17 @@ func (cmd *MapStringInterfaceSliceCmd) SetVal(val []map[string]interface{}) { } func (cmd *MapStringInterfaceSliceCmd) Val() []map[string]interface{} { + cmd.await() return cmd.val } func (cmd *MapStringInterfaceSliceCmd) Result() ([]map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringInterfaceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6119,14 +6517,17 @@ func (cmd *KeyValuesCmd) SetVal(key string, val []string) { } func (cmd *KeyValuesCmd) Val() (string, []string) { + cmd.await() return cmd.key, cmd.val } func (cmd *KeyValuesCmd) Result() (string, []string, error) { + cmd.await() return cmd.key, cmd.val, cmd.err } func (cmd *KeyValuesCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6195,14 +6596,17 @@ func (cmd *ZSliceWithKeyCmd) SetVal(key string, val []Z) { } func (cmd *ZSliceWithKeyCmd) Val() (string, []Z) { + cmd.await() return cmd.key, cmd.val } func (cmd *ZSliceWithKeyCmd) Result() (string, []Z, error) { + cmd.await() return cmd.key, cmd.val, cmd.err } func (cmd *ZSliceWithKeyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6230,6 +6634,9 @@ func (cmd *ZSliceWithKeyCmd) readReply(rd *proto.Reader) (err error) { if array { cmd.val = make([]Z, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the sorted set array, wanted a multiple of 2", n) + } cmd.val = make([]Z, n/2) } @@ -6301,18 +6708,22 @@ func (cmd *FunctionListCmd) SetVal(val []Library) { } func (cmd *FunctionListCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *FunctionListCmd) Val() []Library { + cmd.await() return cmd.val } func (cmd *FunctionListCmd) Result() ([]Library, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FunctionListCmd) First() (*Library, error) { + cmd.await() if cmd.err != nil { return nil, cmd.err } @@ -6514,14 +6925,17 @@ func (cmd *FunctionStatsCmd) SetVal(val FunctionStats) { } func (cmd *FunctionStatsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *FunctionStatsCmd) Val() FunctionStats { + cmd.await() return cmd.val } func (cmd *FunctionStatsCmd) Result() (FunctionStats, error) { + cmd.await() return cmd.val, cmd.err } @@ -6615,11 +7029,18 @@ func (cmd *FunctionStatsCmd) readEngines(rd *proto.Reader) ([]Engine, error) { for i := 0; i < 2; i++ { key, err := rd.ReadString() + if err != nil { + return nil, err + } switch key { case "libraries_count": engine.LibrariesCount, err = rd.ReadInt() case "functions_count": engine.FunctionsCount, err = rd.ReadInt() + default: + // Unknown field: drain its value so the reader stays aligned + // with the rest of the reply. + err = rd.DiscardNext() } if err != nil { return nil, err @@ -6779,14 +7200,17 @@ func (cmd *LCSCmd) SetVal(val *LCSMatch) { } func (cmd *LCSCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *LCSCmd) Val() *LCSMatch { + cmd.await() return cmd.val } func (cmd *LCSCmd) Result() (*LCSMatch, error) { + cmd.await() return cmd.val, cmd.err } @@ -6827,6 +7251,12 @@ func (cmd *LCSCmd) readReply(rd *proto.Reader) (err error) { if lcs.Len, err = rd.ReadInt(); err != nil { return err } + default: + // Unknown field: drain its value so the reader stays aligned + // with the rest of the reply. + if err = rd.DiscardNext(); err != nil { + return err + } } } } @@ -6929,14 +7359,17 @@ func (cmd *KeyFlagsCmd) SetVal(val []KeyFlags) { } func (cmd *KeyFlagsCmd) Val() []KeyFlags { + cmd.await() return cmd.val } func (cmd *KeyFlagsCmd) Result() ([]KeyFlags, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *KeyFlagsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7032,14 +7465,17 @@ func (cmd *ClusterLinksCmd) SetVal(val []ClusterLink) { } func (cmd *ClusterLinksCmd) Val() []ClusterLink { + cmd.await() return cmd.val } func (cmd *ClusterLinksCmd) Result() ([]ClusterLink, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterLinksCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7147,14 +7583,17 @@ func (cmd *ClusterShardsCmd) SetVal(val []ClusterShard) { } func (cmd *ClusterShardsCmd) Val() []ClusterShard { + cmd.await() return cmd.val } func (cmd *ClusterShardsCmd) Result() ([]ClusterShard, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterShardsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7307,14 +7746,17 @@ func (cmd *RankWithScoreCmd) SetVal(val RankScore) { } func (cmd *RankWithScoreCmd) Val() RankScore { + cmd.await() return cmd.val } func (cmd *RankWithScoreCmd) Result() (RankScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RankWithScoreCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7469,14 +7911,17 @@ func (cmd *ClientInfoCmd) SetVal(val *ClientInfo) { } func (cmd *ClientInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *ClientInfoCmd) Val() *ClientInfo { + cmd.await() return cmd.val } func (cmd *ClientInfoCmd) Result() (*ClientInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -7727,14 +8172,17 @@ func (cmd *ACLLogCmd) SetVal(val []*ACLLogEntry) { } func (cmd *ACLLogCmd) Val() []*ACLLogEntry { + cmd.await() return cmd.val } func (cmd *ACLLogCmd) Result() ([]*ACLLogEntry, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ACLLogCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7908,14 +8356,17 @@ func (cmd *InfoCmd) SetVal(val map[string]map[string]string) { } func (cmd *InfoCmd) Val() map[string]map[string]string { + cmd.await() return cmd.val } func (cmd *InfoCmd) Result() (map[string]map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *InfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7955,6 +8406,7 @@ func (cmd *InfoCmd) readReply(rd *proto.Reader) error { } func (cmd *InfoCmd) Item(section, key string) string { + cmd.await() if cmd.val == nil { return "" } else if cmd.val[section] == nil { @@ -8012,6 +8464,7 @@ func newMonitorCmd(ctx context.Context, ch chan string) *MonitorCmd { } func (cmd *MonitorCmd) String() string { + cmd.await() return cmdString(cmd, nil) } @@ -8096,14 +8549,17 @@ func (cmd *VectorScoreSliceCmd) SetVal(val []VectorScore) { } func (cmd *VectorScoreSliceCmd) Val() []VectorScore { + cmd.await() return cmd.val } func (cmd *VectorScoreSliceCmd) Result() ([]VectorScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8180,14 +8636,17 @@ func (cmd *VectorScoreSliceSliceCmd) SetVal(val [][]VectorScore) { } func (cmd *VectorScoreSliceSliceCmd) Val() [][]VectorScore { + cmd.await() return cmd.val } func (cmd *VectorScoreSliceSliceCmd) Result() ([][]VectorScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreSliceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8307,14 +8766,17 @@ func (cmd *VectorAttribSliceCmd) SetVal(val []VectorAttrib) { } func (cmd *VectorAttribSliceCmd) Val() []VectorAttrib { + cmd.await() return cmd.val } func (cmd *VectorAttribSliceCmd) Result() ([]VectorAttrib, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorAttribSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8395,14 +8857,17 @@ func (cmd *VectorScoreAttribSliceCmd) SetVal(val []VectorScoreAttrib) { } func (cmd *VectorScoreAttribSliceCmd) Val() []VectorScoreAttrib { + cmd.await() return cmd.val } func (cmd *VectorScoreAttribSliceCmd) Result() ([]VectorScoreAttrib, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreAttribSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8926,6 +9391,13 @@ func ExtractCommandValue(cmd interface{}) (interface{}, error) { }); ok { return tsTimestampValueSliceCmd.Val(), tsTimestampValueSliceCmd.Err() } + case CmdTypeTSNRangePivotRowSlice: + if tsNRangePivotRowSliceCmd, ok := cmd.(interface { + Val() []TSNRangePivotRow + Err() error + }); ok { + return tsNRangePivotRowSliceCmd.Val(), tsNRangePivotRowSliceCmd.Err() + } case CmdTypeStringSlice: if stringSliceCmd, ok := cmd.(interface { Val() []string @@ -9064,11 +9536,18 @@ func NewIncrEXIntCmd(ctx context.Context, args ...interface{}) *IncrEXIntCmd { } func (cmd *IncrEXIntCmd) SetVal(val IncrEXIntResult) { cmd.val = val } -func (cmd *IncrEXIntCmd) Val() IncrEXIntResult { return cmd.val } +func (cmd *IncrEXIntCmd) Val() IncrEXIntResult { + cmd.await() + return cmd.val +} func (cmd *IncrEXIntCmd) Result() (IncrEXIntResult, error) { + cmd.await() return cmd.val, cmd.err } -func (cmd *IncrEXIntCmd) String() string { return cmdString(cmd, cmd.val) } +func (cmd *IncrEXIntCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} func (cmd *IncrEXIntCmd) readReply(rd *proto.Reader) error { if err := rd.ReadFixedArrayLen(2); err != nil { @@ -9118,11 +9597,18 @@ func NewIncrEXFloatCmd(ctx context.Context, args ...interface{}) *IncrEXFloatCmd } func (cmd *IncrEXFloatCmd) SetVal(val IncrEXFloatResult) { cmd.val = val } -func (cmd *IncrEXFloatCmd) Val() IncrEXFloatResult { return cmd.val } +func (cmd *IncrEXFloatCmd) Val() IncrEXFloatResult { + cmd.await() + return cmd.val +} func (cmd *IncrEXFloatCmd) Result() (IncrEXFloatResult, error) { + cmd.await() return cmd.val, cmd.err } -func (cmd *IncrEXFloatCmd) String() string { return cmdString(cmd, cmd.val) } +func (cmd *IncrEXFloatCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} func (cmd *IncrEXFloatCmd) readReply(rd *proto.Reader) error { if err := rd.ReadFixedArrayLen(2); err != nil { @@ -9172,14 +9658,17 @@ func (cmd *AREntrySliceCmd) SetVal(val []AREntry) { } func (cmd *AREntrySliceCmd) Val() []AREntry { + cmd.await() return cmd.val } func (cmd *AREntrySliceCmd) Result() ([]AREntry, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *AREntrySliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } diff --git a/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go b/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go index da8c6d31..11fc5d1c 100644 --- a/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go +++ b/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go @@ -109,6 +109,13 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{ Request: routing.ReqDefault, Response: routing.RespDefaultKeyless, }, + "aliaslist": { + Request: routing.ReqDefault, + Response: routing.RespDefaultKeyless, + Tips: map[string]string{ + routing.ReadOnlyCMD: "", + }, + }, "info": { Request: routing.ReqDefault, Response: routing.RespDefaultKeyless, @@ -156,6 +163,26 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{ }, } +// defaultPolicyKeyless reports whether name (e.g. "ft.aliaslist") is registered +// in the static policy table as a plain keyless command: default request +// routing with a keyless response policy. Commands whose slot comes from a key +// (RespDefaultHashSlot, e.g. ft.suglen) or with special request routing +// (ReqSpecial, e.g. ft.cursor) are excluded — their key position must still be +// resolved. cmdFirstKeyPosWithInfo consults this so the initial slot +// computation on a cold command-info cache matches the policy the router +// applies once the command reaches routeAndRun. +func defaultPolicyKeyless(name string) bool { + i := strings.IndexByte(name, '.') + if i < 0 { + return false + } + policy, ok := defaultPolicies[name[:i]][name[i+1:]] + if !ok { + return false + } + return policy.Request == routing.ReqDefault && policy.Response == routing.RespDefaultKeyless +} + type CommandInfoResolveFunc func(ctx context.Context, cmd Cmder) *routing.CommandPolicy type commandInfoResolver struct { diff --git a/vendor/github.com/redis/go-redis/v9/commands.go b/vendor/github.com/redis/go-redis/v9/commands.go index d347ffeb..c574f6e6 100644 --- a/vendor/github.com/redis/go-redis/v9/commands.go +++ b/vendor/github.com/redis/go-redis/v9/commands.go @@ -199,6 +199,9 @@ type Cmdable interface { ClientUnblock(ctx context.Context, id int64) *IntCmd ClientUnblockWithError(ctx context.Context, id int64) *IntCmd ClientMaintNotifications(ctx context.Context, enabled bool, endpointType string) *StatusCmd + ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd + ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd + ClientTrackingOff(ctx context.Context) *StatusCmd ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd ConfigResetStat(ctx context.Context) *StatusCmd ConfigSet(ctx context.Context, parameter, value string) *StatusCmd @@ -209,6 +212,7 @@ type Cmdable interface { FlushDB(ctx context.Context) *StatusCmd FlushDBAsync(ctx context.Context) *StatusCmd Info(ctx context.Context, section ...string) *StringCmd + InfoMap(ctx context.Context, section ...string) *InfoCmd LastSave(ctx context.Context) *IntCmd Save(ctx context.Context) *StatusCmd Shutdown(ctx context.Context) *StatusCmd @@ -296,8 +300,8 @@ func (c cmdable) Wait(ctx context.Context, numSlaves int, timeout time.Duration) return cmd } -func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond)) +func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntSliceCmd { + cmd := NewIntSliceCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond)) cmd.setReadTimeout(timeout) _ = c(ctx, cmd) return cmd @@ -565,6 +569,95 @@ func (c cmdable) ClientMaintNotifications(ctx context.Context, enabled bool, end return cmd } +// ClientTrackingOptions configures CLIENT TRACKING ON. See +// https://redis.io/commands/client-tracking/ for semantics. +type ClientTrackingOptions struct { + Redirect int64 + Bcast bool + Prefixes []string + OptIn bool + OptOut bool + NoLoop bool +} + +// ClientTracking enables or disables server-assisted client-side caching for +// the ONE connection that happens to serve this command. On a pooled client +// that connection is arbitrary, so this is only meaningful on a dedicated +// connection (see Client.Conn). When on is false, opt is ignored. Invalid +// option combinations are reported via the returned command's Err and nothing +// is sent to the server. +// +// Must not be combined with the built-in client-side cache: on a client +// configured with Options.ClientSideCache or ClientSideCacheConfig this +// command is rejected, because changing a pool connection's tracking state +// would silently break the cache's invalidation. +func (c cmdable) ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd { + if !on { + return c.ClientTrackingOff(ctx) + } + return c.ClientTrackingOn(ctx, opt) +} + +// ClientTrackingOn enables tracking on the serving connection. See +// ClientTracking for the pooled-client and built-in-CSC caveats. +func (c cmdable) ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd { + args := []interface{}{"client", "tracking", "on"} + if opt != nil { + if err := validateClientTrackingOptions(opt); err != nil { + cmd := NewStatusCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + args = appendClientTrackingOptions(args, opt) + } + cmd := NewStatusCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// ClientTrackingOff disables tracking on the serving connection. See +// ClientTracking for the pooled-client and built-in-CSC caveats. +func (c cmdable) ClientTrackingOff(ctx context.Context) *StatusCmd { + cmd := NewStatusCmd(ctx, "client", "tracking", "off") + _ = c(ctx, cmd) + return cmd +} + +func validateClientTrackingOptions(opt *ClientTrackingOptions) error { + if opt.OptIn && opt.OptOut { + return errors.New("redis: CLIENT TRACKING OPTIN and OPTOUT are mutually exclusive") + } + if opt.Bcast && (opt.OptIn || opt.OptOut) { + return errors.New("redis: CLIENT TRACKING BCAST cannot be combined with OPTIN or OPTOUT") + } + if len(opt.Prefixes) > 0 && !opt.Bcast { + return errors.New("redis: CLIENT TRACKING PREFIX requires BCAST") + } + return nil +} + +func appendClientTrackingOptions(args []interface{}, opt *ClientTrackingOptions) []interface{} { + if opt.Redirect != 0 { + args = append(args, "redirect", opt.Redirect) + } + if opt.Bcast { + args = append(args, "bcast") + } + for _, p := range opt.Prefixes { + args = append(args, "prefix", p) + } + if opt.OptIn { + args = append(args, "optin") + } + if opt.OptOut { + args = append(args, "optout") + } + if opt.NoLoop { + args = append(args, "noloop") + } + return args +} + // ------------------------------------------------------------------------------------------------ func (c cmdable) ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd { @@ -706,7 +799,7 @@ func (c cmdable) ReplicaOf(ctx context.Context, host, port string) *StatusCmd { } func (c cmdable) SlowLogGet(ctx context.Context, num int64) *SlowLogCmd { - cmd := NewSlowLogCmd(context.Background(), "slowlog", "get", num) + cmd := NewSlowLogCmd(ctx, "slowlog", "get", num) _ = c(ctx, cmd) return cmd } @@ -797,6 +890,11 @@ func (c *ModuleLoadexConfig) toArgs() []interface{} { // ModuleLoadex Redis `MODULE LOADEX path [CONFIG name value [CONFIG name value ...]] [ARGS args [args ...]]` command. func (c cmdable) ModuleLoadex(ctx context.Context, conf *ModuleLoadexConfig) *StringCmd { + if conf == nil { + cmd := NewStringCmd(ctx) + cmd.SetErr(errors.New("redis: ModuleLoadex nil config")) + return cmd + } cmd := NewStringCmd(ctx, conf.toArgs()...) _ = c(ctx, cmd) return cmd diff --git a/vendor/github.com/redis/go-redis/v9/csc_commands.go b/vendor/github.com/redis/go-redis/v9/csc_commands.go new file mode 100644 index 00000000..8c6a2b24 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/csc_commands.go @@ -0,0 +1,257 @@ +package redis + +import ( + "bytes" + "strconv" + "strings" + + "github.com/redis/go-redis/v9/internal/proto" +) + +// defaultCacheableCommands is the allow-list of read-only, deterministic +// commands whose responses may be stored in the client-side cache. Keys are +// lowercase to match baseCmd.Name() on the hot path. +var defaultCacheableCommands = map[string]struct{}{ + // String commands + "get": {}, "mget": {}, "getbit": {}, "getrange": {}, + "strlen": {}, "substr": {}, + // Hash commands + "hget": {}, "hgetall": {}, "hmget": {}, + "hkeys": {}, "hvals": {}, "hlen": {}, + "hexists": {}, "hstrlen": {}, + // List commands + "lindex": {}, "llen": {}, "lpos": {}, "lrange": {}, + // Set commands + "scard": {}, "sismember": {}, "smembers": {}, "smismember": {}, + "sdiff": {}, "sinter": {}, "sintercard": {}, "sunion": {}, + // Sorted-set commands + "zcard": {}, "zcount": {}, "zlexcount": {}, "zmscore": {}, + "zrange": {}, "zrangebylex": {}, "zrangebyscore": {}, + "zrank": {}, "zrevrange": {}, "zrevrangebylex": {}, + "zrevrangebyscore": {}, "zrevrank": {}, "zscore": {}, + "zdiff": {}, "zinter": {}, "zunion": {}, + // Bit commands + "bitcount": {}, "bitfield_ro": {}, "bitpos": {}, + // Key/generic commands + "exists": {}, "type": {}, "sort_ro": {}, "lcs": {}, + // Geo commands + "geodist": {}, "geohash": {}, "geopos": {}, "geosearch": {}, + "georadiusbymember_ro": {}, "georadius_ro": {}, + // Stream commands. XREAD is deliberately excluded: it supports BLOCK, and + // its $/+ IDs are state-relative, so identical args are not deterministic. + // XPENDING is excluded for the same class of reason: its extended form + // returns wall-clock-relative idle times and its IDLE filter is + // time-dependent, so identical args yield different correct results with + // no key modification (and therefore no invalidation). + "xlen": {}, "xrange": {}, "xrevrange": {}, + // JSON (RedisJSON) commands + "json.get": {}, "json.mget": {}, "json.arrindex": {}, "json.arrlen": {}, + "json.objkeys": {}, "json.objlen": {}, "json.resp": {}, + "json.strlen": {}, "json.type": {}, + // TimeSeries commands + "ts.get": {}, "ts.info": {}, "ts.range": {}, "ts.revrange": {}, +} + +// isCacheable reports whether cmd is eligible for client-side caching: its +// name is on the allow-list and it operates on at least one key. +func isCacheable(cmd Cmder) bool { + // Commands such as RawWriteToCmd stream replies directly to an io.Writer. + // Capturing their replies for CSC would buffer the entire response first, + // defeating their streaming and allocation guarantees. + if cmd.NoRetry() { + return false + } + if _, ok := defaultCacheableCommands[cmd.Name()]; !ok { + return false + } + // SORT_RO ... BY/GET reads pattern keys that extractRedisKeys can't + // enumerate, so its invalidations would be dropped and the result go stale. + // Plain SORT_RO is fine. + if cmd.Name() == "sort_ro" && sortROHasByGet(cmd) { + return false + } + return cmdFirstKeyPosWithInfo(cmd, nil) != 0 +} + +// sortROHasByGet reports whether a SORT_RO invocation uses BY or GET +// (case-insensitive), scanning past the command name and key. stringArg +// normalizes string, *string, and []byte tokens. +func sortROHasByGet(cmd Cmder) bool { + for i := 2; i < len(cmd.Args()); i++ { + if s := cmd.stringArg(i); strings.EqualFold(s, "by") || strings.EqualFold(s, "get") { + return true + } + } + return false +} + +// isClientTrackingCmd reports whether cmd is a CLIENT TRACKING subcommand (any +// mode: ON, OFF, or with options). Name and stringArg normalize string, +// *string, and []byte arguments. +func isClientTrackingCmd(cmd Cmder) bool { + return cmd.Name() == "client" && strings.EqualFold(cmd.stringArg(1), "tracking") +} + +// isSelectCmd reports whether cmd changes the selected database on its +// connection. CSC keys are namespaced with Options.DB, so a runtime SELECT +// would make the connection's actual database diverge from the cache namespace. +func isSelectCmd(cmd Cmder) bool { + return cmd.Name() == "select" +} + +// isAuthCmd reports whether cmd changes the authenticated user on its +// connection. The cache namespace is fixed from Options.Username, so runtime +// authentication would make the connection identity diverge from it. +func isAuthCmd(cmd Cmder) bool { + return cmd.Name() == "auth" +} + +// isProtocolChangingHelloCmd reports whether HELLO includes a protocol version +// (and can therefore switch a tracked RESP3 connection to RESP2). A bare HELLO +// only reports connection properties and is safe. +func isProtocolChangingHelloCmd(cmd Cmder) bool { + return cmd.Name() == "hello" && len(cmd.Args()) > 1 +} + +// isResetCmd reports whether cmd resets all server-side connection state. +// RESET disables tracking, switches to RESP2, deauthenticates, and changes +// other state that a pooled CSC connection relies on. +func isResetCmd(cmd Cmder) bool { + return cmd.Name() == "reset" +} + +// isSubscribeCmd reports whether a raw command would turn an ordinary pooled +// connection into a Pub/Sub connection. Pub/Sub pushes are deliberately left +// for the dedicated PubSub reader, so the CSC drainer cannot safely own such a +// connection. +func isSubscribeCmd(cmd Cmder) bool { + switch cmd.Name() { + case "subscribe", "psubscribe", "ssubscribe": + return true + default: + return false + } +} + +// buildCacheKey returns the RESP-encoded form of the command's argument list, +// used as a collision-free canonical cache key. ok is false when the writer +// cannot marshal the arguments, in which case the caller must skip caching +// rather than bucket the command under an empty key. +func buildCacheKey(cmd Cmder) (string, bool) { + args := cmd.Args() + if len(args) == 0 { + return "", false + } + var buf bytes.Buffer + if err := proto.NewWriter(&buf).WriteArgs(args); err != nil { + return "", false + } + return buf.String(), true +} + +// keyArg renders the key argument at pos exactly as proto.Writer sends it to +// the server, so invalidation lookups match the key names in the server's +// "invalidate" pushes. Only types whose stringArg rendering is byte-identical +// to the wire encoding are accepted (fmt.Sprint of any integer matches the +// writer's base-10 strconv output); for anything else — pointers, bools, +// times, durations, floats, BinaryMarshaler values — the rendering can +// diverge, the invalidation would never match, and the entry would be served +// stale forever, so ok=false and the caller skips caching (see processCached). +func keyArg(cmd Cmder, pos int) (string, bool) { + args := cmd.Args() + if pos < 0 || pos >= len(args) { + return "", false + } + switch args[pos].(type) { + case string, []byte, + int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64: + return cmd.stringArg(pos), true + } + return "", false +} + +// extractRedisKeys returns the Redis key arguments from cmd. The result lets +// the cache map incoming invalidations back to affected entries. Returns nil +// (caller skips caching) when any key +// argument cannot be rendered in its wire form (see keyArg). +func extractRedisKeys(cmd Cmder) []string { + firstKey := cmdFirstKeyPosWithInfo(cmd, nil) + if firstKey == 0 { + return nil + } + + argsLen := len(cmd.Args()) + if firstKey >= argsLen { + return nil + } + + switch cmd.Name() { + // All remaining args from firstKeyPos are keys. + case "mget", "exists", "sdiff", "sinter", "sunion": + keys := make([]string, 0, argsLen-firstKey) + for i := firstKey; i < argsLen; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + + // Numkeys pattern: numkeys at args[1], keys from args[2]. + case "sintercard", "zdiff", "zinter", "zunion": + if argsLen < 3 { + return nil + } + numKeys, err := strconv.Atoi(cmd.stringArg(1)) + if err != nil || numKeys <= 0 { + return nil + } + keys := make([]string, 0, numKeys) + for i := 2; i < 2+numKeys && i < argsLen; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + + // LCS: exactly two consecutive keys starting at firstKeyPos. + case "lcs": + if firstKey+1 >= argsLen { + return nil + } + k1, ok1 := keyArg(cmd, firstKey) + k2, ok2 := keyArg(cmd, firstKey+1) + if !ok1 || !ok2 { + return nil + } + return []string{k1, k2} + + // JSON.MGET: keys from firstKeyPos to second-to-last (last arg is the + // JSON path, not a key). + case "json.mget": + lastKey := argsLen - 2 + if lastKey < firstKey { + return nil + } + keys := make([]string, 0, lastKey-firstKey+1) + for i := firstKey; i <= lastKey; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + } + + // Single key at firstKeyPos (GET, HGET, LRANGE, ...). + k, ok := keyArg(cmd, firstKey) + if !ok { + return nil + } + return []string{k} +} diff --git a/vendor/github.com/redis/go-redis/v9/csc_integration.go b/vendor/github.com/redis/go-redis/v9/csc_integration.go new file mode 100644 index 00000000..f9b41352 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/csc_integration.go @@ -0,0 +1,922 @@ +package redis + +import ( + "bytes" + "context" + "errors" + "reflect" + "runtime" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" + "github.com/redis/go-redis/v9/push" +) + +// cscRegisterCleanups arranges for a client dropped without Close to stop its +// background CSC drainer. The drainer's exit path revokes its pool's cache +// coverage; the runtime cleanup itself stays non-blocking and never captures +// *Client, so the wrapper remains collectible. +func cscRegisterCleanups(c *Client) { + h := c.baseClient.cscDrainHandle + if h == nil { + return + } + // Capture cscActive (a standalone *atomic.Bool, not *Client) so the cleanup + // also stops clones from serving once the drainer is gone. + active := c.baseClient.cscActive + runtime.AddCleanup(c, func(h *cscDrainHandle) { + if active != nil { + active.Store(false) + } + h.signalStop() + }, h) +} + +// ClientSideCacheConfig configures the built-in client-side cache. Pass a +// non-nil value to Options.ClientSideCacheConfig to enable caching on a RESP3 +// client. +// +// Experimental: this API may change in a minor release. +type ClientSideCacheConfig = CacheConfig + +const ( + invalidatePushName = "invalidate" + // cscNamespaceSep separates fixed-width/logically-delimited namespace parts + // from the command or Redis key. + cscNamespaceSep = "\x00" +) + +// cscNamespacePrefix scopes a shared cache by database and fixed ACL identity. +// Password rotation does not change identity; provider-backed identities are +// rejected before attachment. +func cscNamespacePrefix(db int, username string) string { + return strconv.Itoa(db) + cscNamespaceSep + + strconv.Itoa(len(username)) + ":" + username + cscNamespaceSep +} + +func cscNamespacedKey(prefix, key string) string { + return prefix + key +} + +// invalidateHandler propagates RESP3 "invalidate" push notifications into the +// shared client-side cache. keyPrefix scopes incoming key names so a shared +// cache cannot collide across databases or fixed ACL identities. +// +// The binding (cache, keyPrefix) is mutable under mu: the owning client's teardown +// RELEASES it (cache=nil) instead of unregistering the handler, so the handler +// can stay registered protected — application code holding the processor +// cannot silently unregister invalidation out from under a live client — while +// a successor client on the same processor can still rebind it (see +// registerInvalidateHandler). +type invalidateHandler struct { + mu sync.RWMutex + cache Cache + keyPrefix string + users int +} + +// HandlePushNotification decodes ["invalidate", ] notifications. A nil +// payload is emitted on FLUSHDB/FLUSHALL and triggers a full cache flush. +func (h *invalidateHandler) HandlePushNotification( + _ context.Context, _ push.NotificationHandlerContext, notification []interface{}, +) error { + h.mu.RLock() + cache, keyPrefix := h.cache, h.keyPrefix + h.mu.RUnlock() + if cache == nil || len(notification) < 2 { + return nil + } + + switch payload := notification[1].(type) { + case nil: + cache.Flush() + case []interface{}: + for _, k := range payload { + var name string + switch v := k.(type) { + case string: + name = v + case []byte: + name = string(v) + default: + continue + } + cache.DeleteByRedisKey(cscNamespacedKey(keyPrefix, name)) + } + } + return nil +} + +func (h *invalidateHandler) release() { + h.mu.Lock() + if h.users > 0 { + h.releaseLocked() + } + h.mu.Unlock() +} + +func (h *invalidateHandler) releaseLocked() { + h.users-- + if h.users == 0 { + h.cache = nil + h.keyPrefix = "" + } +} + +// sameCache compares Cache interface values without panicking when an +// implementation uses a non-comparable value type. +func sameCache(a, b Cache) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + typ := reflect.TypeOf(a) + return typ == reflect.TypeOf(b) && typ.Comparable() && a == b +} + +func isNilCache(cache Cache) bool { + if cache == nil { + return true + } + v := reflect.ValueOf(cache) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + +// errInvalidateHandlerBound: piggybacking on a handler bound to a live +// different cache would leave the new cache uninvalidated. +var errInvalidateHandlerBound = errors.New(`csc: a different "invalidate" push handler is already registered`) + +// bindTo binds the handler to (cache, keyPrefix). Success when that is already the +// binding (a derived Client.Conn sharing the parent's processor and cache) or +// when the handler was released by a previous owner's teardown (rebind); +// errInvalidateHandlerBound otherwise. +func (h *invalidateHandler) bindTo(cache Cache, keyPrefix string) error { + h.mu.Lock() + defer h.mu.Unlock() + switch { + case sameCache(h.cache, cache) && h.keyPrefix == keyPrefix: + h.users++ + return nil + case h.cache == nil: + h.cache, h.keyPrefix = cache, keyPrefix + h.users = 1 + return nil + default: + return errInvalidateHandlerBound + } +} + +// lookupInvalidateHandler returns the processor's CSC invalidate handler, nil +// when absent or foreign. +func lookupInvalidateHandler(p push.NotificationProcessor) *invalidateHandler { + if p == nil { + return nil + } + h, _ := p.GetHandler(invalidatePushName).(*invalidateHandler) + return h +} + +func registerInvalidateHandler(p push.NotificationProcessor, cache Cache, keyPrefix string) error { + if p == nil || cache == nil { + return nil + } + if existing := p.GetHandler(invalidatePushName); existing != nil { + h, ok := existing.(*invalidateHandler) + if !ok { + return errInvalidateHandlerBound + } + return h.bindTo(cache, keyPrefix) + } + // VoidProcessor (RESP2) returns an error here; the caller treats it as + // "CSC not available" rather than fatal. Registered PROTECTED: application + // code holding the processor must not be able to unregister invalidation + // under a live client (that would serve unbounded-stale hits with no + // signal); owner teardown releases the BINDING instead of the handler. + err := p.RegisterHandler(invalidatePushName, &invalidateHandler{ + cache: cache, + keyPrefix: keyPrefix, + users: 1, + }, true) + if err == nil { + return nil + } + // Another client can register the same protected handler between GetHandler + // and RegisterHandler. Re-read it and accept the compatible binding. + if existing := p.GetHandler(invalidatePushName); existing != nil { + h, ok := existing.(*invalidateHandler) + if !ok { + return errInvalidateHandlerBound + } + return h.bindTo(cache, keyPrefix) + } + return err +} + +// attachCSC dispatches to the invalidation strategy in +// Options.ClientSideCacheStrategy. Safe with a nil cache; on failure c.csc stays +// nil and commands fall back to normal round-trips. Adding a strategy: a new +// CSCStrategy constant plus cases in Options.init and here. +func (c *baseClient) attachCSC(ctx context.Context, cache Cache) { + if isNilCache(cache) || c.opt.Protocol != 3 { + return + } + // Credential providers may return a different ACL identity over the + // client's lifetime (or per context/connection), while the cache namespace + // is fixed when the client is created. Fixed credentials remain safe because + // the ACL username is included in the length-delimited namespace below. + if c.opt.StreamingCredentialsProvider != nil || + c.opt.CredentialsProviderContext != nil || + c.opt.CredentialsProvider != nil { + internal.Logger.Printf(ctx, + "redis: client-side caching is disabled with credential providers") + return + } + c.cscKeyPrefix = cscNamespacePrefix(c.opt.DB, c.opt.Username) + switch c.opt.ClientSideCacheStrategy { + case CSCStrategySharedTracking: + c.attachSharedTrackingCSC(ctx, cache) + default: + // Options.init clamps unknown strategies to SharedTracking; delegate anyway. + c.attachSharedTrackingCSC(ctx, cache) + } +} + +// attachSharedTrackingCSC wires SharedTracking: one shared cache, per-conn CLIENT +// TRACKING, a background drainer, and the owning-conn eviction hook. DB-0 only: +// tracking is bound to the conn's DB and a runtime SELECT does not re-key it. +func (c *baseClient) attachSharedTrackingCSC(ctx context.Context, cache Cache) { + if c.opt.DB != 0 { + internal.Logger.Printf(ctx, + "csc: client-side caching is restricted to DB 0; disabling CSC for client configured with DB=%d. "+ + "Use one client per DB if you need caching against non-zero databases.", c.opt.DB) + return + } + // A pooler without idle-conn draining (e.g. Client.Conn's StickyConnPool) + // can't apply buffered invalidations, so stay uncached. + if _, ok := c.connPool.(idleConnDrainer); !ok { + return + } + // The lifecycle hook serializes cache publication with connection removal + // and socket replacement. Without it, a reply can become visible after its + // tracking coverage is gone. + reg, ok := c.connPool.(poolHookSupport) + if !ok || !reg.SupportsPoolHooks() { + return + } + if err := registerInvalidateHandler(c.pushProcessor, cache, c.cscKeyPrefix); err != nil { + internal.Logger.Printf(ctx, "csc: failed to register invalidate handler: %v", err) + return + } + c.csc = cache + c.registerConnEvictHook(cache, reg) + c.startBackgroundDrainer() +} + +// cscHook returns the shared evict-on-remove hook, nil when CSC is off. +func (c *baseClient) cscHook() *cscEvictOnRemoveHook { + h, _ := c.cscPoolHook.(*cscEvictOnRemoveHook) + return h +} + +// cscInstallConnCloseHook evicts cn's owned entries on any close — including the +// ConnMaxLifetime/idle retirement path (CloseConn) that bypasses the OnRemove +// hook — so entries don't outlive the server tracking dropped at close. Uses the +// onCscClose slot so it doesn't clobber streaming-credentials cleanup. +func (c *baseClient) cscInstallConnCloseHook(cn *pool.Conn) { + cn.SetOnCscClose(func() error { + c.cscOnConnClose(cn.GetID()) + return nil + }) +} + +// cscInstallConnReinitHook invalidates the old socket's cache coverage before +// SetNetConnAndInitConn replaces it. The later init can then safely enable +// tracking for the new socket without a post-swap publication window. +func (c *baseClient) cscInstallConnReinitHook(cn *pool.Conn) { + cn.SetOnCscReinit(func() { + c.cscEvictOwnedEntries(cn.GetID()) + }) +} + +// cscOnConnClose evicts a closing conn's entries: via the shared hook (which +// records the removed-ring, closing the close-before-fulfill race), else scoped +// EvictByConn on the owning cache. +func (c *baseClient) cscOnConnClose(connID uint64) { + if h := c.cscHook(); h != nil { + h.markRemoved(connID) + return + } + if c.csc != nil { + c.csc.EvictByConn(connID) + } +} + +// poolHookSupport is the pool capability SharedTracking needs to serialize +// cache publication with connection removal and reinitialization. +type poolHookSupport interface { + AddPoolHook(hook pool.PoolHook) + RemovePoolHook(hook pool.PoolHook) + SupportsPoolHooks() bool +} + +// cscEvictOnRemoveHook evicts a connection's owned entries when the pool removes +// it (the server stops delivering their invalidations — Window 2), and tracks +// per-conn init generations so fulfillCached can catch a value whose owning +// conn was removed or re-initialized mid-fetch. +type cscEvictOnRemoveHook struct { + evictor Cache + + mu sync.Mutex + // initGen counts a live conn's socket (re)initializations: bumped by + // cscEvictOwnedEntries before its eviction (first init included, so every + // serving conn has gen >= 1), deleted on removal/close. fulfillCached + // compares it with the generation captured at reply time. + initGen map[uint64]uint64 +} + +func (h *cscEvictOnRemoveHook) OnGet(_ context.Context, _ *pool.Conn, _ bool) (bool, error) { + return true, nil +} + +func (h *cscEvictOnRemoveHook) OnPut(_ context.Context, _ *pool.Conn) (shouldPool, shouldRemove bool, err error) { + return true, false, nil +} + +func (h *cscEvictOnRemoveHook) OnRemove(_ context.Context, cn *pool.Conn, _ error) { + if cn == nil { + return + } + h.markRemoved(cn.GetID()) +} + +// markRemoved forgets connID's generation, then evicts. Forgetting before +// evicting lets a racing fulfillCached see the change (a served conn's captured +// generation is >= 1, an absent entry reads 0) and drop an entry created after +// the eviction — closing the close-before-fulfill race. +func (h *cscEvictOnRemoveHook) markRemoved(connID uint64) { + h.forgetConn(connID) + h.evictor.EvictByConn(connID) +} + +// bumpInitGen advances connID's coverage generation. On reinit it is called by +// the pre-swap hook, before the old socket and its server-side tracking table +// are replaced. +func (h *cscEvictOnRemoveHook) bumpInitGen(connID uint64) { + h.mu.Lock() + if h.initGen == nil { + h.initGen = make(map[uint64]uint64) + } + h.initGen[connID]++ + h.mu.Unlock() +} + +// invalidateConnCoverage revokes all cache coverage associated with connID. +// Bumping before eviction also rejects an in-flight fetch that completed on the +// connection just before it left the parent's invalidation drainer. +func (h *cscEvictOnRemoveHook) invalidateConnCoverage(connID uint64) { + h.bumpInitGen(connID) + h.evictor.EvictByConn(connID) +} + +// initGenOf returns connID's current init generation (0 if never bumped). +func (h *cscEvictOnRemoveHook) initGenOf(connID uint64) uint64 { + h.mu.Lock() + defer h.mu.Unlock() + return h.initGen[connID] +} + +// forgetConn drops connID's init-generation entry: the conn was removed/closed, +// or its init failed before ever serving (the pubsub path would otherwise leak +// the entry — no OnRemove hook, close hook not yet installed). +func (h *cscEvictOnRemoveHook) forgetConn(connID uint64) { + h.mu.Lock() + delete(h.initGen, connID) + h.mu.Unlock() +} + +// fulfillOwnedIfCovered linearizes the final coverage check with connection +// removal/re-init generation changes. Holding h.mu through FulfillOwned means +// either the old generation is rejected before the placeholder becomes valid, +// or publication wins first and the subsequent lifecycle path evicts it before +// closing/replacing the tracked socket. +func (h *cscEvictOnRemoveHook) fulfillOwnedIfCovered( + cacheKey string, + token, ownerConnID, capturedGen uint64, + value []byte, +) bool { + h.mu.Lock() + defer h.mu.Unlock() + if h.initGen[ownerConnID] != capturedGen { + return false + } + return h.evictor.FulfillOwned(cacheKey, token, ownerConnID, value) +} + +// invalidateAllCoverage revokes every connection generation known to this +// client's pool and evicts the entries those connections own. Incrementing +// instead of deleting keeps in-flight fetches that captured an old generation +// from publishing after a drainer stops. +func (h *cscEvictOnRemoveHook) invalidateAllCoverage() { + h.mu.Lock() + connIDs := make([]uint64, 0, len(h.initGen)) + for connID := range h.initGen { + h.initGen[connID]++ + connIDs = append(connIDs, connID) + } + h.mu.Unlock() + + for _, connID := range connIDs { + h.evictor.EvictByConn(connID) + } +} + +// registerConnEvictHook wires the required OnRemove eviction hook. +func (c *baseClient) registerConnEvictHook(cache Cache, reg poolHookSupport) { + h := &cscEvictOnRemoveHook{evictor: cache, initGen: make(map[uint64]uint64)} + reg.AddPoolHook(h) + c.cscPoolHook = h +} + +// cscEvictOwnedEntries evicts connID's entries on first init or immediately +// before a reinit/handoff replaces the socket and its tracking table. It +// prefers the shared hook (so Conn/Tx, which carry it but have a nil csc, still +// evict from the parent cache). Scoped only — no removed-ring (the conn keeps +// serving, and the ring never ages out); the fulfill-vs-re-init race is closed +// by the init-generation bump instead. No custom-cache flush (this also runs on +// first init). +func (c *baseClient) cscEvictOwnedEntries(connID uint64) { + if h := c.cscHook(); h != nil { + h.invalidateConnCoverage(connID) + return + } + if c.csc == nil { + return + } + c.csc.EvictByConn(connID) +} + +// newStickyConnPool creates a derived sticky pool and revokes the claimed +// connection's parent-cache ownership before it becomes unreachable to the +// parent's idle-connection drainer. +func (c *baseClient) newStickyConnPool() *pool.StickyConnPool { + sticky := pool.NewStickyConnPool(c.connPool) + if h := c.cscHook(); h != nil { + sticky.SetOnFirstConn(func(cn *pool.Conn) { + if cn != nil { + h.invalidateConnCoverage(cn.GetID()) + } + }) + } + return sticky +} + +// cscFetchCapture receives, from the successful attempt's reply read — while +// the serving connection is still held — everything the CSC fetch path needs to +// attribute the cached entry: the raw RESP reply, the conn id, and the conn's +// CSC init generation. The generation must be captured before the conn is +// released: a handoff queued at Put can re-init the socket (bumping the +// generation) before fulfillCached runs. +type cscFetchCapture struct { + raw []byte + connID uint64 + initGen uint64 +} + +// cscConnInitGen returns connID's CSC init generation, captured by _process at +// reply time (while the conn is still held) and compared by fulfillCached via +// fulfillOwnedIfCovered. Zero without an active evict-on-remove hook. +func (c *baseClient) cscConnInitGen(connID uint64) uint64 { + if h := c.cscHook(); h != nil { + return h.initGenOf(connID) + } + return 0 +} + +// cscForgetConn drops connID's init-generation entry when initialization does +// not establish tracked coverage, either because init failed or tracking was +// rejected and CSC was disabled. +func (c *baseClient) cscForgetConn(connID uint64) { + if h := c.cscHook(); h != nil { + h.forgetConn(connID) + } +} + +// errClientTrackingWithCSC rejects CLIENT TRACKING on clients with built-in CSC +// (see the guards in baseClient.process and generalProcessPipeline). The raw +// escape hatches — Do(ctx, "client", "tracking", ...) with string or []byte +// args, and pipelines — are also caught: the guard matches on the command's +// leading args, not the typed method. +var errClientTrackingWithCSC = errors.New( + "redis: CLIENT TRACKING is not allowed when client-side caching is enabled") + +// errSelectWithCSC rejects runtime SELECT on clients with built-in CSC. Cache +// keys use Options.DB, while SELECT mutates only the chosen pool connection. +var errSelectWithCSC = errors.New( + "redis: SELECT is not allowed when client-side caching is enabled") + +// errAuthWithCSC rejects runtime authentication because it can change one +// connection's ACL identity without changing the client's fixed cache namespace. +var errAuthWithCSC = errors.New( + "redis: AUTH is not allowed when client-side caching is enabled") + +// errHelloWithCSC rejects HELLO with arguments because it can switch a tracked +// connection out of RESP3 (and can also change authentication). +var errHelloWithCSC = errors.New( + "redis: HELLO with arguments is not allowed when client-side caching is enabled") + +// errResetWithCSC rejects RESET because it disables tracking and switches the +// connection to RESP2. +var errResetWithCSC = errors.New( + "redis: RESET is not allowed when client-side caching is enabled") + +// errSubscribeWithCSC rejects raw subscriptions on the ordinary pool. The +// typed Subscribe methods use dedicated PubSub connections and remain allowed. +var errSubscribeWithCSC = errors.New( + "redis: SUBSCRIBE is not allowed on pooled connections when client-side caching is enabled") + +// cscCommandError rejects commands that can make a pooled connection's state +// diverge from the assumptions used by CSC. +func (c *baseClient) cscCommandError(cmd Cmder) error { + // The successful attachment signal is shared with derived clients. + // initConn's internal command wrapper is exempt during library setup. + if !c.cscTrackingRequested() || c.allowClientTracking { + return nil + } + switch { + case isClientTrackingCmd(cmd): + return errClientTrackingWithCSC + case isSelectCmd(cmd): + return errSelectWithCSC + case isAuthCmd(cmd): + return errAuthWithCSC + case isProtocolChangingHelloCmd(cmd): + return errHelloWithCSC + case isResetCmd(cmd): + return errResetWithCSC + case isSubscribeCmd(cmd): + return errSubscribeWithCSC + default: + return nil + } +} + +// cscDrainHandle owns the drainer lifecycle and serializes client teardown. +// stop signals shutdown; done is closed on exit so Close can join. +type cscDrainHandle struct { + stop chan struct{} + done chan struct{} + stopOnce sync.Once + teardownOnce sync.Once + handlerCloseOnce sync.Once + closeOnce sync.Once + closeErr error + invalidateHandler *invalidateHandler +} + +// signalStop closes stop at most once (so Close and the AddCleanup safety net +// can't double-close) and does not join — a GC cleanup must not block. +func (h *cscDrainHandle) signalStop() { + h.stopOnce.Do(func() { close(h.stop) }) +} + +// cscHandlerClient is exposed only through the background drainer's handler +// context. Close must return before the handler does, otherwise it would wait +// for the drainer goroutine that is currently invoking the handler. +type cscHandlerClient struct { + *baseClient +} + +func (c cscHandlerClient) Close() error { + h := c.cscDrainHandle + if h == nil { + return c.baseClient.Close() + } + h.handlerCloseOnce.Do(func() { + // Close has logically started: stop cache hits immediately and let the + // drainer exit as soon as this handler returns. + if c.cscActive != nil { + c.cscActive.Store(false) + } + h.signalStop() + go func() { + if err := c.baseClient.Close(); err != nil { + internal.Logger.Printf(context.Background(), "csc: deferred client close failed: %v", err) + } + }() + }) + return nil +} + +// cscMinDrainInterval floors a user-supplied DrainInterval: sub-millisecond +// timers are unreliable (https://github.com/golang/go/issues/53824). +const cscMinDrainInterval = time.Millisecond + +// cscDrainInterval returns DrainInterval clamped to cscMinDrainInterval, or the +// default (cscDrainSkipWindow) when unset. +func (c *baseClient) cscDrainInterval() time.Duration { + if cfg := c.opt.ClientSideCacheConfig; cfg != nil && cfg.DrainInterval > 0 { + if cfg.DrainInterval < cscMinDrainInterval { + return cscMinDrainInterval + } + return cfg.DrainInterval + } + return cscDrainSkipWindow +} + +// idleConnDrainer is the pooler capability the drainer needs (*pool.ConnPool has +// it). attachSharedTrackingCSC leaves a pooler without it uncached, rather than +// serve entries nothing would invalidate. +type idleConnDrainer interface { + DrainIdleConns(ctx context.Context, st *pool.DrainState, fn func(cn *pool.Conn) error) +} + +// startBackgroundDrainer launches the per-client invalidation drainer: each tick +// runs one pool.DrainIdleConns pass, draining idle conns' buffered push frames. +// No-op for poolers that don't implement idleConnDrainer. +func (c *baseClient) startBackgroundDrainer() { + cp, ok := c.connPool.(idleConnDrainer) + if !ok { + return + } + if c.cscDrainHandle != nil { + return // already running (startBackgroundDrainer runs once, in NewClient) + } + h := &cscDrainHandle{ + stop: make(chan struct{}), + done: make(chan struct{}), + invalidateHandler: lookupInvalidateHandler(c.pushProcessor), + } + c.cscDrainHandle = h + active := &atomic.Bool{} + active.Store(true) + c.cscActive = active + interval := c.cscDrainInterval() + // Custom-processor drain errors are connection-fatal (drainPushNotifications), + // so a PERSISTENTLY failing custom processor would turn every tick into a + // conn removal + redial — a sustained dial storm. Damping: after + // cscDrainCustomErrCap consecutive fatal custom-processor drains, disable + // CSC serving and stop the drainer (with one log line) instead of churning. + // Built-in processor errors are real conn desyncs and are never damped. + _, builtinProc := c.pushProcessor.(*push.Processor) + go func() { + defer func() { + active.Store(false) + if c.cscPoolHook != nil { + if reg, ok := c.connPool.(poolHookSupport); ok { + reg.RemovePoolHook(c.cscPoolHook) + } + } + if hook := c.cscHook(); hook != nil { + hook.invalidateAllCoverage() + } + if h.invalidateHandler != nil { + h.invalidateHandler.release() + } + close(h.done) + }() + ticker := time.NewTicker(interval) + defer ticker.Stop() + // st persists round/visited across ticks; single-goroutine, no lock. + var st pool.DrainState + consecFatal := 0 + drain := func(cn *pool.Conn) error { + processorSucceeded, err := c.drainPushNotifications(cn) + switch { + case err != nil: + consecFatal++ + case processorSucceeded: + // A successful processor invocation resets consecutive + // failures. A conn skipped without invoking the processor — + // including a clean replacement after a fatal drain — does + // not reset the counter. + consecFatal = 0 + } + return err + } + for { + select { + case <-h.stop: + return + case <-ticker.C: + if !active.Load() { + return + } + // ctx bounds the whole pass; the drain read has its own hard deadline. + cycleCtx, cancel := context.WithTimeout(context.Background(), interval/2) + cp.DrainIdleConns(cycleCtx, &st, drain) + cancel() + if !builtinProc && consecFatal >= cscDrainCustomErrCap { + internal.Logger.Printf(context.Background(), + "csc: disabling client-side caching: the custom push notification processor failed %d consecutive drains "+ + "(each failure removes a connection because the reader may be mid-frame); "+ + "caching cannot be kept fresh safely with this processor", consecFatal) + return + } + } + } + }() +} + +// disableCSCServing atomically stops cache hits and revokes all tracked +// connection coverage. The owner drainer observes the shared active flag on its +// next tick, including when a derived Conn or Tx discovered the incompatibility. +func (c *baseClient) disableCSCServing(ctx context.Context, reason string) { + active := c.cscActive + if active == nil || !active.CompareAndSwap(true, false) { + return + } + if hook := c.cscHook(); hook != nil { + hook.invalidateAllCoverage() + } + internal.Logger.Printf(ctx, "csc: disabling client-side caching: %s", reason) +} + +// stopBackgroundDrainer joins the drainer goroutine and flushes an owned cache. +// The drainer's exit path releases its handler binding and pool hook, including +// when it stops itself. Owner-only: clones have no handle and return early. +// The fields are never cleared here — fulfillCached reads cscPoolHook on the hot +// path, so niling under a concurrent Close would race; teardownOnce makes repeat +// Close idempotent instead. +func (c *baseClient) stopBackgroundDrainer() { + h := c.cscDrainHandle + if h == nil { + return + } + h.teardownOnce.Do(func() { + // Stop serving cache hits on any clone before the drainer is gone. + if c.cscActive != nil { + c.cscActive.Store(false) + } + h.signalStop() + <-h.done + // The drainer's exit defer revoked and evicted this pool's coverage + // before closing done, including for injected caches shared elsewhere. + if c.cscOwnsCache && c.csc != nil { + c.csc.Flush() + } + }) +} + +// applyCachedReply populates cmd from a previously captured raw RESP reply by +// replaying it through the command's own readReply. +func applyCachedReply(cmd Cmder, raw []byte) error { + return cmd.readReply(proto.NewReaderSize(bytes.NewReader(raw), len(raw)+1)) +} + +// isCacheableReplyResult reports whether a fully read Redis reply can be +// cached. redis.Nil is a normal negative lookup, not a transport/protocol +// failure; tracking will invalidate it if the key is later created. +func isCacheableReplyResult(err error) bool { + return err == nil || err == Nil +} + +// cscDrainSkipWindow is the default SharedTracking drain period (overridable via +// ClientSideCacheConfig.DrainInterval). A buffered invalidation is picked up within +// roughly one round; MaxStaleness, when configured, is the hard time-based backstop. +const cscDrainSkipWindow = 5 * time.Millisecond + +// cscDrainHardReadCap is the hard socket read deadline the drainer applies via +// Conn.WithReaderHardDeadline. It bounds only a rare partial-frame mid-read. A +// var (not const) so the tuning harness can sweep it. +var cscDrainHardReadCap = 50 * time.Millisecond + +// cscDrainProbeReadCap bounds the non-consuming one-byte probe used only when +// an opaque transport may hold data that the socket readiness check cannot see. +const cscDrainProbeReadCap = 50 * time.Microsecond + +// cscDrainCustomErrCap is the number of CONSECUTIVE fatal custom-processor +// drain errors after which the drainer disables CSC instead of removing (and +// redialing) a connection per tick indefinitely. +const cscDrainCustomErrCap = 8 + +// processCached runs the Get-Reserve-Fulfill lifecycle for a cacheable command. +// Only invoked after process has verified that CSC is active and cmd is +// eligible. +func (c *baseClient) processCached(ctx context.Context, cmd Cmder, state *processState) error { + if err := ctx.Err(); err != nil { + return err + } + + // Once the drainer has stopped (owner Close, or the owner dropped without + // Close), no invalidations flow — a surviving clone must not serve stale hits. + if a := c.cscActive; a != nil && !a.Load() { + return c.processWithRetry(ctx, cmd, nil, state) + } + + rawKey, ok := buildCacheKey(cmd) + if !ok { + return c.processWithRetry(ctx, cmd, nil, state) + } + + redisKeys := extractRedisKeys(cmd) + if len(redisKeys) == 0 { + // Without a key list we cannot react to invalidations for this command. + return c.processWithRetry(ctx, cmd, nil, state) + } + + keyPrefix := c.cscKeyPrefix + if keyPrefix == "" { + // A successfully attached client always has a namespace. Fail closed if + // an incomplete custom baseClient reaches this path. + return c.processWithRetry(ctx, cmd, nil, state) + } + key := cscNamespacedKey(keyPrefix, rawKey) + nsRedisKeys := make([]string, len(redisKeys)) + for i, k := range redisKeys { + nsRedisKeys[i] = cscNamespacedKey(keyPrefix, k) + } + + // Serve hits straight from the cache. + if data, ok := c.csc.Get(ctx, key); ok { + if err := ctx.Err(); err != nil { + return err + } + if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) { + return err + } + c.csc.DeleteByCacheKey(key) + } + + token, shouldFetch := c.csc.Reserve(key, nsRedisKeys) + if !shouldFetch { + // Another goroutine is fetching; Get below waits until it completes. + if data, ok := c.csc.Get(ctx, key); ok { + if err := ctx.Err(); err != nil { + return err + } + if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) { + return err + } + c.csc.DeleteByCacheKey(key) + } + // Original fetcher cancelled or its value was invalidated; try to take + // over so later waiters still benefit from the cache. + token, shouldFetch = c.csc.Reserve(key, nsRedisKeys) + } + + var fc cscFetchCapture + var capture *cscFetchCapture + if shouldFetch { + capture = &fc + // Release the placeholder if processWithRetry panics; Cancel on a + // stale token is a no-op. + defer func() { + if capture != nil { + c.csc.Cancel(key, token) + } + }() + } + + err := c.processWithRetry(ctx, cmd, capture, state) + + if shouldFetch { + capture = nil // disarm the deferred Cancel + if isCacheableReplyResult(err) { + c.fulfillCached(key, token, &fc) + } else { + c.csc.Cancel(key, token) + } + } + return err +} + +// fulfillCached stores a fetched value, attributing it to its serving conn when +// an evict-on-remove hook is active so EvictByConn can drop it if that conn is +// removed. It also closes the attribute-vs-coverage races: the conn is released +// before this runs, so its OnRemove eviction — or a handoff re-init's scoped +// eviction — may fire before the entry exists. Publication is serialized with +// the hook's init-generation changes, so a reply whose invalidation coverage +// was already lost never becomes visible and never wakes waiters with stale +// data. +func (c *baseClient) fulfillCached(key string, token uint64, fc *cscFetchCapture) bool { + if active := c.cscActive; active != nil && !active.Load() { + c.csc.Cancel(key, token) + return false + } + if hook := c.cscHook(); hook != nil { + if fc.connID == 0 { + // Invariant: an active hook always gets a real conn id (>=1). A zero id + // would leave the entry unattributed and un-evictable, so fail closed. + c.csc.Cancel(key, token) + return false + } + if !hook.fulfillOwnedIfCovered(key, token, fc.connID, fc.initGen, fc.raw) { + // A coverage mismatch leaves the reservation IN_PROGRESS because + // FulfillOwned was deliberately skipped. Cancel wakes its waiters + // as misses so one can safely refetch on a covered connection. + c.csc.Cancel(key, token) + return false + } + return true + } + return c.csc.FulfillOwned(key, token, 0, fc.raw) +} diff --git a/vendor/github.com/redis/go-redis/v9/csc_stats.go b/vendor/github.com/redis/go-redis/v9/csc_stats.go new file mode 100644 index 00000000..d14d3fcf --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/csc_stats.go @@ -0,0 +1,36 @@ +package redis + +// CSCStats reports cumulative client-side cache activity and current +// residency. +// +// Experimental: this API may change in a minor release. +type CSCStats struct { + Hits uint64 + Misses uint64 + Entries int + MemoryUsageBytes int64 +} + +// cacheStatsReporter is an optional interface a Cache implementation may +// satisfy to expose statistics. The built-in LocalCache does; user +// implementations are not required to. +type cacheStatsReporter interface { + Stats() CSCStats +} + +// CSCStats returns statistics for this client's client-side cache, read from +// the shared cache when its implementation exposes them (the built-in +// LocalCache does). +// +// It returns a zero value when CSC is not configured or stats are unavailable. +// +// Experimental: this API may change in a minor release. +func (c *Client) CSCStats() CSCStats { + if c == nil || c.baseClient.csc == nil { + return CSCStats{} + } + if r, ok := c.baseClient.csc.(cacheStatsReporter); ok { + return r.Stats() + } + return CSCStats{} +} diff --git a/vendor/github.com/redis/go-redis/v9/docker-compose.yml b/vendor/github.com/redis/go-redis/v9/docker-compose.yml index fed908be..de9683a6 100644 --- a/vendor/github.com/redis/go-redis/v9/docker-compose.yml +++ b/vendor/github.com/redis/go-redis/v9/docker-compose.yml @@ -1,6 +1,6 @@ --- -x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.8.0} +x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.10.0} services: redis: diff --git a/vendor/github.com/redis/go-redis/v9/error.go b/vendor/github.com/redis/go-redis/v9/error.go index 06ecca74..ad3a81b7 100644 --- a/vendor/github.com/redis/go-redis/v9/error.go +++ b/vendor/github.com/redis/go-redis/v9/error.go @@ -168,6 +168,9 @@ func shouldRetry(err error, retryTimeout bool) bool { return true } + // Other server errors are not retried. This includes the logical + // -SEARCH_TIMEOUT (search-on-timeout fail): retrying would just repeat the + // same expensive query. return false } diff --git a/vendor/github.com/redis/go-redis/v9/generic_commands.go b/vendor/github.com/redis/go-redis/v9/generic_commands.go index c7100222..6d6d8e77 100644 --- a/vendor/github.com/redis/go-redis/v9/generic_commands.go +++ b/vendor/github.com/redis/go-redis/v9/generic_commands.go @@ -129,6 +129,9 @@ func (c cmdable) ExpireAt(ctx context.Context, key string, tm time.Time) *BoolCm return cmd } +// ExpireTime returns the absolute expiration time of key as a Unix timestamp +// encoded in *DurationCmd (seconds since the epoch), not a remaining TTL. +// Convert with: time.Unix(int64(d/time.Second), 0). Use TTL/PTTL for remaining TTL. func (c cmdable) ExpireTime(ctx context.Context, key string) *DurationCmd { cmd := NewDurationCmd(ctx, time.Second, "expiretime", key) _ = c(ctx, cmd) @@ -209,6 +212,9 @@ func (c cmdable) PExpireAt(ctx context.Context, key string, tm time.Time) *BoolC return cmd } +// PExpireTime returns the absolute expiration time of key as a Unix timestamp +// encoded in *DurationCmd (milliseconds since the epoch), not a remaining TTL. +// Convert with: time.UnixMilli(int64(d/time.Millisecond)). Use TTL/PTTL for remaining TTL. func (c cmdable) PExpireTime(ctx context.Context, key string) *DurationCmd { cmd := NewDurationCmd(ctx, time.Millisecond, "pexpiretime", key) _ = c(ctx, cmd) diff --git a/vendor/github.com/redis/go-redis/v9/hash_commands.go b/vendor/github.com/redis/go-redis/v9/hash_commands.go index 256b8746..3174eab7 100644 --- a/vendor/github.com/redis/go-redis/v9/hash_commands.go +++ b/vendor/github.com/redis/go-redis/v9/hash_commands.go @@ -44,6 +44,11 @@ type HashCmdable interface { HPExpireTime(ctx context.Context, key string, fields ...string) *IntSliceCmd HTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd HPTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd + // note: the HIMPORT API is experimental and may be subject to change. + HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd + HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd + HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd + HImportDiscardAll(ctx context.Context) *IntCmd } func (c cmdable) HDel(ctx context.Context, key string, fields ...string) *IntCmd { diff --git a/vendor/github.com/redis/go-redis/v9/himport.go b/vendor/github.com/redis/go-redis/v9/himport.go new file mode 100644 index 00000000..a1d9125a --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/himport.go @@ -0,0 +1,473 @@ +package redis + +import ( + "context" + "strings" + "sync" + + "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" +) + +// himportFieldset is the client-side record of a fieldset registered with +// HImportPrepare. +type himportFieldset struct { + fields []string + version uint64 +} + +// himportRegistry remembers fieldsets registered through a client so HIMPORT +// SET can lazily prepare them on whichever pooled connection it executes. +// Versions increase monotonically and start at 1; re-registering a name under +// a new version invalidates every connection's prepared flag for it, so a +// replaced fieldset is re-prepared before its next use. +// +// Discards propagate lazily as well: a discarded name is kept as a tombstone +// and the discard-all counter as an epoch, and connections whose sessions +// still hold discarded fieldsets replay HIMPORT DISCARD/DISCARDALL before +// their next HIMPORT command (see baseClient.himportInjectedCmds). +type himportRegistry struct { + mu sync.RWMutex + nextVersion uint64 + fieldsets map[string]himportFieldset + // tombstones holds names discarded through this client whose server-side + // copies may survive on pooled connections that prepared them. An entry + // is removed when the name is registered again (the new version replaces + // the fieldset on the server, so no discard is needed) or by discardAll. + // Known limitation: a workload discarding many uniquely-named fieldsets + // grows this map for the client's lifetime and pays an O(tombstones) + // snapshot per HIMPORT round trip; HImportDiscardAll resets it. + tombstones map[string]struct{} + // discardAllEpoch increments on every successful HImportDiscardAll. + discardAllEpoch uint64 +} + +func newHImportRegistry() *himportRegistry { + return &himportRegistry{} +} + +// register stores the fieldset and returns its new version together with the +// current discard-all epoch. +func (r *himportRegistry) register(name string, fields []string) (version, epoch uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.fieldsets == nil { + r.fieldsets = make(map[string]himportFieldset) + } + delete(r.tombstones, name) + r.nextVersion++ + r.fieldsets[name] = himportFieldset{ + fields: append([]string(nil), fields...), + version: r.nextVersion, + } + return r.nextVersion, r.discardAllEpoch +} + +func (r *himportRegistry) lookup(name string) (himportFieldset, bool) { + if r == nil { + return himportFieldset{}, false + } + r.mu.RLock() + fs, ok := r.fieldsets[name] + r.mu.RUnlock() + return fs, ok +} + +// discard removes the fieldset and leaves a tombstone so connections whose +// sessions still hold it replay the DISCARD before their next HIMPORT +// command. It reports whether the fieldset was registered. +func (r *himportRegistry) discard(name string) bool { + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.fieldsets[name]; !ok { + return false + } + delete(r.fieldsets, name) + if r.tombstones == nil { + r.tombstones = make(map[string]struct{}) + } + r.tombstones[name] = struct{}{} + return true +} + +// discardAll drops every fieldset and tombstone and moves to a new epoch; +// connections prepared under an older epoch replay HIMPORT DISCARDALL before +// their next HIMPORT command. It returns the new epoch and the number of +// fieldsets that were registered. +func (r *himportRegistry) discardAll() (epoch uint64, removed int) { + r.mu.Lock() + removed = len(r.fieldsets) + r.fieldsets = nil + r.tombstones = nil + r.discardAllEpoch++ + epoch = r.discardAllEpoch + r.mu.Unlock() + return epoch, removed +} + +// discardVersion withdraws a registration whose fan-out PREPARE was rejected +// by a server — but only while the entry is still at that version, so a +// concurrent re-registration is not clobbered. A tombstone is left: the +// fan-out may have succeeded on some masters before another rejected it +// (per-node ACLs, rolling upgrades), and those sessions hold the withdrawn +// fieldset; the tombstone makes their next HIMPORT command discard it +// instead of leaving a fieldset the client can no longer address. +func (r *himportRegistry) discardVersion(name string, version uint64) { + r.mu.Lock() + if fs, ok := r.fieldsets[name]; ok && fs.version == version { + delete(r.fieldsets, name) + if r.tombstones == nil { + r.tombstones = make(map[string]struct{}) + } + r.tombstones[name] = struct{}{} + } + r.mu.Unlock() +} + +// refreshVersion bumps a registered fieldset to a new version, keeping its +// fields — but only while the entry is still at the given version, so a +// concurrent re-registration is not disturbed. Every connection's prepared +// flag becomes stale, forcing a re-prepare before the fieldset's next use on +// each of them. Used when a "no such fieldset" reply signals session loss +// that may have hit more connections than the one that reported it (failover, +// cross-region switch, reset storms). +func (r *himportRegistry) refreshVersion(name string, version uint64) { + r.mu.Lock() + if fs, ok := r.fieldsets[name]; ok && fs.version == version { + r.nextVersion++ + fs.version = r.nextVersion + r.fieldsets[name] = fs + } + r.mu.Unlock() +} + +// idle reports whether the registry implies no injection work at all: no +// fieldsets to replay, no tombstones to discard, and no discard-all epoch a +// session could be behind. +func (r *himportRegistry) idle() bool { + if r == nil { + return true + } + r.mu.RLock() + idle := len(r.fieldsets) == 0 && len(r.tombstones) == 0 && r.discardAllEpoch == 0 + r.mu.RUnlock() + return idle +} + +// cleanupSnapshot returns the current epoch and the tombstoned names. +func (r *himportRegistry) cleanupSnapshot() (epoch uint64, tombstones []string) { + r.mu.RLock() + epoch = r.discardAllEpoch + if len(r.tombstones) > 0 { + tombstones = make([]string, 0, len(r.tombstones)) + for name := range r.tombstones { + tombstones = append(tombstones, name) + } + } + r.mu.RUnlock() + return epoch, tombstones +} + +// himportNoSuchFieldset reports whether err is the server's "no such +// fieldset" reply, i.e. an HIMPORT SET executed on a connection whose session +// does not hold the referenced fieldset. +func himportNoSuchFieldset(err error) bool { + return isRedisError(err) && strings.Contains(err.Error(), "no such fieldset") +} + +// himportInjectedCmds returns the HIMPORT commands to write to cn ahead of a +// batch, in order: +// +// 1. HIMPORT DISCARDALL when cn's session was prepared under an older +// discard-all epoch; +// 2. HIMPORT DISCARD for each discarded fieldset the session still holds; +// 3. HIMPORT PREPARE for each registered fieldset referenced by an HIMPORT +// SET in the batch that the session lacks at the current version. +// +// A fieldset covered by a user-issued PREPARE earlier in the batch needs no +// injection — the server session holds it by the time the SET runs. Returns +// nil when the batch contains no HIMPORT commands: sessions holding only +// discarded fieldsets are cleaned up on their next HIMPORT use, not on +// unrelated traffic. +func (c *baseClient) himportInjectedCmds(ctx context.Context, cn *pool.Conn, cmds []Cmder) []Cmder { + if c.himport.idle() { + return nil + } + hasHImport := false + for _, cmd := range cmds { + if _, ok := cmd.(himportCmder); ok { + hasHImport = true + break + } + } + if !hasHImport { + return nil + } + + var injected []Cmder + + // Discards first: a session behind the discard-all epoch is wiped + // entirely; otherwise individual tombstoned fieldsets it still holds are + // discarded. + epoch, tombstones := c.himport.cleanupSnapshot() + sessionWiped := false + if cn.HasPreparedFieldsets() && cn.FieldsetEpoch() != epoch { + da := NewHImportDiscardAllCmd(ctx) + da.registryEpoch = epoch + injected = append(injected, da) + sessionWiped = true + } else { + for _, name := range tombstones { + if cn.FieldsetPreparedVersion(name) != 0 { + injected = append(injected, NewHImportDiscardCmd(ctx, name)) + } + } + } + + // Prepares for registered fieldsets the batch's SETs reference. + var covered map[string]struct{} + cover := func(name string) { + if covered == nil { + covered = make(map[string]struct{}) + } + covered[name] = struct{}{} + } + for _, cmd := range cmds { + switch hc := cmd.(type) { + case *HImportPrepareCmd: + cover(hc.fieldsetName) + case *HImportSetCmd: + if _, ok := covered[hc.fieldsetName]; ok { + continue + } + fs, ok := c.himport.lookup(hc.fieldsetName) + if !ok { + continue + } + if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) == fs.version { + continue + } + // The session holds an older version. Discard it before the + // re-prepare: the SET behind it is already on the wire, and if + // the re-prepare fails the SET must answer "no such fieldset" + // rather than silently writing the old version's field names. + if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) != 0 { + injected = append(injected, NewHImportDiscardCmd(ctx, hc.fieldsetName)) + } + prep := NewHImportPrepareCmd(ctx, hc.fieldsetName, fs.fields...) + prep.registryVersion = fs.version + prep.registryEpoch = epoch + injected = append(injected, prep) + cover(hc.fieldsetName) + } + } + return injected +} + +// himportReadInjectedReplies consumes the replies of injected HIMPORT +// commands. Server errors are recorded on the command and the connection is +// left readable; transport errors are returned. Successful commands apply +// their prepared-flag bookkeeping on cn. +func (c *baseClient) himportReadInjectedReplies(ctx context.Context, cn *pool.Conn, rd *proto.Reader, injected []Cmder) error { + for _, cmd := range injected { + if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } + err := cmd.readReply(rd) + cmd.SetErr(err) + if err != nil { + if !isRedisError(err) { + return err + } + // A failed injected PREPARE becomes the root cause of the + // dependent SETs' errors downstream; a failed injected discard + // only delays cleanup until the next HIMPORT command. + internal.Logger.Printf(ctx, "himport: injected %s failed: %v", cmd.Name(), err) + continue + } + switch hc := cmd.(type) { + case *HImportPrepareCmd: + cn.MarkFieldsetPrepared(hc.fieldsetName, hc.registryVersion, hc.registryEpoch) + case *HImportDiscardCmd: + cn.UnmarkFieldsetPrepared(hc.fieldsetName) + case *HImportDiscardAllCmd: + cn.ClearPreparedFieldsets(hc.registryEpoch) + } + } + return nil +} + +// himportAfterCmd applies registry and prepared-flag updates after a +// user-issued HIMPORT command completed successfully on cn. +func (c *baseClient) himportAfterCmd(cn *pool.Conn, hc himportCmder) { + if c.himport == nil { + return + } + switch cmd := hc.(type) { + case *HImportPrepareCmd: + version, epoch := cmd.registryVersion, cmd.registryEpoch + if version == 0 { + version, epoch = c.himport.register(cmd.fieldsetName, cmd.fields) + } + // A pre-assigned version marks a fan-out copy: the fieldset was + // registered once at the cluster/ring level; only mark the + // executing connection. + cn.MarkFieldsetPrepared(cmd.fieldsetName, version, epoch) + case *HImportDiscardCmd: + registered := c.himport.discard(cmd.fieldsetName) + cn.UnmarkFieldsetPrepared(cmd.fieldsetName) + // The managed API reports the registry lifecycle: 1 when the + // fieldset was registered on this client and is now removed. The + // executing connection's session count stands only for fieldsets + // the registry never knew (raw usage). + if registered { + cmd.SetVal(1) + } + case *HImportDiscardAllCmd: + // A pre-assigned epoch marks a fan-out copy: the registry was + // already wiped at the cluster/ring level; only move the executing + // connection to that epoch. + if cmd.registryEpoch != 0 { + cn.ClearPreparedFieldsets(cmd.registryEpoch) + return + } + epoch, removed := c.himport.discardAll() + cn.ClearPreparedFieldsets(epoch) + // Same registry semantics: report how many registered fieldsets + // were removed, not how many the executing session happened to + // hold. + if removed > 0 { + cmd.SetVal(int64(removed)) + } + } +} + +// himportAfterBatch runs after all replies of a batch were read: it surfaces +// an injected PREPARE failure as the root cause on the HIMPORT SET commands +// that depended on it (their own reply is the secondary "no such fieldset" +// error), invalidates stale prepared flags for SETs that found their +// registered fieldset missing server-side, and applies registry updates for +// user-issued HIMPORT commands that succeeded in the batch. +// rawErr throughout: this runs on the execution path, before an async +// autopipeline batch completes (its ready channel closes only after the +// pipeline hook chain returns) — Err() on a user command would await and +// self-deadlock the dispatcher. +func (c *baseClient) himportAfterBatch(cn *pool.Conn, injected []Cmder, cmds []Cmder) { + var failed map[string]error + var refreshed map[string]struct{} + for _, cmd := range injected { + if prep, ok := cmd.(*HImportPrepareCmd); ok { + if err := prep.Err(); err != nil { + if failed == nil { + failed = make(map[string]error) + } + failed[prep.fieldsetName] = err + } + } + } + for _, cmd := range cmds { + hc, ok := cmd.(himportCmder) + if !ok { + continue + } + if set, ok := hc.(*HImportSetCmd); ok { + if rootCause, ok := failed[set.fieldsetName]; ok && himportNoSuchFieldset(set.rawErr()) { + set.SetErr(rootCause) + continue + } + // The session lost a fieldset the flags claim is prepared (e.g. + // RESET) — and the same event may have wiped other sessions + // whose flags also still look current. Bump the fieldset + // version once so the SET's re-issue, the cluster re-queue on + // whichever connection it lands, or the caller's transaction + // retry replays the PREPARE. + if himportNoSuchFieldset(set.rawErr()) { + if _, done := refreshed[set.fieldsetName]; !done { + if refreshed == nil { + refreshed = make(map[string]struct{}) + } + refreshed[set.fieldsetName] = struct{}{} + if fs, registered := c.himport.lookup(set.fieldsetName); registered { + c.himport.refreshVersion(set.fieldsetName, fs.version) + } + } + } + continue + } + if hc.rawErr() == nil { + c.himportAfterCmd(cn, hc) + } + } +} + +// himportRetryFailedSets re-issues, once, the HIMPORT SET commands of a +// pipeline batch that failed with "no such fieldset" while their fieldset is +// registered — the error must not surface for managed fieldsets (NF.4). Only +// the SETs are re-sent: HIMPORT SET is a full replace, so re-execution is +// idempotent, and no other command of the batch runs again. Their prepared +// flags were invalidated by himportAfterBatch, so himportInjectedCmds +// regenerates the PREPAREs for this connection. Transport errors are +// returned; server errors stay recorded on the commands. +// (The retry does not carry an ASKING prefix. A redirected [ASKING, SET] +// pair whose injected PREPARE failed is excluded by the root-cause swap in +// himportAfterBatch; one that lost its session without an injection can be +// re-issued here, and the bare SET then draws a fresh MOVED/ASK that the +// outer cluster redirect handling resolves.) +func (c *baseClient) himportRetryFailedSets(ctx context.Context, cn *pool.Conn, cmds []Cmder) error { + if c.himport.idle() { + return nil + } + var retry []Cmder + for _, cmd := range cmds { + // rawErr: execution path, same self-deadlock rule as himportAfterBatch. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) { + if _, registered := c.himport.lookup(set.fieldsetName); registered { + retry = append(retry, set) + } + } + } + if len(retry) == 0 { + return nil + } + + injected := c.himportInjectedCmds(ctx, cn, retry) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } + return writeCmds(wr, retry) + }); err != nil { + return err + } + return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + return err + } + err := c.pipelineReadCmds(ctx, cn, rd, retry) + if err != nil && !isRedisError(err) { + return err + } + // Server errors (including a repeated failure) stay on the + // individual commands; the batch as a whole is done. + c.himportAfterBatch(cn, injected, retry) + return nil + }) +} + +// himportShouldRetrySet reports whether a retry of cmd may succeed after it +// failed with "no such fieldset": true when the fieldset is registered +// client-side — the executing connection lost its server session state (for +// example a RESET, or a concurrent discard). The connection's prepared flag +// was already invalidated inside _process, while that goroutine still owned +// the connection, so the retry re-prepares lazily wherever it lands. +func (c *baseClient) himportShouldRetrySet(cmd Cmder, err error) bool { + set, ok := cmd.(*HImportSetCmd) + if !ok || !himportNoSuchFieldset(err) { + return false + } + _, registered := c.himport.lookup(set.fieldsetName) + return registered +} diff --git a/vendor/github.com/redis/go-redis/v9/himport_cluster.go b/vendor/github.com/redis/go-redis/v9/himport_cluster.go new file mode 100644 index 00000000..57cddb0d --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/himport_cluster.go @@ -0,0 +1,206 @@ +package redis + +import "context" + +// Cluster and ring support for the HIMPORT command family. +// +// Correctness comes from the shared registry: every node/shard client holds +// the same himportRegistry (wired at client construction), so any connection +// executing an HIMPORT SET lazily replays the PREPARE, MOVED/ASK redirects +// re-prepare on the target node, and discards propagate through tombstones +// and the discard-all epoch. Replicas share the registry too — roles change +// with the topology, and a promoted replica's connections simply carry no +// prepared flags, so their first SET self-prepares. +// +// On top of that, user-issued PREPARE/DISCARD/DISCARDALL fan out eagerly to +// all masters (R.4): one connection per master is prepared/cleaned up front, +// server-side validation surfaces immediately, and leftover session state is +// bounded. The fan-out is best-effort — any connection it does not reach is +// covered by the lazy replay. + +// The fan-out helpers execute the per-node copies through each node client's +// Process, so node-level hooks observe them; the cluster/ring-level +// ProcessHook chain sees only the user's command object, not the fan-out. +// +// Known limitation: an HImportPrepare pipelined together with HImportSets of +// the same new fieldset in one ClusterClient Exec is not ordered across +// nodes — per-node sub-batches run concurrently, and the registration +// happens when the PREPARE's node completes, so SETs routed to other nodes +// can race it and fail with "no such fieldset". Register the fieldset with +// the client-level HImportPrepare before pipelining (the HLD's back-to-back +// PREPARE+SET pattern is a single-connection guarantee). + +// himportForEach runs fn on a set of clients (all cluster masters, or all +// ring shards). +type himportForEach func(ctx context.Context, fn func(ctx context.Context, client *Client) error) error + +// himportRequeueFailedSets re-queues HIMPORT SETs of registered fieldsets +// that failed with "no such fieldset" — their stale prepared flags were just +// invalidated by himportAfterBatch, so the next pipeline attempt re-prepares +// lazily and re-executes only those SETs (a full replace, so idempotent). +// Bounded by the cluster pipeline's attempt budget. +func (c *ClusterClient) himportRequeueFailedSets(ctx context.Context, cmds []Cmder, failedCmds *cmdsMap) { + for _, cmd := range cmds { + // rawErr: runs on the per-node execution goroutine, same + // self-deadlock rule as himportAfterBatch. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) { + if _, registered := c.himport.lookup(set.fieldsetName); registered { + _ = c.mapCmdsByNode(ctx, failedCmds, []Cmder{set}) + } + } + } +} + +// himportFanOutPrepare registers the fieldset once in the shared registry +// and executes a pre-versioned PREPARE copy on every client; each copy marks +// its executing connection without registering again. A deterministic server +// rejection (e.g. duplicate field name) withdraws the registration; a +// transport failure keeps it, and lazy replay covers the connections the +// fan-out missed (all-succeeded semantics: the first error is reported). +func himportFanOutPrepare(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportPrepareCmd) { + version, epoch := registry.register(cmd.fieldsetName, cmd.fields) + err := forEach(ctx, func(ctx context.Context, client *Client) error { + fanCmd := NewHImportPrepareCmd(ctx, cmd.fieldsetName, cmd.fields...) + fanCmd.registryVersion = version + fanCmd.registryEpoch = epoch + return client.Process(ctx, fanCmd) + }) + if err != nil { + if isRedisError(err) { + // Withdraw the registration; the tombstone cleans the sessions + // on which the fan-out succeeded before the rejection. + registry.discardVersion(cmd.fieldsetName, version) + } + cmd.SetErr(err) + return + } + cmd.SetVal("OK") +} + +// himportFanOutDiscard removes the fieldset from the shared registry +// (leaving the tombstone that lazily cleans the connections the fan-out does +// not reach) and discards it on one connection of every client. +func himportFanOutDiscard(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardCmd) { + registered := registry.discard(cmd.fieldsetName) + err := forEach(ctx, func(ctx context.Context, client *Client) error { + return client.Process(ctx, NewHImportDiscardCmd(ctx, cmd.fieldsetName)) + }) + if err != nil { + cmd.SetErr(err) + return + } + if registered { + cmd.SetVal(1) + } else { + cmd.SetVal(0) + } +} + +// himportFanOutDiscardAll wipes the shared registry once and executes a +// pre-epoch DISCARDALL copy on every client; each copy moves its executing +// connection to the new epoch without bumping the registry again. +func himportFanOutDiscardAll(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardAllCmd) { + epoch, removed := registry.discardAll() + err := forEach(ctx, func(ctx context.Context, client *Client) error { + fanCmd := NewHImportDiscardAllCmd(ctx) + fanCmd.registryEpoch = epoch + return client.Process(ctx, fanCmd) + }) + if err != nil { + cmd.SetErr(err) + return + } + cmd.SetVal(int64(removed)) +} + +// HImportPrepare registers the fieldset in the cluster-wide registry and +// eagerly prepares one connection on every master; all other connections — +// including those of replicas promoted later and masters added by +// resharding — are prepared lazily before their first HImportSet. See +// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics. +// +// The fan-out is best-effort and reports the first error: on a server +// rejection (e.g. duplicate field name) the registration is withdrawn and +// any sessions the fan-out already prepared are cleaned lazily; on a +// transport failure the registration is kept and lazy replay covers the +// connections the fan-out missed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + himportFanOutPrepare(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes the fieldset from the cluster-wide registry and +// discards it on every master; connections the fan-out does not reach +// replay the discard before their next HIMPORT command. It returns 1 if the +// fieldset was registered on this client and is now removed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + himportFanOutDiscard(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the cluster-wide registry and +// wipes them on every master; connections the fan-out does not reach replay +// the wipe before their next HIMPORT command. It returns the number of +// fieldsets removed from the registry. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + himportFanOutDiscardAll(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.IntCmd +} + +// HImportPrepare registers the fieldset in the ring-wide registry and +// eagerly prepares one connection on every shard; all other connections are +// prepared lazily before their first HImportSet. The fan-out is best-effort +// with the same failure semantics as ClusterClient.HImportPrepare. See +// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + himportFanOutPrepare(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes the fieldset from the ring-wide registry and +// discards it on every shard; connections the fan-out does not reach replay +// the discard before their next HIMPORT command. It returns 1 if the +// fieldset was registered on this client and is now removed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + himportFanOutDiscard(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the ring-wide registry and +// wipes them on every shard; connections the fan-out does not reach replay +// the wipe before their next HIMPORT command. It returns the number of +// fieldsets removed from the registry. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + himportFanOutDiscardAll(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.IntCmd +} diff --git a/vendor/github.com/redis/go-redis/v9/himport_commands.go b/vendor/github.com/redis/go-redis/v9/himport_commands.go new file mode 100644 index 00000000..77e1a98c --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/himport_commands.go @@ -0,0 +1,253 @@ +package redis + +import "context" + +// The HIMPORT command family (Redis 8.10+, "hinted hash templates") provides +// fast ingestion of many hashes sharing the same field names. HIMPORT PREPARE +// registers the field names once under a fieldset name, then HIMPORT SET +// creates hashes by sending only the values. +// +// The server scopes a fieldset to the physical connection that prepared it. +// Because go-redis pools connections, the client additionally keeps a +// client-side registry of fieldsets registered through HImportPrepare and +// lazily replays the PREPARE (at most once per connection session) on any +// pooled connection about to execute an HImportSet that references it. See +// himport.go. +// +// The whole HIMPORT surface — the typed methods, the HImport*Cmd types and +// their constructors — is experimental and may be subject to change. + +// himportCmder marks HIMPORT commands that participate in client-side +// fieldset tracking. Process paths do a single interface assertion on the +// hot path and inspect the concrete type only for HIMPORT commands. +type himportCmder interface { + Cmder + himportCmd() +} + +var ( + _ himportCmder = (*HImportPrepareCmd)(nil) + _ himportCmder = (*HImportSetCmd)(nil) + _ himportCmder = (*HImportDiscardCmd)(nil) + _ himportCmder = (*HImportDiscardAllCmd)(nil) +) + +// HImportPrepareCmd represents an HIMPORT PREPARE command. +type HImportPrepareCmd struct { + StatusCmd + + fieldsetName string + fields []string + + // registryVersion and registryEpoch are set only on commands injected by + // the client to replay a registered fieldset onto a connection; on + // success the connection is marked as prepared at this version under + // this discard-all epoch. + registryVersion uint64 + registryEpoch uint64 +} + +func (cmd *HImportPrepareCmd) himportCmd() {} + +// NewHImportPrepareCmd returns an HIMPORT PREPARE command. +func NewHImportPrepareCmd(ctx context.Context, fieldsetName string, fields ...string) *HImportPrepareCmd { + args := make([]interface{}, 3+len(fields)) + args[0] = "himport" + args[1] = "prepare" + args[2] = fieldsetName + for i, field := range fields { + args[3+i] = field + } + return &HImportPrepareCmd{ + StatusCmd: StatusCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeStatus, + }, + }, + fieldsetName: fieldsetName, + fields: append([]string(nil), fields...), + } +} + +// HImportSetCmd represents an HIMPORT SET command. +type HImportSetCmd struct { + StatusCmd + + fieldsetName string +} + +func (cmd *HImportSetCmd) himportCmd() {} + +// NewHImportSetCmd returns an HIMPORT SET command. +func NewHImportSetCmd(ctx context.Context, key, fieldsetName string, values ...interface{}) *HImportSetCmd { + args := make([]interface{}, 4+len(values)) + args[0] = "himport" + args[1] = "set" + args[2] = key + args[3] = fieldsetName + copy(args[4:], values) + cmd := &HImportSetCmd{ + StatusCmd: StatusCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeStatus, + }, + }, + fieldsetName: fieldsetName, + } + cmd.SetFirstKeyPos(2) + return cmd +} + +// HImportDiscardCmd represents an HIMPORT DISCARD command. +type HImportDiscardCmd struct { + IntCmd + + fieldsetName string +} + +func (cmd *HImportDiscardCmd) himportCmd() {} + +// NewHImportDiscardCmd returns an HIMPORT DISCARD command. +func NewHImportDiscardCmd(ctx context.Context, fieldsetName string) *HImportDiscardCmd { + return &HImportDiscardCmd{ + IntCmd: IntCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: []interface{}{"himport", "discard", fieldsetName}, + cmdType: CmdTypeInt, + }, + }, + fieldsetName: fieldsetName, + } +} + +// HImportDiscardAllCmd represents an HIMPORT DISCARDALL command. +type HImportDiscardAllCmd struct { + IntCmd + + // registryEpoch is set only on commands injected by the client to wipe a + // session that predates the registry's discard-all epoch; on success the + // connection adopts this epoch. + registryEpoch uint64 +} + +func (cmd *HImportDiscardAllCmd) himportCmd() {} + +// NewHImportDiscardAllCmd returns an HIMPORT DISCARDALL command. +func NewHImportDiscardAllCmd(ctx context.Context) *HImportDiscardAllCmd { + return &HImportDiscardAllCmd{ + IntCmd: IntCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: []interface{}{"himport", "discardall"}, + cmdType: CmdTypeInt, + }, + }, + } +} + +// HImportPrepare registers an ordered list of hash field names under +// fieldsetName for use by subsequent HImportSet calls: +// +// HIMPORT PREPARE fieldset_name field [field ...] +// +// The server keeps the fieldset in the session of the connection that +// executed the command. On pooled clients (Client, Conn, Pipeline, Tx) the +// fieldset is also remembered client-side and the PREPARE is replayed +// lazily — at most once per connection session — on any pooled connection +// about to execute an HImportSet referencing it, so HImportSet works +// transparently across the pool. Preparing an existing fieldset name again +// silently replaces it. +// +// ClusterClient and Ring override this method (see himport_cluster.go): the +// fieldset registers in a registry shared by every node/shard client and the +// PREPARE additionally fans out eagerly to all masters/shards. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + _ = c(ctx, cmd) + return &cmd.StatusCmd +} + +// HImportSet creates or fully replaces the hash at key using the field list +// registered under fieldsetName, pairing values positionally with the +// prepared fields: +// +// HIMPORT SET key fieldset_name value [value ...] +// +// The number of values must equal the fieldset's field count. The resulting +// key is a regular hash readable and writable by all hash commands. If the +// fieldset was registered through HImportPrepare on this client, it is +// prepared automatically on whichever pooled connection executes the command; +// otherwise the fieldset must have been prepared on the executing connection +// or the server replies "ERR no such fieldset". +// +// "no such fieldset" never surfaces for a registered fieldset: a +// single-command HImportSet whose connection lost its session state (e.g. +// RESET) is transparently re-prepared and retried once — the failure also +// stales every other connection's prepared flag, so the retry re-prepares +// wherever it lands, and this recovery attempt is granted even when retries +// are disabled (MaxRetries -1). In pipelines the failed HImportSets — and +// only those — are re-prepared and re-issued once on the same connection +// (HIMPORT SET is a full replace, so the re-execution is idempotent and no +// other command of the batch runs again). Inside transactions the error does +// surface after EXEC — an executed transaction cannot be partially re-run — +// but the prepared flags are invalidated, so retrying the transaction +// succeeds. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd { + cmd := NewHImportSetCmd(ctx, key, fieldsetName, values...) + _ = c(ctx, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes fieldsetName from the executing connection's session +// and from the client-side registry, stopping further automatic replay: +// +// HIMPORT DISCARD fieldset_name +// +// It returns 1 if the fieldset was registered on this client and is now +// removed, 0 otherwise (for names never registered through the managed API, +// the executing connection's session reply passes through unchanged). Pooled +// connections whose sessions still hold the fieldset replay the DISCARD +// before their next HIMPORT command, so a subsequent HImportSet fails with +// "no such fieldset" on every connection, exactly as on a single connection. +// Hashes already created through the fieldset are not affected. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + _ = c(ctx, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the executing connection's +// session and clears the client-side registry: +// +// HIMPORT DISCARDALL +// +// It returns the number of fieldsets removed from the client-side registry +// (when none were registered, the executing connection's session count +// passes through). Other pooled connections whose sessions were prepared +// earlier replay HIMPORT DISCARDALL before their next HIMPORT command. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + _ = c(ctx, cmd) + return &cmd.IntCmd +} diff --git a/vendor/github.com/redis/go-redis/v9/internal/once.go b/vendor/github.com/redis/go-redis/v9/internal/once.go index b81244fd..2d3a8cdc 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/once.go +++ b/vendor/github.com/redis/go-redis/v9/internal/once.go @@ -27,7 +27,7 @@ import ( // and is re-armed on failure. type Once struct { m sync.Mutex - done uint32 + done atomic.Uint32 } // Do calls the function f if and only if Do has not been invoked @@ -46,17 +46,17 @@ type Once struct { // // err := config.once.Do(func() error { return config.init(filename) }) func (o *Once) Do(f func() error) error { - if atomic.LoadUint32(&o.done) == 1 { + if o.done.Load() == 1 { return nil } // Slow-path. o.m.Lock() defer o.m.Unlock() var err error - if o.done == 0 { + if o.done.Load() == 0 { err = f() if err == nil { - atomic.StoreUint32(&o.done, 1) + o.done.Store(1) } } return err diff --git a/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go b/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go index a3f23fff..2e234ba8 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go +++ b/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go @@ -155,6 +155,15 @@ func getRecorder() Recorder { return r } +// Enabled reports whether a real recorder is installed. Callers use it to +// skip metric work whose INPUTS are expensive to obtain — e.g. reading a +// command's result, which on the async autopipeline face blocks until the +// command executes. +func Enabled() bool { + _, noop := getRecorder().(noopRecorder) + return !noop +} + // SetGlobalRecorder sets the global recorder (called by Init() in extra/redisotel-native) func SetGlobalRecorder(r Recorder) { recorderMu.Lock() @@ -266,10 +275,15 @@ func (noopRecorder) RecordStreamLag(context.Context, time.Duration, *pool.Conn, func (noopRecorder) RecordConnectionCount(context.Context, int, *pool.Conn, string, bool) {} func (noopRecorder) RecordPendingRequests(context.Context, int, *pool.Conn, string) {} -// RegisterPools registers connection pools with the global recorder. -func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) { - // Check if the global recorder implements PoolRegistrar - if registrar, ok := globalRecorder.(PoolRegistrar); ok { +// RegisterPools registers connection pools with the global recorder. pipelinePool +// is the optional dedicated pipeline connection pool (nil when not configured); +// it is registered as a regular pool under a "_pipeline" name suffix. +func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler, addr string) { + // Check if the global recorder implements PoolRegistrar. Read it through + // getRecorder: SetGlobalRecorder writes globalRecorder under recorderMu, and + // clients are created (and closed) concurrently with telemetry being + // installed, so an unlocked read here is a data race -race reports. + if registrar, ok := getRecorder().(PoolRegistrar); ok { // Generate a unique ID for this client's pools uniqueID := generateUniqueID() @@ -281,18 +295,27 @@ func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) { poolName := addr + "_" + uniqueID + "_pubsub" registrar.RegisterPubSubPool(poolName, pubSubPool) } + if pipelinePool != nil { + poolName := addr + "_" + uniqueID + "_pipeline" + registrar.RegisterPool(poolName, pipelinePool) + } } } -// UnregisterPools removes connection pools from the global recorder -func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler) { - // Check if the global recorder implements PoolRegistrar - if registrar, ok := globalRecorder.(PoolRegistrar); ok { +// UnregisterPools removes connection pools from the global recorder. pipelinePool +// is the optional dedicated pipeline connection pool (nil when not configured). +func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler) { + // Check if the global recorder implements PoolRegistrar (see RegisterPools + // for why this goes through getRecorder rather than reading directly). + if registrar, ok := getRecorder().(PoolRegistrar); ok { if connPool != nil { registrar.UnregisterPool(connPool) } if pubSubPool != nil { registrar.UnregisterPubSubPool(pubSubPool) } + if pipelinePool != nil { + registrar.UnregisterPool(pipelinePool) + } } } diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go b/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go index 1e836ad4..80f3b77c 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go @@ -45,7 +45,7 @@ func GetCachedTimeNs() int64 { } // Global atomic counter for connection IDs -var connIDCounter uint64 +var connIDCounter atomic.Uint64 // HandoffState represents the atomic state for connection handoffs // This struct is stored atomically to prevent race conditions between @@ -63,7 +63,7 @@ type atomicNetConn struct { // generateConnID generates a fast unique identifier for a connection with zero allocations func generateConnID() uint64 { - return atomic.AddUint64(&connIDCounter, 1) + return connIDCounter.Add(1) } type Conn struct { @@ -109,6 +109,18 @@ type Conn struct { expiresAt time.Time poolName string // Name of the pool this connection belongs to (for metrics) + // preparedFieldsets tracks HIMPORT fieldsets prepared on this + // connection's current server session: fieldset name -> client-side + // registry version. The server drops fieldsets when the session ends, + // so the map is cleared whenever the underlying network connection is + // replaced. preparedFieldsetsEpoch records the registry's discard-all + // epoch the session was prepared under; a session behind the current + // epoch replays HIMPORT DISCARDALL before its next HIMPORT command. + // Guarded by preparedFieldsetsMu; the map is nil until first use. + preparedFieldsetsMu sync.Mutex + preparedFieldsets map[string]uint64 + preparedFieldsetsEpoch uint64 + // When a goroutine closes a connection, it usually knows the reason, so closeReason is not needed. // closeReason is only used when an in-use connection is closed by another goroutine, // to inform the goroutine using the connection why the connection was closed. @@ -135,6 +147,24 @@ type Conn struct { initConnFunc func(context.Context, *Conn) error onClose func() error + + // onCscClose is the client-side-caching close hook, kept separate from + // onClose (streaming-credentials cleanup) so neither clobbers the other. + // Both keep overwrite semantics, so re-running initConn can't accumulate them. + onCscClose func() error + + // onCscReinit runs after the connection is claimed for reinitialization but + // before its socket is replaced. CSC uses it to invalidate entries whose + // server-side tracking coverage belongs to the old socket. + onCscReinit func() + + // cscReadPending requests one conservative drain after a command read through + // a transport whose buffered state cannot be fully observed by MaybeHasData. + cscReadPending atomic.Bool + + // lastCscPeriodicProbeNs throttles bounded fallback reads on platforms and + // opaque transports without a non-consuming readiness mechanism. + lastCscPeriodicProbeNs atomic.Int64 } func NewConn(netConn net.Conn) *Conn { @@ -638,6 +668,18 @@ func (cn *Conn) SetOnClose(fn func() error) { cn.onClose = fn } +// SetOnCscClose sets the client-side-caching close hook, overwriting any +// previous one. It runs on Close in addition to the SetOnClose callback. +func (cn *Conn) SetOnCscClose(fn func() error) { + cn.onCscClose = fn +} + +// SetOnCscReinit sets the client-side-caching pre-reinitialization hook, +// overwriting any previous one. +func (cn *Conn) SetOnCscReinit(fn func()) { + cn.onCscReinit = fn +} + // SetInitConnFunc sets the connection initialization function to be called on reconnections. func (cn *Conn) SetInitConnFunc(fn func(context.Context, *Conn) error) { cn.initConnFunc = fn @@ -661,6 +703,85 @@ func (cn *Conn) SetNetConn(netConn net.Conn) { cn.readerMu.Unlock() cn.bw.Reset(netConn) + + // A new socket is a new server session with no HIMPORT fieldsets and + // nothing left to discard. + cn.ClearPreparedFieldsets(0) +} + +// FieldsetPreparedVersion returns the client-side registry version at which +// the named HIMPORT fieldset was prepared on this connection's current server +// session, or 0 if it was not prepared on it (registry versions start at 1). +func (cn *Conn) FieldsetPreparedVersion(name string) uint64 { + cn.preparedFieldsetsMu.Lock() + version := cn.preparedFieldsets[name] + cn.preparedFieldsetsMu.Unlock() + return version +} + +// MarkFieldsetPrepared records that the named HIMPORT fieldset was prepared +// on this connection's current server session at the given registry version. +// A session acquiring its first fieldset adopts the given discard-all epoch +// (fieldsets prepared after an HIMPORT DISCARDALL are not subject to it); +// the epoch never moves backwards, so a mark carrying an older snapshot +// cannot regress a session already wiped at a newer epoch. +func (cn *Conn) MarkFieldsetPrepared(name string, version, epoch uint64) { + cn.preparedFieldsetsMu.Lock() + if len(cn.preparedFieldsets) == 0 { + cn.preparedFieldsets = make(map[string]uint64) + if epoch > cn.preparedFieldsetsEpoch { + cn.preparedFieldsetsEpoch = epoch + } + } + cn.preparedFieldsets[name] = version + cn.preparedFieldsetsMu.Unlock() +} + +// UnmarkFieldsetPrepared forgets that the named HIMPORT fieldset was prepared +// on this connection, forcing a replay before the next HIMPORT SET using it. +func (cn *Conn) UnmarkFieldsetPrepared(name string) { + cn.preparedFieldsetsMu.Lock() + delete(cn.preparedFieldsets, name) + cn.preparedFieldsetsMu.Unlock() +} + +// HasPreparedFieldsets reports whether any HIMPORT fieldset is prepared on +// this connection's current server session. +func (cn *Conn) HasPreparedFieldsets() bool { + cn.preparedFieldsetsMu.Lock() + n := len(cn.preparedFieldsets) + cn.preparedFieldsetsMu.Unlock() + return n > 0 +} + +// PreparedFieldsetNames returns the names of the HIMPORT fieldsets prepared +// on this connection's current server session. +func (cn *Conn) PreparedFieldsetNames() []string { + cn.preparedFieldsetsMu.Lock() + names := make([]string, 0, len(cn.preparedFieldsets)) + for name := range cn.preparedFieldsets { + names = append(names, name) + } + cn.preparedFieldsetsMu.Unlock() + return names +} + +// FieldsetEpoch returns the discard-all epoch this connection's prepared +// fieldsets belong to (0 when none were ever prepared on the session). +func (cn *Conn) FieldsetEpoch() uint64 { + cn.preparedFieldsetsMu.Lock() + epoch := cn.preparedFieldsetsEpoch + cn.preparedFieldsetsMu.Unlock() + return epoch +} + +// ClearPreparedFieldsets forgets all HIMPORT fieldsets prepared on this +// connection and records the discard-all epoch the wipe corresponds to. +func (cn *Conn) ClearPreparedFieldsets(epoch uint64) { + cn.preparedFieldsetsMu.Lock() + cn.preparedFieldsets = nil + cn.preparedFieldsetsEpoch = epoch + cn.preparedFieldsetsMu.Unlock() } // GetNetConn safely returns the current network connection using atomic load (lock-free). @@ -694,6 +815,10 @@ func (cn *Conn) SetNetConnAndInitConn(ctx context.Context, netConn net.Conn) err return fmt.Errorf("cannot initialize connection from state %s: %w", finalState, err) } + if cn.onCscReinit != nil { + cn.onCscReinit() + } + // Replace the underlying connection cn.SetNetConn(netConn) @@ -774,8 +899,13 @@ func (cn *Conn) MarkQueuedForHandoff() error { // Already unusable - this is fine, keep the new handoff state return nil } - // Restore the original state if transition fails for other reasons - cn.handoffStateAtomic.Store(currentState) + // Restore the original handoff state only if nothing else changed it + // since our CAS above. A concurrent handoff worker may have completed + // the handoff and run ClearHandoffState in this window; a plain Store + // would clobber that, resurrecting ShouldHandoff=true and wedging the + // connection so it can never be acquired again. The CAS leaves the + // worker's state intact when it has taken over. + cn.handoffStateAtomic.CompareAndSwap(newState, currentState) return fmt.Errorf("failed to mark connection as unusable: %w", err) } return nil @@ -871,6 +1001,18 @@ func (cn *Conn) PeekReplyTypeSafe() (byte, error) { return cn.rd.PeekReplyType() } +// PeekReplyTypeForCheck peeks at the reply type while holding readerMu, so it is +// safe against a concurrent SetNetConn resetting the reader during handoff. +// Unlike PeekReplyTypeSafe it does not require the data to already be buffered: +// the pool health check calls it after connCheck reports unexpected socket data, +// and connCheck only MSG_PEEKs, so the byte still has to be pulled from the +// socket into the reader here. +func (cn *Conn) PeekReplyTypeForCheck() (byte, error) { + cn.readerMu.RLock() + defer cn.readerMu.RUnlock() + return cn.rd.PeekReplyType() +} + func (cn *Conn) Write(b []byte) (int, error) { // Lock-free netConn access for better performance if netConn := cn.getNetConn(); netConn != nil { @@ -907,6 +1049,29 @@ func (cn *Conn) WithReader( return fn(cn.rd) } +// WithReaderHardDeadline runs fn under a HARD read deadline of now+timeout, +// bypassing getEffectiveReadTimeout so a relaxed maintenance timeout can't extend +// it (used by the CSC drainer). Takes no context: an expired cycle ctx must not +// become the socket deadline, or the read surfaces context.DeadlineExceeded, which +// isBadConn treats as fatal. +func (cn *Conn) WithReaderHardDeadline( + timeout time.Duration, fn func(rd *proto.Reader) error, +) (err error) { + netConn := cn.getNetConn() + if netConn == nil { + return errConnectionNotAvailable + } + if err := netConn.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return err + } + defer func() { + if clearErr := netConn.SetReadDeadline(time.Time{}); clearErr != nil { + err = clearErr + } + }() + return fn(cn.rd) +} + func (cn *Conn) WithWriter( ctx context.Context, timeout time.Duration, fn func(wr *proto.Writer) error, ) error { @@ -944,17 +1109,29 @@ func (cn *Conn) IsClosed() bool { } func (cn *Conn) Close() error { - if cn.IsClosed() { - return nil + for { + state := cn.stateMachine.GetState() + if state == StateClosed { + return nil + } + if cn.stateMachine.TryTransitionFast(state, StateClosed) { + // TryTransitionFast deliberately skips waiter notification; Close + // still needs to wake any goroutine waiting on initialization. + cn.stateMachine.notifyWaiters() + break + } } - // Transition to CLOSED state - cn.stateMachine.Transition(StateClosed) if cn.onClose != nil { // ignore error _ = cn.onClose() cn.onClose = nil } + if cn.onCscClose != nil { + // ignore error + _ = cn.onCscClose() + cn.onCscClose = nil + } // Lock-free netConn access for better performance if netConn := cn.getNetConn(); netConn != nil { @@ -974,6 +1151,56 @@ func (cn *Conn) MaybeHasData() bool { return false } +// CheckForData reports whether the socket has data ready and surfaces a +// detected closed or failed socket. +func (cn *Conn) CheckForData() (bool, error) { + if netConn := cn.getNetConn(); netConn != nil { + return checkForData(netConn) + } + return false, nil +} + +// MarkCscReadPending requests one conservative CSC drain after a command read +// when the transport may retain data that MaybeHasData cannot observe. +func (cn *Conn) MarkCscReadPending() { + netConn := cn.getNetConn() + if netConn == nil { + return + } + if needsCscReadProbe(netConn) { + cn.cscReadPending.Store(true) + } +} + +// TakeCscReadPending consumes the post-command conservative-drain request. +func (cn *Conn) TakeCscReadPending() bool { + return cn.cscReadPending.Swap(false) +} + +// TakeCscPeriodicReadPending schedules a throttled conservative read for +// transports with no readiness mechanism. It returns true at most once per +// interval, including when several drainer passes race. +func (cn *Conn) TakeCscPeriodicReadPending(interval time.Duration) bool { + netConn := cn.getNetConn() + if netConn == nil || interval <= 0 || !needsCscPeriodicProbe(netConn) { + return false + } + + now := time.Since(cn.createdAt).Nanoseconds() + if now <= 0 { + now = 1 + } + for { + last := cn.lastCscPeriodicProbeNs.Load() + if last != 0 && now >= last && now-last < int64(interval) { + return false + } + if cn.lastCscPeriodicProbeNs.CompareAndSwap(last, now) { + return true + } + } +} + // deadline computes the effective deadline time based on context and timeout. // It updates the usedAt timestamp to now. // Uses cached time to avoid expensive syscall (max 50ms staleness is acceptable for deadline calculation). diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go b/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go index 9e83dd83..333c34ba 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go @@ -19,10 +19,19 @@ func connCheck(conn net.Conn) error { // Reset previous timeout. _ = conn.SetDeadline(time.Time{}) + // Health checks deliberately inspect only the outer connection. Unwrapping a + // buffered transport such as crypto/tls.Conn can reveal an encrypted + // post-handshake record and make isHealthyConn call PeekReplyType on the TLS + // stream. With the deadline cleared above, TLS may consume that control record + // and then wait forever for application data. sysConn, ok := conn.(syscall.Conn) if !ok { return nil } + return checkSyscallConn(sysConn) +} + +func checkSyscallConn(sysConn syscall.Conn) error { rawConn, err := sysConn.SyscallConn() if err != nil { return err @@ -53,7 +62,62 @@ func connCheck(conn net.Conn) error { return sysErr } +// underlyingSyscallConn unwraps connections that expose their transport through +// NetConn (notably crypto/tls.Conn). Limit the walk so a broken wrapper cannot +// loop forever. +func underlyingSyscallConn(conn net.Conn) (syscall.Conn, bool) { + for range 8 { + if sysConn, ok := conn.(syscall.Conn); ok { + return sysConn, true + } + unwrapper, ok := conn.(interface{ NetConn() net.Conn }) + if !ok { + return nil, false + } + conn = unwrapper.NetConn() + if conn == nil { + return nil, false + } + } + return nil, false +} + // maybeHasData checks if there is data in the socket without consuming it func maybeHasData(conn net.Conn) bool { - return connCheck(conn) == errUnexpectedRead + hasData, _ := checkForData(conn) + return hasData +} + +func checkForData(conn net.Conn) (bool, error) { + // Unlike the general health check, CSC only uses this as a readiness hint + // before a bounded read, so unwrapping TLS is safe and avoids blocking. + _ = conn.SetDeadline(time.Time{}) + sysConn, ok := underlyingSyscallConn(conn) + if !ok { + return false, nil + } + switch err := checkSyscallConn(sysConn); err { + case nil: + return false, nil + case errUnexpectedRead: + return true, nil + default: + return false, err + } +} + +// needsCscReadProbe reports whether a command read may leave data hidden from +// maybeHasData. On Unix a direct syscall.Conn has no intermediate buffering; +// TLS and opaque wrappers need one bounded post-command probe. +func needsCscReadProbe(conn net.Conn) bool { + _, direct := conn.(syscall.Conn) + return !direct +} + +// needsCscPeriodicProbe reports whether the platform can inspect the transport +// at all. Opaque wrappers get a throttled bounded fallback so invalidations that +// arrive after the post-command probe are still eventually consumed. +func needsCscPeriodicProbe(conn net.Conn) bool { + _, ok := underlyingSyscallConn(conn) + return !ok } diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go b/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go index f971d94c..94770661 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go @@ -14,7 +14,21 @@ func connCheck(_ net.Conn) error { return nil } -// since we can't check for data on the socket, we just assume there is some +// There is no portable non-consuming readiness check on this platform. +// Returning true would force every idle CSC connection through a timed read on +// every drainer tick. The CSC drainer uses needsCscPeriodicProbe instead. func maybeHasData(_ net.Conn) bool { + return false +} + +func checkForData(_ net.Conn) (bool, error) { + return false, nil +} + +func needsCscReadProbe(_ net.Conn) bool { + return true +} + +func needsCscPeriodicProbe(_ net.Conn) bool { return true } diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go b/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go index 8f648ffb..b79c2bac 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go @@ -72,6 +72,11 @@ var ( // errConnNotPooled is returned when trying to return a non-pooled connection to the pool. errConnNotPooled = errors.New("connection not pooled") + + // errConnEvictedIdle is passed to OnRemove hooks when a pooled connection is evicted on + // Put because the idle pool is already at MaxIdleConns. + errConnEvictedIdle = errors.New("connection evicted: idle pool at capacity") + // metricCallbackMu protects all global metric callback functions for thread-safe access. metricCallbackMu sync.RWMutex @@ -304,6 +309,9 @@ func getMetricPendingRequestsCallback() func(ctx context.Context, delta int, cn } // Stats contains pool state information and accumulated stats. +// +// TODO(cxl): the uint32/int64 fields below will be changed to atomic value +// types (atomic.Uint32/atomic.Int64) in v10, which is a breaking API change. type Stats struct { Hits uint32 // number of times free connection was found in the pool Misses uint32 // number of times free connection was NOT found in the pool @@ -318,6 +326,11 @@ type Stats struct { PendingRequests uint32 // number of pending requests waiting for a connection PubSubStats PubSubStats + + // PipelineStats holds the stats of the separate pipeline connection pool + // when one is configured (PipelineReadBufferSize/PipelineWriteBufferSize). + // nil when pipelines share the main pool. + PipelineStats *Stats } type ConnRetirer interface { @@ -394,7 +407,7 @@ type lastDialErrorWrap struct { type ConnPool struct { cfg *Options - dialErrorsNum uint32 // atomic + dialErrorsNum atomic.Uint32 lastDialError atomic.Value dialsInProgress chan struct{} @@ -415,11 +428,23 @@ type ConnPool struct { stats Stats waitDurationNs atomic.Int64 - _closed uint32 // atomic + _closed atomic.Uint32 - // Pool hooks manager for flexible connection processing - // Using atomic.Pointer for lock-free reads in hot paths (Get/Put) + // Pool hooks manager. atomic.Pointer keeps hot-path reads (Get/Put) + // lock-free; hookMu serializes Add/RemovePoolHook's read-clone-store so + // concurrent mutators (e.g. maintnotifications and CSC) can't lose an update. hookManager atomic.Pointer[PoolHookManager] + hookMu sync.Mutex + + // drainMu/drainDone coordinate the CSC drainer's temporary idle-connection + // claim with Get. The normal semaphore retains its PoolSize capacity (and + // therefore the established MaxActiveConns/ErrPoolExhausted behavior); a Get + // that finds the idle list empty only because the drainer borrowed a conn + // waits for that short claim to finish instead of opening an overflow conn. + drainMu sync.Mutex + drainDone chan struct{} + drainBorrowed int + drainGeneration atomic.Uint64 } var _ Pooler = (*ConnPool)(nil) @@ -453,7 +478,10 @@ func (p *ConnPool) initializeHooks() { // AddPoolHook adds a pool hook to the pool. func (p *ConnPool) AddPoolHook(hook PoolHook) { - // Lock-free read of current manager + // Serialize so a concurrent Add/Remove can't clobber this change. + p.hookMu.Lock() + defer p.hookMu.Unlock() + manager := p.hookManager.Load() if manager == nil { p.initializeHooks() @@ -464,12 +492,22 @@ func (p *ConnPool) AddPoolHook(hook PoolHook) { newManager := manager.Clone() newManager.AddHook(hook) - // Atomically swap to new manager + // Atomically swap to new manager (hot-path readers load lock-free) p.hookManager.Store(newManager) } +// SupportsPoolHooks reports that AddPoolHook and RemovePoolHook are functional. +// Pooler adapters with no-op hook methods intentionally do not expose this +// optional capability. +func (p *ConnPool) SupportsPoolHooks() bool { + return true +} + // RemovePoolHook removes a pool hook from the pool. func (p *ConnPool) RemovePoolHook(hook PoolHook) { + p.hookMu.Lock() + defer p.hookMu.Unlock() + manager := p.hookManager.Load() if manager != nil { // Create new manager with removed hook @@ -651,7 +689,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) { return nil, ErrClosed } - if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.cfg.PoolSize) { + if p.dialErrorsNum.Load() >= uint32(p.cfg.PoolSize) { return nil, p.getLastDialError() } @@ -724,7 +762,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) { internal.Logger.Printf(ctx, "redis: connection pool: failed to dial after %d attempts: %v", attempt, lastErr) // All retries failed - handle error tracking p.setLastDialError(lastErr) - if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.cfg.PoolSize) { + if p.dialErrorsNum.Add(1) == uint32(p.cfg.PoolSize) { go p.tryDial() } return nil, lastErr @@ -789,7 +827,7 @@ func (p *ConnPool) tryDial() { continue } - atomic.StoreUint32(&p.dialErrorsNum, 0) + p.dialErrorsNum.Store(0) _ = conn.Close() return } @@ -835,28 +873,30 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { cb(ctx, -1, nil, poolName) } } - }() - - // Track wait time - only call time.Now() if callback is registered - var waitStart time.Time - waitTimeCallback := getMetricConnectionWaitTimeCallback() - if waitTimeCallback != nil { - waitStart = time.Now() - } - if err = p.waitTurn(ctx); err != nil { - // Record timeout if applicable if err == ErrPoolTimeout { + atomic.AddUint32(&p.stats.Timeouts, 1) if cb := getMetricConnectionTimeoutCallback(); cb != nil { cb(ctx, nil, "pool") } - // Record general error metric for pool timeout if cb := GetMetricErrorCallback(); cb != nil { cb(ctx, "POOL_TIMEOUT", nil, "POOL_TIMEOUT", true, 0) } } + }() + + // PoolTimeout is one budget for both the pool turn and a drainer handoff. + poolDeadline := time.Now().Add(p.cfg.PoolTimeout) + + // Connection wait time measures only semaphore acquisition. + var waitStart time.Time + var waitDuration time.Duration + waitTimeCallback := getMetricConnectionWaitTimeCallback() + if waitTimeCallback != nil { + waitStart = time.Now() + } + if err = p.waitTurn(ctx); err != nil { return nil, err } - var waitDuration time.Duration if waitTimeCallback != nil { waitDuration = time.Since(waitStart) } @@ -867,6 +907,8 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { // Lock-free atomic read - no mutex overhead! hookManager := p.hookManager.Load() +retryIdle: + drainGeneration := p.drainGeneration.Load() for attempts := 0; attempts < getAttempts; attempts++ { p.connsMu.Lock() @@ -937,6 +979,21 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { return cn, nil } + // If the CSC drainer removed the only idle connection during this scan, + // wait for that bounded maintenance claim and retry. The generation closes + // the race where the drainer returns the connection between popIdle and this + // check. Normal MaxActiveConns exhaustion still proceeds to newConn and + // returns ErrPoolExhausted immediately, preserving the existing contract. + if done, retry := p.drainerWaitState(drainGeneration); done != nil { + if err = p.waitForDrainer(ctx, done, poolDeadline); err != nil { + p.freeTurn() + return nil, err + } + goto retryIdle + } else if retry { + goto retryIdle + } + atomic.AddUint32(&p.stats.Misses, 1) var newcn *Conn @@ -964,7 +1021,7 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { // causing IsInited()=true. This means _getConn() in redis.go will take the // early return path and never reach its create time recording. // When hookManager is nil, _getConn() handles both initialization and create time recording. - if dialStartNs := newcn.GetDialStartNs(); dialStartNs > 0 { + if dialStartNs := newcn.GetDialStartNs(); newcn.IsInited() && dialStartNs > 0 { if cb := GetMetricConnectionCreateTimeCallback(); cb != nil { duration := time.Duration(time.Now().UnixNano() - dialStartNs) cb(ctx, duration, newcn) @@ -1120,23 +1177,83 @@ func (p *ConnPool) waitTurn(ctx context.Context) error { // Slow path: need to wait start := time.Now() err := p.semaphore.Acquire(ctx, p.cfg.PoolTimeout, ErrPoolTimeout) - - switch err { - case nil: - // Successfully acquired after waiting - p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano()) - atomic.AddUint32(&p.stats.WaitCount, 1) - case ErrPoolTimeout: - atomic.AddUint32(&p.stats.Timeouts, 1) + if err != nil { + return err } - return err + p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano()) + atomic.AddUint32(&p.stats.WaitCount, 1) + return nil } func (p *ConnPool) freeTurn() { p.semaphore.Release() } +func (p *ConnPool) beginDrainerBorrow() { + p.drainMu.Lock() + if p.drainBorrowed == 0 { + p.drainDone = make(chan struct{}) + } + p.drainBorrowed++ + p.drainMu.Unlock() +} + +func (p *ConnPool) endDrainerBorrow() { + p.drainMu.Lock() + p.drainBorrowed-- + if p.drainBorrowed == 0 { + close(p.drainDone) + p.drainDone = nil + p.drainGeneration.Add(1) + } + p.drainMu.Unlock() +} + +// drainerWaitState returns the current drain epoch's completion channel. If no +// drain is active, retry reports whether an epoch completed during the caller's +// idle scan and the idle list therefore needs to be checked again. +func (p *ConnPool) drainerWaitState(generation uint64) (done <-chan struct{}, retry bool) { + p.drainMu.Lock() + defer p.drainMu.Unlock() + if p.drainBorrowed > 0 { + return p.drainDone, false + } + return nil, p.drainGeneration.Load() != generation +} + +func (p *ConnPool) waitForDrainer( + ctx context.Context, done <-chan struct{}, poolDeadline time.Time, +) error { + if err := ctx.Err(); err != nil { + return err + } + select { + case <-done: + return nil + default: + } + remaining := time.Until(poolDeadline) + if remaining <= 0 { + return ErrPoolTimeout + } + timer := time.NewTimer(remaining) + defer timer.Stop() + + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + // Prefer a caller cancellation that raced with the pool timeout. + if err := ctx.Err(); err != nil { + return err + } + return ErrPoolTimeout + } +} + func (p *ConnPool) popIdle() (*Conn, error) { if p.closed() { return nil, ErrClosed @@ -1290,6 +1407,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) { // expected state, don't log it case StateClosed: internal.Logger.Printf(ctx, "Unexpected conn[%d] state changed by hook to %v, closing it", cn.GetID(), currentState) + if hookManager != nil { + hookManager.ProcessOnRemove(ctx, cn, errHookRequestedRemoval) + } shouldCloseConn = true removedFromPool = p.removeConnWithLock(cn) default: @@ -1357,6 +1477,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) { } } else { shouldCloseConn = true + if hookManager != nil { + hookManager.ProcessOnRemove(ctx, cn, errConnEvictedIdle) + } removedFromPool = p.removeConnWithLock(cn) // Only emit if we actually removed it from the map (not already taken by Close()). @@ -1548,6 +1671,13 @@ func (p *ConnPool) IdleLen() int { return int(n) } +// Name returns the pool's configured name, which is stamped on every +// connection it creates (Conn.PoolName). Callers holding a Pooler can type +// assert to interface{ Name() string } to find which pool owns a connection — +// used by maintnotifications to route a handoff to the hook that owns the +// conn's pool rather than always the primary one. +func (p *ConnPool) Name() string { return p.cfg.Name } + // Size returns the maximum pool size (capacity). // // This is used by the streaming credentials manager to size the re-auth worker pool, @@ -1573,7 +1703,7 @@ func (p *ConnPool) Stats() *Stats { } func (p *ConnPool) closed() bool { - return atomic.LoadUint32(&p._closed) == 1 + return p._closed.Load() == 1 } func (p *ConnPool) RetireConns(ctx context.Context, conns []*Conn, reason string) { @@ -1652,8 +1782,142 @@ func (p *ConnPool) Filter(fn func(*Conn) bool) error { return firstErr } +type drainConn struct { + conn *Conn + idleIndex int +} + +// DrainState carries the cross-pass round bookkeeping for the CSC drainer. +// It is owned by one drainer goroutine, so no synchronization is needed. +type DrainState struct { + // round contains only initialized idle connections. Entries are processed + // from the end so their snapshot indexes remain stable as connections are + // removed and returned. Connections that go idle mid-round are deferred. + round []drainConn + next int +} + +// DrainIdleConns runs one pass of the CSC invalidation drainer over the current +// round, holding AT MOST ONE connection and its pool turn at a time (ctx is the +// per-cycle deadline). A round = idle conn ids snapshotted at start; mid-round +// arrivals are deferred. Each member is drainerPop'd and drained by fn, or — if no +// longer a claimable idle conn — reconciled (marked visited) so it can't hang the +// round. The drainer yields when no turn is immediately available, giving command +// traffic priority. Handles at least one member before honoring ctx (so a tiny +// DrainInterval can't stall it). No-ops if the pool is closed. +func (p *ConnPool) DrainIdleConns(ctx context.Context, st *DrainState, fn func(cn *Conn) error) { + if st == nil || fn == nil || p.closed() { + return + } + + if st.round == nil { + st.round = p.idleConnsSnapshot() + st.next = len(st.round) + if len(st.round) == 0 { + st.round = nil + return + } + } + + handled := 0 + for st.next > 0 { + // Min-progress: handle at least one member — drained OR reconciled — before + // honoring the per-cycle deadline, so a pass does a bounded amount of work + // while ignoring an expired ctx. A deadline-truncated round resumes on the + // next pass. + if handled > 0 && ctx.Err() != nil { + return + } + + // Account for the borrowed connection exactly like Get. Without a turn, + // a concurrent Get can observe the temporarily-empty idle pool and either + // exceed PoolSize or fail at MaxActiveConns. Maintenance never waits for a + // turn, so command traffic wins under contention. + if !p.semaphore.TryAcquire() { + return + } + st.next-- + cn := p.drainerPop(ctx, st.round[st.next]) + if cn == nil { + p.freeTurn() + // Reconcile: not a claimable idle member right now (closed, in use, + // unusable, or moved in idleConns by concurrent traffic). Covered by + // the command-path drain and/or the next round. + handled++ + continue + } + + func() { + defer p.endDrainerBorrow() + if err := fn(cn); err != nil { + // Fatal drain error (read/protocol/connection). + p.removeConnInternal(ctx, cn, err, true) + } else { + // Normal return: runs OnPut (queues any maintenance handoff). + p.putConn(ctx, cn, true) + } + }() + handled++ + } + + // Every member handled — round complete; snapshot a fresh round next pass. + st.round = nil + st.next = 0 +} + +// idleConnsSnapshot returns initialized idle connections and their current +// indexes. StateCreated MinIdleConns are intentionally excluded. +func (p *ConnPool) idleConnsSnapshot() []drainConn { + p.connsMu.Lock() + defer p.connsMu.Unlock() + if len(p.idleConns) == 0 { + return nil + } + round := make([]drainConn, 0, len(p.idleConns)) + for idx, cn := range p.idleConns { + if cn.stateMachine.GetState() == StateIdle { + round = append(round, drainConn{conn: cn, idleIndex: idx}) + } + } + return round +} + +// drainerPop claims a snapshotted connection (strict IDLE->IN_USE) and removes +// it from idleConns in O(1). Entries are processed in reverse index order, so +// swap removal cannot move an unprocessed round member. If concurrent pool +// traffic changed the slot, the member is deferred to the next round. +func (p *ConnPool) drainerPop(ctx context.Context, member drainConn) *Conn { + p.connsMu.Lock() + defer p.connsMu.Unlock() + if p.closed() { + return nil + } + idx := member.idleIndex + if idx < 0 || idx >= len(p.idleConns) || p.idleConns[idx] != member.conn { + return nil + } + cn := member.conn + if !cn.stateMachine.TryTransitionFast(StateIdle, StateInUse) { + return nil + } + p.beginDrainerBorrow() + last := len(p.idleConns) - 1 + p.idleConns[idx] = p.idleConns[last] + p.idleConns[last] = nil + p.idleConns = p.idleConns[:last] + p.idleConnsLen.Add(-1) + if cb := getMetricConnectionStateChangeCallback(); cb != nil { + cb(ctx, cn, MetricStateIdle, MetricStateUsed) + } + if cb := getMetricConnectionCountCallback(); cb != nil { + cb(ctx, -1, cn, "idle", false) + cb(ctx, 1, cn, "used", false) + } + return cn +} + func (p *ConnPool) Close() error { - if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) { + if !p._closed.CompareAndSwap(0, 1) { return ErrClosed } @@ -1735,8 +1999,11 @@ func (p *ConnPool) isHealthyConn(cn *Conn, nowNs int64) bool { if err := connCheck(cn.getNetConn()); err != nil { // If there's unexpected data, it might be push notifications (RESP3) if p.cfg.PushNotificationsEnabled && err == errUnexpectedRead { - // Peek at the reply type to check if it's a push notification - if replyType, err := cn.rd.PeekReplyType(); err == nil && replyType == proto.RespPush { + // Peek at the reply type to check if it's a push notification. + // Use the readerMu-guarded peek: a concurrent handoff may be + // resetting cn.rd via SetNetConn on a connection popped by Get + // before the OnGet state check rejects it. + if replyType, err := cn.PeekReplyTypeForCheck(); err == nil && replyType == proto.RespPush { // For RESP3 connections with push notifications, we allow some buffered data // The client will process these notifications before using the connection internal.Logger.Printf( diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go b/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go index 6763299e..e2206c82 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go @@ -35,11 +35,16 @@ func (e BadConnError) Unwrap() error { type StickyConnPool struct { pool Pooler - shared int32 // atomic + shared atomic.Int32 - state uint32 // atomic + state atomic.Uint32 ch chan *Conn + // onFirstConn runs once when this sticky pool claims a connection from its + // parent. CSC uses it to revoke cache ownership before the connection leaves + // the parent's background drainer. + onFirstConn func(*Conn) + _badConnError atomic.Value } @@ -53,7 +58,7 @@ func NewStickyConnPool(pool Pooler) *StickyConnPool { ch: make(chan *Conn, 1), } } - atomic.AddInt32(&p.shared, 1) + p.shared.Add(1) return p } @@ -68,13 +73,16 @@ func (p *StickyConnPool) CloseConn(ctx context.Context, cn *Conn, reason string, func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) { // In worst case this races with Close which is not a very common operation. for i := 0; i < 1000; i++ { - switch atomic.LoadUint32(&p.state) { + switch p.state.Load() { case stateDefault: cn, err := p.pool.Get(ctx) if err != nil { return nil, err } - if atomic.CompareAndSwapUint32(&p.state, stateDefault, stateInited) { + if p.state.CompareAndSwap(stateDefault, stateInited) { + if p.onFirstConn != nil { + p.onFirstConn(cn) + } return cn, nil } p.pool.Remove(ctx, cn, ErrClosed) @@ -96,12 +104,26 @@ func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) { return nil, fmt.Errorf("redis: StickyConnPool.Get: infinite loop") } +// SetOnFirstConn configures a callback that runs when the sticky pool first +// claims a parent connection. It must be called before the pool is used. +func (p *StickyConnPool) SetOnFirstConn(fn func(*Conn)) { + p.onFirstConn = fn +} + func (p *StickyConnPool) Put(ctx context.Context, cn *Conn) { defer func() { if recover() != nil { p.freeConn(ctx, cn) } }() + // A connection marked for removal on release (it may hold unread + // replies) must not be served to the next Get: record it as a bad + // connection — exactly like Remove — so Get refuses and the underlying + // connection is removed from the parent pool when the sticky pool + // unwinds (the parent's Put honors the same mark). + if reason := cn.CloseOnPutReason(); reason != "" { + p._badConnError.Store(BadConnError{wrapped: errors.New(reason)}) + } p.ch <- cn } @@ -130,16 +152,16 @@ func (p *StickyConnPool) RemoveWithoutTurn(ctx context.Context, cn *Conn, reason } func (p *StickyConnPool) Close() error { - if shared := atomic.AddInt32(&p.shared, -1); shared > 0 { + if shared := p.shared.Add(-1); shared > 0 { return nil } for i := 0; i < 1000; i++ { - state := atomic.LoadUint32(&p.state) + state := p.state.Load() if state == stateClosed { return ErrClosed } - if atomic.CompareAndSwapUint32(&p.state, state, stateClosed) { + if p.state.CompareAndSwap(state, stateClosed) { close(p.ch) cn, ok := <-p.ch if ok { @@ -168,8 +190,8 @@ func (p *StickyConnPool) Reset(ctx context.Context) error { return errors.New("redis: StickyConnPool does not have a Conn") } - if !atomic.CompareAndSwapUint32(&p.state, stateInited, stateDefault) { - state := atomic.LoadUint32(&p.state) + if !p.state.CompareAndSwap(stateInited, stateDefault) { + state := p.state.Load() return fmt.Errorf("redis: invalid StickyConnPool state: %d", state) } @@ -186,7 +208,7 @@ func (p *StickyConnPool) badConnError() error { } func (p *StickyConnPool) Len() int { - switch atomic.LoadUint32(&p.state) { + switch p.state.Load() { case stateDefault: return 0 case stateInited: diff --git a/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go b/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go index 8cfa8678..34c329ff 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go +++ b/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go @@ -7,6 +7,10 @@ import ( "sync/atomic" ) +// PubSubStats contains pub/sub connection pool stats. +// +// TODO(cxl): the uint32 fields below will be changed to atomic.Uint32 in v10, +// which is a breaking API change. type PubSubStats struct { Created uint32 Untracked uint32 diff --git a/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go b/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go index 33b027f7..28e1a781 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go +++ b/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go @@ -76,6 +76,11 @@ func (r *Reader) Buffered() int { return r.rd.Buffered() } +// Size returns the size of the underlying read buffer. +func (r *Reader) Size() int { + return r.rd.Size() +} + func (r *Reader) Peek(n int) ([]byte, error) { return r.rd.Peek(n) } @@ -100,18 +105,30 @@ func (r *Reader) PeekReplyType() (byte, error) { return b[0], nil } +// MinRESP3ReadBufferSize is the minimum buffer size used when RESP3 push +// notifications must be inspected without consuming them. +const MinRESP3ReadBufferSize = 128 + +// ErrPushNotificationNameTooLong is returned when the push header does not fit +// in the bounded peek window. Callers should consume the frame with ReadReply. +var ErrPushNotificationNameTooLong = errors.New("redis: push notification name exceeds peek window") + // PeekPushNotificationName returns the notification name of the next RESP3 // push frame without consuming it. The caller is expected to have already // verified that the next reply is a push notification (e.g. via PeekReplyType // returning RespPush). // -// To identify the name the method may block briefly reading more bytes from -// the underlying connection. That is safe: once the push marker '>' has been -// observed, the server is committed to sending the rest of the frame, so -// fetching the next few header bytes does not race with anything the caller -// could be waiting on. Blocking is preferred to a truncated peek, which would -// silently misidentify the notification and cause the caller's ReadReply to -// consume (and drop) the frame; see issue #3839. +// To identify the name the method may block reading more bytes from the +// underlying connection, but only ever waits for one byte beyond the valid +// frame prefix it has already seen. That byte is guaranteed to arrive: an +// incomplete prefix means the server is still committed to sending the rest +// of the frame. Demanding any fixed amount instead can deadlock — a complete +// frame such as a subscribe confirmation for a short channel name can be +// smaller than the fixed window, and once it is buffered the server has +// nothing more to send (issue #3935). Blocking for in-flight bytes is +// preferred to a truncated peek, which would silently misidentify the +// notification and cause the caller's ReadReply to consume (and drop) the +// frame; see issue #3839. func (r *Reader) PeekPushNotificationName() (string, error) { c, err := r.rd.Peek(1) if err != nil { @@ -121,16 +138,18 @@ func (r *Reader) PeekPushNotificationName() (string, error) { return "", fmt.Errorf("redis: can't peek push notification name, next reply is not a push notification") } - // Start with a peek window that covers every Redis-defined notification - // header (MOVING, MIGRATING, FAILED_OVER, message, pmessage, smessage, - // subscribe, unsubscribe, ...). If a longer name is encountered, grow - // the window up to maxPushHeaderPeek before giving up. - const initialPeek = 36 const maxPushHeaderPeek = 4096 - peekSize := initialPeek for { - buf, peekErr := r.rd.Peek(peekSize) + // Parse from what is already buffered; this never blocks. + avail := r.rd.Buffered() + if avail > maxPushHeaderPeek { + avail = maxPushHeaderPeek + } + buf, peekErr := r.rd.Peek(avail) + if peekErr != nil { + return "", peekErr + } name, complete, parseErr := parsePushNotificationName(buf) if parseErr != nil { return "", parseErr @@ -138,17 +157,17 @@ func (r *Reader) PeekPushNotificationName() (string, error) { if complete { return name, nil } - // Parser ran out of bytes. Surface a failed underlying read before - // growing further; otherwise grow the peek window and retry. - if peekErr != nil { - return "", peekErr - } - if peekSize >= maxPushHeaderPeek { - return "", fmt.Errorf("redis: push notification header exceeds %d bytes", maxPushHeaderPeek) + if avail >= maxPushHeaderPeek { + return "", ErrPushNotificationNameTooLong } - peekSize *= 2 - if peekSize > maxPushHeaderPeek { - peekSize = maxPushHeaderPeek + // Valid but incomplete prefix: the rest of the frame is in flight. + // Block for exactly one more byte — the read that delivers it picks + // up whatever else has already arrived — then re-parse. + if _, err := r.rd.Peek(avail + 1); err != nil { + if errors.Is(err, bufio.ErrBufferFull) { + return "", ErrPushNotificationNameTooLong + } + return "", err } } } @@ -440,6 +459,16 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) { if err != nil { return nil, err } + + // Reject unhashable keys (arrays/maps) before they are used as a map + // key, which would otherwise panic. This check must run before the + // value is read so it also guards the Nil and RedisError paths below, + // which write the key into the map and continue. + switch k.(type) { + case []interface{}, map[interface{}]interface{}: + return nil, fmt.Errorf("redis: RESP3 map key must be a scalar type, got %T", k) + } + v, err := r.ReadReply() if err != nil { if err == Nil { @@ -452,6 +481,7 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) { } return nil, err } + m[k] = v } return m, nil @@ -591,10 +621,33 @@ func (r *Reader) ReadStringInto(buf []byte) (int, error) { // bufio.Reader.Read first drains its internal buffer, then for // remaining data larger than its buffer size reads directly from the // underlying reader (socket) — effectively zero-copy. + // + // Fast path: when the caller VISIBLY hands over room for the trailing + // CRLF too (len(buf) >= n+2), read the payload and the CRLF in a + // single io.ReadFull. For large values this is one direct socket read + // instead of a big read followed by a tiny separate Discard(2) read, + // which is what makes GetToBuffer beat a regular Get (no payload + // allocation and the same number of reads). The 2 trailing bytes land + // past the returned length and are ignored. + // + // The gate is on len, NOT cap: a sub-slice of a larger buffer (e.g. + // packed segments big[i*slot:(i+1)*slot]) exposes trailing capacity + // that belongs to the caller's NEXT segment — writing the CRLF there + // would silently corrupt caller-owned memory outside the slice they + // passed. Callers who want the fast path pass len == payload+2 (the + // returned length is still the payload length). + if len(buf) >= n+2 { + full := buf[:n+2] + if _, err := io.ReadFull(r.rd, full); err != nil { + return 0, err + } + return n, nil + } + // Slow path: buffer is exactly large enough for the payload only, so + // read the payload into it and discard the CRLF separately. if _, err := io.ReadFull(r.rd, buf[:n]); err != nil { return 0, err } - // Discard trailing \r\n. if _, err := r.rd.Discard(2); err != nil { return 0, err } @@ -738,7 +791,15 @@ func (r *Reader) Discard(line []byte) (err error) { } n, err := replyLen(line) - if err != nil && err != Nil { + if err != nil { + if err == Nil { + // A nil reply ($-1, =-1, !-1, *-1, %-1) carries no payload; the + // header line was already consumed by readLine, so there is + // nothing to discard. Falling through would Discard(n+2)==2 bytes + // that belong to the next reply and desync the stream, matching + // how readRawReplyBuf/readRawReplyWriteTo already treat Nil. + return nil + } return err } @@ -755,8 +816,14 @@ func (r *Reader) Discard(line []byte) (err error) { } return nil case RespMap, RespAttr: - // Read key & value. - for i := 0; i < n*2; i++ { + // Iterate over the n key/value pairs rather than n*2 elements: a count + // above MaxInt/2 makes n*2 overflow to a negative loop bound, which + // would skip the body entirely and return nil, leaving the map bytes in + // the stream for the next reply to consume (a silent desync). + for i := 0; i < n; i++ { + if err = r.DiscardNext(); err != nil { + return err + } if err = r.DiscardNext(); err != nil { return err } @@ -849,10 +916,12 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) { } return buf, err } - for i := 0; i < n*2; i++ { - buf, err = r.readRawReplyBuf(buf) - if err != nil { - return buf, err + for i := 0; i < n; i++ { + for pair := 0; pair < 2; pair++ { + buf, err = r.readRawReplyBuf(buf) + if err != nil { + return buf, err + } } } return buf, nil @@ -867,11 +936,15 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) { } return buf, err } - // Read the attribute key-value pairs - for i := 0; i < n*2; i++ { - buf, err = r.readRawReplyBuf(buf) - if err != nil { - return buf, err + // Read the attribute key-value pairs. Iterate over pairs rather than + // n*2 elements so a count above MaxInt/2 can't overflow int to a + // negative loop bound and skip the body. + for i := 0; i < n; i++ { + for pair := 0; pair < 2; pair++ { + buf, err = r.readRawReplyBuf(buf) + if err != nil { + return buf, err + } } } // Read the command reply that follows the attribute @@ -948,11 +1021,13 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) { } return written, err } - for i := 0; i < count*2; i++ { - n, err := r.readRawReplyWriteTo(w) - written += n - if err != nil { - return written, err + for i := 0; i < count; i++ { + for pair := 0; pair < 2; pair++ { + n, err := r.readRawReplyWriteTo(w) + written += n + if err != nil { + return written, err + } } } return written, nil @@ -967,12 +1042,16 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) { } return written, err } - // Read the attribute key-value pairs - for i := 0; i < count*2; i++ { - n, err := r.readRawReplyWriteTo(w) - written += n - if err != nil { - return written, err + // Read the attribute key-value pairs. Iterate over pairs rather than + // count*2 elements so a count above MaxInt/2 can't overflow int to a + // negative loop bound and skip the body. + for i := 0; i < count; i++ { + for pair := 0; pair < 2; pair++ { + n, err := r.readRawReplyWriteTo(w) + written += n + if err != nil { + return written, err + } } } // Read the command reply that follows the attribute diff --git a/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go b/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go index 38e66c68..e3eff8b6 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go +++ b/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go @@ -118,7 +118,7 @@ func (w *Writer) WriteArg(v interface{}) error { return w.uint(uint64(v)) case *uint8: if v == nil { - return w.string("") + return w.uint(0) } return w.uint(uint64(*v)) case uint16: diff --git a/vendor/github.com/redis/go-redis/v9/internal/semaphore.go b/vendor/github.com/redis/go-redis/v9/internal/semaphore.go index a7f40466..b8d60a7b 100644 --- a/vendor/github.com/redis/go-redis/v9/internal/semaphore.go +++ b/vendor/github.com/redis/go-redis/v9/internal/semaphore.go @@ -14,6 +14,28 @@ var semTimers = sync.Pool{ }, } +// putSemTimer stops a pooled timer and drains a stale fire before reuse, +// portably across both timer-channel semantics (the drain must never block): +// +// - main module on go >= 1.23 (synchronous channels): Stop returns TRUE for +// an expired-but-undelivered fire — the delivery is aborted — and false +// only once the value was actually received. With the sole receiver +// being Acquire's own select, the drain branch is unreachable; the +// select-with-default is a safety net so a future semantics shift cannot +// turn it into a blocking receive (reviewed on #3942). +// - GODEBUG=asynctimerchan=1 (consumer main module on go < 1.23, old +// buffered channels): Stop returns false and the fired value sits in the +// buffer; the drain consumes it so the timer is clean for Reset-reuse. +func putSemTimer(t *time.Timer) { + if !t.Stop() { + select { + case <-t.C: + default: + } + } + semTimers.Put(t) +} + // FastSemaphore is a channel-based semaphore optimized for performance. // It uses a fast path that avoids timer allocation when tokens are available. // The channel is pre-filled with tokens: Acquire = receive, Release = send. @@ -70,19 +92,13 @@ func (s *FastSemaphore) Acquire(ctx context.Context, timeout time.Duration, time // Slow path: need to wait with timeout timer := semTimers.Get().(*time.Timer) - defer semTimers.Put(timer) + defer putSemTimer(timer) timer.Reset(timeout) select { case <-s.tokens: - if !timer.Stop() { - <-timer.C - } return nil case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } return ctx.Err() case <-timer.C: return timeoutErr @@ -152,42 +168,20 @@ func (s *FIFOSemaphore) TryAcquire() bool { func (s *FIFOSemaphore) Acquire(ctx context.Context, timeout time.Duration, timeoutErr error) error { // No fast path - always use timer to guarantee FIFO timer := semTimers.Get().(*time.Timer) - defer semTimers.Put(timer) + defer putSemTimer(timer) timer.Reset(timeout) select { case <-s.tokens: - if !timer.Stop() { - <-timer.C - } return nil case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } return ctx.Err() case <-timer.C: return timeoutErr } } -// AcquireBlocking acquires a token, blocking indefinitely until one is available. -func (s *FIFOSemaphore) AcquireBlocking() { - <-s.tokens -} - // Release releases a token back to the semaphore. func (s *FIFOSemaphore) Release() { s.tokens <- struct{}{} } - -// Close closes the semaphore, unblocking all waiting goroutines. -// After close, all Acquire calls will receive a closed channel signal. -func (s *FIFOSemaphore) Close() { - close(s.tokens) -} - -// Len returns the current number of acquired tokens. -func (s *FIFOSemaphore) Len() int32 { - return s.max - int32(len(s.tokens)) -} diff --git a/vendor/github.com/redis/go-redis/v9/iterator.go b/vendor/github.com/redis/go-redis/v9/iterator.go index cd1a8285..a0cf1b35 100644 --- a/vendor/github.com/redis/go-redis/v9/iterator.go +++ b/vendor/github.com/redis/go-redis/v9/iterator.go @@ -46,6 +46,14 @@ func (it *ScanIterator) Next(ctx context.Context) bool { if err != nil { return false } + // Await the fetch before reading page/cursor: on the deferred + // autopipeline face process() only enqueues, and reading the raw + // fields of a not-yet-executed command would spin re-issuing SCANs + // with a stale cursor forever. Err() blocks until executed there and + // is a no-op read everywhere else. + if err := it.cmd.Err(); err != nil { + return false + } it.pos = 1 diff --git a/vendor/github.com/redis/go-redis/v9/json.go b/vendor/github.com/redis/go-redis/v9/json.go index 2bcad0b7..b3878260 100644 --- a/vendor/github.com/redis/go-redis/v9/json.go +++ b/vendor/github.com/redis/go-redis/v9/json.go @@ -96,6 +96,7 @@ func newJSONCmd(ctx context.Context, args ...interface{}) *JSONCmd { } func (cmd *JSONCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -105,6 +106,7 @@ func (cmd *JSONCmd) SetVal(val string) { // Val returns the result of the JSON.GET command as a string. func (cmd *JSONCmd) Val() string { + cmd.await() if len(cmd.val) == 0 && cmd.expanded != nil { val, err := json.Marshal(cmd.expanded) if err != nil { @@ -119,11 +121,13 @@ func (cmd *JSONCmd) Val() string { } func (cmd *JSONCmd) Result() (string, error) { + cmd.await() return cmd.Val(), cmd.Err() } // Expanded returns the result of the JSON.GET command as unmarshalled JSON. func (cmd *JSONCmd) Expanded() (interface{}, error) { + cmd.await() if len(cmd.val) != 0 && cmd.expanded == nil { err := json.Unmarshal([]byte(cmd.val), &cmd.expanded) if err != nil { @@ -136,15 +140,18 @@ func (cmd *JSONCmd) Expanded() (interface{}, error) { func (cmd *JSONCmd) readReply(rd *proto.Reader) error { // nil response from JSON.(M)GET (cmd.baseCmd.err will be "redis: nil") - // This happens when the key doesn't exist - if cmd.baseCmd.Err() == Nil { + // This happens when the key doesn't exist. + // Use rawErr() (not Err()): readReply runs inside the batch's Exec, before + // the autopipeline batch's done channel is closed, so Err()->await() would + // deadlock on the very Exec that is calling readReply. + if cmd.baseCmd.rawErr() == Nil { cmd.val = "" return Nil } // Handle other base command errors - if cmd.baseCmd.Err() != nil { - return cmd.baseCmd.Err() + if cmd.baseCmd.rawErr() != nil { + return cmd.baseCmd.rawErr() } if readType, err := rd.PeekReplyType(); err != nil { @@ -212,6 +219,7 @@ func NewJSONSliceCmd(ctx context.Context, args ...interface{}) *JSONSliceCmd { } func (cmd *JSONSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -220,15 +228,19 @@ func (cmd *JSONSliceCmd) SetVal(val []interface{}) { } func (cmd *JSONSliceCmd) Val() []interface{} { + cmd.await() return cmd.val } func (cmd *JSONSliceCmd) Result() ([]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error { - if cmd.baseCmd.Err() == Nil { + // rawErr(), not Err(): readReply runs inside Exec before the batch's done + // channel closes, so Err()->await() would deadlock (see JSONCmd.readReply). + if cmd.baseCmd.rawErr() == Nil { cmd.val = nil return Nil } @@ -238,7 +250,7 @@ func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error { } else if readType == proto.RespArray { response, err := rd.ReadReply() if err != nil { - return nil + return err } else { cmd.val = response.([]interface{}) } @@ -299,6 +311,7 @@ func NewIntPointerSliceCmd(ctx context.Context, args ...interface{}) *IntPointer } func (cmd *IntPointerSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -307,10 +320,12 @@ func (cmd *IntPointerSliceCmd) SetVal(val []*int64) { } func (cmd *IntPointerSliceCmd) Val() []*int64 { + cmd.await() return cmd.val } func (cmd *IntPointerSliceCmd) Result() ([]*int64, error) { + cmd.await() return cmd.val, cmd.err } diff --git a/vendor/github.com/redis/go-redis/v9/list_commands.go b/vendor/github.com/redis/go-redis/v9/list_commands.go index 9d9e16c6..afaf58e1 100644 --- a/vendor/github.com/redis/go-redis/v9/list_commands.go +++ b/vendor/github.com/redis/go-redis/v9/list_commands.go @@ -34,6 +34,8 @@ type ListCmdable interface { RPushX(ctx context.Context, key string, values ...interface{}) *IntCmd LMove(ctx context.Context, source, destination, srcpos, destpos string) *StringCmd BLMove(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration) *StringCmd + LMoveM(ctx context.Context, source, destination, srcpos, destpos string, args LMoveMArgs) *StringSliceCmd + BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, args LMoveMArgs) *StringSliceCmd } func (c cmdable) BLPop(ctx context.Context, timeout time.Duration, keys ...string) *StringSliceCmd { @@ -156,6 +158,45 @@ type LPosArgs struct { Rank, MaxLen int64 } +// LMoveMMode is the count semantics for LMOVEM/BLMOVEM. +type LMoveMMode string + +const ( + LMoveMCount LMoveMMode = "COUNT" // up to Count + LMoveMExactly LMoveMMode = "EXACTLY" // exactly Count, or nothing +) + +// LMoveMOrder is the destination ordering for LMOVEM/BLMOVEM. +type LMoveMOrder string + +const ( + LMoveMOBO LMoveMOrder = "OBO" // one-by-one, order reversed + LMoveMBulk LMoveMOrder = "BULK" // preserve order +) + +// LMoveMArgs configures the optional count group of LMOVEM/BLMOVEM. +// Count <= 0 moves a single element. Mode defaults to COUNT, Order to BULK. +type LMoveMArgs struct { + Mode LMoveMMode + Count int64 + Order LMoveMOrder +} + +func (a LMoveMArgs) appendArgs(args []interface{}) []interface{} { + if a.Count <= 0 { + return args + } + mode := a.Mode + if mode == "" { + mode = LMoveMCount + } + order := a.Order + if order == "" { + order = LMoveMBulk + } + return append(args, string(mode), a.Count, string(order)) +} + func (c cmdable) LPos(ctx context.Context, key string, value string, a LPosArgs) *IntCmd { args := []interface{}{"lpos", key, value} if a.Rank != 0 { @@ -295,3 +336,27 @@ func (c cmdable) BLMove( _ = c(ctx, cmd) return cmd } + +// LMoveM atomically moves multiple elements between lists (Redis 8.10+). +// srcpos/destpos are "LEFT" or "RIGHT". Returns moved elements, or redis.Nil if none. +func (c cmdable) LMoveM(ctx context.Context, source, destination, srcpos, destpos string, a LMoveMArgs) *StringSliceCmd { + args := make([]interface{}, 5, 8) + args[0], args[1], args[2], args[3], args[4] = "lmovem", source, destination, srcpos, destpos + args = a.appendArgs(args) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// BLMoveM is the blocking variant of LMoveM (Redis 8.10+); timeout 0 blocks forever. +// Returns moved elements, or redis.Nil on timeout. +func (c cmdable) BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, a LMoveMArgs) *StringSliceCmd { + args := make([]interface{}, 6, 9) + args[0], args[1], args[2], args[3], args[4] = "blmovem", source, destination, srcpos, destpos + args[5] = formatSec(ctx, timeout) + args = a.appendArgs(args) + cmd := NewStringSliceCmd(ctx, args...) + cmd.setReadTimeout(timeout) + _ = c(ctx, cmd) + return cmd +} diff --git a/vendor/github.com/redis/go-redis/v9/local_cache.go b/vendor/github.com/redis/go-redis/v9/local_cache.go new file mode 100644 index 00000000..2a5579c6 --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/local_cache.go @@ -0,0 +1,774 @@ +package redis + +import ( + "context" + "math" + "sync" + "sync/atomic" + "time" +) + +// cacheEntryState tracks the lifecycle of a local cache entry. +type cacheEntryState uint8 + +const ( + // cacheEntryInProgress marks a placeholder entry while a value is being fetched. + cacheEntryInProgress cacheEntryState = iota + // cacheEntryValid marks an entry that contains a value that can be returned. + cacheEntryValid +) + +// cacheEntry represents a cached command reply and its Redis-key associations. +type cacheEntry struct { + cacheKey string + redisKeys []string + value []byte + state cacheEntryState + + token uint64 + sizeBytes int64 + reservedAt time.Time + waitCh chan struct{} + waitClosed bool + + // lastAccessNs is a recency token for LRU eviction: a global atomic counter + // bumped on every access, stored atomically so the read path can mark a + // touch under the shard's RLock without upgrading to a write lock. + lastAccessNs atomic.Int64 + + // validAt retains time.Now's monotonic component for the MaxStaleness + // backstop, so wall-clock corrections cannot extend an entry's lifetime. + // Written under Lock (Set/Fulfill), read under RLock (get). + validAt time.Time + + // ownerConnID is the conn that fetched this entry (set by FulfillOwned; 0 = + // none). Default CLIENT TRACKING sends a key's invalidation only to that + // conn, so the entry must be evicted when it goes away (see EvictByConn). + ownerConnID uint64 +} + +// lruSequence is the global monotonic counter feeding lastAccessNs. It totally +// orders recency across all entries in all shards for approximate-LRU eviction. +var lruSequence atomic.Int64 + +// nextLRUToken returns the next strictly-greater LRU token. +func nextLRUToken() int64 { + return lruSequence.Add(1) +} + +// CacheSizer calculates estimated memory usage in bytes for a cache entry. +// +// Experimental: this API may change in a minor release. +type CacheSizer func(cacheKey string, redisKeys []string, value []byte) int64 + +// CacheConfig configures a local cache instance. +// +// Experimental: this API may change in a minor release. +type CacheConfig struct { + // MaxEntries limits the number of entries. Zero or negative means unlimited. + MaxEntries int + // MaxMemoryBytes limits estimated memory usage in bytes. Zero or negative means unlimited. + // + // If both MaxEntries and MaxMemoryBytes are unlimited, MaxEntries defaults to + // defaultCacheMaxEntries so the cache cannot grow without bound. The cache is + // sharded 16 ways (above small thresholds) and each shard enforces its 1/16 + // share, so an entry larger than MaxMemoryBytes/16 is never admitted — + // size it to at least 16× your largest reply. + MaxMemoryBytes int64 + // Sizer estimates memory usage per entry. If nil, a built-in approximation is used. + // + // Sizer may be invoked concurrently from multiple goroutines and must be + // thread-safe. It must return quickly and must not call back into the + // cache (Get, Set, Delete*, Flush, etc.): some call sites hold an internal + // shard lock, so re-entry can deadlock. + Sizer CacheSizer + // StaleTimeout is the duration after which an IN_PROGRESS placeholder is + // considered stale and eligible for takeover by a new Reserve call. + // If zero, defaults to defaultStaleTimeout (5s). + StaleTimeout time.Duration + + // DrainInterval is the background-drainer period (default 5ms; zero uses the + // default): how often idle pool conns are swept for buffered "invalidate" + // frames, roughly bounding cache-hit staleness. Values below 1ms are clamped + // to 1ms. + DrainInterval time.Duration + + // MaxStaleness caps how long a cached entry is served after it became valid, + // regardless of invalidation. It is a correctness + // BACKSTOP for lost invalidations or connection-lifecycle gaps ("Window 2"), not + // the primary freshness mechanism. Keep it well above the invalidation round-trip + // (e.g. seconds); per-entry refetch overhead scales ~1/MaxStaleness. + // + // Default: 0 (disabled). + MaxStaleness time.Duration +} + +// Cache is the thread-safe storage contract used by client-side caching. +// +// All methods may be called concurrently. Cache keys and Redis keys are opaque +// strings and must be preserved exactly. Removing a reservation must wake any +// Get calls waiting for it. +// +// Reserve must allow only one caller to fetch a missing key and return a token +// that is valid until FulfillOwned, Cancel, or an eviction removes that +// reservation. FulfillOwned and Cancel must modify only a reservation with the +// matching token. Get may wait for an in-progress reservation and must stop +// waiting when ctx is done. +// +// Experimental: this API may change in a minor release. +type Cache interface { + Get(ctx context.Context, cacheKey string) ([]byte, bool) + Reserve(cacheKey string, redisKeys []string) (token uint64, shouldFetch bool) + // FulfillOwned publishes a reserved value and records the connection that + // fetched it so the entry can be evicted if that connection loses tracking. + FulfillOwned(cacheKey string, token, ownerConnID uint64, value []byte) bool + Cancel(cacheKey string, token uint64) bool + DeleteByRedisKey(redisKey string) int + DeleteByCacheKey(cacheKey string) bool + // EvictByConn removes every entry fetched by connID. + EvictByConn(connID uint64) int + Flush() int +} + +const ( + defaultStaleTimeout = 5 * time.Second + defaultCacheShardCount = 16 + + // defaultCacheMaxEntries bounds the cache when the config leaves both + // MaxEntries and MaxMemoryBytes unlimited (matches the 10k-entry default + // other Redis clients use, e.g. redis-py). + defaultCacheMaxEntries = 10000 + + // shardingThresholdEntries / shardingThresholdBytes: caches with capacity + // below these thresholds fall back to a single shard so global LRU / + // memory-cap semantics behave exactly as a non-sharded cache would. + shardingThresholdEntries = 64 + shardingThresholdBytes = 64 * 1024 +) + +// NewLocalCache creates a thread-safe local cache with approximate-LRU +// eviction. The cache is internally sharded by cache-key hash to reduce +// mutex contention under high concurrent access. +// +// Experimental: this API may change in a minor release. +func NewLocalCache(cfg CacheConfig) *LocalCache { + sizer := cfg.Sizer + if sizer == nil { + sizer = defaultCacheSizer + } + + staleTimeout := cfg.StaleTimeout + if staleTimeout <= 0 { + staleTimeout = defaultStaleTimeout + } + + maxEntries := cfg.MaxEntries + maxMemoryBytes := cfg.MaxMemoryBytes + // An unbounded cache can grow until the process OOMs; require at least + // one limit. + if maxEntries <= 0 && maxMemoryBytes <= 0 { + maxEntries = defaultCacheMaxEntries + } + + shardCount := defaultCacheShardCount + if maxEntries > 0 && maxEntries < shardingThresholdEntries { + shardCount = 1 + } + if maxMemoryBytes > 0 && maxMemoryBytes < int64(shardingThresholdBytes) { + shardCount = 1 + } + + c := &LocalCache{ + shards: make([]cacheShard, shardCount), + shardCount: uint32(shardCount), + shardMask: uint32(shardCount - 1), + sizer: sizer, + } + for i := range c.shards { + s := &c.shards[i] + s.entries = make(map[string]*cacheEntry) + s.byRedisKey = make(map[string]map[string]struct{}) + s.byConnID = make(map[uint64]map[string]struct{}) + // Distribute capacity so the per-shard caps sum to exactly the + // configured limits; a ceil-per-shard split would let total residency + // exceed MaxEntries/MaxMemoryBytes. + if maxEntries > 0 { + s.maxEntries = maxEntries / shardCount + if i < maxEntries%shardCount { + s.maxEntries++ + } + } + if maxMemoryBytes > 0 { + s.maxMemoryBytes = maxMemoryBytes / int64(shardCount) + if int64(i) < maxMemoryBytes%int64(shardCount) { + s.maxMemoryBytes++ + } + } + s.maxStaleness = cfg.MaxStaleness + s.sizer = sizer + s.staleTimeout = staleTimeout + } + return c +} + +// LocalCache is the built-in sharded approximate-LRU cache. +// +// Experimental: this API may change in a minor release. +type LocalCache struct { + shards []cacheShard + shardCount uint32 + shardMask uint32 + sizer CacheSizer + + nextToken atomic.Uint64 + hits atomic.Uint64 + misses atomic.Uint64 +} + +var _ Cache = (*LocalCache)(nil) + +// cacheShard holds the state for one shard of LocalCache. The mutex +// protects entries, byRedisKey, byConnID, and usedBytes. +type cacheShard struct { + mu sync.RWMutex + entries map[string]*cacheEntry + byRedisKey map[string]map[string]struct{} + // byConnID is the owning-conn reverse index (twin of byRedisKey): conn id -> + // its cache keys. Populated by FulfillOwned, cleaned in removeEntryLocked, + // consumed by EvictByConn. + byConnID map[uint64]map[string]struct{} + usedBytes int64 + + maxEntries int + maxMemoryBytes int64 + maxStaleness time.Duration + sizer CacheSizer + staleTimeout time.Duration +} + +// shardFor returns the shard responsible for cacheKey. +func (c *LocalCache) shardFor(cacheKey string) *cacheShard { + if c.shardCount == 1 { + return &c.shards[0] + } + return &c.shards[fnv1a32(cacheKey)&c.shardMask] +} + +// fnv1a32 returns the FNV-1a 32-bit hash of s. Allocation-free. +func fnv1a32(s string) uint32 { + const ( + offset uint32 = 2166136261 + prime uint32 = 16777619 + ) + h := offset + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= prime + } + return h +} + +const defaultCacheEntryOverhead int64 = 96 + +func defaultCacheSizer(cacheKey string, redisKeys []string, value []byte) int64 { + size := defaultCacheEntryOverhead + int64(len(cacheKey)+len(value)) + for _, key := range redisKeys { + size += int64(len(key)) + 16 + } + if size < 0 { + return 0 + } + return size +} + +// Get returns a copy of a cached value, waiting for an in-progress fetch when +// necessary. +func (c *LocalCache) Get(ctx context.Context, cacheKey string) ([]byte, bool) { + if ctx == nil { + ctx = context.Background() + } + value, ok := c.shardFor(cacheKey).get(ctx, cacheKey) + if ok { + c.hits.Add(1) + } else { + c.misses.Add(1) + } + return value, ok +} + +// get is the read-side hot path. Holds only the shard's read lock; updates +// the LRU recency timestamp via atomic store on the entry — no write-lock +// upgrade is needed. +func (s *cacheShard) get(ctx context.Context, cacheKey string) ([]byte, bool) { + for { + s.mu.RLock() + entry, ok := s.entries[cacheKey] + if !ok { + s.mu.RUnlock() + return nil, false + } + + if entry.state == cacheEntryInProgress { + waitCh := entry.waitCh + // Bound the wait by the placeholder's remaining stale window so an + // abandoned reservation cannot block waiters indefinitely. + remaining := s.staleTimeout - time.Since(entry.reservedAt) + s.mu.RUnlock() + if waitCh == nil { + // Defensive: treat a missing waitCh as a miss to avoid busy-looping. + return nil, false + } + if remaining <= 0 { + // Placeholder already stale; miss so the caller refetches. + return nil, false + } + // Wait for the in-flight fetch to either publish (Fulfill) or abort (Cancel/Delete/Flush). + timer := time.NewTimer(remaining) + select { + case <-waitCh: + timer.Stop() + case <-ctx.Done(): + timer.Stop() + return nil, false + case <-timer.C: + return nil, false + } + continue + } + + if entry.state != cacheEntryValid { + s.mu.RUnlock() + return nil, false + } + + // Max-staleness backstop: a Valid entry older than maxStaleness is treated + // as a miss and evicted, so a lost invalidation or connection-lifecycle + // staleness (Window 2) cannot keep a stale value resident past MaxStaleness. + // Evict under the write lock so the next access re-fetches — a stale-but-present + // entry would otherwise suppress the re-fetch via Reserve. + if s.maxStaleness > 0 && time.Since(entry.validAt) > s.maxStaleness { + s.mu.RUnlock() + s.mu.Lock() + if cur, ok := s.entries[cacheKey]; ok && cur == entry { + s.removeEntryLocked(cacheKey) + } + s.mu.Unlock() + return nil, false + } + + value := cloneBytes(entry.value) + // Record access timestamp without upgrading the lock. Last writer + // wins; cross-goroutine ordering of timestamps is fine for + // approximate-LRU semantics. + entry.lastAccessNs.Store(nextLRUToken()) + s.mu.RUnlock() + return value, true + } +} + +// Stats returns cumulative activity and current residency. +func (c *LocalCache) Stats() CSCStats { + return CSCStats{ + Hits: c.hits.Load(), + Misses: c.misses.Load(), + Entries: c.Len(), + MemoryUsageBytes: c.MemoryUsage(), + } +} + +// Reserve claims a missing cache key for fetching. +func (c *LocalCache) Reserve(cacheKey string, redisKeys []string) (token uint64, shouldFetch bool) { + keysCopy := cloneStrings(redisKeys) + waitCh := make(chan struct{}) + reservedAt := time.Now() + sizeBytes := c.sizer(cacheKey, keysCopy, nil) + if sizeBytes < 0 { + sizeBytes = 0 + } + newToken := c.nextToken.Add(1) + + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + if entry, ok := s.entries[cacheKey]; ok { + switch entry.state { + case cacheEntryValid: + // Existing-VALID hit: record access; caller will re-Get to + // retrieve. + entry.lastAccessNs.Store(nextLRUToken()) + return 0, false + case cacheEntryInProgress: + if time.Since(entry.reservedAt) < s.staleTimeout { + return 0, false + } + s.removeEntryLocked(cacheKey) + default: + return 0, false + } + } + + if s.maxMemoryBytes > 0 && sizeBytes > s.maxMemoryBytes { + return 0, true + } + + entry := &cacheEntry{ + cacheKey: cacheKey, + redisKeys: keysCopy, + state: cacheEntryInProgress, + token: newToken, + reservedAt: reservedAt, + waitCh: waitCh, + sizeBytes: sizeBytes, + } + entry.lastAccessNs.Store(nextLRUToken()) + + s.setEntryLocked(entry) + // Evict only Valid victims. If still over capacity the shard holds only + // in-flight placeholders: rather than abort a peer's fetch, drop this + // reservation (the caller fetches uncached). The hard cap holds either way. + s.evictValidLocked() + if s.overCapacityLocked() { + s.removeEntryLocked(cacheKey) + return 0, true + } + if s.entries[cacheKey] != entry { + return 0, true + } + return newToken, true +} + +// FulfillOwned publishes a reserved value and records ownerConnID so +// EvictByConn can drop it when that connection is removed. ownerConnID == 0 +// leaves the value unowned. +func (c *LocalCache) FulfillOwned(cacheKey string, token, ownerConnID uint64, value []byte) bool { + return c.fulfill(cacheKey, token, ownerConnID, value) +} + +func (c *LocalCache) fulfill(cacheKey string, token, ownerConnID uint64, value []byte) bool { + valueCopy := cloneBytes(value) + + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[cacheKey] + if !ok || entry.state != cacheEntryInProgress || entry.token != token { + return false + } + + valueSize := s.sizer(cacheKey, entry.redisKeys, valueCopy) + if valueSize < 0 { + valueSize = 0 + } + if s.maxMemoryBytes > 0 && valueSize > s.maxMemoryBytes { + s.removeEntryLocked(cacheKey) + return false + } + + s.usedBytes += valueSize - entry.sizeBytes + entry.value = valueCopy + entry.sizeBytes = valueSize + entry.state = cacheEntryValid + entry.validAt = time.Now() + entry.token = 0 + entry.lastAccessNs.Store(nextLRUToken()) + if ownerConnID != 0 { + entry.ownerConnID = ownerConnID + s.indexConnLocked(ownerConnID, cacheKey) + } + s.closeWaitersLocked(entry) + + s.evictIfNeededLocked() + current, stillExists := s.entries[cacheKey] + return stillExists && current == entry && entry.state == cacheEntryValid +} + +// EvictByConn removes every entry fetched by connID and returns the count. +// Called when a conn is removed/swapped: the server stops delivering those +// keys' invalidations, so keeping them risks stale serves. Errs toward a miss. +func (c *LocalCache) EvictByConn(connID uint64) int { + if connID == 0 { + return 0 + } + removed := 0 + for i := range c.shards { + removed += c.shards[i].evictByConn(connID) + } + return removed +} + +func (s *cacheShard) evictByConn(connID uint64) int { + s.mu.Lock() + defer s.mu.Unlock() + + cacheKeys, ok := s.byConnID[connID] + if !ok { + return 0 + } + toRemove := make([]string, 0, len(cacheKeys)) + for cacheKey := range cacheKeys { + toRemove = append(toRemove, cacheKey) + } + removed := 0 + for _, cacheKey := range toRemove { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// indexConnLocked records cacheKey under connID in the owning-connection index. +func (s *cacheShard) indexConnLocked(connID uint64, cacheKey string) { + cacheKeys := s.byConnID[connID] + if cacheKeys == nil { + cacheKeys = make(map[string]struct{}) + s.byConnID[connID] = cacheKeys + } + cacheKeys[cacheKey] = struct{}{} +} + +// Cancel removes the reservation matching token. +func (c *LocalCache) Cancel(cacheKey string, token uint64) bool { + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[cacheKey] + if !ok || entry.state != cacheEntryInProgress || entry.token != token { + return false + } + + s.removeEntryLocked(cacheKey) + return true +} + +// DeleteByRedisKey removes entries associated with redisKey. +func (c *LocalCache) DeleteByRedisKey(redisKey string) int { + removed := 0 + for i := range c.shards { + removed += c.shards[i].deleteByRedisKey(redisKey) + } + return removed +} + +func (s *cacheShard) deleteByRedisKey(redisKey string) int { + s.mu.Lock() + defer s.mu.Unlock() + + cacheKeys, ok := s.byRedisKey[redisKey] + if !ok { + return 0 + } + + // Remove IN_PROGRESS placeholders too: an invalidation can arrive on a + // different stream than the in-flight reply (the background drainer), so the + // fetch may predate the write. Removing makes the racing Fulfill fail and + // waiters refetch, so a raced-invalidation value is never published. + toRemove := make([]string, 0, len(cacheKeys)) + for cacheKey := range cacheKeys { + toRemove = append(toRemove, cacheKey) + } + + removed := 0 + for _, cacheKey := range toRemove { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// DeleteByCacheKey removes one entry by its internal cache key. +func (c *LocalCache) DeleteByCacheKey(cacheKey string) bool { + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + return s.removeEntryLocked(cacheKey) +} + +// Flush removes all entries. +func (c *LocalCache) Flush() int { + removed := 0 + for i := range c.shards { + removed += c.shards[i].flush() + } + return removed +} + +func (s *cacheShard) flush() int { + s.mu.Lock() + defer s.mu.Unlock() + + // Flush placeholders too (see deleteByRedisKey): a flush (FLUSHDB, or the + // owned-cache flush on Close) means everything, including in-flight fetches, + // may be stale. + removed := 0 + for cacheKey := range s.entries { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// Len returns the current number of entries and reservations. +func (c *LocalCache) Len() int { + n := 0 + for i := range c.shards { + s := &c.shards[i] + s.mu.RLock() + n += len(s.entries) + s.mu.RUnlock() + } + return n +} + +// MemoryUsage returns the cache's estimated memory usage in bytes. +func (c *LocalCache) MemoryUsage() int64 { + var total int64 + for i := range c.shards { + s := &c.shards[i] + s.mu.RLock() + total += s.usedBytes + s.mu.RUnlock() + } + return total +} + +func (s *cacheShard) setEntryLocked(entry *cacheEntry) { + if old, exists := s.entries[entry.cacheKey]; exists { + s.removeEntryLocked(old.cacheKey) + } + + s.entries[entry.cacheKey] = entry + s.usedBytes += entry.sizeBytes + + for _, redisKey := range entry.redisKeys { + cacheKeys := s.byRedisKey[redisKey] + if cacheKeys == nil { + cacheKeys = make(map[string]struct{}) + s.byRedisKey[redisKey] = cacheKeys + } + cacheKeys[entry.cacheKey] = struct{}{} + } +} + +func (s *cacheShard) removeEntryLocked(cacheKey string) bool { + entry, exists := s.entries[cacheKey] + if !exists { + return false + } + + delete(s.entries, cacheKey) + s.usedBytes -= entry.sizeBytes + if s.usedBytes < 0 { + s.usedBytes = 0 + } + + for _, redisKey := range entry.redisKeys { + cacheKeys := s.byRedisKey[redisKey] + if cacheKeys == nil { + continue + } + delete(cacheKeys, cacheKey) + if len(cacheKeys) == 0 { + delete(s.byRedisKey, redisKey) + } + } + + if entry.ownerConnID != 0 { + if cacheKeys := s.byConnID[entry.ownerConnID]; cacheKeys != nil { + delete(cacheKeys, cacheKey) + if len(cacheKeys) == 0 { + delete(s.byConnID, entry.ownerConnID) + } + } + } + + s.closeWaitersLocked(entry) + return true +} + +func (s *cacheShard) closeWaitersLocked(entry *cacheEntry) { + if entry.waitCh != nil && !entry.waitClosed { + close(entry.waitCh) + entry.waitClosed = true + } +} + +func (s *cacheShard) overCapacityLocked() bool { + if s.maxEntries > 0 && len(s.entries) > s.maxEntries { + return true + } + if s.maxMemoryBytes > 0 && s.usedBytes > s.maxMemoryBytes { + return true + } + return false +} + +// evictIfNeededLocked evicts by approximate LRU (O(N) scan; rare in +// well-sized caches) until under capacity. Used by Set/Fulfill: it prefers a +// Valid victim but falls back to the oldest IN_PROGRESS placeholder to keep the +// hard cap (that placeholder's Fulfill then fails and its waiters refetch). +func (s *cacheShard) evictIfNeededLocked() { + for s.overCapacityLocked() { + victim := s.oldestLocked(cacheEntryValid) + if victim == nil { + victim = s.oldestLocked(cacheEntryInProgress) + } + if victim == nil { + return + } + s.removeEntryLocked(victim.cacheKey) + } +} + +// evictValidLocked evicts only Valid entries until under capacity. Unlike +// evictIfNeededLocked it never evicts a placeholder, so Reserve can't abort a +// peer's in-flight fetch. +func (s *cacheShard) evictValidLocked() { + for s.overCapacityLocked() { + victim := s.oldestLocked(cacheEntryValid) + if victim == nil { + return + } + s.removeEntryLocked(victim.cacheKey) + } +} + +// oldestLocked returns the entry in the given state with the smallest +// lastAccessNs (the least-recently-used), or nil when none exists. +func (s *cacheShard) oldestLocked(state cacheEntryState) *cacheEntry { + var victim *cacheEntry + var oldestNs int64 = math.MaxInt64 + for _, e := range s.entries { + if e.state != state { + continue + } + if ns := e.lastAccessNs.Load(); ns < oldestNs { + oldestNs = ns + victim = e + } + } + return victim +} + +func cloneBytes(src []byte) []byte { + if src == nil { + return nil + } + dst := make([]byte, len(src)) + copy(dst, src) + return dst +} + +func cloneStrings(src []string) []string { + if len(src) == 0 { + return nil + } + dst := make([]string, len(src)) + copy(dst, src) + return dst +} diff --git a/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go b/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go index ff54c717..bf32389a 100644 --- a/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go +++ b/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go @@ -77,17 +77,33 @@ type Manager struct { activeOperationCount atomic.Int64 // Number of active operations closed atomic.Bool // Manager closed state + // shutdownTimeout bounds each pool hook's Shutdown during Close. A field + // (not a constant) so tests can exercise the failed-Close-then-retry + // path without waiting out the real budget. + shutdownTimeout time.Duration + // Notification hooks for extensibility hooks []NotificationHook hooksMu sync.RWMutex // Protects hooks slice poolHooksRef *PoolHook + // additionalPoolHooks are pool hooks bound to pools other than the primary + // one (e.g. a dedicated pipeline connection pool). Each is an independent + // *PoolHook bound to its own pool because the hook's failed-handoff removal + // target (HandoffRequest.Pool) is taken from the hook's single pool field, + // so one hook cannot safely serve two pools. They share this Manager as + // their operations manager, keeping MOVING/MIGRATING tracking centralized. + additionalPoolHooks []additionalPoolHook + // Connections that successfully enabled maintnotifications. These need to be // retired before the pool-level listeners are removed. maintNotificationsConns sync.Map // connID -> *pool.Conn - // Cluster state reload callback for SMIGRATED notifications - clusterStateReloadCallback ClusterStateReloadCallback + // Cluster state reload callback for SMIGRATED notifications. + // Stored atomically because it is set from the OnNewNode hook while a node + // client is being created and read from the SMIGRATED push handler on that + // node's connections, which can overlap during connection init. + clusterStateReloadCallback atomic.Pointer[ClusterStateReloadCallback] } // MovingOperation tracks an active MOVING operation. @@ -113,11 +129,12 @@ func NewManager(client interfaces.ClientInterface, pool pool.Pooler, config *Con } hm := &Manager{ - client: client, - pool: pool, - options: client.GetOptions(), - config: config.Clone(), - hooks: make([]NotificationHook, 0), + client: client, + pool: pool, + options: client.GetOptions(), + config: config.Clone(), + hooks: make([]NotificationHook, 0), + shutdownTimeout: 10 * time.Second, } // Set up push notification handling @@ -134,6 +151,78 @@ func (hm *Manager) InitPoolHook(baseDialer func(context.Context, string, string) hm.pool.AddPoolHook(poolHook) } +// additionalPoolHook pairs a pool hook with the pool it was attached to so the +// manager can shut it down and detach it on Close. +type additionalPoolHook struct { + pool pool.Pooler + hook *PoolHook +} + +// InitPoolHookForPool attaches a maintnotifications pool hook to an additional +// pool (e.g. a client's dedicated pipeline connection pool). A fresh, independent +// *PoolHook is created and bound to the given pool so that connections which fail +// handoff are removed from the correct pool — the hook's removal target is its own +// single pool field, so the primary hook cannot be reused for a second pool. The +// new hook shares this Manager as its operations manager, so MOVING/MIGRATING +// tracking and notification handling stay centralized across both pools. +func (hm *Manager) InitPoolHookForPool(p pool.Pooler, baseDialer func(context.Context, string, string) (net.Conn, error)) { + if p == nil { + return + } + poolSize := 0 + network := "" + if hm.options != nil { + poolSize = hm.options.GetPoolSize() + network = hm.options.GetNetwork() + } + hook := NewPoolHookWithPoolSize(baseDialer, network, hm.config, hm, poolSize) + hook.SetPool(p) + // The closed check, the append, AND the AddPoolHook must all be atomic with + // respect to Close's snapshot: hold the lock across all three so either Close + // runs first (closed==true here, so we neither register nor attach) or we + // register+attach first (Close's snapshot then includes this hook and tears + // it down). Attaching outside the lock left a window where Close could + // snapshot/tear-down between the append and the attach, then AddPoolHook + // would re-attach to a closed manager's pool — leaking an active hook. + // p.AddPoolHook is lock-free (atomic swap on the pool's own hook manager) and + // never calls back into this manager, so holding hooksMu across it is safe. + hm.hooksMu.Lock() + defer hm.hooksMu.Unlock() + if hm.closed.Load() { + return + } + hm.additionalPoolHooks = append(hm.additionalPoolHooks, additionalPoolHook{pool: p, hook: hook}) + p.AddPoolHook(hook) +} + +// hookForConn returns the pool hook that owns cn's pool: an additional hook +// when the connection came from a secondary pool (e.g. a client's dedicated +// pipeline pool), the primary hook otherwise. Handoffs must be queued through +// the owning hook — the HandoffRequest carries that hook's pool, and a failed +// handoff removes the connection from it, so queuing a pipeline-pool +// connection on the primary hook would close the connection without freeing +// its slot in the pipeline pool's bookkeeping. +func (hm *Manager) hookForConn(cn *pool.Conn) *PoolHook { + if cn == nil { + return hm.poolHooksRef + } + name := cn.PoolName() + if name == "" { + return hm.poolHooksRef + } + hm.hooksMu.RLock() + defer hm.hooksMu.RUnlock() + for _, ah := range hm.additionalPoolHooks { + // Pooler does not expose the name; the concrete pool does. A Pooler + // implementation without it simply never matches and falls through to + // the primary hook — the pre-existing behavior. + if np, ok := ah.pool.(interface{ Name() string }); ok && np.Name() == name { + return ah.hook + } + } + return hm.poolHooksRef +} + // setupPushNotifications sets up push notification handling by registering with the client's processor. func (hm *Manager) setupPushNotifications() error { processor := hm.client.GetPushProcessor() @@ -284,17 +373,38 @@ func (hm *Manager) maintNotificationsConnSnapshot() []*pool.Conn { func (hm *Manager) retireMaintNotificationsConns(ctx context.Context) { conns := hm.maintNotificationsConnSnapshot() - if len(conns) == 0 || hm.pool == nil { + if len(conns) == 0 { return } - if retirer, ok := hm.pool.(pool.ConnRetirer); ok { - retirer.RetireConns(ctx, conns, pool.CloseReasonMaintNotificationsDisabled) - return + // Tracked connections can live in the primary pool OR in any additional + // pool this manager attached a hook to (e.g. a client's dedicated pipeline + // connection pool — its conns run initConn and are tracked exactly like + // primary ones). Retire through every pool: RetireConns skips connections + // a pool does not own, so offering the full snapshot to each pool is safe. + // Missing the additional pools left pipeline connections in service with + // maintnotifications enabled but no hook attached after a runtime + // downgrade — pushes on them were silently dropped. + pools := make([]pool.Pooler, 0, 1+len(hm.additionalPoolHooks)) + if hm.pool != nil { + pools = append(pools, hm.pool) + } + hm.hooksMu.RLock() + for _, ah := range hm.additionalPoolHooks { + if ah.pool != nil { + pools = append(pools, ah.pool) + } } + hm.hooksMu.RUnlock() - for _, cn := range conns { - _ = hm.pool.CloseConn(ctx, cn, pool.CloseReasonMaintNotificationsDisabled, pool.MetricStateIdle) + for _, pl := range pools { + if retirer, ok := pl.(pool.ConnRetirer); ok { + retirer.RetireConns(ctx, conns, pool.CloseReasonMaintNotificationsDisabled) + continue + } + for _, cn := range conns { + _ = pl.CloseConn(ctx, cn, pool.CloseReasonMaintNotificationsDisabled, pool.MetricStateIdle) + } } } @@ -312,7 +422,7 @@ func (hm *Manager) Close() error { // Shutdown the pool hook if it exists if hm.poolHooksRef != nil { // Use a timeout to prevent hanging indefinitely - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), hm.shutdownTimeout) defer cancel() err := hm.poolHooksRef.Shutdown(shutdownCtx) @@ -327,6 +437,36 @@ func (hm *Manager) Close() error { } } + // Shutdown and detach any hooks bound to additional pools (e.g. a dedicated + // pipeline pool). Snapshot under the lock so we don't iterate concurrently + // with a registering InitPoolHookForPool; Shutdown itself runs unlocked. + hm.hooksMu.Lock() + additional := hm.additionalPoolHooks + hm.additionalPoolHooks = nil + hm.hooksMu.Unlock() + for i, ah := range additional { + shutdownCtx, cancel := context.WithTimeout(context.Background(), hm.shutdownTimeout) + err := ah.hook.Shutdown(shutdownCtx) + cancel() + if err != nil { + // Could not cleanly shut down this hook. Put it and the ones not + // yet processed back so a retried Close still sees them, then stay + // open so the caller can retry, matching the primary-hook behavior + // above. Hooks before i already shut down and detached. + hm.hooksMu.Lock() + remaining := make([]additionalPoolHook, 0, len(additional)-i+len(hm.additionalPoolHooks)) + remaining = append(remaining, additional[i:]...) + remaining = append(remaining, hm.additionalPoolHooks...) + hm.additionalPoolHooks = remaining + hm.hooksMu.Unlock() + hm.closed.Store(false) + return err + } + if ah.pool != nil { + ah.pool.RemovePoolHook(ah.hook) + } + } + // Clear all active operations hm.activeMovingOps.Range(func(key, value interface{}) bool { hm.activeMovingOps.Delete(key) @@ -401,13 +541,13 @@ func (hm *Manager) AddNotificationHook(notificationHook NotificationHook) { // SetClusterStateReloadCallback sets the callback function that will be called when a SMIGRATED notification is received. // This allows node clients to notify their parent ClusterClient to reload cluster state. func (hm *Manager) SetClusterStateReloadCallback(callback ClusterStateReloadCallback) { - hm.clusterStateReloadCallback = callback + hm.clusterStateReloadCallback.Store(&callback) } // TriggerClusterStateReload calls the cluster state reload callback if it's set. // This is called when a SMIGRATED notification is received. func (hm *Manager) TriggerClusterStateReload(ctx context.Context, hostPort string, slotRanges []string) { - if hm.clusterStateReloadCallback != nil { - hm.clusterStateReloadCallback(ctx, hostPort, slotRanges) + if cb := hm.clusterStateReloadCallback.Load(); cb != nil { + (*cb)(ctx, hostPort, slotRanges) } } diff --git a/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go b/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go index 752abc71..71d14a7f 100644 --- a/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go +++ b/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go @@ -164,12 +164,17 @@ func (ph *PoolHook) OnPut(ctx context.Context, conn *pool.Conn) (shouldPool bool } if err := conn.MarkQueuedForHandoff(); err != nil { - // If marking fails, check if handoff was processed in the meantime + // Marking can fail if a worker advanced the connection's state between + // our queueHandoff above and here. Re-check ShouldHandoff: with the CAS + // rollback in Conn.MarkQueuedForHandoff, a worker that already cleared + // the handoff state is no longer misreported as ShouldHandoff=true, so a + // cleared connection is reliably detected and pooled here. if !conn.ShouldHandoff() { - // Handoff was processed - this is normal, pool the connection + // Handoff was processed - this is normal, pool the connection. return true, false, nil } - // Other error - remove the connection + // Still marked for handoff in an ambiguous state — remove it rather than + // returning a connection a queued worker may still close or replace. return false, true, nil } internal.Logger.Printf(ctx, logs.MarkedForHandoff(conn.GetID())) diff --git a/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go b/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go index 7108265b..26ae7fd4 100644 --- a/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go +++ b/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go @@ -161,8 +161,15 @@ func (snh *NotificationHandler) handleMoving(ctx context.Context, handlerCtx pus // If the connection is in use (StateInUse), it will be queued when returned to the pool via OnPut. // This handles the case where the connection is idle and might never be retrieved again. if poolConn.GetStateMachine().GetState() == pool.StateIdle { - if snh.manager.poolHooksRef != nil && snh.manager.poolHooksRef.workerManager != nil { - if err := snh.manager.poolHooksRef.workerManager.queueHandoff(poolConn); err != nil { + // Queue on the hook that owns this connection's pool, not + // unconditionally on the primary one: the request carries the + // hook's pool, and a failed handoff removes the connection + // from it — for a dedicated pipeline-pool connection the + // primary pool cannot do that, which would close the + // connection while leaving a dead slot behind. + owner := snh.manager.hookForConn(poolConn) + if owner != nil && owner.workerManager != nil { + if err := owner.workerManager.queueHandoff(poolConn); err != nil { internal.Logger.Printf(context.Background(), logs.FailedToQueueHandoff(poolConn.GetID(), err)) } else { // Mark the connection as queued for handoff to prevent it from being retrieved diff --git a/vendor/github.com/redis/go-redis/v9/options.go b/vendor/github.com/redis/go-redis/v9/options.go index ba45a0cb..0863b386 100644 --- a/vendor/github.com/redis/go-redis/v9/options.go +++ b/vendor/github.com/redis/go-redis/v9/options.go @@ -16,6 +16,7 @@ import ( "time" "github.com/redis/go-redis/v9/auth" + "github.com/redis/go-redis/v9/internal" "github.com/redis/go-redis/v9/internal/pool" "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/internal/util" @@ -121,12 +122,12 @@ type Options struct { // MinRetryBackoff is the minimum backoff between each retry. // -1 disables backoff. // - // default: 8 milliseconds + // default: 10 milliseconds MinRetryBackoff time.Duration // MaxRetryBackoff is the maximum backoff between each retry. // -1 disables backoff. - // default: 512 milliseconds; + // default: 1 second; MaxRetryBackoff time.Duration // DialTimeout for establishing new connections. @@ -157,7 +158,7 @@ type Options struct { // - `-1` - no timeout (block indefinitely). // - `-2` - disables SetReadDeadline calls completely. // - // default: 3 seconds + // default: 5 seconds ReadTimeout time.Duration // WriteTimeout for socket writes. If reached, commands will fail @@ -166,7 +167,7 @@ type Options struct { // - `-1` - no timeout (block indefinitely). // - `-2` - disables SetWriteDeadline calls completely. // - // default: 3 seconds + // default: 5 seconds (same as ReadTimeout, which it follows when unset) WriteTimeout time.Duration // ContextTimeoutEnabled controls whether the client respects context timeouts and deadlines. @@ -187,6 +188,86 @@ type Options struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize is the size of the bufio.Reader buffer for pipeline connections. + // If set to a value > 0, a separate connection pool will be created specifically for + // pipelining operations (Pipeline, AutoPipeline and AsyncAutoPipeline) with + // this buffer size. + // + // This allows you to use large buffers for pipelining (to reduce syscalls and improve + // throughput) while keeping regular command buffers small (to save memory). + // + // If not set (0), pipeline operations will use the regular connection pool with + // ReadBufferSize buffers. + // + // Recommended: 64–128 KiB for high-throughput pipelining. The benefit here is + // on the READ side: a batch's replies arrive as one large stream, and a bigger + // buffer consumes them in fewer syscalls instead of refilling repeatedly + // mid-batch. Size it to roughly the reply volume of a typical batch — which + // for read-heavy pipelines is dominated by value sizes, not command count. + // (The write-side counterpart, sizing to the outgoing wire bytes so the batch + // flushes without overflowing mid-write, belongs to PipelineWriteBufferSize.) + // Benchmarks show throughput climbs from the 32 KiB default up to ~64 KiB and + // then plateaus; going beyond ~128 KiB gives no further gain and very large + // buffers (≥512 KiB) can regress throughput and waste memory. Bigger is not + // better. + // + // Example: + // client := redis.NewClient(&redis.Options{ + // Addr: "localhost:6379", + // ReadBufferSize: 32 * 1024, // 32 KiB for regular commands + // PipelineReadBufferSize: 128 * 1024, // 128 KiB for pipelining + // PipelineWriteBufferSize: 128 * 1024, + // }) + // + // Memory impact: With PoolSize=100 and PipelinePoolSize=10: + // - Without pipeline pool: 100 conns × 128 KiB = 12.8 MB (if all use 128 KiB buffers) + // - With pipeline pool: (100 × 32 KiB) + (10 × 128 KiB) = 4.5 MB (~65% savings) + // + // default: 0 (use ReadBufferSize) + PipelineReadBufferSize int + + // PipelineWriteBufferSize is the size of the bufio.Writer buffer for pipeline connections. + // If set to a value > 0, a separate connection pool will be created specifically for + // pipelining operations (Pipeline, AutoPipeline and AsyncAutoPipeline) with + // this buffer size. + // + // This allows you to use large buffers for pipelining (to reduce syscalls and improve + // throughput) while keeping regular command buffers small (to save memory). + // + // If not set (0), pipeline operations will use the regular connection pool with + // WriteBufferSize buffers. + // + // Recommended: 64–128 KiB for high-throughput pipelining (size to roughly + // MaxBatchSize × average-command-bytes). Throughput plateaus past ~64 KiB and + // gains nothing beyond ~128 KiB; very large buffers (≥512 KiB) can regress it. + // See PipelineReadBufferSize for the full rationale. + // + // default: 0 (use WriteBufferSize) + PipelineWriteBufferSize int + + // PipelinePoolSize is the pool size for the separate pipeline connection pool. + // Only used if PipelineReadBufferSize or PipelineWriteBufferSize is set. + // + // Pipelining typically needs fewer connections than regular operations because + // batching reduces connection contention. A smaller pool saves memory while + // maintaining high throughput. + // + // If not set (0), defaults to 10 connections. + // + // default: 10 + PipelinePoolSize int + + // AutoPipelineOptions is the default config for BOTH autopipeliner faces: + // AutoPipeline and AsyncAutoPipeline use it when called without an + // explicit config, falling back to their per-face defaults + // (DefaultBlockingAutoPipelineOptions / DefaultAutoPipelineOptions) when it + // is nil. Pass a config to either method to override. Commands issued + // through an autopipeliner are batched into pipelines to cut round-trips + // and raise throughput. + // + // EXPERIMENTAL: this API is subject to change, use with caution. + AutoPipelineOptions *AutoPipelineOptions + // PoolFIFO type of connection pool. // // - true for FIFO pool @@ -300,6 +381,8 @@ type Options struct { // PushNotificationProcessor is the processor for handling push notifications. // If nil, a default processor will be created for RESP3 connections. + // With client-side caching, a custom processor runs while an idle connection + // is borrowed from the pool and should return promptly. PushNotificationProcessor push.NotificationProcessor // FailingTimeoutSeconds is the timeout in seconds for marking a cluster node as failing. @@ -313,12 +396,76 @@ type Options struct { // transitions seamlessly. Requires Protocol: 3 (RESP3) for push notifications. // If nil, maintnotifications are in "auto" mode and will be enabled if the server supports it. MaintNotificationsConfig *maintnotifications.Config + + // ClientSideCacheConfig enables client-side caching when non-nil. Together + // with ClientSideCache it is the on/off switch for the feature: leave both + // nil to disable CSC, set either one to enable it. If ClientSideCache is also set, it + // takes precedence over this config. + // + // Client-side caching is disabled when CredentialsProvider, + // CredentialsProviderContext, or StreamingCredentialsProvider is set: + // provider-backed credentials can change the ACL identity after the cache + // namespace is selected. Fixed Username/Password values are supported and + // included in the cache namespace. + // + // Experimental: this API may change in a minor release. + ClientSideCacheConfig *ClientSideCacheConfig + + // ClientSideCache is an explicit Cache implementation used for client-side + // caching. When set, it overrides ClientSideCacheConfig. Intended for + // advanced users that want to share a cache across clients or supply a + // custom implementation. + // + // A shared Cache is only safe across clients on the same server and DB. + // Clients with different fixed Username/Password values are isolated by a + // username namespace. + // Client-side caching is restricted to DB 0 and disabled with a warning + // otherwise. It is also disabled with any credential provider; see + // ClientSideCacheConfig. + // + // Experimental: this API may change in a minor release. + ClientSideCache Cache + + // ClientSideCacheStrategy selects the invalidation architecture used when + // client-side caching is enabled (via ClientSideCacheConfig or + // ClientSideCache); it is ignored when CSC is disabled. The zero value is + // CSCStrategySharedTracking, currently the only implemented strategy. + // + // Experimental: this API may change in a minor release. + ClientSideCacheStrategy CSCStrategy } +// CSCStrategy selects the client-side caching invalidation architecture. Set via +// Options.ClientSideCacheStrategy; fixed for the client's lifetime. +// +// CSCStrategySharedTracking is currently the only implemented strategy; the type +// exists as an extension point for additional architectures (e.g. a BCAST sidecar) +// without a breaking API change. +// +// Experimental: this API may change in a minor release. +type CSCStrategy int + +const ( + // CSCStrategySharedTracking (default, the zero value): one shared cache; every + // pool connection runs plain CLIENT TRACKING ON and a background drainer applies + // buffered invalidations. Portable (no BCAST), and matches the other Redis clients. + CSCStrategySharedTracking CSCStrategy = iota +) + func (opt *Options) init() { if opt.Addr == "" { opt.Addr = "localhost:6379" } + // An unknown strategy would thread the CSC gates inconsistently (e.g. tracking + // on with no drainer), serving stale data. Clamp to the only supported value. + switch opt.ClientSideCacheStrategy { + case CSCStrategySharedTracking: + default: + internal.Logger.Printf(context.Background(), + "redis: unknown ClientSideCacheStrategy %d; falling back to CSCStrategySharedTracking", + opt.ClientSideCacheStrategy) + opt.ClientSideCacheStrategy = CSCStrategySharedTracking + } if opt.Network == "" { if strings.HasPrefix(opt.Addr, "/") { opt.Network = "unix" @@ -357,6 +504,13 @@ func (opt *Options) init() { } if opt.ReadBufferSize == 0 { opt.ReadBufferSize = proto.DefaultBufferSize + } else if opt.Protocol == 3 && opt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + // Too small to hold a push header, the processor would consume frames before + // knowing their name and could swallow a Pub/Sub frame. Clamp to the minimum. + internal.Logger.Printf(context.Background(), + "redis: ReadBufferSize=%d is below the RESP3 minimum %d; clamping.", + opt.ReadBufferSize, proto.MinRESP3ReadBufferSize) + opt.ReadBufferSize = proto.MinRESP3ReadBufferSize } if opt.WriteBufferSize == 0 { opt.WriteBufferSize = proto.DefaultBufferSize @@ -367,7 +521,7 @@ func (opt *Options) init() { case -1: opt.ReadTimeout = 0 case 0: - opt.ReadTimeout = 3 * time.Second + opt.ReadTimeout = 5 * time.Second } switch opt.WriteTimeout { case -2: @@ -400,19 +554,24 @@ func (opt *Options) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.FailingTimeoutSeconds == 0 { opt.FailingTimeoutSeconds = 15 } + if opt.Protocol == 2 && (opt.ClientSideCache != nil || opt.ClientSideCacheConfig != nil) { + internal.Logger.Printf(context.Background(), + "redis: client-side caching requires Protocol: 3 (RESP3); caching is disabled") + } + opt.MaintNotificationsConfig = opt.MaintNotificationsConfig.ApplyDefaultsWithPoolConfig(opt.PoolSize, opt.MaxActiveConns) // auto-detect endpoint type if not specified @@ -442,13 +601,24 @@ func (opt *Options) NewDialer() func(context.Context, string, string) (net.Conn, return NewDialer(opt) } +// defaultKeepAliveConfig is the TCP keep-alive policy of the default dialers +// here and in sentinel.go: start probing after 30s idle (below typical LB/NAT +// idle timeouts), then declare the peer dead after 3 unanswered probes 5s +// apart. +var defaultKeepAliveConfig = net.KeepAliveConfig{ + Enable: true, + Idle: 30 * time.Second, + Interval: 5 * time.Second, + Count: 3, +} + // NewDialer returns a function that will be used as the default dialer // when none is specified in Options.Dialer. func NewDialer(opt *Options) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, network, addr string) (net.Conn, error) { netDialer := &net.Dialer{ - Timeout: opt.DialTimeout, - KeepAlive: 5 * time.Minute, + Timeout: opt.DialTimeout, + KeepAliveConfig: defaultKeepAliveConfig, } if opt.TLSConfig == nil { return netDialer.DialContext(ctx, network, addr) @@ -547,10 +717,12 @@ func setupTCPConn(u *url.URL) (*Options, error) { // a host and a port. If the host is missing, it defaults to localhost // and if the port is missing, it defaults to 6379. func getHostPortWithDefaults(u *url.URL) (string, string) { - host, port, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Host - } + // u.Hostname and u.Port strip the surrounding brackets from IPv6 literals + // (e.g. "[::1]" -> "::1") and handle the missing-port case, which + // net.SplitHostPort instead reports as an error. Relying on them avoids + // leaving the brackets on the host, which the caller's net.JoinHostPort + // would wrap again and turn "redis://[::1]" into "[[::1]]:6379". + host, port := u.Hostname(), u.Port() if host == "" { host = "localhost" } @@ -627,6 +799,10 @@ func (o *queryOptions) duration(name string) time.Duration { } dur, err := time.ParseDuration(s) if err == nil { + if dur <= 0 { + // disable timeouts + return -1 + } return dur } if o.err == nil { diff --git a/vendor/github.com/redis/go-redis/v9/osscluster.go b/vendor/github.com/redis/go-redis/v9/osscluster.go index efd52960..c6f97a40 100644 --- a/vendor/github.com/redis/go-redis/v9/osscluster.go +++ b/vendor/github.com/redis/go-redis/v9/osscluster.go @@ -142,6 +142,19 @@ type ClusterOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining on + // each node, with its own (typically larger) buffers. See the same-named + // fields on Options for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + + // AutoPipelineOptions is the default config for BOTH autopipeliner faces + // (AutoPipeline and AsyncAutoPipeline), applied when they are called + // without explicit options. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + TLSConfig *tls.Config // DisableRoutingPolicies disables the request/response policy routing system. @@ -190,7 +203,9 @@ type ClusterOptions struct { ShardPicker routing.ShardPicker // ClusterStateReloadInterval is the interval for reloading the cluster state. - // Default is 10 seconds. + // MOVED/ASK redirects still trigger an immediate reactive reload, so this + // only bounds how stale a topology can get without traffic errors. + // Default is 60 seconds. ClusterStateReloadInterval time.Duration } @@ -235,7 +250,7 @@ func (opt *ClusterOptions) init() { case -1: opt.ReadTimeout = 0 case 0: - opt.ReadTimeout = 3 * time.Second + opt.ReadTimeout = 5 * time.Second } switch opt.WriteTimeout { case -1: @@ -251,13 +266,13 @@ func (opt *ClusterOptions) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.NewClient == nil { @@ -273,7 +288,7 @@ func (opt *ClusterOptions) init() { } if opt.ClusterStateReloadInterval == 0 { - opt.ClusterStateReloadInterval = 10 * time.Second + opt.ClusterStateReloadInterval = 60 * time.Second } } @@ -455,11 +470,15 @@ func (opt *ClusterOptions) clientOptions() *Options { ConnMaxLifetimeJitter: opt.ConnMaxLifetimeJitter, ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, - DisableIdentity: opt.DisableIdentity, - DisableIndentity: opt.DisableIdentity, - IdentitySuffix: opt.IdentitySuffix, - FailingTimeoutSeconds: opt.FailingTimeoutSeconds, - TLSConfig: opt.TLSConfig, + + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + DisableIdentity: opt.DisableIdentity, + DisableIndentity: opt.DisableIndentity, + IdentitySuffix: opt.IdentitySuffix, + FailingTimeoutSeconds: opt.FailingTimeoutSeconds, + TLSConfig: opt.TLSConfig, // If ClusterSlots is populated, then we probably have an artificial // cluster whose nodes are not in clustering mode (otherwise there isn't // much use for ClusterSlots config). This means we cannot execute the @@ -477,13 +496,13 @@ func (opt *ClusterOptions) clientOptions() *Options { type clusterNode struct { Client *Client - latency uint32 // atomic - generation uint32 // atomic - failing uint32 // atomic - loaded uint32 // atomic + latency atomic.Uint32 + generation atomic.Uint32 + failing atomic.Uint32 + loaded atomic.Uint32 // last time the latency measurement was performed for the node, stored in nanoseconds from epoch - lastLatencyMeasurement int64 // atomic + lastLatencyMeasurement atomic.Int64 } func newClusterNodeWithNodeAddress(clOpt *ClusterOptions, addr, nodeAddress string) *clusterNode { @@ -494,7 +513,7 @@ func newClusterNodeWithNodeAddress(clOpt *ClusterOptions, addr, nodeAddress stri Client: clOpt.NewClient(opt), } - node.latency = math.MaxUint32 + node.latency.Store(math.MaxUint32) if clOpt.RouteByLatency { go node.updateLatency() } @@ -536,46 +555,46 @@ func (n *clusterNode) updateLatency() { } else { latency = float64(dur) / float64(successes) } - atomic.StoreUint32(&n.latency, uint32(latency+0.5)) + n.latency.Store(uint32(latency + 0.5)) n.SetLastLatencyMeasurement(time.Now()) } func (n *clusterNode) Latency() time.Duration { - latency := atomic.LoadUint32(&n.latency) + latency := n.latency.Load() return time.Duration(latency) * time.Microsecond } func (n *clusterNode) MarkAsFailing() { - atomic.StoreUint32(&n.failing, uint32(time.Now().Unix())) - atomic.StoreUint32(&n.loaded, 0) + n.failing.Store(uint32(time.Now().Unix())) + n.loaded.Store(0) } func (n *clusterNode) Failing() bool { timeout := int64(n.Client.opt.FailingTimeoutSeconds) - failing := atomic.LoadUint32(&n.failing) + failing := n.failing.Load() if failing == 0 { return false } if time.Now().Unix()-int64(failing) < timeout { return true } - atomic.StoreUint32(&n.failing, 0) + n.failing.Store(0) return false } func (n *clusterNode) Generation() uint32 { - return atomic.LoadUint32(&n.generation) + return n.generation.Load() } func (n *clusterNode) LastLatencyMeasurement() int64 { - return atomic.LoadInt64(&n.lastLatencyMeasurement) + return n.lastLatencyMeasurement.Load() } func (n *clusterNode) SetGeneration(gen uint32) { for { - v := atomic.LoadUint32(&n.generation) - if gen < v || atomic.CompareAndSwapUint32(&n.generation, v, gen) { + v := n.generation.Load() + if gen < v || n.generation.CompareAndSwap(v, gen) { break } } @@ -583,15 +602,15 @@ func (n *clusterNode) SetGeneration(gen uint32) { func (n *clusterNode) SetLastLatencyMeasurement(t time.Time) { for { - v := atomic.LoadInt64(&n.lastLatencyMeasurement) - if t.UnixNano() < v || atomic.CompareAndSwapInt64(&n.lastLatencyMeasurement, v, t.UnixNano()) { + v := n.lastLatencyMeasurement.Load() + if t.UnixNano() < v || n.lastLatencyMeasurement.CompareAndSwap(v, t.UnixNano()) { break } } } func (n *clusterNode) Loading() bool { - loaded := atomic.LoadUint32(&n.loaded) + loaded := n.loaded.Load() if loaded == 1 { return false } @@ -603,7 +622,7 @@ func (n *clusterNode) Loading() bool { err := n.Client.Ping(ctx).Err() loading := err != nil && isLoadingError(err) if !loading { - atomic.StoreUint32(&n.loaded, 1) + n.loaded.Store(1) } return loading } @@ -620,7 +639,7 @@ type clusterNodes struct { closed bool onNewNode []func(rdb *Client) - generation uint32 // atomic + generation atomic.Uint32 } func newClusterNodes(opt *ClusterOptions) *clusterNodes { @@ -685,7 +704,7 @@ func (c *clusterNodes) Addrs() ([]string, error) { } func (c *clusterNodes) NextGeneration() uint32 { - return atomic.AddUint32(&c.generation, 1) + return c.generation.Add(1) } // GC removes unused nodes. @@ -1064,8 +1083,8 @@ type clusterStateHolder struct { reloadInterval time.Duration state atomic.Value - reloading uint32 // atomic - reloadPending uint32 // atomic - set to 1 when reload is requested during active reload + reloading atomic.Uint32 + reloadPending atomic.Uint32 // set to 1 when reload is requested during active reload } func newClusterStateHolder(load func(ctx context.Context) (*clusterState, error), reloadInterval time.Duration) *clusterStateHolder { @@ -1086,8 +1105,8 @@ func (c *clusterStateHolder) Reload(ctx context.Context) (*clusterState, error) func (c *clusterStateHolder) LazyReload() { // If already reloading, mark that another reload is pending - if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { - atomic.StoreUint32(&c.reloadPending, 1) + if !c.reloading.CompareAndSwap(0, 1) { + c.reloadPending.Store(1) return } @@ -1095,22 +1114,22 @@ func (c *clusterStateHolder) LazyReload() { for { _, err := c.Reload(context.Background()) if err != nil { - atomic.StoreUint32(&c.reloadPending, 0) - atomic.StoreUint32(&c.reloading, 0) + c.reloadPending.Store(0) + c.reloading.Store(0) return } // Clear pending flag after reload completes, before cooldown // This captures notifications that arrived during the reload - atomic.StoreUint32(&c.reloadPending, 0) + c.reloadPending.Store(0) // Wait cooldown period time.Sleep(200 * time.Millisecond) // Check if another reload was requested during cooldown - if atomic.LoadUint32(&c.reloadPending) == 0 { + if c.reloadPending.Load() == 0 { // No pending reload, we're done - atomic.StoreUint32(&c.reloading, 0) + c.reloading.Store(0) return } @@ -1153,6 +1172,17 @@ type ClusterClient struct { cmdInfoResolver *commandInfoResolver cmdable hooksMixin + + // himport is the cluster-wide HIMPORT fieldset registry, shared with + // every node client (masters and replicas alike — roles change with the + // topology) so any connection serving an HIMPORT SET can lazily replay + // the PREPARE (see himport.go, himport_cluster.go). + himport *himportRegistry + + autopipelinerMu *sync.Mutex // guards the autopipeliner fields against concurrent first-call creation + autopipeliner *AutoPipeliner // blocking face (ClusterClient.AutoPipeline) + asyncAutopipeliner *AutoPipeliner // deferred face (ClusterClient.AsyncAutoPipeline) + autopipelinerClosed bool // set by Close: refuse to resurrect a pipeliner on a closed client } // NewClusterClient returns a Redis Cluster client as described in @@ -1165,10 +1195,19 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt.init() c := &ClusterClient{ - opt: opt, - nodes: newClusterNodes(opt), + opt: opt, + nodes: newClusterNodes(opt), + himport: newHImportRegistry(), + autopipelinerMu: &sync.Mutex{}, } + // Every node client shares the cluster-wide fieldset registry, replicas + // included: a promoted replica's connections carry no prepared flags, so + // the first HIMPORT SET routed to it replays the PREPARE lazily. + c.nodes.OnNewNode(func(nodeClient *Client) { + nodeClient.himport = c.himport + }) + c.cmdsInfoCache = newCmdsInfoCache(c.cmdsInfo) c.state = newClusterStateHolder(c.loadState, opt.ClusterStateReloadInterval) @@ -1223,7 +1262,26 @@ func (c *ClusterClient) ReloadState(ctx context.Context) { // It is rare to Close a ClusterClient, as the ClusterClient is meant // to be long-lived and shared between many goroutines. func (c *ClusterClient) Close() error { - return c.nodes.Close() + // Stop both cached autopipeliners (blocking and async faces) before + // closing nodes, so its background flusher goroutines don't outlive the + // client. AutoPipeliner.Close is idempotent and nil-safe here. + c.autopipelinerMu.Lock() + ap, async := c.autopipeliner, c.asyncAutopipeliner + c.autopipeliner, c.asyncAutopipeliner = nil, nil + c.autopipelinerClosed = true // getters refuse to resurrect on a closed client + c.autopipelinerMu.Unlock() + var firstErr error + for _, p := range []*AutoPipeliner{ap, async} { + if p != nil { + if err := p.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + } + if err := c.nodes.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr } func (c *ClusterClient) Process(ctx context.Context, cmd Cmder) error { @@ -1555,6 +1613,155 @@ func (c *ClusterClient) Pipeline() Pipeliner { return &pipe } +// clusterAutoPipelineOptions applies the cluster shard-count default: commands +// are routed to shards by slot (see installAutoPipelineSharding), so unlike a +// standalone client — which defaults to a single deep queue — a cluster client +// wants several shards to keep concurrent nodes' batches separate. The caller's +// config is copied before the default is filled in, never mutated. +func clusterAutoPipelineOptions(cfg *AutoPipelineOptions) *AutoPipelineOptions { + c2 := *cfg + if c2.NumShards == 0 { + c2.NumShards = numAutoPipelineShards() + } + // A cluster always routes by slot, so per-key order holds regardless of shard + // count; mark it so construction's NumShards ordering check (which targets + // round-robin sharding) does not reject the cluster default or an explicit + // NumShards on the deferred (async) face. + c2.contentSharded = true + return &c2 +} + +// AutoPipeline returns the blocking autopipeliner for this cluster client: each +// command call blocks until executed (drop-in shape) while the engine batches +// concurrent callers into pipelines. Commands keep per-goroutine order; across +// nodes, ordering is per key (slot routing keeps a key on one shard and node +// sub-pipelines execute concurrently). Use AutoPipelineWithOptions to override +// DefaultBlockingAutoPipelineOptions. Cached/shared; first call's config wins. +// Close it (or the client) to release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is AutoPipeline with explicit options instead of +// ClusterOptions.AutoPipelineOptions / the default. Cached/shared; first call wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.autopipeliner, &c.autopipelinerClosed, nil, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultBlockingAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { + ap, err := newAutoPipeliner(c, clusterAutoPipelineOptions(cfg), true) + if err != nil { + return nil, err + } + c.installAutoPipelineSharding(ap) + return ap, nil + }) +} + +// installAutoPipelineSharding routes commands to shards by cluster slot so each +// shard's batch lands on a single master node, keeping per-node pipelines deep +// instead of splitting every batch across all nodes at flush. Cluster slots are +// contiguous per node, so bucketing by slot range (slot*shards/16384) keeps a +// node's slots together. Keyless commands hash to slot -1 → bucket 0; multi-node +// commands are already rejected from pipelines, so only single-node commands +// reach here. +func (c *ClusterClient) installAutoPipelineSharding(ap *AutoPipeliner) { + // Reject commands whose request policy cannot ride a pipeline (ReqAllNodes/ + // ReqAllShards/ReqMultiShard) at submit, BEFORE they can join a merged + // batch: mapCmdsByNode fails a whole mapping on such a command (user + // pipelines are all-or-nothing), and one autopipeline caller must not be + // able to poison unrelated callers' batches. Rejecting here also keeps the + // lone-command fast path consistent with batched dispatch — the command is + // refused regardless of what it happens to coalesce with. + ap.setPreflight(func(ctx context.Context, cmd Cmder) error { + if c.cmdInfoResolver == nil { + return nil + } + if policy := c.cmdInfoResolver.GetCommandPolicy(ctx, cmd); policy != nil && !policy.CanBeUsedInPipeline() { + return fmt.Errorf( + "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), + ) + } + return nil + }) + // Commands whose routing is not slot-derived must not be coalesced: a solo + // flush reaches ClusterClient.process and its special handling (FT.CURSOR + // READ/DEL are sticky to the node holding the cursor), but inside a batch + // mapCmdsByNode routes by slot and can hit the wrong shard — visible only + // under concurrent traffic, which is the worst way to find it. Divert them + // instead of rejecting: they work fine on their own connection (review + // finding by codex on #3942). + ap.setMustDivert(func(ctx context.Context, cmd Cmder) bool { + if c.cmdInfoResolver == nil { + return false + } + policy := c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) + return policy != nil && policy.Request == routing.ReqSpecial + }) + + const slots = 16384 + n := ap.numShards() + ap.setShardFn(func(cmd Cmder) int { + // Compute the exact slot once and cache it on the command; the flush + // router (mapCmdsByNode) reuses the cached value, so the slot is resolved + // once per command, not twice. Keyless (slot -1) buckets to shard 0. + slot := c.cmdSlot(cmd, -1) + if slot < 0 { + return 0 + } + return slot * n / slots + }) +} + +// AsyncAutoPipeline returns the deferred autopipeliner: command calls return +// immediately and the result accessors block. Submit a window then read results +// for the highest throughput. By default, +// ClusterOptions.AutoPipelineOptions is used if set, otherwise +// DefaultAutoPipelineOptions. Ordering across nodes is per key: slot routing +// keeps a key on one shard, and node sub-pipelines execute concurrently. Use +// AsyncAutoPipelineWithOptions to override. Cached/shared; first call's config wins. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is AsyncAutoPipeline with an explicit config +// instead of ClusterOptions.AutoPipelineOptions / the default. Cached/shared. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.asyncAutopipeliner, &c.autopipelinerClosed, nil, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { + ap, err := newAutoPipeliner(c, clusterAutoPipelineOptions(cfg), false) + if err != nil { + return nil, err + } + c.installAutoPipelineSharding(ap) + return ap, nil + }) +} + func (c *ClusterClient) Pipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(ctx, fn) } @@ -1638,9 +1845,18 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd policy = c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) } if policy != nil && !policy.CanBeUsedInPipeline() { - return fmt.Errorf( + // All-or-nothing: a user Pipeline() relies on the whole batch + // either dispatching or failing before anything executes, so a + // non-pipelineable command fails the entire mapping pre-dispatch. + // Autopipeline batches never reach here with such a command: the + // cluster face rejects them at submit (see the preflight installed + // by installAutoPipelineSharding), so one caller's bad command + // cannot poison a merged batch. + err := fmt.Errorf( "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), ) + setCmdsErr(cmds, err) + return err } slot := c.cmdSlot(cmd, -1) var node *clusterNode @@ -1649,10 +1865,15 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd if len(state.Masters) == 0 { return errClusterNoNodes } - // For read-only keyless commands, pick from all nodes (masters + slaves) - allNodes := append(state.Masters, state.Slaves...) - idx := c.opt.ShardPicker.Next(len(allNodes)) - node = allNodes[idx] + // For read-only keyless commands, pick from all nodes (masters + slaves). + // Index directly instead of building a combined slice, which would + // append into the shared snapshot's spare capacity and race. + idx := c.opt.ShardPicker.Next(len(state.Masters) + len(state.Slaves)) + if idx < len(state.Masters) { + node = state.Masters[idx] + } else { + node = state.Slaves[idx-len(state.Masters)] + } } else { node, err = c.slotReadOnlyNode(state, slot) if err != nil { @@ -1670,9 +1891,18 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd policy = c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) } if policy != nil && !policy.CanBeUsedInPipeline() { - return fmt.Errorf( + // All-or-nothing: a user Pipeline() relies on the whole batch + // either dispatching or failing before anything executes, so a + // non-pipelineable command fails the entire mapping pre-dispatch. + // Autopipeline batches never reach here with such a command: the + // cluster face rejects them at submit (see the preflight installed + // by installAutoPipelineSharding), so one caller's bad command + // cannot poison a merged batch. + err := fmt.Errorf( "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), ) + setCmdsErr(cmds, err) + return err } slot := c.cmdSlot(cmd, -1) var node *clusterNode @@ -1707,31 +1937,76 @@ func (c *ClusterClient) cmdsAreReadOnly(ctx context.Context, cmds []Cmder) bool func (c *ClusterClient) processPipelineNode( ctx context.Context, node *clusterNode, cmds []Cmder, failedCmds *cmdsMap, ) { - _ = node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { - cn, err := node.Client.getConn(ctx) - if err != nil { + // This call runs on a per-node fan-out goroutine, so register it as an + // executor of every deferred-face batch among cmds: a NODE-level hook + // (OnNewNode — redisotel's tracing) reading a result before next() must + // get the not-yet-executed view from the accessor guards instead of + // blocking on a batch only this call chain completes (reproduced as a + // permanent wedge with a rediscmd-shaped Err() peek). + unregister := registerBatchExecutors(cmds) + defer unregister() + + // executed guards against a node-level hook short-circuiting (returning + // without calling next): the inner callback then never runs, and without + // surfacing the chain's error the cluster pipeline would report success + // for commands that were never sent. + executed := false + err := node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { + executed = true + // Acquire through the node's dedicated pipeline pool when one is + // configured (Pipeline*BufferSize propagate to node clients via + // clientOptions); withPipelineConn falls back to the main pool + // otherwise, preserving the previous behavior. entered distinguishes + // an acquisition failure (fn never ran) from an execution error. + entered := false + err := node.Client.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + entered = true + return c.processPipelineNodeConn(ctx, node, cn, cmds, failedCmds) + }) + if err != nil && !entered { if !isContextError(err) { node.MarkAsFailing() } _ = c.mapCmdsByNode(ctx, failedCmds, cmds) setCmdsErr(cmds, err) - return err } - - var processErr error - defer func() { - node.Client.releaseConn(ctx, cn, processErr) - }() - processErr = c.processPipelineNodeConn(ctx, node, cn, cmds, failedCmds) - - return processErr + return err }) + if !executed { + // A hook returned without calling next. If it supplied an error that is + // a deliberate abort: set it and do not remap for retry (a retry would + // re-run the same hook). If it returned nil it short-circuited + // SUCCESSFULLY, having served the batch itself — the same thing a plain + // Pipeline hook may do — so setCmdsErr(nil) leaves the values it set + // intact (review finding by codex on #3942). + setCmdsErr(cmds, err) + return + } + if err != nil && cmdsFirstErr(cmds) == nil { + // Post-next verdict from a node-level hook on an all-clean sub-batch: + // the exec fully succeeded, so the error can only be the hook's own — + // apply it, mirroring AutoPipeliner.dispatchCmds. On a mixed batch the + // exec-recorded outcomes win (hooks conventionally echo next's error, + // and stamping the echo would overwrite successful replies). No remap: + // retrying would re-run the same hook. + setCmdsErr(cmds, err) + } } func (c *ClusterClient) processPipelineNodeConn( ctx context.Context, node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, ) error { + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the batch references get written ahead of + // the batch (see himport.go). + injected := node.Client.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { if isBadConn(err, false, node.Client.getAddr()) { @@ -1745,18 +2020,56 @@ func (c *ClusterClient) processPipelineNodeConn( } return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { - return c.pipelineReadCmds(ctx, node, rd, cmds, failedCmds) + if err := node.Client.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with the batch replies unread: same handling + // as a write error — the batch may be retried on a fresh + // connection. + if isBadConn(err, false, node.Client.getAddr()) { + node.MarkAsFailing() + } + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + _ = c.mapCmdsByNode(ctx, failedCmds, cmds) + } + setCmdsErr(cmds, err) + return err + } + err := c.pipelineReadCmds(ctx, node, cn, rd, cmds, failedCmds) + if err == nil || isRedisError(err) { + node.Client.himportAfterBatch(cn, injected, cmds) + // SETs of registered fieldsets that lost their session state + // re-queue for the next attempt, which re-prepares lazily — + // the cluster equivalent of himportRetryFailedSets, bounded by + // the pipeline's attempt budget. A non-nil redis error here + // means pipelineReadCmds already re-queued the whole batch + // (retryable first-command error); adding the SETs again would + // duplicate them in the next attempt. + if err == nil { + c.himportRequeueFailedSets(ctx, cmds, failedCmds) + } + } + return err }) } func (c *ClusterClient) pipelineReadCmds( ctx context.Context, node *clusterNode, + cn *pool.Conn, rd *proto.Reader, cmds []Cmder, failedCmds *cmdsMap, ) error { for i, cmd := range cmds { + // Drain any buffered RESP3 push notifications before reading each + // reply — otherwise a push frame (e.g. a maintnotifications MOVING + // notification) is consumed AS the command's reply and every + // subsequent reply in the pipeline shifts by one command. The + // standalone pipeline and the cluster TxPipeline read loops already + // do this; this loop was the only push-blind reader, and the + // autopipeliner routes all cluster traffic through it. + if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } err := cmd.readReply(rd) cmd.SetErr(err) @@ -1781,7 +2094,8 @@ func (c *ClusterClient) pipelineReadCmds( } } - if err := cmds[0].Err(); err != nil && shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + // rawErr: execution path; never await an async command's batch here. + if err := cmds[0].rawErr(); err != nil && shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { _ = c.mapCmdsByNode(ctx, failedCmds, cmds) return err } @@ -1832,18 +2146,108 @@ func (c *ClusterClient) TxPipelined(ctx context.Context, fn func(Pipeliner) erro return c.TxPipeline().Pipelined(ctx, fn) } -func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) error { - // Only call time.Now() if pipeline operation duration callback is set to avoid overhead +// A cluster tx pipeline sends MULTI, c1..cN, EXEC — N+2 commands, or N+3 with a +// leading ASKING — and always receives exactly that many replies, so every +// redirect/abort path leaves the connection clean. +// +// Possible reply sequences: +// 1. Slot owned here, no migration: +// +OK, +QUEUED x N, *N (array of N results) -> success +// 2. Slot already migrated away: +// +OK, -MOVED x N, -EXECABORT -> re-route whole tx +// 3. Slot in migrating state (still owned here, keys draining out). Per +// cmd, the queue reply is +QUEUED / -ASK / -TRYAGAIN (keys present / +// all gone / some gone); any -ASK or -TRYAGAIN dirties the tx, so +// EXEC is -EXECABORT. Still N+2 replies, like the cases above: +// +OK, (+QUEUED|-ASK|-TRYAGAIN) x N, -EXECABORT -> follow first redirect +// 4. Narrow race (all +QUEUED, slot moves before EXEC): +// +OK, +QUEUED x N, -MOVED -> re-route whole tx +// 5. Non-cluster command error (arity / ACL / unknown): +// +OK, +QUEUED..., -ERR..., -EXECABORT -> surface, not retryable +// 6. Narrow race (all +QUEUED, slot still migrating, keys drain before EXEC): +// +OK, +QUEUED x N, -ASK / -TRYAGAIN -> re-route on -ASK, back off on -TRYAGAIN +// +// EXEC reply — the reply that decides the outcome: +// +// *N success; read N per-command results +// -EXECABORT a queue-stage command failed; follow the first queue +// redirect (MOVED/ASK/TRYAGAIN), else surface the trigger +// -MOVED case 4; re-route whole tx to addr, reload topology +// -ASK race: slot entered migrating state; re-route to addr +// with a top-level ASKING before MULTI +// -TRYAGAIN race: migrating with split keys, or slot being trimmed +// (CLUSTER_REDIR_TRIMMING on a write); back off and retry +// the whole tx (same node still owns it) +// -CLUSTERDOWN cluster degraded; back off and retry whole tx +// +// ASK retry: the ASKING flag is NOT cleared between commands inside a MULTI +// so one top-level ASKING before MULTI covers the whole tx and lets the importing +// slot serve at EXEC. ASKING placed inside the MULTI would be queued and leave +// the flag unset during queueing, so the keyed commands would still get MOVED. +// +// Out of scope: WATCH's null-array EXEC and -CROSSSLOT; +// cluster TxPipeline is not used with WATCH and cross-slot is rejected client-side. + +type txOutcomeKind int + +const ( + txSuccess txOutcomeKind = iota // transaction executed; per-command results are set + txRetryMoved // MOVED: reload topology and re-route the whole tx + txRetryAsk // ASK: re-route to the target with a top-level ASKING + txRetryTryAgain // TRYAGAIN: back off and re-route the whole tx + txRetryConn // connection/write/read failure: re-route the whole tx + txFatal // non-retryable error; surface to the caller +) + +// txOutcome is the result of a single tx attempt. err is the error to report +// when the redirect/retry loop is exhausted (or the fatal error to surface); +// addr is the ASK target; execErr is the EXEC reply error used to mark +// aborted commands; unreadReplies forces the connection to be discarded +// when the read loop exited before consuming all N+2 replies, leaving bytes +// on the wire. +type txOutcome struct { + kind txOutcomeKind + err error + addr string + execErr error + unreadReplies bool +} + +// txRedirect records the first queue-stage redirect (MOVED/ASK/TRYAGAIN) seen +// while reading +QUEUED replies. Redis dirties and aborts the transaction on +// any such reply, so the EXEC reply will be EXECABORT and the client must +// follow the recorded redirect with the whole transaction. +type txRedirect struct { + moved bool + ask bool + tryAgain bool + addr string + err error +} + +// errTxDirtyConn forces releaseConn to discard a connection that may still have +// unread transaction replies on it (an early exit before consuming all N+2). +var errTxDirtyConn = errors.New("redis: connection has unread transaction replies") + +func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) (retErr error) { var operationStart time.Time pipelineOpDurationCallback := otel.GetPipelineOperationDurationCallback() if pipelineOpDurationCallback != nil { operationStart = time.Now() } totalAttempts := 0 + var lastErr error + + defer func() { + if pipelineOpDurationCallback == nil { + return + } + finalErr := cmp.Or(retErr, cmdsFirstErr(cmds), lastErr) + pipelineOpDurationCallback(ctx, time.Since(operationStart), "MULTI", len(cmds), totalAttempts, finalErr, nil, 0) + }() // Trim multi .. exec. cmds = cmds[1 : len(cmds)-1] - if len(cmds) == 0 { return nil } @@ -1851,10 +2255,6 @@ func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) err state, err := c.state.Get(ctx) if err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, err, nil, 0) - } return err } @@ -1866,77 +2266,85 @@ func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) err case 1: for sl := range keyedCmdsBySlot { slot = sl - break } default: // TxPipeline does not support cross slot transaction. setCmdsErr(cmds, ErrCrossSlot) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, ErrCrossSlot, nil, 0) - } return ErrCrossSlot } node, err := state.slotMasterNode(slot) if err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, err, nil, 0) - } return err } - var lastErr error - cmdsMap := map[*clusterNode][]Cmder{node: cmds} + asking := false + // MOVED/ASK are routing changes, not transient failures: follow them immediately. + redirected := false for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { totalAttempts++ - if attempt > 0 { + if attempt > 0 && !redirected { if err := internal.Sleep(ctx, c.retryBackoff(attempt)); err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), totalAttempts, err, nil, 0) - } return err } } - failedCmds := newCmdsMap() - var wg sync.WaitGroup - - for node, cmds := range cmdsMap { - wg.Add(1) - go func(node *clusterNode, cmds []Cmder) { - defer wg.Done() - c.processTxPipelineNode(ctx, node, cmds, failedCmds) - }(node, cmds) - } - - wg.Wait() - if len(failedCmds.m) == 0 { - break + outcome := c.processTxPipelineNode(ctx, node, cmds, asking) + lastErr = outcome.err + redirected = false + switch outcome.kind { + case txSuccess: + return cmdsFirstErr(cmds) + case txRetryMoved: + // Route directly to the authoritative addr from the MOVED; the + // cached slot state may be stale until LazyReload lands. + redirected = true + asking = false + c.state.LazyReload() + if node, err = c.nodes.GetOrCreate(outcome.addr); err != nil { + setCmdsErr(cmds, err) + return err + } + case txRetryAsk: + redirected = true + asking = true + if node, err = c.nodes.GetOrCreate(outcome.addr); err != nil { + setCmdsErr(cmds, err) + return err + } + case txRetryTryAgain, txRetryConn: + // Same node, fresh connection: TRYAGAIN comes from the migrating + // source (still the owner), and a conn failure only needs a new + // connection. Preserve a prior ASKING flag: if we followed an ASK + // to the importing target, the retry must still send ASKING (the + // slot is still importing). ASKING is harmless if the migration + // has since completed, since the flag is only consulted for + // importing slots. + case txFatal: + // Mark every queued-but-never-executed command with the abort + // error; the command that triggered EXECABORT already has its + // own error and keeps it, so callers can tell what went wrong. + abortErr := cmp.Or(outcome.execErr, outcome.err) + for _, cmd := range cmds { + if cmd.Err() == nil { + cmd.SetErr(abortErr) + } + } + return lastErr } - cmdsMap = failedCmds.m - lastErr = cmdsFirstErr(cmds) } - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - finalErr := cmdsFirstErr(cmds) - if finalErr == nil { - finalErr = lastErr - } - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), totalAttempts, finalErr, nil, 0) + if lastErr != nil { + setCmdsErr(cmds, lastErr) } - return cmdsFirstErr(cmds) } // slottedKeyedCommands returns a map of slot to commands taking into account // only commands that have keys. -func (c *ClusterClient) slottedKeyedCommands(ctx context.Context, cmds []Cmder) map[int][]Cmder { +func (c *ClusterClient) slottedKeyedCommands(_ context.Context, cmds []Cmder) map[int][]Cmder { cmdsSlots := map[int][]Cmder{} // Peek once outside the loop, one RLock for the whole batch instead of @@ -1967,151 +2375,255 @@ func (c *ClusterClient) slottedKeyedCommands(ctx context.Context, cmds []Cmder) } func (c *ClusterClient) processTxPipelineNode( - ctx context.Context, node *clusterNode, cmds []Cmder, failedCmds *cmdsMap, -) { - cmds = wrapMultiExec(ctx, cmds) - _ = node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { - cn, err := node.Client.getConn(ctx) - if err != nil { - _ = c.mapCmdsByNode(ctx, failedCmds, cmds) - setCmdsErr(cmds, err) - return err + ctx context.Context, node *clusterNode, cmds []Cmder, asking bool, +) *txOutcome { + wire := wrapMultiExec(ctx, cmds) + if asking { + // ASKING must precede MULTI so the flag stays set for the whole tx. + wire = append([]Cmder{NewCmd(ctx, "asking")}, wire...) + } + + var outcome *txOutcome + // executed guards against a node-level hook short-circuiting (returning + // without calling next) — same treatment as processPipelineNode. + executed := false + chainErr := node.Client.withProcessPipelineHook(ctx, wire, func(ctx context.Context, wire []Cmder) error { + executed = true + // Acquire through the node's dedicated pipeline pool when configured + // (same routing as processPipelineNode); withPipelineConn falls back + // to the main pool otherwise. The inner fn's return value drives the + // connection release exactly like the explicit releaseConn did: + // redis errors keep the conn poolable, unread replies poison it. + entered := false + err := node.Client.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + entered = true + outcome = c.processTxPipelineNodeConn(ctx, node, cn, wire, cmds, asking) + connErr := outcome.err + if isRedisError(outcome.err) { + connErr = nil + } + if outcome.unreadReplies { + connErr = errTxDirtyConn + } + return connErr + }) + if !entered && err != nil { + // Connection acquisition failed — fn never ran. + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + outcome = &txOutcome{kind: txRetryConn, err: err} + } else { + outcome = &txOutcome{kind: txFatal, err: err} + } } - - var processErr error - defer func() { - node.Client.releaseConn(ctx, cn, processErr) - }() - processErr = c.processTxPipelineNodeConn(ctx, node, cn, cmds, failedCmds) - - return processErr + return err }) + + if !executed && chainErr != nil { + // A node-level hook aborted with an error: surface its verdict. A hook + // that returned nil short-circuited successfully (it served the batch), + // which is legal for plain pipelines too, so it is not turned into a + // fatal outcome (review finding by codex on #3942). + outcome = &txOutcome{kind: txFatal, err: chainErr} + } + if outcome == nil { + outcome = &txOutcome{kind: txFatal, err: fmt.Errorf("redis: tx pipeline produced no outcome")} + } + return outcome } func (c *ClusterClient) processTxPipelineNodeConn( - ctx context.Context, node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, -) error { + ctx context.Context, node *clusterNode, cn *pool.Conn, wire []Cmder, cmds []Cmder, asking bool, +) *txOutcome { + // HIMPORT bookkeeping: pending discards and PREPAREs for registered + // fieldsets the transaction references get written ahead of the wire + // batch (before ASKING/MULTI; the session state is visible at EXEC). + injected := node.Client.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { - return writeCmds(wr, cmds) + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } + return writeCmds(wr, wire) }); err != nil { + // Write failure: re-route the whole tx on a fresh connection. if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { - _ = c.mapCmdsByNode(ctx, failedCmds, cmds) + return &txOutcome{kind: txRetryConn, err: err} } - setCmdsErr(cmds, err) - return err + return &txOutcome{kind: txFatal, err: err} } - return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { - statusCmd := cmds[0].(*StatusCmd) - // Trim multi and exec. - trimmedCmds := cmds[1 : len(cmds)-1] - - if err := c.txPipelineReadQueued( - ctx, node, cn, rd, statusCmd, trimmedCmds, failedCmds, - ); err != nil { - setCmdsErr(cmds, err) - - moved, ask, addr := isMovedError(err) - if moved || ask { - return c.cmdsMoved(ctx, trimmedCmds, moved, ask, addr, failedCmds) - } - - return err + var outcome *txOutcome + readErr := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := node.Client.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with the tx replies unread; the batch was + // written and may have committed — fatal, discard the conn. + outcome = c.txReadFatal(err) + return nil } - - return node.Client.pipelineReadCmds(ctx, cn, rd, trimmedCmds) + outcome = c.readTxPipelineReplies(ctx, node, cn, rd, cmds, asking) + if outcome != nil && outcome.kind == txSuccess { + node.Client.himportAfterBatch(cn, injected, cmds) + } + return nil }) + + if readErr != nil { + // Reader-level failure (deadline setup, nil conn) around the read loop. + // The batch was already written, so the server may have committed; + // surface the error as fatal and discard the suspect connection rather + // than re-executing the transaction. + return c.txReadFatal(readErr) + } + return outcome } -func (c *ClusterClient) txPipelineReadQueued( - ctx context.Context, - node *clusterNode, - cn *pool.Conn, - rd *proto.Reader, - statusCmd *StatusCmd, - cmds []Cmder, - failedCmds *cmdsMap, -) error { - // Parse queued replies. - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) +// readTxPipelineReplies reads the replies of one MULTI..EXEC unit and +// classifies the outcome. The reply count always matches the number of sent +// commands, so success/redirect paths leave the connection clean; only an early +// MULTI read failure can leave unread replies. +func (c *ClusterClient) readTxPipelineReplies( + ctx context.Context, node *clusterNode, cn *pool.Conn, rd *proto.Reader, cmds []Cmder, asking bool, +) *txOutcome { + scratch := NewStatusCmd(ctx) + + readStatus := func() error { + c.txProcessPush(ctx, node, cn, rd) + return scratch.readReply(rd) } - if err := statusCmd.readReply(rd); err != nil { - return err + + // Optional top-level ASKING reply (+OK, or a retryable error such as -LOADING). + if asking { + if err := readStatus(); err != nil { + return c.txPreQueueErrorOutcome(err, cmds) + } + } + + // MULTI reply (+OK, or an error such as -LOADING during failover). + if err := readStatus(); err != nil { + return c.txPreQueueErrorOutcome(err, cmds) } + // Queue replies: +QUEUED, or a redirect / command error that dirties the tx. + var firstRedirect *txRedirect + var firstFatal error for _, cmd := range cmds { - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + err := readStatus() + if err == nil { + continue // +QUEUED } - err := statusCmd.readReply(rd) - if err != nil { - if c.checkMovedErr(ctx, cmd, err, failedCmds) { - // will be processed later - continue + if !isRedisError(err) { + return c.txReadFatal(err) // IO error + } + if moved, ask, addr := isMovedError(err); moved || ask { + if firstRedirect == nil { + firstRedirect = &txRedirect{moved: moved, ask: ask, addr: addr, err: err} } - cmd.SetErr(err) - if !isRedisError(err) { - return err + continue + } + if proto.IsTryAgainError(err) { + if firstRedirect == nil { + firstRedirect = &txRedirect{tryAgain: true, err: err} } + continue + } + // Non-redirect command error (e.g. wrong arity) dirties the tx. + cmd.SetErr(err) + if firstFatal == nil { + firstFatal = err } } - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) - } - // Parse number of replies. + // EXEC reply. ReadLine parses error lines into typed errors, so a non-nil + // err means EXEC returned an error rather than the result array. + c.txProcessPush(ctx, node, cn, rd) line, err := rd.ReadLine() if err != nil { - if err == Nil { - err = TxFailedErr + if !isRedisError(err) { + return c.txReadFatal(err) // IO error } - return err + return c.classifyExecError(err, firstRedirect, firstFatal) } if line[0] != proto.RespArray { - return fmt.Errorf("redis: expected '*', but got line %q", line) + err := fmt.Errorf("redis: unexpected EXEC reply %q", line) + setCmdsErr(cmds, err) + // A non-array aggregate reply may carry an unread payload. + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} } - return nil + // Success: read the N command results. + if err := node.Client.pipelineReadCmds(ctx, cn, rd, cmds); err != nil && !isRedisError(err) { + return c.txReadFatal(err) // IO error mid-results + } + return &txOutcome{kind: txSuccess} } -func (c *ClusterClient) cmdsMoved( - ctx context.Context, cmds []Cmder, - moved, ask bool, - addr string, - failedCmds *cmdsMap, -) error { - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - return err +func (c *ClusterClient) txProcessPush(ctx context.Context, node *clusterNode, cn *pool.Conn, rd *proto.Reader) { + if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) } +} - if moved { - c.state.LazyReload() - for _, cmd := range cmds { - failedCmds.Add(node, cmd) - } - return nil +// txReadFatal classifies a read-phase IO error. The MULTI..EXEC batch was +// already written, so the server may have committed the transaction; retrying +// would re-execute it, double-applying non-idempotent commands (INCR/APPEND, +// which are not NoRetry). Surface the error as fatal and discard the +// connection, since replies may still be unread on the wire. +func (c *ClusterClient) txReadFatal(err error) *txOutcome { + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} +} + +// txPreQueueErrorOutcome classifies a setup-phase reply error: the top-level +// ASKING reply or the MULTI reply. The transaction body never executes (EXEC +// returns -EXECABORT), so retryable errors such as -LOADING are safe to retry +// on a fresh connection. A failed setup reply still leaves the remaining +// replies on the wire -- the server replies to each following command and to +// EXEC regardless -- so the connection is always discarded. +func (c *ClusterClient) txPreQueueErrorOutcome(err error, cmds []Cmder) *txOutcome { + if !isRedisError(err) { + return c.txReadFatal(err) } + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + return &txOutcome{kind: txRetryConn, err: err, unreadReplies: true} + } + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} +} - if ask { - for _, cmd := range cmds { - failedCmds.Add(node, NewCmd(ctx, "asking"), cmd) +// classifyExecError turns an EXEC reply error into a retry/fatal outcome. +func (c *ClusterClient) classifyExecError(execErr error, firstRedirect *txRedirect, firstFatal error) *txOutcome { + if moved, ask, addr := isMovedError(execErr); moved || ask { + // Narrow race: the slot moved after every command was queued. + if ask { + return &txOutcome{kind: txRetryAsk, err: execErr, addr: addr} } - return nil + return &txOutcome{kind: txRetryMoved, err: execErr, addr: addr} } - - return nil + if proto.IsTryAgainError(execErr) { + return &txOutcome{kind: txRetryTryAgain, err: execErr} + } + if proto.IsClusterDownError(execErr) { + // Cluster degraded: back off and retry. Replies were fully consumed. + return &txOutcome{kind: txRetryConn, err: execErr} + } + if proto.IsExecAbortError(execErr) { + if firstFatal != nil { + return &txOutcome{kind: txFatal, err: firstFatal, execErr: execErr} + } + if firstRedirect != nil { + switch { + case firstRedirect.moved: + return &txOutcome{kind: txRetryMoved, err: firstRedirect.err, addr: firstRedirect.addr} + case firstRedirect.ask: + return &txOutcome{kind: txRetryAsk, err: firstRedirect.err, addr: firstRedirect.addr} + case firstRedirect.tryAgain: + return &txOutcome{kind: txRetryTryAgain, err: firstRedirect.err} + } + } + return &txOutcome{kind: txFatal, err: execErr, execErr: execErr} + } + return &txOutcome{kind: txFatal, err: execErr} } func (c *ClusterClient) Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error { @@ -2359,8 +2871,22 @@ func (c *ClusterClient) cmdInfoPeek(name string) *CommandInfo { } func (c *ClusterClient) cmdSlot(cmd Cmder, prefferedSlot int) int { + // Serve/populate the per-command slot cache only on the natural-slot path + // (prefferedSlot == -1). A forced prefferedSlot (retry re-routing) must not be + // cached or served from cache. The cache lets the autopipeline shard router + // and the pipeline-flush router (mapCmdsByNode) share one slot computation + // instead of each recomputing it. + if prefferedSlot == -1 { + if slot, ok := cmd.cachedSlot(); ok { + return slot + } + } info := c.cmdInfoPeek(cmd.Name()) - return c.cmdSlotWithPos(cmd, cmdFirstKeyPosWithInfo(cmd, info), prefferedSlot) + slot := c.cmdSlotWithPos(cmd, cmdFirstKeyPosWithInfo(cmd, info), prefferedSlot) + if prefferedSlot == -1 && slot >= 0 { + cmd.setCachedSlot(slot) + } + return slot } // cmdSlotWithPos computes the cluster slot for cmd given a pre-resolved first key @@ -2520,10 +3046,8 @@ func (c *ClusterClient) NewDynamicResolver() *commandInfoResolver { } func appendIfNotExist[T comparable](vals []T, newVal T) []T { - for _, v := range vals { - if v == newVal { - return vals - } + if slices.Contains(vals, newVal) { + return vals } return append(vals, newVal) } diff --git a/vendor/github.com/redis/go-redis/v9/osscluster_commands.go b/vendor/github.com/redis/go-redis/v9/osscluster_commands.go index b13f8e7e..bc6f60fd 100644 --- a/vendor/github.com/redis/go-redis/v9/osscluster_commands.go +++ b/vendor/github.com/redis/go-redis/v9/osscluster_commands.go @@ -9,19 +9,19 @@ import ( func (c *ClusterClient) DBSize(ctx context.Context) *IntCmd { cmd := NewIntCmd(ctx, "dbsize") _ = c.withProcessHook(ctx, cmd, func(ctx context.Context, _ Cmder) error { - var size int64 + var size atomic.Int64 err := c.ForEachMaster(ctx, func(ctx context.Context, master *Client) error { n, err := master.DBSize(ctx).Result() if err != nil { return err } - atomic.AddInt64(&size, n) + size.Add(n) return nil }) if err != nil { cmd.SetErr(err) } else { - cmd.val = size + cmd.val = size.Load() } return nil }) diff --git a/vendor/github.com/redis/go-redis/v9/osscluster_router.go b/vendor/github.com/redis/go-redis/v9/osscluster_router.go index 0da29530..b56271d1 100644 --- a/vendor/github.com/redis/go-redis/v9/osscluster_router.go +++ b/vendor/github.com/redis/go-redis/v9/osscluster_router.go @@ -85,7 +85,9 @@ func (c *ClusterClient) executeOnAllNodes(ctx context.Context, cmd Cmder, policy return err } - nodes := append(state.Masters, state.Slaves...) + nodes := make([]*clusterNode, 0, len(state.Masters)+len(state.Slaves)) + nodes = append(nodes, state.Masters...) + nodes = append(nodes, state.Slaves...) if len(nodes) == 0 { return errClusterNoNodes } @@ -494,10 +496,14 @@ func (c *ClusterClient) pickArbitraryNode(ctx context.Context) *clusterNode { return nil } - allNodes := append(state.Masters, state.Slaves...) - - idx := c.opt.ShardPicker.Next(len(allNodes)) - return allNodes[idx] + // Index into masters+slaves without materializing a combined slice. + // append(state.Masters, state.Slaves...) writes into the shared snapshot's + // spare capacity and races other routers, so pick directly. + idx := c.opt.ShardPicker.Next(len(state.Masters) + len(state.Slaves)) + if idx < len(state.Masters) { + return state.Masters[idx] + } + return state.Slaves[idx-len(state.Masters)] } // hasKeys checks if a command operates on keys diff --git a/vendor/github.com/redis/go-redis/v9/probabilistic.go b/vendor/github.com/redis/go-redis/v9/probabilistic.go index ee67911e..f4c40262 100644 --- a/vendor/github.com/redis/go-redis/v9/probabilistic.go +++ b/vendor/github.com/redis/go-redis/v9/probabilistic.go @@ -233,6 +233,7 @@ func newScanDumpCmd(ctx context.Context, args ...interface{}) *ScanDumpCmd { } func (cmd *ScanDumpCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -241,10 +242,12 @@ func (cmd *ScanDumpCmd) SetVal(val ScanDump) { } func (cmd *ScanDumpCmd) Result() (ScanDump, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ScanDumpCmd) Val() ScanDump { + cmd.await() return cmd.val } @@ -316,14 +319,17 @@ func (cmd *BFInfoCmd) SetVal(val BFInfo) { } func (cmd *BFInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *BFInfoCmd) Val() BFInfo { + cmd.await() return cmd.val } func (cmd *BFInfoCmd) Result() (BFInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -653,14 +659,17 @@ func (cmd *CFInfoCmd) SetVal(val CFInfo) { } func (cmd *CFInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *CFInfoCmd) Val() CFInfo { + cmd.await() return cmd.val } func (cmd *CFInfoCmd) Result() (CFInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -823,14 +832,17 @@ func (cmd *CMSInfoCmd) SetVal(val CMSInfo) { } func (cmd *CMSInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *CMSInfoCmd) Val() CMSInfo { + cmd.await() return cmd.val } func (cmd *CMSInfoCmd) Result() (CMSInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -1024,14 +1036,17 @@ func (cmd *TopKInfoCmd) SetVal(val TopKInfo) { } func (cmd *TopKInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *TopKInfoCmd) Val() TopKInfo { + cmd.await() return cmd.val } func (cmd *TopKInfoCmd) Result() (TopKInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -1279,14 +1294,17 @@ func (cmd *TDigestInfoCmd) SetVal(val TDigestInfo) { } func (cmd *TDigestInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *TDigestInfoCmd) Val() TDigestInfo { + cmd.await() return cmd.val } func (cmd *TDigestInfoCmd) Result() (TDigestInfo, error) { + cmd.await() return cmd.val, cmd.err } diff --git a/vendor/github.com/redis/go-redis/v9/pubsub_commands.go b/vendor/github.com/redis/go-redis/v9/pubsub_commands.go index ccc0ed52..b0857b9c 100644 --- a/vendor/github.com/redis/go-redis/v9/pubsub_commands.go +++ b/vendor/github.com/redis/go-redis/v9/pubsub_commands.go @@ -20,8 +20,15 @@ type PubSubCmdable interface { func (c cmdable) Publish(ctx context.Context, channel string, message interface{}) *IntCmd { cmd := NewIntCmd(ctx, "publish", channel, message) _ = c(ctx, cmd) - // Record PubSub message sent (if command succeeded) - if cmd.Err() == nil { + // Record PubSub message sent (if command succeeded). Gated on the result + // being readable WITHOUT blocking: on the deferred autopipeline face the + // call above only enqueues, so reading the outcome here would await the + // batch and turn a fire-and-forget publish into a blocking call — i.e. + // enabling telemetry would change the async call shape (review finding by + // codex on #3942). The metric is therefore skipped for a submission that + // has not executed yet; recording it from the execution path instead is a + // follow-up in the OTel wiring, not something the command wrapper can do. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { otel.RecordPubSubMessage(ctx, nil, "sent", channel, false) } return cmd @@ -30,8 +37,9 @@ func (c cmdable) Publish(ctx context.Context, channel string, message interface{ func (c cmdable) SPublish(ctx context.Context, channel string, message interface{}) *IntCmd { cmd := NewIntCmd(ctx, "spublish", channel, message) _ = c(ctx, cmd) - // Record PubSub message sent (if command succeeded) - if cmd.Err() == nil { + // Record PubSub message sent (if command succeeded). See Publish for why + // this is gated on the result being readable without blocking. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { otel.RecordPubSubMessage(ctx, nil, "sent", channel, true) } return cmd diff --git a/vendor/github.com/redis/go-redis/v9/push/processor.go b/vendor/github.com/redis/go-redis/v9/push/processor.go index b8112ddc..264d779d 100644 --- a/vendor/github.com/redis/go-redis/v9/push/processor.go +++ b/vendor/github.com/redis/go-redis/v9/push/processor.go @@ -2,6 +2,7 @@ package push import ( "context" + "errors" "github.com/redis/go-redis/v9/internal" "github.com/redis/go-redis/v9/internal/proto" @@ -26,6 +27,15 @@ type Processor struct { registry *Registry } +type timeoutError interface { + Timeout() bool +} + +func isTimeoutError(err error) bool { + var timeoutErr timeoutError + return errors.As(err, &timeoutErr) && timeoutErr.Timeout() +} + // NewProcessor creates a new push notification processor func NewProcessor() *Processor { return &Processor{ @@ -52,17 +62,59 @@ func (p *Processor) UnregisterHandler(pushNotificationName string) error { // This method should be called by the client in WithReader before reading the reply // It will try to read from the socket and if it is empty - it may block. func (p *Processor) ProcessPendingNotifications(ctx context.Context, handlerCtx NotificationHandlerContext, rd *proto.Reader) error { + return p.processPendingNotifications(ctx, handlerCtx, rd, false) +} + +// ProcessPendingNotificationsBuffered processes one pending push notification +// and then continues only through frames already buffered by that read. It is +// used by callers that have already established socket readiness and must not +// wait for another frame after draining the current batch. +func (p *Processor) ProcessPendingNotificationsBuffered( + ctx context.Context, handlerCtx NotificationHandlerContext, rd *proto.Reader, +) error { + return p.processPendingNotifications(ctx, handlerCtx, rd, true) +} + +func (p *Processor) processPendingNotifications( + ctx context.Context, + handlerCtx NotificationHandlerContext, + rd *proto.Reader, + bufferedContinuation bool, +) error { if rd == nil { return nil } - for { + processed := false + for !bufferedContinuation || !processed || rd.Buffered() > 0 { // Check if there's data available to read - replyType, err := rd.PeekReplyType() - if err != nil { - // No more data available or error reading - // if timeout, it will be handled by the caller - break + var replyType byte + if bufferedContinuation { + for { + b, err := rd.Peek(1) + if err != nil { + if isTimeoutError(err) { + return nil + } + return err + } + replyType = b[0] + if replyType != proto.RespAttr { + break + } + // Unlike Peek, DiscardNext consumes bytes. Any error here is + // fatal because the reader may be left mid-frame. + if err := rd.DiscardNext(); err != nil { + return err + } + } + } else { + var err error + replyType, err = rd.PeekReplyType() + if err != nil { + // No more data available or error reading. + break + } } // Only process push notifications (arrays starting with >) @@ -73,19 +125,31 @@ func (p *Processor) ProcessPendingNotifications(ctx context.Context, handlerCtx // see if we should skip this notification notificationName, err := rd.PeekPushNotificationName() if err != nil { + // Name too long to peek: consume & dispatch below rather than leave the + // frame at the buffer head (which would stall and desync the next reply). + if !errors.Is(err, proto.ErrPushNotificationNameTooLong) { + if bufferedContinuation { + if isTimeoutError(err) { + return nil + } + return err + } + break + } + } else if willHandleNotificationInClient(notificationName) { break } - if willHandleNotificationInClient(notificationName) { - break - } - - // Read the push notification + // Surface a ReadReply error (unlike the boundary peek errors above, + // which consumed nothing): it happens mid-frame after bytes are + // consumed, so the conn is desynced and the CSC drainer must remove it. + // Normal reply-read callers log-and-ignore this and let their own read fail. reply, err := rd.ReadReply() if err != nil { internal.Logger.Printf(ctx, "push: error reading push notification: %v", err) - break + return err } + processed = true // Convert to slice of interfaces notification, ok := reply.([]interface{}) @@ -165,10 +229,12 @@ func (v *VoidProcessor) ProcessPendingNotifications(_ context.Context, handlerCt // see if we should skip this notification notificationName, err := rd.PeekPushNotificationName() if err != nil { - break - } - - if willHandleNotificationInClient(notificationName) { + // Name too long to peek: still consume the frame below so it isn't + // misread as a reply. + if !errors.Is(err, proto.ErrPushNotificationNameTooLong) { + break + } + } else if willHandleNotificationInClient(notificationName) { break } diff --git a/vendor/github.com/redis/go-redis/v9/redis.go b/vendor/github.com/redis/go-redis/v9/redis.go index 3bb759c3..8883a1ad 100644 --- a/vendor/github.com/redis/go-redis/v9/redis.go +++ b/vendor/github.com/redis/go-redis/v9/redis.go @@ -1,6 +1,7 @@ package redis import ( + "bytes" "context" "errors" "fmt" @@ -64,17 +65,61 @@ type ( ) type hooksMixin struct { - hooksMu *sync.RWMutex + // hooksMu serializes writers (AddHook); readers never take it. + hooksMu *sync.Mutex + // state holds the immutable hook snapshot. Readers Load it lock-free; + // writers publish a replacement copy-on-write under hooksMu. + state *atomic.Pointer[hooksState] +} +// hooksState is an immutable snapshot of the hook configuration. Once stored +// in hooksMixin.state it is never mutated; AddHook builds a fresh copy. +type hooksState struct { slice []Hook initial hooks current hooks } +// rebuild recomputes current from initial + slice. It mutates the receiver, so +// it must only run on a state that has not yet been published. +func (s *hooksState) rebuild() { + s.initial.setDefaults() + + s.current.dial = s.initial.dial + s.current.process = s.initial.process + s.current.pipeline = s.initial.pipeline + s.current.txPipeline = s.initial.txPipeline + + for i := len(s.slice) - 1; i >= 0; i-- { + if wrapped := s.slice[i].DialHook(s.current.dial); wrapped != nil { + s.current.dial = wrapped + } + if wrapped := s.slice[i].ProcessHook(s.current.process); wrapped != nil { + s.current.process = wrapped + } + if wrapped := s.slice[i].ProcessPipelineHook(s.current.pipeline); wrapped != nil { + s.current.pipeline = wrapped + } + if wrapped := s.slice[i].ProcessPipelineHook(s.current.txPipeline); wrapped != nil { + s.current.txPipeline = wrapped + } + } +} + func (hs *hooksMixin) initHooks(hooks hooks) { - hs.hooksMu = new(sync.RWMutex) - hs.initial = hooks - hs.chain() + var slice []Hook + if hs.state != nil { + if old := hs.state.Load(); old != nil { + slice = old.slice + } + } + + hs.hooksMu = new(sync.Mutex) + hs.state = new(atomic.Pointer[hooksState]) + + state := &hooksState{slice: slice, initial: hooks} + state.rebuild() + hs.state.Store(state) } type hooks struct { @@ -136,51 +181,42 @@ func (h *hooks) setDefaults() { // Please note: "next(ctx, cmd)" is very important, it will call the next hook, // if "next(ctx, cmd)" is not executed, the redis command will not be executed. func (hs *hooksMixin) AddHook(hook Hook) { - hs.slice = append(hs.slice, hook) - hs.chain() -} - -func (hs *hooksMixin) chain() { - hs.initial.setDefaults() - hs.hooksMu.Lock() defer hs.hooksMu.Unlock() - hs.current.dial = hs.initial.dial - hs.current.process = hs.initial.process - hs.current.pipeline = hs.initial.pipeline - hs.current.txPipeline = hs.initial.txPipeline - - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].DialHook(hs.current.dial); wrapped != nil { - hs.current.dial = wrapped - } - if wrapped := hs.slice[i].ProcessHook(hs.current.process); wrapped != nil { - hs.current.process = wrapped - } - if wrapped := hs.slice[i].ProcessPipelineHook(hs.current.pipeline); wrapped != nil { - hs.current.pipeline = wrapped - } - if wrapped := hs.slice[i].ProcessPipelineHook(hs.current.txPipeline); wrapped != nil { - hs.current.txPipeline = wrapped - } + old := hs.state.Load() + state := &hooksState{ + slice: make([]Hook, len(old.slice)+1), + initial: old.initial, } + copy(state.slice, old.slice) + state.slice[len(old.slice)] = hook + state.rebuild() + + hs.state.Store(state) } func (hs *hooksMixin) clone() hooksMixin { - hs.hooksMu.Lock() - defer hs.hooksMu.Unlock() + old := hs.state.Load() + l := len(old.slice) + state := &hooksState{ + slice: old.slice[:l:l], + initial: old.initial, + current: old.current, + } - clone := *hs - l := len(clone.slice) - clone.slice = clone.slice[:l:l] - clone.hooksMu = new(sync.RWMutex) + clone := hooksMixin{ + hooksMu: new(sync.Mutex), + state: new(atomic.Pointer[hooksState]), + } + clone.state.Store(state) return clone } func (hs *hooksMixin) withProcessHook(ctx context.Context, cmd Cmder, hook ProcessHook) error { - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].ProcessHook(hook); wrapped != nil { + slice := hs.state.Load().slice + for i := len(slice) - 1; i >= 0; i-- { + if wrapped := slice[i].ProcessHook(hook); wrapped != nil { hook = wrapped } } @@ -190,8 +226,9 @@ func (hs *hooksMixin) withProcessHook(ctx context.Context, cmd Cmder, hook Proce func (hs *hooksMixin) withProcessPipelineHook( ctx context.Context, cmds []Cmder, hook ProcessPipelineHook, ) error { - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].ProcessPipelineHook(hook); wrapped != nil { + slice := hs.state.Load().slice + for i := len(slice) - 1; i >= 0; i-- { + if wrapped := slice[i].ProcessPipelineHook(hook); wrapped != nil { hook = wrapped } } @@ -199,26 +236,26 @@ func (hs *hooksMixin) withProcessPipelineHook( } func (hs *hooksMixin) dialHook(ctx context.Context, network, addr string) (net.Conn, error) { - // Access to hs.current is guarded by a read-only lock since it may be mutated by AddHook(...) - // while this dialer is concurrently accessed by the background connection pool population - // routine when MinIdleConns > 0. - hs.hooksMu.RLock() - current := hs.current - hs.hooksMu.RUnlock() + return hs.state.Load().current.dial(ctx, network, addr) +} - return current.dial(ctx, network, addr) +// hookCount reports how many user hooks are installed. The autopipeliner +// arms its await() self-deadlock guard only when hooks exist, keeping the +// guard a single atomic load on hook-free clients. +func (hs *hooksMixin) hookCount() int { + return len(hs.state.Load().slice) } func (hs *hooksMixin) processHook(ctx context.Context, cmd Cmder) error { - return hs.current.process(ctx, cmd) + return hs.state.Load().current.process(ctx, cmd) } func (hs *hooksMixin) processPipelineHook(ctx context.Context, cmds []Cmder) error { - return hs.current.pipeline(ctx, cmds) + return hs.state.Load().current.pipeline(ctx, cmds) } func (hs *hooksMixin) processTxPipelineHook(ctx context.Context, cmds []Cmder) error { - return hs.current.txPipeline(ctx, cmds) + return hs.state.Load().current.txPipeline(ctx, cmds) } //------------------------------------------------------------------------------ @@ -314,10 +351,26 @@ func (h *onCloseHooks) run() error { } type baseClient struct { + // apClosed flips when the shared pools begin closing; every wrapper and + // every clone SHARING those pools refuses to build a new autopipeliner + // from then on. A pointer: withTimeout/clone copy it, so the flag is one + // per pool-set, not one per wrapper. See baseClient.Close. + apClosed *atomic.Bool + opt *Options optLock sync.RWMutex connPool pool.Pooler pubSubPool *pool.PubSubPool + // pipelinePool is an optional separate connection pool for pipelining + // operations, used when PipelineReadBufferSize/PipelineWriteBufferSize is + // set so pipelines can use large buffers without bloating the main pool. + // nil means pipelines use connPool. + pipelinePool pool.Pooler + // pipelinePoolName is the pool name assigned to pipelinePool's connections + // (pool.Conn.PoolName()). It lets poolForConn route a connection back to the + // pool that owns it — e.g. so streaming-credentials re-auth closes/accounts a + // failed pipeline connection against pipelinePool, not connPool. + pipelinePoolName string hooksMixin // onClose holds named callbacks invoked when the client is closed. @@ -337,6 +390,51 @@ type baseClient struct { // streamingCredentialsManager is used to manage streaming credentials streamingCredentialsManager *streaming.Manager + + // himport is the client-side registry of HIMPORT fieldsets, used to + // lazily replay HIMPORT PREPARE onto pooled connections (see himport.go). + // Shared by clones and by Conn instances derived from the same pool. + himport *himportRegistry + + // csc is the shared client-side cache; nil when CSC is disabled. + csc Cache + + // cscKeyPrefix namespaces a shared cache by DB and fixed authentication + // identity. It is computed once during attachment and copied with the cache. + cscKeyPrefix string + + // allowClientTracking exempts a client from the CLIENT TRACKING guard (see + // process and generalProcessPipeline). Set only on initConn's internal conn + // wrapper, whose init pipeline legitimately issues CLIENT TRACKING ON; + // never set on user-visible clients. + allowClientTracking bool + + // The following are OWNER-ONLY and NOT copied by clone(): derived clients + // (Conn/WithTimeout) share the cache but must not stop the owner's + // goroutines or flush its cache on their own Close. + + // cscOwnsCache is true only when this client constructed its LocalCache (not + // an injected/shared one); it gates the defensive flush on drainer stop. + cscOwnsCache bool + + // cscDrainHandle is the background drainer handle (nil when none). Held + // on the client, not a global registry, so an un-Closed client stays + // GC-collectible and a runtime.AddCleanup net can stop the goroutine. Its + // presence also identifies the owner (the only client that runs the drainer + // and thus the one that deregisters cscPoolHook). + cscDrainHandle *cscDrainHandle + + // cscPoolHook is the evict-on-remove pool hook (nil when CSC is off). Unlike + // the owner-only fields above it IS copied by clone(): a clone reads it in + // processCached to attribute fetches to the shared hook. Only the owner (the + // one with cscDrainHandle) deregisters it when the drainer exits. + cscPoolHook pool.PoolHook + + // cscActive is allocated only after CSC attaches successfully and becomes + // false once the drainer stops (owner Close, GC cleanup, or damping). It is + // shared with derived clients so they initialize borrowed pool connections + // with tracking only while the parent's CSC is actually operational. + cscActive *atomic.Bool } func (c *baseClient) clone() *baseClient { @@ -345,13 +443,24 @@ func (c *baseClient) clone() *baseClient { c.maintNotificationsManagerLock.RUnlock() clone := &baseClient{ + apClosed: c.apClosed, opt: c.opt, connPool: c.connPool, + pipelinePool: c.pipelinePool, + pipelinePoolName: c.pipelinePoolName, pubSubPool: c.pubSubPool, onClose: c.onClose, pushProcessor: c.pushProcessor, maintNotificationsManager: maintNotificationsManager, streamingCredentialsManager: c.streamingCredentialsManager, + himport: c.himport, + csc: c.csc, + // cscPoolHook and cscActive travel with the cache (read in processCached); + // the owner-only fields — cscDrainHandle, cscOwnsCache — do not, so a clone's + // Close never tears down the owner's resources. + cscPoolHook: c.cscPoolHook, + cscActive: c.cscActive, + cscKeyPrefix: c.cscKeyPrefix, } return clone } @@ -405,16 +514,30 @@ func (c *baseClient) _getConn(ctx context.Context) (*pool.Conn, error) { return nil, err } + if err := c.initPooledConn(ctx, c.connPool, cn); err != nil { + return nil, err + } + + return cn, nil +} + +// initPooledConn brings a conn freshly obtained from p to a usable state: it +// runs the connection handshake if needed, records the connection-create-time +// metric, and re-acquires the conn after initConn parks it IDLE. On failure +// the conn is Removed from p (never leaked) and the error is unwrapped to the +// caller-visible cause. Shared by the main-pool path (_getConn) and the +// dedicated pipeline-pool path (withPipelineConn) so the two cannot drift. +func (c *baseClient) initPooledConn(ctx context.Context, p pool.Pooler, cn *pool.Conn) error { if cn.IsInited() { - return cn, nil + return nil } if err := c.initConn(ctx, cn); err != nil { - c.connPool.Remove(ctx, cn, err) - if err := errors.Unwrap(err); err != nil { - return nil, err + p.Remove(ctx, cn, err) + if unwrapped := errors.Unwrap(err); unwrapped != nil { + return unwrapped } - return nil, err + return err } if dialStartNs := cn.GetDialStartNs(); dialStartNs > 0 { @@ -427,10 +550,25 @@ func (c *baseClient) _getConn(ctx context.Context) (*pool.Conn, error) { // initConn will transition to IDLE state, so we need to acquire it // before returning it to the user. if !cn.TryAcquire() { - return nil, fmt.Errorf("redis: connection is not usable") + err := fmt.Errorf("redis: connection is not usable") + // Remove rather than abandon: an unacquirable conn left outside the + // pool's accounting would leak its slot. + p.Remove(ctx, cn, err) + return err } - return cn, nil + return nil +} + +// poolForConn returns the pool that owns cn — the dedicated pipeline pool when +// cn was dialed there, otherwise the main pool. Re-auth close/accounting must +// target the owning pool so a failed pipeline connection is removed from the +// pipeline pool's books, not the main pool's. +func (c *baseClient) poolForConn(cn *pool.Conn) pool.Pooler { + if c.pipelinePool != nil && c.pipelinePoolName != "" && cn.PoolName() == c.pipelinePoolName { + return c.pipelinePool + } + return c.connPool } func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth.Credentials) error { @@ -441,10 +579,11 @@ func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth // Use background context - timeout is handled by ReadTimeout in WithReader/WithWriter ctx := context.Background() - connPool := pool.NewSingleConnPool(c.connPool, poolCn) + connPool := pool.NewSingleConnPool(c.poolForConn(poolCn), poolCn) - // Pass hooks so that reauth commands are recorded/traced - cn := newConn(c.opt, connPool, &c.hooksMixin) + // Pass hooks so that reauth commands are recorded/traced; share the + // HIMPORT registry for the same reason as in initConn. + cn := newConn(c.opt, connPool, &c.hooksMixin, c.himport) if username != "" { err = cn.AuthACL(ctx, username, password).Err() @@ -455,6 +594,7 @@ func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth return err } } + func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { return func(poolCn *pool.Conn, err error) { if err != nil { @@ -464,7 +604,7 @@ func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { // waits for IDLE state before transitioning to UNUSABLE for re-auth). // From metrics perspective, the connection was never "used" by a client. // Note: Using context.Background() as this callback doesn't have access to caller's context. - err := c.connPool.CloseConn(context.Background(), poolCn, pool.CloseReasonAuthError, pool.MetricStateIdle) + err := c.poolForConn(poolCn).CloseConn(context.Background(), poolCn, pool.CloseReasonAuthError, pool.MetricStateIdle) if err != nil { internal.Logger.Printf(context.Background(), "redis: failed to close connection: %v", err) // try to close the network connection directly @@ -480,6 +620,25 @@ func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { } } +// resolveCredentials returns the username/password to authenticate with, using +// the non-streaming credential sources in precedence order: +// CredentialsProviderContext, then CredentialsProvider, then the static +// Username/Password fields. The StreamingCredentialsProvider path is handled +// separately by initConn (it requires per-connection listener wiring) and is +// intentionally not covered here. Returns empty strings when no credentials +// are configured. +func (opt *Options) resolveCredentials(ctx context.Context) (username, password string, err error) { + switch { + case opt.CredentialsProviderContext != nil: + return opt.CredentialsProviderContext(ctx) + case opt.CredentialsProvider != nil: + username, password = opt.CredentialsProvider() + case opt.Username != "" || opt.Password != "": + username, password = opt.Username, opt.Password + } + return username, password, nil +} + func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // This function is called in two scenarios: // 1. First-time init: Connection is in CREATED state (from pool.Get()) @@ -558,7 +717,22 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // If we fail, we must transition to CLOSED var initErr error connPool := pool.NewSingleConnPool(c.connPool, cn) - conn := newConn(c.opt, connPool, &c.hooksMixin) + // The handshake Conn (handed to OnConnect) must share the client's + // HIMPORT registry: a private registry restarts versions at 1, so an + // OnConnect prepare would mark the pooled connection with a version + // number that collides with the client registry's and silently skips + // the replay of a different fieldset definition. + conn := newConn(c.opt, connPool, &c.hooksMixin, c.himport) + // The internal wrapper does not serve cached reads, but it needs the + // successful-attachment signal both to issue CLIENT TRACKING during init + // and to guard the user-visible OnConnect callback below. + conn.baseClient.cscActive = c.cscActive + // This internal conn's init pipeline issues CLIENT TRACKING ON itself; + // exempt it from the guard that blocks user-issued CLIENT TRACKING. Setting + // the field after newConn is safe: initHooks bound the pipeline hook as a + // method value on the addressable baseClient, so the guard reads the + // updated field. + conn.baseClient.allowClientTracking = true username, password := "", "" if c.opt.StreamingCredentialsProvider != nil { @@ -598,16 +772,12 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { cn.SetOnClose(unsubscribeFromCredentialsProvider) username, password = credentials.BasicAuth() - } else if c.opt.CredentialsProviderContext != nil { - username, password, initErr = c.opt.CredentialsProviderContext(ctx) + } else { + username, password, initErr = c.opt.resolveCredentials(ctx) if initErr != nil { cn.GetStateMachine().Transition(pool.StateClosed) - return fmt.Errorf("failed to get credentials from context provider: %w", initErr) + return fmt.Errorf("failed to resolve credentials: %w", initErr) } - } else if c.opt.CredentialsProvider != nil { - username, password = c.opt.CredentialsProvider() - } else if c.opt.Username != "" || c.opt.Password != "" { - username, password = c.opt.Username, c.opt.Password } // for redis-server versions that do not support the HELLO command, @@ -616,6 +786,10 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // falls back to RESP2 regardless of c.opt.Protocol, and features that // require RESP3 (e.g. maintenance notifications) must be skipped. helloOK := false + // For redis-server versions that do not support HELLO, RESP2 continues to + // be used. Remember that negotiated fallback: configured Protocol remains 3, + // but CSC must not serve without RESP3 invalidations. + helloFallbackToRESP2 := false if initErr = conn.Hello(ctx, c.opt.Protocol, username, password, c.opt.ClientName).Err(); initErr == nil { // Authentication successful with HELLO command helloOK = true @@ -629,20 +803,40 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // difficult to rely on error strings to determine all results. cn.GetStateMachine().Transition(pool.StateClosed) return initErr - } else if password != "" { - // Try legacy AUTH command if HELLO failed - if username != "" { - initErr = conn.AuthACL(ctx, username, password).Err() - } else { - initErr = conn.Auth(ctx, password).Err() - } - if initErr != nil { - cn.GetStateMachine().Transition(pool.StateClosed) - return fmt.Errorf("failed to authenticate: %w", initErr) + } else { + helloFallbackToRESP2 = c.opt.Protocol == 3 + if password != "" { + // Try legacy AUTH command if HELLO failed. + if username != "" { + initErr = conn.AuthACL(ctx, username, password).Err() + } else { + initErr = conn.Auth(ctx, password).Err() + } + if initErr != nil { + cn.GetStateMachine().Transition(pool.StateClosed) + return fmt.Errorf("failed to authenticate: %w", initErr) + } } } - - _, initErr = conn.Pipelined(ctx, func(pipe Pipeliner) error { + if helloFallbackToRESP2 { + c.disableCSCServing(ctx, "HELLO 3 was rejected and the connection negotiated RESP2") + } + + // trackingEnabled reports whether THIS pool connection must issue + // CLIENT TRACKING ON during init. True when CSC (SharedTracking) is enabled: + // the shared cache is fed by per-connection tracking + the background + // drainer. Once CSC serving stops (owner Close, GC cleanup, or drainer + // damping), new and re-inited conns skip tracking — nothing consumes the + // pushes into the cache anymore. + trackingEnabled := !helloFallbackToRESP2 && !cn.IsPubSub() && c.cscTrackingRequested() + if trackingEnabled && c.cscConnInitGen(cn.GetID()) == 0 { + // First initialization establishes generation 1. Reinitialization + // already bumped and evicted through onCscReinit before replacing the + // socket, so it must not bump a second time here. + c.cscEvictOwnedEntries(cn.GetID()) + } + var trackingCmd *StatusCmd + initCmds, initErr := conn.Pipelined(ctx, func(pipe Pipeliner) error { if c.opt.DB > 0 { pipe.Select(ctx, c.opt.DB) } @@ -655,13 +849,56 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { pipe.ClientSetName(ctx, c.opt.ClientName) } + if trackingEnabled { + // Must run before any cacheable command is issued on this conn. + trackingCmd = pipe.ClientTrackingOn(ctx, nil) + } + return nil }) + // The exemption is init-only. OnConnect is user code and must go through + // the same CSC connection-state guard as every other public command path. + conn.baseClient.allowClientTracking = false + trackingRejected := trackingCmd != nil && isRedisError(trackingCmd.Err()) + for _, cmd := range initCmds { + if cmd != trackingCmd && cmd.Err() != nil { + trackingRejected = false + break + } + } + if trackingRejected { + // A server-side rejection means tracking is unavailable, but the + // connection and the preceding init commands are still usable. Disable + // CSC globally and continue without caching. Transport and protocol + // failures still take the normal connection-failure path below. + c.disableCSCServing(ctx, fmt.Sprintf("CLIENT TRACKING ON was rejected: %v", trackingCmd.Err())) + c.cscForgetConn(cn.GetID()) + trackingEnabled = false + initErr = nil + } if initErr != nil { + if trackingEnabled { + // cscEvictOwnedEntries above bumped this conn's init generation; a + // failed init never serves, and the pubsub path has no OnRemove + // hook (and the close hook below is not yet installed), so drop + // the entry here to keep the map bounded to live conns. + c.cscForgetConn(cn.GetID()) + } cn.GetStateMachine().Transition(pool.StateClosed) return fmt.Errorf("failed to initialize connection options: %w", initErr) } + if trackingEnabled { + // Evict this conn's entries on any close (incl. the ConnMaxLifetime/idle + // path that bypasses the OnRemove hook), since the server drops its + // tracking table on close. + c.cscInstallConnCloseHook(cn) + // A handoff replaces the socket before initConn runs. Bump and evict at + // the pre-swap boundary so fulfillCached cannot publish an old-socket + // reply during that gap. + c.cscInstallConnReinitHook(cn) + } + // Enable maintnotifications if maintnotifications are configured c.optLock.RLock() maintNotifEnabled := c.opt.MaintNotificationsConfig != nil && c.opt.MaintNotificationsConfig.Mode != maintnotifications.ModeDisabled @@ -799,16 +1036,38 @@ func (c *baseClient) releaseConn(ctx context.Context, cn *pool.Conn, err error) if c.opt.Limiter != nil { c.opt.Limiter.ReportResult(err) } + c.releaseConnToPool(ctx, c.connPool, cn, err) +} +// releaseConnToPool returns a conn to p after a command or pipeline ran on +// it: bad conns are Removed, pending push notifications are drained (a +// mid-frame drain failure also Removes — the reply stream may be +// desynchronized), and a client-side-cache post-read probe is requested when +// tracking is on. Limiter accounting stays with the callers, whose shapes +// differ. Shared by releaseConn and withPipelineConn so the two cannot drift. +func (c *baseClient) releaseConnToPool(ctx context.Context, p pool.Pooler, cn *pool.Conn, err error) { if isBadConn(err, false, c.opt.Addr) { - c.connPool.Remove(ctx, cn, err) - } else { - // process any pending push notifications before returning the connection to the pool - if err := c.processPushNotifications(ctx, cn); err != nil { - internal.Logger.Printf(ctx, "push: error processing pending notifications before releasing connection: %v", err) + p.Remove(ctx, cn, err) + return + } + // process any pending push notifications before returning the connection to the pool + if err := c.processPushNotifications(ctx, cn); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before releasing connection: %v", err) + if isBadConn(err, false, c.opt.Addr) { + // A mid-frame read failure may leave the reply stream + // desynchronized, so the connection cannot be reused. + p.Remove(ctx, cn, err) + return } - c.connPool.Put(ctx, cn) } + if c.cscTrackingRequested() { + // A TLS-like wrapper can retain decrypted bytes after the command + // reply even when its raw socket is empty. Ask the background + // drainer for one bounded post-read probe before relying on raw + // socket peeks again. + cn.MarkCscReadPending() + } + p.Put(ctx, cn) } func (c *baseClient) withConn( @@ -829,39 +1088,158 @@ func (c *baseClient) withConn( return fnErr } +// withPipelineConn executes fn with a connection from the pipeline pool when +// one is configured (PipelineReadBufferSize/PipelineWriteBufferSize set), +// otherwise it falls back to the regular pool via withConn. +// withPipelineConn is withConn/releaseConn for the DEDICATED pipeline pool. +// Conn preparation and release go through the shared pool-parameterized +// helpers (initPooledConn, releaseConnToPool) — the paths used to mirror each +// other by hand and drifted three times (a Limiter-ordering divergence, a +// missed drain-error removal, a missed client-side-cache probe), so only the +// Limiter shape is allowed to live here. +func (c *baseClient) withPipelineConn( + ctx context.Context, fn func(context.Context, *pool.Conn) error, +) (retErr error) { + // Use pipeline pool if available, otherwise fall back to regular pool. + if c.pipelinePool == nil { + return c.withConn(ctx, fn) + } + + // Honor the Limiter on the dedicated pipeline-pool path too, mirroring + // getConn/releaseConn: Allow() before acquiring and ReportResult() on every + // exit (including the early init/re-acquire failures below). Without this, + // enabling the pipeline pool would silently bypass throttling and failure + // reporting for callers that set a Limiter. + if c.opt.Limiter != nil { + if err := c.opt.Limiter.Allow(); err != nil { + return err + } + } + + // One deferred exit for both concerns, because their ORDER is part of the + // contract: releaseConn reports the result BEFORE the connection becomes + // available again, so a limiter or circuit breaker observes the failure + // before it can admit the next operation. Two separate defers would run + // LIFO and release first, letting another pipelined operation through + // against a breaker that has not seen the failure yet (review finding by + // codex on #3942). cn is nil on the acquire/init failure paths, which still + // must report. + var cn *pool.Conn + var fnErr error + defer func() { + if c.opt.Limiter != nil { + c.opt.Limiter.ReportResult(retErr) + } + if cn != nil { + c.releaseConnToPool(ctx, c.pipelinePool, cn, fnErr) + } + }() + + cn, retErr = c.pipelinePool.Get(ctx) + if retErr != nil { + cn = nil // nothing acquired: no release, but still report above + return retErr + } + + if err := c.initPooledConn(ctx, c.pipelinePool, cn); err != nil { + // initPooledConn already removed the conn from the pool on failure. + cn = nil + retErr = err + return retErr + } + + fnErr = fn(ctx, cn) + retErr = fnErr + return retErr +} + func (c *baseClient) dial(ctx context.Context, network, addr string) (net.Conn, error) { return c.opt.Dialer(ctx, network, addr) } +// cscTrackingRequested reports whether initConn must issue CLIENT TRACKING ON. +// cscActive is allocated only after attachment succeeds and is shared with +// derived clients: a conn initialized by Conn/Tx may later return to the +// parent's pool, but a configured cache whose attachment failed must not turn +// tracking on. +func (c *baseClient) cscTrackingRequested() bool { + if c.opt.Protocol != 3 || c.cscActive == nil || !c.cscActive.Load() { + return false + } + return c.opt.DB == 0 +} + func (c *baseClient) process(ctx context.Context, cmd Cmder) error { - // Start measuring total operation duration (includes all retries) - // Only call time.Now() if operation duration callback is set to avoid overhead - var operationStart time.Time opDurationCallback := otel.GetOperationDurationCallback() - if opDurationCallback != nil { - operationStart = time.Now() + if opDurationCallback == nil { + return c.processCommand(ctx, cmd, nil) + } + + start := time.Now() + var state processState + err := c.processCommand(ctx, cmd, &state) + opDurationCallback(ctx, time.Since(start), cmd, state.attempts, err, state.lastConn, c.opt.DB) + return err +} + +type processState struct { + attempts int + lastConn *pool.Conn +} + +func (c *baseClient) processCommand(ctx context.Context, cmd Cmder, state *processState) error { + // Reject commands that would make one pooled connection diverge from CSC's + // tracking or database assumptions. Pipelines mirror this guard below. + if err := c.cscCommandError(cmd); err != nil { + return err + } + if c.csc != nil && isCacheable(cmd) { + return c.processCached(ctx, cmd, state) } + return c.processWithRetry(ctx, cmd, nil, state) +} + +// processWithRetry runs cmd through the retry loop. capture (optional) is +// filled by the successful attempt's reply read for the CSC fetch path (see +// cscFetchCapture). +func (c *baseClient) processWithRetry( + ctx context.Context, cmd Cmder, capture *cscFetchCapture, state *processState, +) error { var lastConn *pool.Conn var lastErr error totalAttempts := 0 - for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + maxRetries := c.opt.MaxRetries + himportRetried := false + for attempt := 0; attempt <= maxRetries; attempt++ { totalAttempts++ attempt := attempt - retry, cn, err := c._process(ctx, cmd, attempt) + retry, cn, err := c._process(ctx, cmd, attempt, capture) if cn != nil { lastConn = cn } + if state != nil { + state.attempts = totalAttempts + state.lastConn = lastConn + } + // A "no such fieldset" reply for a registered fieldset means the + // connection lost its server session state (e.g. RESET, concurrent + // discard). The stale prepared flag was invalidated inside _process + // while the connection was still held; grant a single extra attempt + // so the retry re-prepares lazily on whichever connection it lands. + if err != nil && !retry && !himportRetried && !cmd.NoRetry() && + c.himportShouldRetrySet(cmd, err) { + himportRetried = true + if attempt == maxRetries { + maxRetries++ + } + lastErr = err + continue + } // Don't retry if command explicitly disables retries (e.g., RawWriteToCmd // which writes directly to an io.Writer and cannot undo partial writes) if err == nil || !retry || cmd.NoRetry() { - // Record total operation duration - if opDurationCallback != nil { - operationDuration := time.Since(operationStart) - opDurationCallback(ctx, operationDuration, cmd, totalAttempts, err, lastConn, c.opt.DB) - } - if err != nil { if errorCallback := pool.GetMetricErrorCallback(); errorCallback != nil { errorType, statusCode, isInternal := classifyCommandError(err) @@ -874,12 +1252,6 @@ func (c *baseClient) process(ctx context.Context, cmd Cmder) error { lastErr = err } - // Record failed operation after all retries - if opDurationCallback != nil { - operationDuration := time.Since(operationStart) - opDurationCallback(ctx, operationDuration, cmd, totalAttempts, lastErr, lastConn, c.opt.DB) - } - // Record error metric for exhausted retries if errorCallback := pool.GetMetricErrorCallback(); errorCallback != nil { errorType, statusCode, isInternal := classifyCommandError(lastErr) @@ -949,13 +1321,7 @@ func classifyCommandError(err error) (errorType, statusCode string, isInternal b return "UNKNOWN", "UNKNOWN", true } -func (c *baseClient) assertUnstableCommand(cmd Cmder) (bool, error) { - // All search commands (FTSearchCmd, AggregateCmd, FTInfoCmd, FTSpellCheckCmd, FTSynDumpCmd) - // now have stable RESP3 parsing. No commands require the UnstableResp3 flag anymore. - return false, nil -} - -func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool, *pool.Conn, error) { +func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int, capture *cscFetchCapture) (bool, *pool.Conn, error) { if attempt > 0 { if err := internal.Sleep(ctx, c.retryBackoff(attempt)); err != nil { return false, nil, err @@ -963,7 +1329,7 @@ func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool } var usedConn *pool.Conn - retryTimeout := uint32(0) + var retryTimeout atomic.Uint32 if err := c.withConn(ctx, func(ctx context.Context, cn *pool.Conn) error { usedConn = cn // Process any pending push notifications before executing the command @@ -971,41 +1337,110 @@ func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool internal.Logger.Printf(ctx, "push: error processing pending notifications before command: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and the + // PREPARE for an HIMPORT SET's registered fieldset are written in + // the same round trip, right before the command. + var injected []Cmder + if _, ok := cmd.(himportCmder); ok { + injected = c.himportInjectedCmds(ctx, cn, []Cmder{cmd}) + } + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmd(wr, cmd) }); err != nil { - atomic.StoreUint32(&retryTimeout, 1) + retryTimeout.Store(1) return err } readReplyFunc := cmd.readReply - // Apply unstable RESP3 search module. - if c.opt.Protocol != 2 { - useRawReply, err := c.assertUnstableCommand(cmd) - if err != nil { - return err - } - if useRawReply { - readReplyFunc = cmd.readRawReply + // When the caller requested raw-reply capture (client-side cache), + // read the reply as raw RESP bytes and re-parse them through the + // command's normal reply handler. This reuses proto.Reader rather + // than duplicating parsing logic in a bespoke cache serializer. + if capture != nil { + origRead := readReplyFunc + readReplyFunc = func(rd *proto.Reader) error { + raw, err := rd.ReadRawReply() + if err != nil { + return err + } + capture.raw = raw + return origRead(proto.NewReaderSize(bytes.NewReader(raw), len(raw)+1)) } } - if err := cn.WithReader(c.context(ctx), c.cmdTimeout(cmd), func(rd *proto.Reader) error { + readErr := cn.WithReader(c.context(ctx), c.cmdTimeout(cmd), func(rd *proto.Reader) error { // To be sure there are no buffered push notifications, we process them before reading the reply if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) } - return readReplyFunc(rd) - }); err != nil { + if len(injected) > 0 { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + return err + } + // A push notification can arrive between the injected + // replies and the command reply; drain again so the + // reply read below does not consume it as the command's. + if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } + } + err := readReplyFunc(rd) + // Assert the command type before touching the error: the + // errors.As chain inside himportNoSuchFieldset allocates, and + // this is the per-command hot path. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(err) { + // A failed injected PREPARE is the root cause of the + // command's "no such fieldset" reply (drained above). + for _, ic := range injected { + if prep, ok := ic.(*HImportPrepareCmd); ok && + prep.fieldsetName == set.fieldsetName && prep.Err() != nil { + err = prep.Err() + break + } + } + // The session lost a registered fieldset the flags claim is + // prepared — and the same event (failover, cross-region + // switch, reset storm) may have wiped other sessions whose + // flags also still look current. Bump the fieldset version + // so every connection re-prepares before its next use, + // wherever the retry granted by process() lands. + if himportNoSuchFieldset(err) { + if fs, registered := c.himport.lookup(set.fieldsetName); registered { + c.himport.refreshVersion(set.fieldsetName, fs.version) + } + } + } + return err + }) + // redis.Nil is a complete, valid negative reply. For a CSC fetch, retain + // its connection attribution before returning Nil to the caller so the + // raw reply can be cached and invalidated like any other read result. + if readErr != nil && (capture == nil || readErr != Nil) { if cmd.readTimeout() == nil { - atomic.StoreUint32(&retryTimeout, 1) + retryTimeout.Store(1) } else { - atomic.StoreUint32(&retryTimeout, 0) + retryTimeout.Store(0) } - return err + return readErr + } + if capture != nil { + // Attribute while the conn is still held: once it is released, a + // queued handoff may swap the socket and bump the generation, and + // this capture is what fulfillCached compares against. + capture.connID = cn.GetID() + capture.initGen = c.cscConnInitGen(capture.connID) } - return nil + if hc, ok := cmd.(himportCmder); ok { + c.himportAfterCmd(cn, hc) + } + return readErr }); err != nil { - retry := shouldRetry(err, atomic.LoadUint32(&retryTimeout) == 1) + retry := shouldRetry(err, retryTimeout.Load() == 1) return retry, usedConn, err } @@ -1064,6 +1499,13 @@ func (c *baseClient) enableMaintNotificationsUpgrades() error { // Initialize pool hook (safe to call without lock since manager is now set) manager.InitPoolHook(c.dialHook) + // If a dedicated pipeline connection pool is in use, attach an independent + // maintnotifications hook to it as well. Otherwise autopipelined/pipelined + // commands run on pipeline-pool connections that never receive MOVING/ + // MIGRATING handoff handling. + if c.pipelinePool != nil { + manager.InitPoolHookForPool(c.pipelinePool, c.dialHook) + } return nil } @@ -1088,8 +1530,30 @@ func (c *baseClient) disableMaintNotificationsUpgrades() error { // It is rare to Close a Client, as the Client is meant to be // long-lived and shared between many goroutines. func (c *baseClient) Close() error { + // The pools this baseClient owns are shared with every WithTimeout/ + // WithReadTimeout clone. Once ANY sharer closes them, no wrapper may + // build a fresh autopipeliner against them — its flushers would run + // against closed pools forever. The atomic is checked by the + // AutoPipeline getters of every wrapper sharing this base. + if c.apClosed != nil { + c.apClosed.Store(true) + } + if h := c.cscDrainHandle; h != nil { + h.closeOnce.Do(func() { + h.closeErr = c.closeResources() + }) + return h.closeErr + } + return c.closeResources() +} + +func (c *baseClient) closeResources() error { var firstErr error + // CSC teardown (no-op when CSC is not active): stop the background + // invalidation drainer before the pool it walks is torn down. + c.stopBackgroundDrainer() + // Close maintnotifications manager first if err := c.disableMaintNotificationsUpgrades(); err != nil { firstErr = err @@ -1100,13 +1564,18 @@ func (c *baseClient) Close() error { } // Unregister pools from OTel before closing them - otel.UnregisterPools(c.connPool, c.pubSubPool) + otel.UnregisterPools(c.connPool, c.pubSubPool, c.pipelinePool) if c.connPool != nil { if err := c.connPool.Close(); err != nil && firstErr == nil { firstErr = err } } + if c.pipelinePool != nil { + if err := c.pipelinePool.Close(); err != nil && firstErr == nil { + firstErr = err + } + } if c.pubSubPool != nil { if err := c.pubSubPool.Close(); err != nil && firstErr == nil { firstErr = err @@ -1138,6 +1607,14 @@ type pipelineProcessor func(context.Context, *pool.Conn, []Cmder) (bool, error) func (c *baseClient) generalProcessPipeline( ctx context.Context, cmds []Cmder, p pipelineProcessor, operationName string, ) error { + // Pipeline commands never pass through process, so apply the same CSC state + // guard here. initConn's internal client is exempt. + for _, cmd := range cmds { + if err := c.cscCommandError(cmd); err != nil { + setCmdsErr(cmds, err) + return err + } + } // Only call time.Now() if pipeline operation duration callback is set to avoid overhead var operationStart time.Time pipelineOpDurationCallback := otel.GetPipelineOperationDurationCallback() @@ -1163,7 +1640,9 @@ func (c *baseClient) generalProcessPipeline( // Enable retries by default to retry dial errors returned by withConn. canRetry := true - lastErr = c.withConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + // Route pipelines through the dedicated pipeline pool when configured; + // withPipelineConn falls back to the regular pool when it is not. + lastErr = c.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { lastConn = cn // Process any pending push notifications before executing the pipeline if err := c.processPushNotifications(ctx, cn); err != nil { @@ -1196,6 +1675,17 @@ func (c *baseClient) generalProcessPipeline( } } + // Retries exhausted on a retryable error: the loop fell through without the + // early-exit branch running, so the commands were never populated with the + // failure. Mirror that branch here so callers that observe results only + // per-command — notably AutoPipeline, which discards this function's returned + // error — see the error instead of a nil error and a zero value. Guard on + // !isRedisError so a per-command redis error (e.g. LOADING) keeps its own + // reply rather than being overwritten. + if !isRedisError(lastErr) { + setCmdsErr(cmds, lastErr) + } + if pipelineOpDurationCallback != nil { operationDuration := time.Since(operationStart) pipelineOpDurationCallback(ctx, operationDuration, operationName, len(cmds), totalAttempts, lastErr, lastConn, c.opt.DB) @@ -1217,21 +1707,69 @@ func (c *baseClient) pipelineProcessCmds( internal.Logger.Printf(ctx, "push: error processing pending notifications before writing pipeline: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the batch references get written ahead of + // the batch. + injected := c.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { setCmdsErr(cmds, err) return true, err } + var readErr error if err := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with every batch reply unreadXX: stamp the + // batch like a write failure. The outer retry loop stamps only + // on its exit branch, not when attempts run out, so without + // this a batch that keeps dying here would surface an Exec + // error while every command still reports Err() == nil. + setCmdsErr(cmds, err) + return err + } // read all replies - return c.pipelineReadCmds(ctx, cn, rd, cmds) + readErr = c.pipelineReadCmds(ctx, cn, rd, cmds) + if readErr != nil && !isRedisError(readErr) { + return readErr + } + c.himportAfterBatch(cn, injected, cmds) + return nil }); err != nil { return true, err } - return false, nil + // Registered fieldsets whose SETs came back "no such fieldset" (the + // session was lost between prepare and use) are re-prepared and those + // SETs re-issued once on the same connection; the error must not + // surface for managed fieldsets. + // + // A transport failure here must neither retry nor fail the batch: the + // first round trip was fully consumed and its results delivered, so + // re-executing would double-apply non-idempotent commands and failing + // would stamp a spurious error onto commands that succeeded. The + // re-issue errors stay on the retried SETs; the connection, which may + // hold unread replies, is marked for removal when released. + if err := c.himportRetryFailedSets(ctx, cn, cmds); err != nil { + internal.Logger.Printf(ctx, "himport: pipeline set re-issue failed: %v", err) + cn.MarkCloseOnPut("himport: transport error during set re-issue") + } + + // Preserve retryable first-command errors (e.g. LOADING) for the outer + // loop; the re-issue above may have cleared it. rawErr: this runs on the + // execution path; never await here (an async autopipeline command's ready + // channel is closed by this very batch — Err() would self-deadlock). + if readErr != nil { + readErr = cmds[0].rawErr() + } + return readErr != nil, readErr } func (c *baseClient) pipelineReadCmds(ctx context.Context, cn *pool.Conn, rd *proto.Reader, cmds []Cmder) error { @@ -1248,7 +1786,9 @@ func (c *baseClient) pipelineReadCmds(ctx context.Context, cn *pool.Conn, rd *pr } } // Retry errors like "LOADING redis is loading the dataset in memory". - return cmds[0].Err() + // rawErr: this runs on the execution path; never await here (an async + // autopipeline command's ready channel is closed by this very batch). + return cmds[0].rawErr() } func (c *baseClient) txPipelineProcessCmds( @@ -1259,7 +1799,17 @@ func (c *baseClient) txPipelineProcessCmds( internal.Logger.Printf(ctx, "push: error processing pending notifications before transaction: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the transaction references get written ahead + // of MULTI; the session state is visible inside the transaction. + injected := c.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { setCmdsErr(cmds, err) @@ -1267,6 +1817,13 @@ func (c *baseClient) txPipelineProcessCmds( } if err := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with every transaction reply unread: stamp + // the batch like a write failure (see pipelineProcessCmds). + setCmdsErr(cmds, err) + return err + } + statusCmd := cmds[0].(*StatusCmd) // Trim multi and exec. trimmedCmds := cmds[1 : len(cmds)-1] @@ -1277,7 +1834,11 @@ func (c *baseClient) txPipelineProcessCmds( } // Read replies. - return c.pipelineReadCmds(ctx, cn, rd, trimmedCmds) + err := c.pipelineReadCmds(ctx, cn, rd, trimmedCmds) + if err == nil || isRedisError(err) { + c.himportAfterBatch(cn, injected, trimmedCmds) + } + return err }); err != nil { return false, err } @@ -1341,6 +1902,16 @@ func (c *baseClient) txPipelineReadQueued(ctx context.Context, cn *pool.Conn, rd type Client struct { *baseClient cmdable + + // cscLifecycleOwner keeps the canonical Client wrapper (the one whose GC + // cleanup owns the drainer) reachable while a WithTimeout clone can still + // serve from its cache. Nil on the canonical wrapper and on non-CSC clones. + cscLifecycleOwner *Client + + autopipelinerMu *sync.Mutex // guards the autopipeliner fields against concurrent first-call creation + autopipeliner *AutoPipeliner // blocking face (Client.AutoPipeline) + asyncAutopipeliner *AutoPipeliner // deferred face (Client.AsyncAutoPipeline) + autopipelinerClosed bool // set by Close: refuse to resurrect a pipeliner on a closed client } // NewClient returns a client to the Redis Server specified by Options. @@ -1357,8 +1928,10 @@ func NewClient(opt *Options) *Client { c := Client{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, + himport: newHImportRegistry(), }, } c.init() @@ -1385,9 +1958,61 @@ func NewClient(opt *Options) *Client { panic(fmt.Errorf("redis: failed to create pubsub pool: %w", err)) } + // Optionally create a separate connection pool for pipelining, with its own + // (typically larger) buffers, so pipelines can use big buffers without + // bloating the main pool. Enabled when either pipeline buffer size is set. + if opt.PipelineReadBufferSize > 0 || opt.PipelineWriteBufferSize > 0 { + pipelineOpt := opt.clone() + if opt.PipelineReadBufferSize > 0 { + pipelineOpt.ReadBufferSize = opt.PipelineReadBufferSize + // Same clamp Options.init applies to the main pool: RESP3 push + // parsing needs a minimum read buffer, and a tiny pipeline reader + // would break push-notification handling on pipeline conns. + if pipelineOpt.Protocol == 3 && pipelineOpt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + pipelineOpt.ReadBufferSize = proto.MinRESP3ReadBufferSize + } + } + if opt.PipelineWriteBufferSize > 0 { + pipelineOpt.WriteBufferSize = opt.PipelineWriteBufferSize + } + if opt.PipelinePoolSize > 0 { + pipelineOpt.PoolSize = opt.PipelinePoolSize + } else { + pipelineOpt.PoolSize = 10 // default smaller pool for pipelining + } + pipelinePoolName := opt.Addr + "_" + uniqueID + "_pipeline" + c.pipelinePoolName = pipelinePoolName + c.pipelinePool, err = newConnPool(pipelineOpt, c.dialHook, pipelinePoolName) + if err != nil { + panic(fmt.Errorf("redis: failed to create pipeline connection pool: %w", err)) + } + } + if opt.StreamingCredentialsProvider != nil { c.streamingCredentialsManager = streaming.NewManager(c.connPool, c.opt.PoolTimeout) c.connPool.AddPoolHook(c.streamingCredentialsManager.PoolHook()) + if c.pipelinePool != nil { + c.pipelinePool.AddPoolHook(c.streamingCredentialsManager.PoolHook()) + } + } + + // CSC wiring (SharedTracking): shared cache + per-connection CLIENT TRACKING + + // background drainer. attachCSC is the strategy dispatch entry. + if opt.Protocol == 3 { + var cache Cache + if explicit := opt.ClientSideCache; explicit != nil { + cache = explicit + } else if cfg := opt.ClientSideCacheConfig; cfg != nil { + cache = NewLocalCache(*cfg) + // We constructed it, so we own it (may flush on drainer stop). + c.baseClient.cscOwnsCache = true + } + c.baseClient.attachCSC(context.Background(), cache) + + // Safety net for a client dropped without Close: the goroutines hold + // *baseClient (never *Client), so dropping *Client (returned as &c) + // triggers these cleanups, which stop them. See cscRegisterCleanups. + cscRegisterCleanups(&c) } // Initialize maintnotifications first if enabled and protocol is RESP3 @@ -1413,12 +2038,17 @@ func NewClient(opt *Options) *Client { // Register pools with OTel recorder if it supports pool registration // This allows async gauge metrics to pull stats from pools periodically - otel.RegisterPools(c.connPool, c.pubSubPool, opt.Addr) + otel.RegisterPools(c.connPool, c.pubSubPool, c.pipelinePool, opt.Addr) return &c } func (c *Client) init() { + // Fresh per-Client guard and no inherited autopipeliner: a WithTimeout clone + // (clone := *c) must not share the parent's mutex or AutoPipeliner instance. + c.autopipelinerMu = &sync.Mutex{} + c.autopipeliner = nil + c.asyncAutopipeliner = nil c.cmdable = c.Process c.initHooks(hooks{ dial: c.baseClient.dial, @@ -1428,15 +2058,89 @@ func (c *Client) init() { }) } +// WithTimeout returns a clone sharing the parent's connection pools with the +// given read/write timeout. The clone caches its own autopipeliners: an +// AutoPipeline()/AsyncAutoPipeline() created on the clone is NOT stopped by +// the parent's Close — call Close on the clone's autopipeliner explicitly. func (c *Client) WithTimeout(timeout time.Duration) *Client { + // Snapshot under the guard: AutoPipeline()/Close() mutate the + // autopipeliner fields concurrently, so a bare struct copy of them is a + // data race (init below discards the copied values either way). + c.autopipelinerMu.Lock() clone := *c + c.autopipelinerMu.Unlock() + if c.cscLifecycleOwner != nil { + clone.cscLifecycleOwner = c.cscLifecycleOwner + } else if c.baseClient.cscDrainHandle != nil { + clone.cscLifecycleOwner = c + } clone.baseClient = c.baseClient.withTimeout(timeout) clone.init() return &clone } +// Close closes the client, stopping both cached autopipeliners (the blocking +// AutoPipeline instance and the async AsyncAutoPipeline instance, if created) +// before releasing the underlying resources, so their background flusher +// goroutines don't outlive the client. AutoPipeliner.Close is idempotent and +// safe to call here even if autopipelining was never used. +// A WithTimeout clone delegates CSC teardown to the canonical wrapper that +// owns the background drainer. +func (c *Client) Close() error { + c.autopipelinerMu.Lock() + ap, async := c.autopipeliner, c.asyncAutopipeliner + c.autopipeliner, c.asyncAutopipeliner = nil, nil + // A later AutoPipeline()/AsyncAutoPipeline() call must not build a fresh + // pipeliner against the closed pools: nothing would ever close it and its + // flusher goroutines would leak. The getters check this flag. + c.autopipelinerClosed = true + c.autopipelinerMu.Unlock() + var firstErr error + for _, p := range []*AutoPipeliner{ap, async} { + if p != nil { + if err := p.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + } + if c.cscLifecycleOwner != nil { + // Delegate through the OWNER's *Client.Close, not its baseClient: + // the owner may hold cached autopipeliners of its own whose flusher + // goroutines must stop with the shared pools, and its + // autopipelinerClosed flag must flip so later owner getters cannot + // resurrect a pipeliner against closed pools. Client.Close is + // idempotent through baseClient.Close, so an owner also closed + // directly is fine. + if err := c.cscLifecycleOwner.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr + } + if err := c.baseClient.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr +} + func (c *Client) Conn() *Conn { - return newConn(c.opt, pool.NewStickyConnPool(c.connPool), &c.hooksMixin) + // Share the HIMPORT fieldset registry: the sticky pool borrows + // connections from this client's pool, so fieldsets prepared on them + // stay valid after the connections are returned. + conn := newConn(c.opt, c.baseClient.newStickyConnPool(), &c.hooksMixin, c.himport) + // A sticky client does not serve cache hits, but a new pool connection first + // initialized through it may later be reused by the parent. Share the + // successful-attachment signal so that connection is tracked exactly when + // the parent's CSC is active. + conn.baseClient.cscActive = c.baseClient.cscActive + // No-op today: the strategy needs an idle-conn drainer and a StickyConnPool + // has none, so CSC isn't active on a Conn() (its reads hit the server). Kept + // so a future sticky-pool-capable strategy attaches here. + conn.baseClient.attachCSC(context.Background(), c.csc) + // Carry the parent's shared eviction hook so that if this derived client + // initializes a pool conn, the close hook it installs still evicts from the + // parent cache (its own csc is nil). + conn.baseClient.cscPoolHook = c.baseClient.cscPoolHook + return conn } func (c *Client) Process(ctx context.Context, cmd Cmder) error { @@ -1505,7 +2209,10 @@ type PoolStats pool.Stats // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { stats := c.connPool.Stats() - stats.PubSubStats = *(c.pubSubPool.Stats()) + stats.PubSubStats = *c.pubSubPool.Stats() + if c.pipelinePool != nil { + stats.PipelineStats = c.pipelinePool.Stats() + } return (*PoolStats)(stats) } @@ -1521,6 +2228,79 @@ func (c *Client) Pipeline() Pipeliner { return &pipe } +// AutoPipeline returns the blocking autopipeliner for this client: a drop-in +// replacement for the normal command surface where each command call (ap.Set, +// ap.Get, ...) blocks until executed, exactly like a plain client — but the +// engine batches concurrent callers' commands into pipelines, so throughput is +// far higher (measured locally over loopback: ~1M+ SET/sec vs ~100k; indicative, not a guarantee). Commands keep per-goroutine order. +// +// By default, Options.AutoPipelineOptions is used if set, +// otherwise DefaultBlockingAutoPipelineOptions (a single ordered batch stream, +// which maximizes throughput and minimizes latency for the blocking face — see +// its doc). The instance is cached and shared; the first +// call's config wins and later calls return the same instance until it is closed. +// It must be closed (or close the client) to release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is AutoPipeline with explicit options instead of +// Options.AutoPipelineOptions / the default. The instance is cached and shared; +// the first call's config wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.autopipeliner, &c.autopipelinerClosed, c.baseClient.apClosed, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultBlockingAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { return newAutoPipeliner(c, cfg, true) }) +} + +// AsyncAutoPipeline returns the deferred (async) autopipeliner: command calls +// return immediately and the result accessors (Val/Result/Err) block until the +// command has executed. Submit a window of commands, then read their results, to +// keep each pipeline deep and reach the highest throughput (measured locally over loopback: ~2-3M SET/sec; indicative). +// +// By default, Options.AutoPipelineOptions is used if set, +// otherwise DefaultAutoPipelineOptions (ordered, MaxConcurrentBatches: 1) — a +// single goroutine's deferred commands execute in submit order. Use AsyncAutoPipelineWithOptions +// to override (and, for parallel batches, set Unordered). The instance is +// cached and shared; the first call's config wins. Close it (or the client) to +// release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is AsyncAutoPipeline with an explicit config +// instead of Options.AutoPipelineOptions / the default. The instance is cached +// and shared; the first call's config wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.asyncAutopipeliner, &c.autopipelinerClosed, c.baseClient.apClosed, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { return newAutoPipeliner(c, cfg, false) }) +} + func (c *Client) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().Pipelined(ctx, fn) } @@ -1539,7 +2319,7 @@ func (c *Client) TxPipeline() Pipeliner { func (c *Client) pubSub() *PubSub { pubsub := &PubSub{ - opt: c.opt, + opt: c.cloneOpt(), newConn: func(ctx context.Context, addr string, channels []string) (*pool.Conn, error) { cn, err := c.pubSubPool.NewConn(ctx, c.opt.Network, addr, channels) if err != nil { @@ -1634,14 +2414,20 @@ type Conn struct { } // newConn is a helper func to create a new Conn instance. -// the Conn instance is not thread-safe and should not be shared between goroutines. -// the parentHooks will be cloned, no need to clone before passing it. -func newConn(opt *Options, connPool pool.Pooler, parentHooks *hooksMixin) *Conn { +// The Conn instance is not thread-safe and should not be shared between goroutines. +// The parentHooks will be cloned, no need to clone before passing it. +// himport is the HIMPORT fieldset registry the Conn participates in — pass +// the owning client's registry (a private one would restart versions at 1 +// and collide with the client's version space on the shared pooled +// connections); nil disables HIMPORT tracking. +func newConn(opt *Options, connPool pool.Pooler, parentHooks *hooksMixin, himport *himportRegistry) *Conn { c := Conn{ baseClient: baseClient{ + apClosed: &atomic.Bool{}, opt: opt, connPool: connPool, onClose: &onCloseHooks{}, + himport: himport, }, } @@ -1739,25 +2525,116 @@ func (c *baseClient) processPushNotifications(ctx context.Context, cn *pool.Conn } } - // Check if there is any data to read before processing - // This is an optimization on UNIX systems where MaybeHasData is a syscall - // On Windows, MaybeHasData always returns true, so this check is a no-op + return c.peekAndProcessPushNotifications(ctx, cn) +} + +// peekAndProcessPushNotifications peeks the socket and processes any pending +// push notifications on cn unconditionally, bypassing the recent-health-check +// shortcut in processPushNotifications. Required on paths that do not follow +// up with a reply read on the same connection (e.g. the CSC cache-hit drain), +// where the shortcut would otherwise suppress invalidations buffered since the +// last health check. +func (c *baseClient) peekAndProcessPushNotifications(ctx context.Context, cn *pool.Conn) error { + if c.opt.Protocol != 3 || c.pushProcessor == nil { + return nil + } + if !cn.MaybeHasData() { return nil } - // Use WithReader to access the reader and process push notifications - // This is critical for maintnotifications to work properly - // NOTE: almost no timeouts are set for this read, so it should not block - // longer than necessary, 10us should be plenty of time to read if there are any push notifications - // on the socket. - return cn.WithReader(ctx, 10*time.Microsecond, func(rd *proto.Reader) error { - // Create handler context with client, connection pool, and connection information + // Short read timeout: MaybeHasData confirmed kernel-buffered bytes, so + // the first read returns immediately — the deadline only needs to cover + // scheduler pauses, not network waits. 10us was routinely lost to + // scheduling on loaded machines: the peek then timed out with nothing + // consumed, the processor treated that as "no pending data", and a + // connection with buffered push bytes was returned to the pool instead + // of being drained (or removed, when the frame turns out partial). + return cn.WithReader(ctx, time.Millisecond, func(rd *proto.Reader) error { handlerCtx := c.pushNotificationHandlerContext(cn) return c.pushProcessor.ProcessPendingNotifications(ctx, handlerCtx, rd) }) } +// cscFallbackProbeInterval bounds how often an idle connection without a +// portable readiness mechanism is subjected to a timed read. Post-command +// probes remain immediate; this is only the eventual invalidation fallback. +const cscFallbackProbeInterval = 100 * time.Millisecond + +// drainPushNotifications drains push frames buffered on a connection the CSC +// drainer has claimed, under a HARD read deadline. processorSucceeded reports a +// successful processor invocation; it resets custom-processor damping even when +// the frame was hidden inside a transport wrapper. A non-nil error is +// connection-fatal (the drainer removes the conn), including a read timeout +// after reply consumption starts: the reader may be desynchronized. A custom +// processor's error is also fatal because its contract cannot prove no bytes +// were consumed. +func (c *baseClient) drainPushNotifications(cn *pool.Conn) (processorSucceeded bool, err error) { + if c.opt.Protocol != 3 || c.pushProcessor == nil { + return false, nil + } + // Skip only when nothing is buffered (reader) AND nothing on the socket: + // MaybeHasData peeks only the socket, but an invalidate can sit in cn.rd. + readPending := cn.TakeCscReadPending() + periodicReadPending := cn.TakeCscPeriodicReadPending(cscFallbackProbeInterval) + socketData, socketErr := cn.CheckForData() + if socketErr != nil { + return false, socketErr + } + hasData := cn.HasBufferedData() || socketData + if !readPending && !periodicReadPending && !hasData { + return false, nil + } + if !hasData { + // TLS and opaque wrappers can hide bytes from the socket readiness + // check. Probe one byte without consuming it under a tiny deadline; + // only a confirmed byte gets the longer fragmented-frame budget below. + err := cn.WithReaderHardDeadline(cscDrainProbeReadCap, func(rd *proto.Reader) error { + _, err := rd.Peek(1) + return err + }) + if err != nil { + if isTimeout, hasTimeoutFlag := isTimeoutError(err); isTimeout && hasTimeoutFlag { + return false, nil + } + return false, err + } + } + + handlerCtx := c.pushNotificationHandlerContext(cn) + handlerCtx.Client = cscHandlerClient{baseClient: c} + err = cn.WithReaderHardDeadline(cscDrainHardReadCap, func(rd *proto.Reader) error { + if processor, ok := c.pushProcessor.(*push.Processor); ok { + return processor.ProcessPendingNotificationsBuffered( + context.Background(), handlerCtx, rd) + } + return c.pushProcessor.ProcessPendingNotifications(context.Background(), handlerCtx, rd) + }) + if err != nil { + // The built-in processor surfaces mid-frame ReadReply errors (a benign + // boundary peek timeout returns nil). allowTimeout=false: such an error + // means bytes were consumed mid-frame, leaving the conn desynced — + // re-pooling would corrupt the next command's reply, so remove it. + if _, builtin := c.pushProcessor.(*push.Processor); builtin { + if isBadConn(err, false, c.opt.Addr) { + return true, err // fatal read/protocol/connection error — remove the conn + } + return true, nil + } + // A CUSTOM processor's error contract is unknown: it may have consumed + // part of a frame before failing, and a mid-frame reader silently + // corrupts the next command's reply. The conn is idle and held solely + // by the drainer, so the safe default — removal — costs one reconnect; + // persistent failures are damped by the drainer (cscDrainCustomErrCap). + internal.Logger.Printf(context.Background(), "csc: drain: custom push processor error (removing conn): %v", err) + return true, err + } + // The processor ran successfully. This is stronger evidence than a clean + // connection on which it was never invoked, and prevents successful TLS- + // buffered drains from being counted as if failures were consecutive. + return true, nil +} + // processPendingPushNotificationWithReader processes all pending push notifications on a connection // This method should be called by the client in WithReader before reading the reply func (c *baseClient) processPendingPushNotificationWithReader(ctx context.Context, cn *pool.Conn, rd *proto.Reader) error { diff --git a/vendor/github.com/redis/go-redis/v9/ring.go b/vendor/github.com/redis/go-redis/v9/ring.go index b60d3eab..a22e1667 100644 --- a/vendor/github.com/redis/go-redis/v9/ring.go +++ b/vendor/github.com/redis/go-redis/v9/ring.go @@ -48,6 +48,10 @@ type RingOptions struct { // NewClient creates a shard client with provided options. NewClient func(opt *Options) *Client + // himport is the ring-wide HIMPORT fieldset registry, set by NewRing and + // shared with every shard client (see himport.go, himport_cluster.go). + himport *himportRegistry + // ClientName will execute the `CLIENT SETNAME ClientName` command for each conn. ClientName string @@ -143,6 +147,14 @@ type RingOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining on + // each shard, with its own (typically larger) buffers. See the same-named + // fields on Options for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + TLSConfig *tls.Config Limiter Limiter @@ -194,13 +206,13 @@ func (opt *RingOptions) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.ReadBufferSize == 0 { @@ -247,6 +259,10 @@ func (opt *RingOptions) clientOptions() *Options { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + TLSConfig: opt.TLSConfig, Limiter: opt.Limiter, @@ -262,7 +278,7 @@ func (opt *RingOptions) clientOptions() *Options { type ringShard struct { Client *Client - down int32 + down atomic.Int32 addr string } @@ -270,10 +286,16 @@ func newRingShard(opt *RingOptions, addr string) *ringShard { clopt := opt.clientOptions() clopt.Addr = addr - return &ringShard{ + shard := &ringShard{ Client: opt.NewClient(clopt), addr: addr, } + // Share the ring-wide HIMPORT fieldset registry so any shard connection + // serving an HIMPORT SET can lazily replay the PREPARE. + if opt.himport != nil { + shard.Client.himport = opt.himport + } + return shard } func (shard *ringShard) String() string { @@ -288,7 +310,7 @@ func (shard *ringShard) String() string { func (shard *ringShard) IsDown() bool { const threshold = 3 - return atomic.LoadInt32(&shard.down) >= threshold + return shard.down.Load() >= threshold } func (shard *ringShard) IsUp() bool { @@ -299,7 +321,7 @@ func (shard *ringShard) IsUp() bool { func (shard *ringShard) Vote(up bool) bool { if up { changed := shard.IsDown() - atomic.StoreInt32(&shard.down, 0) + shard.down.Store(0) return changed } @@ -307,7 +329,7 @@ func (shard *ringShard) Vote(up bool) bool { return false } - atomic.AddInt32(&shard.down, 1) + shard.down.Add(1) return shard.IsDown() } @@ -369,9 +391,10 @@ func (c *ringSharding) SetAddrs(addrs map[string]string) { return } existing := c.shards + onNewNode := c.onNewNode c.mu.RUnlock() - shards, created, unused := c.newRingShards(addrs, existing) + shards, created, unused := c.newRingShards(addrs, existing, onNewNode) c.mu.Lock() if c.closed { @@ -387,7 +410,7 @@ func (c *ringSharding) SetAddrs(addrs map[string]string) { } func (c *ringSharding) newRingShards( - addrs map[string]string, existing *ringShards, + addrs map[string]string, existing *ringShards, onNewNode []func(rdb *Client), ) (shards *ringShards, created, unused map[string]*ringShard) { shards = &ringShards{m: make(map[string]*ringShard, len(addrs))} created = make(map[string]*ringShard) // indexed by addr @@ -408,7 +431,7 @@ func (c *ringSharding) newRingShards( shards.m[name] = shard created[addr] = shard - for _, fn := range c.onNewNode { + for _, fn := range onNewNode { fn(shard.Client) } } @@ -604,7 +627,16 @@ func NewRing(opt *RingOptions) *Ring { if opt == nil { panic("redis: NewRing nil options") } + // Shallow-copy the options: the ring-wide HIMPORT registry is carried + // through them to shard construction, and reusing one caller-owned + // RingOptions across several rings must not make the rings share (or + // clobber each other's) registry. + optCopy := *opt + opt = &optCopy opt.init() + // The registry must exist before the first shard is created; shards + // adopt it in newRingShard. + opt.himport = newHImportRegistry() hbCtx, hbCancel := context.WithCancel(context.Background()) @@ -714,6 +746,17 @@ func (c *Ring) SSubscribe(ctx context.Context, channels ...string) *PubSub { return shard.Client.SSubscribe(ctx, channels...) } +// Publish posts the message to the channel +func (c *Ring) Publish(ctx context.Context, channel string, message interface{}) *IntCmd { + shard, err := c.sharding.GetByKey(channel) + if err != nil { + cmd := NewIntCmd(ctx, "publish", channel, message) + cmd.SetErr(err) + return cmd + } + return shard.Client.Publish(ctx, channel, message) +} + func (c *Ring) OnNewNode(fn func(rdb *Client)) { c.sharding.OnNewNode(fn) } @@ -820,6 +863,34 @@ func (c *Ring) Pipeline() Pipeliner { return &pipe } +// ErrRingAutoPipelineUnsupported is returned by Ring's AutoPipeline / +// AsyncAutoPipeline (and their WithOptions forms); check for it with +// errors.Is. Autopipelining is not implemented for Ring; use the per-shard +// clients or a ClusterClient. Ring is part of the UniversalClient +// interface, so these methods exist to satisfy it and fail explicitly rather +// than being silently absent. +var ErrRingAutoPipelineUnsupported = errors.New("redis: AutoPipeline is not supported by Ring") + +// AutoPipeline is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return nil, ErrRingAutoPipelineUnsupported +} + +// AsyncAutoPipeline is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return nil, ErrRingAutoPipelineUnsupported +} + func (c *Ring) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().Pipelined(ctx, fn) } diff --git a/vendor/github.com/redis/go-redis/v9/script.go b/vendor/github.com/redis/go-redis/v9/script.go index 92d508f9..5b649f66 100644 --- a/vendor/github.com/redis/go-redis/v9/script.go +++ b/vendor/github.com/redis/go-redis/v9/script.go @@ -192,7 +192,7 @@ func (s *Script) EvalShaRO(ctx context.Context, c Scripter, keys []string, args // it is retried using EVAL. func (s *Script) Run(ctx context.Context, c Scripter, keys []string, args ...interface{}) *Cmd { r := s.EvalSha(ctx, c, keys, args...) - if errors.Is(r.Err(), ErrNoScript) { + if isNoScriptErr(r.Err()) { return s.Eval(ctx, c, keys, args...) } return r @@ -202,8 +202,20 @@ func (s *Script) Run(ctx context.Context, c Scripter, keys []string, args ...int // it is retried using EVAL_RO. func (s *Script) RunRO(ctx context.Context, c Scripter, keys []string, args ...interface{}) *Cmd { r := s.EvalShaRO(ctx, c, keys, args...) - if errors.Is(r.Err(), ErrNoScript) { + if isNoScriptErr(r.Err()) { return s.EvalRO(ctx, c, keys, args...) } return r } + +// isNoScriptErr reports whether err means "this digest is not cached", whether +// it arrived already normalized to ErrNoScript or as the server's raw NOSCRIPT +// error. Both are accepted because the Eval wrappers only normalize when the +// result is readable without blocking — on the deferred autopipeline face the +// raw error reaches here untouched (see cmdable.eval). +func isNoScriptErr(err error) bool { + if err == nil { + return false + } + return errors.Is(err, ErrNoScript) || HasErrorPrefix(err, "NOSCRIPT") +} diff --git a/vendor/github.com/redis/go-redis/v9/scripting_commands.go b/vendor/github.com/redis/go-redis/v9/scripting_commands.go index 3310b9d0..6845f398 100644 --- a/vendor/github.com/redis/go-redis/v9/scripting_commands.go +++ b/vendor/github.com/redis/go-redis/v9/scripting_commands.go @@ -60,8 +60,16 @@ func (c cmdable) eval(ctx context.Context, name, payload string, keys []string, cmd.SetFirstKeyPos(3) } _ = c(ctx, cmd) - if err := cmd.Err(); err != nil { - if HasErrorPrefix(err, "NOSCRIPT") { + // Normalize NOSCRIPT to ErrNoScript for Script.Run/RunRO's EVAL fallback, + // but only when the result is already readable: on the deferred + // autopipeline face the call above merely enqueues, and reading the outcome + // here would await execution — making the whole Eval family synchronous on + // a face whose contract is to return immediately (review finding by codex + // on #3942). When the result is still pending the normalization is skipped; + // Script.Run/RunRO also match the raw NOSCRIPT prefix, so the fallback + // keeps working on that face. + if cmd.resultReady() { + if err := cmd.rawErr(); err != nil && HasErrorPrefix(err, "NOSCRIPT") { cmd.SetErr(ErrNoScript) } } diff --git a/vendor/github.com/redis/go-redis/v9/search_builders.go b/vendor/github.com/redis/go-redis/v9/search_builders.go index a6c6718c..094441c7 100644 --- a/vendor/github.com/redis/go-redis/v9/search_builders.go +++ b/vendor/github.com/redis/go-redis/v9/search_builders.go @@ -313,6 +313,31 @@ func (b *AggregateBuilder) ReduceAs(fn SearchAggregator, alias string, args ...i return b } +// Collect adds a REDUCE COLLECT clause to the last step, which must be a +// GROUPBY. The COLLECT options (FIELDS/DISTINCT/SORTBY/LIMIT/AS) are rendered +// and the argument count is computed automatically; field and sort names are +// normalized to a single "@" prefix. Set FTAggregateCollect.As to alias the +// output column. +// +// If the last step is not a GROUPBY, or the options are invalid (no FIELDS +// selector), Run returns the recorded error without issuing the command. +// COLLECT requires Redis 8.8+ with unstable features enabled. +func (b *AggregateBuilder) Collect(o FTAggregateCollect) *AggregateBuilder { + n := len(b.options.Steps) + if n == 0 || b.options.Steps[n-1].GroupBy == nil { + b.setErr(fmt.Errorf("FT.AGGREGATE: Collect must follow a GroupBy step")) + return b + } + reducer, err := NewCollectReducer(o) + if err != nil { + b.setErr(err) + return b + } + g := b.options.Steps[n-1].GroupBy + g.Reduce = append(g.Reduce, reducer) + return b +} + // SortBy adds SORTBY ASC|DESC. Consecutive SortBy calls (with no // other step in between) are merged into a single SORTBY clause so fields // act as tiebreakers. A SortBy call after a non-SortBy step starts a new diff --git a/vendor/github.com/redis/go-redis/v9/search_collect.go b/vendor/github.com/redis/go-redis/v9/search_collect.go new file mode 100644 index 00000000..48389a0c --- /dev/null +++ b/vendor/github.com/redis/go-redis/v9/search_collect.go @@ -0,0 +1,235 @@ +package redis + +import ( + "fmt" + "strings" +) + +// ---------------------- +// FT.AGGREGATE COLLECT reducer +// ---------------------- +// +// COLLECT is a GROUPBY reducer for FT.AGGREGATE (Redis 8.8+, gated behind +// search-enable-unstable-features). Within each group it projects a chosen +// set of fields from every row, optionally deduplicates, sorts, and limits +// them, and emits the result as an array of per-entry maps under the reducer +// alias. +// +// COLLECT is not a standalone command; it is a REDUCE clause inside +// FT.AGGREGATE. The helpers below assemble the reducer token list and compute +// its argument count, so callers do not have to hand-write FIELDS/SORTBY/LIMIT +// tokens or remember to @-prefix every name. + +// FTAggregateCollect describes a COLLECT reducer. It is rendered into a +// standard FTAggregateReducer via NewCollectReducer, or appended to a builder +// via AggregateBuilder.Collect. +// +// Field and sort names may be supplied with or without a leading "@"; each is +// normalized to a single "@" on the wire. Output map keys returned by +// the server are the bare names (see AggregateRow.Collect). +type FTAggregateCollect struct { + // FieldsAll emits FIELDS *, projecting every field present in the + // pipeline at the COLLECT stage. It is not a whole-document fetch; pair + // it with an upstream LOAD * to collect complete documents. FieldsAll + // takes precedence over Fields when both are set. + FieldsAll bool + + // Fields is the explicit list of fields to project (FIELDS @f ...). + // Ignored when FieldsAll is true. Exactly one of FieldsAll or a non-empty + // Fields must be set. + Fields []string + + // Distinct emits DISTINCT, deduplicating entries with identical projected + // fields. + // + // NOTE: DISTINCT is specified by the product but not yet implemented by + // the server. Sending it currently produces a server error. The option is + // kept for forward compatibility; leave it false unless the target server + // supports it. + Distinct bool + + // SortBy orders entries within each group. Direction defaults to ASC when + // neither Asc nor Desc is set. With Limit, SORTBY acts as a top-N + // selection. Reuses FTAggregateSortBy for consistency with the rest of the + // aggregate API. + SortBy []FTAggregateSortBy + + // Limit returns at most Count entries per group after skipping Offset. + // nil means no LIMIT clause (distinct from LIMIT 0 0). + Limit *FTAggregateCollectLimit + + // As sets the reducer output column name (AS ). It is emitted + // outside the reducer argument count. + As string +} + +// FTAggregateCollectLimit is the LIMIT clause of a COLLECT +// reducer. Numeric bounds are enforced by the server, not the client. +type FTAggregateCollectLimit struct { + Offset int + Count int +} + +// ensureAtPrefix normalizes a field or sort name to exactly one leading "@", +// collapsing any number of leading "@" (including none) to a single prefix. +func ensureAtPrefix(name string) string { + return "@" + strings.TrimLeft(name, "@") +} + +// buildCollectArgs renders a FTAggregateCollect into the reducer argument +// token list (everything after "REDUCE COLLECT ", excluding AS ). +// The serializer computes as len(args), which matches the COLLECT +// contract: narg counts every FIELDS/DISTINCT/SORTBY/LIMIT token. +func buildCollectArgs(o FTAggregateCollect) ([]interface{}, error) { + args := make([]interface{}, 0, 8) + + // FIELDS (required): either * or a counted list of @-names. + switch { + case o.FieldsAll: + args = append(args, "FIELDS", "*") + case len(o.Fields) > 0: + args = append(args, "FIELDS", len(o.Fields)) + for _, f := range o.Fields { + if strings.TrimLeft(f, "@") == "" { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: empty field name in Fields") + } + args = append(args, ensureAtPrefix(f)) + } + default: + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT requires FieldsAll or a non-empty Fields list") + } + + // DISTINCT (optional, forward-compatible). + if o.Distinct { + args = append(args, "DISTINCT") + } + + // SORTBY (optional). sort_narg counts each field plus its optional + // direction token. + if len(o.SortBy) > 0 { + sortTokens := make([]interface{}, 0, len(o.SortBy)*2) + for _, s := range o.SortBy { + if strings.TrimLeft(s.FieldName, "@") == "" { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: empty field name in SortBy") + } + if s.Asc && s.Desc { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: ASC and DESC are mutually exclusive") + } + sortTokens = append(sortTokens, ensureAtPrefix(s.FieldName)) + switch { + case s.Desc: + sortTokens = append(sortTokens, "DESC") + case s.Asc: + sortTokens = append(sortTokens, "ASC") + // neither set: ASC is the server default; emit nothing. + } + } + args = append(args, "SORTBY", len(sortTokens)) + args = append(args, sortTokens...) + } + + // LIMIT (optional). + if o.Limit != nil { + args = append(args, "LIMIT", o.Limit.Offset, o.Limit.Count) + } + + return args, nil +} + +// NewCollectReducer builds a COLLECT FTAggregateReducer for use with +// FTAggregateOptions.GroupBy[i].Reduce. It normalizes field/sort names and +// computes the argument count automatically. +// +// It returns an error only for local API misuse: a missing FIELDS selector, an +// empty field name (in Fields or SortBy), or a SortBy entry with both Asc and +// Desc set. Numeric bounds and the unstable-features gate are enforced by the +// server and surface unchanged through the command reply. +func NewCollectReducer(o FTAggregateCollect) (FTAggregateReducer, error) { + args, err := buildCollectArgs(o) + if err != nil { + return FTAggregateReducer{}, err + } + return FTAggregateReducer{Reducer: SearchCollect, Args: args, As: o.As}, nil +} + +// ---------------------- +// COLLECT response decoding +// ---------------------- + +// CollectEntry is a single collected row: a sparse map of bare field name to +// value. A field absent from a row is omitted from its entry (no NULL +// placeholder), so entries in the same column may have different key sets. +type CollectEntry = map[string]interface{} + +// CollectColumn is the value stored under a COLLECT reducer alias: the ordered +// list of collected entries for a group. +type CollectColumn = []CollectEntry + +// Collect decodes the COLLECT reducer column stored under alias in this row +// into a uniform CollectColumn, hiding the RESP2/RESP3 representation +// difference (RESP3 entries are maps; RESP2 entries are flat key/value +// arrays). +// +// It returns (nil, nil) when the alias is absent from the row. Entry order is +// preserved as returned by the server; it is meaningful only when the COLLECT +// reducer was given a SORTBY. +func (r AggregateRow) Collect(alias string) (CollectColumn, error) { + v, ok := r.Fields[alias] + if !ok { + return nil, nil + } + return parseCollectValue(v) +} + +// parseCollectValue decodes a raw COLLECT alias value (an array of entries) +// into a CollectColumn. +func parseCollectValue(v interface{}) (CollectColumn, error) { + if v == nil { + return nil, nil + } + arr, ok := v.([]interface{}) + if !ok { + return nil, fmt.Errorf("redis: COLLECT value has type %T, want array of entries", v) + } + out := make(CollectColumn, 0, len(arr)) + for i, e := range arr { + entry, err := parseCollectEntry(e) + if err != nil { + return nil, fmt.Errorf("redis: COLLECT entry %d: %w", i, err) + } + out = append(out, entry) + } + return out, nil +} + +// parseCollectEntry decodes a single collected entry from either the RESP3 +// map form or the RESP2 flat key/value array form into a CollectEntry. Keys +// are passed through as-is: the server already returns them without the "@" +// prefix. +func parseCollectEntry(e interface{}) (CollectEntry, error) { + switch m := e.(type) { + case map[interface{}]interface{}: // RESP3 + out := make(CollectEntry, len(m)) + for k, val := range m { + out[fmt.Sprint(k)] = val + } + return out, nil + case map[string]interface{}: // already string-keyed + return m, nil + case []interface{}: // RESP2 flat [field, value, field, value, ...] + if len(m)%2 != 0 { + return nil, fmt.Errorf("odd-length key/value array of length %d", len(m)) + } + out := make(CollectEntry, len(m)/2) + for i := 0; i < len(m); i += 2 { + key, ok := m[i].(string) + if !ok { + key = fmt.Sprint(m[i]) + } + out[key] = m[i+1] + } + return out, nil + default: + return nil, fmt.Errorf("unexpected type %T, want map or key/value array", e) + } +} diff --git a/vendor/github.com/redis/go-redis/v9/search_commands.go b/vendor/github.com/redis/go-redis/v9/search_commands.go index b13aa5be..588a30e7 100644 --- a/vendor/github.com/redis/go-redis/v9/search_commands.go +++ b/vendor/github.com/redis/go-redis/v9/search_commands.go @@ -18,6 +18,7 @@ type SearchCmdable interface { FTAggregateWithArgs(ctx context.Context, index string, query string, options *FTAggregateOptions) *AggregateCmd FTAliasAdd(ctx context.Context, index string, alias string) *StatusCmd FTAliasDel(ctx context.Context, alias string) *StatusCmd + FTAliasList(ctx context.Context, index string) *StringSliceCmd FTAliasUpdate(ctx context.Context, index string, alias string) *StatusCmd FTAlter(ctx context.Context, index string, skipInitialScan bool, definition []interface{}) *StatusCmd FTConfigGet(ctx context.Context, option string) *MapMapStringInterfaceCmd @@ -107,6 +108,13 @@ type FTHNSWOptions struct { MaxAllowedEdgesPerNode int EFRunTime int Epsilon float64 + // Rerank toggles the exact re-scoring pass over approximate candidates on + // disk-backed HNSW indexes (Redis 8.10+), where the server requires it to + // be set explicitly. Rerank=true emits RERANK TRUE on its own; to emit + // RERANK FALSE, set HasRerank=true with Rerank=false, so that an explicit + // false can be distinguished from unset (omitted). + Rerank bool + HasRerank bool } type FTVamanaOptions struct { @@ -157,6 +165,12 @@ const ( SearchToList SearchFirstValue SearchRandomSample + // SearchCollect is the COLLECT reducer for FT.AGGREGATE. Within each + // GROUPBY group it projects a chosen set of fields from every row and + // emits them as an array of per-entry maps under the reducer alias. + // Requires Redis 8.8+ with unstable features enabled + // (CONFIG SET search-enable-unstable-features yes). + SearchCollect ) func (a SearchAggregator) String() string { @@ -187,6 +201,8 @@ func (a SearchAggregator) String() string { return "FIRST_VALUE" case SearchRandomSample: return "RANDOM_SAMPLE" + case SearchCollect: + return "COLLECT" default: return "" } @@ -418,8 +434,15 @@ type FTHybridVectorExpression struct { VectorParamName string Method FTHybridVectorMethod MethodParams []interface{} - Filter string - YieldScoreAs string + // ShardKRatio controls how many results each shard returns relative to the + // requested KNN K, trading recall for latency in Redis cluster setups. + // Valid range: 0.1 - 1.0. The zero value means "unset" and falls back to + // the server default of 1.0 (no per-shard reduction). Has no effect on + // standalone Redis, and only applies to the KNN method. Requires Redis 8.8+. + // See https://redis.io/docs/latest/develop/ai/search-and-query/query/vector-search/ + ShardKRatio float64 + Filter string + YieldScoreAs string } // FTHybridCombineOptions represents options for result fusion @@ -487,8 +510,10 @@ type FTSynDumpCmd struct { // FTAggregateResult represents the result of an aggregate operation // NOTE: For RESP3 Total is not reliable (before Redis 8.8) type FTAggregateResult struct { - Total int - Rows []AggregateRow + Total int + Rows []AggregateRow + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict). RESP3 only; the fail policy returns an error instead. Warnings []string } @@ -619,8 +644,10 @@ type SpellCheckSuggestion struct { } type FTSearchResult struct { - Total int - Docs []Document + Total int + Docs []Document + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict). RESP3 only; the fail policy returns an error instead. Warnings []string } @@ -936,22 +963,27 @@ func (cmd *AggregateCmd) SetVal(val *FTAggregateResult) { } func (cmd *AggregateCmd) Val() *FTAggregateResult { + cmd.await() return cmd.val } func (cmd *AggregateCmd) Result() (*FTAggregateResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *AggregateCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *AggregateCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } func (cmd *AggregateCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1268,6 +1300,20 @@ func (c cmdable) FTAliasDel(ctx context.Context, alias string) *StatusCmd { return cmd } +// FTAliasList - Lists all aliases associated with an index. +// The 'index' parameter specifies the index whose aliases are listed; it must +// be the name of an index created with FT.CREATE, not an alias. +// The reply is an unordered collection of alias names, already deduplicated +// by the server; an index with no aliases yields an empty result, not an +// error. Available since Redis 8.10. +// For more information, please refer to the Redis documentation: +// [FT.ALIASLIST]: (https://redis.io/commands/ft.aliaslist/) +func (c cmdable) FTAliasList(ctx context.Context, index string) *StringSliceCmd { + cmd := NewStringSliceCmd(ctx, "FT.ALIASLIST", index) + _ = c(ctx, cmd) + return cmd +} + // FTAliasUpdate - Updates an alias to an index. // The 'index' parameter specifies the index to which the alias is updated, and the 'alias' parameter specifies the alias. // If the alias already exists for a different index, it updates the alias to point to the specified index instead. @@ -1484,6 +1530,13 @@ func (c cmdable) FTCreate(ctx context.Context, index string, options *FTCreateOp if schema.VectorArgs.HNSWOptions.Epsilon > 0 { hnswArgs = append(hnswArgs, "EPSILON", schema.VectorArgs.HNSWOptions.Epsilon) } + if schema.VectorArgs.HNSWOptions.Rerank || schema.VectorArgs.HNSWOptions.HasRerank { + rerank := "FALSE" + if schema.VectorArgs.HNSWOptions.Rerank { + rerank = "TRUE" + } + hnswArgs = append(hnswArgs, "RERANK", rerank) + } args = append(args, len(hnswArgs)) args = append(args, hnswArgs...) } @@ -1564,7 +1617,6 @@ func (c cmdable) FTCreate(ctx context.Context, index string, options *FTCreateOp } if schema.IndexMissing { args = append(args, "INDEXMISSING") - } } cmd := NewStatusCmd(ctx, args...) @@ -2127,6 +2179,7 @@ func newFTInfoCmd(ctx context.Context, args ...interface{}) *FTInfoCmd { } func (cmd *FTInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2135,20 +2188,25 @@ func (cmd *FTInfoCmd) SetVal(val FTInfoResult) { } func (cmd *FTInfoCmd) Result() (FTInfoResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTInfoCmd) Val() FTInfoResult { + cmd.await() return cmd.val } func (cmd *FTInfoCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTInfoCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } + func (cmd *FTInfoCmd) readReply(rd *proto.Reader) (err error) { readType, err := rd.PeekReplyType() if err != nil { @@ -2329,6 +2387,7 @@ func newFTSpellCheckCmd(ctx context.Context, args ...interface{}) *FTSpellCheckC } func (cmd *FTSpellCheckCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2337,18 +2396,22 @@ func (cmd *FTSpellCheckCmd) SetVal(val []SpellCheckResult) { } func (cmd *FTSpellCheckCmd) Result() ([]SpellCheckResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSpellCheckCmd) Val() []SpellCheckResult { + cmd.await() return cmd.val } func (cmd *FTSpellCheckCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSpellCheckCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -2645,6 +2708,7 @@ func newFTSearchCmd(ctx context.Context, options *FTSearchOptions, args ...inter } func (cmd *FTSearchCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2653,18 +2717,22 @@ func (cmd *FTSearchCmd) SetVal(val FTSearchResult) { } func (cmd *FTSearchCmd) Result() (FTSearchResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSearchCmd) Val() FTSearchResult { + cmd.await() return cmd.val } func (cmd *FTSearchCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSearchCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -2890,8 +2958,10 @@ func (cmd *FTSearchCmd) Clone() Cmder { // FTHybridResult represents the result of a hybrid search operation type FTHybridResult struct { - TotalResults int - Results []map[string]interface{} + TotalResults int + Results []map[string]interface{} + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict), on RESP2 and RESP3; the fail policy returns an error. Warnings []string ExecutionTime float64 } @@ -2926,6 +2996,7 @@ func newFTHybridCmd(ctx context.Context, options *FTHybridOptions, args ...inter } func (cmd *FTHybridCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2934,26 +3005,32 @@ func (cmd *FTHybridCmd) SetVal(val FTHybridResult) { } func (cmd *FTHybridCmd) Result() (FTHybridResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTHybridCmd) CursorResult() (*FTHybridCursorResult, error) { + cmd.await() return cmd.cursorVal, cmd.err } func (cmd *FTHybridCmd) Val() FTHybridResult { + cmd.await() return cmd.val } func (cmd *FTHybridCmd) CursorVal() *FTHybridCursorResult { + cmd.await() return cmd.cursorVal } func (cmd *FTHybridCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTHybridCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -3034,9 +3111,13 @@ func parseFTHybrid(data []interface{}, withCursor bool) (FTHybridResult, *FTHybr results = append(results, itemMap) } - // Parse warnings (optional field) + // Optional warnings; accept both "warning" (as FT.SEARCH/FT.AGGREGATE) and "warnings". var warnings []string - if warningsData, ok := resultMap["warnings"].([]interface{}); ok { + warningsData, ok := resultMap["warning"].([]interface{}) + if !ok { + warningsData, ok = resultMap["warnings"].([]interface{}) + } + if ok { warnings = make([]string, 0, len(warningsData)) for _, w := range warningsData { if ws, ok := w.(string); ok { @@ -3460,6 +3541,7 @@ func NewFTSynDumpCmd(ctx context.Context, args ...interface{}) *FTSynDumpCmd { } func (cmd *FTSynDumpCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3468,18 +3550,22 @@ func (cmd *FTSynDumpCmd) SetVal(val []FTSynDumpResult) { } func (cmd *FTSynDumpCmd) Val() []FTSynDumpResult { + cmd.await() return cmd.val } func (cmd *FTSynDumpCmd) Result() ([]FTSynDumpResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSynDumpCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSynDumpCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -3795,6 +3881,22 @@ func (c cmdable) FTHybridWithArgs(ctx context.Context, index string, options *FT } } + // SHARD_K_RATIO applies to the KNN method only (Redis 8.8+, cluster only). + // Zero means "unset" and falls back to the server default of 1.0. + if vectorExpr.ShardKRatio > 0 { + if vectorExpr.Method != "KNN" { + cmd := newFTHybridCmd(ctx, options, args...) + cmd.SetErr(fmt.Errorf("FT.HYBRID: SHARD_K_RATIO requires KNN method")) + return cmd + } + if vectorExpr.ShardKRatio < 0.1 || vectorExpr.ShardKRatio > 1.0 { + cmd := newFTHybridCmd(ctx, options, args...) + cmd.SetErr(fmt.Errorf("FT.HYBRID: SHARD_K_RATIO must be between 0.1 and 1.0")) + return cmd + } + args = append(args, "SHARD_K_RATIO", vectorExpr.ShardKRatio) + } + if vectorExpr.Filter != "" { args = append(args, "FILTER", vectorExpr.Filter) } diff --git a/vendor/github.com/redis/go-redis/v9/sentinel.go b/vendor/github.com/redis/go-redis/v9/sentinel.go index 055b3101..5720fc47 100644 --- a/vendor/github.com/redis/go-redis/v9/sentinel.go +++ b/vendor/github.com/redis/go-redis/v9/sentinel.go @@ -12,11 +12,14 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/redis/go-redis/v9/auth" "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/otel" "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/maintnotifications" "github.com/redis/go-redis/v9/push" ) @@ -123,6 +126,18 @@ type FailoverOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining, with + // its own (typically larger) buffers. See the same-named fields on Options + // for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + + // AutoPipelineOptions is the default config for the client's autopipeliner + // faces. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + PoolFIFO bool PoolSize int @@ -176,7 +191,7 @@ type FailoverOptions struct { // seamlessly. Requires Protocol: 3 (RESP3) for push notifications. // If nil, maintnotifications upgrades are disabled. // (however if Mode is nil, it defaults to "auto" - enable if server supports it) - //MaintNotificationsConfig *maintnotifications.Config + // MaintNotificationsConfig *maintnotifications.Config } func (opt *FailoverOptions) clientOptions() *Options { @@ -202,6 +217,11 @@ func (opt *FailoverOptions) clientOptions() *Options { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + AutoPipelineOptions: opt.AutoPipelineOptions, + DialTimeout: opt.DialTimeout, DialerRetries: opt.DialerRetries, DialerRetryTimeout: opt.DialerRetryTimeout, @@ -318,6 +338,11 @@ func (opt *FailoverOptions) clusterOptions() *ClusterOptions { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + AutoPipelineOptions: opt.AutoPipelineOptions, + DialTimeout: opt.DialTimeout, DialerRetries: opt.DialerRetries, DialerRetryTimeout: opt.DialerRetryTimeout, @@ -536,8 +561,10 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { rdb := &Client{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, + himport: newHImportRegistry(), }, } rdb.init() @@ -561,6 +588,40 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { panic(fmt.Errorf("redis: failed to create pubsub pool: %w", err)) } + // Optionally create a separate connection pool for pipelining, with its own + // (typically larger) buffers. Enabled when either pipeline buffer size is set. + if opt.PipelineReadBufferSize > 0 || opt.PipelineWriteBufferSize > 0 { + pipelineOpt := opt.clone() + if opt.PipelineReadBufferSize > 0 { + pipelineOpt.ReadBufferSize = opt.PipelineReadBufferSize + // Same clamp Options.init applies to the main pool: RESP3 push + // parsing needs a minimum read buffer, and a tiny pipeline reader + // would break push-notification handling on pipeline conns. + if pipelineOpt.Protocol == 3 && pipelineOpt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + pipelineOpt.ReadBufferSize = proto.MinRESP3ReadBufferSize + } + } + if opt.PipelineWriteBufferSize > 0 { + pipelineOpt.WriteBufferSize = opt.PipelineWriteBufferSize + } + if opt.PipelinePoolSize > 0 { + pipelineOpt.PoolSize = opt.PipelinePoolSize + } else { + pipelineOpt.PoolSize = 10 // default smaller pool for pipelining + } + rdb.pipelinePoolName = mainPoolName + "_pipeline" + rdb.pipelinePool, err = newConnPool(pipelineOpt, rdb.dialHook, rdb.pipelinePoolName) + if err != nil { + panic(fmt.Errorf("redis: failed to create pipeline connection pool: %w", err)) + } + } + + // Register pools for OTel async gauge metrics, matching NewClient (the + // failover client previously registered none, so pool gauges were silent + // for the identical standalone setup). The pipeline pool is nil when not + // configured. + otel.RegisterPools(rdb.connPool, rdb.pubSubPool, rdb.pipelinePool, opt.Addr) + rdb.onClose.register(onCloseHookIDSentinelFailover, failover.Close) failover.mu.Lock() @@ -570,6 +631,13 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { return cn.RemoteAddr().String() != addr }) } + // Drop stale pipeline-pool connections dialed to the demoted master too; + // otherwise pipelined traffic keeps using the old address after failover. + if pipelinePool, ok := rdb.pipelinePool.(*pool.ConnPool); ok { + _ = pipelinePool.Filter(func(cn *pool.Conn) bool { + return cn.RemoteAddr().String() != addr + }) + } } failover.mu.Unlock() @@ -599,8 +667,8 @@ func masterReplicaDialer( } netDialer := &net.Dialer{ - Timeout: failover.opt.DialTimeout, - KeepAlive: 5 * time.Minute, + Timeout: failover.opt.DialTimeout, + KeepAliveConfig: defaultKeepAliveConfig, } if failover.opt.TLSConfig == nil { return netDialer.DialContext(ctx, network, addr) @@ -625,8 +693,9 @@ func NewSentinelClient(opt *Options) *SentinelClient { opt.init() c := &SentinelClient{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, }, } @@ -678,7 +747,7 @@ func (c *SentinelClient) Process(ctx context.Context, cmd Cmder) error { func (c *SentinelClient) pubSub() *PubSub { pubsub := &PubSub{ - opt: c.opt, + opt: c.cloneOpt(), newConn: func(ctx context.Context, addr string, channels []string) (*pool.Conn, error) { cn, err := c.pubSubPool.NewConn(ctx, c.opt.Network, addr, channels) if err != nil { diff --git a/vendor/github.com/redis/go-redis/v9/set_commands.go b/vendor/github.com/redis/go-redis/v9/set_commands.go index 2a465728..94074555 100644 --- a/vendor/github.com/redis/go-redis/v9/set_commands.go +++ b/vendor/github.com/redis/go-redis/v9/set_commands.go @@ -12,6 +12,7 @@ type SetCmdable interface { SAdd(ctx context.Context, key string, members ...interface{}) *IntCmd SCard(ctx context.Context, key string) *IntCmd SDiff(ctx context.Context, keys ...string) *StringSliceCmd + SDiffCard(ctx context.Context, opts *SDiffCardOptions, keys ...string) *IntCmd SDiffStore(ctx context.Context, destination string, keys ...string) *IntCmd SInter(ctx context.Context, keys ...string) *StringSliceCmd SInterCard(ctx context.Context, limit int64, keys ...string) *IntCmd @@ -28,9 +29,21 @@ type SetCmdable interface { SRem(ctx context.Context, key string, members ...interface{}) *IntCmd SScan(ctx context.Context, key string, cursor uint64, match string, count int64) *ScanCmd SUnion(ctx context.Context, keys ...string) *StringSliceCmd + SUnionCard(ctx context.Context, opts *SUnionCardOptions, keys ...string) *IntCmd SUnionStore(ctx context.Context, destination string, keys ...string) *IntCmd } +// SUnionCardOptions are the options for SUnionCard. +type SUnionCardOptions struct { + Approx bool // use an approximate (HyperLogLog) count. + Limit int64 // cap the result; 0 means no limit. +} + +// SDiffCardOptions are the options for SDiffCard. +type SDiffCardOptions struct { + Limit int64 // cap the result; 0 means no limit. +} + // Returns the number of elements that were added to the set, not including all // the elements already present in the set. // @@ -96,6 +109,30 @@ func (c cmdable) SDiffStore(ctx context.Context, destination string, keys ...str return cmd } +// Returns the cardinality of the difference of the first set and the rest. +// Missing keys are treated as empty sets. +// +// For more information about the command please refer to [SDIFFCARD]. +// +// [SDIFFCARD]: (https://redis.io/docs/latest/commands/sdiffcard/) +func (c cmdable) SDiffCard(ctx context.Context, opts *SDiffCardOptions, keys ...string) *IntCmd { + if opts == nil { + opts = &SDiffCardOptions{} + } + numKeys := len(keys) + args := make([]interface{}, 0, 4+numKeys) + args = append(args, "sdiffcard", numKeys) + for _, key := range keys { + args = append(args, key) + } + args = append(args, "limit", opts.Limit) + cmd := NewIntCmd(ctx, args...) + // Keys start after the numkeys arg: ["sdiffcard", numKeys, key1, ...]. + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + // Returns the members of the set resulting from the intersection of all the given sets. // Keys that do not exist are considered to be empty sets. // With one of the keys being an empty set, the resulting set is also empty. @@ -328,6 +365,33 @@ func (c cmdable) SUnionStore(ctx context.Context, destination string, keys ...st return cmd } +// Returns the cardinality of the union of all the given sets. +// Missing keys are treated as empty sets. +// +// For more information about the command please refer to [SUNIONCARD]. +// +// [SUNIONCARD]: (https://redis.io/docs/latest/commands/sunioncard/) +func (c cmdable) SUnionCard(ctx context.Context, opts *SUnionCardOptions, keys ...string) *IntCmd { + if opts == nil { + opts = &SUnionCardOptions{} + } + numKeys := len(keys) + args := make([]interface{}, 0, 4+numKeys+1) + args = append(args, "sunioncard", numKeys) + for _, key := range keys { + args = append(args, key) + } + if opts.Approx { + args = append(args, "approx") + } + args = append(args, "limit", opts.Limit) + cmd := NewIntCmd(ctx, args...) + // Keys start after the numkeys arg: ["sunioncard", numKeys, key1, ...]. + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + // Incrementally iterates the set elements stored at key. // This is a cursor-based iterator that allows scanning large sets efficiently. // diff --git a/vendor/github.com/redis/go-redis/v9/stream_commands.go b/vendor/github.com/redis/go-redis/v9/stream_commands.go index e2b2a9e2..95531eb1 100644 --- a/vendor/github.com/redis/go-redis/v9/stream_commands.go +++ b/vendor/github.com/redis/go-redis/v9/stream_commands.go @@ -227,14 +227,16 @@ func (c cmdable) XRevRangeN(ctx context.Context, stream, start, stop string, cou } type XReadArgs struct { - Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 - Count int64 - Block time.Duration - ID string + Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 + Count int64 + MaxCount int64 // cumulative cap on total entries across all streams (Redis >= 8.10) + MaxSize int64 // soft cumulative cap on total reply size in bytes across all streams (Redis >= 8.10) + Block time.Duration + ID string } func (c cmdable) XRead(ctx context.Context, a *XReadArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 2*len(a.Streams)+6) + args := make([]interface{}, 0, 2*len(a.Streams)+10) args = append(args, "xread") keyPos := int8(1) @@ -243,6 +245,14 @@ func (c cmdable) XRead(ctx context.Context, a *XReadArgs) *XStreamSliceCmd { args = append(args, a.Count) keyPos += 2 } + if a.MaxCount > 0 { + args = append(args, "maxcount", a.MaxCount) + keyPos += 2 + } + if a.MaxSize > 0 { + args = append(args, "maxsize", a.MaxSize) + keyPos += 2 + } if a.Block >= 0 { args = append(args, "block") args = append(args, int64(a.Block/time.Millisecond)) @@ -322,13 +332,15 @@ type XReadGroupArgs struct { Consumer string Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 Count int64 + MaxCount int64 // cumulative cap on total entries across all streams (Redis >= 8.10) + MaxSize int64 // soft cumulative cap on total reply size in bytes across all streams (Redis >= 8.10) Block time.Duration NoAck bool Claim time.Duration // Claim idle pending entries older than this duration } func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 10+len(a.Streams)) + args := make([]interface{}, 0, 14+len(a.Streams)) args = append(args, "xreadgroup", "group", a.Group, a.Consumer) keyPos := int8(4) @@ -336,6 +348,14 @@ func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSlic args = append(args, "count", a.Count) keyPos += 2 } + if a.MaxCount > 0 { + args = append(args, "maxcount", a.MaxCount) + keyPos += 2 + } + if a.MaxSize > 0 { + args = append(args, "maxsize", a.MaxSize) + keyPos += 2 + } if a.Block >= 0 { args = append(args, "block", int64(a.Block/time.Millisecond)) keyPos += 2 @@ -361,8 +381,15 @@ func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSlic cmd.SetFirstKeyPos(keyPos) _ = c(ctx, cmd) - // Record stream lag for each message (if command succeeded) - if cmd.Err() == nil { + // Record stream lag for each message (if command succeeded). Gated on the + // result being readable WITHOUT blocking: this command carries a + // read-timeout marker, so on the deferred autopipeline face it is diverted + // and still running when we get here — and the default Block: 0 form can + // wait indefinitely for messages, so reading the outcome would block the + // submit call instead of returning a future (review finding by codex on + // #3942). Skipped for a submission that has not executed yet; emitting + // this from the execution path is a follow-up in the OTel wiring. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { streams := cmd.Val() for _, stream := range streams { for _, msg := range stream.Messages { diff --git a/vendor/github.com/redis/go-redis/v9/string_commands.go b/vendor/github.com/redis/go-redis/v9/string_commands.go index 6731c09f..3c50715b 100644 --- a/vendor/github.com/redis/go-redis/v9/string_commands.go +++ b/vendor/github.com/redis/go-redis/v9/string_commands.go @@ -194,6 +194,12 @@ func (c cmdable) GetDel(ctx context.Context, key string) *StringCmd { // (including redis.Nil when the key does not exist) via Err(). If buf is too // small to hold the value, Err() returns a "buffer too small" error. // +// Nothing is ever written past len(buf). When len(buf) >= value length + 2, +// the read takes a fast path that pulls the payload and the protocol's +// trailing CRLF in a single socket read, using the two bytes after the +// payload as scratch — size buffers with 2 spare bytes to opt in (see +// example/zerocopy-buffer). +// // This command opts out of automatic retries because partial data from a // failed attempt would already be sitting in the caller's buffer. func (c cmdable) GetToBuffer(ctx context.Context, key string, buf []byte) *ZeroCopyStringCmd { diff --git a/vendor/github.com/redis/go-redis/v9/timeseries_commands.go b/vendor/github.com/redis/go-redis/v9/timeseries_commands.go index db00db80..673e0e57 100644 --- a/vendor/github.com/redis/go-redis/v9/timeseries_commands.go +++ b/vendor/github.com/redis/go-redis/v9/timeseries_commands.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/internal/util" @@ -30,6 +31,8 @@ type TimeseriesCmdable interface { TSInfoWithArgs(ctx context.Context, key string, options *TSInfoOptions) *MapStringInterfaceCmd TSMAdd(ctx context.Context, ktvSlices [][]interface{}) *IntSliceCmd TSQueryIndex(ctx context.Context, filterExpr []string) *StringSliceCmd + TSQueryLabels(ctx context.Context, filterExpr []string) *StringSliceCmd + TSQueryLabelValues(ctx context.Context, label string, filterExpr []string) *StringSliceCmd TSRevRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd TSRevRangeWithArgs(ctx context.Context, key string, fromTimestamp int, toTimestamp int, options *TSRevRangeOptions) *TSTimestampValueSliceCmd TSRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd @@ -40,6 +43,27 @@ type TimeseriesCmdable interface { TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRevRangeOptions) *MapStringSliceInterfaceCmd TSMGet(ctx context.Context, filters []string) *MapStringSliceInterfaceCmd TSMGetWithArgs(ctx context.Context, filters []string, options *TSMGetOptions) *MapStringSliceInterfaceCmd + TSNRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd + TSNRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRangeOptions) *TSNRangePivotRowSliceCmd + TSNRevRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd + TSNRevRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRevRangeOptions) *TSNRangePivotRowSliceCmd + TSRead(ctx context.Context, key string, timestamp interface{}) *TSTimestampValueSliceCmd + TSReadWithArgs(ctx context.Context, key string, timestamp interface{}, options *TSReadOptions) *TSTimestampValueSliceCmd +} + +// TS.READ timestamp cursor sentinels. +const ( + TSReadEarliest = "-" // read from the earliest sample + TSReadLatest = "+" // latest sample, inclusive + TSReadNew = "$" // only samples added after the call +) + +// TSReadOptions holds the optional TS.READ arguments. +type TSReadOptions struct { + Block bool // wait for samples (emits the BLOCK group) + Timeout time.Duration // max wait; 0 blocks indefinitely + MinCount int // unblock threshold; defaults to 1 + MaxCount int // reply cap; 0 is unlimited } type TSOptions struct { @@ -145,6 +169,7 @@ func (a Aggregator) String() string { var ( errTSMultiAggregationGroupBy = errors.New("redis: GROUPBY is not allowed when multiple aggregators are specified") errTSAggregationConflict = errors.New("redis: setting both Aggregator and Aggregators is not allowed; use Aggregators instead because Aggregator is deprecated") + errTSExcludeEmptyGroupBy = errors.New("redis: EXCLUDEEMPTY is not allowed with GROUPBY") ) func formatAggregationArgs(aggregator Aggregator, aggregators []Aggregator) (string, int, error) { @@ -227,8 +252,10 @@ type TSMRangeOptions struct { BucketDuration int BucketTimestamp interface{} Empty bool - GroupByLabel interface{} - Reducer interface{} + // ExcludeEmpty omits matching series that have no samples. Not allowed with GroupByLabel/Reducer. Redis 8.10+. + ExcludeEmpty bool + GroupByLabel interface{} + Reducer interface{} } type TSMRevRangeOptions struct { @@ -245,8 +272,10 @@ type TSMRevRangeOptions struct { BucketDuration int BucketTimestamp interface{} Empty bool - GroupByLabel interface{} - Reducer interface{} + // ExcludeEmpty omits matching series that have no samples. Not allowed with GroupByLabel/Reducer. Redis 8.10+. + ExcludeEmpty bool + GroupByLabel interface{} + Reducer interface{} } type TSMGetOptions struct { @@ -255,6 +284,36 @@ type TSMGetOptions struct { SelectedLabels []interface{} } +type TSNRangeOptions struct { + Latest bool + FilterByTS []int + FilterByValue []float64 // exactly two elements: [min, max] + Count int + Align interface{} + // Aggregators holds exactly one aggregator spec per key. Each spec lists one or + // more aggregators applied to that key and is sent as a single comma-joined token + // (e.g. {{Min, Max}, {Sum}} -> AGGREGATION MIN,MAX SUM ). + Aggregators [][]Aggregator + BucketDuration int + BucketTimestamp interface{} + Empty bool +} + +type TSNRevRangeOptions struct { + Latest bool + FilterByTS []int + FilterByValue []float64 // exactly two elements: [min, max] + Count int + Align interface{} + // Aggregators holds exactly one aggregator spec per key. Each spec lists one or + // more aggregators applied to that key and is sent as a single comma-joined token + // (e.g. {{Min, Max}, {Sum}} -> AGGREGATION MIN,MAX SUM ). + Aggregators [][]Aggregator + BucketDuration int + BucketTimestamp interface{} + Empty bool +} + // TSAdd - Adds one or more observations to a t-digest sketch. // For more information - https://redis.io/commands/ts.add/ func (c cmdable) TSAdd(ctx context.Context, key string, timestamp interface{}, value float64) *IntCmd { @@ -563,6 +622,7 @@ func newTSTimestampValueCmd(ctx context.Context, args ...interface{}) *TSTimesta } func (cmd *TSTimestampValueCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -571,10 +631,12 @@ func (cmd *TSTimestampValueCmd) SetVal(val TSTimestampValue) { } func (cmd *TSTimestampValueCmd) Result() (TSTimestampValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TSTimestampValueCmd) Val() TSTimestampValue { + cmd.await() return cmd.val } @@ -666,6 +728,53 @@ func (c cmdable) TSQueryIndex(ctx context.Context, filterExpr []string) *StringS return cmd } +// TSQueryLabels - Returns the set of label names present on the time series +// matching the filter expressions. Passing no filter expressions queries all +// indexed series. The reply is unordered and already deduplicated by the +// server; it includes the label names used in the filter itself, and an +// empty reply is a valid result, not an error. +// filterExpr uses the same filter language as TSQueryIndex and is passed to +// the server verbatim. Available since Redis 8.10. +// For more information - https://redis.io/commands/ts.querylabels/ +func (c cmdable) TSQueryLabels(ctx context.Context, filterExpr []string) *StringSliceCmd { + args := []interface{}{"TS.QUERYLABELS", "LABELS"} + args = appendTSFilter(args, filterExpr) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// TSQueryLabelValues - Returns the set of values assigned to the given label +// name across the time series matching the filter expressions. Passing no +// filter expressions queries all indexed series. The label name is matched +// byte-exactly; a label present on no matching series yields an empty reply, +// not an error. The reply is unordered and already deduplicated by the +// server. +// filterExpr uses the same filter language as TSQueryIndex and is passed to +// the server verbatim. Available since Redis 8.10. +// For more information - https://redis.io/commands/ts.querylabels/ +func (c cmdable) TSQueryLabelValues(ctx context.Context, label string, filterExpr []string) *StringSliceCmd { + args := []interface{}{"TS.QUERYLABELS", "VALUES", label} + args = appendTSFilter(args, filterExpr) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// appendTSFilter appends the FILTER token followed by the filter expressions, +// or nothing when no expressions are given: the server rejects a bare FILTER +// token, and omitting it is the documented way to query all indexed series. +func appendTSFilter(args []interface{}, filterExpr []string) []interface{} { + if len(filterExpr) == 0 { + return args + } + args = append(args, "FILTER") + for _, f := range filterExpr { + args = append(args, f) + } + return args +} + // TSRevRange - Returns a range of samples from a time-series key in reverse order. // For more information - https://redis.io/commands/ts.revrange/ func (c cmdable) TSRevRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd { @@ -790,6 +899,47 @@ func (c cmdable) TSRangeWithArgs(ctx context.Context, key string, fromTimestamp return cmd } +// TSRead - Returns samples at or after timestamp, in ascending order. +// timestamp is a non-negative Unix-ms integer or a sentinel (TSReadEarliest, +// TSReadLatest, TSReadNew). +// For more information - https://redis.io/commands/ts.read/ +func (c cmdable) TSRead(ctx context.Context, key string, timestamp interface{}) *TSTimestampValueSliceCmd { + args := []interface{}{"TS.READ", key, timestamp} + cmd := newTSTimestampValueSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// TSReadWithArgs - TS.READ with the optional BLOCK and MAX_COUNT groups. +// When options.Block is set it waits for options.MinCount samples or until +// options.Timeout elapses. Blocking calls must not be used in a pipeline or MULTI. +// For more information - https://redis.io/commands/ts.read/ +func (c cmdable) TSReadWithArgs(ctx context.Context, key string, timestamp interface{}, options *TSReadOptions) *TSTimestampValueSliceCmd { + args := []interface{}{"TS.READ", key, timestamp} + blocking := false + var blockTimeout time.Duration + if options != nil { + if options.Block { + blocking = true + blockTimeout = options.Timeout + minCount := options.MinCount + if minCount <= 0 { + minCount = 1 + } + args = append(args, "BLOCK", formatMs(ctx, options.Timeout), minCount) + } + if options.MaxCount != 0 { + args = append(args, "MAX_COUNT", options.MaxCount) + } + } + cmd := newTSTimestampValueSliceCmd(ctx, args...) + if blocking { + cmd.setReadTimeout(blockTimeout) + } + _ = c(ctx, cmd) + return cmd +} + type TSTimestampValueSliceCmd struct { baseCmd val []TSTimestampValue @@ -806,6 +956,7 @@ func newTSTimestampValueSliceCmd(ctx context.Context, args ...interface{}) *TSTi } func (cmd *TSTimestampValueSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -814,10 +965,12 @@ func (cmd *TSTimestampValueSliceCmd) SetVal(val []TSTimestampValue) { } func (cmd *TSTimestampValueSliceCmd) Result() ([]TSTimestampValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TSTimestampValueSliceCmd) Val() []TSTimestampValue { + cmd.await() return cmd.val } @@ -896,11 +1049,8 @@ func (c cmdable) TSMRange(ctx context.Context, fromTimestamp int, toTimestamp in return cmd } -// TSMRangeWithArgs - Returns a range of samples from multiple time-series keys with additional options. -// This function allows for specifying additional options such as: -// Latest, FilterByTS, FilterByValue, WithLabels, SelectedLabels, -// Count, Align, Aggregator, BucketDuration, BucketTimestamp, -// Empty, GroupByLabel and Reducer. +// TSMRangeWithArgs - Returns a range of samples from multiple time-series keys. +// Options are set via TSMRangeOptions. // For more information - https://redis.io/commands/ts.mrange/ func (c cmdable) TSMRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRangeOptions) *MapStringSliceInterfaceCmd { args := []interface{}{"TS.MRANGE", fromTimestamp, toTimestamp} @@ -953,12 +1103,20 @@ func (c cmdable) TSMRangeWithArgs(ctx context.Context, fromTimestamp int, toTime if options.Empty { args = append(args, "EMPTY") } + if options.ExcludeEmpty { + args = append(args, "EXCLUDEEMPTY") + } } args = append(args, "FILTER") for _, f := range filterExpr { args = append(args, f) } if options != nil { + if options.ExcludeEmpty && (options.GroupByLabel != nil || options.Reducer != nil) { + cmd := NewMapStringSliceInterfaceCmd(ctx, args...) + cmd.SetErr(errTSExcludeEmptyGroupBy) + return cmd + } if multiAggregationCount > 1 && (options.GroupByLabel != nil || options.Reducer != nil) { cmd := NewMapStringSliceInterfaceCmd(ctx, args...) cmd.SetErr(errTSMultiAggregationGroupBy) @@ -988,11 +1146,8 @@ func (c cmdable) TSMRevRange(ctx context.Context, fromTimestamp int, toTimestamp return cmd } -// TSMRevRangeWithArgs - Returns a range of samples from multiple time-series keys in reverse order with additional options. -// This function allows for specifying additional options such as: -// Latest, FilterByTS, FilterByValue, WithLabels, SelectedLabels, -// Count, Align, Aggregator, BucketDuration, BucketTimestamp, -// Empty, GroupByLabel and Reducer. +// TSMRevRangeWithArgs - Returns a range of samples from multiple time-series keys in reverse order. +// Options are set via TSMRevRangeOptions. // For more information - https://redis.io/commands/ts.mrevrange/ func (c cmdable) TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRevRangeOptions) *MapStringSliceInterfaceCmd { args := []interface{}{"TS.MREVRANGE", fromTimestamp, toTimestamp} @@ -1045,12 +1200,20 @@ func (c cmdable) TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toT if options.Empty { args = append(args, "EMPTY") } + if options.ExcludeEmpty { + args = append(args, "EXCLUDEEMPTY") + } } args = append(args, "FILTER") for _, f := range filterExpr { args = append(args, f) } if options != nil { + if options.ExcludeEmpty && (options.GroupByLabel != nil || options.Reducer != nil) { + cmd := NewMapStringSliceInterfaceCmd(ctx, args...) + cmd.SetErr(errTSExcludeEmptyGroupBy) + return cmd + } if multiAggregationCount > 1 && (options.GroupByLabel != nil || options.Reducer != nil) { cmd := NewMapStringSliceInterfaceCmd(ctx, args...) cmd.SetErr(errTSMultiAggregationGroupBy) @@ -1106,3 +1269,279 @@ func (c cmdable) TSMGetWithArgs(ctx context.Context, filters []string, options * _ = c(ctx, cmd) return cmd } + +// TSNRangePivotRow represents a single row in the pivot response from TS.NRANGE / TS.NREVRANGE. +// Timestamp is the row's timestamp. Without aggregation, Values holds one float64 per input key +// in input-key order. With aggregation, Values holds one float64 per requested (key, aggregator) +// pair, flattened in input-key order with each key's aggregators in spec order. +// Missing samples and missing aggregation buckets are represented as NaN. +type TSNRangePivotRow struct { + Timestamp int64 + Values []float64 +} + +type TSNRangePivotRowSliceCmd struct { + baseCmd + val []TSNRangePivotRow +} + +func newTSNRangePivotRowSliceCmd(ctx context.Context, args ...interface{}) *TSNRangePivotRowSliceCmd { + return &TSNRangePivotRowSliceCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeTSNRangePivotRowSlice, + }, + } +} + +func (cmd *TSNRangePivotRowSliceCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} + +func (cmd *TSNRangePivotRowSliceCmd) SetVal(val []TSNRangePivotRow) { + cmd.val = val +} + +func (cmd *TSNRangePivotRowSliceCmd) Result() ([]TSNRangePivotRow, error) { + cmd.await() + return cmd.val, cmd.err +} + +func (cmd *TSNRangePivotRowSliceCmd) Val() []TSNRangePivotRow { + cmd.await() + return cmd.val +} + +func (cmd *TSNRangePivotRowSliceCmd) readReply(rd *proto.Reader) error { + n, err := rd.ReadArrayLen() + if err != nil { + return err + } + cmd.val = make([]TSNRangePivotRow, n) + for i := 0; i < n; i++ { + // Each row is a 2-element array: [timestamp, [value_0, value_1, ...]] + if _, err = rd.ReadArrayLen(); err != nil { + return err + } + timestamp, err := rd.ReadInt() + if err != nil { + return err + } + cmd.val[i].Timestamp = timestamp + + valCount, err := rd.ReadArrayLen() + if err != nil { + return err + } + cmd.val[i].Values = make([]float64, valCount) + for j := 0; j < valCount; j++ { + s, err := rd.ReadString() + if err != nil { + return err + } + cmd.val[i].Values[j], err = util.ParseStringToFloat(s) + if err != nil { + return err + } + } + } + return nil +} + +func (cmd *TSNRangePivotRowSliceCmd) Clone() Cmder { + var val []TSNRangePivotRow + if cmd.val != nil { + val = make([]TSNRangePivotRow, len(cmd.val)) + copy(val, cmd.val) + for i := range cmd.val { + if cmd.val[i].Values != nil { + val[i].Values = make([]float64, len(cmd.val[i].Values)) + copy(val[i].Values, cmd.val[i].Values) + } + } + } + return &TSNRangePivotRowSliceCmd{ + baseCmd: cmd.cloneBaseCmd(), + val: val, + } +} + +// buildNRangeAggregationArgs validates and returns one aggregator spec string per key for +// TS.NRANGE / TS.NREVRANGE. The number of specs must equal the number of keys. Each spec +// lists one or more aggregators for its key and is emitted as a single comma-joined wire +// token; specs for different keys are separate wire tokens. +func buildNRangeAggregationArgs(keys []string, aggregators [][]Aggregator) ([]string, error) { + if len(aggregators) != len(keys) { + return nil, fmt.Errorf("redis: TS.NRANGE/TS.NREVRANGE requires exactly %d aggregator spec(s), got %d", len(keys), len(aggregators)) + } + parts := make([]string, len(aggregators)) + for i, spec := range aggregators { + if len(spec) == 0 { + return nil, fmt.Errorf("redis: empty timeseries aggregator spec at index %d", i) + } + names := make([]string, len(spec)) + for j, agg := range spec { + if agg == Invalid { + return nil, fmt.Errorf("redis: invalid timeseries aggregator at index %d[%d]: Invalid (%d)", i, j, agg) + } + s := agg.String() + if s == "" { + return nil, fmt.Errorf("redis: invalid timeseries aggregator at index %d[%d]: %d", i, j, agg) + } + names[j] = s + } + parts[i] = strings.Join(names, ",") + } + return parts, nil +} + +// appendNRangeOptions appends optional TS.NRANGE / TS.NREVRANGE arguments to args. +func appendNRangeOptions( + args []interface{}, + keys []string, + latest bool, + filterByTS []int, + filterByValue []float64, + count int, + align interface{}, + aggregators [][]Aggregator, + bucketDuration int, + bucketTimestamp interface{}, + empty bool, +) ([]interface{}, error) { + if latest { + args = append(args, "LATEST") + } + if len(filterByTS) > 0 { + args = append(args, "FILTER_BY_TS") + for _, ts := range filterByTS { + args = append(args, ts) + } + } + if len(filterByValue) > 0 { + if len(filterByValue) != 2 { + return args, fmt.Errorf("redis: FILTER_BY_VALUE requires exactly 2 elements [min, max], got %d", len(filterByValue)) + } + args = append(args, "FILTER_BY_VALUE", filterByValue[0], filterByValue[1]) + } + if count != 0 { + args = append(args, "COUNT", count) + } + if align != nil { + args = append(args, "ALIGN", align) + } + if len(aggregators) > 0 { + aggParts, err := buildNRangeAggregationArgs(keys, aggregators) + if err != nil { + return args, err + } + args = append(args, "AGGREGATION") + for _, a := range aggParts { + args = append(args, a) + } + if bucketDuration != 0 { + args = append(args, bucketDuration) + } + if bucketTimestamp != nil { + args = append(args, "BUCKETTIMESTAMP", bucketTimestamp) + } + if empty { + args = append(args, "EMPTY") + } + } + return args, nil +} + +// TSNRange - Queries multiple time-series keys and returns a pivot response in forward (ascending) order. +// For more information - https://redis.io/commands/ts.nrange/ +func (c cmdable) TSNRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRangeWithArgs - Queries multiple time-series keys and returns a pivot response in forward (ascending) order with additional options. +// This function allows for specifying additional options such as: +// Latest, FilterByTS, FilterByValue, Count, Align, Aggregators, BucketDuration, BucketTimestamp and Empty. +// Aggregators must contain exactly one spec per key; each spec lists one or more aggregators +// for its key and is emitted as a single comma-joined wire token. +// For more information - https://redis.io/commands/ts.nrange/ +func (c cmdable) TSNRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRangeOptions) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + if options != nil { + var err error + args, err = appendNRangeOptions(args, keys, + options.Latest, options.FilterByTS, options.FilterByValue, + options.Count, options.Align, options.Aggregators, + options.BucketDuration, options.BucketTimestamp, options.Empty) + if err != nil { + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + } + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRevRange - Queries multiple time-series keys and returns a pivot response in reverse (descending) order. +// For more information - https://redis.io/commands/ts.nrevrange/ +func (c cmdable) TSNRevRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NREVRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRevRangeWithArgs - Queries multiple time-series keys and returns a pivot response in reverse (descending) order with additional options. +// This function allows for specifying additional options such as: +// Latest, FilterByTS, FilterByValue, Count, Align, Aggregators, BucketDuration, BucketTimestamp and Empty. +// Aggregators must contain exactly one spec per key; each spec lists one or more aggregators +// for its key and is emitted as a single comma-joined wire token. +// For more information - https://redis.io/commands/ts.nrevrange/ +func (c cmdable) TSNRevRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRevRangeOptions) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NREVRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + if options != nil { + var err error + args, err = appendNRangeOptions(args, keys, + options.Latest, options.FilterByTS, options.FilterByValue, + options.Count, options.Align, options.Aggregators, + options.BucketDuration, options.BucketTimestamp, options.Empty) + if err != nil { + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + } + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} diff --git a/vendor/github.com/redis/go-redis/v9/tx.go b/vendor/github.com/redis/go-redis/v9/tx.go index 179230e3..0e4e33da 100644 --- a/vendor/github.com/redis/go-redis/v9/tx.go +++ b/vendor/github.com/redis/go-redis/v9/tx.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/redis/go-redis/v9/internal/pool" "github.com/redis/go-redis/v9/internal/proto" ) @@ -44,10 +43,20 @@ func (c *Client) newTx() *Tx { tx := Tx{ baseClient: baseClient{ opt: c.cloneOpt(), // Clone options under optLock to avoid race with initConn - connPool: pool.NewStickyConnPool(c.connPool), + connPool: c.baseClient.newStickyConnPool(), hooksMixin: c.hooksMixin.clone(), pushProcessor: c.pushProcessor, // Copy push processor from parent client onClose: &onCloseHooks{}, + // Share the HIMPORT fieldset registry: the sticky pool borrows + // connections from the parent client's pool, so fieldsets + // prepared on them stay valid after the connections are + // returned. + himport: c.himport, + // Carry the shared eviction hook (not csc: a sticky Tx must not serve + // cached reads) so close/reinit hooks on a Watch-initialized conn still + // evict from the parent cache. + cscPoolHook: c.cscPoolHook, + cscActive: c.cscActive, }, } tx.init() diff --git a/vendor/github.com/redis/go-redis/v9/universal.go b/vendor/github.com/redis/go-redis/v9/universal.go index b623460c..40f72ac5 100644 --- a/vendor/github.com/redis/go-redis/v9/universal.go +++ b/vendor/github.com/redis/go-redis/v9/universal.go @@ -149,8 +149,31 @@ type UniversalOptions struct { // IsClusterMode can be used when only one Addrs is provided (e.g. Elasticache supports setting up cluster mode with configuration endpoint). IsClusterMode bool + // AutoPipelineOptions is the default config for the client's + // autopipeliner faces (AutoPipeline / AsyncAutoPipeline), applied when + // they are called without explicit options. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + // MaintNotificationsConfig provides configuration for maintnotifications upgrades. MaintNotificationsConfig *maintnotifications.Config + + // ClientSideCacheConfig enables client-side caching when NewUniversalClient + // selects a standalone Client. See Options.ClientSideCacheConfig. + // + // Experimental: this API may change in a minor release. + ClientSideCacheConfig *ClientSideCacheConfig + + // ClientSideCache supplies an explicit cache when NewUniversalClient selects + // a standalone Client. See Options.ClientSideCache. + // + // Experimental: this API may change in a minor release. + ClientSideCache Cache + + // ClientSideCacheStrategy selects the standalone client's invalidation + // strategy. See Options.ClientSideCacheStrategy. + // + // Experimental: this API may change in a minor release. + ClientSideCacheStrategy CSCStrategy } // Cluster returns cluster options created from the universal options. @@ -208,6 +231,7 @@ func (o *UniversalOptions) Cluster() *ClusterOptions { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, FailingTimeoutSeconds: o.FailingTimeoutSeconds, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, @@ -276,6 +300,7 @@ func (o *UniversalOptions) Failover() *FailoverOptions { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, // Note: MaintNotificationsConfig not supported for FailoverOptions @@ -334,9 +359,13 @@ func (o *UniversalOptions) Simple() *Options { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, MaintNotificationsConfig: o.MaintNotificationsConfig, + ClientSideCacheConfig: o.ClientSideCacheConfig, + ClientSideCache: o.ClientSideCache, + ClientSideCacheStrategy: o.ClientSideCacheStrategy, } } @@ -352,6 +381,15 @@ type UniversalClient interface { Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error Do(ctx context.Context, args ...interface{}) *Cmd Process(ctx context.Context, cmd Cmder) error + // AutoPipeline / AsyncAutoPipeline return an AutoPipeliner for the concrete + // client. Supported on *Client (including sentinel-backed failover clients) + // and *ClusterClient; *Ring returns an error (not supported). + // + // EXPERIMENTAL: this API is subject to change, use with caution. + AutoPipeline() (*AutoPipeliner, error) + AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) + AsyncAutoPipeline() (*AutoPipeliner, error) + AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) Subscribe(ctx context.Context, channels ...string) *PubSub PSubscribe(ctx context.Context, channels ...string) *PubSub SSubscribe(ctx context.Context, channels ...string) *PubSub @@ -363,6 +401,9 @@ var ( _ UniversalClient = (*Client)(nil) _ UniversalClient = (*ClusterClient)(nil) _ UniversalClient = (*Ring)(nil) + // AutoPipeliner is a drop-in for the real clients; non-data operations + // delegate to the underlying client. + _ UniversalClient = (*AutoPipeliner)(nil) ) // NewUniversalClient returns a new multi client. The type of the returned client depends diff --git a/vendor/github.com/redis/go-redis/v9/version.go b/vendor/github.com/redis/go-redis/v9/version.go index c6cabc69..9ce635e1 100644 --- a/vendor/github.com/redis/go-redis/v9/version.go +++ b/vendor/github.com/redis/go-redis/v9/version.go @@ -2,5 +2,5 @@ package redis // Version is the current release version. func Version() string { - return "9.21.0" + return "9.22.0" } diff --git a/vendor/golang.org/x/sys/cpu/asm_aix_ppc64.s b/vendor/golang.org/x/sys/cpu/asm_aix_ppc64.s new file mode 100644 index 00000000..269e173c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/asm_aix_ppc64.s @@ -0,0 +1,17 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +#include "textflag.h" + +// +// System calls for ppc64, AIX are implemented in runtime/syscall_aix.go +// + +TEXT ·syscall6(SB),NOSPLIT,$0-88 + JMP syscall·syscall6(SB) + +TEXT ·rawSyscall6(SB),NOSPLIT,$0-88 + JMP syscall·rawSyscall6(SB) diff --git a/vendor/golang.org/x/sys/cpu/asm_darwin_arm64_gc.s b/vendor/golang.org/x/sys/cpu/asm_darwin_arm64_gc.s new file mode 100644 index 00000000..e07fa75e --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/asm_darwin_arm64_gc.s @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && arm64 && gc + +#include "textflag.h" + +TEXT libc_sysctlbyname_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctlbyname(SB) +GLOBL ·libc_sysctlbyname_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctlbyname_trampoline_addr(SB)/8, $libc_sysctlbyname_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s b/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s new file mode 100644 index 00000000..ec2acfe5 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +#include "textflag.h" + +TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctl(SB) +GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) + +TEXT libc_sysctlbyname_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctlbyname(SB) +GLOBL ·libc_sysctlbyname_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctlbyname_trampoline_addr(SB)/8, $libc_sysctlbyname_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/cpu/byteorder.go b/vendor/golang.org/x/sys/cpu/byteorder.go new file mode 100644 index 00000000..271055be --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/byteorder.go @@ -0,0 +1,66 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "runtime" +) + +// byteOrder is a subset of encoding/binary.ByteOrder. +type byteOrder interface { + Uint32([]byte) uint32 + Uint64([]byte) uint64 +} + +type littleEndian struct{} +type bigEndian struct{} + +func (littleEndian) Uint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func (littleEndian) Uint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func (bigEndian) Uint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func (bigEndian) Uint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +// hostByteOrder returns littleEndian on little-endian machines and +// bigEndian on big-endian machines. +func hostByteOrder() byteOrder { + switch runtime.GOARCH { + case "386", "amd64", "amd64p32", + "alpha", + "arm", "arm64", + "loong64", + "mipsle", "mips64le", "mips64p32le", + "nios2", + "ppc64le", + "riscv", "riscv64", + "sh": + return littleEndian{} + case "armbe", "arm64be", + "m68k", + "mips", "mips64", "mips64p32", + "ppc", "ppc64", + "s390", "s390x", + "shbe", + "sparc", "sparc64": + return bigEndian{} + } + panic("unknown architecture") +} diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go new file mode 100644 index 00000000..f1ce515d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -0,0 +1,343 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection for +// various CPU architectures. +package cpu + +import ( + "os" + "strings" +) + +// Initialized reports whether the CPU features were initialized. +// +// For some GOOS/GOARCH combinations initialization of the CPU features depends +// on reading an operating specific file, e.g. /proc/self/auxv on linux/arm +// Initialized will report false if reading the file fails. +var Initialized bool + +// CacheLinePad is used to pad structs to avoid false sharing. +type CacheLinePad struct{ _ [cacheLineSize]byte } + +// X86 contains the supported CPU features of the +// current X86/AMD64 platform. If the current platform +// is not X86/AMD64 then all feature flags are false. +// +// X86 is padded to avoid false sharing. Further the HasAVX +// and HasAVX2 are only set if the OS supports XMM and YMM +// registers in addition to the CPUID feature bit being set. +var X86 struct { + _ CacheLinePad + HasAES bool // AES hardware implementation (AES NI) + HasADX bool // Multi-precision add-carry instruction extensions + HasAVX bool // Advanced vector extension + HasAVX2 bool // Advanced vector extension 2 + HasAVX512 bool // Advanced vector extension 512 + HasAVX512F bool // Advanced vector extension 512 Foundation Instructions + HasAVX512CD bool // Advanced vector extension 512 Conflict Detection Instructions + HasAVX512ER bool // Advanced vector extension 512 Exponential and Reciprocal Instructions + HasAVX512PF bool // Advanced vector extension 512 Prefetch Instructions + HasAVX512VL bool // Advanced vector extension 512 Vector Length Extensions + HasAVX512BW bool // Advanced vector extension 512 Byte and Word Instructions + HasAVX512DQ bool // Advanced vector extension 512 Doubleword and Quadword Instructions + HasAVX512IFMA bool // Advanced vector extension 512 Integer Fused Multiply Add + HasAVX512VBMI bool // Advanced vector extension 512 Vector Byte Manipulation Instructions + HasAVX5124VNNIW bool // Advanced vector extension 512 Vector Neural Network Instructions Word variable precision + HasAVX5124FMAPS bool // Advanced vector extension 512 Fused Multiply Accumulation Packed Single precision + HasAVX512VPOPCNTDQ bool // Advanced vector extension 512 Double and quad word population count instructions + HasAVX512VPCLMULQDQ bool // Advanced vector extension 512 Vector carry-less multiply operations + HasAVX512VNNI bool // Advanced vector extension 512 Vector Neural Network Instructions + HasAVX512GFNI bool // Advanced vector extension 512 Galois field New Instructions + HasAVX512VAES bool // Advanced vector extension 512 Vector AES instructions + HasAVX512VBMI2 bool // Advanced vector extension 512 Vector Byte Manipulation Instructions 2 + HasAVX512BITALG bool // Advanced vector extension 512 Bit Algorithms + HasAVX512BF16 bool // Advanced vector extension 512 BFloat16 Instructions + HasAMXTile bool // Advanced Matrix Extension Tile instructions + HasAMXInt8 bool // Advanced Matrix Extension Int8 instructions + HasAMXBF16 bool // Advanced Matrix Extension BFloat16 instructions + HasBMI1 bool // Bit manipulation instruction set 1 + HasBMI2 bool // Bit manipulation instruction set 2 + HasCX16 bool // Compare and exchange 16 Bytes + HasERMS bool // Enhanced REP for MOVSB and STOSB + HasFMA bool // Fused-multiply-add instructions + HasOSXSAVE bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. + HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM + HasPOPCNT bool // Hamming weight instruction POPCNT. + HasRDRAND bool // RDRAND instruction (on-chip random number generator) + HasRDSEED bool // RDSEED instruction (on-chip random number generator) + HasSSE2 bool // Streaming SIMD extension 2 (always available on amd64) + HasSSE3 bool // Streaming SIMD extension 3 + HasSSSE3 bool // Supplemental streaming SIMD extension 3 + HasSSE41 bool // Streaming SIMD extension 4 and 4.1 + HasSSE42 bool // Streaming SIMD extension 4 and 4.2 + HasAVXIFMA bool // Advanced vector extension Integer Fused Multiply Add + HasAVXVNNI bool // Advanced vector extension Vector Neural Network Instructions + HasAVXVNNIInt8 bool // Advanced vector extension Vector Neural Network Int8 instructions + _ CacheLinePad +} + +// ARM64 contains the supported CPU features of the +// current ARMv8(aarch64) platform. If the current platform +// is not arm64 then all feature flags are false. +var ARM64 struct { + _ CacheLinePad + HasFP bool // Floating-point instruction set (always available) + HasASIMD bool // Advanced SIMD (always available) + HasEVTSTRM bool // Event stream support + HasAES bool // AES hardware implementation + HasPMULL bool // Polynomial multiplication instruction set + HasSHA1 bool // SHA1 hardware implementation + HasSHA2 bool // SHA2 hardware implementation + HasCRC32 bool // CRC32 hardware implementation + HasATOMICS bool // Atomic memory operation instruction set + HasFPHP bool // Half precision floating-point instruction set + HasASIMDHP bool // Advanced SIMD half precision instruction set + HasCPUID bool // CPUID identification scheme registers + HasASIMDRDM bool // Rounding double multiply add/subtract instruction set + HasJSCVT bool // Javascript conversion from floating-point to integer + HasFCMA bool // Floating-point multiplication and addition of complex numbers + HasLRCPC bool // Release Consistent processor consistent support + HasDCPOP bool // Persistent memory support + HasSHA3 bool // SHA3 hardware implementation + HasSM3 bool // SM3 hardware implementation + HasSM4 bool // SM4 hardware implementation + HasASIMDDP bool // Advanced SIMD double precision instruction set + HasSHA512 bool // SHA512 hardware implementation + HasSVE bool // Scalable Vector Extensions + HasSVE2 bool // Scalable Vector Extensions 2 + HasASIMDFHM bool // Advanced SIMD multiplication FP16 to FP32 + HasDIT bool // Data Independent Timing support + HasI8MM bool // Advanced SIMD Int8 matrix multiplication instructions + _ CacheLinePad +} + +// ARM contains the supported CPU features of the current ARM (32-bit) platform. +// All feature flags are false if: +// 1. the current platform is not arm, or +// 2. the current operating system is not Linux. +var ARM struct { + _ CacheLinePad + HasSWP bool // SWP instruction support + HasHALF bool // Half-word load and store support + HasTHUMB bool // ARM Thumb instruction set + Has26BIT bool // Address space limited to 26-bits + HasFASTMUL bool // 32-bit operand, 64-bit result multiplication support + HasFPA bool // Floating point arithmetic support + HasVFP bool // Vector floating point support + HasEDSP bool // DSP Extensions support + HasJAVA bool // Java instruction set + HasIWMMXT bool // Intel Wireless MMX technology support + HasCRUNCH bool // MaverickCrunch context switching and handling + HasTHUMBEE bool // Thumb EE instruction set + HasNEON bool // NEON instruction set + HasVFPv3 bool // Vector floating point version 3 support + HasVFPv3D16 bool // Vector floating point version 3 D8-D15 + HasTLS bool // Thread local storage support + HasVFPv4 bool // Vector floating point version 4 support + HasIDIVA bool // Integer divide instruction support in ARM mode + HasIDIVT bool // Integer divide instruction support in Thumb mode + HasVFPD32 bool // Vector floating point version 3 D15-D31 + HasLPAE bool // Large Physical Address Extensions + HasEVTSTRM bool // Event stream support + HasAES bool // AES hardware implementation + HasPMULL bool // Polynomial multiplication instruction set + HasSHA1 bool // SHA1 hardware implementation + HasSHA2 bool // SHA2 hardware implementation + HasCRC32 bool // CRC32 hardware implementation + _ CacheLinePad +} + +// The booleans in Loong64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var Loong64 struct { + _ CacheLinePad + HasLSX bool // support 128-bit vector extension + HasLASX bool // support 256-bit vector extension + HasCRC32 bool // support CRC instruction + HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} + HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction + HasLLACQ_SCREL bool // support LLACQ.{W/D}, SCREL.{W/D} instruction + HasSCQ bool // support SC.Q instruction + HasDBAR_HINTS bool // supports finer-grained DBAR hints + + _ CacheLinePad +} + +// MIPS64X contains the supported CPU features of the current mips64/mips64le +// platforms. If the current platform is not mips64/mips64le or the current +// operating system is not Linux then all feature flags are false. +var MIPS64X struct { + _ CacheLinePad + HasMSA bool // MIPS SIMD architecture + _ CacheLinePad +} + +// PPC64 contains the supported CPU features of the current ppc64/ppc64le platforms. +// If the current platform is not ppc64/ppc64le then all feature flags are false. +// +// For ppc64/ppc64le, it is safe to check only for ISA level starting on ISA v3.00, +// since there are no optional categories. There are some exceptions that also +// require kernel support to work (DARN, SCV), so there are feature bits for +// those as well. The struct is padded to avoid false sharing. +var PPC64 struct { + _ CacheLinePad + HasDARN bool // Hardware random number generator (requires kernel enablement) + HasSCV bool // Syscall vectored (requires kernel enablement) + IsPOWER8 bool // ISA v2.07 (POWER8) + IsPOWER9 bool // ISA v3.00 (POWER9), implies IsPOWER8 + _ CacheLinePad +} + +// S390X contains the supported CPU features of the current IBM Z +// (s390x) platform. If the current platform is not IBM Z then all +// feature flags are false. +// +// S390X is padded to avoid false sharing. Further HasVX is only set +// if the OS supports vector registers in addition to the STFLE +// feature bit being set. +var S390X struct { + _ CacheLinePad + HasZARCH bool // z/Architecture mode is active [mandatory] + HasSTFLE bool // store facility list extended + HasLDISP bool // long (20-bit) displacements + HasEIMM bool // 32-bit immediates + HasDFP bool // decimal floating point + HasETF3EH bool // ETF-3 enhanced + HasMSA bool // message security assist (CPACF) + HasAES bool // KM-AES{128,192,256} functions + HasAESCBC bool // KMC-AES{128,192,256} functions + HasAESCTR bool // KMCTR-AES{128,192,256} functions + HasAESGCM bool // KMA-GCM-AES{128,192,256} functions + HasGHASH bool // KIMD-GHASH function + HasSHA1 bool // K{I,L}MD-SHA-1 functions + HasSHA256 bool // K{I,L}MD-SHA-256 functions + HasSHA512 bool // K{I,L}MD-SHA-512 functions + HasSHA3 bool // K{I,L}MD-SHA3-{224,256,384,512} and K{I,L}MD-SHAKE-{128,256} functions + HasVX bool // vector facility + HasVXE bool // vector-enhancements facility 1 + _ CacheLinePad +} + +// RISCV64 contains the supported CPU features and performance characteristics for riscv64 +// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate +// the presence of RISC-V extensions. +// +// It is safe to assume that all the RV64G extensions are supported and so they are omitted from +// this structure. As riscv64 Go programs require at least RV64G, the code that populates +// this structure cannot run successfully if some of the RV64G extensions are missing. +// The struct is padded to avoid false sharing. +var RISCV64 struct { + _ CacheLinePad + HasFastMisaligned bool // Fast misaligned accesses + HasC bool // Compressed instruction-set extension + HasV bool // Vector extension compatible with RVV 1.0 + HasZba bool // Address generation instructions extension + HasZbb bool // Basic bit-manipulation extension + HasZbs bool // Single-bit instructions extension + HasZbc bool // Carryless multiplication extension + HasZvbb bool // Vector Basic Bit-manipulation + HasZvbc bool // Vector Carryless Multiplication + HasZvkb bool // Vector Cryptography Bit-manipulation + HasZvkt bool // Vector Data-Independent Execution Latency + HasZvkg bool // Vector GCM/GMAC + HasZvkn bool // NIST Algorithm Suite (AES/SHA256/SHA512) + HasZvknc bool // NIST Algorithm Suite with carryless multiply + HasZvkng bool // NIST Algorithm Suite with GCM + HasZvks bool // ShangMi Algorithm Suite + HasZvksc bool // ShangMi Algorithm Suite with carryless multiplication + HasZvksg bool // ShangMi Algorithm Suite with GCM + _ CacheLinePad +} + +func init() { + archInit() + initOptions() + processOptions() +} + +// options contains the cpu debug options that can be used in GODEBUG. +// Options are arch dependent and are added by the arch specific initOptions functions. +// Features that are mandatory for the specific GOARCH should have the Required field set +// (e.g. SSE2 on amd64). +var options []option + +// Option names should be lower case. e.g. avx instead of AVX. +type option struct { + Name string + Feature *bool + Specified bool // whether feature value was specified in GODEBUG + Enable bool // whether feature should be enabled + Required bool // whether feature is mandatory and can not be disabled +} + +func processOptions() { + env := os.Getenv("GODEBUG") +field: + for env != "" { + field := "" + i := strings.IndexByte(env, ',') + if i < 0 { + field, env = env, "" + } else { + field, env = env[:i], env[i+1:] + } + if len(field) < 4 || field[:4] != "cpu." { + continue + } + i = strings.IndexByte(field, '=') + if i < 0 { + print("GODEBUG sys/cpu: no value specified for \"", field, "\"\n") + continue + } + key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on" + + var enable bool + switch value { + case "on": + enable = true + case "off": + enable = false + default: + print("GODEBUG sys/cpu: value \"", value, "\" not supported for cpu option \"", key, "\"\n") + continue field + } + + if key == "all" { + for i := range options { + options[i].Specified = true + options[i].Enable = enable || options[i].Required + } + continue field + } + + for i := range options { + if options[i].Name == key { + options[i].Specified = true + options[i].Enable = enable + continue field + } + } + + print("GODEBUG sys/cpu: unknown cpu feature \"", key, "\"\n") + } + + for _, o := range options { + if !o.Specified { + continue + } + + if o.Enable && !*o.Feature { + print("GODEBUG sys/cpu: can not enable \"", o.Name, "\", missing CPU support\n") + continue + } + + if !o.Enable && o.Required { + print("GODEBUG sys/cpu: can not disable \"", o.Name, "\", required CPU feature\n") + continue + } + + *o.Feature = o.Enable + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_aix.go b/vendor/golang.org/x/sys/cpu/cpu_aix.go new file mode 100644 index 00000000..9bf0c32e --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_aix.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix + +package cpu + +const ( + // getsystemcfg constants + _SC_IMPL = 2 + _IMPL_POWER8 = 0x10000 + _IMPL_POWER9 = 0x20000 +) + +func archInit() { + impl := getsystemcfg(_SC_IMPL) + if impl&_IMPL_POWER8 != 0 { + PPC64.IsPOWER8 = true + } + if impl&_IMPL_POWER9 != 0 { + PPC64.IsPOWER8 = true + PPC64.IsPOWER9 = true + } + + Initialized = true +} + +func getsystemcfg(label int) (n uint64) { + r0, _ := callgetsystemcfg(label) + n = uint64(r0) + return +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm.go b/vendor/golang.org/x/sys/cpu/cpu_arm.go new file mode 100644 index 00000000..301b752e --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm.go @@ -0,0 +1,73 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 32 + +// HWCAP/HWCAP2 bits. +// These are specific to Linux. +const ( + hwcap_SWP = 1 << 0 + hwcap_HALF = 1 << 1 + hwcap_THUMB = 1 << 2 + hwcap_26BIT = 1 << 3 + hwcap_FAST_MULT = 1 << 4 + hwcap_FPA = 1 << 5 + hwcap_VFP = 1 << 6 + hwcap_EDSP = 1 << 7 + hwcap_JAVA = 1 << 8 + hwcap_IWMMXT = 1 << 9 + hwcap_CRUNCH = 1 << 10 + hwcap_THUMBEE = 1 << 11 + hwcap_NEON = 1 << 12 + hwcap_VFPv3 = 1 << 13 + hwcap_VFPv3D16 = 1 << 14 + hwcap_TLS = 1 << 15 + hwcap_VFPv4 = 1 << 16 + hwcap_IDIVA = 1 << 17 + hwcap_IDIVT = 1 << 18 + hwcap_VFPD32 = 1 << 19 + hwcap_LPAE = 1 << 20 + hwcap_EVTSTRM = 1 << 21 + + hwcap2_AES = 1 << 0 + hwcap2_PMULL = 1 << 1 + hwcap2_SHA1 = 1 << 2 + hwcap2_SHA2 = 1 << 3 + hwcap2_CRC32 = 1 << 4 +) + +func initOptions() { + options = []option{ + {Name: "pmull", Feature: &ARM.HasPMULL}, + {Name: "sha1", Feature: &ARM.HasSHA1}, + {Name: "sha2", Feature: &ARM.HasSHA2}, + {Name: "swp", Feature: &ARM.HasSWP}, + {Name: "thumb", Feature: &ARM.HasTHUMB}, + {Name: "thumbee", Feature: &ARM.HasTHUMBEE}, + {Name: "tls", Feature: &ARM.HasTLS}, + {Name: "vfp", Feature: &ARM.HasVFP}, + {Name: "vfpd32", Feature: &ARM.HasVFPD32}, + {Name: "vfpv3", Feature: &ARM.HasVFPv3}, + {Name: "vfpv3d16", Feature: &ARM.HasVFPv3D16}, + {Name: "vfpv4", Feature: &ARM.HasVFPv4}, + {Name: "half", Feature: &ARM.HasHALF}, + {Name: "26bit", Feature: &ARM.Has26BIT}, + {Name: "fastmul", Feature: &ARM.HasFASTMUL}, + {Name: "fpa", Feature: &ARM.HasFPA}, + {Name: "edsp", Feature: &ARM.HasEDSP}, + {Name: "java", Feature: &ARM.HasJAVA}, + {Name: "iwmmxt", Feature: &ARM.HasIWMMXT}, + {Name: "crunch", Feature: &ARM.HasCRUNCH}, + {Name: "neon", Feature: &ARM.HasNEON}, + {Name: "idivt", Feature: &ARM.HasIDIVT}, + {Name: "idiva", Feature: &ARM.HasIDIVA}, + {Name: "lpae", Feature: &ARM.HasLPAE}, + {Name: "evtstrm", Feature: &ARM.HasEVTSTRM}, + {Name: "aes", Feature: &ARM.HasAES}, + {Name: "crc32", Feature: &ARM.HasCRC32}, + } + +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go new file mode 100644 index 00000000..5fc09e29 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -0,0 +1,191 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import "runtime" + +// cacheLineSize is used to prevent false sharing of cache lines. +// We choose 128 because Apple Silicon, a.k.a. M1, has 128-byte cache line size. +// It doesn't cost much and is much more future-proof. +const cacheLineSize = 128 + +func initOptions() { + options = []option{ + {Name: "fp", Feature: &ARM64.HasFP}, + {Name: "asimd", Feature: &ARM64.HasASIMD}, + {Name: "evstrm", Feature: &ARM64.HasEVTSTRM}, + {Name: "aes", Feature: &ARM64.HasAES}, + {Name: "fphp", Feature: &ARM64.HasFPHP}, + {Name: "jscvt", Feature: &ARM64.HasJSCVT}, + {Name: "lrcpc", Feature: &ARM64.HasLRCPC}, + {Name: "pmull", Feature: &ARM64.HasPMULL}, + {Name: "sha1", Feature: &ARM64.HasSHA1}, + {Name: "sha2", Feature: &ARM64.HasSHA2}, + {Name: "sha3", Feature: &ARM64.HasSHA3}, + {Name: "sha512", Feature: &ARM64.HasSHA512}, + {Name: "sm3", Feature: &ARM64.HasSM3}, + {Name: "sm4", Feature: &ARM64.HasSM4}, + {Name: "sve", Feature: &ARM64.HasSVE}, + {Name: "sve2", Feature: &ARM64.HasSVE2}, + {Name: "crc32", Feature: &ARM64.HasCRC32}, + {Name: "atomics", Feature: &ARM64.HasATOMICS}, + {Name: "asimdhp", Feature: &ARM64.HasASIMDHP}, + {Name: "cpuid", Feature: &ARM64.HasCPUID}, + {Name: "asimrdm", Feature: &ARM64.HasASIMDRDM}, + {Name: "fcma", Feature: &ARM64.HasFCMA}, + {Name: "dcpop", Feature: &ARM64.HasDCPOP}, + {Name: "asimddp", Feature: &ARM64.HasASIMDDP}, + {Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM}, + {Name: "dit", Feature: &ARM64.HasDIT}, + {Name: "i8mm", Feature: &ARM64.HasI8MM}, + } +} + +func archInit() { + if runtime.GOOS == "freebsd" { + readARM64Registers() + } else { + // Most platforms don't seem to allow directly reading these registers. + doinit() + } +} + +// setMinimalFeatures fakes the minimal ARM64 features expected by +// TestARM64minimalFeatures. +func setMinimalFeatures() { + ARM64.HasASIMD = true + ARM64.HasFP = true +} + +func readARM64Registers() { + Initialized = true + + parseARM64SystemRegisters(getisar0(), getisar1(), getpfr0()) +} + +func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) { + // ID_AA64ISAR0_EL1 + switch extractBits(isar0, 4, 7) { + case 1: + ARM64.HasAES = true + case 2: + ARM64.HasAES = true + ARM64.HasPMULL = true + } + + switch extractBits(isar0, 8, 11) { + case 1: + ARM64.HasSHA1 = true + } + + switch extractBits(isar0, 12, 15) { + case 1: + ARM64.HasSHA2 = true + case 2: + ARM64.HasSHA2 = true + ARM64.HasSHA512 = true + } + + switch extractBits(isar0, 16, 19) { + case 1: + ARM64.HasCRC32 = true + } + + switch extractBits(isar0, 20, 23) { + case 2: + ARM64.HasATOMICS = true + } + + switch extractBits(isar0, 28, 31) { + case 1: + ARM64.HasASIMDRDM = true + } + + switch extractBits(isar0, 32, 35) { + case 1: + ARM64.HasSHA3 = true + } + + switch extractBits(isar0, 36, 39) { + case 1: + ARM64.HasSM3 = true + } + + switch extractBits(isar0, 40, 43) { + case 1: + ARM64.HasSM4 = true + } + + switch extractBits(isar0, 44, 47) { + case 1: + ARM64.HasASIMDDP = true + } + + // ID_AA64ISAR1_EL1 + switch extractBits(isar1, 0, 3) { + case 1: + ARM64.HasDCPOP = true + } + + switch extractBits(isar1, 12, 15) { + case 1: + ARM64.HasJSCVT = true + } + + switch extractBits(isar1, 16, 19) { + case 1: + ARM64.HasFCMA = true + } + + switch extractBits(isar1, 20, 23) { + case 1: + ARM64.HasLRCPC = true + } + + switch extractBits(isar1, 52, 55) { + case 1: + ARM64.HasI8MM = true + } + + // ID_AA64PFR0_EL1 + switch extractBits(pfr0, 16, 19) { + case 0: + ARM64.HasFP = true + case 1: + ARM64.HasFP = true + ARM64.HasFPHP = true + } + + switch extractBits(pfr0, 20, 23) { + case 0: + ARM64.HasASIMD = true + case 1: + ARM64.HasASIMD = true + ARM64.HasASIMDHP = true + } + + switch extractBits(pfr0, 32, 35) { + case 1: + ARM64.HasSVE = true + + parseARM64SVERegister(getzfr0()) + } + + switch extractBits(pfr0, 48, 51) { + case 1: + ARM64.HasDIT = true + } +} + +func parseARM64SVERegister(zfr0 uint64) { + switch extractBits(zfr0, 0, 3) { + case 1: + ARM64.HasSVE2 = true + } +} + +func extractBits(data uint64, start, end uint) uint { + return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.s b/vendor/golang.org/x/sys/cpu/cpu_arm64.s new file mode 100644 index 00000000..3b0450a0 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.s @@ -0,0 +1,35 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +#include "textflag.h" + +// func getisar0() uint64 +TEXT ·getisar0(SB),NOSPLIT,$0-8 + // get Instruction Set Attributes 0 into x0 + MRS ID_AA64ISAR0_EL1, R0 + MOVD R0, ret+0(FP) + RET + +// func getisar1() uint64 +TEXT ·getisar1(SB),NOSPLIT,$0-8 + // get Instruction Set Attributes 1 into x0 + MRS ID_AA64ISAR1_EL1, R0 + MOVD R0, ret+0(FP) + RET + +// func getpfr0() uint64 +TEXT ·getpfr0(SB),NOSPLIT,$0-8 + // get Processor Feature Register 0 into x0 + MRS ID_AA64PFR0_EL1, R0 + MOVD R0, ret+0(FP) + RET + +// func getzfr0() uint64 +TEXT ·getzfr0(SB),NOSPLIT,$0-8 + // get SVE Feature Register 0 into x0 + MRS ID_AA64ZFR0_EL1, R0 + MOVD R0, ret+0(FP) + RET diff --git a/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64.go new file mode 100644 index 00000000..0b470744 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64.go @@ -0,0 +1,67 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && arm64 && gc + +package cpu + +func doinit() { + setMinimalFeatures() + + // The feature flags are explained in [Instruction Set Detection]. + // There are some differences between MacOS versions: + // + // MacOS 11 and 12 do not have "hw.optional" sysctl values for some of the features. + // + // MacOS 13 changed some of the naming conventions to align with ARM Architecture Reference Manual. + // For example "hw.optional.armv8_2_sha512" became "hw.optional.arm.FEAT_SHA512". + // It currently checks both to stay compatible with MacOS 11 and 12. + // The old names also work with MacOS 13, however it's not clear whether + // they will continue working with future OS releases. + // + // Once MacOS 12 is no longer supported the old names can be removed. + // + // [Instruction Set Detection]: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics + + // Encryption, hashing and checksum capabilities + + // For the following flags there are no MacOS 11 sysctl flags. + ARM64.HasAES = true || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_AES\x00")) + ARM64.HasPMULL = true || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_PMULL\x00")) + ARM64.HasSHA1 = true || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SHA1\x00")) + ARM64.HasSHA2 = true || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SHA256\x00")) + + ARM64.HasSHA3 = darwinSysctlEnabled([]byte("hw.optional.armv8_2_sha3\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SHA3\x00")) + ARM64.HasSHA512 = darwinSysctlEnabled([]byte("hw.optional.armv8_2_sha512\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SHA512\x00")) + + ARM64.HasCRC32 = darwinSysctlEnabled([]byte("hw.optional.armv8_crc32\x00")) + + // Atomic and memory ordering + ARM64.HasATOMICS = darwinSysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_LSE\x00")) + ARM64.HasLRCPC = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_LRCPC\x00")) + + // SIMD and floating point capabilities + ARM64.HasFPHP = darwinSysctlEnabled([]byte("hw.optional.neon_fp16\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_FP16\x00")) + ARM64.HasASIMDHP = darwinSysctlEnabled([]byte("hw.optional.neon_hpfp\x00")) || darwinSysctlEnabled([]byte("hw.optional.AdvSIMD_HPFPCvt\x00")) + ARM64.HasASIMDRDM = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_RDM\x00")) + ARM64.HasASIMDDP = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_DotProd\x00")) + ARM64.HasASIMDFHM = darwinSysctlEnabled([]byte("hw.optional.armv8_2_fhm\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_FHM\x00")) + ARM64.HasI8MM = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_I8MM\x00")) + + ARM64.HasJSCVT = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_JSCVT\x00")) + ARM64.HasFCMA = darwinSysctlEnabled([]byte("hw.optional.armv8_3_compnum\x00")) || darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_FCMA\x00")) + + // Miscellaneous + ARM64.HasDCPOP = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_DPB\x00")) + ARM64.HasEVTSTRM = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_ECV\x00")) + ARM64.HasDIT = darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_DIT\x00")) + + // Not supported, but added for completeness + ARM64.HasCPUID = false + + ARM64.HasSM3 = false // darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SM3\x00")) + ARM64.HasSM4 = false // darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SM4\x00")) + ARM64.HasSVE = false // darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SVE\x00")) + ARM64.HasSVE2 = false // darwinSysctlEnabled([]byte("hw.optional.arm.FEAT_SVE2\x00")) +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64_other.go b/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64_other.go new file mode 100644 index 00000000..37ecc664 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_darwin_arm64_other.go @@ -0,0 +1,31 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && arm64 && !gc + +package cpu + +import "runtime" + +func doinit() { + setMinimalFeatures() + + ARM64.HasASIMD = true + ARM64.HasFP = true + + // Go already assumes these to be available because they were on the M1 + // and these are supported on all Apple arm64 chips. + ARM64.HasAES = true + ARM64.HasPMULL = true + ARM64.HasSHA1 = true + ARM64.HasSHA2 = true + + if runtime.GOOS != "ios" { + // Apple A7 processors do not support these, however + // M-series SoCs are at least armv8.4-a + ARM64.HasCRC32 = true // armv8.1 + ARM64.HasATOMICS = true // armv8.2 + ARM64.HasJSCVT = true // armv8.3, if HasFP + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go b/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go new file mode 100644 index 00000000..b838cb9e --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +package cpu + +// darwinSupportsAVX512 checks Darwin kernel for AVX512 support via sysctl +// call (see issue 43089). It also restricts AVX512 support for Darwin to +// kernel version 21.3.0 (MacOS 12.2.0) or later (see issue 49233). +// +// Background: +// Darwin implements a special mechanism to economize on thread state when +// AVX512 specific registers are not in use. This scheme minimizes state when +// preempting threads that haven't yet used any AVX512 instructions, but adds +// special requirements to check for AVX512 hardware support at runtime (e.g. +// via sysctl call or commpage inspection). See issue 43089 and link below for +// full background: +// https://github.com/apple-oss-distributions/xnu/blob/xnu-11215.1.10/osfmk/i386/fpu.c#L214-L240 +// +// Additionally, all versions of the Darwin kernel from 19.6.0 through 21.2.0 +// (corresponding to MacOS 10.15.6 - 12.1) have a bug that can cause corruption +// of the AVX512 mask registers (K0-K7) upon signal return. For this reason +// AVX512 is considered unsafe to use on Darwin for kernel versions prior to +// 21.3.0, where a fix has been confirmed. See issue 49233 for full background. +func darwinSupportsAVX512() bool { + return darwinSysctlEnabled([]byte("hw.optional.avx512f\x00")) && darwinKernelVersionCheck(21, 3, 0) +} + +// Ensure Darwin kernel version is at least major.minor.patch, avoiding dependencies +func darwinKernelVersionCheck(major, minor, patch int) bool { + var release [256]byte + err := darwinOSRelease(&release) + if err != nil { + return false + } + + var mmp [3]int + c := 0 +Loop: + for _, b := range release[:] { + switch { + case b >= '0' && b <= '9': + mmp[c] = 10*mmp[c] + int(b-'0') + case b == '.': + c++ + if c > 2 { + return false + } + case b == 0: + break Loop + default: + return false + } + } + if c != 2 { + return false + } + return mmp[0] > major || mmp[0] == major && (mmp[1] > minor || mmp[1] == minor && mmp[2] >= patch) +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go new file mode 100644 index 00000000..6ac6e1ef --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go @@ -0,0 +1,12 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +package cpu + +func getisar0() uint64 +func getisar1() uint64 +func getpfr0() uint64 +func getzfr0() uint64 diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_gc_s390x.go new file mode 100644 index 00000000..c8ae6ddc --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_s390x.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +package cpu + +// haveAsmFunctions reports whether the other functions in this file can +// be safely called. +func haveAsmFunctions() bool { return true } + +// The following feature detection functions are defined in cpu_s390x.s. +// They are likely to be expensive to call so the results should be cached. +func stfle() facilityList +func kmQuery() queryResult +func kmcQuery() queryResult +func kmctrQuery() queryResult +func kmaQuery() queryResult +func kimdQuery() queryResult +func klmdQuery() queryResult diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go new file mode 100644 index 00000000..32a44514 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -0,0 +1,15 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (386 || amd64 || amd64p32) && gc + +package cpu + +// cpuid is implemented in cpu_gc_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_gc_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func xgetbv() (eax, edx uint32) diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s new file mode 100644 index 00000000..ce208ce6 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s @@ -0,0 +1,26 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (386 || amd64 || amd64p32) && gc + +#include "textflag.h" + +// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL eaxArg+0(FP), AX + MOVL ecxArg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv() (eax, edx uint32) +TEXT ·xgetbv(SB), NOSPLIT, $0-8 + MOVL $0, CX + XGETBV + MOVL AX, eax+0(FP) + MOVL DX, edx+4(FP) + RET diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go new file mode 100644 index 00000000..05913081 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go @@ -0,0 +1,12 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gccgo + +package cpu + +func getisar0() uint64 { return 0 } +func getisar1() uint64 { return 0 } +func getpfr0() uint64 { return 0 } +func getzfr0() uint64 { return 0 } diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo_s390x.go new file mode 100644 index 00000000..9526d2ce --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo_s390x.go @@ -0,0 +1,22 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gccgo + +package cpu + +// haveAsmFunctions reports whether the other functions in this file can +// be safely called. +func haveAsmFunctions() bool { return false } + +// TODO(mundaym): the following feature detection functions are currently +// stubs. See https://golang.org/cl/162887 for how to fix this. +// They are likely to be expensive to call so the results should be cached. +func stfle() facilityList { panic("not implemented for gccgo") } +func kmQuery() queryResult { panic("not implemented for gccgo") } +func kmcQuery() queryResult { panic("not implemented for gccgo") } +func kmctrQuery() queryResult { panic("not implemented for gccgo") } +func kmaQuery() queryResult { panic("not implemented for gccgo") } +func kimdQuery() queryResult { panic("not implemented for gccgo") } +func klmdQuery() queryResult { panic("not implemented for gccgo") } diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.c b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.c new file mode 100644 index 00000000..3f73a05d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.c @@ -0,0 +1,37 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (386 || amd64 || amd64p32) && gccgo + +#include +#include +#include + +// Need to wrap __get_cpuid_count because it's declared as static. +int +gccgoGetCpuidCount(uint32_t leaf, uint32_t subleaf, + uint32_t *eax, uint32_t *ebx, + uint32_t *ecx, uint32_t *edx) +{ + return __get_cpuid_count(leaf, subleaf, eax, ebx, ecx, edx); +} + +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#pragma GCC push_options +#pragma GCC target("xsave") +#pragma clang attribute push (__attribute__((target("xsave"))), apply_to=function) + +// xgetbv reads the contents of an XCR (Extended Control Register) +// specified in the ECX register into registers EDX:EAX. +// Currently, the only supported value for XCR is 0. +void +gccgoXgetbv(uint32_t *eax, uint32_t *edx) +{ + uint64_t v = _xgetbv(0); + *eax = v & 0xffffffff; + *edx = v >> 32; +} + +#pragma clang attribute pop +#pragma GCC pop_options diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go new file mode 100644 index 00000000..170d21dd --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (386 || amd64 || amd64p32) && gccgo + +package cpu + +//extern gccgoGetCpuidCount +func gccgoGetCpuidCount(eaxArg, ecxArg uint32, eax, ebx, ecx, edx *uint32) + +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) { + var a, b, c, d uint32 + gccgoGetCpuidCount(eaxArg, ecxArg, &a, &b, &c, &d) + return a, b, c, d +} + +//extern gccgoXgetbv +func gccgoXgetbv(eax, edx *uint32) + +func xgetbv() (eax, edx uint32) { + var a, d uint32 + gccgoXgetbv(&a, &d) + return a, d +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux.go b/vendor/golang.org/x/sys/cpu/cpu_linux.go new file mode 100644 index 00000000..743eb543 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux.go @@ -0,0 +1,15 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !386 && !amd64 && !amd64p32 && !arm64 + +package cpu + +func archInit() { + if err := readHWCAP(); err != nil { + return + } + doinit() + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_arm.go b/vendor/golang.org/x/sys/cpu/cpu_linux_arm.go new file mode 100644 index 00000000..2057006d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_arm.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +func doinit() { + ARM.HasSWP = isSet(hwCap, hwcap_SWP) + ARM.HasHALF = isSet(hwCap, hwcap_HALF) + ARM.HasTHUMB = isSet(hwCap, hwcap_THUMB) + ARM.Has26BIT = isSet(hwCap, hwcap_26BIT) + ARM.HasFASTMUL = isSet(hwCap, hwcap_FAST_MULT) + ARM.HasFPA = isSet(hwCap, hwcap_FPA) + ARM.HasVFP = isSet(hwCap, hwcap_VFP) + ARM.HasEDSP = isSet(hwCap, hwcap_EDSP) + ARM.HasJAVA = isSet(hwCap, hwcap_JAVA) + ARM.HasIWMMXT = isSet(hwCap, hwcap_IWMMXT) + ARM.HasCRUNCH = isSet(hwCap, hwcap_CRUNCH) + ARM.HasTHUMBEE = isSet(hwCap, hwcap_THUMBEE) + ARM.HasNEON = isSet(hwCap, hwcap_NEON) + ARM.HasVFPv3 = isSet(hwCap, hwcap_VFPv3) + ARM.HasVFPv3D16 = isSet(hwCap, hwcap_VFPv3D16) + ARM.HasTLS = isSet(hwCap, hwcap_TLS) + ARM.HasVFPv4 = isSet(hwCap, hwcap_VFPv4) + ARM.HasIDIVA = isSet(hwCap, hwcap_IDIVA) + ARM.HasIDIVT = isSet(hwCap, hwcap_IDIVT) + ARM.HasVFPD32 = isSet(hwCap, hwcap_VFPD32) + ARM.HasLPAE = isSet(hwCap, hwcap_LPAE) + ARM.HasEVTSTRM = isSet(hwCap, hwcap_EVTSTRM) + ARM.HasAES = isSet(hwCap2, hwcap2_AES) + ARM.HasPMULL = isSet(hwCap2, hwcap2_PMULL) + ARM.HasSHA1 = isSet(hwCap2, hwcap2_SHA1) + ARM.HasSHA2 = isSet(hwCap2, hwcap2_SHA2) + ARM.HasCRC32 = isSet(hwCap2, hwcap2_CRC32) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go new file mode 100644 index 00000000..f1caf0f7 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go @@ -0,0 +1,120 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "strings" + "syscall" +) + +// HWCAP/HWCAP2 bits. These are exposed by Linux. +const ( + hwcap_FP = 1 << 0 + hwcap_ASIMD = 1 << 1 + hwcap_EVTSTRM = 1 << 2 + hwcap_AES = 1 << 3 + hwcap_PMULL = 1 << 4 + hwcap_SHA1 = 1 << 5 + hwcap_SHA2 = 1 << 6 + hwcap_CRC32 = 1 << 7 + hwcap_ATOMICS = 1 << 8 + hwcap_FPHP = 1 << 9 + hwcap_ASIMDHP = 1 << 10 + hwcap_CPUID = 1 << 11 + hwcap_ASIMDRDM = 1 << 12 + hwcap_JSCVT = 1 << 13 + hwcap_FCMA = 1 << 14 + hwcap_LRCPC = 1 << 15 + hwcap_DCPOP = 1 << 16 + hwcap_SHA3 = 1 << 17 + hwcap_SM3 = 1 << 18 + hwcap_SM4 = 1 << 19 + hwcap_ASIMDDP = 1 << 20 + hwcap_SHA512 = 1 << 21 + hwcap_SVE = 1 << 22 + hwcap_ASIMDFHM = 1 << 23 + hwcap_DIT = 1 << 24 + + hwcap2_SVE2 = 1 << 1 + hwcap2_I8MM = 1 << 13 +) + +// linuxKernelCanEmulateCPUID reports whether we're running +// on Linux 4.11+. Ideally we'd like to ask the question about +// whether the current kernel contains +// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=77c97b4ee21290f5f083173d957843b615abbff2 +// but the version number will have to do. +func linuxKernelCanEmulateCPUID() bool { + var un syscall.Utsname + syscall.Uname(&un) + var sb strings.Builder + for _, b := range un.Release[:] { + if b == 0 { + break + } + sb.WriteByte(byte(b)) + } + major, minor, _, ok := parseRelease(sb.String()) + return ok && (major > 4 || major == 4 && minor >= 11) +} + +func doinit() { + if err := readHWCAP(); err != nil { + // We failed to read /proc/self/auxv. This can happen if the binary has + // been given extra capabilities(7) with /bin/setcap. + // + // When this happens, we have two options. If the Linux kernel is new + // enough (4.11+), we can read the arm64 registers directly which'll + // trap into the kernel and then return back to userspace. + // + // But on older kernels, such as Linux 4.4.180 as used on many Synology + // devices, calling readARM64Registers (specifically getisar0) will + // cause a SIGILL and we'll die. So for older kernels, parse /proc/cpuinfo + // instead. + // + // See golang/go#57336. + if linuxKernelCanEmulateCPUID() { + readARM64Registers() + } else { + readLinuxProcCPUInfo() + } + return + } + + // HWCAP feature bits + ARM64.HasFP = isSet(hwCap, hwcap_FP) + ARM64.HasASIMD = isSet(hwCap, hwcap_ASIMD) + ARM64.HasEVTSTRM = isSet(hwCap, hwcap_EVTSTRM) + ARM64.HasAES = isSet(hwCap, hwcap_AES) + ARM64.HasPMULL = isSet(hwCap, hwcap_PMULL) + ARM64.HasSHA1 = isSet(hwCap, hwcap_SHA1) + ARM64.HasSHA2 = isSet(hwCap, hwcap_SHA2) + ARM64.HasCRC32 = isSet(hwCap, hwcap_CRC32) + ARM64.HasATOMICS = isSet(hwCap, hwcap_ATOMICS) + ARM64.HasFPHP = isSet(hwCap, hwcap_FPHP) + ARM64.HasASIMDHP = isSet(hwCap, hwcap_ASIMDHP) + ARM64.HasCPUID = isSet(hwCap, hwcap_CPUID) + ARM64.HasASIMDRDM = isSet(hwCap, hwcap_ASIMDRDM) + ARM64.HasJSCVT = isSet(hwCap, hwcap_JSCVT) + ARM64.HasFCMA = isSet(hwCap, hwcap_FCMA) + ARM64.HasLRCPC = isSet(hwCap, hwcap_LRCPC) + ARM64.HasDCPOP = isSet(hwCap, hwcap_DCPOP) + ARM64.HasSHA3 = isSet(hwCap, hwcap_SHA3) + ARM64.HasSM3 = isSet(hwCap, hwcap_SM3) + ARM64.HasSM4 = isSet(hwCap, hwcap_SM4) + ARM64.HasASIMDDP = isSet(hwCap, hwcap_ASIMDDP) + ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512) + ARM64.HasSVE = isSet(hwCap, hwcap_SVE) + ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM) + ARM64.HasDIT = isSet(hwCap, hwcap_DIT) + + // HWCAP2 feature bits + ARM64.HasSVE2 = isSet(hwCap2, hwcap2_SVE2) + ARM64.HasI8MM = isSet(hwCap2, hwcap2_I8MM) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go new file mode 100644 index 00000000..4f341143 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go @@ -0,0 +1,22 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +// HWCAP bits. These are exposed by the Linux kernel. +const ( + hwcap_LOONGARCH_LSX = 1 << 4 + hwcap_LOONGARCH_LASX = 1 << 5 +) + +func doinit() { + // TODO: Features that require kernel support like LSX and LASX can + // be detected here once needed in std library or by the compiler. + Loong64.HasLSX = hwcIsSet(hwCap, hwcap_LOONGARCH_LSX) + Loong64.HasLASX = hwcIsSet(hwCap, hwcap_LOONGARCH_LASX) +} + +func hwcIsSet(hwc uint, val uint) bool { + return hwc&val != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_linux_mips64x.go new file mode 100644 index 00000000..4686c1d5 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_mips64x.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips64 || mips64le) + +package cpu + +// HWCAP bits. These are exposed by the Linux kernel 5.4. +const ( + // CPU features + hwcap_MIPS_MSA = 1 << 1 +) + +func doinit() { + // HWCAP feature bits + MIPS64X.HasMSA = isSet(hwCap, hwcap_MIPS_MSA) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go new file mode 100644 index 00000000..a428dec9 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go @@ -0,0 +1,9 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !arm && !arm64 && !loong64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x && !riscv64 + +package cpu + +func doinit() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go new file mode 100644 index 00000000..197188e6 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go @@ -0,0 +1,30 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (ppc64 || ppc64le) + +package cpu + +// HWCAP/HWCAP2 bits. These are exposed by the kernel. +const ( + // ISA Level + _PPC_FEATURE2_ARCH_2_07 = 0x80000000 + _PPC_FEATURE2_ARCH_3_00 = 0x00800000 + + // CPU features + _PPC_FEATURE2_DARN = 0x00200000 + _PPC_FEATURE2_SCV = 0x00100000 +) + +func doinit() { + // HWCAP2 feature bits + PPC64.IsPOWER8 = isSet(hwCap2, _PPC_FEATURE2_ARCH_2_07) + PPC64.IsPOWER9 = isSet(hwCap2, _PPC_FEATURE2_ARCH_3_00) + PPC64.HasDARN = isSet(hwCap2, _PPC_FEATURE2_DARN) + PPC64.HasSCV = isSet(hwCap2, _PPC_FEATURE2_SCV) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go new file mode 100644 index 00000000..f4fb52ee --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go @@ -0,0 +1,162 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "syscall" + "unsafe" +) + +// RISC-V extension discovery code for Linux. The approach here is to first try the riscv_hwprobe +// syscall falling back to HWCAP to check for the C extension if riscv_hwprobe is not available. +// +// A note on detection of the Vector extension using HWCAP. +// +// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5. +// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe +// syscall is not available then neither is the Vector extension (which needs kernel support). +// The riscv_hwprobe syscall should then be all we need to detect the Vector extension. +// However, some RISC-V board manufacturers ship boards with an older kernel on top of which +// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe +// patches. These kernels advertise support for the Vector extension using HWCAP. Falling +// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not +// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option. +// +// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by +// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board +// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified +// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use +// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector +// extension are binary incompatible. HWCAP can then not be used in isolation to populate the +// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0. +// +// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector +// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype +// register. This check would allow us to safely detect version 1.0 of the Vector extension +// with HWCAP, if riscv_hwprobe were not available. However, the check cannot +// be added until the assembler supports the Vector instructions. +// +// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the +// extensions it advertises support for are explicitly versioned. It's also worth noting that +// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zba. +// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority +// of RISC-V extensions. +// +// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information. + +// golang.org/x/sys/cpu is not allowed to depend on golang.org/x/sys/unix so we must +// reproduce the constants, types and functions needed to make the riscv_hwprobe syscall +// here. + +const ( + // Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. + riscv_HWPROBE_KEY_IMA_EXT_0 = 0x4 + riscv_HWPROBE_IMA_C = 0x2 + riscv_HWPROBE_IMA_V = 0x4 + riscv_HWPROBE_EXT_ZBA = 0x8 + riscv_HWPROBE_EXT_ZBB = 0x10 + riscv_HWPROBE_EXT_ZBS = 0x20 + riscv_HWPROBE_EXT_ZBC = 0x80 + riscv_HWPROBE_EXT_ZVBB = 0x20000 + riscv_HWPROBE_EXT_ZVBC = 0x40000 + riscv_HWPROBE_EXT_ZVKB = 0x80000 + riscv_HWPROBE_EXT_ZVKG = 0x100000 + riscv_HWPROBE_EXT_ZVKNED = 0x200000 + riscv_HWPROBE_EXT_ZVKNHB = 0x800000 + riscv_HWPROBE_EXT_ZVKSED = 0x1000000 + riscv_HWPROBE_EXT_ZVKSH = 0x2000000 + riscv_HWPROBE_EXT_ZVKT = 0x4000000 + riscv_HWPROBE_KEY_CPUPERF_0 = 0x5 + riscv_HWPROBE_MISALIGNED_FAST = 0x3 + riscv_HWPROBE_MISALIGNED_MASK = 0x7 +) + +const ( + // sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go. + sys_RISCV_HWPROBE = 258 +) + +// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. +type riscvHWProbePairs struct { + key int64 + value uint64 +} + +const ( + // CPU features + hwcap_RISCV_ISA_C = 1 << ('C' - 'A') +) + +func doinit() { + // A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key + // field should be initialised with one of the key constants defined above, e.g., + // RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value. + // If the kernel does not recognise a key it will set the key field to -1 and the value field to 0. + + pairs := []riscvHWProbePairs{ + {riscv_HWPROBE_KEY_IMA_EXT_0, 0}, + {riscv_HWPROBE_KEY_CPUPERF_0, 0}, + } + + // This call only indicates that extensions are supported if they are implemented on all cores. + if riscvHWProbe(pairs, 0) { + if pairs[0].key != -1 { + v := uint(pairs[0].value) + RISCV64.HasC = isSet(v, riscv_HWPROBE_IMA_C) + RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V) + RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA) + RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB) + RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS) + RISCV64.HasZbc = isSet(v, riscv_HWPROBE_EXT_ZBC) + RISCV64.HasZvbb = isSet(v, riscv_HWPROBE_EXT_ZVBB) + RISCV64.HasZvbc = isSet(v, riscv_HWPROBE_EXT_ZVBC) + RISCV64.HasZvkb = isSet(v, riscv_HWPROBE_EXT_ZVKB) + RISCV64.HasZvkg = isSet(v, riscv_HWPROBE_EXT_ZVKG) + RISCV64.HasZvkt = isSet(v, riscv_HWPROBE_EXT_ZVKT) + // Cryptography shorthand extensions + RISCV64.HasZvkn = isSet(v, riscv_HWPROBE_EXT_ZVKNED) && + isSet(v, riscv_HWPROBE_EXT_ZVKNHB) && RISCV64.HasZvkb && RISCV64.HasZvkt + RISCV64.HasZvknc = RISCV64.HasZvkn && RISCV64.HasZvbc + RISCV64.HasZvkng = RISCV64.HasZvkn && RISCV64.HasZvkg + RISCV64.HasZvks = isSet(v, riscv_HWPROBE_EXT_ZVKSED) && + isSet(v, riscv_HWPROBE_EXT_ZVKSH) && RISCV64.HasZvkb && RISCV64.HasZvkt + RISCV64.HasZvksc = RISCV64.HasZvks && RISCV64.HasZvbc + RISCV64.HasZvksg = RISCV64.HasZvks && RISCV64.HasZvkg + } + if pairs[1].key != -1 { + v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK + RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST + } + } + + // Let's double check with HWCAP if the C extension does not appear to be supported. + // This may happen if we're running on a kernel older than 6.4. + + if !RISCV64.HasC { + RISCV64.HasC = isSet(hwCap, hwcap_RISCV_ISA_C) + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} + +// riscvHWProbe is a simplified version of the generated wrapper function found in +// golang.org/x/sys/unix/zsyscall_linux_riscv64.go. We simplify it by removing the +// cpuCount and cpus parameters which we do not need. We always want to pass 0 for +// these parameters here so the kernel only reports the extensions that are present +// on all cores. +func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool { + var _zero uintptr + var p0 unsafe.Pointer + if len(pairs) > 0 { + p0 = unsafe.Pointer(&pairs[0]) + } else { + p0 = unsafe.Pointer(&_zero) + } + + _, _, e1 := syscall.Syscall6(sys_RISCV_HWPROBE, uintptr(p0), uintptr(len(pairs)), uintptr(0), uintptr(0), uintptr(flags), 0) + return e1 == 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go new file mode 100644 index 00000000..1517ac61 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go @@ -0,0 +1,40 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const ( + // bit mask values from /usr/include/bits/hwcap.h + hwcap_ZARCH = 2 + hwcap_STFLE = 4 + hwcap_MSA = 8 + hwcap_LDISP = 16 + hwcap_EIMM = 32 + hwcap_DFP = 64 + hwcap_ETF3EH = 256 + hwcap_VX = 2048 + hwcap_VXE = 8192 +) + +func initS390Xbase() { + // test HWCAP bit vector + has := func(featureMask uint) bool { + return hwCap&featureMask == featureMask + } + + // mandatory + S390X.HasZARCH = has(hwcap_ZARCH) + + // optional + S390X.HasSTFLE = has(hwcap_STFLE) + S390X.HasLDISP = has(hwcap_LDISP) + S390X.HasEIMM = has(hwcap_EIMM) + S390X.HasETF3EH = has(hwcap_ETF3EH) + S390X.HasDFP = has(hwcap_DFP) + S390X.HasMSA = has(hwcap_MSA) + S390X.HasVX = has(hwcap_VX) + if S390X.HasVX { + S390X.HasVXE = has(hwcap_VXE) + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_loong64.go b/vendor/golang.org/x/sys/cpu/cpu_loong64.go new file mode 100644 index 00000000..8c234b44 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_loong64.go @@ -0,0 +1,62 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 + +package cpu + +const cacheLineSize = 64 + +// Bit fields for CPUCFG registers, Related reference documents: +// https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_cpucfg +const ( + // CPUCFG1 bits + cpucfg1_CRC32 = 1 << 25 + + // CPUCFG2 bits + cpucfg2_LAM_BH = 1 << 27 + cpucfg2_LAMCAS = 1 << 28 + cpucfg2_LLACQ_SCREL = 1 << 29 + cpucfg2_SCQ = 1 << 30 + + // CPUCFG3 bits + cpucfg3_DBAR_HINTS = 1 << 17 +) + +func initOptions() { + options = []option{ + {Name: "lsx", Feature: &Loong64.HasLSX}, + {Name: "lasx", Feature: &Loong64.HasLASX}, + {Name: "crc32", Feature: &Loong64.HasCRC32}, + {Name: "lam_bh", Feature: &Loong64.HasLAM_BH}, + {Name: "lamcas", Feature: &Loong64.HasLAMCAS}, + {Name: "llacq_screl", Feature: &Loong64.HasLLACQ_SCREL}, + {Name: "scq", Feature: &Loong64.HasSCQ}, + {Name: "dbar_hints", Feature: &Loong64.HasDBAR_HINTS}, + } + + // The CPUCFG data on Loong64 only reflects the hardware capabilities, + // not the kernel support status, so features such as LSX and LASX that + // require kernel support cannot be obtained from the CPUCFG data. + // + // These features only require hardware capability support and do not + // require kernel specific support, so they can be obtained directly + // through CPUCFG + cfg1 := get_cpucfg(1) + cfg2 := get_cpucfg(2) + cfg3 := get_cpucfg(3) + + Loong64.HasCRC32 = cfgIsSet(cfg1, cpucfg1_CRC32) + Loong64.HasLAMCAS = cfgIsSet(cfg2, cpucfg2_LAMCAS) + Loong64.HasLAM_BH = cfgIsSet(cfg2, cpucfg2_LAM_BH) + Loong64.HasLLACQ_SCREL = cfgIsSet(cfg2, cpucfg2_LLACQ_SCREL) + Loong64.HasSCQ = cfgIsSet(cfg2, cpucfg2_SCQ) + Loong64.HasDBAR_HINTS = cfgIsSet(cfg3, cpucfg3_DBAR_HINTS) +} + +func get_cpucfg(reg uint32) uint32 + +func cfgIsSet(cfg uint32, val uint32) bool { + return cfg&val != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_loong64.s b/vendor/golang.org/x/sys/cpu/cpu_loong64.s new file mode 100644 index 00000000..71cbaf1c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_loong64.s @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func get_cpucfg(reg uint32) uint32 +TEXT ·get_cpucfg(SB), NOSPLIT|NOFRAME, $0 + MOVW reg+0(FP), R5 + // CPUCFG R5, R4 = 0x00006ca4 + WORD $0x00006ca4 + MOVW R4, ret+8(FP) + RET diff --git a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go new file mode 100644 index 00000000..fedb00cc --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go @@ -0,0 +1,15 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips64 || mips64le + +package cpu + +const cacheLineSize = 32 + +func initOptions() { + options = []option{ + {Name: "msa", Feature: &MIPS64X.HasMSA}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go new file mode 100644 index 00000000..ffb4ec7e --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go @@ -0,0 +1,11 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips || mipsle + +package cpu + +const cacheLineSize = 32 + +func initOptions() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go new file mode 100644 index 00000000..ebfb3fc8 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go @@ -0,0 +1,173 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "syscall" + "unsafe" +) + +// Minimal copy of functionality from x/sys/unix so the cpu package can call +// sysctl without depending on x/sys/unix. + +const ( + _CTL_QUERY = -2 + + _SYSCTL_VERS_1 = 0x1000000 +) + +var _zero uintptr + +func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { + var _p0 unsafe.Pointer + if len(mib) > 0 { + _p0 = unsafe.Pointer(&mib[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + _, _, errno := syscall.Syscall6( + syscall.SYS___SYSCTL, + uintptr(_p0), + uintptr(len(mib)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen)) + if errno != 0 { + return errno + } + return nil +} + +type sysctlNode struct { + Flags uint32 + Num int32 + Name [32]int8 + Ver uint32 + __rsvd uint32 + Un [16]byte + _sysctl_size [8]byte + _sysctl_func [8]byte + _sysctl_parent [8]byte + _sysctl_desc [8]byte +} + +func sysctlNodes(mib []int32) ([]sysctlNode, error) { + var olen uintptr + + // Get a list of all sysctl nodes below the given MIB by performing + // a sysctl for the given MIB with CTL_QUERY appended. + mib = append(mib, _CTL_QUERY) + qnode := sysctlNode{Flags: _SYSCTL_VERS_1} + qp := (*byte)(unsafe.Pointer(&qnode)) + sz := unsafe.Sizeof(qnode) + if err := sysctl(mib, nil, &olen, qp, sz); err != nil { + return nil, err + } + + // Now that we know the size, get the actual nodes. + nodes := make([]sysctlNode, olen/sz) + np := (*byte)(unsafe.Pointer(&nodes[0])) + if err := sysctl(mib, np, &olen, qp, sz); err != nil { + return nil, err + } + + return nodes, nil +} + +func nametomib(name string) ([]int32, error) { + // Split name into components. + var parts []string + last := 0 + for i := 0; i < len(name); i++ { + if name[i] == '.' { + parts = append(parts, name[last:i]) + last = i + 1 + } + } + parts = append(parts, name[last:]) + + mib := []int32{} + // Discover the nodes and construct the MIB OID. + for partno, part := range parts { + nodes, err := sysctlNodes(mib) + if err != nil { + return nil, err + } + for _, node := range nodes { + n := make([]byte, 0) + for i := range node.Name { + if node.Name[i] != 0 { + n = append(n, byte(node.Name[i])) + } + } + if string(n) == part { + mib = append(mib, int32(node.Num)) + break + } + } + if len(mib) != partno+1 { + return nil, err + } + } + + return mib, nil +} + +// aarch64SysctlCPUID is struct aarch64_sysctl_cpu_id from NetBSD's +type aarch64SysctlCPUID struct { + midr uint64 /* Main ID Register */ + revidr uint64 /* Revision ID Register */ + mpidr uint64 /* Multiprocessor Affinity Register */ + aa64dfr0 uint64 /* A64 Debug Feature Register 0 */ + aa64dfr1 uint64 /* A64 Debug Feature Register 1 */ + aa64isar0 uint64 /* A64 Instruction Set Attribute Register 0 */ + aa64isar1 uint64 /* A64 Instruction Set Attribute Register 1 */ + aa64mmfr0 uint64 /* A64 Memory Model Feature Register 0 */ + aa64mmfr1 uint64 /* A64 Memory Model Feature Register 1 */ + aa64mmfr2 uint64 /* A64 Memory Model Feature Register 2 */ + aa64pfr0 uint64 /* A64 Processor Feature Register 0 */ + aa64pfr1 uint64 /* A64 Processor Feature Register 1 */ + aa64zfr0 uint64 /* A64 SVE Feature ID Register 0 */ + mvfr0 uint32 /* Media and VFP Feature Register 0 */ + mvfr1 uint32 /* Media and VFP Feature Register 1 */ + mvfr2 uint32 /* Media and VFP Feature Register 2 */ + pad uint32 + clidr uint64 /* Cache Level ID Register */ + ctr uint64 /* Cache Type Register */ +} + +func sysctlCPUID(name string) (*aarch64SysctlCPUID, error) { + mib, err := nametomib(name) + if err != nil { + return nil, err + } + + out := aarch64SysctlCPUID{} + n := unsafe.Sizeof(out) + _, _, errno := syscall.Syscall6( + syscall.SYS___SYSCTL, + uintptr(unsafe.Pointer(&mib[0])), + uintptr(len(mib)), + uintptr(unsafe.Pointer(&out)), + uintptr(unsafe.Pointer(&n)), + uintptr(0), + uintptr(0)) + if errno != 0 { + return nil, errno + } + return &out, nil +} + +func doinit() { + cpuid, err := sysctlCPUID("machdep.cpu0.cpu_id") + if err != nil { + setMinimalFeatures() + return + } + parseARM64SystemRegisters(cpuid.aa64isar0, cpuid.aa64isar1, cpuid.aa64pfr0) + + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.go new file mode 100644 index 00000000..85b64d5c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.go @@ -0,0 +1,65 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "syscall" + "unsafe" +) + +// Minimal copy of functionality from x/sys/unix so the cpu package can call +// sysctl without depending on x/sys/unix. + +const ( + // From OpenBSD's sys/sysctl.h. + _CTL_MACHDEP = 7 + + // From OpenBSD's machine/cpu.h. + _CPU_ID_AA64ISAR0 = 2 + _CPU_ID_AA64ISAR1 = 3 +) + +// Implemented in the runtime package (runtime/sys_openbsd3.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 + +func sysctl(mib []uint32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { + _, _, errno := syscall_syscall6(libc_sysctl_trampoline_addr, uintptr(unsafe.Pointer(&mib[0])), uintptr(len(mib)), uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)), uintptr(unsafe.Pointer(new)), uintptr(newlen)) + if errno != 0 { + return errno + } + return nil +} + +var libc_sysctl_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_sysctl sysctl "libc.so" + +func sysctlUint64(mib []uint32) (uint64, bool) { + var out uint64 + nout := unsafe.Sizeof(out) + if err := sysctl(mib, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); err != nil { + return 0, false + } + return out, true +} + +func doinit() { + setMinimalFeatures() + + // Get ID_AA64ISAR0 and ID_AA64ISAR1 from sysctl. + isar0, ok := sysctlUint64([]uint32{_CTL_MACHDEP, _CPU_ID_AA64ISAR0}) + if !ok { + return + } + isar1, ok := sysctlUint64([]uint32{_CTL_MACHDEP, _CPU_ID_AA64ISAR1}) + if !ok { + return + } + parseARM64SystemRegisters(isar0, isar1, 0) + + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.s b/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.s new file mode 100644 index 00000000..054ba05d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.s @@ -0,0 +1,11 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctl(SB) + +GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_arm.go b/vendor/golang.org/x/sys/cpu/cpu_other_arm.go new file mode 100644 index 00000000..e9ecf2a4 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_arm.go @@ -0,0 +1,9 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux && arm + +package cpu + +func archInit() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_other_arm64.go new file mode 100644 index 00000000..6c7c5bfd --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_arm64.go @@ -0,0 +1,11 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !darwin && !linux && !netbsd && !openbsd && !windows && arm64 + +package cpu + +func doinit() { + setMinimalFeatures() +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_other_mips64x.go new file mode 100644 index 00000000..5f8f2419 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_mips64x.go @@ -0,0 +1,11 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux && (mips64 || mips64le) + +package cpu + +func archInit() { + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_other_ppc64x.go new file mode 100644 index 00000000..89608fba --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_ppc64x.go @@ -0,0 +1,12 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !aix && !linux && (ppc64 || ppc64le) + +package cpu + +func archInit() { + PPC64.IsPOWER8 = true + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_other_riscv64.go new file mode 100644 index 00000000..5ab87808 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_riscv64.go @@ -0,0 +1,11 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux && riscv64 + +package cpu + +func archInit() { + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_x86.go b/vendor/golang.org/x/sys/cpu/cpu_other_x86.go new file mode 100644 index 00000000..a0fd7e2f --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_x86.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64p32 || (amd64 && (!darwin || !gc)) + +package cpu + +func darwinSupportsAVX512() bool { + panic("only implemented for gc && amd64 && darwin") +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 00000000..c14f12b1 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le + +package cpu + +const cacheLineSize = 128 + +func initOptions() { + options = []option{ + {Name: "darn", Feature: &PPC64.HasDARN}, + {Name: "scv", Feature: &PPC64.HasSCV}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go new file mode 100644 index 00000000..d4e9885f --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build riscv64 + +package cpu + +const cacheLineSize = 64 + +func initOptions() { + options = []option{ + {Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned}, + {Name: "c", Feature: &RISCV64.HasC}, + {Name: "v", Feature: &RISCV64.HasV}, + {Name: "zba", Feature: &RISCV64.HasZba}, + {Name: "zbb", Feature: &RISCV64.HasZbb}, + {Name: "zbs", Feature: &RISCV64.HasZbs}, + {Name: "zbc", Feature: &RISCV64.HasZbc}, + // RISC-V Cryptography Extensions + {Name: "zvbb", Feature: &RISCV64.HasZvbb}, + {Name: "zvbc", Feature: &RISCV64.HasZvbc}, + {Name: "zvkb", Feature: &RISCV64.HasZvkb}, + {Name: "zvkg", Feature: &RISCV64.HasZvkg}, + {Name: "zvkt", Feature: &RISCV64.HasZvkt}, + {Name: "zvkn", Feature: &RISCV64.HasZvkn}, + {Name: "zvknc", Feature: &RISCV64.HasZvknc}, + {Name: "zvkng", Feature: &RISCV64.HasZvkng}, + {Name: "zvks", Feature: &RISCV64.HasZvks}, + {Name: "zvksc", Feature: &RISCV64.HasZvksc}, + {Name: "zvksg", Feature: &RISCV64.HasZvksg}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 00000000..5881b883 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,172 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 + +func initOptions() { + options = []option{ + {Name: "zarch", Feature: &S390X.HasZARCH, Required: true}, + {Name: "stfle", Feature: &S390X.HasSTFLE, Required: true}, + {Name: "ldisp", Feature: &S390X.HasLDISP, Required: true}, + {Name: "eimm", Feature: &S390X.HasEIMM, Required: true}, + {Name: "dfp", Feature: &S390X.HasDFP}, + {Name: "etf3eh", Feature: &S390X.HasETF3EH}, + {Name: "msa", Feature: &S390X.HasMSA}, + {Name: "aes", Feature: &S390X.HasAES}, + {Name: "aescbc", Feature: &S390X.HasAESCBC}, + {Name: "aesctr", Feature: &S390X.HasAESCTR}, + {Name: "aesgcm", Feature: &S390X.HasAESGCM}, + {Name: "ghash", Feature: &S390X.HasGHASH}, + {Name: "sha1", Feature: &S390X.HasSHA1}, + {Name: "sha256", Feature: &S390X.HasSHA256}, + {Name: "sha3", Feature: &S390X.HasSHA3}, + {Name: "sha512", Feature: &S390X.HasSHA512}, + {Name: "vx", Feature: &S390X.HasVX}, + {Name: "vxe", Feature: &S390X.HasVXE}, + } +} + +// bitIsSet reports whether the bit at index is set. The bit index +// is in big endian order, so bit index 0 is the leftmost bit. +func bitIsSet(bits []uint64, index uint) bool { + return bits[index/64]&((1<<63)>>(index%64)) != 0 +} + +// facility is a bit index for the named facility. +type facility uint8 + +const ( + // mandatory facilities + zarch facility = 1 // z architecture mode is active + stflef facility = 7 // store-facility-list-extended + ldisp facility = 18 // long-displacement + eimm facility = 21 // extended-immediate + + // miscellaneous facilities + dfp facility = 42 // decimal-floating-point + etf3eh facility = 30 // extended-translation 3 enhancement + + // cryptography facilities + msa facility = 17 // message-security-assist + msa3 facility = 76 // message-security-assist extension 3 + msa4 facility = 77 // message-security-assist extension 4 + msa5 facility = 57 // message-security-assist extension 5 + msa8 facility = 146 // message-security-assist extension 8 + msa9 facility = 155 // message-security-assist extension 9 + + // vector facilities + vx facility = 129 // vector facility + vxe facility = 135 // vector-enhancements 1 + vxe2 facility = 148 // vector-enhancements 2 +) + +// facilityList contains the result of an STFLE call. +// Bits are numbered in big endian order so the +// leftmost bit (the MSB) is at index 0. +type facilityList struct { + bits [4]uint64 +} + +// Has reports whether the given facilities are present. +func (s *facilityList) Has(fs ...facility) bool { + if len(fs) == 0 { + panic("no facility bits provided") + } + for _, f := range fs { + if !bitIsSet(s.bits[:], uint(f)) { + return false + } + } + return true +} + +// function is the code for the named cryptographic function. +type function uint8 + +const ( + // KM{,A,C,CTR} function codes + aes128 function = 18 // AES-128 + aes192 function = 19 // AES-192 + aes256 function = 20 // AES-256 + + // K{I,L}MD function codes + sha1 function = 1 // SHA-1 + sha256 function = 2 // SHA-256 + sha512 function = 3 // SHA-512 + sha3_224 function = 32 // SHA3-224 + sha3_256 function = 33 // SHA3-256 + sha3_384 function = 34 // SHA3-384 + sha3_512 function = 35 // SHA3-512 + shake128 function = 36 // SHAKE-128 + shake256 function = 37 // SHAKE-256 + + // KLMD function codes + ghash function = 65 // GHASH +) + +// queryResult contains the result of a Query function +// call. Bits are numbered in big endian order so the +// leftmost bit (the MSB) is at index 0. +type queryResult struct { + bits [2]uint64 +} + +// Has reports whether the given functions are present. +func (q *queryResult) Has(fns ...function) bool { + if len(fns) == 0 { + panic("no function codes provided") + } + for _, f := range fns { + if !bitIsSet(q.bits[:], uint(f)) { + return false + } + } + return true +} + +func doinit() { + initS390Xbase() + + // We need implementations of stfle, km and so on + // to detect cryptographic features. + if !haveAsmFunctions() { + return + } + + // optional cryptographic functions + if S390X.HasMSA { + aes := []function{aes128, aes192, aes256} + + // cipher message + km, kmc := kmQuery(), kmcQuery() + S390X.HasAES = km.Has(aes...) + S390X.HasAESCBC = kmc.Has(aes...) + if S390X.HasSTFLE { + facilities := stfle() + if facilities.Has(msa4) { + kmctr := kmctrQuery() + S390X.HasAESCTR = kmctr.Has(aes...) + } + if facilities.Has(msa8) { + kma := kmaQuery() + S390X.HasAESGCM = kma.Has(aes...) + } + } + + // compute message digest + kimd := kimdQuery() // intermediate (no padding) + klmd := klmdQuery() // last (padding) + S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1) + S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256) + S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512) + S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist + sha3 := []function{ + sha3_224, sha3_256, sha3_384, sha3_512, + shake128, shake256, + } + S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...) + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.s b/vendor/golang.org/x/sys/cpu/cpu_s390x.s new file mode 100644 index 00000000..1fb4b701 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.s @@ -0,0 +1,57 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +#include "textflag.h" + +// func stfle() facilityList +TEXT ·stfle(SB), NOSPLIT|NOFRAME, $0-32 + MOVD $ret+0(FP), R1 + MOVD $3, R0 // last doubleword index to store + XC $32, (R1), (R1) // clear 4 doublewords (32 bytes) + WORD $0xb2b01000 // store facility list extended (STFLE) + RET + +// func kmQuery() queryResult +TEXT ·kmQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KM-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xB92E0024 // cipher message (KM) + RET + +// func kmcQuery() queryResult +TEXT ·kmcQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMC-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xB92F0024 // cipher message with chaining (KMC) + RET + +// func kmctrQuery() queryResult +TEXT ·kmctrQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMCTR-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xB92D4024 // cipher message with counter (KMCTR) + RET + +// func kmaQuery() queryResult +TEXT ·kmaQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMA-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xb9296024 // cipher message with authentication (KMA) + RET + +// func kimdQuery() queryResult +TEXT ·kimdQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KIMD-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xB93E0024 // compute intermediate message digest (KIMD) + RET + +// func klmdQuery() queryResult +TEXT ·klmdQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KLMD-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + WORD $0xB93F0024 // compute last message digest (KLMD) + RET diff --git a/vendor/golang.org/x/sys/cpu/cpu_wasm.go b/vendor/golang.org/x/sys/cpu/cpu_wasm.go new file mode 100644 index 00000000..384787ea --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_wasm.go @@ -0,0 +1,17 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build wasm + +package cpu + +// We're compiling the cpu package for an unknown (software-abstracted) CPU. +// Make CacheLinePad an empty struct and hope that the usual struct alignment +// rules are good enough. + +const cacheLineSize = 0 + +func initOptions() {} + +func archInit() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_windows.go b/vendor/golang.org/x/sys/cpu/cpu_windows.go new file mode 100644 index 00000000..99ec8fdf --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_windows.go @@ -0,0 +1,26 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +//go:generate go run golang.org/x/sys/windows/mkwinsyscall -systemdll=false -output zcpu_windows.go cpu_windows.go + +//sys isProcessorFeaturePresent(ProcessorFeature uint32) (ret bool) = kernel32.IsProcessorFeaturePresent + +// The processor features to be tested for IsProcessorFeaturePresent, see +// https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent +const ( + _PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE = 30 + _PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE = 31 + _PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE = 34 + _PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE = 43 + + _PF_ARM_V83_JSCVT_INSTRUCTIONS_AVAILABLE = 44 + _PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE = 45 + _PF_ARM_SVE_INSTRUCTIONS_AVAILABLE = 46 + _PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE = 47 + + _PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE = 64 + _PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE = 65 +) diff --git a/vendor/golang.org/x/sys/cpu/cpu_windows_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_windows_arm64.go new file mode 100644 index 00000000..034732e5 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_windows_arm64.go @@ -0,0 +1,38 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +func doinit() { + // set HasASIMD and HasFP to true as per + // https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#base-requirements + // + // The ARM64 version of Windows always presupposes that it's running on an ARMv8 or later architecture. + // Both floating-point and NEON support are presumed to be present in hardware. + // + ARM64.HasASIMD = true + ARM64.HasFP = true + + if isProcessorFeaturePresent(_PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) { + ARM64.HasAES = true + ARM64.HasPMULL = true + ARM64.HasSHA1 = true + ARM64.HasSHA2 = true + } + ARM64.HasSHA3 = isProcessorFeaturePresent(_PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE) + ARM64.HasCRC32 = isProcessorFeaturePresent(_PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) + ARM64.HasSHA512 = isProcessorFeaturePresent(_PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE) + ARM64.HasATOMICS = isProcessorFeaturePresent(_PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE) + if isProcessorFeaturePresent(_PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) { + ARM64.HasASIMDDP = true + ARM64.HasASIMDRDM = true + } + if isProcessorFeaturePresent(_PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE) { + ARM64.HasLRCPC = true + ARM64.HasSM3 = true + } + ARM64.HasSVE = isProcessorFeaturePresent(_PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) + ARM64.HasSVE2 = isProcessorFeaturePresent(_PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) + ARM64.HasJSCVT = isProcessorFeaturePresent(_PF_ARM_V83_JSCVT_INSTRUCTIONS_AVAILABLE) +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go new file mode 100644 index 00000000..f5723d4f --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -0,0 +1,236 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 || amd64p32 + +package cpu + +import "runtime" + +const cacheLineSize = 64 + +func initOptions() { + options = []option{ + {Name: "adx", Feature: &X86.HasADX}, + {Name: "aes", Feature: &X86.HasAES}, + {Name: "avx", Feature: &X86.HasAVX}, + {Name: "avx2", Feature: &X86.HasAVX2}, + {Name: "avx512", Feature: &X86.HasAVX512}, + {Name: "avx512f", Feature: &X86.HasAVX512F}, + {Name: "avx512cd", Feature: &X86.HasAVX512CD}, + {Name: "avx512er", Feature: &X86.HasAVX512ER}, + {Name: "avx512pf", Feature: &X86.HasAVX512PF}, + {Name: "avx512vl", Feature: &X86.HasAVX512VL}, + {Name: "avx512bw", Feature: &X86.HasAVX512BW}, + {Name: "avx512dq", Feature: &X86.HasAVX512DQ}, + {Name: "avx512ifma", Feature: &X86.HasAVX512IFMA}, + {Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI}, + {Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW}, + {Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS}, + {Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ}, + {Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ}, + {Name: "avx512vnni", Feature: &X86.HasAVX512VNNI}, + {Name: "avx512gfni", Feature: &X86.HasAVX512GFNI}, + {Name: "avx512vaes", Feature: &X86.HasAVX512VAES}, + {Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2}, + {Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG}, + {Name: "avx512bf16", Feature: &X86.HasAVX512BF16}, + {Name: "amxtile", Feature: &X86.HasAMXTile}, + {Name: "amxint8", Feature: &X86.HasAMXInt8}, + {Name: "amxbf16", Feature: &X86.HasAMXBF16}, + {Name: "bmi1", Feature: &X86.HasBMI1}, + {Name: "bmi2", Feature: &X86.HasBMI2}, + {Name: "cx16", Feature: &X86.HasCX16}, + {Name: "erms", Feature: &X86.HasERMS}, + {Name: "fma", Feature: &X86.HasFMA}, + {Name: "osxsave", Feature: &X86.HasOSXSAVE}, + {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, + {Name: "popcnt", Feature: &X86.HasPOPCNT}, + {Name: "rdrand", Feature: &X86.HasRDRAND}, + {Name: "rdseed", Feature: &X86.HasRDSEED}, + {Name: "sse3", Feature: &X86.HasSSE3}, + {Name: "sse41", Feature: &X86.HasSSE41}, + {Name: "sse42", Feature: &X86.HasSSE42}, + {Name: "ssse3", Feature: &X86.HasSSSE3}, + {Name: "avxifma", Feature: &X86.HasAVXIFMA}, + {Name: "avxvnni", Feature: &X86.HasAVXVNNI}, + {Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8}, + + // These capabilities should always be enabled on amd64: + {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"}, + } +} + +func archInit() { + + // From internal/cpu + const ( + // eax bits + cpuid_AVXVNNI = 1 << 4 + + // ecx bits + cpuid_SSE3 = 1 << 0 + cpuid_PCLMULQDQ = 1 << 1 + cpuid_AVX512VBMI = 1 << 1 + cpuid_AVX512VBMI2 = 1 << 6 + cpuid_SSSE3 = 1 << 9 + cpuid_AVX512GFNI = 1 << 8 + cpuid_AVX512VAES = 1 << 9 + cpuid_AVX512VNNI = 1 << 11 + cpuid_AVX512BITALG = 1 << 12 + cpuid_FMA = 1 << 12 + cpuid_AVX512VPOPCNTDQ = 1 << 14 + cpuid_SSE41 = 1 << 19 + cpuid_SSE42 = 1 << 20 + cpuid_POPCNT = 1 << 23 + cpuid_AES = 1 << 25 + cpuid_OSXSAVE = 1 << 27 + cpuid_AVX = 1 << 28 + + // "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0 + cpuid_BMI1 = 1 << 3 + cpuid_AVX2 = 1 << 5 + cpuid_BMI2 = 1 << 8 + cpuid_ERMS = 1 << 9 + cpuid_AVX512F = 1 << 16 + cpuid_AVX512DQ = 1 << 17 + cpuid_ADX = 1 << 19 + cpuid_AVX512CD = 1 << 28 + cpuid_SHA = 1 << 29 + cpuid_AVX512BW = 1 << 30 + cpuid_AVX512VL = 1 << 31 + + // "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0 + cpuid_AVX512_VBMI = 1 << 1 + cpuid_AVX512_VBMI2 = 1 << 6 + cpuid_GFNI = 1 << 8 + cpuid_AVX512VPCLMULQDQ = 1 << 10 + cpuid_AVX512_BITALG = 1 << 12 + + // edx bits + cpuid_FSRM = 1 << 4 + // edx bits for CPUID 0x80000001 + cpuid_RDTSCP = 1 << 27 + ) + // Additional constants not in internal/cpu + const ( + // eax=1: edx + cpuid_SSE2 = 1 << 26 + // eax=1: ecx + cpuid_CX16 = 1 << 13 + cpuid_RDRAND = 1 << 30 + // eax=7,ecx=0: ebx + cpuid_RDSEED = 1 << 18 + cpuid_AVX512IFMA = 1 << 21 + cpuid_AVX512PF = 1 << 26 + cpuid_AVX512ER = 1 << 27 + // eax=7,ecx=0: edx + cpuid_AVX5124VNNIW = 1 << 2 + cpuid_AVX5124FMAPS = 1 << 3 + cpuid_AMXBF16 = 1 << 22 + cpuid_AMXTile = 1 << 24 + cpuid_AMXInt8 = 1 << 25 + // eax=7,ecx=1: eax + cpuid_AVX512BF16 = 1 << 5 + cpuid_AVXIFMA = 1 << 23 + // eax=7,ecx=1: edx + cpuid_AVXVNNIInt8 = 1 << 4 + ) + + Initialized = true + + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(edx1, cpuid_SSE2) + + X86.HasSSE3 = isSet(ecx1, cpuid_SSE3) + X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ) + X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3) + X86.HasFMA = isSet(ecx1, cpuid_FMA) + X86.HasCX16 = isSet(ecx1, cpuid_CX16) + X86.HasSSE41 = isSet(ecx1, cpuid_SSE41) + X86.HasSSE42 = isSet(ecx1, cpuid_SSE42) + X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT) + X86.HasAES = isSet(ecx1, cpuid_AES) + X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE) + X86.HasRDRAND = isSet(ecx1, cpuid_RDRAND) + + var osSupportsAVX, osSupportsAVX512 bool + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2) + + if runtime.GOOS == "darwin" { + // Darwin requires special AVX512 checks, see cpu_darwin_x86.go + osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512() + } else { + // Check if OPMASK and ZMM registers have OS support. + osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7) + } + } + + X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX + + if maxID < 7 { + return + } + + eax7, ebx7, ecx7, edx7 := cpuid(7, 0) + X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) + X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX + X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) + X86.HasERMS = isSet(ebx7, cpuid_ERMS) + X86.HasRDSEED = isSet(ebx7, cpuid_RDSEED) + X86.HasADX = isSet(ebx7, cpuid_ADX) + + X86.HasAVX512 = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 // Because avx-512 foundation is the core required extension + if X86.HasAVX512 { + X86.HasAVX512F = true + X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD) + X86.HasAVX512ER = isSet(ebx7, cpuid_AVX512ER) + X86.HasAVX512PF = isSet(ebx7, cpuid_AVX512PF) + X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) + X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW) + X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ) + X86.HasAVX512IFMA = isSet(ebx7, cpuid_AVX512IFMA) + X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI) + X86.HasAVX5124VNNIW = isSet(edx7, cpuid_AVX5124VNNIW) + X86.HasAVX5124FMAPS = isSet(edx7, cpuid_AVX5124FMAPS) + X86.HasAVX512VPOPCNTDQ = isSet(ecx7, cpuid_AVX512VPOPCNTDQ) + X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ) + X86.HasAVX512VNNI = isSet(ecx7, cpuid_AVX512VNNI) + X86.HasAVX512GFNI = isSet(ecx7, cpuid_AVX512GFNI) + X86.HasAVX512VAES = isSet(ecx7, cpuid_AVX512VAES) + X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512VBMI2) + X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512BITALG) + } + + X86.HasAMXTile = isSet(edx7, cpuid_AMXTile) + X86.HasAMXInt8 = isSet(edx7, cpuid_AMXInt8) + X86.HasAMXBF16 = isSet(edx7, cpuid_AMXBF16) + + // These features depend on the second level of extended features. + if eax7 >= 1 { + eax71, _, _, edx71 := cpuid(7, 1) + if X86.HasAVX512 { + X86.HasAVX512BF16 = isSet(eax71, cpuid_AVX512BF16) + } + if X86.HasAVX { + X86.HasAVXIFMA = isSet(eax71, cpuid_AVXIFMA) + X86.HasAVXVNNI = isSet(eax71, cpuid_AVXVNNI) + X86.HasAVXVNNIInt8 = isSet(edx71, cpuid_AVXVNNIInt8) + } + } +} + +func isSet(hwc uint32, value uint32) bool { + return hwc&value != 0 +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_zos.go b/vendor/golang.org/x/sys/cpu/cpu_zos.go new file mode 100644 index 00000000..5f54683a --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_zos.go @@ -0,0 +1,10 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +func archInit() { + doinit() + Initialized = true +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_zos_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_zos_s390x.go new file mode 100644 index 00000000..ccb1b708 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_zos_s390x.go @@ -0,0 +1,25 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +func initS390Xbase() { + // get the facilities list + facilities := stfle() + + // mandatory + S390X.HasZARCH = facilities.Has(zarch) + S390X.HasSTFLE = facilities.Has(stflef) + S390X.HasLDISP = facilities.Has(ldisp) + S390X.HasEIMM = facilities.Has(eimm) + + // optional + S390X.HasETF3EH = facilities.Has(etf3eh) + S390X.HasDFP = facilities.Has(dfp) + S390X.HasMSA = facilities.Has(msa) + S390X.HasVX = facilities.Has(vx) + if S390X.HasVX { + S390X.HasVXE = facilities.Has(vxe) + } +} diff --git a/vendor/golang.org/x/sys/cpu/endian_big.go b/vendor/golang.org/x/sys/cpu/endian_big.go new file mode 100644 index 00000000..7fe04b0a --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/endian_big.go @@ -0,0 +1,10 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build armbe || arm64be || m68k || mips || mips64 || mips64p32 || ppc || ppc64 || s390 || s390x || shbe || sparc || sparc64 + +package cpu + +// IsBigEndian records whether the GOARCH's byte order is big endian. +const IsBigEndian = true diff --git a/vendor/golang.org/x/sys/cpu/endian_little.go b/vendor/golang.org/x/sys/cpu/endian_little.go new file mode 100644 index 00000000..48eccc4c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/endian_little.go @@ -0,0 +1,10 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 || amd64p32 || alpha || arm || arm64 || loong64 || mipsle || mips64le || mips64p32le || nios2 || ppc64le || riscv || riscv64 || sh || wasm + +package cpu + +// IsBigEndian records whether the GOARCH's byte order is big endian. +const IsBigEndian = false diff --git a/vendor/golang.org/x/sys/cpu/hwcap_linux.go b/vendor/golang.org/x/sys/cpu/hwcap_linux.go new file mode 100644 index 00000000..34e49f95 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/hwcap_linux.go @@ -0,0 +1,71 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "os" +) + +const ( + _AT_HWCAP = 16 + _AT_HWCAP2 = 26 + + procAuxv = "/proc/self/auxv" + + uintSize = int(32 << (^uint(0) >> 63)) +) + +// For those platforms don't have a 'cpuid' equivalent we use HWCAP/HWCAP2 +// These are initialized in cpu_$GOARCH.go +// and should not be changed after they are initialized. +var hwCap uint +var hwCap2 uint + +func readHWCAP() error { + // For Go 1.21+, get auxv from the Go runtime. + if a := getAuxv(); len(a) > 0 { + for len(a) >= 2 { + tag, val := a[0], uint(a[1]) + a = a[2:] + switch tag { + case _AT_HWCAP: + hwCap = val + case _AT_HWCAP2: + hwCap2 = val + } + } + return nil + } + + buf, err := os.ReadFile(procAuxv) + if err != nil { + // e.g. on android /proc/self/auxv is not accessible, so silently + // ignore the error and leave Initialized = false. On some + // architectures (e.g. arm64) doinit() implements a fallback + // readout and will set Initialized = true again. + return err + } + bo := hostByteOrder() + for len(buf) >= 2*(uintSize/8) { + var tag, val uint + switch uintSize { + case 32: + tag = uint(bo.Uint32(buf[0:])) + val = uint(bo.Uint32(buf[4:])) + buf = buf[8:] + case 64: + tag = uint(bo.Uint64(buf[0:])) + val = uint(bo.Uint64(buf[8:])) + buf = buf[16:] + } + switch tag { + case _AT_HWCAP: + hwCap = val + case _AT_HWCAP2: + hwCap2 = val + } + } + return nil +} diff --git a/vendor/golang.org/x/sys/cpu/parse.go b/vendor/golang.org/x/sys/cpu/parse.go new file mode 100644 index 00000000..12a99af5 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/parse.go @@ -0,0 +1,55 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import "strconv" + +// parseRelease parses a dot-separated version number from the prefix +// of rel. It returns ok=true only if at least the major and minor +// components were successfully parsed; the patch component is +// best-effort. Trailing vendor or build suffixes such as +// "-generic", "+", "_hi3535", or "-rc1" are ignored. +// +// This is a copy of the Go runtime's parseRelease from +// https://golang.org/cl/209597, updated in https://golang.org/cl/781800. +func parseRelease(rel string) (major, minor, patch int, ok bool) { + // next consumes a run of decimal digits from the front of rel, + // returning the parsed value. If the digits are followed by a + // '.', it is consumed and more is set so the caller knows to + // parse another component; otherwise scanning terminates and + // the rest of rel is discarded. + next := func() (n int, more, ok bool) { + i := 0 + for i < len(rel) && rel[i] >= '0' && rel[i] <= '9' { + i++ + } + if i == 0 { + return 0, false, false + } + n, err := strconv.Atoi(rel[:i]) + if err != nil { + return 0, false, false + } + if i < len(rel) && rel[i] == '.' { + rel = rel[i+1:] + return n, true, true + } + rel = "" + return n, false, true + } + + var more bool + if major, more, ok = next(); !ok || !more { + return 0, 0, 0, false + } + if minor, more, ok = next(); !ok { + return 0, 0, 0, false + } + if !more { + return major, minor, 0, true + } + patch, _, _ = next() + return major, minor, patch, true +} diff --git a/vendor/golang.org/x/sys/cpu/proc_cpuinfo_linux.go b/vendor/golang.org/x/sys/cpu/proc_cpuinfo_linux.go new file mode 100644 index 00000000..4cd64c70 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/proc_cpuinfo_linux.go @@ -0,0 +1,53 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && arm64 + +package cpu + +import ( + "errors" + "io" + "os" + "strings" +) + +func readLinuxProcCPUInfo() error { + f, err := os.Open("/proc/cpuinfo") + if err != nil { + return err + } + defer f.Close() + + var buf [1 << 10]byte // enough for first CPU + n, err := io.ReadFull(f, buf[:]) + if err != nil && err != io.ErrUnexpectedEOF { + return err + } + in := string(buf[:n]) + const features = "\nFeatures : " + i := strings.Index(in, features) + if i == -1 { + return errors.New("no CPU features found") + } + in = in[i+len(features):] + if i := strings.Index(in, "\n"); i != -1 { + in = in[:i] + } + m := map[string]*bool{} + + initOptions() // need it early here; it's harmless to call twice + for _, o := range options { + m[o.Name] = o.Feature + } + // The EVTSTRM field has alias "evstrm" in Go, but Linux calls it "evtstrm". + m["evtstrm"] = &ARM64.HasEVTSTRM + + for _, f := range strings.Fields(in) { + if p, ok := m[f]; ok { + *p = true + } + } + return nil +} diff --git a/vendor/golang.org/x/sys/cpu/runtime_auxv.go b/vendor/golang.org/x/sys/cpu/runtime_auxv.go new file mode 100644 index 00000000..5f92ac9a --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/runtime_auxv.go @@ -0,0 +1,16 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +// getAuxvFn is non-nil on Go 1.21+ (via runtime_auxv_go121.go init) +// on platforms that use auxv. +var getAuxvFn func() []uintptr + +func getAuxv() []uintptr { + if getAuxvFn == nil { + return nil + } + return getAuxvFn() +} diff --git a/vendor/golang.org/x/sys/cpu/runtime_auxv_go121.go b/vendor/golang.org/x/sys/cpu/runtime_auxv_go121.go new file mode 100644 index 00000000..4c9788ea --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/runtime_auxv_go121.go @@ -0,0 +1,18 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package cpu + +import ( + _ "unsafe" // for linkname +) + +//go:linkname runtime_getAuxv runtime.getAuxv +func runtime_getAuxv() []uintptr + +func init() { + getAuxvFn = runtime_getAuxv +} diff --git a/vendor/golang.org/x/sys/cpu/syscall_aix_gccgo.go b/vendor/golang.org/x/sys/cpu/syscall_aix_gccgo.go new file mode 100644 index 00000000..1b9ccb09 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/syscall_aix_gccgo.go @@ -0,0 +1,26 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Recreate a getsystemcfg syscall handler instead of +// using the one provided by x/sys/unix to avoid having +// the dependency between them. (See golang.org/issue/32102) +// Moreover, this file will be used during the building of +// gccgo's libgo and thus must not used a CGo method. + +//go:build aix && gccgo + +package cpu + +import ( + "syscall" +) + +//extern getsystemcfg +func gccgoGetsystemcfg(label uint32) (r uint64) + +func callgetsystemcfg(label int) (r1 uintptr, e1 syscall.Errno) { + r1 = uintptr(gccgoGetsystemcfg(uint32(label))) + e1 = syscall.GetErrno() + return +} diff --git a/vendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go b/vendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go new file mode 100644 index 00000000..e8b6cdbe --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go @@ -0,0 +1,35 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimal copy of x/sys/unix so the cpu package can make a +// system call on AIX without depending on x/sys/unix. +// (See golang.org/issue/32102) + +//go:build aix && ppc64 && gc + +package cpu + +import ( + "syscall" + "unsafe" +) + +//go:cgo_import_dynamic libc_getsystemcfg getsystemcfg "libc.a/shr_64.o" + +//go:linkname libc_getsystemcfg libc_getsystemcfg + +type syscallFunc uintptr + +var libc_getsystemcfg syscallFunc + +type errno = syscall.Errno + +// Implemented in runtime/syscall_aix.go. +func rawSyscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err errno) +func syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err errno) + +func callgetsystemcfg(label int) (r1 uintptr, e1 errno) { + r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_getsystemcfg)), 1, uintptr(label), 0, 0, 0, 0, 0) + return +} diff --git a/vendor/golang.org/x/sys/cpu/syscall_darwin_arm64_gc.go b/vendor/golang.org/x/sys/cpu/syscall_darwin_arm64_gc.go new file mode 100644 index 00000000..7b4e67ff --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/syscall_darwin_arm64_gc.go @@ -0,0 +1,54 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimal copy from internal/cpu and runtime to make sysctl calls. + +//go:build darwin && arm64 && gc + +package cpu + +import ( + "syscall" + "unsafe" +) + +type Errno = syscall.Errno + +// adapted from internal/cpu/cpu_arm64_darwin.go +func darwinSysctlEnabled(name []byte) bool { + out := int32(0) + nout := unsafe.Sizeof(out) + if ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); ret != nil { + return false + } + return out > 0 +} + +//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" + +var libc_sysctlbyname_trampoline_addr uintptr + +// adapted from runtime/sys_darwin.go in the pattern of sysctl() above, as defined in x/sys/unix +func sysctlbyname(name *byte, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + if _, _, err := syscall_syscall6( + libc_sysctlbyname_trampoline_addr, + uintptr(unsafe.Pointer(name)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + 0, + ); err != 0 { + return err + } + + return nil +} + +//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib" + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 diff --git a/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go b/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go new file mode 100644 index 00000000..4d0888b0 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go @@ -0,0 +1,98 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimal copy of x/sys/unix so the cpu package can make a +// system call on Darwin without depending on x/sys/unix. + +//go:build darwin && amd64 && gc + +package cpu + +import ( + "syscall" + "unsafe" +) + +type _C_int int32 + +// adapted from unix.Uname() at x/sys/unix/syscall_darwin.go L419 +func darwinOSRelease(release *[256]byte) error { + // from x/sys/unix/zerrors_openbsd_amd64.go + const ( + CTL_KERN = 0x1 + KERN_OSRELEASE = 0x2 + ) + + mib := []_C_int{CTL_KERN, KERN_OSRELEASE} + n := unsafe.Sizeof(*release) + + return sysctl(mib, &release[0], &n, nil, 0) +} + +type Errno = syscall.Errno + +var _zero uintptr // Single-word zero for use when we need a valid pointer to 0 bytes. + +// from x/sys/unix/zsyscall_darwin_amd64.go L791-807 +func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + var _p0 unsafe.Pointer + if len(mib) > 0 { + _p0 = unsafe.Pointer(&mib[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + if _, _, err := syscall_syscall6( + libc_sysctl_trampoline_addr, + uintptr(_p0), + uintptr(len(mib)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + ); err != 0 { + return err + } + + return nil +} + +var libc_sysctl_trampoline_addr uintptr + +// adapted from internal/cpu/cpu_arm64_darwin.go +func darwinSysctlEnabled(name []byte) bool { + out := int32(0) + nout := unsafe.Sizeof(out) + if ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); ret != nil { + return false + } + return out > 0 +} + +//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" + +var libc_sysctlbyname_trampoline_addr uintptr + +// adapted from runtime/sys_darwin.go in the pattern of sysctl() above, as defined in x/sys/unix +func sysctlbyname(name *byte, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + if _, _, err := syscall_syscall6( + libc_sysctlbyname_trampoline_addr, + uintptr(unsafe.Pointer(name)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + 0, + ); err != 0 { + return err + } + + return nil +} + +//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib" + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 diff --git a/vendor/golang.org/x/sys/cpu/zcpu_windows.go b/vendor/golang.org/x/sys/cpu/zcpu_windows.go new file mode 100644 index 00000000..6411a7a7 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/zcpu_windows.go @@ -0,0 +1,48 @@ +// Code generated by 'go generate'; DO NOT EDIT. + +package cpu + +import ( + "syscall" + "unsafe" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) + errERROR_EINVAL error = syscall.EINVAL +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return errERROR_EINVAL + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + + procIsProcessorFeaturePresent = modkernel32.NewProc("IsProcessorFeaturePresent") +) + +func isProcessorFeaturePresent(ProcessorFeature uint32) (ret bool) { + r0, _, _ := syscall.SyscallN(procIsProcessorFeaturePresent.Addr(), uintptr(ProcessorFeature)) + ret = r0 != 0 + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c35529d6..0eeea33c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -113,7 +113,7 @@ github.com/pelletier/go-toml/v2/unstable # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors -# github.com/redis/go-redis/v9 v9.21.0 +# github.com/redis/go-redis/v9 v9.22.0 ## explicit; go 1.24 github.com/redis/go-redis/v9 github.com/redis/go-redis/v9/auth @@ -199,6 +199,7 @@ go.uber.org/ratelimit go.yaml.in/yaml/v3 # golang.org/x/sys v0.47.0 ## explicit; go 1.25.0 +golang.org/x/sys/cpu golang.org/x/sys/execabs golang.org/x/sys/unix golang.org/x/sys/windows