From 417b6799f66a4b16a7d744b120826cf064a4ae29 Mon Sep 17 00:00:00 2001 From: "dev.bo" <129651996+devdotbo@users.noreply.github.com> Date: Fri, 12 Dec 2025 21:46:04 +0400 Subject: [PATCH 1/6] fix(docker): resolve zallet permission error by relocating config mounts (#11) Fixes #9 The zallet container was failing with "Permission denied (os error 13)" due to overlapping Docker mounts. Config files were bind-mounted inside the data volume at /var/lib/zallet/, causing Docker to create them with root ownership while the container runs as UID 65532. This commit resolves the issue by: - Mounting config files to /etc/zallet/ instead of /var/lib/zallet/ - Adding --config flag to zallet command to specify config location - Using absolute path for encryption_identity in zallet.toml The fix works on both mainnet and testnet as it addresses Docker configuration, not network-specific settings. Changes: - docker-compose.yml: Updated zallet command and volume mount paths - config/zallet.toml: Changed encryption_identity to absolute path Tested on testnet - zallet now starts successfully without permission errors. --- config/zallet.toml | 2 +- docker-compose.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/zallet.toml b/config/zallet.toml index 4792c45..1d35a63 100644 --- a/config/zallet.toml +++ b/config/zallet.toml @@ -41,7 +41,7 @@ validator_cookie_path = "/var/run/auth/.cookie" [keystore] # Age encryption identity file (mounted from ./config/zallet_identity.txt) -encryption_identity = "identity.txt" +encryption_identity = "/etc/zallet/identity.txt" [note_management] # Note management - using defaults diff --git a/docker-compose.yml b/docker-compose.yml index a3db421..e3ff24b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,7 +101,7 @@ services: container_name: z3_zallet restart: unless-stopped user: "65532:65532" - command: ["--datadir", "/var/lib/zallet", "start"] + command: ["--datadir", "/var/lib/zallet", "--config", "/etc/zallet/zallet.toml", "start"] depends_on: zebra: condition: service_healthy @@ -116,9 +116,9 @@ services: - ${Z3_ZALLET_DATA_PATH}:/var/lib/zallet # Cookie authentication for Zebra access - ${Z3_COOKIE_PATH}:${COOKIE_AUTH_FILE_DIR}:ro - # Configuration files - - ./config/zallet.toml:/var/lib/zallet/zallet.toml:ro - - ./config/zallet_identity.txt:/var/lib/zallet/identity.txt:ro + # Configuration files (mounted outside datadir to avoid permission conflicts) + - ./config/zallet.toml:/etc/zallet/zallet.toml:ro + - ./config/zallet_identity.txt:/etc/zallet/identity.txt:ro ports: - "${ZALLET_HOST_RPC_PORT}:${ZALLET_RPC_PORT}" networks: From 64bb4e02f68891c7a97dbaa7f56a685fc6b94241 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 29 Jan 2026 15:53:50 +0000 Subject: [PATCH 2/6] feat(docker): update to latest zaino and official zallet image (#12) * feat(docker): update to latest zaino and official zallet image Update submodules and Docker configuration: - Zaino: use GHCR image sha-417b679 (built from zingolabs/zaino:dev) - Zallet: v0.1.0-alpha.3 (electriccoinco/zallet official image) - Zebra: v3.1.0 (unchanged) Add custom entrypoint for Zaino to resolve Docker DNS hostnames to IP addresses, required because ValidatorConfig uses SocketAddr type which only accepts IP:port format, not hostname:port. NOTE: The entrypoint workaround can be removed once zaino PR #784 is merged and a new image is built: https://github.com/zingolabs/zaino/pull/784 New files: - config/zaino/docker-entrypoint.sh (temporary workaround) - config/zaino/zindexer.toml * refactor(docker): remove zaino DNS workaround and update submodules Remove the docker-entrypoint.sh workaround for Zaino hostname resolution now that upstream PR #784 adds native support for hostname:port format. Changes: - Remove config/zaino/docker-entrypoint.sh (no longer needed) - Simplify config/zaino/zindexer.toml to minimal config-rs requirement - Update docker-compose.yml to use native hostname resolution - Remove env_file from zaino service (conflicts with config-rs) - Update submodules to latest versions: - zaino: 49e5241d (config-rs migration + hostname support) - zebra: e04845f3e - zcashd: cfcfcd93b (v6.11.0) * chore(docker): update zaino image to sha-1871eba --- .env | 4 ++-- .gitignore | 4 ++++ config/zaino/zindexer.toml | 1 + config/zallet.toml | 6 ++---- docker-compose.yml | 42 ++++++++++++++------------------------ zaino | 2 +- zallet | 2 +- zcashd | 2 +- zebra | 2 +- 9 files changed, 28 insertions(+), 37 deletions(-) create mode 100644 config/zaino/zindexer.toml diff --git a/.env b/.env index 058b1eb..a17fe1c 100644 --- a/.env +++ b/.env @@ -72,8 +72,8 @@ Z3_ZALLET_DATA_PATH=zallet_data # ============================================================================= # Shared variables used by multiple services, mapped in docker-compose.yml: # NETWORK_NAME → ZEBRA_NETWORK__NETWORK, ZAINO_NETWORK -# ENABLE_COOKIE_AUTH → ZEBRA_RPC__ENABLE_COOKIE_AUTH, ZAINO_VALIDATOR_COOKIE_AUTH -# COOKIE_AUTH_FILE_DIR → ZEBRA_RPC__COOKIE_DIR, ZAINO_VALIDATOR_COOKIE_PATH +# ENABLE_COOKIE_AUTH → ZEBRA_RPC__ENABLE_COOKIE_AUTH +# COOKIE_AUTH_FILE_DIR → ZEBRA_RPC__COOKIE_DIR, ZAINO_VALIDATOR_SETTINGS__VALIDATOR_COOKIE_PATH # Network name for all services (e.g., Mainnet, Testnet, Regtest) NETWORK_NAME=Mainnet diff --git a/.gitignore b/.gitignore index bd498cf..5a50148 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,9 @@ config/tls/* # Then un-ignore the .gitkeep file within 'tls' !config/tls/.gitkeep +# Zaino config (required by config-rs) +!config/zaino/ +!config/zaino/zindexer.toml + # Un-ignore .gitkeep directly under config !config/.gitkeep diff --git a/config/zaino/zindexer.toml b/config/zaino/zindexer.toml new file mode 100644 index 0000000..d4c0bc7 --- /dev/null +++ b/config/zaino/zindexer.toml @@ -0,0 +1 @@ +# Minimal Zaino config - most settings come from environment variables diff --git a/config/zallet.toml b/config/zallet.toml index 1d35a63..7c0f338 100644 --- a/config/zallet.toml +++ b/config/zallet.toml @@ -21,7 +21,7 @@ network = "main" # External settings - using defaults [features] -as_of_version = "0.1.0-alpha.1" +as_of_version = "0.1.0-alpha.3" [features.deprecated] # No deprecated features enabled @@ -34,9 +34,7 @@ as_of_version = "0.1.0-alpha.1" # to fetch blockchain data. The validator_address MUST point to Zebra (not the # standalone Zaino service). Service name 'zebra' and port from Z3_ZEBRA_RPC_PORT in .env validator_address = "zebra:18232" - -# Cookie authentication (matches Zebra's ENABLE_COOKIE_AUTH=true) -validator_cookie_auth = true +# Cookie authentication path (matches Zebra's ENABLE_COOKIE_AUTH=true) validator_cookie_path = "/var/run/auth/.cookie" [keystore] diff --git a/docker-compose.yml b/docker-compose.yml index e3ff24b..2af5b9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,41 +36,37 @@ services: start_period: 90s zaino: - # No pre-built image available for this commit - must build locally - # Built from zingolabs/zaino:dev - commit 66b3199f (before config restructure) - # Run 'docker compose build zaino' to build from ./zaino submodule - image: ghcr.io/zcashfoundation/zaino:sha-934f857 + # No official image from zingolabs yet - built from zingolabs/zaino:dev + image: ghcr.io/zcashfoundation/zaino:sha-1871eba platform: ${DOCKER_PLATFORM:-linux/amd64} build: context: ./zaino dockerfile: Dockerfile container_name: z3_zaino restart: unless-stopped + command: ["zainod", "--config", "/etc/zaino/zindexer.toml"] depends_on: zebra: condition: service_healthy - env_file: - - ./.env environment: - RUST_LOG=${ZAINO_RUST_LOG} - RUST_BACKTRACE=${ZAINO_RUST_BACKTRACE} - ZAINO_NETWORK=${NETWORK_NAME} - # Zebra connection and authentication - - ZAINO_VALIDATOR_LISTEN_ADDRESS=zebra:${Z3_ZEBRA_RPC_PORT} - - ZAINO_VALIDATOR_COOKIE_AUTH=${ENABLE_COOKIE_AUTH} - - ZAINO_VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie - # Zaino RPC services - - ZAINO_GRPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_GRPC_PORT} - - ZAINO_JSON_RPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_JSON_RPC_PORT} + - ZAINO_VALIDATOR_SETTINGS__VALIDATOR_JSONRPC_LISTEN_ADDRESS=zebra:${Z3_ZEBRA_RPC_PORT} + - ZAINO_VALIDATOR_SETTINGS__VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie + # gRPC server + - ZAINO_GRPC_SETTINGS__LISTEN_ADDRESS=0.0.0.0:${ZAINO_GRPC_PORT} + # JSON-RPC server + - ZAINO_JSON_SERVER_SETTINGS__JSON_RPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_JSON_RPC_PORT} # TLS configuration - - ZAINO_GRPC_TLS=${ZAINO_GRPC_TLS_ENABLE} - - ZAINO_TLS_CERT_PATH=${ZAINO_GRPC_TLS_CERT_PATH} - - ZAINO_TLS_KEY_PATH=${ZAINO_GRPC_TLS_KEY_PATH} + - ZAINO_GRPC_SETTINGS__TLS__CERT_PATH=${ZAINO_GRPC_TLS_CERT_PATH} + - ZAINO_GRPC_SETTINGS__TLS__KEY_PATH=${ZAINO_GRPC_TLS_KEY_PATH} volumes: # Indexer state (defaults to named volume 'zaino_data') - ${Z3_ZAINO_DATA_PATH}:/home/zaino/.cache/zaino # Cookie authentication - ${Z3_COOKIE_PATH}:${COOKIE_AUTH_FILE_DIR}:ro + - ./config/zaino/zindexer.toml:/etc/zaino/zindexer.toml:ro configs: - source: zaino_tls_cert target: ${ZAINO_GRPC_TLS_CERT_PATH} @@ -90,17 +86,11 @@ services: start_period: 60s zallet: - # No pre-built image available - must build locally - # Built from zcash/wallet commit 60b0235 (uses zaino 13919816, before config restructure) - # Run 'docker compose build zallet' to build from ./zallet submodule - image: z3-zallet:local + image: electriccoinco/zallet:v0.1.0-alpha.3 platform: ${DOCKER_PLATFORM:-linux/amd64} - build: - context: ./zallet - dockerfile: Dockerfile container_name: z3_zallet restart: unless-stopped - user: "65532:65532" + user: "1000:1000" command: ["--datadir", "/var/lib/zallet", "--config", "/etc/zallet/zallet.toml", "start"] depends_on: zebra: @@ -123,9 +113,7 @@ services: - "${ZALLET_HOST_RPC_PORT}:${ZALLET_RPC_PORT}" networks: - z3_net - # NOTE: Healthcheck disabled - distroless image has no shell/curl - # Zallet will restart automatically if it crashes (restart: unless-stopped) - # Monitor logs or use external monitoring for service health + # No healthcheck: distroless image has no shell/curl volumes: zebra_data: diff --git a/zaino b/zaino index 66b3199..49e5241 160000 --- a/zaino +++ b/zaino @@ -1 +1 @@ -Subproject commit 66b3199fc07542159f1ef1d6c7149fd3193294c6 +Subproject commit 49e5241def007353033900eaefc84de5f6c13fcc diff --git a/zallet b/zallet index 60b0235..f0db32d 160000 --- a/zallet +++ b/zallet @@ -1 +1 @@ -Subproject commit 60b0235f292268bdc9a91b004e1e61ed9a73c9e9 +Subproject commit f0db32d23de36b9a8e0c48b4438d22ab076aca58 diff --git a/zcashd b/zcashd index 16ac743..cfcfcd9 160000 --- a/zcashd +++ b/zcashd @@ -1 +1 @@ -Subproject commit 16ac743764a513e41dafb2cd79c2417c5bb41e81 +Subproject commit cfcfcd93b06d2ee897f1d24eb62692c9e9e0e66d diff --git a/zebra b/zebra index 930fb7f..e04845f 160000 --- a/zebra +++ b/zebra @@ -1 +1 @@ -Subproject commit 930fb7f4b43a817b2a08d4a4ad94049efdf61887 +Subproject commit e04845f3e4a0976031f35c6564aa56ea4d722371 From 85b868ed6ebe06a7b162501f65f2c6117ca960cf Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 29 Jan 2026 18:49:30 +0000 Subject: [PATCH 3/6] feat(docker): add monitoring stack with Prometheus, Grafana, Jaeger, AlertManager Add optional observability stack enabled via Docker Compose profiles: - Prometheus (v3.2.0) for metrics collection - Grafana (11.5.1) with 14 pre-built Zebra dashboards - Jaeger (2.1.0) for distributed tracing (future Zebra releases) - AlertManager (v0.28.1) for alert routing Usage: docker compose --profile monitoring up -d Also updates: - Zebra image from 3.1.0 to 4.0.0 - README to reflect all services now use pre-built remote images - .env with monitoring configuration options --- .env | 22 +- .gitignore | 3 + README.md | 27 +- docker-compose.yml | 117 +- observability/README.md | 163 ++ observability/alertmanager/alertmanager.yml | 28 + observability/grafana/README.md | 123 + .../dashboards/block_verification.json | 529 ++++ .../dashboards/checkpoint_verification.json | 2084 ++++++++++++++++ observability/grafana/dashboards/errors.json | 469 ++++ observability/grafana/dashboards/mempool.json | 943 +++++++ .../grafana/dashboards/network_health.json | 2186 +++++++++++++++++ .../grafana/dashboards/network_messages.json | 868 +++++++ observability/grafana/dashboards/peers.json | 620 +++++ observability/grafana/dashboards/rocksdb.json | 925 +++++++ .../grafana/dashboards/rpc_metrics.json | 820 +++++++ .../grafana/dashboards/rpc_tracing.json | 639 +++++ observability/grafana/dashboards/syncer.json | 1319 ++++++++++ .../dashboards/transaction-verification.json | 978 ++++++++ .../grafana/dashboards/value_pools.json | 725 ++++++ .../grafana/dashboards/zebra_overview.json | 1047 ++++++++ .../provisioning/dashboards/default.yml | 13 + .../provisioning/datasources/datasources.yml | 19 + observability/jaeger/README.md | 343 +++ observability/jaeger/config.yaml | 86 + observability/prometheus/prometheus.yaml | 42 + .../prometheus/rules/zebra_alerts.yml | 149 ++ 27 files changed, 15267 insertions(+), 20 deletions(-) create mode 100644 observability/README.md create mode 100644 observability/alertmanager/alertmanager.yml create mode 100644 observability/grafana/README.md create mode 100644 observability/grafana/dashboards/block_verification.json create mode 100644 observability/grafana/dashboards/checkpoint_verification.json create mode 100644 observability/grafana/dashboards/errors.json create mode 100644 observability/grafana/dashboards/mempool.json create mode 100644 observability/grafana/dashboards/network_health.json create mode 100644 observability/grafana/dashboards/network_messages.json create mode 100644 observability/grafana/dashboards/peers.json create mode 100644 observability/grafana/dashboards/rocksdb.json create mode 100644 observability/grafana/dashboards/rpc_metrics.json create mode 100644 observability/grafana/dashboards/rpc_tracing.json create mode 100644 observability/grafana/dashboards/syncer.json create mode 100644 observability/grafana/dashboards/transaction-verification.json create mode 100644 observability/grafana/dashboards/value_pools.json create mode 100644 observability/grafana/dashboards/zebra_overview.json create mode 100644 observability/grafana/provisioning/dashboards/default.yml create mode 100644 observability/grafana/provisioning/datasources/datasources.yml create mode 100644 observability/jaeger/README.md create mode 100644 observability/jaeger/config.yaml create mode 100644 observability/prometheus/prometheus.yaml create mode 100644 observability/prometheus/rules/zebra_alerts.yml diff --git a/.env b/.env index a17fe1c..0520b29 100644 --- a/.env +++ b/.env @@ -154,6 +154,26 @@ ZALLET_CONF_PATH=/etc/zallet/zallet.toml # Zallet application internal data directory ZALLET_DATA_DIR=/home/zallet/.data # Example path for a CA certificate file that Zallet might use to trust Zaino's gRPC TLS certificate. -# If Zaino uses a self-signed certificate or a certificate from a private CA, Zallet would need to be +# If Zaino uses a self-signed certificate or a certificate from a private CA, Zallet would need to be # configured to trust it. The actual environment variable name and mechanism depend on Zallet's implementation. # ZALLET_INDEXER_CA_PATH=/path/to/trusted/zaino_ca.crt + +# ============================================================================= +# Monitoring Configuration (--profile monitoring) +# ============================================================================= +# Enable monitoring with: docker compose --profile monitoring up -d +# +# To enable Zebra metrics, uncomment this variable: +ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999 +# +# NOTE: OpenTelemetry tracing (Jaeger) is not yet available in Zebra 4.0.0. +# It will be supported in a future Zebra release. +# +# Service ports (defaults shown, customize if needed): +# GRAFANA_PORT=3000 +# PROMETHEUS_PORT=9094 +# JAEGER_UI_PORT=16686 +# ALERTMANAGER_PORT=9093 +# +# Grafana admin password (default: admin, prompted to change on first login): +# GRAFANA_ADMIN_PASSWORD=your_secure_password diff --git a/.gitignore b/.gitignore index 5a50148..0ee1c67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# macOS +.DS_Store + # Generated by Cargo # will have compiled files and executables debug/ diff --git a/README.md b/README.md index 1bf1b21..76a4f1d 100644 --- a/README.md +++ b/README.md @@ -108,37 +108,28 @@ docker compose ps ## Docker Images -> [!IMPORTANT] -> **Current Status:** Zaino and Zallet require local builds. Pre-built images are available for Zebra only. - ### Image Sources | Service | Image | Source | |---------|-------|--------| -| **Zebra** | `zfnd/zebra:3.1.0` | Pre-built from [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) | -| **Zaino** | `z3-zaino:local` | Must build locally from submodule | -| **Zallet** | `z3-zallet:local` | Must build locally from submodule | +| **Zebra** | `zfnd/zebra:4.0.0` | [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) | +| **Zaino** | `ghcr.io/zcashfoundation/zaino:sha-1871eba` | [ZcashFoundation/zaino](https://github.com/ZcashFoundation/zaino) | +| **Zallet** | `electriccoinco/zallet:v0.1.0-alpha.3` | [Electric Coin Co](https://github.com/Electric-Coin-Company/zallet) | -### Building Local Images +### Building Local Images (Optional) + +To build from local submodules instead of using pre-built images: ```bash # Initialize submodules git submodule update --init --recursive -# Build zaino and zallet -docker compose build zaino zallet +# Build all services locally +docker compose build ``` > [!NOTE] -> Local builds are required because Zaino and Zallet are under active development and require specific version pinning for compatibility. - -### Why Local Builds? - -Zallet embeds Zaino libraries internally. Both must use compatible versions of the Zaino codebase. The submodules in this repository are pinned to tested, compatible commits. - -**For production deployments**, use official release images when available: -- Zebra: [zfnd/zebra](https://hub.docker.com/r/zfnd/zebra) (stable releases) -- Zaino/Zallet: Official releases when published +> The submodules in this repository are pinned to tested, compatible commits if you prefer to build locally. ## Prerequisites diff --git a/docker-compose.yml b/docker-compose.yml index 2af5b9b..ccbf839 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: zebra: # Run 'docker compose build' to build locally from ./zebra submodule instead - image: zfnd/zebra:3.1.0 + image: zfnd/zebra:4.0.0 build: context: ./zebra dockerfile: docker/Dockerfile @@ -115,11 +115,124 @@ services: - z3_net # No healthcheck: distroless image has no shell/curl + # ============================================================================= + # Monitoring Stack (enabled with --profile monitoring) + # ============================================================================= + # Usage: docker compose --profile monitoring up -d + # Access: + # - Grafana: http://localhost:3000 (admin/admin) + # - Prometheus: http://localhost:9094 + # - Jaeger: http://localhost:16686 + # - AlertManager: http://localhost:9093 + + jaeger: + image: jaegertracing/jaeger:2.1.0 + container_name: z3_jaeger + profiles: [monitoring] + restart: unless-stopped + volumes: + - ./observability/jaeger/config.yaml:/etc/jaeger/config.yaml:ro + command: + - --config=/etc/jaeger/config.yaml + ports: + - "${JAEGER_UI_PORT:-16686}:16686" + - "${JAEGER_OTLP_GRPC_PORT:-4317}:4317" + - "${JAEGER_OTLP_HTTP_PORT:-4318}:4318" + - "${JAEGER_SPANMETRICS_PORT:-8889}:8889" + networks: + - z3_net + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:16686/ || exit 1"] + interval: 5s + timeout: 3s + retries: 10 + + prometheus: + image: prom/prometheus:v3.2.0 + container_name: z3_prometheus + profiles: [monitoring] + restart: unless-stopped + volumes: + - prometheus_data:/prometheus + - ./observability/prometheus/rules:/etc/prometheus/rules:ro + configs: + - source: prometheus_config + target: /etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.enable-lifecycle' + - '--web.enable-admin-api' + ports: + - "${PROMETHEUS_PORT:-9094}:9090" + networks: + - z3_net + depends_on: + jaeger: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:9090/-/healthy || exit 1"] + interval: 10s + timeout: 3s + retries: 3 + + grafana: + image: grafana/grafana:11.5.1 + container_name: z3_grafana + profiles: [monitoring] + restart: unless-stopped + volumes: + - ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./observability/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + environment: + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/zebra_overview.json + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + ports: + - "${GRAFANA_PORT:-3000}:3000" + networks: + - z3_net + depends_on: + prometheus: + condition: service_healthy + jaeger: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s + + alertmanager: + image: prom/alertmanager:v0.28.1 + container_name: z3_alertmanager + profiles: [monitoring] + restart: unless-stopped + volumes: + - ./observability/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + command: + - '--config.file=/etc/alertmanager/alertmanager.yml' + - '--storage.path=/alertmanager' + ports: + - "${ALERTMANAGER_PORT:-9093}:9093" + networks: + - z3_net + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:9093/-/healthy || exit 1"] + interval: 10s + timeout: 3s + retries: 3 + volumes: zebra_data: zaino_data: zallet_data: shared_cookie_volume: + prometheus_data: + grafana_data: + alertmanager_data: networks: z3_net: @@ -130,3 +243,5 @@ configs: file: ./config/tls/zaino.crt zaino_tls_key: file: ./config/tls/zaino.key + prometheus_config: + file: ./observability/prometheus/prometheus.yaml diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 0000000..287e5f2 --- /dev/null +++ b/observability/README.md @@ -0,0 +1,163 @@ +# Z3 Observability Stack + +Metrics, alerting, and dashboards for the Z3 stack (Zebra, Zaino, Zallet). + +## Quick Start + +```bash +# 1. Enable Zebra metrics in .env (uncomment this line): +ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999 + +# 2. Start the full stack with monitoring +docker compose --profile monitoring up -d + +# 3. View logs +docker compose logs -f zebra +``` + +> **Note**: OpenTelemetry tracing (Jaeger) is not yet available in Zebra 4.0.0. +> Jaeger is included for use with future Zebra releases that support tracing. + +## Components + +| Component | Port | URL | Purpose | +|-----------|------|-----|---------| +| **Zebra** | 9999 | - | Zcash node with metrics and tracing | +| **Prometheus** | 9094 | | Metrics collection and storage | +| **Grafana** | 3000 | | Dashboards and visualization | +| **Jaeger** | 16686 | | Distributed tracing UI | +| **AlertManager** | 9093 | | Alert routing | + +Default Grafana credentials: `admin` / `admin` (you'll be prompted to change on first login) + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Zebra Node │ +│ ┌─────────────────┐ ┌─────────────────────────────┐ │ +│ │ Metrics │ │ Tracing (OpenTelemetry) │ │ +│ │ :9999/metrics │ │ OTLP HTTP → Jaeger │ │ +│ └────────┬────────┘ └──────────────┬──────────────┘ │ +└───────────│──────────────────────────────────────│──────────────────┘ + │ │ + ▼ ▼ +┌───────────────────┐ ┌───────────────────────────┐ +│ Prometheus │ │ Jaeger │ +│ :9094 │ │ :16686 (UI) │ +│ │◄─────────────────│ :8889 (spanmetrics) │ +│ Scrapes metrics │ Span metrics │ :4318 (OTLP HTTP) │ +└─────────┬─────────┘ └───────────────────────────┘ + │ │ + ▼ │ +┌───────────────────┐ │ +│ Grafana │◄─────────────────────────────┘ +│ :3000 │ Trace queries +│ │ +│ Dashboards for │ +│ metrics + traces │ +└─────────┬─────────┘ + │ + ▼ +┌───────────────────┐ +│ AlertManager │ +│ :9093 │ +│ │ +│ Routes alerts │ +└───────────────────┘ +``` + +## What Each Component Provides + +### Metrics (Prometheus + Grafana) + +Quantitative data about Zebra's behavior over time: + +- **Network health**: Peer connections, bandwidth, message rates +- **Sync progress**: Block height, checkpoint verification, chain tip +- **Performance**: Block/transaction verification times +- **Resources**: Memory, connections, queue depths + +See [grafana/README.md](grafana/README.md) for dashboard details. + +### Tracing (Jaeger) + +> **Note**: OpenTelemetry tracing is not yet available in Zebra 4.0.0. +> This feature will be supported in a future Zebra release. + +Once available, Jaeger will provide: + +- **Distributed traces**: Follow a request through all components +- **Latency breakdown**: See where time is spent in each operation +- **Error analysis**: Identify failure points and error propagation +- **Service Performance Monitoring (SPM)**: RED metrics for RPC endpoints + +See [jaeger/README.md](jaeger/README.md) for tracing details. + +### Alerts (AlertManager) + +Automated notifications for operational issues: + +- Critical: Negative value pools (ZIP-209 violation) +- Warning: High RPC latency, sync stalls, peer connection issues + +Configure alert destinations in [alertmanager/alertmanager.yml](alertmanager/alertmanager.yml). + +## Configuration + +### Environment Variables + +Add this to your `.env` file to enable Zebra metrics: + +| Variable | Default | Description | +|----------|---------|-------------| +| `ZEBRA_METRICS__ENDPOINT_ADDR` | - | Prometheus metrics endpoint (e.g., `0.0.0.0:9999`) | + +### Port Customization + +Override default ports in `.env`: + +```bash +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9094 +JAEGER_UI_PORT=16686 +ALERTMANAGER_PORT=9093 +``` + +## Common Tasks + +### View Zebra's current metrics + +```bash +curl -s http://localhost:9999/metrics | grep zcash +``` + +### Query Prometheus directly + +```bash +# Current block height +curl -s 'http://localhost:9094/api/v1/query?query=zcash_state_tip_height' +``` + +## Troubleshooting + +### No metrics in Grafana + +1. Verify `ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999` is set in `.env` +2. Restart Zebra: `docker compose restart zebra` +3. Check Zebra is exposing metrics: `docker compose exec zebra wget -qO- http://localhost:9999/metrics | head` +4. Check Prometheus targets: + +## Running Without Monitoring + +To run the Z3 stack without monitoring: + +```bash +docker compose up -d # Only starts zebra, zaino, zallet +``` + +To add monitoring later: + +```bash +docker compose --profile monitoring up -d +``` diff --git a/observability/alertmanager/alertmanager.yml b/observability/alertmanager/alertmanager.yml new file mode 100644 index 0000000..5b3351e --- /dev/null +++ b/observability/alertmanager/alertmanager.yml @@ -0,0 +1,28 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'severity'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'default' + routes: + - match: + severity: critical + receiver: 'critical' + - match: + severity: warning + receiver: 'warning' + +receivers: + - name: 'default' + - name: 'critical' + - name: 'warning' + +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname'] diff --git a/observability/grafana/README.md b/observability/grafana/README.md new file mode 100644 index 0000000..ed4455d --- /dev/null +++ b/observability/grafana/README.md @@ -0,0 +1,123 @@ +# Zebra Grafana Dashboards + +Pre-built dashboards for monitoring Zebra nodes. + +## Quick Start + +```bash +# From repository root - starts Zebra + all observability tools +docker compose -f docker/docker-compose.observability.yml up -d +``` + +Access Grafana at (admin/admin - you'll be prompted to change on first login). + +For full stack documentation, see the [Observability README](../README.md). + +## Dashboards + +| Dashboard | Description | +|-----------|-------------| +| `network_health.json` | Peer connections, bandwidth (default home) | +| `syncer.json` | Sync progress, block downloads | +| `mempool.json` | Transaction pool metrics | +| `peers.json` | Peer connection details | +| `block_verification.json` | Block verification stats | +| `checkpoint_verification.json` | Checkpoint sync progress | +| `transaction-verification.json` | Transaction verification | +| `network_messages.json` | P2P protocol messages | +| `errors.json` | Error tracking | + +## Datasources + +Grafana is provisioned with two datasources: + +| Datasource | UID | Description | +|------------|-----|-------------| +| Prometheus | `zebra-prometheus` | Metrics storage and queries | +| Jaeger | `zebra-jaeger` | Distributed tracing | + +Configuration: `provisioning/datasources/datasources.yml` + +### Using Jaeger in Grafana + +The Jaeger datasource allows you to: + +- Search traces by service name +- View trace details and span timelines +- Correlate traces with metrics (via trace IDs) + +To explore traces: + +1. Go to **Explore** in Grafana +2. Select **Jaeger** datasource +3. Search for service `zebra` + +Or access Jaeger UI directly at for full trace exploration. +See the [Jaeger README](../jaeger/README.md) for detailed tracing documentation. + +## Dashboard Configuration + +### Rate Window Requirements + +Dashboards use `rate()` functions for per-second metrics. The rate window must +contain at least 2 data points to calculate a rate. + +| Scrape Interval | Minimum Rate Window | +|-----------------|---------------------| +| 500ms | 1s | +| 15s (default) | 30s | +| 30s | 1m | + +Current dashboards use `[1m]` windows, compatible with the default 15s scrape interval. + +If you modify `../prometheus/prometheus.yaml` scrape_interval, update dashboard queries accordingly. + +### Job Label + +The `$job` variable in dashboards is populated from Prometheus. The default job +name is `zebra` (configured in `../prometheus/prometheus.yaml`). + +## Creating New Dashboards + +### Option 1: Grafana UI Export (Recommended) + +1. Create panel in Grafana UI +2. Click panel title → "Inspect" → "Panel JSON" +3. Add to dashboard file +4. Commit + +### Option 2: Copy Existing Panel + +1. Find similar panel in existing dashboard +2. Copy JSON, update metric names and titles +3. Test in Grafana + +### Panel Template + +```json +{ + "title": "Your Metric", + "type": "timeseries", + "targets": [ + { + "expr": "rate(your_metric_total[1m])", + "legendFormat": "{{label}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + } + } +} +``` + +## Validation + +```bash +# Check JSON syntax +for f in dashboards/*.json; do jq . "$f" > /dev/null && echo "$f: OK"; done + +# List all metrics used +jq -r '.panels[].targets[]?.expr' dashboards/*.json | sort -u +``` diff --git a/observability/grafana/dashboards/block_verification.json b/observability/grafana/dashboards/block_verification.json new file mode 100644 index 0000000..0c99019 --- /dev/null +++ b/observability/grafana/dashboards/block_verification.json @@ -0,0 +1,529 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "full verified", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "checkpoint verified", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_memory_queued_max_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full queued max", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_memory_queued_min_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full queued min", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint queued max", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_checkpoint_queued_min_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint queued min", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_memory_sent_block_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full sent", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "state_checkpoint_sent_block_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint sent", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "zcash_chain_verified_block_height{job=\"$job\"}", + "interval": "", + "legendFormat": "committed", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_finalized_block_height{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "finalized", + "range": true, + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Verified Block Height - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:84", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:85", + "format": "none", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "zcash_chain_verified_block_total[1m]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_downloaded_block_count", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_downloads_in_flight", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(sync_verified_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_verified_block_count", + "refId": "J" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Block Sync Count - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:167", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:168", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "zcash_chain_verified_block_total[1m]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(gossip_downloaded_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_downloaded_block_count[1m]", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rate(gossip_verified_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_verified_block_count[1m]", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "gossip_queued_block_count{job=\"$job\"}", + "interval": "", + "legendFormat": "gossip_queued_block_count", + "refId": "E" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Block Gossip Count - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(zcash_chain_verified_block_height, job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_chain_verified_block_height, job)", + "refId": "Prometheus-Zebra-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "block verification", + "uid": "rO_Cl5tGz", + "version": 18, + "weekStart": "" +} diff --git a/observability/grafana/dashboards/checkpoint_verification.json b/observability/grafana/dashboards/checkpoint_verification.json new file mode 100644 index 0000000..3ab5d3e --- /dev/null +++ b/observability/grafana/dashboards/checkpoint_verification.json @@ -0,0 +1,2084 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "iteration": 1633652714499, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "decimals": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "checkpoint_processing_next_height{job=\"$job\"}", + "interval": "", + "legendFormat": "next_check", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_continuous_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_cont", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_verified_height{job=\"$job\"}", + "interval": "", + "legendFormat": "verified", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_max", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_committed_block_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_commit", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_q_max", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Height - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:84", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:85", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "decimals": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 0 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "checkpoint_processing_next_height{job=\"$job\"}", + "interval": "", + "legendFormat": "next_check", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_continuous_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_cont", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_verified_height{job=\"$job\"}", + "interval": "", + "legendFormat": "verified", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_max", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_committed_block_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_commit", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_q_max", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Height - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:84", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:85", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "decimals": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "checkpoint_processing_next_height{job=\"$job\"}", + "interval": "", + "legendFormat": "next_check", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_continuous_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_cont", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_verified_height{job=\"$job\"}", + "interval": "", + "legendFormat": "verified", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_max", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_committed_block_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_commit", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_q_max", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Height - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:84", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:85", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "decimals": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "checkpoint_processing_next_height{job=\"$job\"}", + "interval": "", + "legendFormat": "next_check", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_continuous_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_cont", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_verified_height{job=\"$job\"}", + "interval": "", + "legendFormat": "verified", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "queue_max", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_committed_block_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_commit", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "interval": "", + "legendFormat": "state_q_max", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Height - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:84", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:85", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(checkpoint_verified_block_count[1m])", + "interval": "", + "legendFormat": "checkpoint verify rate [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(state_checkpoint_committed_block_count[1m])", + "interval": "", + "legendFormat": "state commit rate [1m]", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "sync download rate [1m]", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "gossip download rate [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Pipeline Throughput - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 8, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(checkpoint_verified_block_count[1m])", + "interval": "", + "legendFormat": "checkpoint verify rate [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(state_checkpoint_committed_block_count[1m])", + "interval": "", + "legendFormat": "state commit rate [1m]", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "sync download rate [1m]", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "gossip download rate [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Pipeline Throughput - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 8, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(checkpoint_verified_block_count[1m])", + "interval": "", + "legendFormat": "checkpoint verify rate [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(state_checkpoint_committed_block_count[1m])", + "interval": "", + "legendFormat": "state commit rate [1m]", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "sync download rate [1m]", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "gossip download rate [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Pipeline Throughput - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 8, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(checkpoint_verified_block_count[1m])", + "interval": "", + "legendFormat": "checkpoint verify rate [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(state_checkpoint_committed_block_count[1m])", + "interval": "", + "legendFormat": "state commit rate [1m]", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "sync download rate [1m]", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_downloaded_block_count[1m])", + "interval": "", + "legendFormat": "gossip download rate [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Pipeline Throughput - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "checkpoint_queued_slots{job=\"$job\"}", + "interval": "", + "legendFormat": "checkpoint_queued_slots", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_block_count{job=\"$job\"}", + "interval": "", + "legendFormat": "state_finalized_queued_block_count", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_prospective_tips_len", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_downloads_in_flight", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_pending_blocks_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_pending_blocks_len", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_download_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_download_count[1m]", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_verify_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_verify_count[1m]", + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_queued_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_queued_block_count[1m]", + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Queues - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:337", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:338", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 14 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 4, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "checkpoint_queued_slots{job=\"$job\"}", + "interval": "", + "legendFormat": "checkpoint_queued_slots", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_block_count{job=\"$job\"}", + "interval": "", + "legendFormat": "state_finalized_queued_block_count", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_prospective_tips_len", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_downloads_in_flight", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_pending_blocks_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_pending_blocks_len", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_download_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_download_count[1m]", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_verify_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_verify_count[1m]", + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_queued_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_queued_block_count[1m]", + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Queues - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:337", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:338", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 4, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "checkpoint_queued_slots{job=\"$job\"}", + "interval": "", + "legendFormat": "checkpoint_queued_slots", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_block_count{job=\"$job\"}", + "interval": "", + "legendFormat": "state_finalized_queued_block_count", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_prospective_tips_len", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_downloads_in_flight", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_pending_blocks_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_pending_blocks_len", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_download_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_download_count[1m]", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_verify_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_verify_count[1m]", + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_queued_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_queued_block_count[1m]", + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Queues - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:337", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:338", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1633652714499, + "repeatPanelId": 4, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "checkpoint_queued_slots{job=\"$job\"}", + "interval": "", + "legendFormat": "checkpoint_queued_slots", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "state_checkpoint_queued_block_count{job=\"$job\"}", + "interval": "", + "legendFormat": "state_finalized_queued_block_count", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_prospective_tips_len", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_downloads_in_flight", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sync_pending_blocks_len{job=\"$job\"}", + "interval": "", + "legendFormat": "sync_pending_blocks_len", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_download_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_download_count[1m]", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(sync_cancelled_verify_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "sync_cancelled_verify_count[1m]", + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "rate(gossip_queued_block_count{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "gossip_queued_block_count[1m]", + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Queues - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:337", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:338", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(sync_prospective_tips_len, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(sync_prospective_tips_len, job)", + "refId": "Prometheus-Zebra-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "checkpoint verification", + "uid": "o4LmN_OMk", + "version": 12 +} diff --git a/observability/grafana/dashboards/errors.json b/observability/grafana/dashboards/errors.json new file mode 100644 index 0000000..c5bdfde --- /dev/null +++ b/observability/grafana/dashboards/errors.json @@ -0,0 +1,469 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 6, + "iteration": 1616480577841, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zebra_error_sapling_binding{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sapling_spend{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sprout_joinsplit{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sighash Errors - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1616480577841, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet-tmp-1", + "value": "zebrad-mainnet-tmp-1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zebra_error_sapling_binding{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sapling_spend{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sprout_joinsplit{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sighash Errors - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1616480577841, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-testnet-tmp-1", + "value": "zebrad-testnet-tmp-1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zebra_error_sapling_binding{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sapling_spend{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "zebra_sighash_error_sprout_joinsplit{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sighash Errors - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": [ + "zebrad-mainnet", + "zebrad-mainnet-tmp-1", + "zebrad-testnet-tmp-1" + ], + "value": [ + "zebrad-mainnet", + "zebrad-mainnet-tmp-1", + "zebrad-testnet-tmp-1" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(zcash_net_peers, job)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": "label_values(zcash_net_peers, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "errors", + "uid": "IhbO11wGk", + "version": 4 +} diff --git a/observability/grafana/dashboards/mempool.json b/observability/grafana/dashboards/mempool.json new file mode 100644 index 0000000..606eb00 --- /dev/null +++ b/observability/grafana/dashboards/mempool.json @@ -0,0 +1,943 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 15, + "iteration": 1634239984015, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Transactions" + }, + "properties": [ + { + "id": "displayName", + "value": "transactions" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Serialized Bytes" + }, + "properties": [ + { + "id": "displayName", + "value": "serialized bytes" + } + ] + } + ] + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [ + { + "$$hashKey": "object:232", + "alias": "transactions", + "yaxis": 1 + }, + { + "$$hashKey": "object:239", + "alias": "serialized bytes", + "yaxis": 2 + }, + { + "alias": "rejected serialized bytes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "zcash_mempool_size_transactions{job=\"$job\"}", + "interval": "", + "legendFormat": "transactions", + "refId": "Transactions", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "zcash_mempool_size_bytes{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": " serialized bytes", + "refId": "Serialized Bytes", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_currently_queued_transactions{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "queued transactions", + "refId": "Queued Transactions", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mempool Storage - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "decimals": null, + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Transactions" + }, + "properties": [ + { + "id": "displayName", + "value": "transactions" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Serialized Bytes" + }, + "properties": [ + { + "id": "displayName", + "value": "serialized bytes" + } + ] + } + ] + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:232", + "alias": "transactions", + "yaxis": 1 + }, + { + "$$hashKey": "object:239", + "alias": "serialized bytes", + "yaxis": 2 + }, + { + "alias": "rejected serialized bytes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "mempool_rejected_transaction_ids{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "rejected transactions", + "refId": "Rejected Transactions IDs", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_rejected_transaction_ids_bytes{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "rejected serialized bytes", + "refId": "Rejected Serialized TXID Bytes", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mempool Rejected Storage - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "decimals": null, + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1m])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "sync download", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])", + "interval": "", + "legendFormat": "state commit", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Rates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:80", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:81", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:232", + "alias": "transactions", + "yaxis": 1 + }, + { + "$$hashKey": "object:239", + "alias": "serialized bytes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "mempool_queued_transactions_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "queued", + "refId": "Queued", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_downloaded_transactions_total{job=\"$job\"}", + "interval": "", + "legendFormat": "downloaded", + "refId": "Downloaded", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_pushed_transactions_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "pushed", + "refId": "Pushed", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_verified_transactions_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "verified", + "refId": "Verified", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_cancelled_verify_tasks_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "cancelled", + "refId": "Cancelled", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_failed_verify_tasks_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "failed - {{reason}}", + "refId": "Failed", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "mempool_gossiped_transactions_total{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "gossiped", + "refId": "Gossiped", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transaction Downloader and Verifier, Gossiper - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "decimals": null, + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(mempool_downloaded_transactions_total{job=\"$job\"}[1m]) * 60", + "interval": "", + "legendFormat": "downloaded per min", + "refId": "Downloaded", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(mempool_verified_transactions_total{job=\"$job\"}[1m]) * 60", + "interval": "", + "legendFormat": "verified per min", + "refId": "Verified", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(mempool_queued_transactions_total{job=\"$job\"}[1m]) * 60", + "interval": "", + "legendFormat": "queued per min", + "refId": "Queued", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transaction Downloader and Verifier (Rates) - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1174", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1175", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:232", + "alias": "transactions", + "yaxis": 1 + }, + { + "$$hashKey": "object:239", + "alias": "serialized bytes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (version) (mempool_downloaded_transactions_total{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "{{version}}", + "refId": "Downloaded", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Downloaded Txs by Version - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "decimals": null, + "format": "decbytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "mempool", + "uid": "wVXGE6v7z", + "version": 8 +} diff --git a/observability/grafana/dashboards/network_health.json b/observability/grafana/dashboards/network_health.json new file mode 100644 index 0000000..cf3035f --- /dev/null +++ b/observability/grafana/dashboards/network_health.json @@ -0,0 +1,2186 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 4, + "iteration": 1639361606831, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(zcash_net_in_bytes_total{job=\"$job\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "bytes read [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sum(rate(zcash_net_out_bytes_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "bytes written [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "bytes - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:93", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:94", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(zcash_net_in_bytes_total{job=\"$job\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "bytes read [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sum(rate(zcash_net_out_bytes_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "bytes written [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "bytes - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:93", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:94", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(zcash_net_in_bytes_total{job=\"$job\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "bytes read [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sum(rate(zcash_net_out_bytes_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "bytes written [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "bytes - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:93", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:94", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(zcash_net_in_bytes_total{job=\"$job\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "bytes read [1m]", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "sum(rate(zcash_net_out_bytes_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "bytes written [1m]", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "bytes - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:93", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:94", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "repeatDirection": "h", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zcash_net_peers{job=\"$job\"}", + "interval": "", + "legendFormat": "total peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_ready{job=\"$job\"}", + "interval": "", + "legendFormat": "ready peers", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_unready{job=\"$job\"}", + "interval": "", + "legendFormat": "unready peers", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "peer readiness - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 6, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zcash_net_peers{job=\"$job\"}", + "interval": "", + "legendFormat": "total peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_ready{job=\"$job\"}", + "interval": "", + "legendFormat": "ready peers", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_unready{job=\"$job\"}", + "interval": "", + "legendFormat": "unready peers", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "peer readiness - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 6, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zcash_net_peers{job=\"$job\"}", + "interval": "", + "legendFormat": "total peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_ready{job=\"$job\"}", + "interval": "", + "legendFormat": "ready peers", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_unready{job=\"$job\"}", + "interval": "", + "legendFormat": "unready peers", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "peer readiness - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1639361606831, + "repeatPanelId": 6, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "zcash_net_peers{job=\"$job\"}", + "interval": "", + "legendFormat": "total peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_ready{job=\"$job\"}", + "interval": "", + "legendFormat": "ready peers", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "pool_num_unready{job=\"$job\"}", + "interval": "", + "legendFormat": "unready peers", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "peer readiness - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "candidate_set_disconnected{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "recently stopped peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_failed{job=\"$job\"}", + "interval": "", + "legendFormat": "failed candidates", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_gossiped{job=\"$job\"}", + "interval": "", + "legendFormat": "never attempted candidates", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_pending{job=\"$job\"}", + "interval": "", + "legendFormat": "connection attempt pending", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_responded{job=\"$job\"}", + "interval": "", + "legendFormat": "recent peers", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_recently_live{job=\"$job\"}", + "interval": "", + "legendFormat": "recently live peers", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "candidate set - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 7, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "candidate_set_disconnected{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "recently stopped peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_failed{job=\"$job\"}", + "interval": "", + "legendFormat": "failed candidates", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_gossiped{job=\"$job\"}", + "interval": "", + "legendFormat": "never attempted candidates", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_pending{job=\"$job\"}", + "interval": "", + "legendFormat": "connection attempt pending", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_responded{job=\"$job\"}", + "interval": "", + "legendFormat": "recent peers", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_recently_live{job=\"$job\"}", + "interval": "", + "legendFormat": "recently live peers", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "candidate set - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 7, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "candidate_set_disconnected{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "recently stopped peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_failed{job=\"$job\"}", + "interval": "", + "legendFormat": "failed candidates", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_gossiped{job=\"$job\"}", + "interval": "", + "legendFormat": "never attempted candidates", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_pending{job=\"$job\"}", + "interval": "", + "legendFormat": "connection attempt pending", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_responded{job=\"$job\"}", + "interval": "", + "legendFormat": "recent peers", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_recently_live{job=\"$job\"}", + "interval": "", + "legendFormat": "recently live peers", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "candidate set - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 7, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "candidate_set_disconnected{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "recently stopped peers", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_failed{job=\"$job\"}", + "interval": "", + "legendFormat": "failed candidates", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_gossiped{job=\"$job\"}", + "interval": "", + "legendFormat": "never attempted candidates", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_pending{job=\"$job\"}", + "interval": "", + "legendFormat": "connection attempt pending", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_responded{job=\"$job\"}", + "interval": "", + "legendFormat": "recent peers", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "expr": "candidate_set_recently_live{job=\"$job\"}", + "interval": "", + "legendFormat": "recently live peers", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "candidate set - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "sum by(command) (zebra_net_connection_state{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "connection state - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 18 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 11, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet-tmp", + "value": "zebrad-mainnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "sum by(command) (zebra_net_connection_state{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "connection state - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 11, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "sum by(command) (zebra_net_connection_state{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "connection state - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 18 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1639361606831, + "repeatPanelId": 11, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet-tmp", + "value": "zebrad-testnet-tmp" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "sum by(command) (zebra_net_connection_state{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "connection state - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s" + ] + }, + "timezone": "", + "title": "network health", + "uid": "320aS_dMk", + "version": 6 +} diff --git a/observability/grafana/dashboards/network_messages.json b/observability/grafana/dashboards/network_messages.json new file mode 100644 index 0000000..5b91950 --- /dev/null +++ b/observability/grafana/dashboards/network_messages.json @@ -0,0 +1,868 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 6, + "iteration": 1639360549666, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (command) (zebra_net_in_requests{job=\"$job\"})", + "interval": "", + "legendFormat": "Req::{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sum by (command) (zebra_net_out_responses{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "Rsp::{{command}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "inbound requests & responses - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (command) (zebra_net_out_requests{job=\"$job\"})", + "interval": "", + "legendFormat": "Req::{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sum by (command) (zebra_net_in_responses{job=\"$job\"})", + "hide": false, + "interval": "", + "legendFormat": "Rsp::{{command}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "outbound requests & responses - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (command) (zebra_net_out_requests_canceled{job=\"$job\"})", + "interval": "", + "legendFormat": "Req::{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "canceled outbound requests - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (command) (zcash_net_in_messages{job=\"$job\"})", + "interval": "", + "legendFormat": "{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "inbound message types - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (command) (zcash_net_out_messages{job=\"$job\"})", + "interval": "", + "legendFormat": "{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "outbound message types - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (addr) (zcash_net_in_messages{job=\"$job\"})", + "interval": "", + "legendFormat": "{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "inbound message peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": true, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (addr) (zcash_net_out_messages{job=\"$job\"})", + "interval": "", + "legendFormat": "{{command}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "outbound message peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": [ + "zebrad-mainnet" + ], + "value": [ + "zebrad-mainnet" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "network messages", + "uid": "YQ3yxiVnk", + "version": 9 +} diff --git a/observability/grafana/dashboards/peers.json b/observability/grafana/dashboards/peers.json new file mode 100644 index 0000000..5a7574d --- /dev/null +++ b/observability/grafana/dashboards/peers.json @@ -0,0 +1,620 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "iteration": 1635278363376, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (label_replace(zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"}, \"seed\", \"$1\", \"seed\", \"(.*):1?8233\") * on(remote_ip) group_left(remote_version) (count_values by (remote_ip) (\"remote_version\", zcash_net_peers_version_connected{job=\"$job\"})))", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version, seed) (label_replace(zcash_net_peers_initial{job=\"$job\",seed=~\"$seed\"}, \"seed\", \"$1\", \"seed\", \"(.*):1?8233\") * on(remote_ip) group_left(remote_version) (count_values by (remote_ip) (\"remote_version\", zcash_net_peers_version_obsolete{job=\"$job\"})))", + "instant": false, + "interval": "", + "legendFormat": "{{remote_version}} - {{seed}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Seed Peers - $job", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_connected{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compatible Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (remote_version) (zcash_net_peers_obsolete{job=\"$job\"})", + "interval": "", + "legendFormat": "{{remote_version}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Obsolete Peers - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (user_agent) (zcash_net_peers_connected{job=\"$job\"})", + "interval": "", + "legendFormat": "{{user_agent}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer User Agents - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 31, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_peers_initial, seed)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "seed", + "options": [], + "query": { + "query": "label_values(zcash_net_peers_initial, seed)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "peers", + "uid": "S29TgUH7k", + "version": 6 +} diff --git a/observability/grafana/dashboards/rocksdb.json b/observability/grafana/dashboards/rocksdb.json new file mode 100644 index 0000000..c93a2f1 --- /dev/null +++ b/observability/grafana/dashboards/rocksdb.json @@ -0,0 +1,925 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "RocksDB database metrics for Zebra", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_total_disk_size_bytes{job=~\"$job\"}", + "legendFormat": "Total Disk Size", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_live_data_size_bytes{job=~\"$job\"}", + "legendFormat": "Live Data Size", + "refId": "B" + } + ], + "title": "Database Disk Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_total_memory_size_bytes{job=~\"$job\"}", + "legendFormat": "Total Memory Size", + "refId": "A" + } + ], + "title": "Database Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "topk(10, zebra_state_rocksdb_cf_disk_size_bytes{job=~\"$job\"})", + "legendFormat": "{{cf}}", + "refId": "A" + } + ], + "title": "Top 10 Column Families by Disk Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "topk(10, zebra_state_rocksdb_cf_memory_size_bytes{job=~\"$job\"})", + "legendFormat": "{{cf}}", + "refId": "A" + } + ], + "title": "Top 10 Column Families by Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_total_disk_size_bytes{job=~\"$job\"}", + "legendFormat": "Total Disk", + "refId": "A" + } + ], + "title": "Total Disk Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "noValue": "NO DATA", + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_live_data_size_bytes{job=~\"$job\"}", + "legendFormat": "Live Data", + "refId": "A" + } + ], + "title": "Live Data Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000000000 + }, + { + "color": "red", + "value": 2000000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_total_memory_size_bytes{job=~\"$job\"}", + "legendFormat": "Memory", + "refId": "A" + } + ], + "title": "Total Memory Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_state_rocksdb_live_data_size_bytes{job=~\"$job\"} / zebra_state_rocksdb_total_disk_size_bytes{job=~\"$job\"}", + "legendFormat": "Compaction Efficiency", + "refId": "A" + } + ], + "title": "Compaction Efficiency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "id": 100, + "panels": [], + "title": "I/O Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "RocksDB batch commit latency percentiles (p95, p99)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "line" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.1 }, + { "color": "red", "value": 0.5 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "id": 10, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_batch_commit_duration_seconds{job=~\"$job\", quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_batch_commit_duration_seconds{job=~\"$job\", quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "B" + } + ], + "title": "Batch Commit Latency", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Block cache memory usage", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, + "id": 11, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_block_cache_usage_bytes{job=~\"$job\"}", + "legendFormat": "Block Cache", + "refId": "A" + } + ], + "title": "Block Cache Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, + "id": 101, + "panels": [], + "title": "Compaction", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Compaction status - pending bytes and running compactions", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Running Compactions" }, + "properties": [ + { "id": "custom.axisPlacement", "value": "right" }, + { "id": "unit", "value": "none" } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "id": 12, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_compaction_pending_bytes{job=~\"$job\"}", + "legendFormat": "Pending Bytes", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_compaction_running{job=~\"$job\"}", + "legendFormat": "Running Compactions", + "refId": "B" + } + ], + "title": "Compaction Status", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Number of SST files at each RocksDB level (L0-L6)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, + "id": 13, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_num_files_at_level{job=~\"$job\"}", + "legendFormat": "Level {{level}}", + "refId": "A" + } + ], + "title": "SST Files by Level", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 27, + "style": "dark", + "tags": ["rocksdb", "database"], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(zebra_state_rocksdb_total_disk_size_bytes, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zebra_state_rocksdb_total_disk_size_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RocksDB Database", + "uid": "zebra-rocksdb", + "version": 1 +} diff --git a/observability/grafana/dashboards/rpc_metrics.json b/observability/grafana/dashboards/rpc_metrics.json new file mode 100644 index 0000000..a433d11 --- /dev/null +++ b/observability/grafana/dashboards/rpc_metrics.json @@ -0,0 +1,820 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "JSON-RPC endpoint metrics - request rates, latency, and errors", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rpc_active_requests", + "legendFormat": "Active Requests", + "refId": "A" + } + ], + "title": "Active Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(rpc_requests_total[5m]))", + "legendFormat": "Requests/s", + "refId": "A" + } + ], + "title": "Request Rate (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(rpc_requests_total{status=\"error\"}[5m])) / sum(rate(rpc_requests_total[5m]))", + "legendFormat": "Error Rate", + "refId": "A" + } + ], + "title": "Error Rate (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99 Latency", + "refId": "A" + } + ], + "title": "p99 Latency (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "req/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(rpc_requests_total[5m])) by (method)", + "legendFormat": "{{ method }}", + "refId": "A" + } + ], + "title": "Request Rate by Method", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method))", + "legendFormat": "{{ method }} p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method))", + "legendFormat": "{{ method }} p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method))", + "legendFormat": "{{ method }} p99", + "refId": "C" + } + ], + "title": "Request Latency Percentiles by Method", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "errors/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(rpc_errors_total[5m])) by (method, error_code)", + "legendFormat": "{{ method }} ({{ error_code }})", + "refId": "A" + } + ], + "title": "Errors by Method and Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "rpc_active_requests", + "legendFormat": "Active Requests", + "refId": "A" + } + ], + "title": "Active Requests Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Rate" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 9, + "options": { + "footer": { + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Rate" + } + ] + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(rpc_requests_total[5m])) by (method)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "Rate" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "p99" + } + ], + "title": "Method Statistics", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "method" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true + }, + "indexByName": {}, + "renameByName": { + "Value #Rate": "Rate", + "Value #p50": "p50", + "Value #p99": "p99", + "method": "Method" + } + } + } + ], + "type": "table" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": ["zebra", "zcash", "rpc", "json-rpc"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Zebra RPC Metrics", + "uid": "zebra-rpc-metrics", + "version": 1, + "weekStart": "" +} diff --git a/observability/grafana/dashboards/rpc_tracing.json b/observability/grafana/dashboards/rpc_tracing.json new file mode 100644 index 0000000..8e6585b --- /dev/null +++ b/observability/grafana/dashboards/rpc_tracing.json @@ -0,0 +1,639 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "RPC Request Rate by Method", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum by (rpc_method) (rate(traces_spanmetrics_calls_total{service=\"zebra\", span_name=\"rpc_request\"}[1m]))", + "legendFormat": "{{rpc_method}}", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "RPC Error Rate by Method", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum by (rpc_method) (rate(traces_spanmetrics_calls_total{service=\"zebra\", span_name=\"rpc_request\", otel_status_code=\"ERROR\"}[1m])) / sum by (rpc_method) (rate(traces_spanmetrics_calls_total{service=\"zebra\", span_name=\"rpc_request\"}[1m]))", + "legendFormat": "{{rpc_method}}", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "RPC Latency P50 by Method", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (rpc_method, le) (rate(traces_spanmetrics_latency_bucket{service=\"zebra\", span_name=\"rpc_request\"}[1m])))", + "legendFormat": "{{rpc_method}}", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "RPC Latency P99 by Method", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (rpc_method, le) (rate(traces_spanmetrics_latency_bucket{service=\"zebra\", span_name=\"rpc_request\"}[1m])))", + "legendFormat": "{{rpc_method}}", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "title": "Total RPC Requests", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(increase(traces_spanmetrics_calls_total{service=\"zebra\", span_name=\"rpc_request\"}[1h]))", + "legendFormat": "Total", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "title": "Total RPC Errors", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(increase(traces_spanmetrics_calls_total{service=\"zebra\", span_name=\"rpc_request\", otel_status_code=\"ERROR\"}[1h]))", + "legendFormat": "Errors", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "title": "Average Latency", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(traces_spanmetrics_latency_sum{service=\"zebra\", span_name=\"rpc_request\"}[5m])) / sum(rate(traces_spanmetrics_latency_count{service=\"zebra\", span_name=\"rpc_request\"}[5m]))", + "legendFormat": "Avg", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "title": "P99 Latency", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"zebra\", span_name=\"rpc_request\"}[5m])) by (le))", + "legendFormat": "P99", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["zebra", "rpc", "tracing"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "RPC Tracing", + "uid": "zebra-rpc-tracing", + "version": 1, + "weekStart": "" +} diff --git a/observability/grafana/dashboards/syncer.json b/observability/grafana/dashboards/syncer.json new file mode 100644 index 0000000..9441540 --- /dev/null +++ b/observability/grafana/dashboards/syncer.json @@ -0,0 +1,1319 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "iteration": 1633496321106, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 200, + "panels": [], + "title": "Key Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "Current blockchain height", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 201, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_block_height", + "legendFormat": "Height", + "refId": "A" + } + ], + "title": "Block Height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "Total blocks downloaded during sync", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 202, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_downloaded_block_count", + "legendFormat": "Downloaded", + "refId": "A" + } + ], + "title": "Downloaded Blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "Total blocks verified during sync", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 203, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_verified_block_count", + "legendFormat": "Verified", + "refId": "A" + } + ], + "title": "Verified Blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "Block downloads currently in progress", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 204, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_downloads_in_flight", + "legendFormat": "In-Flight", + "refId": "A" + } + ], + "title": "In-Flight Downloads", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sync_obtain_queued_hash_count{job=\"$job\"}", + "interval": "", + "legendFormat": "obtain tips", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_extend_queued_hash_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "extend tips", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Queued Downloads - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1633496321106, + "repeatPanelId": 2, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sync_obtain_queued_hash_count{job=\"$job\"}", + "interval": "", + "legendFormat": "obtain tips", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_extend_queued_hash_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "extend tips", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Queued Downloads - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 14 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 4, + "legend": { + "show": false + }, + "pluginVersion": "7.5.7", + "repeat": "job", + "reverseYBuckets": false, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sync_obtain_response_hash_count{job=\"$job\"}", + "format": "heatmap", + "interval": "", + "legendFormat": "obtain tips", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_extend_response_hash_count{job=\"$job\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "extend tips", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Sync Peer Responses - $job", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 14 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 9, + "legend": { + "show": false + }, + "pluginVersion": "7.5.7", + "repeatIteration": 1633496321106, + "repeatPanelId": 4, + "reverseYBuckets": false, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sync_obtain_response_hash_count{job=\"$job\"}", + "format": "heatmap", + "interval": "", + "legendFormat": "obtain tips", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_extend_response_hash_count{job=\"$job\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "extend tips", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Sync Peer Responses - $job", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "job", + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-mainnet", + "value": "zebrad-mainnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sync_cancelled_download_count{job=\"$job\"}", + "interval": "", + "legendFormat": "cancelled downloads", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_cancelled_verify_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "cancelled verifications", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "in-flight downloads", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "prospective tips", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_downloaded_block_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "downloaded blocks", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_verified_block_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "verified blocks", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Status - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1633496321106, + "repeatPanelId": 7, + "scopedVars": { + "job": { + "selected": false, + "text": "zebrad-testnet", + "value": "zebrad-testnet" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sync_cancelled_download_count{job=\"$job\"}", + "interval": "", + "legendFormat": "cancelled downloads", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_cancelled_verify_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "cancelled verifications", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_downloads_in_flight{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "in-flight downloads", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_prospective_tips_len{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "prospective tips", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_downloaded_block_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "downloaded blocks", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "sync_verified_block_count{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "verified blocks", + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Status - $job", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:65", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:66", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 100, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_stage_duration_seconds{quantile=\"0.5\"}", + "legendFormat": "{{stage}} p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_stage_duration_seconds{quantile=\"0.95\"}", + "legendFormat": "{{stage}} p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_stage_duration_seconds{quantile=\"0.99\"}", + "legendFormat": "{{stage}} p99", + "refId": "C" + } + ], + "title": "Sync Stage Duration Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 101, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_block_download_duration_seconds{quantile=\"0.95\"}", + "legendFormat": "download {{result}} p95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sync_block_verify_duration_seconds{quantile=\"0.95\"}", + "legendFormat": "verify {{result}} p95", + "refId": "B" + } + ], + "title": "Block Download/Verify Duration (p95)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "label_values(zcash_net_in_bytes_total, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(zcash_net_in_bytes_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "syncer", + "uid": "Sl3h19Gnk", + "version": 11 +} diff --git a/observability/grafana/dashboards/transaction-verification.json b/observability/grafana/dashboards/transaction-verification.json new file mode 100644 index 0000000..21fb739 --- /dev/null +++ b/observability/grafana/dashboards/transaction-verification.json @@ -0,0 +1,978 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.5", + "targets": [ + { + "exemplar": true, + "expr": "zcash_chain_verified_block_height{}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "title": "Block height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{instance=\"localhost:9999\", job=\"zebrad\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RedPallas" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RedJubjub" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ed25519" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(signatures_ed25519_validated{}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Ed25519", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(signatures_redjubjub_validated{}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "RedJubjub", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(signatures_redpallas_validated{}[$__interval])", + "hide": false, + "legendFormat": "RedPallas", + "range": true, + "refId": "C" + } + ], + "title": "Signatures validated", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.5", + "targets": [ + { + "exemplar": true, + "expr": "rate(state_finalized_cumulative_transactions{}[$__interval])", + "interval": "", + "legendFormat": "Transactions finalized", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "title": "Transactions finalized", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Halo2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "exemplar": true, + "expr": "rate(proofs_groth16_verified{}[$__interval])", + "interval": "", + "legendFormat": "Groth16", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "editorMode": "code", + "expr": "rate(proofs_halo2_verified{}[$__interval])", + "hide": false, + "legendFormat": "Halo2", + "range": true, + "refId": "B" + } + ], + "title": "Proofs verified", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Transparent newOuts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Transparent prevOuts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(state_finalized_cumulative_transparent_newouts{}[$__interval])", + "interval": "", + "legendFormat": "Transparent newOuts", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + }, + { + "exemplar": true, + "expr": "rate(state_finalized_cumulative_transparent_prevouts{}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Transparent prevOuts", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + } + } + ], + "title": "Transparent Outpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Orchard" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Sapling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Sprout" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "exemplar": true, + "expr": "rate(state_finalized_cumulative_sprout_nullifiers{}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Sprout", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "exemplar": true, + "expr": "rate(state_finalized_cumulative_sapling_nullifiers{}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Sapling", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "exemplar": true, + "expr": "rate(state_finalized_cumulative_orchard_nullifiers{}[$__interval])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Orchard", + "refId": "C" + } + ], + "title": "Nullifiers revealed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 100, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_consensus_batch_duration_seconds{quantile=\"0.5\"}", + "legendFormat": "{{verifier}} p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_consensus_batch_duration_seconds{quantile=\"0.95\"}", + "legendFormat": "{{verifier}} p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "zebra_consensus_batch_duration_seconds{quantile=\"0.99\"}", + "legendFormat": "{{verifier}} p99", + "refId": "C" + } + ], + "title": "Batch Verification Duration by Verifier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 101, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(zebra_consensus_batch_duration_seconds_count{result=\"success\"}[5m])) by (verifier)", + "legendFormat": "{{verifier}} success/s", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "sum(rate(zebra_consensus_batch_duration_seconds_count{result=\"failure\"}[5m])) by (verifier)", + "legendFormat": "{{verifier}} failure/s", + "refId": "B" + } + ], + "title": "Verification Throughput by Verifier", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "🔎", + "uid": "UXVRR1v7z", + "version": 23, + "weekStart": "" +} diff --git a/observability/grafana/dashboards/value_pools.json b/observability/grafana/dashboards/value_pools.json new file mode 100644 index 0000000..b1d8a5a --- /dev/null +++ b/observability/grafana/dashboards/value_pools.json @@ -0,0 +1,725 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Value Pool Monitoring - Track shielded pool balances and total supply. Zebra enforces ZIP-209 internally.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ZEC", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["last"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_chain_supply_total / 100000000", + "legendFormat": "Total Supply (ZEC)", + "refId": "A" + } + ], + "title": "Total Chain Supply", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ZEC", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Transparent" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Sprout" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Sapling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Orchard" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deferred" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["last", "mean"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_transparent / 100000000", + "legendFormat": "Transparent", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_sprout / 100000000", + "legendFormat": "Sprout", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_sapling / 100000000", + "legendFormat": "Sapling", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_orchard / 100000000", + "legendFormat": "Orchard", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_deferred / 100000000", + "legendFormat": "Deferred", + "refId": "E" + } + ], + "title": "Value Pools by Type (Stacked)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_transparent / 100000000", + "legendFormat": "Transparent ZEC", + "refId": "A" + } + ], + "title": "Transparent Pool", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#F2CC0C", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_sprout / 100000000", + "legendFormat": "Sprout ZEC", + "refId": "A" + } + ], + "title": "Sprout Pool", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_sapling / 100000000", + "legendFormat": "Sapling ZEC", + "refId": "A" + } + ], + "title": "Sapling Pool", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_value_pool_orchard / 100000000", + "legendFormat": "Orchard ZEC", + "refId": "A" + } + ], + "title": "Orchard Pool", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "description": "Pool Health Check: All pools should be non-negative (enforced internally by Zebra)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "HEALTHY" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "red", + "index": 1, + "text": "VIOLATION" + }, + "to": 999999 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "(state_finalized_value_pool_transparent < 0) + (state_finalized_value_pool_sprout < 0) + (state_finalized_value_pool_sapling < 0) + (state_finalized_value_pool_orchard < 0)", + "legendFormat": "Pool Health Status", + "refId": "A" + } + ], + "title": "Pool Health Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "expr": "state_finalized_block_height", + "legendFormat": "Block Height", + "refId": "A" + } + ], + "title": "Current Block Height", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": ["zebra", "zcash", "value-pools"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Zebra Value Pools", + "uid": "zebra-value-pools", + "version": 1, + "weekStart": "" +} diff --git a/observability/grafana/dashboards/zebra_overview.json b/observability/grafana/dashboards/zebra_overview.json new file mode 100644 index 0000000..2b23bc5 --- /dev/null +++ b/observability/grafana/dashboards/zebra_overview.json @@ -0,0 +1,1047 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Zebra Node Overview - At-a-glance health and status for node operators", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "panels": [], + "title": "Node Identity", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Zebra node software version", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 3, "w": 8, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^version$/", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebrad_build_info", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Current finalized blockchain height", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 3, "w": 8, "x": 8, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "state_finalized_block_height", + "legendFormat": "Height", + "refId": "A" + } + ], + "title": "Block Height", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Total ZEC supply on the blockchain", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "orange", "value": null }] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 3, "w": 8, "x": 16, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "state_finalized_chain_supply_total / 100000000", + "legendFormat": "ZEC", + "refId": "A" + } + ], + "title": "Total Supply (ZEC)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, + "id": 101, + "panels": [], + "title": "Health Status", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Peer connectivity - gauge shows current count relative to healthy thresholds", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "max": 50, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "#EAB839", "value": 8 }, + { "color": "green", "value": 20 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 4, "x": 0, "y": 5 }, + "id": 10, + "options": { + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": true, + "showThresholdMarkers": true + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zcash_net_peers", + "legendFormat": "Peers", + "refId": "A" + } + ], + "title": "Peer Count", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Block verification activity (ACTIVE = processing blocks, IDLE = waiting for new blocks)", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { + "options": { "from": 0.001, "result": { "color": "green", "index": 0, "text": "ACTIVE" }, "to": 1000000 }, + "type": "range" + }, + { + "options": { "from": 0, "result": { "color": "blue", "index": 1, "text": "IDLE" }, "to": 0.001 }, + "type": "range" + } + ], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 0.001 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 5, "x": 4, "y": 5 }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "rate(zcash_chain_verified_block_total[5m])", + "legendFormat": "Rate", + "refId": "A" + } + ], + "title": "Block Activity", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Blocks behind estimated network tip", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { + "options": { "from": 0, "result": { "color": "green", "index": 0, "text": "SYNCED" }, "to": 2 }, + "type": "range" + } + ], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 3 }, + { "color": "orange", "value": 50 }, + { "color": "red", "value": 100 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 5, "x": 9, "y": 5 }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "sync_estimated_distance_to_tip{job=~\"$job\"}", + "legendFormat": "Distance", + "refId": "A" + } + ], + "title": "Sync Distance", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "ZIP-209 compliance - all value pools must be non-negative", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "options": { "0": { "color": "green", "index": 0, "text": "HEALTHY" } }, "type": "value" }, + { + "options": { "from": 1, "result": { "color": "red", "index": 1, "text": "VIOLATION" }, "to": 10 }, + "type": "range" + } + ], + "noValue": "NO DATA", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 5, "x": 14, "y": 5 }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "count(state_finalized_value_pool_transparent < 0 or state_finalized_value_pool_sprout < 0 or state_finalized_value_pool_sapling < 0 or state_finalized_value_pool_orchard < 0) or vector(0)", + "legendFormat": "Violations", + "refId": "A" + } + ], + "title": "Value Pools", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "RPC endpoint health based on error rate (no data = RPC disabled)", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "options": { "from": 0, "result": { "color": "green", "index": 0, "text": "HEALTHY" }, "to": 0.05 }, "type": "range" }, + { "options": { "from": 0.05, "result": { "color": "yellow", "index": 1, "text": "DEGRADED" }, "to": 0.1 }, "type": "range" }, + { "options": { "from": 0.1, "result": { "color": "red", "index": 2, "text": "UNHEALTHY" }, "to": 1 }, "type": "range" } + ], + "noValue": "DISABLED", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "red", "value": 0.1 } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 5, "x": 19, "y": 5 }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "sum(rate(rpc_requests_total{status=\"error\"}[5m])) / sum(rate(rpc_requests_total[5m])) or vector(0)", + "legendFormat": "Error Rate", + "refId": "A" + } + ], + "title": "RPC Health", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, + "id": 102, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Network bandwidth - bytes transferred per second", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "binBps" + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Inbound" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Outbound" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 11 }, + "id": 20, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "sum(rate(zcash_net_in_bytes_total[1m]))", "legendFormat": "Inbound", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "sum(rate(zcash_net_out_bytes_total[1m]))", "legendFormat": "Outbound", "refId": "B" } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "P2P message rate - messages per second", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "msg/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "ops" + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Inbound" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Outbound" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 11 }, + "id": 21, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "sum(rate(zcash_net_in_messages[1m]))", "legendFormat": "Inbound", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "sum(rate(zcash_net_out_messages[1m]))", "legendFormat": "Outbound", "refId": "B" } + ], + "title": "Message Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Distribution of connected peers by client software", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 11 }, + "id": 22, + "options": { + "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": ["value", "percent"] }, + "pieType": "pie", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "sum by (user_agent) (zcash_net_peers_connected)", + "legendFormat": "{{user_agent}}", + "refId": "A" + } + ], + "title": "Peer Distribution", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, + "id": 103, + "panels": [], + "title": "Consensus & Mempool", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Mempool transaction count and size", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Transactions" }, "properties": [{ "id": "custom.axisPlacement", "value": "left" }] }, + { "matcher": { "id": "byName", "options": "Size" }, "properties": [{ "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "bytes" }] } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 }, + "id": 30, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "zcash_mempool_size_transactions", "legendFormat": "Transactions", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "zcash_mempool_size_bytes", "legendFormat": "Size", "refId": "B" } + ], + "title": "Mempool Size", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Cryptographic proof verification rate (Halo2 + Groth16)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "proofs/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 }, + "id": 31, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "rate(proofs_halo2_verified[1m])", "legendFormat": "Halo2", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "rate(proofs_groth16_verified[1m])", "legendFormat": "Groth16", "refId": "B" } + ], + "title": "Proof Verification Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Block download and verification rate", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "blocks/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 }, + "id": 32, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "rate(zcash_chain_verified_block_total[1m])", "legendFormat": "Verified", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, "expr": "rate(sync_downloaded_block_count[1m])", "legendFormat": "Downloaded", "refId": "B" } + ], + "title": "Block Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "id": 104, + "panels": [], + "title": "Infrastructure", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "RocksDB database disk usage", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80000000000 }, + { "color": "red", "value": 150000000000 } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 29 }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zebra_state_rocksdb_total_disk_size_bytes", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "title": "Database Size", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "RPC endpoint p99 latency (5m window)", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 2 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 29 }, + "id": 41, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "RPC p99 Latency", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Peer count trend over time with health thresholds", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "line" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 8 }, + { "color": "green", "value": 20 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 29 }, + "id": 42, + "options": { + "legend": { "calcs": ["lastNotNull", "min", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "zcash_net_peers", + "legendFormat": "Peers", + "refId": "A" + } + ], + "title": "Peer Count Trend", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, + "id": 106, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Process CPU usage rate", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 34 }, + "id": 60, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=~\"$job\"}[1m])", + "legendFormat": "CPU", + "refId": "A" + } + ], + "title": "Process CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Process memory usage (resident and virtual)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 34 }, + "id": 61, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "process_resident_memory_bytes{job=~\"$job\"}", + "legendFormat": "Resident", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "process_virtual_memory_bytes{job=~\"$job\"}", + "legendFormat": "Virtual", + "refId": "B" + } + ], + "title": "Process Memory", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "description": "Open and maximum file descriptors", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 34 }, + "id": 62, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "process_open_fds{job=~\"$job\"}", + "legendFormat": "Open", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "zebra-prometheus" }, + "expr": "process_max_fds{job=~\"$job\"}", + "legendFormat": "Max", + "refId": "B" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + } + ], + "title": "System Resources", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, + "id": 105, + "panels": [], + "title": "Dashboard Links", + "type": "row" + }, + { + "datasource": { "type": "datasource", "uid": "grafana" }, + "gridPos": { "h": 3, "w": 24, "x": 0, "y": 35 }, + "id": 50, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "### Detailed Dashboards\n\n| [Syncer](/d/Sl3h19Gnk/syncer) | [Network Health](/d/320aS_dMk/network-health) | [Peers](/d/S29TgUH7k/peers) | [RPC Metrics](/d/zebra-rpc-metrics/zebra-rpc-metrics) | [RPC Tracing](/d/zebra-rpc-tracing/zebra-rpc-tracing) | [Value Pools](/d/zebra-value-pools/zebra-value-pools) | [Mempool](/d/wVXGE6v7z/mempool) | [Database](/d/zebra-rocksdb/rocksdb-database) | [Tx Verification](/d/UXVRR1v7z/transaction-verification) | [Block Verification](/d/rO_Cl5tGz/block-verification) |", + "mode": "markdown" + }, + "pluginVersion": "10.0.0", + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["zebra", "zcash", "overview"], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "zebra-prometheus" + }, + "definition": "label_values(state_finalized_block_height, job)", + "description": "Filter by Prometheus job label", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(state_finalized_block_height, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Zebra Overview", + "uid": "zebra-overview", + "version": 1, + "weekStart": "" +} diff --git a/observability/grafana/provisioning/dashboards/default.yml b/observability/grafana/provisioning/dashboards/default.yml new file mode 100644 index 0000000..c364c8b --- /dev/null +++ b/observability/grafana/provisioning/dashboards/default.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'Zebra Dashboards' + orgId: 1 + folder: 'Zebra' + folderUid: 'zebra-dashboards' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/observability/grafana/provisioning/datasources/datasources.yml b/observability/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..114fcb0 --- /dev/null +++ b/observability/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + # Prometheus for metrics (Zebra metrics + Jaeger spanmetrics) + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + uid: zebra-prometheus + + # Jaeger for distributed tracing + - name: Jaeger + type: jaeger + access: proxy + url: http://jaeger:16686 + editable: false + uid: zebra-jaeger diff --git a/observability/jaeger/README.md b/observability/jaeger/README.md new file mode 100644 index 0000000..6f74d88 --- /dev/null +++ b/observability/jaeger/README.md @@ -0,0 +1,343 @@ +# Jaeger Distributed Tracing + +Jaeger shows how requests flow through Zebra, where time is spent, and where errors occur. + +## Why Jaeger? + +While Prometheus metrics show **what** is happening (counters, gauges, histograms), Jaeger traces show **how** it's happening: + +| Metrics (Prometheus) | Traces (Jaeger) | +|---------------------|-----------------| +| "RPC latency P95 is 500ms" | "This specific getblock call spent 400ms in state lookup" | +| "10 requests/second" | "Request X called service A → B → C with these timings" | +| "5% error rate" | "This error started in component Y and propagated to Z" | + +Jaeger is used by other blockchain clients including Lighthouse, Reth, Hyperledger Besu, and Hyperledger Fabric for similar observability needs. + +## Accessing Jaeger + +Open http://localhost:16686 in your browser. + +## Concepts + +### Traces and Spans + +- **Trace**: A complete request journey through the system (e.g., an RPC call from start to finish) +- **Span**: A single operation within a trace (e.g., "verify block", "read from database") +- **Parent-child relationships**: Spans form a tree showing how operations nest + +### Span Kinds + +Jaeger categorizes spans by their role: + +| Kind | Description | Example in Zebra | +|------|-------------|------------------| +| **SERVER** | Entry point handling external requests | JSON-RPC endpoints (`getblock`, `getinfo`) | +| **INTERNAL** | Internal operations within the service | Block verification, state operations | +| **CLIENT** | Outgoing calls to other services | (Not currently used) | + +This distinction is important for the Monitor tab (see below). + +## Monitor Tab (Service Performance Monitoring) + +The Monitor tab provides RED metrics (Rate, Errors, Duration) aggregated from traces. + +### Understanding the Dashboard + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Service: zebra Span Kind: [Internal ▼] │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Latency (s) Error rate (%) Request rate (req/s) │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ /\ │ │ │ │ ___ │ │ +│ │ / \ │ │ ────── │ │ / \ │ │ +│ │ / \ │ │ │ │ / \ │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +│ │ +├─────────────────────────────────────────────────────────────────────┤ +│ Operations metrics │ +│ ┌────────────────────────────────────────────────────────────────┐│ +│ │ Name P95 Latency Request rate Error ││ +│ │ state 10s 10.27 req/s < 0.1% ││ +│ │ connector 4.52s 0.24 req/s < 0.1% ││ +│ │ checkpoint 95.88ms 6.69 req/s < 0.1% ││ +│ │ best_tip_height 4.75ms 58.84 req/s < 0.1% ││ +│ │ download_and_verify 2.32s 0.11 req/s < 0.1% ││ +│ └────────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Columns Explained + +| Column | Meaning | What to Look For | +|--------|---------|------------------| +| **Name** | Span/operation name | Group of related operations | +| **P95 Latency** | 95th percentile duration | High values indicate slow operations | +| **Request rate** | Operations per second | Throughput of each operation type | +| **Error rate** | Percentage of failures | Any value > 0% needs investigation | +| **Impact** | Relative contribution to overall latency | Focus optimization on high-impact operations | + +### Span Kind Filter + +The "Span Kind" dropdown filters which operations you see: + +- **Internal**: All internal Zebra operations (default view) + - Block verification, state operations, network handling + - High volume during sync + +- **Server**: External-facing request handlers + - JSON-RPC endpoints (`getblock`, `getinfo`, `sendrawtransaction`) + - Use this for RPC performance monitoring + +### Interpreting Common Operations + +| Operation | Description | Normal Latency | +|-----------|-------------|----------------| +| `state` | State database operations | 1-10s during sync | +| `connector` | Peer connection establishment | 1-5s | +| `dial` | Network dial attempts | 1-5s | +| `checkpoint` | Checkpoint verification | 50-200ms | +| `best_tip_height` | Chain tip queries | < 10ms | +| `download_and_verify` | Block download + verification | 1-5s | +| `block_commitment_is_valid_for_chain_history` | Block validation | 50-200ms | +| `rpc_request` | JSON-RPC handler (SERVER span) | Method-dependent | + +## Search Tab + +Use the Search tab to find specific traces. + +### Finding Traces + +1. **Service**: Select `zebra` +2. **Operation**: Choose a specific operation or "all" +3. **Tags**: Filter by attributes (e.g., `rpc.method=getblock`) +4. **Min/Max Duration**: Find slow or fast requests +5. **Limit**: Number of results (default: 20) + +### Useful Search Queries + +**Find slow RPC calls:** +``` +Service: zebra +Operation: rpc_request +Min Duration: 1s +``` + +**Find failed operations:** +``` +Service: zebra +Tags: error=true +``` + +**Find specific RPC method:** +``` +Service: zebra +Tags: rpc.method=getblock +``` + +## Trace Detail View + +Clicking a trace opens the detail view showing: + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Trace: abc123 (5 spans, 234ms) │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ├─ rpc_request [SERVER] ─────────────────────────────── 234ms ────┤│ +│ │ rpc.method: getblock ││ +│ │ rpc.system: jsonrpc ││ +│ │ ││ +│ │ ├─ state::read_block ─────────────────────── 180ms ───────────┤││ +│ │ │ block.height: 2000000 │││ +│ │ │ │││ +│ │ │ ├─ db::get ───────────────────── 150ms ──────────────────┤│││ +│ │ │ │ cf: blocks ││││ +│ │ │ └─────────────────────────────────────────────────────────┘│││ +│ │ └─────────────────────────────────────────────────────────────┘││ +│ └─────────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Reading the Waterfall + +- **Horizontal bars**: Duration of each span +- **Nesting**: Parent-child relationships +- **Colors**: Different services/components +- **Tags**: Attributes attached to each span + +### Span Attributes + +| Attribute | Meaning | +|-----------|---------| +| `otel.kind` | Span type (server, internal) | +| `rpc.method` | JSON-RPC method name | +| `rpc.system` | Protocol (jsonrpc) | +| `otel.status_code` | ERROR on failure | +| `rpc.error_code` | JSON-RPC error code | +| `error.message` | Error description | + +## Debugging with Traces + +### Performance Issues + +1. **Find slow traces**: Use Min Duration filter in Search +2. **Identify bottleneck**: Look for the longest span in the waterfall +3. **Check children**: See if parent is slow due to child operations +4. **Compare traces**: Compare fast vs slow traces for same operation + +### Error Investigation + +1. **Find failed traces**: Search with `error=true` tag +2. **Locate error span**: Look for spans with `otel.status_code=ERROR` +3. **Read error message**: Check `error.message` attribute +4. **Trace propagation**: See how error affected parent operations + +### Example: Debugging Slow RPC + +``` +Problem: getblock calls are slow + +1. Search: + - Service: zebra + - Operation: rpc_request + - Tags: rpc.method=getblock + - Min Duration: 500ms + +2. Open a slow trace + +3. Examine waterfall: + - rpc_request: 800ms total + └─ state::read_block: 750ms ← Most time spent here + └─ db::get: 700ms ← Database is the bottleneck + +4. Conclusion: Database read is slow, possibly due to: + - Disk I/O + - Large block size + - Missing cache +``` + +## RPC Tracing + +Zebra's JSON-RPC endpoints are instrumented with `SPAN_KIND_SERVER` spans, enabling: + +- **Jaeger SPM**: View RPC metrics in the Monitor tab (select "Server" span kind) +- **Per-method analysis**: Filter by `rpc.method` to see specific endpoint performance +- **Error tracking**: Failed RPC calls have `otel.status_code=ERROR` + +### RPC Span Attributes + +Each RPC request includes: + +| Attribute | Example | Description | +|-----------|---------|-------------| +| `otel.kind` | `server` | Marks as server-side handler | +| `rpc.method` | `getblock` | JSON-RPC method name | +| `rpc.system` | `jsonrpc` | Protocol identifier | +| `otel.status_code` | `ERROR` | Present on failure | +| `rpc.error_code` | `-8` | JSON-RPC error code | + +## Compare Tab + +Use Compare to diff two traces: + +1. Select two trace IDs +2. View side-by-side comparison +3. Identify differences in timing or structure + +Useful for: +- Comparing fast vs slow versions of same request +- Before/after optimization comparisons +- Debugging intermittent issues + +## System Architecture Tab + +Shows service dependencies based on trace data: + +- Nodes: Services (currently just `zebra`) +- Edges: Communication between services +- Useful when Zebra calls external services + +## Configuration + +Jaeger v2 configuration is in `config.yaml`: + +```yaml +# Key settings +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 # OTLP HTTP (used by Zebra) + +processors: + batch: {} # Batches spans for efficiency + +connectors: + spanmetrics: # Generates Prometheus metrics from spans + namespace: traces.spanmetrics + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 # Spanmetrics for Prometheus +``` + +### Spanmetrics + +Jaeger automatically generates Prometheus metrics from spans: + +```bash +# View spanmetrics +curl -s http://localhost:8889/metrics | grep traces_spanmetrics +``` + +These power the Monitor tab and can be scraped by Prometheus for Grafana dashboards. + +## Usage Tips + +### During Initial Sync + +- **Reduce sampling** to 1-10% to avoid overwhelming Jaeger +- **Focus on errors** rather than complete traces +- **Use metrics** (Prometheus) for high-level sync progress + +### Steady State Operation + +- **Increase sampling** to 10-50% for better visibility +- **Monitor RPC latency** in the Monitor tab (Server spans) +- **Set up alerts** for high error rates + +### Debugging Sessions + +- **Set sampling to 100%** temporarily +- **Use specific searches** to find relevant traces +- **Compare traces** to identify anomalies +- **Remember to reduce sampling** after debugging + +## Troubleshooting + +### No traces appearing + +1. Check OTLP endpoint: `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` +2. Verify Jaeger health: http://localhost:16686 +3. Check Jaeger logs: `docker compose logs jaeger` + +### Monitor tab shows "No Data" + +1. Ensure spanmetrics connector is configured +2. Wait for traces to accumulate (needs a few minutes) +3. Select the correct Span Kind filter + +### High memory usage + +1. Reduce sampling: `OTEL_TRACES_SAMPLER_ARG=10` +2. Restart Jaeger to clear memory +3. Consider using external storage for production + +## Further Reading + +- [Jaeger Documentation](https://www.jaegertracing.io/docs/) +- [OpenTelemetry Tracing Concepts](https://opentelemetry.io/docs/concepts/signals/traces/) +- [Jaeger v2 Architecture](https://www.jaegertracing.io/docs/2.0/architecture/) diff --git a/observability/jaeger/config.yaml b/observability/jaeger/config.yaml new file mode 100644 index 0000000..21c4f1e --- /dev/null +++ b/observability/jaeger/config.yaml @@ -0,0 +1,86 @@ +# Jaeger v2 Configuration +# Based on OpenTelemetry Collector architecture +# Reference: https://www.jaegertracing.io/docs/2.0/getting-started/ + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 1s + send_batch_size: 1024 + +connectors: + # Spanmetrics connector generates metrics from spans for Jaeger SPM + # Reference: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/spanmetricsconnector + spanmetrics: + namespace: span_metrics + metrics_flush_interval: 15s + # Histogram buckets for latency distribution + histogram: + explicit: + buckets: [1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s] + dimensions: + - name: rpc.method + - name: rpc.system + - name: otel.status_code + +exporters: + # Jaeger storage (in-memory for development) + jaeger_storage_exporter: + trace_storage: memstore + + # Prometheus metrics endpoint for spanmetrics (used by Jaeger SPM) + prometheus: + endpoint: 0.0.0.0:8889 + namespace: traces + const_labels: + service: zebra + +extensions: + jaeger_storage: + backends: + memstore: + memory: + max_traces: 100000 + # Metrics storage for SPM - reads from Prometheus + metric_backends: + prometheus: + prometheus: + endpoint: http://prometheus:9090 + normalize_calls: true + normalize_duration: true + + jaeger_query: + storage: + traces: memstore + metrics: prometheus + ui: + config_file: "" + + healthcheckv2: + use_v2: true + http: + endpoint: 0.0.0.0:13133 + +service: + extensions: + - jaeger_storage + - jaeger_query + - healthcheckv2 + + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [jaeger_storage_exporter, spanmetrics] + + # Spanmetrics pipeline - exports to Prometheus + metrics/spanmetrics: + receivers: [spanmetrics] + exporters: [prometheus] diff --git a/observability/prometheus/prometheus.yaml b/observability/prometheus/prometheus.yaml new file mode 100644 index 0000000..9c56994 --- /dev/null +++ b/observability/prometheus/prometheus.yaml @@ -0,0 +1,42 @@ +# Prometheus configuration for Zebra metrics and tracing +# +# IMPORTANT: The scrape_interval affects Grafana dashboard queries. +# The rate() function requires at least 2 data points, so rate windows +# must be > 2x scrape_interval. With 15s scrape_interval: +# - rate(...[1m]) works (4 samples) +# - rate(...[30s]) works (2 samples minimum) +# - rate(...[1s]) does NOT work (insufficient samples) +# +# If you change scrape_interval, update dashboard rate() windows accordingly. + +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/rules/*.yml + +alerting: + alertmanagers: + - static_configs: + - targets: ['alertmanager:9093'] + +scrape_configs: + # Zebra node metrics (z3 stack uses z3_zebra container name) + - job_name: "zebra" + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: "/metrics" + static_configs: + - targets: ["zebra:9999"] + labels: + stack: "z3" + + # Jaeger spanmetrics for RPC tracing + # These metrics are generated by the spanmetrics connector in Jaeger + # and provide RED metrics (Rate, Errors, Duration) for RPC calls + - job_name: "jaeger-spanmetrics" + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: ["jaeger:8889"] diff --git a/observability/prometheus/rules/zebra_alerts.yml b/observability/prometheus/rules/zebra_alerts.yml new file mode 100644 index 0000000..688ea07 --- /dev/null +++ b/observability/prometheus/rules/zebra_alerts.yml @@ -0,0 +1,149 @@ +# Zebra Alerting Rules +# +# These rules are evaluated by Prometheus and sent to AlertManager. +# AlertManager handles routing, grouping, and notifications. +# +# To receive notifications, configure receivers in: +# docker/observability/alertmanager/alertmanager.yml + +groups: + - name: zebra + rules: + # Node is down or metrics endpoint unreachable + - alert: ZebraDown + expr: up{job="zebra"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Zebra node is down" + description: "Zebra metrics endpoint has been unreachable for 2 minutes." + + # Block height not increasing (sync may be stalled) + - alert: ZebraSyncStalled + expr: changes(zcash_chain_verified_block_height[15m]) == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Block height stalled" + description: "Zebra block height has not increased in the last 15 minutes. Node may be stuck or network may be partitioned." + + # Low peer count (degraded connectivity) + - alert: ZebraLowPeers + expr: zcash_net_peers < 3 + for: 5m + labels: + severity: warning + annotations: + summary: "Low peer count" + description: "Zebra has fewer than 3 peers for 5 minutes. Network connectivity may be degraded." + + # No peers at all (network isolation) + - alert: ZebraNoPeers + expr: zcash_net_peers == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "No peers connected" + description: "Zebra has 0 peers for 2 minutes. Check network connectivity and firewall rules." + + # High error rate + - alert: ZebraHighErrorRate + expr: rate(zebra_errors_total[5m]) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High error rate" + description: "Zebra is experiencing elevated error rates (> 0.1/s for 5 minutes)." + + # Value Pool Alerts (Monitoring) + - name: zebra-value-pools + rules: + # Critical: Negative value pool would indicate a bug (should never happen - Zebra rejects such blocks) + - alert: ValuePoolNegative + expr: state_finalized_value_pool_transparent < 0 or state_finalized_value_pool_sprout < 0 or state_finalized_value_pool_sapling < 0 or state_finalized_value_pool_orchard < 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Negative value pool detected" + description: "A pool has a negative balance in metrics. Zebra enforces ZIP-209 internally, so this should not occur in normal operation." + + # Warning: Value pool not updating (may indicate sync issues) + - alert: ValuePoolStale + expr: changes(state_finalized_chain_supply_total[15m]) == 0 and state_finalized_block_height > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Value pool not updating" + description: "Chain supply has not changed in 15 minutes while blocks exist. Sync may be stalled." + + # RPC Alerts + - name: zebra-rpc + rules: + # High RPC latency + - alert: RPCHighLatency + expr: histogram_quantile(0.99, sum(rate(rpc_request_duration_seconds_bucket[5m])) by (le, method)) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "High RPC latency detected" + description: "RPC method {{ $labels.method }} has p99 latency > 2 seconds for 5 minutes." + + # High RPC error rate + - alert: RPCHighErrorRate + expr: sum(rate(rpc_errors_total[5m])) by (method) / sum(rate(rpc_requests_total[5m])) by (method) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High RPC error rate" + description: "RPC method {{ $labels.method }} has error rate > 10% for 5 minutes." + + # RPC endpoint overloaded + - alert: RPCOverloaded + expr: rpc_active_requests > 100 + for: 2m + labels: + severity: warning + annotations: + summary: "RPC endpoint overloaded" + description: "More than 100 concurrent RPC requests for 2 minutes. Consider rate limiting." + + # Peer Health Alerts + - name: zebra-peer-health + rules: + # High handshake failure rate + - alert: HandshakeFailureRateHigh + expr: sum(rate(zcash_net_peer_handshake_failures_total[5m])) / (sum(rate(zcash_net_peer_handshake_duration_seconds_count[5m])) + 0.001) > 0.5 + for: 5m + labels: + severity: warning + annotations: + summary: "High peer handshake failure rate" + description: "More than 50% of peer handshakes are failing for 5 minutes. Check network connectivity." + + # Specific failure reason spike (e.g., obsolete version) + - alert: ObsoleteVersionHandshakes + expr: rate(zcash_net_peer_handshake_failures_total{reason="obsolete_version"}[5m]) > 0.1 + for: 5m + labels: + severity: info + annotations: + summary: "Many obsolete version handshakes" + description: "Elevated rate of peer handshake failures due to obsolete protocol versions. May indicate network upgrade in progress." + + # Slow handshakes + - alert: SlowHandshakes + expr: histogram_quantile(0.95, sum(rate(zcash_net_peer_handshake_duration_seconds_bucket{result="success"}[5m])) by (le)) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Slow peer handshakes" + description: "p95 handshake duration exceeds 10 seconds. Network latency may be high." From 5c95431745c2750a514b05e651140bc4a5d6bac0 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 12 Feb 2026 09:48:51 +0000 Subject: [PATCH 4/6] chore(docker): update Zebra to 4.1.0 --- README.md | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76a4f1d..40b1347 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ docker compose ps | Service | Image | Source | |---------|-------|--------| -| **Zebra** | `zfnd/zebra:4.0.0` | [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) | +| **Zebra** | `zfnd/zebra:4.1.0` | [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) | | **Zaino** | `ghcr.io/zcashfoundation/zaino:sha-1871eba` | [ZcashFoundation/zaino](https://github.com/ZcashFoundation/zaino) | | **Zallet** | `electriccoinco/zallet:v0.1.0-alpha.3` | [Electric Coin Co](https://github.com/Electric-Coin-Company/zallet) | diff --git a/docker-compose.yml b/docker-compose.yml index ccbf839..63847f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: zebra: # Run 'docker compose build' to build locally from ./zebra submodule instead - image: zfnd/zebra:4.0.0 + image: zfnd/zebra:4.1.0 build: context: ./zebra dockerfile: docker/Dockerfile From 2056084e4718ff34b5a02269656b4174de228135 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 12 Feb 2026 10:00:01 +0000 Subject: [PATCH 5/6] docs(observability): update OpenTelemetry tracing instructions for Zebra 4.1.0 OpenTelemetry tracing is available in Zebra 4.x but requires building with the `opentelemetry` cargo feature. Add build instructions and environment variable configuration for enabling tracing with Jaeger. --- .env | 8 ++++++-- docker-compose.yml | 3 +++ observability/README.md | 22 +++++++++++++++++----- observability/jaeger/README.md | 7 ++++--- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.env b/.env index 0520b29..d2049bd 100644 --- a/.env +++ b/.env @@ -166,8 +166,12 @@ ZALLET_DATA_DIR=/home/zallet/.data # To enable Zebra metrics, uncomment this variable: ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999 # -# NOTE: OpenTelemetry tracing (Jaeger) is not yet available in Zebra 4.0.0. -# It will be supported in a future Zebra release. +# To enable OpenTelemetry tracing (Jaeger), build Zebra with OTel support: +# docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra +# Then set the tracing endpoint: +# ZEBRA_TRACING__OPENTELEMETRY_ENDPOINT=http://jaeger:4318 +# ZEBRA_TRACING__OPENTELEMETRY_SERVICE_NAME=zebra +# ZEBRA_TRACING__OPENTELEMETRY_SAMPLE_PERCENT=100 # # Service ports (defaults shown, customize if needed): # GRAFANA_PORT=3000 diff --git a/docker-compose.yml b/docker-compose.yml index 63847f5..6c0446b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,14 @@ services: zebra: # Run 'docker compose build' to build locally from ./zebra submodule instead + # For OpenTelemetry tracing: docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra image: zfnd/zebra:4.1.0 build: context: ./zebra dockerfile: docker/Dockerfile target: runtime + args: + FEATURES: ${ZEBRA_BUILD_FEATURES:-default-release-binaries} container_name: z3_zebra restart: unless-stopped env_file: diff --git a/observability/README.md b/observability/README.md index 287e5f2..ed4c5d6 100644 --- a/observability/README.md +++ b/observability/README.md @@ -15,8 +15,8 @@ docker compose --profile monitoring up -d docker compose logs -f zebra ``` -> **Note**: OpenTelemetry tracing (Jaeger) is not yet available in Zebra 4.0.0. -> Jaeger is included for use with future Zebra releases that support tracing. +> **Note**: OpenTelemetry tracing requires building Zebra with the `opentelemetry` feature. +> The pre-built Docker image does not include it. See the [Tracing section](#tracing-jaeger) for build instructions. ## Components @@ -82,10 +82,22 @@ See [grafana/README.md](grafana/README.md) for dashboard details. ### Tracing (Jaeger) -> **Note**: OpenTelemetry tracing is not yet available in Zebra 4.0.0. -> This feature will be supported in a future Zebra release. +Distributed tracing via OpenTelemetry. Requires building Zebra with the `opentelemetry` feature (not included in the pre-built image): -Once available, Jaeger will provide: +```bash +# Build Zebra with OpenTelemetry support +docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra +``` + +Then enable tracing in `.env`: + +```bash +ZEBRA_TRACING__OPENTELEMETRY_ENDPOINT=http://jaeger:4318 +ZEBRA_TRACING__OPENTELEMETRY_SERVICE_NAME=zebra +ZEBRA_TRACING__OPENTELEMETRY_SAMPLE_PERCENT=100 +``` + +Jaeger provides: - **Distributed traces**: Follow a request through all components - **Latency breakdown**: See where time is spent in each operation diff --git a/observability/jaeger/README.md b/observability/jaeger/README.md index 6f74d88..18d7df2 100644 --- a/observability/jaeger/README.md +++ b/observability/jaeger/README.md @@ -320,9 +320,10 @@ These power the Monitor tab and can be scraped by Prometheus for Grafana dashboa ### No traces appearing -1. Check OTLP endpoint: `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318` -2. Verify Jaeger health: http://localhost:16686 -3. Check Jaeger logs: `docker compose logs jaeger` +1. Ensure Zebra was built with OTel support: `docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra` +2. Check tracing env vars are set: `ZEBRA_TRACING__OPENTELEMETRY_ENDPOINT=http://jaeger:4318` +3. Verify Jaeger health: http://localhost:16686 +4. Check Jaeger logs: `docker compose logs jaeger` ### Monitor tab shows "No Data" From e0853fc56f9a95ba96d8d061a4752930a94bbd6b Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 12 Feb 2026 10:07:19 +0000 Subject: [PATCH 6/6] fix(docs): correct Zaino and Zallet source repository URLs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40b1347..145a68b 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,8 @@ docker compose ps | Service | Image | Source | |---------|-------|--------| | **Zebra** | `zfnd/zebra:4.1.0` | [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) | -| **Zaino** | `ghcr.io/zcashfoundation/zaino:sha-1871eba` | [ZcashFoundation/zaino](https://github.com/ZcashFoundation/zaino) | -| **Zallet** | `electriccoinco/zallet:v0.1.0-alpha.3` | [Electric Coin Co](https://github.com/Electric-Coin-Company/zallet) | +| **Zaino** | `ghcr.io/zcashfoundation/zaino:sha-1871eba` | [zingolabs/zaino](https://github.com/zingolabs/zaino) | +| **Zallet** | `electriccoinco/zallet:v0.1.0-alpha.3` | [zcash/wallet](https://github.com/zcash/wallet) | ### Building Local Images (Optional)