feat(docker): add monitoring stack with Prometheus, Grafana, Jaeger, AlertManager

.env

@@ -72,8 +72,8 @@ Z3_ZALLET_DATA_PATH=zallet_data
 # =============================================================================
 # Shared variables used by multiple services, mapped in docker-compose.yml:
 #   NETWORK_NAME → ZEBRA_NETWORK__NETWORK, ZAINO_NETWORK
-#   ENABLE_COOKIE_AUTH → ZEBRA_RPC__ENABLE_COOKIE_AUTH, ZAINO_VALIDATOR_COOKIE_AUTH
-#   COOKIE_AUTH_FILE_DIR → ZEBRA_RPC__COOKIE_DIR, ZAINO_VALIDATOR_COOKIE_PATH
+#   ENABLE_COOKIE_AUTH → ZEBRA_RPC__ENABLE_COOKIE_AUTH
+#   COOKIE_AUTH_FILE_DIR → ZEBRA_RPC__COOKIE_DIR, ZAINO_VALIDATOR_SETTINGS__VALIDATOR_COOKIE_PATH
 
 # Network name for all services (e.g., Mainnet, Testnet, Regtest)
 NETWORK_NAME=Mainnet
@@ -154,6 +154,30 @@ ZALLET_CONF_PATH=/etc/zallet/zallet.toml
 # Zallet application internal data directory
 ZALLET_DATA_DIR=/home/zallet/.data
 # Example path for a CA certificate file that Zallet might use to trust Zaino's gRPC TLS certificate.
-# If Zaino uses a self-signed certificate or a certificate from a private CA, Zallet would need to be 
+# If Zaino uses a self-signed certificate or a certificate from a private CA, Zallet would need to be
 # configured to trust it. The actual environment variable name and mechanism depend on Zallet's implementation.
 # ZALLET_INDEXER_CA_PATH=/path/to/trusted/zaino_ca.crt
+
+# =============================================================================
+# Monitoring Configuration (--profile monitoring)
+# =============================================================================
+# Enable monitoring with: docker compose --profile monitoring up -d
+#
+# To enable Zebra metrics, uncomment this variable:
+ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999
+#
+# To enable OpenTelemetry tracing (Jaeger), build Zebra with OTel support:
+#   docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra
+# Then set the tracing endpoint:
+# ZEBRA_TRACING__OPENTELEMETRY_ENDPOINT=http://jaeger:4318
+# ZEBRA_TRACING__OPENTELEMETRY_SERVICE_NAME=zebra
+# ZEBRA_TRACING__OPENTELEMETRY_SAMPLE_PERCENT=100
+#
+# Service ports (defaults shown, customize if needed):
+# GRAFANA_PORT=3000
+# PROMETHEUS_PORT=9094
+# JAEGER_UI_PORT=16686
+# ALERTMANAGER_PORT=9093
+#
+# Grafana admin password (default: admin, prompted to change on first login):
+# GRAFANA_ADMIN_PASSWORD=your_secure_password

.gitignore

@@ -1,3 +1,6 @@
+# macOS
+.DS_Store
+
 # Generated by Cargo
 # will have compiled files and executables
 debug/
@@ -29,5 +32,9 @@ config/tls/*
 # Then un-ignore the .gitkeep file within 'tls'
 !config/tls/.gitkeep
 
+# Zaino config (required by config-rs)
+!config/zaino/
+!config/zaino/zindexer.toml
+
 # Un-ignore .gitkeep directly under config
 !config/.gitkeep

README.md

@@ -108,37 +108,28 @@ docker compose ps
 
 ## Docker Images
 
-> [!IMPORTANT]
-> **Current Status:** Zaino and Zallet require local builds. Pre-built images are available for Zebra only.
-
 ### Image Sources
 
 | Service | Image | Source |
 |---------|-------|--------|
-| **Zebra** | `zfnd/zebra:3.1.0` | Pre-built from [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) |
-| **Zaino** | `z3-zaino:local` | Must build locally from submodule |
-| **Zallet** | `z3-zallet:local` | Must build locally from submodule |
+| **Zebra** | `zfnd/zebra:4.1.0` | [ZcashFoundation/zebra](https://github.com/ZcashFoundation/zebra) |
+| **Zaino** | `ghcr.io/zcashfoundation/zaino:sha-1871eba` | [zingolabs/zaino](https://github.com/zingolabs/zaino) |
+| **Zallet** | `electriccoinco/zallet:v0.1.0-alpha.3` | [zcash/wallet](https://github.com/zcash/wallet) |
 
-### Building Local Images
+### Building Local Images (Optional)
+
+To build from local submodules instead of using pre-built images:
 
 ```bash
 # Initialize submodules
 git submodule update --init --recursive
 
-# Build zaino and zallet
-docker compose build zaino zallet
+# Build all services locally
+docker compose build
 ```
 
 > [!NOTE]
-> Local builds are required because Zaino and Zallet are under active development and require specific version pinning for compatibility.
-
-### Why Local Builds?
-
-Zallet embeds Zaino libraries internally. Both must use compatible versions of the Zaino codebase. The submodules in this repository are pinned to tested, compatible commits.
-
-**For production deployments**, use official release images when available:
-- Zebra: [zfnd/zebra](https://hub.docker.com/r/zfnd/zebra) (stable releases)
-- Zaino/Zallet: Official releases when published
+> The submodules in this repository are pinned to tested, compatible commits if you prefer to build locally.
 
 ## Prerequisites
 

config/zaino/zindexer.toml

@@ -0,0 +1 @@
+# Minimal Zaino config - most settings come from environment variables

config/zallet.toml

@@ -21,7 +21,7 @@ network = "main"
 # External settings - using defaults
 
 [features]
-as_of_version = "0.1.0-alpha.1"
+as_of_version = "0.1.0-alpha.3"
 
 [features.deprecated]
 # No deprecated features enabled
@@ -34,14 +34,12 @@ as_of_version = "0.1.0-alpha.1"
 # to fetch blockchain data. The validator_address MUST point to Zebra (not the
 # standalone Zaino service). Service name 'zebra' and port from Z3_ZEBRA_RPC_PORT in .env
 validator_address = "zebra:18232"
-
-# Cookie authentication (matches Zebra's ENABLE_COOKIE_AUTH=true)
-validator_cookie_auth = true
+# Cookie authentication path (matches Zebra's ENABLE_COOKIE_AUTH=true)
 validator_cookie_path = "/var/run/auth/.cookie"
 
 [keystore]
 # Age encryption identity file (mounted from ./config/zallet_identity.txt)
-encryption_identity = "identity.txt"
+encryption_identity = "/etc/zallet/identity.txt"
 
 [note_management]
 # Note management - using defaults

docker-compose.yml

@@ -1,11 +1,14 @@
 services:
   zebra:
     # Run 'docker compose build' to build locally from ./zebra submodule instead
-    image: zfnd/zebra:3.1.0
+    # For OpenTelemetry tracing: docker compose build --build-arg FEATURES="default-release-binaries opentelemetry" zebra
+    image: zfnd/zebra:4.1.0
     build:
       context: ./zebra
       dockerfile: docker/Dockerfile
       target: runtime
+      args:
+        FEATURES: ${ZEBRA_BUILD_FEATURES:-default-release-binaries}
     container_name: z3_zebra
     restart: unless-stopped
     env_file:
@@ -36,41 +39,37 @@ services:
       start_period: 90s
 
   zaino:
-    # No pre-built image available for this commit - must build locally
-    # Built from zingolabs/zaino:dev - commit 66b3199f (before config restructure)
-    # Run 'docker compose build zaino' to build from ./zaino submodule
-    image: ghcr.io/zcashfoundation/zaino:sha-934f857
+    # No official image from zingolabs yet - built from zingolabs/zaino:dev
+    image: ghcr.io/zcashfoundation/zaino:sha-1871eba
     platform: ${DOCKER_PLATFORM:-linux/amd64}
     build:
       context: ./zaino
       dockerfile: Dockerfile
     container_name: z3_zaino
     restart: unless-stopped
+    command: ["zainod", "--config", "/etc/zaino/zindexer.toml"]
     depends_on:
       zebra:
         condition: service_healthy
-    env_file:
-      - ./.env
     environment:
       - RUST_LOG=${ZAINO_RUST_LOG}
       - RUST_BACKTRACE=${ZAINO_RUST_BACKTRACE}
       - ZAINO_NETWORK=${NETWORK_NAME}
-      # Zebra connection and authentication
-      - ZAINO_VALIDATOR_LISTEN_ADDRESS=zebra:${Z3_ZEBRA_RPC_PORT}
-      - ZAINO_VALIDATOR_COOKIE_AUTH=${ENABLE_COOKIE_AUTH}
-      - ZAINO_VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie
-      # Zaino RPC services
-      - ZAINO_GRPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_GRPC_PORT}
-      - ZAINO_JSON_RPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_JSON_RPC_PORT}
+      - ZAINO_VALIDATOR_SETTINGS__VALIDATOR_JSONRPC_LISTEN_ADDRESS=zebra:${Z3_ZEBRA_RPC_PORT}
+      - ZAINO_VALIDATOR_SETTINGS__VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie
+      # gRPC server
+      - ZAINO_GRPC_SETTINGS__LISTEN_ADDRESS=0.0.0.0:${ZAINO_GRPC_PORT}
+      # JSON-RPC server
+      - ZAINO_JSON_SERVER_SETTINGS__JSON_RPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_JSON_RPC_PORT}
       # TLS configuration
-      - ZAINO_GRPC_TLS=${ZAINO_GRPC_TLS_ENABLE}
-      - ZAINO_TLS_CERT_PATH=${ZAINO_GRPC_TLS_CERT_PATH}
-      - ZAINO_TLS_KEY_PATH=${ZAINO_GRPC_TLS_KEY_PATH}
+      - ZAINO_GRPC_SETTINGS__TLS__CERT_PATH=${ZAINO_GRPC_TLS_CERT_PATH}
+      - ZAINO_GRPC_SETTINGS__TLS__KEY_PATH=${ZAINO_GRPC_TLS_KEY_PATH}
     volumes:
       # Indexer state (defaults to named volume 'zaino_data')
       - ${Z3_ZAINO_DATA_PATH}:/home/zaino/.cache/zaino
       # Cookie authentication
       - ${Z3_COOKIE_PATH}:${COOKIE_AUTH_FILE_DIR}:ro
+      - ./config/zaino/zindexer.toml:/etc/zaino/zindexer.toml:ro
     configs:
       - source: zaino_tls_cert
         target: ${ZAINO_GRPC_TLS_CERT_PATH}
@@ -90,18 +89,12 @@ services:
       start_period: 60s
 
   zallet:
-    # No pre-built image available - must build locally
-    # Built from zcash/wallet commit 60b0235 (uses zaino 13919816, before config restructure)
-    # Run 'docker compose build zallet' to build from ./zallet submodule
-    image: z3-zallet:local
+    image: electriccoinco/zallet:v0.1.0-alpha.3
     platform: ${DOCKER_PLATFORM:-linux/amd64}
-    build:
-      context: ./zallet
-      dockerfile: Dockerfile
     container_name: z3_zallet
     restart: unless-stopped
-    user: "65532:65532"
-    command: ["--datadir", "/var/lib/zallet", "start"]
+    user: "1000:1000"
+    command: ["--datadir", "/var/lib/zallet", "--config", "/etc/zallet/zallet.toml", "start"]
     depends_on:
       zebra:
         condition: service_healthy
@@ -116,22 +109,133 @@ services:
       - ${Z3_ZALLET_DATA_PATH}:/var/lib/zallet
       # Cookie authentication for Zebra access
       - ${Z3_COOKIE_PATH}:${COOKIE_AUTH_FILE_DIR}:ro
-      # Configuration files
-      - ./config/zallet.toml:/var/lib/zallet/zallet.toml:ro
-      - ./config/zallet_identity.txt:/var/lib/zallet/identity.txt:ro
+      # Configuration files (mounted outside datadir to avoid permission conflicts)
+      - ./config/zallet.toml:/etc/zallet/zallet.toml:ro
+      - ./config/zallet_identity.txt:/etc/zallet/identity.txt:ro
     ports:
       - "${ZALLET_HOST_RPC_PORT}:${ZALLET_RPC_PORT}"
     networks:
       - z3_net
-    # NOTE: Healthcheck disabled - distroless image has no shell/curl
-    # Zallet will restart automatically if it crashes (restart: unless-stopped)
-    # Monitor logs or use external monitoring for service health
+    # No healthcheck: distroless image has no shell/curl
+
+  # =============================================================================
+  # Monitoring Stack (enabled with --profile monitoring)
+  # =============================================================================
+  # Usage: docker compose --profile monitoring up -d
+  # Access:
+  #   - Grafana:      http://localhost:3000 (admin/admin)
+  #   - Prometheus:   http://localhost:9094
+  #   - Jaeger:       http://localhost:16686
+  #   - AlertManager: http://localhost:9093
+
+  jaeger:
+    image: jaegertracing/jaeger:2.1.0
+    container_name: z3_jaeger
+    profiles: [monitoring]
+    restart: unless-stopped
+    volumes:
+      - ./observability/jaeger/config.yaml:/etc/jaeger/config.yaml:ro
+    command:
+      - --config=/etc/jaeger/config.yaml
+    ports:
+      - "${JAEGER_UI_PORT:-16686}:16686"
+      - "${JAEGER_OTLP_GRPC_PORT:-4317}:4317"
+      - "${JAEGER_OTLP_HTTP_PORT:-4318}:4318"
+      - "${JAEGER_SPANMETRICS_PORT:-8889}:8889"
+    networks:
+      - z3_net
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:16686/ || exit 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+
+  prometheus:
+    image: prom/prometheus:v3.2.0
+    container_name: z3_prometheus
+    profiles: [monitoring]
+    restart: unless-stopped
+    volumes:
+      - prometheus_data:/prometheus
+      - ./observability/prometheus/rules:/etc/prometheus/rules:ro
+    configs:
+      - source: prometheus_config
+        target: /etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.enable-lifecycle'
+      - '--web.enable-admin-api'
+    ports:
+      - "${PROMETHEUS_PORT:-9094}:9090"
+    networks:
+      - z3_net
+    depends_on:
+      jaeger:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:9090/-/healthy || exit 1"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+
+  grafana:
+    image: grafana/grafana:11.5.1
+    container_name: z3_grafana
+    profiles: [monitoring]
+    restart: unless-stopped
+    volumes:
+      - ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - ./observability/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/zebra_overview.json
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    networks:
+      - z3_net
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      jaeger:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/api/health || exit 1"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 10s
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1
+    container_name: z3_alertmanager
+    profiles: [monitoring]
+    restart: unless-stopped
+    volumes:
+      - ./observability/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/alertmanager.yml'
+      - '--storage.path=/alertmanager'
+    ports:
+      - "${ALERTMANAGER_PORT:-9093}:9093"
+    networks:
+      - z3_net
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:9093/-/healthy || exit 1"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
 
 volumes:
   zebra_data:
   zaino_data:
   zallet_data:
   shared_cookie_volume:
+  prometheus_data:
+  grafana_data:
+  alertmanager_data:
 
 networks:
   z3_net:
@@ -142,3 +246,5 @@ configs:
     file: ./config/tls/zaino.crt
   zaino_tls_key:
     file: ./config/tls/zaino.key
+  prometheus_config:
+    file: ./observability/prometheus/prometheus.yaml