From bf26e2cd47684c45ef7f38c806e9df2cf7fbaef6 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 14 May 2025 09:22:53 +0100 Subject: [PATCH 01/11] fix(workflows): correct file extension in GitHub Actions workflow paths --- .github/workflows/build-z3-images.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index 1fa5a7f..4098991 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -10,11 +10,11 @@ on: branches: - dev paths: - - '.github/workflows/build-z3-images.yml' + - '.github/workflows/build-z3-images.yaml' pull_request: paths: - - '.github/workflows/build-z3-images.yml' + - '.github/workflows/build-z3-images.yaml' jobs: build: From 5c0e7bfed7bf1926088dd345511e074b91216631 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 14 May 2025 10:06:58 +0100 Subject: [PATCH 02/11] chore(workflows): change to a forked zaino repo --- .github/workflows/build-z3-images.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index 4098991..33c6ad6 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -24,10 +24,10 @@ jobs: id-token: 'write' uses: ./.github/workflows/sub-build-docker-image.yaml with: - repository: zingolabs/zaino - ref: 'dev' + repository: gustavovalverde/zaino + ref: 'imp-dockerfile' dockerfile_path: ./Dockerfile - dockerfile_target: tests + dockerfile_target: runtime image_name: zaino no_cache: ${{ inputs.no_cache || false }} rust_backtrace: full From 74e820759457cef38033b1816708c31272b74899 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 14 May 2025 11:21:24 +0100 Subject: [PATCH 03/11] chore(workflows): fix permissions issue --- .github/workflows/build-z3-images.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index 33c6ad6..dbaa57d 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -6,6 +6,12 @@ concurrency: on: workflow_dispatch: + inputs: + no_cache: + description: "Disable Docker cache for this build" + required: false + type: boolean + default: false push: branches: - dev @@ -22,6 +28,7 @@ jobs: permissions: contents: 'read' id-token: 'write' + packages: 'write' uses: ./.github/workflows/sub-build-docker-image.yaml with: repository: gustavovalverde/zaino From df9ff54ae5afaf22da1fc53a81940b7e61fdcd21 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 14 May 2025 12:53:15 +0100 Subject: [PATCH 04/11] chore(workflows): fix permissions and cache --- .github/workflows/sub-build-docker-image.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sub-build-docker-image.yaml b/.github/workflows/sub-build-docker-image.yaml index 688ae7d..99b3fd2 100644 --- a/.github/workflows/sub-build-docker-image.yaml +++ b/.github/workflows/sub-build-docker-image.yaml @@ -68,6 +68,7 @@ jobs: permissions: contents: "read" id-token: "write" + packages: "write" env: DOCKER_BUILD_SUMMARY: ${{ vars.DOCKER_BUILD_SUMMARY }} steps: @@ -146,3 +147,5 @@ jobs: # Don't read from the cache if the caller disabled it. # https://docs.docker.com/engine/reference/commandline/buildx_build/#options no-cache: ${{ inputs.no_cache }} + cache-from: type=gha,scope=z3-${{ inputs.image_name }} + cache-to: type=gha,mode=max,scope=z3-${{ inputs.image_name }} From 1040201b6e55994675ba861986beaffcbe33ee91 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 14 May 2025 13:24:16 +0100 Subject: [PATCH 05/11] test: try to build without a repo --- .github/workflows/sub-build-docker-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sub-build-docker-image.yaml b/.github/workflows/sub-build-docker-image.yaml index 99b3fd2..cb4483f 100644 --- a/.github/workflows/sub-build-docker-image.yaml +++ b/.github/workflows/sub-build-docker-image.yaml @@ -100,7 +100,7 @@ jobs: # We only publish images to DockerHub if a release is not a pre-release # Ref: https://github.com/orgs/community/discussions/26281#discussioncomment-3251177 images: | - ghcr.io/${{ env.GITHUB_REPOSITORY_OWNER_PART }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }} + ghcr.io/${{ env.GITHUB_REPOSITORY_OWNER_PART }}/${{ inputs.image_name }} # generate Docker tags based on the following events/attributes tags: | # - `pr-xxx`: Tags images with the pull request number. From 2df84b182d850e197d28684e8648d7c051b8c303 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 15 May 2025 12:11:58 +0100 Subject: [PATCH 06/11] feat(workflows): add zallet Docker build job * Renamed the existing build job to 'build-zaino' for clarity. * Introduced a new job 'build-zallet' to build the zallet Docker image with specified permissions and parameters. --- .github/workflows/build-z3-images.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index dbaa57d..7012cc3 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -23,7 +23,7 @@ on: - '.github/workflows/build-z3-images.yaml' jobs: - build: + build-zaino: name: Build zaino Docker permissions: contents: 'read' @@ -40,3 +40,21 @@ jobs: rust_backtrace: full rust_lib_backtrace: full rust_log: info + + build-zallet: + name: Build zallet Docker + permissions: + contents: 'read' + id-token: 'write' + packages: 'write' + uses: ./.github/workflows/sub-build-docker-image.yaml + with: + repository: gustavovalverde/wallet + ref: 'feat-add-docker' + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runtime + image_name: zallet + no_cache: ${{ inputs.no_cache || false }} + rust_backtrace: full + rust_lib_backtrace: full + rust_log: info From a36a1d576100080889953d72b43c5e92658bd157 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 15 May 2025 18:00:36 +0100 Subject: [PATCH 07/11] fix(workflows): update Dockerfile path in build-z3-images workflow --- .github/workflows/build-z3-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index 7012cc3..8e7d4cd 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -33,7 +33,7 @@ jobs: with: repository: gustavovalverde/zaino ref: 'imp-dockerfile' - dockerfile_path: ./Dockerfile + dockerfile_path: ./docker/Dockerfile dockerfile_target: runtime image_name: zaino no_cache: ${{ inputs.no_cache || false }} From 55fcb6eddb4736f3372dc0d9f0fa76d603782c8c Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 15 May 2025 19:01:59 +0100 Subject: [PATCH 08/11] docs: rename `memory-bank` to `docs` --- {memory-bank => docs}/activeContext.md | 0 {memory-bank => docs}/data/ecosystem_map.md | 0 {memory-bank => docs}/data/rpc_mapping.md | 0 {memory-bank => docs}/data/zcashd_RPC_methods.md | 0 .../data/zcashd_deprecation_team_rpc_method_support_9apr2025.md | 0 .../data/zebra_adoption_tracking/analyze_nodes.sh | 0 {memory-bank => docs}/productContext.md | 0 {memory-bank => docs}/progress.md | 0 {memory-bank => docs}/projectbrief.md | 0 {memory-bank => docs}/systemPatterns.md | 0 {memory-bank => docs}/techContext.md | 0 {memory-bank => docs}/z3_component_interplay.md | 0 {memory-bank => docs}/z3_docker.md | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename {memory-bank => docs}/activeContext.md (100%) rename {memory-bank => docs}/data/ecosystem_map.md (100%) rename {memory-bank => docs}/data/rpc_mapping.md (100%) rename {memory-bank => docs}/data/zcashd_RPC_methods.md (100%) rename {memory-bank => docs}/data/zcashd_deprecation_team_rpc_method_support_9apr2025.md (100%) rename {memory-bank => docs}/data/zebra_adoption_tracking/analyze_nodes.sh (100%) rename {memory-bank => docs}/productContext.md (100%) rename {memory-bank => docs}/progress.md (100%) rename {memory-bank => docs}/projectbrief.md (100%) rename {memory-bank => docs}/systemPatterns.md (100%) rename {memory-bank => docs}/techContext.md (100%) rename {memory-bank => docs}/z3_component_interplay.md (100%) rename {memory-bank => docs}/z3_docker.md (100%) diff --git a/memory-bank/activeContext.md b/docs/activeContext.md similarity index 100% rename from memory-bank/activeContext.md rename to docs/activeContext.md diff --git a/memory-bank/data/ecosystem_map.md b/docs/data/ecosystem_map.md similarity index 100% rename from memory-bank/data/ecosystem_map.md rename to docs/data/ecosystem_map.md diff --git a/memory-bank/data/rpc_mapping.md b/docs/data/rpc_mapping.md similarity index 100% rename from memory-bank/data/rpc_mapping.md rename to docs/data/rpc_mapping.md diff --git a/memory-bank/data/zcashd_RPC_methods.md b/docs/data/zcashd_RPC_methods.md similarity index 100% rename from memory-bank/data/zcashd_RPC_methods.md rename to docs/data/zcashd_RPC_methods.md diff --git a/memory-bank/data/zcashd_deprecation_team_rpc_method_support_9apr2025.md b/docs/data/zcashd_deprecation_team_rpc_method_support_9apr2025.md similarity index 100% rename from memory-bank/data/zcashd_deprecation_team_rpc_method_support_9apr2025.md rename to docs/data/zcashd_deprecation_team_rpc_method_support_9apr2025.md diff --git a/memory-bank/data/zebra_adoption_tracking/analyze_nodes.sh b/docs/data/zebra_adoption_tracking/analyze_nodes.sh similarity index 100% rename from memory-bank/data/zebra_adoption_tracking/analyze_nodes.sh rename to docs/data/zebra_adoption_tracking/analyze_nodes.sh diff --git a/memory-bank/productContext.md b/docs/productContext.md similarity index 100% rename from memory-bank/productContext.md rename to docs/productContext.md diff --git a/memory-bank/progress.md b/docs/progress.md similarity index 100% rename from memory-bank/progress.md rename to docs/progress.md diff --git a/memory-bank/projectbrief.md b/docs/projectbrief.md similarity index 100% rename from memory-bank/projectbrief.md rename to docs/projectbrief.md diff --git a/memory-bank/systemPatterns.md b/docs/systemPatterns.md similarity index 100% rename from memory-bank/systemPatterns.md rename to docs/systemPatterns.md diff --git a/memory-bank/techContext.md b/docs/techContext.md similarity index 100% rename from memory-bank/techContext.md rename to docs/techContext.md diff --git a/memory-bank/z3_component_interplay.md b/docs/z3_component_interplay.md similarity index 100% rename from memory-bank/z3_component_interplay.md rename to docs/z3_component_interplay.md diff --git a/memory-bank/z3_docker.md b/docs/z3_docker.md similarity index 100% rename from memory-bank/z3_docker.md rename to docs/z3_docker.md From ef60d2a1031aae6c6e21e9cd77fa9a712505b7a8 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Thu, 15 May 2025 19:04:59 +0100 Subject: [PATCH 09/11] docs: move some docs to memory-bank --- docs/{ => memory-bank}/activeContext.md | 0 docs/{ => memory-bank}/productContext.md | 0 docs/{ => memory-bank}/progress.md | 0 docs/{ => memory-bank}/projectbrief.md | 0 docs/{ => memory-bank}/systemPatterns.md | 0 docs/{ => memory-bank}/techContext.md | 0 docs/{ => memory-bank}/z3_component_interplay.md | 0 docs/{ => memory-bank}/z3_docker.md | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename docs/{ => memory-bank}/activeContext.md (100%) rename docs/{ => memory-bank}/productContext.md (100%) rename docs/{ => memory-bank}/progress.md (100%) rename docs/{ => memory-bank}/projectbrief.md (100%) rename docs/{ => memory-bank}/systemPatterns.md (100%) rename docs/{ => memory-bank}/techContext.md (100%) rename docs/{ => memory-bank}/z3_component_interplay.md (100%) rename docs/{ => memory-bank}/z3_docker.md (100%) diff --git a/docs/activeContext.md b/docs/memory-bank/activeContext.md similarity index 100% rename from docs/activeContext.md rename to docs/memory-bank/activeContext.md diff --git a/docs/productContext.md b/docs/memory-bank/productContext.md similarity index 100% rename from docs/productContext.md rename to docs/memory-bank/productContext.md diff --git a/docs/progress.md b/docs/memory-bank/progress.md similarity index 100% rename from docs/progress.md rename to docs/memory-bank/progress.md diff --git a/docs/projectbrief.md b/docs/memory-bank/projectbrief.md similarity index 100% rename from docs/projectbrief.md rename to docs/memory-bank/projectbrief.md diff --git a/docs/systemPatterns.md b/docs/memory-bank/systemPatterns.md similarity index 100% rename from docs/systemPatterns.md rename to docs/memory-bank/systemPatterns.md diff --git a/docs/techContext.md b/docs/memory-bank/techContext.md similarity index 100% rename from docs/techContext.md rename to docs/memory-bank/techContext.md diff --git a/docs/z3_component_interplay.md b/docs/memory-bank/z3_component_interplay.md similarity index 100% rename from docs/z3_component_interplay.md rename to docs/memory-bank/z3_component_interplay.md diff --git a/docs/z3_docker.md b/docs/memory-bank/z3_docker.md similarity index 100% rename from docs/z3_docker.md rename to docs/memory-bank/z3_docker.md From 49fd4a06bbd78bf58d31935489d5749b6635f87b Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 20 May 2025 10:20:16 +0100 Subject: [PATCH 10/11] feat(docker): establish comprehensive Z3 Docker Compose setup and guidelines Introduces a structured approach to Z3 service orchestration using Docker Compose. - Adds detailed README.md instructions for setting up and running the Z3 stack via Docker Compose, including .env configuration, certificate/identity file generation, and operational commands. - Implements .gitignore rules to correctly manage the 'config' directory, ensuring essential subdirectories (like 'config/tls') can be tracked via .gitkeep files while ignoring other contents, which is crucial for Docker volume mounts and config sources. - Includes a minor cosmetic update to the name of the GitHub Actions workflow for building Z3 images. This commit provides a foundational, well-documented Docker setup for deploying and managing the Zebra, Zaino, and Zallet services cohesively. --- .env | 70 +++++++++++ .github/workflows/build-z3-images.yaml | 2 +- .gitignore | 17 ++- Dockerfile.zebra | 6 + README.md | 154 +++++++++++++++++++++++-- config/.gitkeep | 0 config/tls/.gitkeep | 0 docker-compose.yml | 151 ++++++++++++++++++++++++ 8 files changed, 390 insertions(+), 10 deletions(-) create mode 100644 .env create mode 100644 Dockerfile.zebra create mode 100644 config/.gitkeep create mode 100644 config/tls/.gitkeep create mode 100644 docker-compose.yml diff --git a/.env b/.env new file mode 100644 index 0000000..182cd4a --- /dev/null +++ b/.env @@ -0,0 +1,70 @@ +# z3/.env + +# z3/docker-compose.yml Environment Variables + +# --- Common Configuration --- +# Network name for all services (e.g., Mainnet, Testnet, Regtest). Referenced by Zebra, Zaino, and Zallet. +NETWORK_NAME=Testnet +# Globally enables RPC cookie authentication. If true, Zebra generates a .cookie file. +# Zaino's use of this cookie (via ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE in compose) is conditional on its ZAINO_VALIDATOR_LISTEN_ADDRESS. +ENABLE_COOKIE_AUTH=true +# In-container directory for the .cookie authentication file (e.g., /var/run/auth). +# Zebra writes its cookie here; Zaino reads from this location via its ZAINO_VALIDATOR_COOKIE_PATH in compose. +COOKIE_AUTH_FILE_DIR=/var/run/auth + +# --- Zebra Configuration --- +# Zebra Rust log level +ZEBRA_RUST_LOG=info +# Zebra's internal RPC port. Zaino connects to this port on the 'zebra' service hostname. +ZEBRA_RPC_PORT=18232 +# Zebra host RPC port (for external access to Zebra) +ZEBRA_HOST_RPC_PORT=18232 + +# --- Zaino Configuration --- +# Zaino Rust log level +ZAINO_RUST_LOG=trace,hyper=info +# Zaino's internal gRPC port. Zallet connects to this port on the 'zaino' service hostname. +ZAINO_GRPC_PORT=8137 +# Enable/disable Zaino's JSON-RPC service +ZAINO_JSON_RPC_ENABLE=false +# Zaino internal JSON-RPC port +ZAINO_JSON_RPC_PORT=8237 +# Zaino gRPC TLS (true/false) +ZAINO_GRPC_TLS_ENABLE=true # Set to true to enforce security policies +# Zaino host gRPC port (for external access to Zaino gRPC) +ZAINO_HOST_GRPC_PORT=8137 +# Zaino host JSON-RPC port (for external access to Zaino JSON-RPC) +ZAINO_HOST_JSONRPC_PORT=8237 +# In-container FHS-compliant paths for Zaino TLS configs +# These are the paths Zaino application expects for its cert and key +ZAINO_GRPC_TLS_CERT_PATH=/var/run/zaino/tls/zaino.crt +ZAINO_GRPC_TLS_KEY_PATH=/var/run/zaino/tls/zaino.key +# For Zaino healthcheck with TLS (adjust curl_opt as needed for your cert) +ZAINO_GRPC_TLS_ENABLE_SCHEME_SUFFIX=s +ZAINO_GRPC_TLS_ENABLE_CURL_OPT=-k # -k allows insecure for self-signed, use --cacert for prod +# Option to disable Zaino's local DB features (skips FinalisedState sync) +ZAINO_NO_DB=true +# Zaino validator username for Zebra RPC +ZAINO_VALIDATOR_USER=__cookie__ +# Zaino Rust backtrace setting +ZAINO_RUST_BACKTRACE=full +# Zaino application internal config path +ZAINO_CONF_PATH=/home/zaino/.config/zaino/zindexer.toml +# Zaino application internal data directory +ZAINO_DATA_DIR=/home/zaino/.cache/zaino + +# --- Zallet Configuration --- +# Zallet Rust log level +ZALLET_RUST_LOG=debug +# Zallet internal RPC port +ZALLET_RPC_PORT=28232 +# Zallet host RPC port (for external access to Zallet RPC) +ZALLET_HOST_RPC_PORT=28232 +# Zallet application internal config path +ZALLET_CONF_PATH=/etc/zallet/zallet.toml +# Zallet application internal data directory +ZALLET_DATA_DIR=/home/zallet/.data +# Example path for a CA certificate file that Zallet might use to trust Zaino's gRPC TLS certificate. +# If Zaino uses a self-signed certificate or a certificate from a private CA, Zallet would need to be +# configured to trust it. The actual environment variable name and mechanism depend on Zallet's implementation. +# ZALLET_INDEXER_CA_PATH=/path/to/trusted/zaino_ca.crt diff --git a/.github/workflows/build-z3-images.yaml b/.github/workflows/build-z3-images.yaml index 8e7d4cd..b8ebff5 100644 --- a/.github/workflows/build-z3-images.yaml +++ b/.github/workflows/build-z3-images.yaml @@ -1,4 +1,4 @@ -name: Build z3 images +name: Build Z3 images concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.gitignore b/.gitignore index 0104787..7801950 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,19 @@ target/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +# Ignore all contents of the 'config' directory recursively +config/** + +# Un-ignore the 'tls' subdirectory itself, so we can look inside it +!config/tls/ + +# Then ignore everything inside 'tls' +config/tls/* + +# Then un-ignore the .gitkeep file within 'tls' +!config/tls/.gitkeep + +# Un-ignore .gitkeep directly under config +!config/.gitkeep diff --git a/Dockerfile.zebra b/Dockerfile.zebra new file mode 100644 index 0000000..374027a --- /dev/null +++ b/Dockerfile.zebra @@ -0,0 +1,6 @@ +FROM zfnd/zebra:latest + +# Install curl +RUN apt-get update && \ + apt-get install -y curl && \ + rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index d1adf04..eca220a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,148 @@ -# z3 -Repository to iterate on the grand unification of Zebra, Zaino and Zallet to replace the venerable Zcashd +# Z3 - Unified Zcash Stack -For details, see: -- [Project Brief](memory-bank/projectbrief.md) for a high-level overview -- [Product Context](memory-bank/productContext.md) for detailed problem analysis and proposed solution -- [Technical Context](memory-bank/techContext.md) for architecture and dependencies -- [System Patterns](memory-bank/systemPatterns.md) for design patterns and conventions -- [Progress](memory-bank/progress.md) for current status and roadmap +This project orchestrates Zebra, Zaino, and Zallet to provide a modern, modular Zcash software stack, intended to replace the legacy `zcashd`. +## Prerequisites + +Before you begin, ensure you have the following installed: + +* **Docker Engine:** [Install Docker](https://docs.docker.com/engine/install/) +* **Docker Compose:** (Usually included with Docker Desktop, or [install separately](https://docs.docker.com/compose/install/)) +* **rage:** For generating the Zallet identity file. Install from [str4d/rage releases](https://github.com/str4d/rage/releases) or build from source. +* **Git:** For cloning the repositories and submodules. + +## Setup + +1. **Clone the Repository and Submodules:** + + If you haven't already, clone this `z3` repository and initialize its submodules (Zebra, Zaino, Wallet/Zallet, Zcashd). The Docker Compose setup currently relies on local builds for Zaino and Zallet if you were to build the images from scratch (though pre-built images are specified in the compose file for ease of use). + + ```bash + git clone + cd z3 + git submodule update --init --recursive + ``` + +2. **Configuration Directories:** + + After cloning the repository, you will find the following configuration directories, which are tracked by Git and will be populated with essential files in subsequent steps: + + * `config/`: This directory is intended to hold user-generated files that are essential for the Z3 stack's operation. Specifically, you will place: + * `zallet_identity.txt` (Zallet age identity file for encryption - _you will generate this in a later step_). + * `config/tls/`: This subdirectory is for TLS certificate files that you will generate: + * `zaino.crt` (Zaino's TLS certificate - _you will generate this_) + * `zaino.key` (Zaino's TLS private key - _you will generate this_) + +3. **Generate Zaino TLS Certificates:** + + Zaino requires a TLS certificate and private key for its gRPC interface. These files should be placed in the `config/tls/` directory. + + * `config/tls/zaino.crt`: The TLS certificate for Zaino. + * `config/tls/zaino.key`: The private key for Zaino's TLS certificate. + + You will need to generate these files using your preferred method (e.g., OpenSSL). For example, to generate a self-signed certificate: + + ```bash + openssl req -x509 -newkey rsa:4096 -keyout config/tls/zaino.key -out config/tls/zaino.crt -sha256 -days 365 -nodes -subj "/CN=localhost" -addext "subjectAltName = DNS:localhost,IP:127.0.0.1" + ``` + + **Note:** For production or more secure setups, use certificates issued by a trusted Certificate Authority (CA). The example above creates a self-signed certificate valid for 365 days and includes `localhost` and `127.0.0.1` as Subject Alternative Names (SANs), which is important for client validation. + +4. **Generate Zallet Identity File:** + + Zallet requires an `age` identity file for wallet encryption. Generate this file using `rage-keygen`: + + ```bash + rage-keygen -o config/zallet_identity.txt + ``` + + This will create `config/zallet_identity.txt`. **Securely back up this file and its corresponding public key.** The public key will be printed to your terminal during generation. + +5. **Understanding Service Configuration:** + + The services within the Z3 stack (Zebra, Zaino, Zallet) come with their own internal default configurations. For the Z3 Docker Compose setup, **all user-driven customization of service operational parameters (such as network settings, ports, log levels, and feature flags) is exclusively managed through environment variables.** These variables are defined in the `z3/.env` file (which you will create in the next step) and are then passed to the services by Docker Compose. + + You do not need to create or modify separate `.toml` configuration files for Zebra, Zaino, or Zallet in the `z3/config/` directory to control their runtime behavior in this setup; the environment variables are the sole interface for these kinds of adjustments. + +6. **Create `.env` File for Docker Compose:** + + The `docker-compose.yml` file is configured to load environment variables from a `.env` file located in the `z3/` directory. This file is essential for customizing network settings, ports, log levels, and feature flags without modifying the `docker-compose.yml` directly. + + Create a `z3/.env` file. You can use the example content below as a starting point, adapting it to your needs. Refer to the comments within the example `z3/.env` or the `docker-compose.yml` for variable details. + A comprehensive example `z3/.env` can be found alongside `docker-compose.yml`. Key variables include: + + ```env + # z3/.env Example Snippet + NETWORK_NAME=Testnet + ENABLE_COOKIE_AUTH=true + COOKIE_AUTH_FILE_DIR=/var/run/auth + + ZEBRA_RUST_LOG=info + ZEBRA_RPC_PORT=18232 + ZEBRA_HOST_RPC_PORT=18232 + + ZAINO_RUST_LOG=trace,hyper=info + ZAINO_GRPC_PORT=8137 + ZAINO_JSON_RPC_ENABLE=false + ZAINO_GRPC_TLS_ENABLE=true + ZAINO_HOST_GRPC_PORT=8137 + ZAINO_GRPC_TLS_CERT_PATH=/var/run/zaino/tls/zaino.crt + ZAINO_GRPC_TLS_KEY_PATH=/var/run/zaino/tls/zaino.key + + ZALLET_RUST_LOG=debug + ZALLET_HOST_RPC_PORT=28232 + ``` + +## Running the Stack + +Once the setup is complete, you can start all services using Docker Compose: + +```bash +cd z3 # Ensure you are in the z3 directory +docker-compose up --build +``` +* `--build`: This flag tells Docker Compose to build the images if they don't exist or if their Dockerfiles have changed. The current `docker-compose.yml` uses pre-built images for `zaino` and `zallet` specified by their SHA, and `zfnd/zebra:latest` for zebra, so `--build` might primarily affect local Dockerfile changes if you were to modify them or switch to local builds. +* To run in detached mode (in the background), add the `-d` flag: `docker-compose up -d --build`. + +## Stopping the Stack + +To stop the services and remove the containers, run: + +```bash +docker-compose down +``` + +If you also want to remove the data volumes (blockchain data, indexer database, wallet database), use: + +```bash +docker-compose down -v +``` + +## Configuration Details + +Understanding how configuration is applied is key to customizing the Z3 stack: + +* **Internal Service Defaults:** Each service (Zebra, Zaino, Zallet) has its own built-in default configuration values. These internal defaults are used unless influenced by environment variables. For this Z3 Docker Compose deployment, you do not directly interact with or provide TOML configuration files for the services in the `z3/config/` directory to alter these defaults for general operational parameters. + +* **Environment Variables (`z3/.env`):** This is the **exclusive method for customizing the operational parameters of the Zebra, Zaino, and Zallet services within the Z3 stack.** Variables defined in the `z3/.env` file are passed into their respective containers by Docker Compose. The services are designed to read these environment variables at startup to configure their behavior (e.g., log levels, network ports, feature enablement). This approach provides a centralized and clear way to manage your deployment settings. + +* **Explicitly Mounted Files & Docker Configs:** Note that specific files *are* sourced from your `z3/config/` directory for distinct purposes, such as `zallet_identity.txt` (volume mounted for Zallet) and the TLS certificates in `z3/config/tls/` (used via Docker `configs` for Zaino). These are for providing essential data or credentials, separate from the environment variable-based parameter tuning. + +* **Docker Compose Overrides (`docker-compose.yml`):** The `environment` section within each service definition in `docker-compose.yml` is used for several purposes: + * **Passing `.env` Variables:** It explicitly lists which variables from `.env` (or your shell environment) are passed into the container (e.g., `RUST_LOG=${ZALLET_RUST_LOG}`). + * **Service Discovery & Internal Settings:** It sets variables crucial for inter-service communication (e.g., `ZAINO_VALIDATOR_LISTEN_ADDRESS=zebra:${ZEBRA_RPC_PORT}`) or paths internal to the container (e.g., `ZAINO_VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie`). + * **Derived or Conditional Values:** Some environment variables might be constructed from others (e.g., combining `COOKIE_AUTH_FILE_DIR` with a filename) or set based on conditions (e.g., `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE=${ENABLE_COOKIE_AUTH}`). + These `docker-compose.yml` environment settings generally take precedence if there's an overlap, as they are the final values passed when the container starts. + +* **Entrypoint Scripts:** Each service's Docker image has an entrypoint script (`entrypoint.sh`). These scripts often perform final configuration steps, such as generating configuration files from templates based on environment variables, or applying conditional logic before starting the main application process. + +## Interacting with Services + +Once the stack is running, services can be accessed via the ports exposed in `docker-compose.yml`: + +* **Zebra RPC:** `http://localhost:${ZEBRA_HOST_RPC_PORT:-18232}` (default: Testnet `http://localhost:18232`) +* **Zaino gRPC:** `localhost:${ZAINO_HOST_GRPC_PORT:-8137}` (default: `localhost:8137`) +* **Zaino JSON-RPC:** `http://localhost:${ZAINO_HOST_JSONRPC_PORT:-8237}` (default: `http://localhost:8237`, if enabled) +* **Zallet RPC:** `http://localhost:${ZALLET_HOST_RPC_PORT:-28232}` (default: `http://localhost:28232`) + +Refer to the individual component documentation for RPC API details. \ No newline at end of file diff --git a/config/.gitkeep b/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/tls/.gitkeep b/config/tls/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bfb977a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,151 @@ +services: + zebra: + # TODO: Use zfnd/zebra:latest when zebra has curl installed + image: zfnd/zebra:latest # Using latest, specify a version tag if preferred + build: + context: . + dockerfile: Dockerfile.zebra + container_name: z3_zebra + restart: unless-stopped + platform: linux/amd64 # Uncomment if you need to specify platform + env_file: + - ./.env + environment: + - RUST_LOG=${ZEBRA_RUST_LOG} + - NETWORK=${NETWORK_NAME} # Zebra's network, derived from global NETWORK_NAME + - ZEBRA_COOKIE_DIR=${COOKIE_AUTH_FILE_DIR} # Zebra entrypoint uses this directory to store its .cookie + # - METRICS_ENDPOINT_ADDR=0.0.0.0:9999 + volumes: + - zebra_data:/home/zebra/.cache/zebra + - shared_cookie_volume:${COOKIE_AUTH_FILE_DIR} # Zebra writes the cookie here + ports: + # Expose to host if direct access is needed + - "${ZEBRA_HOST_RPC_PORT}:${ZEBRA_RPC_PORT}" # Default Testnet RPC + networks: + - z3_net + # TODO: Make this healcheck simpler when we have a way to supply the cookie to Zebra + # This also needs to consider we're extracting the password from the cookie file + # using `cut -d ':' -f 2-`, because the file is formatted as "__cookie__:" + healthcheck: + test: | + echo "Healthcheck script started. ENABLE_COOKIE_AUTH=${ENABLE_COOKIE_AUTH:-false}" + if [ "${ENABLE_COOKIE_AUTH:-false}" = "true" ]; then + echo "Cookie auth is ON." + echo "Attempting to use cookie file at: [${COOKIE_AUTH_FILE_DIR}/.cookie]" + if [ -f "${COOKIE_AUTH_FILE_DIR}/.cookie" ] && [ -s "${COOKIE_AUTH_FILE_DIR}/.cookie" ]; then + echo "Cookie file [${COOKIE_AUTH_FILE_DIR}/.cookie] found and not empty. Attempting curl with cookie." + curl -v --user "__cookie__:$(cat ${COOKIE_AUTH_FILE_DIR}/.cookie | cut -d ':' -f 2-)" --data-binary '{"jsonrpc": "2.0", "id":"healthcheck", "method": "getinfo", "params": [] }' -H 'Content-Type: application/json' http://127.0.0.1:${ZEBRA_RPC_PORT}/ || exit 1 + else + echo "Cookie file [${COOKIE_AUTH_FILE_DIR}/.cookie] not found or empty. Exiting healthcheck with 1." + exit 1 + fi + else + echo "Cookie auth is OFF. Attempting curl without cookie." + curl -s -f --user "" --data-binary '{"jsonrpc": "2.0", "id":"healthcheck", "method": "getinfo", "params": [] }' -H 'Content-Type: application/json' http://127.0.0.1:${ZEBRA_RPC_PORT}/ | grep -q '"result":' || exit 1 + fi + interval: 30s + timeout: 30s + retries: 5 + start_period: 30s + + zaino: + image: docker pull ghcr.io/zcashfoundation/zaino:sha-54e0da6@sha256:5cf6db69280261e836d98e99b47e2601e825935c9f56134e7380d3a10adf7899 + #! You can build the image yourself if you want to use a custom version + # build: + # context: ../zaino + # dockerfile: docker/Dockerfile + container_name: z3_zaino + restart: unless-stopped + # platform: linux/amd64 + depends_on: + zebra: + condition: service_healthy + env_file: + - ./.env + environment: + - RUST_LOG=${ZAINO_RUST_LOG} + - RUST_BACKTRACE=${ZAINO_RUST_BACKTRACE} + - ZAINO_NETWORK=${NETWORK_NAME} + # Connect to Zebra service using its name and port, cookie auth will be used if enabled globally. + - ZAINO_VALIDATOR_LISTEN_ADDRESS=zebra:${ZEBRA_RPC_PORT} + # This assignment enables cookie auth for Zaino if ENABLE_COOKIE_AUTH is true + # AND its ZAINO_VALIDATOR_LISTEN_ADDRESS is non-loopback (which 'zebra:${ZEBRA_RPC_PORT}' is). + - ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE=${ENABLE_COOKIE_AUTH} + - ZAINO_VALIDATOR_COOKIE_PATH=${COOKIE_AUTH_FILE_DIR}/.cookie # Zaino reads the cookie from this specific path + # Zaino's own RPC services + - ZAINO_GRPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_GRPC_PORT} + - ZAINO_JSON_RPC_LISTEN_ADDRESS=0.0.0.0:${ZAINO_JSON_RPC_PORT} + volumes: + - zaino_data:/home/zaino/.cache/zaino + - shared_cookie_volume:${COOKIE_AUTH_FILE_DIR}:ro # Zaino reads the cookie read-only + configs: + - source: zaino_tls_cert # Use the top-level config for Zaino's cert + target: ${ZAINO_GRPC_TLS_CERT_PATH} # Mount path inside container, FHS compliant via env var + - source: zaino_tls_key # Use the top-level config for Zaino's key + target: ${ZAINO_GRPC_TLS_KEY_PATH} # Mount path inside container, FHS compliant via env var + ports: + - "${ZAINO_HOST_GRPC_PORT}:${ZAINO_GRPC_PORT}" + - "${ZAINO_HOST_JSONRPC_PORT}:${ZAINO_JSON_RPC_PORT}" + networks: + - z3_net + healthcheck: + # Healthcheck needs to account for TLS if ZAINO_GRPC_TLS_ENABLE is true + # Use env vars for scheme and curl options, to be defined in .env + test: ["CMD-SHELL", "curl -f -k https://127.0.0.1:${ZAINO_GRPC_PORT} || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + + zallet: + image: ghcr.io/zcashfoundation/zallet:sha-54e0da6@sha256:3710fb41dbad0f0457551b76a81c97e476f4addc9605498a0d0e34fbe4b8d4e9 + #! You can build the image yourself if you want to use a custom version + # build: + # context: ../wallet + # dockerfile: docker/Dockerfile + container_name: z3_zallet + restart: unless-stopped + # platform: linux/amd64 + depends_on: + zaino: + condition: service_healthy + env_file: + - ./.env + environment: + - RUST_LOG=${ZALLET_RUST_LOG} + - ZALLET_NETWORK=${NETWORK_NAME} + # Zallet indexer points to Zaino's gRPC service (hosted at 'zaino:${ZAINO_GRPC_PORT}'). + # TLS for this connection needs to be handled by Zallet (e.g., trusting Zaino's cert). + # Refer to .env for potential CA path configuration if needed. + - ZALLET_INDEXER_VALIDATOR_ADDRESS=zaino:${ZAINO_GRPC_PORT} + - ZALLET_RPC_BIND=0.0.0.0:${ZALLET_RPC_PORT} + volumes: + - zallet_data:/home/zallet/.data + - ./config/zallet_identity.txt:/home/zallet/.data/identity.txt:ro + # - ./config/tls/zaino.crt:/tls_trusted/zaino.crt:ro # Example mount for Zallet to trust Zaino + ports: + - "${ZALLET_HOST_RPC_PORT}:${ZALLET_RPC_PORT}" + networks: + - z3_net + healthcheck: + test: ["CMD-SHELL", "curl -s -f --data-binary '{\"jsonrpc\": \"2.0\", \"id\":\"healthcheck\", \"method\": \"help\", \"params\": [] }' -H 'Content-Type: application/json' http://127.0.0.1:${ZALLET_RPC_PORT}/ || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + +volumes: + zebra_data: + zaino_data: + zallet_data: + shared_cookie_volume: {} # Define the shared volume for the cookie + +networks: + z3_net: + driver: bridge + +configs: + zaino_tls_cert: + file: ./config/tls/zaino.crt # Path on the host for Zaino's TLS cert + zaino_tls_key: + file: ./config/tls/zaino.key # Path on the host for Zaino's TLS key From e4cccd04be7fd2eac5d6feaf735d8b62998a429d Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 21 May 2025 06:17:54 +0100 Subject: [PATCH 11/11] feat(rules): add comprehensive guidelines for Docker Compose configuration and service interactions Introduces detailed documentation for configuring and orchestrating the Z3 services (Zebra, Zaino, Zallet) using Docker Compose. - Adds guidelines for environment variable management, entrypoint script expectations, and shared runtime data handling using Docker configs. - Outlines security policies and networking configurations, including cookie authentication and TLS requirements for service interactions. - Provides specific service configuration details for Zaino and Zallet, emphasizing their dependencies and operational parameters. --- ...ocker-compose-configuration-guidelines.mdc | 48 ++++++++++++++++ .cursor/rules/z3_docker_orchestration.mdc | 26 +++++++++ .cursor/rules/z3_gitignore_config_rules.mdc | 50 +++++++++++++++++ .cursor/rules/z3_security_and_networking.mdc | 52 ++++++++++++++++++ .cursor/rules/z3_service_zaino.mdc | 52 ++++++++++++++++++ .cursor/rules/z3_service_zallet.mdc | 55 +++++++++++++++++++ .cursor/rules/z3_service_zebra.mdc | 38 +++++++++++++ 7 files changed, 321 insertions(+) create mode 100644 .cursor/rules/docker-compose-configuration-guidelines.mdc create mode 100644 .cursor/rules/z3_docker_orchestration.mdc create mode 100644 .cursor/rules/z3_gitignore_config_rules.mdc create mode 100644 .cursor/rules/z3_security_and_networking.mdc create mode 100644 .cursor/rules/z3_service_zaino.mdc create mode 100644 .cursor/rules/z3_service_zallet.mdc create mode 100644 .cursor/rules/z3_service_zebra.mdc diff --git a/.cursor/rules/docker-compose-configuration-guidelines.mdc b/.cursor/rules/docker-compose-configuration-guidelines.mdc new file mode 100644 index 0000000..1bf3181 --- /dev/null +++ b/.cursor/rules/docker-compose-configuration-guidelines.mdc @@ -0,0 +1,48 @@ +--- +description: +globs: +alwaysApply: true +--- +## Docker Compose Configuration Guidelines + +This document outlines best practices for configuring multi-service applications using Docker Compose, drawing from the setup for Zebra, Zaino, and Zallet. + +**Key Files:** +- Environment Variables: [`z3/.env`](mdc:zaino/z3/.env) +- Docker Compose Configuration: [`z3/docker-compose.yml`](mdc:zaino/z3/docker-compose.yml) +- Zebra Entrypoint Script: [`zebra/docker/entrypoint.sh`](mdc:zaino/zebra/docker/entrypoint.sh) +- Zaino Entrypoint Script: [`zaino/docker/entrypoint.sh`](mdc:zaino/zaino/docker/entrypoint.sh) +- Placeholder for Docker `configs`: [`z3/.cookie`](mdc:zaino/z3/.cookie) (example) + +**Core Principles:** + +1. **Centralized Environment Variables (`.env`)**: + * Store common and service-specific configurable parameters in a single `.env` file (e.g., [`z3/.env`](mdc:zaino/z3/.env)). This includes network settings, ports, log levels, and feature flags. + * For shared resources like an RPC authentication cookie directory, define a single global variable in the `.env` file (e.g., `COOKIE_AUTH_FILE_DIR=/var/run/auth`). This directory should be an FHS-compliant path for transient runtime data. + +2. **Respecting Application Entrypoints in `docker-compose.yml`**: + * Each service in [`z3/docker-compose.yml`](mdc:zaino/z3/docker-compose.yml) should have its environment variables set according to what its entrypoint script (e.g., [`zebra/docker/entrypoint.sh`](mdc:zaino/zebra/docker/entrypoint.sh), [`zaino/docker/entrypoint.sh`](mdc:zaino/zaino/docker/entrypoint.sh)) expects. + * If an entrypoint script requires a specific variable (e.g., `ZEBRA_COOKIE_DIR` for Zebra, `ZAINO_VALIDATOR_COOKIE_PATH` for Zaino), these should be set in the `environment` section of the service definition. + * The values for these service-specific variables should be derived from the global variables in the `.env` file. For example: + * Zebra's `ZEBRA_COOKIE_DIR` could be `${COOKIE_AUTH_FILE_DIR}`. + * Zaino's `ZAINO_VALIDATOR_COOKIE_PATH` could be `${COOKIE_AUTH_FILE_DIR}/.cookie`. + +3. **Using Docker `configs` for Shared Runtime Data**: + * For runtime-generated files that need to be shared between services (like RPC cookie files), use Docker `configs`. + * Define a top-level `configs:` block in [`z3/docker-compose.yml`](mdc:zaino/z3/docker-compose.yml), sourcing from a placeholder file on the host (e.g., `file: ./.cookie`). + ```yaml + configs: + cookie_auth_config: # Descriptive name for the shared config object + file: ./.cookie + ``` + * Mount this Docker `config` into each relevant service: + * The service generating the file (e.g., Zebra) mounts the config to the full path where its entrypoint expects to write the file (e.g., `target: ${COOKIE_AUTH_FILE_DIR}/.cookie`). + * The service consuming the file (e.g., Zaino) mounts the same config to the full path where its entrypoint expects to read the file (e.g., `target: ${COOKIE_AUTH_FILE_DIR}/.cookie`). + * This ensures both services are looking at the same underlying data, managed by Docker. The initial placeholder file allows Docker Compose to create the config object. + +4. **FHS-Compliant Paths**: + * For transient runtime data generated within containers (like cookie files or PID files), prefer FHS-compliant directories such as `/var/run//` or `/run//`. + +5. **Clear Naming Conventions**: + * Use descriptive names for environment variables (e.g., `COOKIE_AUTH_FILE_DIR` instead of generic names). + * Name Docker `config` objects clearly to reflect their purpose and shared nature (e.g., `cookie_auth_config`). diff --git a/.cursor/rules/z3_docker_orchestration.mdc b/.cursor/rules/z3_docker_orchestration.mdc new file mode 100644 index 0000000..85abc59 --- /dev/null +++ b/.cursor/rules/z3_docker_orchestration.mdc @@ -0,0 +1,26 @@ +--- +description: +globs: +alwaysApply: true +--- +# Z3 Docker Orchestration Overview + +This project uses Docker Compose to orchestrate the Zebra, Zaino, and Zallet services. The main orchestration file is [z3/docker-compose.yml](mdc:z3/docker-compose.yml). + +## Key Components: + +* **`docker-compose.yml`**: Defines the services (Zebra, Zaino, Zallet), their build contexts (if building locally), images, volumes for data persistence, networks, and mechanisms for specific configuration data (like Docker `configs` for TLS certificates and volume mounts for identity files). It also specifies dependencies between services to manage startup order. +* **`z3/config/` Directory**: This host directory is for user-provided files that are explicitly used by the Docker setup. This includes: + * `[z3/config/zallet_identity.txt](mdc:z3/config/zallet_identity.txt)`: A crucial `age` identity file for Zallet wallet encryption, which must be generated by the user. This file is volume-mounted into the Zallet container. + * TLS certificates (e.g., `[z3/config/tls/zaino.crt](mdc:z3/config/tls/zaino.crt)` and `[z3/config/tls/zaino.key](mdc:z3/config/tls/zaino.key)`): These are sourced by Docker `configs` and made available to services like Zaino. + * It's important to note that general service configuration files like `zebra.toml`, `zaino.toml`, or `zallet.toml` are **not** placed in `z3/config/` by the user to be mounted for overriding base service configurations in this Z3 setup. Services use their internal defaults, and operational parameters are set via environment variables. +* **`.env` File**: The `[z3/.env](mdc:z3/.env)` file is the **exclusive method for users to customize operational parameters** for the Zebra, Zaino, and Zallet services. This includes settings like network choice, ports, log levels, and feature flags. Docker Compose injects these variables into the respective service containers. +* **README**: The main instructions for setup and running are in `[z3/README.md](mdc:z3/README.md)`. + +## Service Interaction: + +* **Zebra** acts as the consensus node. +* **Zaino** connects to Zebra to index blockchain data. +* **Zallet** connects primarily to Zaino for indexed data and wallet operations. It might also interact directly with Zebra for certain functionalities. + +Networking between services is handled by a custom Docker bridge network (`z3_net`) defined in `[z3/docker-compose.yml](mdc:z3/docker-compose.yml)`. Services can reach each other using their service names as hostnames (e.g., `zebra`, `zaino`, `zallet`). diff --git a/.cursor/rules/z3_gitignore_config_rules.mdc b/.cursor/rules/z3_gitignore_config_rules.mdc new file mode 100644 index 0000000..cbbb000 --- /dev/null +++ b/.cursor/rules/z3_gitignore_config_rules.mdc @@ -0,0 +1,50 @@ +--- +description: +globs: +alwaysApply: false +--- +# .gitignore Rules for the z3/config/ Directory + +The `[z3/.gitignore](mdc:z3/z3/.gitignore)` file contains specific rules to manage the contents of the `z3/config/` directory and its subdirectories (like `z3/config/tls/`). The goal is to generally ignore user-generated configuration files or sensitive data placed here, while ensuring that the directory structure itself (including empty directories needed for Docker mounts or `configs`) can be version-controlled using `.gitkeep` files. + +## Key Patterns in `z3/.gitignore` for `config/`: + +1. **General Ignore for `config/` Contents:** + ```gitignore + config/** + ``` + This rule ignores all files and subdirectories recursively within `z3/config/`. + +2. **Un-ignoring the `tls` Subdirectory (to look inside):** + ```gitignore + !config/tls/ + ``` + This allows Git to consider rules pertaining to the contents of `z3/config/tls/`. + +3. **Ignoring Contents of `tls` (before specific un-ignores):** + ```gitignore + config/tls/* + ``` + After un-ignoring the `tls` directory itself, this rule ensures its contents are ignored by default, allowing for specific un-ignores next. + +4. **Un-ignoring `.gitkeep` in `config/tls/`:** + ```gitignore + !config/tls/.gitkeep + ``` + This explicitly prevents `[z3/config/tls/.gitkeep](mdc:z3/z3/config/tls/.gitkeep)` from being ignored, allowing the `z3/config/tls/` directory to be tracked even if empty. + +5. **Un-ignoring `.gitkeep` in `config/`:** + ```gitignore + !config/.gitkeep + ``` + This explicitly prevents `[z3/config/.gitkeep](mdc:z3/z3/config/.gitkeep)` from being ignored, allowing the `z3/config/` directory to be tracked even if empty. + +**Summary of Behavior:** + +* All files and folders within `z3/config/` are ignored by default. +* The `z3/config/tls/` directory is specifically managed to allow its `.gitkeep` file to be tracked. +* The `z3/config/` directory itself has its `.gitkeep` file tracked. +* Any other files (e.g., `config/my_secret.txt`, `config/tls/extra_cert.pem`) will be ignored by Git and should not be committed unless a specific negation rule (`!config/...`) is added for them. +* Note: The `[z3/config/zallet_identity.txt](mdc:z3/z3/config/zallet_identity.txt)` file, while crucial, is covered by `config/**` and will be ignored. It is user-generated and should not be committed to the repository. + +These rules ensure that the necessary directory structure for configurations (especially for Docker `configs` which might require specific source paths) can be maintained in the Git repository without tracking actual sensitive configuration files or runtime data within them. diff --git a/.cursor/rules/z3_security_and_networking.mdc b/.cursor/rules/z3_security_and_networking.mdc new file mode 100644 index 0000000..1df344d --- /dev/null +++ b/.cursor/rules/z3_security_and_networking.mdc @@ -0,0 +1,52 @@ +--- +description: +globs: +alwaysApply: false +--- +# Z3 Service Security and Networking Configuration Guide + +This document outlines the security policies and networking configurations for the Z3 services (Zebra, Zaino, Zallet) as implemented in the `z3/docker-compose.yml` and related entrypoint scripts. + +## Core Security Policies: + +1. **Zebra <-> Zaino (Validator Connection):** + * Zaino connects to Zebra's RPC. If Zebra's address (`ZAINO_VALIDATOR_LISTEN_ADDRESS`) is not a loopback address from Zaino's perspective (e.g., using the service name `zebra:${ZEBRA_RPC_PORT}`), Zaino's internal policy mandates that cookie authentication must be used. + * To comply: + * The `ENABLE_COOKIE_AUTH` environment variable (in `[.env](mdc:zaino/zaino/zaino/z3/.env)`) must be set to `true`. + * Zebra (`[entrypoint.sh](mdc:zaino/zaino/zaino/z3/entrypoint.sh)`) must be configured to enable cookie authentication and generate a `.cookie` file. + * Zaino (`[entrypoint.sh](mdc:zaino/zaino/zaino/zaino/docker/entrypoint.sh)`) must be configured to use cookie authentication and read the shared cookie file. + * A shared cookie mechanism is used: + * A host file `./.cookie` is defined as a Docker `config` named `cookie_auth_config`. + * This config is mounted into both `zebra` and `zaino` containers at `${COOKIE_AUTH_FILE_DIR}/.cookie`. The `COOKIE_AUTH_FILE_DIR` variable (defined in `[.env](mdc:zaino/zaino/zaino/z3/.env)`) should specify an FHS-compliant path (e.g., `/var/run/auth`). + * **Note on Writability**: If Zebra generates the cookie, it needs to write to this file. Docker `configs` are typically read-only. The user must ensure this setup allows Zebra to manage the cookie file when `ENABLE_COOKIE_AUTH=true`. + * **Authentication:** + * **Cookie-based (Primary method if `ENABLE_COOKIE_AUTH=true` in `[.env](mdc:zaino/z3/.env)`):** + * If `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE` is `true` (derived from `${ENABLE_COOKIE_AUTH}`), Zaino reads Zebra's RPC cookie from the path specified by `ZAINO_VALIDATOR_COOKIE_PATH` (e.g., `${COOKIE_AUTH_FILE_DIR}/.cookie`). + * Zaino expects this file to contain the cookie string, typically in the format `__cookie__:`. + * Zaino then uses this cookie to authenticate to Zebra by sending an HTTP `Authorization` header with the value `Basic __cookie__:`. The `__cookie__:` part acts as the username, and the rest as the password in Basic Authentication terms. + * This mechanism relies on Zebra generating the cookie file and Zaino having read access to it via a shared Docker `config` mount. + * **User/Password-based (Basic Auth):** If `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE` is `false`, Zaino can use HTTP Basic Authentication with `ZAINO_VALIDATOR_USER` and `ZAINO_VALIDATOR_PASSWORD` (if these are provided). + * **No Authentication:** If cookie auth is disabled and user/pass are empty, Zaino attempts to connect without authentication, relying on Zebra to permit such connections. + +2. **Zaino's gRPC Server (Zaino <-> Zallet Connection):** + * Zaino's gRPC server listens on `ZAINO_GRPC_LISTEN_ADDRESS`. If this address is `0.0.0.0` (or any non-loopback/non-private IP from Zaino's perspective), Zaino's internal policy requires its gRPC service to use TLS. + * To comply: + * The `ZAINO_GRPC_TLS_ENABLE` environment variable (in `[.env](mdc:zaino/zaino/zaino/z3/.env)`) must be set to `true`. + * TLS certificate and key must be provided to Zaino. These are supplied as Docker `configs` (e.g., `zaino_tls_cert` and `zaino_tls_key`), sourced from host files (e.g., `./config/tls/zaino.crt` and `./config/tls/zaino.key`). + * The Docker configs are mounted into Zaino at FHS-compliant paths defined by environment variables in `[.env](mdc:zaino/zaino/zaino/z3/.env)` (e.g., `ZAINO_TLS_CERT_CONTAINER_PATH=/var/run/zaino/tls/zaino.crt`, `ZAINO_TLS_KEY_CONTAINER_PATH=/var/run/zaino/tls/zaino.key`). + * Zaino's internal configuration then uses these paths, set via `ZAINO_GRPC_TLS_CERT_PATH=${ZAINO_TLS_CERT_CONTAINER_PATH}` and `ZAINO_GRPC_TLS_KEY_PATH=${ZAINO_TLS_KEY_CONTAINER_PATH}` in `docker-compose.yml`. + * The user is responsible for generating these TLS certificate files on the host. + +3. **Zallet's Connection to Zaino's gRPC Server:** + * Since Zaino's gRPC service will be TLS-enabled, Zallet must connect to it using TLS and must trust Zaino's server certificate. + * **Challenge**: Standard Zallet configuration (`[entrypoint.sh](mdc:zaino/zaino/zaino/zaino/zaino/zaino/wallet/docker/entrypoint.sh)`) does not have explicit, documented options for configuring TLS trust (e.g., specifying a CA certificate or trusting a self-signed server certificate) for its indexer connection. + * The user needs to investigate and implement how Zallet will trust Zaino's TLS certificate. This might involve custom modifications to Zallet's container or finding undocumented configuration methods. + +## Docker Compose Configuration (`[docker-compose.yml](mdc:zaino/zaino/zaino/zaino/zaino/zaino/z3/docker-compose.yml)`): + +* Environment variables from `[.env](mdc:zaino/zaino/zaino/zaino/zaino/zaino/z3/.env)` are used extensively to control these security features and other service parameters. +* Zaino's healthcheck uses `curl` and requires additional environment variables (`ZAINO_GRPC_TLS_ENABLE_SCHEME_SUFFIX`, `ZAINO_GRPC_TLS_ENABLE_CURL_OPT`) to adapt to TLS settings. These should be defined in `[.env](mdc:zaino/zaino/zaino/zaino/zaino/zaino/z3/.env)`. + +## Entrypoint Scripts: + +* The entrypoint scripts for `zebra` (`[entrypoint.sh](mdc:zaino/zaino/zaino/zaino/zaino/zaino/z3/entrypoint.sh)`), `zaino` (`[entrypoint.sh](mdc:zaino/zaino/zaino/zaino/zaino/zaino/zaino/docker/entrypoint.sh)`), and `zallet` (`[entrypoint.sh](mdc:zaino/zaino/zaino/zaino/zaino/zaino/wallet/docker/entrypoint.sh)`) have been modified to handle directory ownership (`chown`) failures more gracefully by warning instead of exiting, particularly for directories that might contain read-only mounted files (like configs). However, `mkdir -p` failures will still cause an exit. diff --git a/.cursor/rules/z3_service_zaino.mdc b/.cursor/rules/z3_service_zaino.mdc new file mode 100644 index 0000000..8880ac3 --- /dev/null +++ b/.cursor/rules/z3_service_zaino.mdc @@ -0,0 +1,52 @@ +--- +description: +globs: +alwaysApply: false +--- +# Z3 Service: Zaino + +This rule describes the Zaino service configuration within the Z3 Docker stack, focusing on its interaction with Zebra. + +## Docker Configuration (`docker-compose.yml`): + +* **Image:** `ghcr.io/zcashfoundation/zaino:sha-54e0da6` (specific SHA). +* **Container Name:** `z3_zaino`. +* **Depends On:** `zebra` (waits for Zebra to be healthy). +* **Environment Variables (relevant to Zebra connection): + * `ZAINO_NETWORK`: Specifies the Zcash network, must match Zebra (e.g., `${NETWORK_NAME:-Testnet}`). + * `ZAINO_VALIDATOR_LISTEN_ADDRESS`: Address of the Zebra RPC service (e.g., `zebra:${ZEBRA_RPC_PORT:-18232}`). + * `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE`: Boolean to enable/disable cookie auth for connecting to Zebra (e.g., `${ENABLE_COOKIE_AUTH:-false}`). + * `ZAINO_VALIDATOR_COOKIE_PATH`: Path to Zebra's RPC cookie file *if cookie auth is enabled*. Default is `${ZEBRA_DATA_DIR}/.cookie` which resolves to `/home/zebra/.cache/zebra/.cookie` inside the Zaino container if `ZEBRA_DATA_DIR` is not overridden from its default. (e.g. `/zebra_rpc_cookies/.cookie` if using a shared volume for cookies). + * `ZAINO_VALIDATOR_USER`: RPC username for Zebra (if using basic auth). + * `ZAINO_VALIDATOR_PASSWORD`: RPC password for Zebra (if using basic auth). +* **Volumes:** + * `zaino_data:/home/zaino/.cache/zaino`: Persists Zaino's indexer database. + * `./config/zaino.toml:/home/zaino/.config/zaino/zindexer.toml:ro`: Mounts the Zaino configuration file [z3/config/zaino.toml](mdc:zaino/z3/config/zaino.toml). + * (Conditional) If Zebra uses cookie auth, a shared volume for the cookie file would be mounted here, for example: `- zebra_cookies:/zebra_rpc_cookies:ro`. +* **Healthcheck:** Uses `curl` to check its own gRPC port (`http://127.0.0.1:8137/`). + +## Source and Build (if building locally): + +* **Dockerfile:** [zaino/docker/Dockerfile](mdc:zaino/zaino/docker/Dockerfile) +* **Entrypoint Script:** [zaino/docker/entrypoint.sh](mdc:zaino/zaino/docker/entrypoint.sh) + * Generates `zindexer.toml` (by default at `/home/zaino/.config/zaino/zindexer.toml`) based on environment variables if the file is not present. + * Handles setting up default paths and user permissions. + +## Configuration File (`config/zaino.toml` or generated by entrypoint): + +* Contains settings for connecting to the validator (Zebra), its own gRPC/JSON-RPC server settings, database paths, and network type. +* Key section for Zebra connection: `validator_listen_address`, `validator_cookie_auth`, `validator_cookie_path`, `validator_user`, `validator_password`. + +## Connection to Zebra: + +* Zaino connects to Zebra primarily via **JSON-RPC over HTTP**. +* The connection details are specified in Zaino's configuration (`zindexer.toml`), typically generated or influenced by environment variables like `ZAINO_VALIDATOR_LISTEN_ADDRESS`. +* **Authentication:** + * **Cookie-based:** If `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE` is `true`, Zaino reads Zebra's RPC cookie from `ZAINO_VALIDATOR_COOKIE_PATH` and uses it for an `Authorization: Basic __cookie__:` header. + * **User/Password-based (Basic Auth):** If `ZAINO_VALIDATOR_COOKIE_AUTH_ENABLE` is `false`, Zaino uses HTTP Basic Authentication with `ZAINO_VALIDATOR_USER` and `ZAINO_VALIDATOR_PASSWORD`. + * **No Authentication:** If cookie auth is disabled and user/pass are empty, Zaino attempts to connect without authentication, relying on Zebra to permit such connections. +* **Core Logic:** The connection is managed by `JsonRpSeeConnector` in [zaino/zaino-fetch/src/jsonrpsee/connector.rs](mdc:zaino/zaino/zaino-fetch/src/jsonrpsee/connector.rs), which uses `reqwest`. + +## Role in Z3 Stack: + +Zaino indexes blockchain data fetched from Zebra, providing an efficient query layer for clients like Zallet. It relies on Zebra for raw block data, transaction information, and tree states. diff --git a/.cursor/rules/z3_service_zallet.mdc b/.cursor/rules/z3_service_zallet.mdc new file mode 100644 index 0000000..68704e8 --- /dev/null +++ b/.cursor/rules/z3_service_zallet.mdc @@ -0,0 +1,55 @@ +--- +description: +globs: +alwaysApply: false +--- +# Z3 Service: Zallet + +This rule describes the Zallet (wallet) service configuration within the Z3 Docker stack, focusing on its interactions with Zaino and Zebra. + +## Docker Configuration (`docker-compose.yml`): + +* **Image:** `ghcr.io/zcashfoundation/zallet:sha-18a7f9f` (specific SHA). +* **Container Name:** `z3_zallet`. +* **Depends On:** `zaino` (waits for Zaino to be healthy). +* **Environment Variables (relevant to service connections & core ops): + * `RUST_LOG`: Controls Zallet's logging level (e.g., `${ZALLET_RUST_LOG:-info}`). + * `ZALLET_NETWORK`: Specifies the Zcash network, must match Zebra and Zaino (e.g., `${NETWORK_NAME:-Testnet}`). + * `ZALLET_CONF_PATH`: Path to the main configuration file (e.g., `/etc/zallet/zallet.toml`). + * `ZALLET_DATA_DIR`: Directory for Zallet's persistent data, including its wallet DB and indexer cache (e.g., `/home/zallet/.data`). + * `ZALLET_INDEXER_VALIDATOR_ADDRESS`: Address of the Zaino gRPC service (e.g., `zaino:8137`). + * `ZALLET_RPC_BIND`: Address for Zallet's own RPC server to listen on (e.g., `0.0.0.0:28232`). +* **Volumes:** + * `zallet_data:/home/zallet/.data`: Persists Zallet's wallet database and its cache for data fetched from Zaino. + * `./config/zallet.toml:/etc/zallet/zallet.toml:ro`: Mounts the Zallet configuration file [z3/config/zallet.toml](mdc:zaino/z3/config/zallet.toml). + * `./config/zallet_identity.txt:/home/zallet/.data/identity.txt:ro`: Mounts the user-generated `age` identity file [z3/config/zallet_identity.txt](mdc:zaino/z3/config/zallet_identity.txt), crucial for wallet encryption. +* **Ports:** Exposes Zallet's RPC port to the host (e.g., `"${ZALLET_HOST_RPC_PORT:-28232}:28232"`). + +## Source and Build (if building locally): + +* **Dockerfile:** [wallet/docker/Dockerfile](mdc:zaino/wallet/docker/Dockerfile) +* **Entrypoint Script:** [wallet/docker/entrypoint.sh](mdc:zaino/wallet/docker/entrypoint.sh) + * Generates `zallet.toml` (by default at `/etc/zallet/zallet.toml`) based on environment variables if the file is not present. + * Ensures the identity file path specified in the config is accessible. + * Handles user privilege dropping using `gosu`. + +## Configuration File (`config/zallet.toml` or generated by entrypoint): + +* Defines paths for `wallet_db` and `keystore.identity` (relative to `ZALLET_DATA_DIR` if generated by entrypoint). +* Specifies `indexer.validator_address` to connect to Zaino (e.g., `zaino:8137`). +* Configures `rpc.bind` for its own JSON-RPC server. + +## Connection to Zaino & Zebra: + +* **Primary Data Source (Zaino):** Zallet primarily connects to **Zaino** to fetch indexed blockchain data (compact blocks, tree states) for wallet scanning and account updates. + * This connection is typically **gRPC-based**, using the address specified in `zallet.toml` under `indexer.validator_address` (e.g., `zaino:8137`). + * The logic for this is within Zallet's `ChainView` component ([wallet/zallet/src/components/chain_view.rs](mdc:zaino/wallet/zallet/src/components/chain_view.rs)), which utilizes `zaino-state` and `zaino-fetch` libraries. +* **Transaction Submission & Direct Zebra Interaction (Potential):** + * While Zaino is the main data source, Zallet might also interact directly with **Zebra** for specific operations like transaction submission (if Zaino acts as a relay) or for fetching data not available/indexed by Zaino. + * Zallet includes `zebra-rpc` as a dependency, indicating capability for direct JSON-RPC communication with Zebra. + * The exact flow for transaction submission (Zallet -> Zaino -> Zebra, or Zallet -> Zebra) needs to be confirmed from Zallet's transaction broadcasting logic. +* **Authentication:** Authentication methods for connecting to Zaino (or Zebra directly, if applicable) would be configured within Zallet, likely in its `indexer` section if Zaino/Zebra RPCs are secured. + +## Role in Z3 Stack: + +Zallet is the user-facing wallet. It manages private keys, derives addresses, constructs and signs transactions, and keeps track of account balances by scanning blockchain data provided by Zaino. It exposes its own JSON-RPC interface for wallet commands. diff --git a/.cursor/rules/z3_service_zebra.mdc b/.cursor/rules/z3_service_zebra.mdc new file mode 100644 index 0000000..6ec07d2 --- /dev/null +++ b/.cursor/rules/z3_service_zebra.mdc @@ -0,0 +1,38 @@ +--- +description: +globs: +alwaysApply: false +--- +# Z3 Service: Zebra + +This rule describes the Zebra service configuration within the Z3 Docker stack. + +## Docker Configuration (`docker-compose.yml`): + +* **Image:** `zfnd/zebra:latest` (or a specific version). +* **Container Name:** `z3_zebra`. +* **Environment Variables:** + * `RUST_LOG`: Controls Zebra's logging level (e.g., `${ZEBRA_RUST_LOG:-info}`). + * `NETWORK`: Specifies the Zcash network (e.g., `${NETWORK_NAME:-Testnet}`). + * `ZEBRA_RPC_PORT`: Defines the RPC port Zebra listens on (e.g., `${ZEBRA_RPC_PORT:-18232}`). + * `ENABLE_COOKIE_AUTH`: Toggles RPC cookie authentication (e.g., `${ENABLE_COOKIE_AUTH:-false}`). +* **Volumes:** + * `zebra_data:/home/zebra/.cache/zebra`: Persists Zebra's blockchain data and state. + * `./config/zebra.toml:/home/zebra/.config/zebrad.toml:ro`: Mounts the Zebra configuration file [z3/config/zebra.toml](mdc:zaino/z3/config/zebra.toml). +* **Ports:** Exposes Zebra's RPC port to the host (e.g., `"${ZEBRA_HOST_RPC_PORT:-18232}:${ZEBRA_RPC_PORT:-18232}"`). +* **Healthcheck:** Uses `zebrad getblockchaininfo` to verify service health. + +## Source and Build (if building locally): + +* **Dockerfile:** [zebra/docker/Dockerfile](mdc:zaino/zebra/docker/Dockerfile) +* **Entrypoint Script:** [zebra/docker/entrypoint.sh](mdc:zaino/zebra/docker/entrypoint.sh) + * This script handles initial setup, configuration generation from environment variables if `zebra.toml` is minimal or missing, and user privilege dropping using `gosu`. + +## Configuration File (`config/zebra.toml`): + +* The primary configuration for Zebra. Settings here can be supplemented or overridden by environment variables defined in the `docker-compose.yml` and processed by the `entrypoint.sh`. +* Key settings often include network parameters, RPC listener details, and state cache directory (though the latter is usually managed by the volume mount). + +## Role in Z3 Stack: + +Zebra is the Zcash consensus node. It connects to the Zcash P2P network, downloads and validates the blockchain, and provides an RPC interface for other services (like Zaino and Zallet) to query blockchain data and submit transactions.