diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index abdf44a9..4365594a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,12 +20,12 @@ jobs: branch: ${{ steps.export_outputs.outputs.branch }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true - name: Checkout submodules - run: git submodule update --init + run: git submodule update --init --recursive - name: Install ubuntu dependencies run: | @@ -55,126 +55,84 @@ jobs: release_name: ${{ env.VERSION }} draft: false prerelease: ${{ env.PRERELEASE }} + - name: Export outputs id: export_outputs run: | echo "::set-output name=version::$VERSION" echo "::set-output name=branch::$BRANCH" - build_and_publish_normal: - if: github.event.pull_request.merged - needs: create_release - name: Build and publish for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-22.04 - asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.11 - uses: actions/setup-python@v1 - with: - python-version: 3.11 - - - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-22.04' - run: | - sudo apt-get update - - - name: Checkout submodules - run: git submodule update --init - - - name: Build normal binary - run: | - mkdir -p ./dist - docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal - ls -altr /home/ubuntu/dist/ - docker rm -f $(docker ps -aq) - - - name: Save sha512sum - run: | - sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum - - - name: Upload release binary - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} - asset_name: ${{ matrix.asset_name }} - asset_content_type: application/octet-stream - - - name: Upload release checksum - id: upload-release-checksum - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: /home/ubuntu/dist/sha512sum - asset_name: ${{ matrix.asset_name }}.sha512 - asset_content_type: text/plain - - build_and_publish_sync: + build_and_publish: if: github.event.pull_request.merged needs: create_release - name: Build and publish for ${{ matrix.os }} + name: Build and publish ${{ matrix.build_type }} for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - include: - - os: ubuntu-22.04 - asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync + os: [ubuntu-22.04] + build_type: [normal, sync, mirage] steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.11 - uses: actions/setup-python@v1 - with: - python-version: 3.11 - - - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-22.04' - run: | - sudo apt-get update - - - name: Checkout submodules - run: git submodule update --init - - - name: Build sync release binary - run: | - mkdir -p ./dist - docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - ls -altr /home/ubuntu/dist/ - docker rm -f $(docker ps -aq) - - - name: Save sha512sum - run: | - sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum - - - name: Upload release sync CLI - id: upload-sync-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} - asset_name: ${{ matrix.asset_name }} - asset_content_type: application/octet-stream - - - name: Upload release sync CLI checksum - id: upload-sync-release-checksum - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: /home/ubuntu/dist/sha512sum - asset_name: ${{ matrix.asset_name }}.sha512 - asset_content_type: text/plain + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install ubuntu dependencies + if: matrix.os == 'ubuntu-22.04' + run: | + sudo apt-get update + + - name: Ensure submodules are updated + run: git submodule update --init --recursive + + - name: Define Asset Name + id: asset_details + run: | + ASSET_BASE_NAME="skale-${{ needs.create_release.outputs.version }}-Linux-x86_64" + if [[ "${{ matrix.build_type }}" == "normal" ]]; then + echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}" >> $GITHUB_OUTPUT + else + echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}-${{ matrix.build_type }}" >> $GITHUB_OUTPUT + fi + + - name: Build ${{ matrix.build_type }} release binary + run: | + mkdir -p ${{ github.workspace }}/dist + docker build . -t node-cli-builder + docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ + scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} ${{ matrix.build_type }} + echo "Contents of dist directory:" + ls -altr ${{ github.workspace }}/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum for ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} + run: | + cd ${{ github.workspace }}/dist + sha512sum ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} > ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum + echo "Checksum file created: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum" + cat ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum + + - name: Upload release binary (${{ matrix.build_type }}) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} + asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} + asset_content_type: application/octet-stream + + - name: Upload release checksum (${{ matrix.build_type }}) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum + asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512 + asset_content_type: text/plain \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ca8a514..04bde884 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install -e .[dev] + pip install -e ".[dev]" - name: Generate info run: ./scripts/generate_info.sh 1.0.0 my-branch normal @@ -57,6 +57,16 @@ jobs: - name: Check build - sync run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync + - name: Build binary - mirage + run: | + mkdir -p ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test mirage + docker rm -f $(docker ps -aq) + + - name: Check build - mirage + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-mirage + - name: Run prepare test build run: | scripts/build.sh test test normal diff --git a/.gitignore b/.gitignore index 9ce8b81c..e183f4a7 100644 --- a/.gitignore +++ b/.gitignore @@ -122,3 +122,4 @@ test-env nginx.conf tests/.skale/node_data/docker.json tests/.skale/node_data/node_options.json +tests/.skale/config/nginx.conf.j2 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c77efb4b..c2fc6972 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,6 @@ COPY . . ENV PATH=/app/buildvenv/bin:$PATH ENV PYTHONPATH="{PYTHONPATH}:/usr/lib/python3/dist-packages" -RUN python3.11 -m venv /app/buildvenv && \ - pip install --upgrade pip && \ +RUN pip install --upgrade pip && \ pip install wheel setuptools==63.2.0 && \ pip install -e '.[dev]' diff --git a/README.md b/README.md index b2b2a95e..b4d0e691 100644 --- a/README.md +++ b/README.md @@ -4,63 +4,105 @@ ![Test](https://github.com/skalenetwork/node-cli/workflows/Test/badge.svg) [![Discord](https://img.shields.io/discord/534485763354787851.svg)](https://discord.gg/vvUtWJB) -SKALE Node CLI, part of the SKALE suite of validator tools, is the command line to setup, register and maintain your SKALE node. +SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Sync (for dedicated sChain synchronization), and Mirage (for the Mirage network). ## Table of Contents 1. [Installation](#installation) -2. [CLI usage](#cli-usage) - 2.1 [Top level commands](#top-level-commands) - 2.2 [Node](#node-commands) - 2.3 [Wallet](#wallet-commands) - 2.4 [sChains](#schain-commands) - 2.5 [Health](#health-commands) - 2.6 [SSL](#ssl-commands) - 2.7 [Logs](#logs-commands) - 2.8 [Resources allocation](#resources-allocation-commands) -3. [Sync CLI usage](#sync-cli-usage) - 3.1 [Top level commands](#top-level-commands-sync) + 1.1 [Standard Node Binary](#standard-node-binary) + 1.2 [Sync Node Binary](#sync-node-binary) + 1.3 [Mirage Node Binary](#mirage-node-binary) + 1.4 [Permissions and Testing](#permissions-and-testing) +2. [Standard Node Usage (`skale` - Normal Build)](#standard-node-usage-skale---normal-build) + 2.1 [Top level commands (Standard)](#top-level-commands-standard) + 2.2 [Node commands (Standard)](#node-commands-standard) + 2.3 [Wallet commands (Standard)](#wallet-commands-standard) + 2.4 [sChain commands (Standard)](#schain-commands-standard) + 2.5 [Health commands (Standard)](#health-commands-standard) + 2.6 [SSL commands (Standard)](#ssl-commands-standard) + 2.7 [Logs commands (Standard)](#logs-commands-standard) + 2.8 [Resources allocation commands (Standard)](#resources-allocation-commands-standard) +3. [Sync Node Usage (`skale` - Sync Build)](#sync-node-usage-skale---sync-build) + 3.1 [Top level commands (Sync)](#top-level-commands-sync) 3.2 [Sync node commands](#sync-node-commands) -4. [Exit codes](#exit-codes) -5. [Development](#development) +4. [Mirage Node Usage (`mirage`)](#mirage-node-usage-mirage) + 4.1 [Top level commands (Mirage)](#top-level-commands-mirage) + 4.2 [Mirage Boot commands](#mirage-boot-commands) + 4.3 [Mirage Node commands](#mirage-node-commands) +5. [Exit codes](#exit-codes) +6. [Development](#development) + +--- ## Installation -- Prerequisites +### Prerequisites + +Ensure that the following packages are installed: **docker**, **docker-compose** (1.27.4+) + +### Standard Node Binary + +This binary (`skale-VERSION-OS`) is used for managing standard SKALE validator nodes. + +```shell +# Replace {version} with the desired release version (e.g., 2.6.0) +VERSION_NUM={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" +``` -Ensure that the following package is installed: **docker**, **docker-compose** (1.27.4+) +### Sync Node Binary -- Download the executable +This binary (`skale-VERSION-OS-sync`) is used for managing dedicated Sync nodes. **Ensure you download the correct `-sync` suffixed binary for Sync node operations.** ```shell -VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" +# Replace {version} with the desired release version (e.g., 2.6.0) +VERSION_NUM={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" ``` -For Sync node version: +### Mirage Node Binary + +This binary (`skale-VERSION-OS-mirage`) is used specifically for managing nodes on the Mirage network. It is named `mirage`. ```shell -VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" +# Replace {version} with the desired release version (e.g., 2.6.0) +VERSION_NUM={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-mirage > /usr/local/bin/mirage" ``` -- Apply executable permissions to the downloaded binary: +### Permissions and Testing + +Apply executable permissions to the downloaded binary (adjust name accordingly): ```shell -chmod +x /usr/local/bin/skale +# For Standard or Sync binary +sudo chmod +x /usr/local/bin/skale + +# For Mirage binary +sudo chmod +x /usr/local/bin/mirage ``` -- Test the installation +Test the installation: ```shell +# Standard or Sync build skale --help + +# Mirage build +mirage --help ``` -## CLI usage +--- + +## Standard Node Usage (`skale` - Normal Build) -### Top level commands +Commands available in the **standard `skale` binary** for managing nodes. + +### Top level commands (Standard) #### Info -Print build info +Print build info for the `skale` (normal) binary. ```shell skale info @@ -68,7 +110,7 @@ skale info #### Version -Print version number +Print version number for the `skale` (normal) binary. ```shell skale version @@ -78,13 +120,13 @@ Options: - `--short` - prints version only, without additional text. -### Node commands +### Node commands (Standard) > Prefix: `skale node` #### Node information -Get base info about SKALE node +Get base info about the standard SKALE node. ```shell skale node info @@ -92,11 +134,11 @@ skale node info Options: -`-f/--format json/text` - optional +- `-f/--format json/text` - optional. #### Node initialization -Initialize a SKALE node on current machine +Initialize a standard SKALE node on the current machine. > :warning: **Avoid re-initializing a node that’s already initialized**: Run `skale node info` first to confirm the current initialization state. @@ -106,32 +148,32 @@ skale node init [ENV_FILE] Arguments: -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) +- `ENV_FILE` - path to .env file (required). -You should specify the following environment variables: +Required environment variables in `ENV_FILE`: -- `SGX_SERVER_URL` - SGX server URL -- `DISK_MOUNTPOINT` - disk mount point for storing sChains data -- `DOCKER_LVMPY_STREAM` - stream of `docker-lvmpy` to use -- `CONTAINER_CONFIGS_STREAM` - stream of `skale-node` to use -- `ENDPOINT` - RPC endpoint of the node in the network where SKALE Manager is deployed -- `MANAGER_CONTRACTS` - SKALE Manager `message_proxy_mainnet` contract alias or address -- `IMA_CONTRACTS` - IMA `skale_manager` contract alias or address -- `FILEBEAT_URL` - URL to the Filebeat log server -- `ENV_TYPE` - environment type (e.g., 'mainnet', 'testnet', 'qanet', 'devnet') +- `SGX_SERVER_URL` - SGX server URL. +- `DISK_MOUNTPOINT` - Mount point for storing sChains data. +- `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy` to use. +- `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` to use. +- `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. +- `MANAGER_CONTRACTS` - SKALE Manager `message_proxy_mainnet` contract alias or address. +- `IMA_CONTRACTS` - IMA `skale_manager` contract alias or address. +- `FILEBEAT_HOST` - URL of the Filebeat log server. +- `ENV_TYPE` - Environment type (e.g., 'mainnet', 'testnet', 'qanet', 'devnet'). > In `MANAGER_CONTRACTS` and `IMA_CONTRACTS` fields, if you are using a recognized network (e.g., 'Mainnet', 'Holesky', 'local'), you can use a recognized alias (e.g., 'production', 'grants'). You can check the list of recognized networks and aliases in [contract deployments](https://github.com/skalenetwork/skale-contracts/tree/deployments). -> :warning: If you are using a custom network or a contract which isn't recognized by underlying skale library, you **MUST** provide a direct contract address. +> :warning: If you are using a custom network or a contract which isn't recognized by the underlying `skale-contracts` library, you **MUST** provide a direct contract address. Optional variables: - `TG_API_KEY` - Telegram API key - `TG_CHAT_ID` - Telegram chat ID -- `MONITORING_CONTAINERS` - will enable monitoring containers (`cadvisor`, `node-exporter`). +- `MONITORING_CONTAINERS` - Enable monitoring containers (`cadvisor`, `node-exporter`). #### Node initialization from backup -Restore SKALE node on another machine +Restore a standard SKALE node on another machine from a backup. ```shell skale node restore [BACKUP_PATH] [ENV_FILE] @@ -139,139 +181,148 @@ skale node restore [BACKUP_PATH] [ENV_FILE] Arguments: -- `BACKUP_PATH` - path to the archive with backup data generated by `skale node backup` command -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) +- `BACKUP_PATH` - Path to the archive created by `skale node backup`. +- `ENV_FILE` - Path to .env file with configuration for the restored node. #### Node backup -Generate backup file to restore SKALE node on another machine +Generate a backup archive of the standard SKALE node's state. ```shell -skale node backup [BACKUP_FOLDER_PATH] [ENV_FILE] +skale node backup [BACKUP_FOLDER_PATH] ``` Arguments: -- `BACKUP_FOLDER_PATH` - path to the folder where the backup file will be saved +- `BACKUP_FOLDER_PATH` - Path to the folder where the backup file will be saved. #### Node Registration +Register the standard node with the SKALE Manager contract. + ```shell -skale node register +skale node register --name --ip --domain [--port ] ``` Required arguments: -- `--ip` - public IP for RPC connections and consensus -- `--domain`/`-d` - SKALE node domain name -- `--name` - SKALE node name +- `--ip` - Public IP for RPC connections and consensus. +- `--domain`/`-d` - SKALE node domain name. +- `--name` - SKALE node name. Optional arguments: -- `--port` - public port - beginning of the port range for node SKALE Chains (default: `10000`) +- `--port` - Base port for node sChains (default: `10000`). #### Node update -Update SKALE node on current machine +Update the standard SKALE node software and configuration. ```shell -skale node update [ENV_FILEPATH] +skale node update [ENV_FILEPATH] [--yes] ``` -Options: - -- `--yes` - update without additional confirmation - Arguments: -- `ENV_FILEPATH` - path to env file where parameters are defined +- `ENV_FILEPATH` - Path to the .env file containing potentially updated parameters. + +Options: -You can also specify a file with environment variables -which will update parameters in env file used during skale node init. +- `--yes` - Update without confirmation prompt. #### Node turn-off -Turn-off SKALE node on current machine and optionally set it to the maintenance mode +Turn off the standard SKALE node containers. ```shell -skale node turn-off +skale node turn-off [--maintenance-on] [--yes] ``` Options: -- `--maintenance-on` - set SKALE node into maintenance mode before turning off -- `--yes` - turn off without additional confirmation +- `--maintenance-on` - Set node to maintenance mode before turning off. +- `--yes` - Turn off without confirmation. #### Node turn-on -Turn on SKALE node on current machine and optionally disable maintenance mode +Turn on the standard SKALE node containers. ```shell -skale node turn-on [ENV_FILEPATH] +skale node turn-on [ENV_FILEPATH] [--maintenance-off] [--yes] ``` -Options: - -- `--maintenance-off` - turn off maintenance mode after turning on the node -- `--yes` - turn on without additional confirmation - Arguments: -- `ENV_FILEPATH` - path to env file where parameters are defined +- `ENV_FILEPATH` - Path to the .env file. + +Options: -You can also specify a file with environment variables -which will update parameters in env file used during skale node init. +- `--maintenance-off` - Turn off maintenance mode after turning on. +- `--yes` - Turn on without additional confirmation. #### Node maintenance -Set SKALE node into maintenance mode +Control the node's maintenance status in SKALE Manager. ```shell -skale node maintenance-on +# Set maintenance ON +skale node maintenance-on [--yes] + +# Set maintenance OFF +skale node maintenance-off ``` Options: -- `--yes` - set without additional confirmation +- `--yes` - Perform action without additional confirmation. + +#### Domain name -Switch off maintenance mode +Set the standard node's domain name. ```shell -skale node maintenance-off +skale node set-domain --domain [--yes] ``` -#### Domain name +Required Options: -Set SKALE node domain name +- `--domain`/`-d` - The new SKALE node domain name. + +Options: + +- `--yes` - Set without additional confirmation. + +#### Skale Node Signature + +Get the node signature for a validator ID. ```shell -skale node set-domain +skale node signature ``` -Options: +Arguments: -- `--domain`/`-d` - SKALE node domain name -- `--yes` - set without additional confirmation +- `VALIDATOR_ID` - The ID of the validator requesting the signature. -### Wallet commands +### Wallet commands (Standard) > Prefix: `skale wallet` -Commands related to Ethereum wallet associated with SKALE node +Commands related to the Ethereum wallet associated with the standard SKALE node. #### Wallet information ```shell -skale wallet info +skale wallet info [-f json/text] ``` Options: -`-f/--format json/text` - optional +- `-f/--format json/text` - optional. #### Wallet setting -Set local wallet for the SKALE node +Set the local wallet private key for the node. ```shell skale wallet set --private-key $ETH_PRIVATE_KEY @@ -279,188 +330,197 @@ skale wallet set --private-key $ETH_PRIVATE_KEY #### Send ETH tokens -Send ETH tokens from SKALE node wallet to specific address +Send ETH from the node's wallet. ```shell -skale wallet send [ADDRESS] [AMOUNT] +skale wallet send [--yes] ``` Arguments: -- `ADDRESS` - Ethereum receiver address -- `AMOUNT` - Amount of ETH tokens to send +- `RECEIVER_ADDRESS` - Ethereum receiver address. +- `AMOUNT_ETH` - Amount of ETH tokens to send. Optional arguments: -`--yes` - Send without additional confirmation +- `--yes` - Send without additional confirmation. -### sChain commands +### sChain commands (Standard) > Prefix: `skale schains` -#### SKALE Chain list +Commands for interacting with sChains managed by the standard node. + +#### List sChains -List of SKALE Chains served by connected node +List of SKALE Chains served by connected node. ```shell skale schains ls ``` -#### SKALE Chain configuration +#### Get sChain config + +Show the configuration for a specific SKALE Chain. ```shell -skale schains config SCHAIN_NAME +skale schains config ``` -#### SKALE Chain DKG status +#### Get DKG status -List DKG status for each SKALE Chain on the node +List DKG status for each SKALE Chain on the node. ```shell skale schains dkg ``` -#### SKALE Chain info +#### Get sChain info -Show information about SKALE Chain on node +Show information about a specific SKALE Chain on the node. ```shell -skale schains info SCHAIN_NAME +skale schains info [--json] ``` Options: -- `--json` - Show info in JSON format +- `--json` - Show info in JSON format. -#### SKALE Chain repair +#### Repair sChain -Turn on repair mode for SKALE Chain +Turn on repair mode for a specific SKALE Chain. ```shell -skale schains repair SCHAIN_NAME +skale schains repair ``` -### Health commands +### Health commands (Standard) > Prefix: `skale health` -#### SKALE containers +Commands to check the health of the standard node and its components. -List all SKALE containers running on the connected node +#### List containers + +List all SKALE containers running on the connected node. ```shell -skale health containers +skale health containers [-a/--all] ``` Options: -- `-a/--all` - list all containers (by default - only running) +- `-a/--all` - list all containers (by default - only running). -#### sChains healthchecks +#### Get sChains healthchecks -Show health check results for all SKALE Chains on the node +Show health check results for all SKALE Chains on the node. ```shell -skale health schains +skale health schains [--json] ``` Options: -- `--json` - Show data in JSON format +- `--json` - Show data in JSON format. -#### SGX +#### Check SGX server status Status of the SGX server. Returns the SGX server URL and connection status. ```shell -$ skale health sgx - -SGX server status: -┌────────────────┬────────────────────────────┐ -│ SGX server URL │ https://0.0.0.0:1026/ │ -├────────────────┼────────────────────────────┤ -│ Status │ CONNECTED │ -└────────────────┴────────────────────────────┘ +skale health sgx ``` -### SSL commands +### SSL commands (Standard) > Prefix: `skale ssl` -#### SSL Status +Manage SSL certificates for the standard node. + +#### Check SSL Status -Status of the SSL certificates on the node +Status of the SSL certificates on the node. ```shell skale ssl status ``` -Admin API URL: \[GET] `/api/ssl/status` +Admin API URL: `[GET] /api/ssl/status` #### Upload certificates -Upload new SSL certificates +Upload new SSL certificates. ```shell -skale ssl upload +skale ssl upload -c -k [-f/--force] ``` -##### Options +Options: -- `-c/--cert-path` - Path to the certificate file -- `-k/--key-path` - Path to the key file -- `-f/--force` - Overwrite existing certificates +- `-c/--cert-path` - Path to the certificate file. +- `-k/--key-path` - Path to the key file. +- `-f/--force` - Overwrite existing certificates. -Admin API URL: \[GET] `/api/ssl/upload` +Admin API URL: `[POST] /api/ssl/upload` -#### Check ssl certificate +#### Check certificate -Check SSL certificate by connecting to the health-check SSL server +Check SSL certificate by connecting to the health-check SSL server. ```shell -skale ssl check +skale ssl check [-c ] [-k ] [--type ] [--port ] [--no-client] ``` -##### Options +Options: -- `-c/--cert-path` - Path to the certificate file (default: uploaded using `skale ssl upload` certificate) -- `-k/--key-path` - Path to the key file (default: uploaded using `skale ssl upload` key) -- `--type/-t` - Check type (`openssl` - openssl cli check, `skaled` - skaled-based check, `all` - both) -- `--port/-p` - Port to start healthcheck server (default: `4536`) -- `--no-client` - Skip client connection (only make sure server started without errors) +- `-c/--cert-path` - Path to the certificate file (default: uploaded using `skale ssl upload` certificate). +- `-k/--key-path` - Path to the key file (default: uploaded using `skale ssl upload` key). +- `--type/-t` - Check type (`openssl` - openssl cli check, `skaled` - skaled-based check, `all` - both). +- `--port/-p` - Port to start healthcheck server (default: `4536`). +- `--no-client` - Skip client connection (only make sure server started without errors). -### Logs commands +### Logs commands (Standard) > Prefix: `skale logs` -#### CLI Logs +Access logs for the standard node. + +#### Fetch CLI Logs Fetch node CLI logs: ```shell -skale logs cli +skale logs cli [--debug] ``` Options: -- `--debug` - show debug logs; more detailed output +- `--debug` - show debug logs; more detailed output. -#### Dump Logs +#### Dump All Node Logs Dump all logs from the connected node: ```shell -skale logs dump [PATH] +skale logs dump [TARGET_PATH] [-c/--container ] ``` -Optional arguments: +Arguments: -- `--container`, `-c` - Dump logs only from specified container +- `TARGET_PATH` - Optional path to save the log dump archive. -### Resources allocation commands +Options: + +- `--container`, `-c` - Dump logs only from specified container. + +### Resources allocation commands (Standard) > Prefix: `skale resources-allocation` +Manage the resources allocation file for the standard node. + #### Show allocation file Show resources allocation file: @@ -469,40 +529,43 @@ Show resources allocation file: skale resources-allocation show ``` -#### Generate/update +#### Generate/update allocation file Generate/update allocation file: ```shell -skale resources-allocation generate [ENV_FILE] +skale resources-allocation generate [ENV_FILE] [--yes] [-f/--force] ``` Arguments: -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) +- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command). Options: -- `--yes` - generate without additional confirmation -- `-f/--force` - rewrite allocation file if it exists +- `--yes` - generate without additional confirmation. +- `-f/--force` - rewrite allocation file if it exists. -## Sync CLI usage +--- -A sync node is a node dedicated to synchronizing a single sChain. +## Sync Node Usage (`skale` - Sync Build) -### Top level commands sync +Commands available in the **sync `skale` binary** for managing dedicated Sync nodes. +Note that this binary contains a **different set of commands** compared to the standard build. -#### Info +### Top level commands (Sync) + +#### Info (Sync) -Print build info +Print build info for the `skale` (sync) binary. ```shell skale info ``` -#### Version +#### Version (Sync) -Print version number +Print version number for the `skale` (sync) binary. ```shell skale version @@ -518,69 +581,255 @@ Options: #### Sync node initialization -Initialize full sync SKALE node on current machine +Initialize a dedicated Sync node on the current machine. ```shell -skale sync-node init [ENV_FILE] +skale sync-node init [ENV_FILE] [--indexer | --archive] [--snapshot] [--snapshot-from ] [--yes] ``` Arguments: -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale sync-node init` command) +- `ENV_FILE` - path to .env file (required). -You should specify the following environment variables: +Required environment variables in `ENV_FILE`: -- `DISK_MOUNTPOINT` - disk mount point for storing sChains data -- `DOCKER_LVMPY_STREAM` - stream of `docker-lvmpy` to use -- `CONTAINER_CONFIGS_STREAM` - stream of `skale-node` to use -- `ENDPOINT` - RPC endpoint of the node in the network where SKALE Manager is deployed -- `MANAGER_CONTRACTS` - SKALE Manager main contract alias or address -- `IMA_CONTRACTS` - IMA main contract alias or address -- `SCHAIN_NAME` - name of the SKALE chain to sync -- `ENV_TYPE` - environment type (e.g., 'mainnet', 'testnet', 'qanet', 'devnet') +- `DISK_MOUNTPOINT` - Mount point for storing sChain data. +- `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy`. +- `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node`. +- `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. +- `MANAGER_CONTRACTS` - SKALE Manager alias or address. +- `IMA_CONTRACTS` - IMA alias or address. +- `SCHAIN_NAME` - Name of the specific SKALE chain to sync. +- `ENV_TYPE` - Environment type (e.g., 'mainnet', 'testnet'). > In `MANAGER_CONTRACTS` and `IMA_CONTRACTS` fields, if you are using a recognized network (e.g., 'Mainnet', 'Holesky', 'local'), you can use a recognized alias (e.g., 'production', 'grants'). You can check the list of recognized networks and aliases in [contract deployments](https://github.com/skalenetwork/skale-contracts/tree/deployments). -> :warning: If you are using a custom network or a contract which isn't recognized by underlying skale library, you **MUST** provide a direct contract address. +> :warning: If you are using a custom network or a contract which isn't recognized by the underlying `skale-contracts` library, you **MUST** provide a direct contract address. Options: -- `--indexer` - run sync node in indexer mode (disable block rotation) -- `--archive` - enable historic state and disable block rotation (can't be used with `--indexer`) -- `--snapshot` - start sync node from snapshot -- `--snapshot-from` - specify the IP of the node to take snapshot from -- `--yes` - initialize without additional confirmation +- `--indexer` - Run in indexer mode (disables block rotation). +- `--archive` - Run in archive mode (enable historic state and disable block rotation). +- `--snapshot` - Start sync node from snapshot. +- `--snapshot-from ` - Specify the IP of another node to download a snapshot from. +- `--yes` - Initialize without additional confirmation. #### Sync node update -Update full sync SKALE node on current machine +Update the Sync node software and configuration. ```shell -skale sync-node update [ENV_FILEPATH] +skale sync-node update [ENV_FILEPATH] [--yes] ``` -Options: +Arguments: -- `--yes` - update without additional confirmation +- `ENV_FILEPATH` - Path to the .env file. -Arguments: +Options: -- `ENV_FILEPATH` - path to env file where parameters are defined +- `--yes` - Update without additionalconfirmation. > NOTE: You can just update a file with environment variables used during `skale sync-node init`. #### Sync node cleanup -Cleanup full sync SKALE node on current machine +Remove all data and containers for the Sync node. + +```shell +skale sync-node cleanup [--yes] +``` + +Options: + +- `--yes` - Cleanup without confirmation. + +> WARNING: This command removes all Sync node data. + +--- + +## Mirage Node Usage (`mirage`) + +Commands available in the **`mirage` binary** for managing nodes on the Mirage network. + +### Top level commands (Mirage) + +#### Mirage Info + +Print build info for the `mirage` binary. + +```shell +mirage info +``` + +#### Mirage Version + +Print version number for the `mirage` binary. + +```shell +mirage version [--short] +``` + +Options: + +- `--short` - prints version only, without additional text. + +### Mirage Boot commands + +> Prefix: `mirage boot` + +Commands for a Mirage node in the Boot phase. + +#### Mirage Boot Initialization + +Initialize the Mirage node boot phase. + +```shell +mirage boot init [ENV_FILE] +``` + +Arguments: + +- `ENV_FILE` - path to .env file (required). + +Required environment variables in `ENV_FILE`: + +- `SGX_SERVER_URL` - SGX server URL. +- `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). +- `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` configs. +- `ENDPOINT` - RPC endpoint of the network where Mirage Manager is deployed. +- `MANAGER_CONTRACTS` - SKALE Manager alias or address. +- `IMA_CONTRACTS` - IMA alias or address (_Note: Required by boot service, may not be used by Mirage itself_). +- `FILEBEAT_HOST` - URL/IP:Port of the Filebeat log server. +- `ENV_TYPE` - Environment type (e.g., 'mainnet', 'devnet'). + +Optional variables: + +- `MONITORING_CONTAINERS` - Enable monitoring containers (`cadvisor`, `node-exporter`). + +#### Mirage Boot Registration + +Register the Mirage node with Mirage Manager _during_ the boot phase. + +```shell +mirage boot register --name --ip --domain [--port ] +``` + +Required arguments: + +- `--name`/`-n` - Mirage node name. +- `--ip` - Public IP for RPC connections and consensus. +- `--domain`/`-d` - Mirage node domain name (e.g., `mirage1.example.com`). + +Optional arguments: + +- `--port`/`-p` - Base port for node sChains (default: `10000`). + +#### Mirage Boot Signature + +Get the node signature for a validator ID _during_ the boot phase. + +```shell +mirage boot signature +``` + +Arguments: + +- `VALIDATOR_ID` - The ID of the validator requesting the signature. + +#### Mirage Boot Migrate + +Migrate the Mirage node from the boot phase to the main phase (regular operation). ```shell -skale sync-node cleanup +mirage boot migrate [ENV_FILEPATH] [--yes] ``` +Arguments: + +- `ENV_FILEPATH` - Path to the .env file. + Options: -- `--yes` - cleanup without additional confirmation +- `--yes` - Migrate without confirmation. + +### Mirage Node commands + +> Prefix: `mirage node` + +Commands for managing a Mirage node during its regular operation (main phase). + +#### Mirage Node Initialization (Placeholder) + +Initialize the regular operation phase of the Mirage node. + +```shell +mirage node init +``` + +> **Note:** This command is currently a placeholder and not implemented. + +#### Mirage Node Registration (Placeholder) + +Register the node during regular operation. + +```shell +mirage node register +``` + +> **Note:** This command is currently a placeholder and not implemented. + +#### Mirage Node Update (Placeholder) + +Update the Mirage node during regular operation. + +```shell +mirage node update [ENV_FILEPATH] [--yes] [--unsafe] +``` -> WARNING: This command will remove all data from the node. +> **Note:** This command is currently a placeholder and not implemented. + +#### Mirage Node Signature + +Get the node signature for a validator ID during regular operation. + +```shell +mirage node signature +``` + +Arguments: + +- `VALIDATOR_ID` - The ID of the validator requesting the signature. + +#### Mirage Node Backup + +Generate a backup archive of the Mirage node's state. + +```shell +mirage node backup +``` + +Arguments: + +- `BACKUP_FOLDER_PATH` - Path to the folder where the backup file will be saved. + +#### Mirage Node Restore + +Restore a Mirage node from a backup archive. + +```shell +mirage node restore [--config-only] +``` + +Arguments: + +- `BACKUP_PATH` - Path to the archive. +- `ENV_FILE` - Path to the .env file for the restored node configuration. + +Options: + +- `--config-only` - Only restore configuration files. + +--- ## Exit codes @@ -598,37 +847,74 @@ Exit codes conventions for SKALE CLI tools `*` - `validator-cli` only\ `**` - `node-cli` only +--- + ## Development ### Setup repo +#### Dependencies + +- Python 3.11 +- Git + +#### Clone the repository + +Clone with HTTPS: + +```shell +git clone https://github.com/skalenetwork/node-cli.git +``` + +Or with SSH: + +```shell +git clone git@github.com:skalenetwork/node-cli.git +``` + +#### Create and source virtual environment + +```shell +python3.11 -m venv venv +source venv/bin/activate +``` + #### Install development dependencies ```shell -pip install -e .[dev] +pip install -e ".[dev]" ``` #### Generate info.py locally +Specify the build type (`normal`, `sync`, or `mirage`): + ```shell +# Example for Standard build ./scripts/generate_info.sh 1.0.0 my-branch normal + +# Example for Sync build +./scripts/generate_info.sh 1.0.0 my-branch sync + +# Example for Mirage build +./scripts/generate_info.sh 1.0.0 my-branch mirage ``` -##### Add linting git hook +#### Add linting git hook In file `.git/hooks/pre-commit` add: ```shell #!/bin/sh -ruff check +./venv/bin/ruff check . ``` -### Debugging +> **Note:** This hook assumes your virtual environment is named 'venv' and is located at the root of the repository. -Run commands in dev mode: +Make the hook executable: ```shell -ENV=dev python main.py YOUR_COMMAND +chmod +x .git/hooks/pre-commit ``` ## Contributing diff --git a/helper-scripts b/helper-scripts index 84b57271..7c6ccee7 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit 84b572717eef72feb3c901ea8817f9dcddbca8bb +Subproject commit 7c6ccee7599f30ddec3cf0d7747dd031fd57cb27 diff --git a/node_cli/cli/mirage_boot.py b/node_cli/cli/mirage_boot.py new file mode 100644 index 00000000..4419b92a --- /dev/null +++ b/node_cli/cli/mirage_boot.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.core.node import get_node_signature, register_node as register +from node_cli.core.mirage_boot import init, migrate, update +from node_cli.configs import DEFAULT_NODE_BASE_PORT +from node_cli.utils.helper import streamed_cmd, IP_TYPE, error_exit, abort_if_false + + +@click.group('boot', help='Commands for the Mirage Boot phase.') +def mirage_boot_cli(): + pass + + +@mirage_boot_cli.command('init', help='Initialize Mirage node (Boot Phase).') +@click.argument('env_file') +@streamed_cmd +def init_boot(env_file): + init(env_file) + + +@mirage_boot_cli.command( + 'register', help='Register Mirage node in SKALE Manager (during Boot Phase).' +) +@click.option( + '--name', '-n', required=True, prompt='Enter mirage node name', help='Mirage node name' +) +@click.option( + '--ip', + prompt='Enter node public IP', + type=IP_TYPE, + help='Public IP for RPC connections & consensus (required)', +) +@click.option( + '--port', '-p', default=DEFAULT_NODE_BASE_PORT, type=int, help='Base port for node sChains' +) +@click.option('--domain', '-d', prompt='Enter node domain name', type=str, help='Node domain name') +@streamed_cmd +def register_boot(name, ip, port, domain): + register(name=name, p2p_ip=ip, public_ip=ip, port=port, domain_name=domain) + + +@mirage_boot_cli.command( + 'signature', help='Get mirage node signature for a validator ID (during Boot Phase).' +) +@click.argument('validator_id') +def signature_boot(validator_id): + res = get_node_signature(validator_id) + if isinstance(res, dict) and 'error' in res: + error_exit(f'Error getting signature: {res.get("message", res)}') + print(f'Signature: {res}') + + +@mirage_boot_cli.command( + 'migrate', help='Migrate mirage node from Mirage Boot Phase to Mirage Main Phase.' +) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to mirage node from Mirage Boot Phase to Mirage Main Phase?', +) +@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.argument('env_file') +@streamed_cmd +def migrate_boot(env_file, pull_config_for_schain): + migrate(env_file, pull_config_for_schain) + + +@mirage_boot_cli.command('update', help='Update Mirage node from .env file') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to update Mirage node software?', +) +@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.argument('env_file') +@streamed_cmd +def update_node(env_file, pull_config_for_schain): + update( + env_filepath=env_file, + pull_config_for_schain=pull_config_for_schain, + ) diff --git a/node_cli/cli/mirage_node.py b/node_cli/cli/mirage_node.py new file mode 100644 index 00000000..0fc32668 --- /dev/null +++ b/node_cli/cli/mirage_node.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.core.node import get_node_signature, backup +from node_cli.core.mirage_node import restore_mirage +from node_cli.utils.helper import error_exit, streamed_cmd, abort_if_false + + +@click.group('node', help='Commands for regular Mirage Node operations.') +def mirage_node_cli(): + pass + + +@mirage_node_cli.command('init', help='Initialize regular Mirage node operations (Placeholder).') +def init_node(): + click.echo("Placeholder: Command 'mirage node init' is not yet implemented.") + + +@mirage_node_cli.command( + 'register', help='Register Mirage node (Placeholder for regular operations).' +) +def register_node(): + click.echo("Placeholder: Command 'mirage node register' is not yet implemented.") + + +@mirage_node_cli.command('update', help='Update Mirage.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to update Mirage node software?', +) +@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) +@click.argument('env_file') +@streamed_cmd +def update_node(env_file, pull_config_for_schain, unsafe_ok): + click.echo("Placeholder: Command 'mirage node update' is not yet implemented.") + + +@mirage_node_cli.command('signature', help='Get mirage node signature for a validator ID.') +@click.argument('validator_id') +def signature_node(validator_id): + res = get_node_signature(validator_id) + if isinstance(res, dict) and 'error' in res: + error_exit(f'Error getting signature: {res.get("message", res)}') + print(f'Signature: {res}') + + +@mirage_node_cli.command('backup', help='Generate backup file for the Mirage node.') +@click.argument('backup_folder_path') +@streamed_cmd +def backup_node(backup_folder_path): + backup(backup_folder_path) + + +@mirage_node_cli.command('restore', help='Restore Mirage node from a backup file.') +@click.argument('backup_path') +@click.argument('env_file') +@click.option( + '--config-only', + help='Only restore configuration files in .skale and artifacts', + is_flag=True, + hidden=True, +) +@streamed_cmd +def restore_node(backup_path, env_file, config_only): + restore_mirage(backup_path, env_file, config_only) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index ebddc5c8..4c9e40e3 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -19,8 +19,9 @@ import click -from node_cli.core.node import configure_firewall_rules +from node_cli.cli.info import TYPE from node_cli.core.node import ( + configure_firewall_rules, get_node_signature, init, restore, @@ -83,7 +84,7 @@ def register_node(name, ip, port, domain): @click.argument('env_file') @streamed_cmd def init_node(env_file): - init(env_file) + init(env_filepath=env_file, node_type=TYPE) @node.command('update', help='Update node from .env file') @@ -99,7 +100,12 @@ def init_node(env_file): @click.argument('env_file') @streamed_cmd def update_node(env_file, pull_config_for_schain, unsafe_ok): - update(env_file, pull_config_for_schain, unsafe_ok) + update( + env_filepath=env_file, + pull_config_for_schain=pull_config_for_schain, + node_type=TYPE, + unsafe_ok=unsafe_ok, + ) @node.command('signature', help='Get node signature for given validator id') @@ -130,7 +136,13 @@ def backup_node(backup_folder_path): ) @streamed_cmd def restore_node(backup_path, env_file, no_snapshot, config_only): - restore(backup_path, env_file, no_snapshot, config_only) + restore( + backup_path=backup_path, + env_filepath=env_file, + no_snapshot=no_snapshot, + config_only=config_only, + node_type=TYPE, + ) @node.command('maintenance-on', help='Set SKALE node into maintenance mode') @@ -166,7 +178,7 @@ def remove_node_from_maintenance(): @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe turn-off', hidden=True, is_flag=True) @streamed_cmd def _turn_off(maintenance_on, unsafe_ok): - turn_off(maintenance_on, unsafe_ok) + turn_off(node_type=TYPE, maintenance_on=maintenance_on, unsafe_ok=unsafe_ok) @node.command('turn-on', help='Turn on the node') @@ -189,7 +201,7 @@ def _turn_off(maintenance_on, unsafe_ok): @click.argument('env_file') @streamed_cmd def _turn_on(maintenance_off, sync_schains, env_file): - turn_on(maintenance_off, sync_schains, env_file) + turn_on(maintenance_off, sync_schains, env_file, node_type=TYPE) @node.command('set-domain', help='Set node domain name') @@ -206,7 +218,7 @@ def _set_domain_name(domain): set_domain_name(domain) -@node.command(help='Check if node meet network requirements') +@node.command(help='Check if node meets network requirements') @click.option( '--network', '-n', @@ -215,7 +227,7 @@ def _set_domain_name(domain): help='Network to check', ) def check(network): - run_checks(network) + run_checks(node_type=TYPE, network=network) @node.command(help='Reconfigure nftables rules') diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py index 768690ed..9a518a2d 100644 --- a/node_cli/cli/resources_allocation.py +++ b/node_cli/cli/resources_allocation.py @@ -25,6 +25,7 @@ generate_resource_allocation_config, ) from node_cli.utils.helper import abort_if_false, safe_load_texts +from node_cli.utils.node_type import NodeType TEXTS = safe_load_texts() @@ -59,4 +60,4 @@ def show(): ) @click.option('--force', '-f', is_flag=True, help='Rewrite if already exists') def generate(env_file, force): - generate_resource_allocation_config(env_file=env_file, force=force) + generate_resource_allocation_config(node_type=NodeType.REGULAR, env_file=env_file, force=force) diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index 188b39fe..803ea754 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -32,6 +32,7 @@ show_schains, toggle_schain_repair_mode, ) +from node_cli.cli.info import TYPE @click.group() @@ -107,4 +108,4 @@ def info_(schain_name: str, json_format: bool) -> None: def restore( schain_name: str, snapshot_path: str, schain_type: str, env_type: Optional[str] ) -> None: - restore_schain_from_snapshot(schain_name, snapshot_path) + restore_schain_from_snapshot(schain_name, snapshot_path, node_type=TYPE) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 33ee9ab2..668e8ea1 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -34,7 +34,7 @@ FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR_REGULAR = '/mnt' -SCHAINS_MNT_DIR_SYNC = '/var/lib/skale/schains' +SCHAINS_MNT_DIR_SINGLE_CHAIN = '/var/lib/skale/schains' VOLUME_GROUP = 'schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') @@ -56,7 +56,9 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') SYNC_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-sync.yml') +MIRAGE_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-mirage.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') +MIRAGE_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'mirage_static_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index c6528421..9979beef 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -24,6 +24,7 @@ from node_cli.configs import SKALE_DIR, CONTAINER_CONFIG_PATH from node_cli.configs.alias_address_validation import validate_env_alias_or_address, ContractType +from node_cli.utils.node_type import NodeType from node_cli.utils.helper import error_exit SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') @@ -31,27 +32,37 @@ ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] -REQUIRED_PARAMS: Dict[str, str] = { +CORE_REQUIRED_PARAMS: Dict[str, str] = { 'CONTAINER_CONFIGS_STREAM': '', 'ENDPOINT': '', 'MANAGER_CONTRACTS': '', - 'IMA_CONTRACTS': '', - 'FILEBEAT_HOST': '', 'DISK_MOUNTPOINT': '', 'SGX_SERVER_URL': '', - 'DOCKER_LVMPY_STREAM': '', 'ENV_TYPE': '', } +REQUIRED_PARAMS_SKALE: Dict[str, str] = { + **CORE_REQUIRED_PARAMS, + 'IMA_CONTRACTS': '', + 'DOCKER_LVMPY_STREAM': '', + 'FILEBEAT_HOST': '', +} + +REQUIRED_PARAMS_MIRAGE_BOOT: Dict[str, str] = { + **CORE_REQUIRED_PARAMS, + 'IMA_CONTRACTS': '', + 'FILEBEAT_HOST': '', +} +REQUIRED_PARAMS_MIRAGE: Dict[str, str] = { + **CORE_REQUIRED_PARAMS, + 'FILEBEAT_HOST': '', +} + REQUIRED_PARAMS_SYNC: Dict[str, str] = { + **CORE_REQUIRED_PARAMS, 'SCHAIN_NAME': '', - 'CONTAINER_CONFIGS_STREAM': '', - 'ENDPOINT': '', - 'MANAGER_CONTRACTS': '', 'IMA_CONTRACTS': '', - 'DISK_MOUNTPOINT': '', 'DOCKER_LVMPY_STREAM': '', - 'ENV_TYPE': '', } OPTIONAL_PARAMS: Dict[str, str] = { @@ -76,12 +87,14 @@ def absent_required_params(params: Dict[str, str]) -> List[str]: def get_validated_env_config( - env_filepath: str = SKALE_DIR_ENV_FILEPATH, sync_node: bool = False + node_type: NodeType, + env_filepath: str = SKALE_DIR_ENV_FILEPATH, + is_mirage_boot: bool = False, ) -> Dict[str, str]: load_env_file(env_filepath) - params = build_env_params(sync_node) + params = build_env_params(node_type=node_type, is_mirage_boot=is_mirage_boot) populate_env_params(params) - validate_env_params(params) + validate_env_params(params=params) return params @@ -90,9 +103,19 @@ def load_env_file(env_filepath: str) -> None: error_exit(f'Failed to load environment from {env_filepath}') -def build_env_params(sync_node: bool = False) -> Dict[str, str]: - """Return environment variables dictionary with keys based on node type.""" - params = REQUIRED_PARAMS_SYNC.copy() if sync_node else REQUIRED_PARAMS.copy() +def build_env_params( + node_type: NodeType, + is_mirage_boot: bool = False, +) -> Dict[str, str]: + if node_type == NodeType.MIRAGE and is_mirage_boot: + params = REQUIRED_PARAMS_MIRAGE_BOOT.copy() + elif node_type == NodeType.MIRAGE: + params = REQUIRED_PARAMS_MIRAGE.copy() + elif node_type == NodeType.SYNC: + params = REQUIRED_PARAMS_SYNC.copy() + else: + params = REQUIRED_PARAMS_SKALE.copy() + params.update(OPTIONAL_PARAMS) return params @@ -104,15 +127,19 @@ def populate_env_params(params: Dict[str, str]) -> None: params[key] = str(env_value) -def validate_env_params(params: Dict[str, str]) -> None: +def validate_env_params( + params: Dict[str, str], +) -> None: missing = absent_required_params(params) if missing: error_exit(f'Missing required parameters: {missing}') - validate_env_type(params['ENV_TYPE']) + validate_env_type(env_type=params['ENV_TYPE']) endpoint = params['ENDPOINT'] - validate_env_alias_or_address(params['IMA_CONTRACTS'], ContractType.IMA, endpoint) validate_env_alias_or_address(params['MANAGER_CONTRACTS'], ContractType.MANAGER, endpoint) + if 'IMA_CONTRACTS' in params.keys(): + validate_env_alias_or_address(params['IMA_CONTRACTS'], ContractType.IMA, endpoint) + def validate_env_type(env_type: str) -> None: if env_type not in ALLOWED_ENV_TYPES: diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index f9d42212..1428c440 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -56,9 +56,11 @@ DOCKER_DAEMON_HOSTS, REPORTS_PATH, STATIC_PARAMS_FILEPATH, + MIRAGE_STATIC_PARAMS_FILEPATH, ) from node_cli.core.host import is_ufw_ipv6_chain_exists, is_ufw_ipv6_option_enabled from node_cli.core.resources import get_disk_size +from node_cli.utils.docker_utils import NodeType from node_cli.utils.helper import run_cmd, safe_mkdir logger = logging.getLogger(__name__) @@ -76,9 +78,18 @@ FuncList = List[Func] -def get_static_params(env_type: str = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH) -> Dict: - status_params_filename = os.path.basename(STATIC_PARAMS_FILEPATH) - static_params_filepath = os.path.join(config_path, status_params_filename) +def get_static_params( + node_type: NodeType, + env_type: str = 'mainnet', + config_path: str = CONTAINER_CONFIG_PATH, +) -> Dict: + if node_type == NodeType.MIRAGE: + static_params_base_filepath = MIRAGE_STATIC_PARAMS_FILEPATH + else: + static_params_base_filepath = STATIC_PARAMS_FILEPATH + + static_params_filename = os.path.basename(static_params_base_filepath) + static_params_filepath = os.path.join(config_path, static_params_filename) with open(static_params_filepath) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) return ydata['envs'][env_type] @@ -154,6 +165,9 @@ def merge_reports( class BaseChecker: + def __init__(self, requirements: Dict) -> None: + self.requirements = requirements + def _ok(self, name: str, info: Optional[Union[str, Dict]] = None) -> CheckResult: return CheckResult(name=name, status='ok', info=info) @@ -169,7 +183,8 @@ def get_checks(self, check_type: CheckType = CheckType.ALL) -> FuncList: methods = inspect.getmembers( type(self), predicate=lambda m: inspect.isfunction(m) - and getattr(m, '_check_type', None) in allowed_types, + and getattr(m, '_check_type', None) in allowed_types + and self.requirements.get(m.__name__, None) != 'disabled', ) return [functools.partial(m[1], self) for m in methods] @@ -190,9 +205,9 @@ class MachineChecker(BaseChecker): def __init__( self, requirements: Dict, disk_device: str, network_timeout: Optional[int] = None ) -> None: - self.requirements = requirements self.disk_device = disk_device self.network_timeout = network_timeout or NETWORK_CHECK_TIMEOUT + super().__init__(requirements=requirements) @preinstall def cpu_total(self) -> CheckResult: @@ -274,7 +289,7 @@ def network(self) -> CheckResult: class PackageChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: - self.requirements = requirements + super().__init__(requirements=requirements) def _check_apt_package(self, package_name: str, version: str = None) -> CheckResult: # TODO: check versions @@ -327,7 +342,7 @@ def _version_from_dpkg_output(self, output: str) -> str: class DockerChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: self.docker_client = docker.from_env() - self.requirements = requirements + super().__init__(requirements=requirements) def _check_docker_command(self) -> Optional[str]: return shutil.which('docker') @@ -471,12 +486,13 @@ def get_all_checkers(disk: str, requirements: Dict) -> List[BaseChecker]: def run_checks( disk: str, + node_type: NodeType, env_type: str = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, check_type: CheckType = CheckType.ALL, ) -> ResultList: logger.info('Executing checks. Type: %s', check_type) - requirements = get_static_params(env_type, config_path) + requirements = get_static_params(node_type, env_type, config_path) checkers = get_all_checkers(disk, requirements) checks = get_checks(checkers, check_type) results = [check() for check in checks] diff --git a/node_cli/core/host.py b/node_cli/core/host.py index f4d4e790..e3279b75 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -47,8 +47,8 @@ SKALE_TMP_DIR, UFW_CONFIG_PATH, UFW_IPV6_BEFORE_INPUT_CHAIN, + NGINX_CONFIG_FILEPATH, ) -from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH from node_cli.configs.env import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH from node_cli.core.nftables import NFTablesManager @@ -103,7 +103,7 @@ def prepare_host(env_filepath: str, env_type: str, allocation: bool = False) -> def is_node_inited() -> bool: - return os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) + return os.path.isfile(NGINX_CONFIG_FILEPATH) def make_dirs(): diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 3060b874..67472e96 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -58,7 +58,7 @@ def create_logs_dump(path, filter_container=None): def create_dump_dir(): - time = datetime.datetime.utcnow().strftime('%Y-%m-%d--%H-%M-%S') + time = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d--%H-%M-%S') folder_name = f'skale-logs-dump-{time}' folder_path = os.path.join(SKALE_TMP_DIR, folder_name) containers_path = os.path.join(folder_path, 'containers') diff --git a/node_cli/core/mirage_boot.py b/node_cli/core/mirage_boot.py new file mode 100644 index 00000000..5d16affa --- /dev/null +++ b/node_cli/core/mirage_boot.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import logging +import time + +from node_cli.configs import TM_INIT_TIMEOUT +from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.operations import init_mirage_boot_op, migrate_mirage_boot_op, update_mirage_boot_op +from node_cli.utils.decorators import check_not_inited, check_inited, check_user +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit +from node_cli.utils.node_type import NodeType +from node_cli.utils.print_formatters import print_node_cmd_error + + +logger = logging.getLogger(__name__) + + +@check_not_inited +def init(env_filepath: str) -> None: + env = compose_node_env( + env_filepath, + node_type=NodeType.MIRAGE, + is_mirage_boot=True, + ) + + init_mirage_boot_op(env_filepath, env) + logger.info('Waiting for mirage containers initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(node_type=NodeType.MIRAGE, is_mirage_boot=True): + error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + logger.info('Init mirage procedure finished') + + +@check_inited +@check_user +def migrate(env_filepath: str, pull_config_for_schain: str) -> None: + logger.info('Mirage node migration started') + env = compose_node_env( + env_filepath, + inited_node=True, + sync_schains=False, + pull_config_for_schain=pull_config_for_schain, + node_type=NodeType.MIRAGE, + ) + migrate_ok = migrate_mirage_boot_op(env_filepath, env) + if migrate_ok: + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + alive = is_base_containers_alive(node_type=NodeType.MIRAGE) + if not migrate_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Node migration from Mirage Boot to Mirage Main finished successfully!') + + +@check_inited +@check_user +def update(env_filepath: str, pull_config_for_schain: str) -> None: + logger.info('Mirage boot node update started') + env = compose_node_env( + env_filepath, + inited_node=True, + sync_schains=False, + pull_config_for_schain=pull_config_for_schain, + node_type=NodeType.MIRAGE, + is_mirage_boot=True, + ) + migrate_ok = update_mirage_boot_op(env_filepath, env) + if migrate_ok: + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + alive = is_base_containers_alive(node_type=NodeType.MIRAGE, is_mirage_boot=True) + if not migrate_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Mirage boot node update finished successfully!') diff --git a/node_cli/core/mirage_node.py b/node_cli/core/mirage_node.py new file mode 100644 index 00000000..4a4101f9 --- /dev/null +++ b/node_cli/core/mirage_node.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import logging +import time + +from node_cli.configs import SKALE_DIR, RESTORE_SLEEP_TIMEOUT +from node_cli.core.host import save_env_params +from node_cli.core.node import compose_node_env +from node_cli.utils.decorators import check_not_inited +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit +from node_cli.utils.node_type import NodeType +from node_cli.operations import restore_mirage_op + + +logger = logging.getLogger(__name__) + + +@check_not_inited +def restore_mirage(backup_path, env_filepath, config_only=False): + env = compose_node_env(env_filepath, node_type=NodeType.MIRAGE) + if env is None: + return + save_env_params(env_filepath) + env['SKALE_DIR'] = SKALE_DIR + + restored_ok = restore_mirage_op(env, backup_path, config_only=config_only) + if not restored_ok: + error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + time.sleep(RESTORE_SLEEP_TIMEOUT) + print('Mirage node is restored from backup') diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py index e87b17db..97ea4844 100644 --- a/node_cli/core/nginx.py +++ b/node_cli/core/nginx.py @@ -20,11 +20,12 @@ import logging import os.path -from node_cli.utils.docker_utils import restart_nginx_container, docker_client +from node_cli.cli.info import TYPE from node_cli.configs import NODE_CERTS_PATH, NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH +from node_cli.utils.node_type import NodeType +from node_cli.utils.docker_utils import restart_nginx_container, docker_client from node_cli.utils.helper import process_template - logger = logging.getLogger(__name__) @@ -34,10 +35,12 @@ def generate_nginx_config() -> None: ssl_on = check_ssl_certs() + regular_node = is_regular_node_nginx() template_data = { 'ssl': ssl_on, + 'regular_node': regular_node, } - logger.info(f'Processing nginx template. ssl: {ssl_on}') + logger.info(f'Processing nginx template. ssl: {ssl_on}, regular_node: {regular_node}') process_template(NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, template_data) @@ -47,6 +50,10 @@ def check_ssl_certs(): return os.path.exists(crt_path) and os.path.exists(key_path) +def is_regular_node_nginx() -> bool: + return TYPE in [NodeType.REGULAR, NodeType.SYNC] + + def reload_nginx() -> None: dutils = docker_client() generate_nginx_config() diff --git a/node_cli/core/node.py b/node_cli/core/node.py index ff5da975..19f4a688 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -36,7 +36,7 @@ LOG_PATH, RESTORE_SLEEP_TIMEOUT, SCHAINS_MNT_DIR_REGULAR, - SCHAINS_MNT_DIR_SYNC, + SCHAINS_MNT_DIR_SINGLE_CHAIN, SKALE_DIR, SKALE_STATE_DIR, TM_INIT_TIMEOUT, @@ -73,14 +73,21 @@ from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.decorators import check_not_inited, check_inited, check_user -from node_cli.utils.docker_utils import is_admin_running, is_api_running, is_sync_admin_running +from node_cli.utils.docker_utils import ( + is_admin_running, + is_api_running, + BASE_SKALE_COMPOSE_SERVICES, + BASE_SYNC_COMPOSE_SERVICES, + BASE_MIRAGE_COMPOSE_SERVICES, + BASE_MIRAGE_BOOT_COMPOSE_SERVICES, +) +from node_cli.utils.node_type import NodeType from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 logger = logging.getLogger(__name__) TEXTS = Texts() -SYNC_BASE_CONTAINERS_AMOUNT = 2 BASE_CONTAINERS_AMOUNT = 5 BLUEPRINT_NAME = 'node' @@ -96,11 +103,12 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def is_update_safe(sync_node: bool = False) -> bool: - if not sync_node and not is_admin_running() and not is_api_running(): - return True - if sync_node and not is_sync_admin_running(): - return True +def is_update_safe(node_type: NodeType) -> bool: + if not is_admin_running(node_type): + if node_type == NodeType.SYNC: + return True + elif not is_api_running(node_type): + return True status, payload = get_request(BLUEPRINT_NAME, 'update-safe') if status == 'error': return False @@ -137,15 +145,13 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited -def init(env_filepath): - env = compose_node_env(env_filepath) +def init(env_filepath: str, node_type: NodeType) -> None: + env = compose_node_env(env_filepath=env_filepath, node_type=node_type) - inited_ok = init_op(env_filepath, env) - if not inited_ok: - error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + init_op(env_filepath=env_filepath, env=env, node_type=node_type) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(): + if not is_base_containers_alive(node_type=node_type): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Generating resource allocation file ...') update_resource_allocation(env['ENV_TYPE']) @@ -153,8 +159,8 @@ def init(env_filepath): @check_not_inited -def restore(backup_path, env_filepath, no_snapshot=False, config_only=False): - env = compose_node_env(env_filepath) +def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, config_only=False): + env = compose_node_env(env_filepath=env_filepath, node_type=node_type) if env is None: return save_env_params(env_filepath) @@ -164,7 +170,7 @@ def restore(backup_path, env_filepath, no_snapshot=False, config_only=False): logger.info('Adding BACKUP_RUN to env ...') env['BACKUP_RUN'] = 'True' # should be str - restored_ok = restore_op(env, backup_path, config_only=config_only) + restored_ok = restore_op(env, backup_path, node_type=node_type, config_only=config_only) if not restored_ok: error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) @@ -177,15 +183,13 @@ def restore(backup_path, env_filepath, no_snapshot=False, config_only=False): def init_sync( env_filepath: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: - env = compose_node_env(env_filepath, sync_node=True) + env = compose_node_env(env_filepath, node_type=NodeType.SYNC) if env is None: return - inited_ok = init_sync_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) - if not inited_ok: - error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + init_sync_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(sync_node=True): + if not is_base_containers_alive(node_type=NodeType.SYNC): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Sync node initialized successfully') @@ -197,12 +201,12 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: prev_version = get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(env_filepath, sync_node=True) + env = compose_node_env(env_filepath, node_type=NodeType.SYNC) update_ok = update_sync_op(env_filepath, env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(sync_node=True) + alive = is_base_containers_alive(node_type=NodeType.SYNC) if not update_ok or not alive: print_node_cmd_error() return @@ -213,7 +217,7 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: @check_inited @check_user def cleanup_sync() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, sync_node=True) + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SYNC) schain_name = env['SCHAIN_NAME'] cleanup_sync_op(env, schain_name) logger.info('Sync node was cleaned up, all containers and data removed') @@ -221,20 +225,32 @@ def cleanup_sync() -> None: def compose_node_env( env_filepath: str, + node_type: NodeType, inited_node: bool = False, sync_schains: Optional[bool] = None, pull_config_for_schain: Optional[str] = None, - sync_node: bool = False, save: bool = True, -) -> dict: + is_mirage_boot: bool = False, +) -> dict[str, str]: if env_filepath is not None: - env_params = get_validated_env_config(env_filepath, sync_node=sync_node) + env_params = get_validated_env_config( + node_type=node_type, + env_filepath=env_filepath, + is_mirage_boot=is_mirage_boot, + ) if save: save_env_params(env_filepath) else: - env_params = get_validated_env_config(INIT_ENV_FILEPATH, sync_node=sync_node) - - mnt_dir = SCHAINS_MNT_DIR_SYNC if sync_node else SCHAINS_MNT_DIR_REGULAR + env_params = get_validated_env_config( + node_type=node_type, + env_filepath=INIT_ENV_FILEPATH, + is_mirage_boot=is_mirage_boot, + ) + + if node_type == NodeType.SYNC or node_type == NodeType.MIRAGE: + mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN + else: + mnt_dir = SCHAINS_MNT_DIR_REGULAR env = { 'SKALE_DIR': SKALE_DIR, @@ -244,10 +260,10 @@ def compose_node_env( **env_params, } - if inited_node and not sync_node: + if inited_node and not node_type == NodeType.SYNC: env['FLASK_SECRET_KEY'] = get_flask_secret_key() - if sync_schains and not sync_node: + if sync_schains and not node_type == NodeType.SYNC: env['BACKUP_RUN'] = 'True' if pull_config_for_schain: @@ -258,8 +274,13 @@ def compose_node_env( @check_inited @check_user -def update(env_filepath: str, pull_config_for_schain: str, unsafe_ok: bool = False) -> None: - if not unsafe_ok and not is_update_safe(): +def update( + env_filepath: str, + pull_config_for_schain: Optional[str], + node_type: NodeType, + unsafe_ok: bool = False, +) -> None: + if not unsafe_ok and not is_update_safe(node_type=node_type): error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -272,12 +293,13 @@ def update(env_filepath: str, pull_config_for_schain: str, unsafe_ok: bool = Fal inited_node=True, sync_schains=False, pull_config_for_schain=pull_config_for_schain, + node_type=node_type, ) - update_ok = update_op(env_filepath, env) + update_ok = update_op(env_filepath, env, node_type=node_type) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive() + alive = is_base_containers_alive(node_type=node_type) if not update_ok or not alive: print_node_cmd_error() return @@ -300,7 +322,7 @@ def backup(path): def get_backup_filename(): - time = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S') + time = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d-%H-%M-%S') return f'{BACKUP_ARCHIVE_NAME}-{time}.tar.gz' @@ -327,7 +349,7 @@ def pack_dir(source: str, dest: str, exclude: Tuple[str] = ()): def logfilter(tarinfo): path = Path(tarinfo.name) for e in exclude: - logger.debug('Cheking if %s is parent of %s', e, tarinfo.name) + logger.debug('Checking if %s is parent of %s', e, tarinfo.name) try: path.relative_to(e) except ValueError: @@ -379,24 +401,26 @@ def set_maintenance_mode_off(): @check_inited @check_user -def turn_off(maintenance_on: bool = False, unsafe_ok: bool = False) -> None: - if not unsafe_ok and not is_update_safe(): +def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool = False) -> None: + if not unsafe_ok and not is_update_safe(node_type=node_type): error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False) - turn_off_op(env=env) + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type) + turn_off_op(node_type=node_type, env=env) @check_inited @check_user -def turn_on(maintenance_off, sync_schains, env_file): - env = compose_node_env(env_file, inited_node=True, sync_schains=sync_schains) - turn_on_op(env) +def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> None: + env = compose_node_env( + env_file, inited_node=True, sync_schains=sync_schains, node_type=node_type + ) + turn_on_op(env=env, node_type=node_type) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(): + if not is_base_containers_alive(node_type=node_type): print_node_cmd_error() return logger.info('Node turned on') @@ -404,12 +428,30 @@ def turn_on(maintenance_off, sync_schains, env_file): set_maintenance_mode_off() -def is_base_containers_alive(sync_node: bool = False): +def get_expected_container_names(node_type: NodeType, is_mirage_boot: bool) -> list[str]: + if node_type == NodeType.MIRAGE and is_mirage_boot: + services = BASE_MIRAGE_BOOT_COMPOSE_SERVICES + elif node_type == NodeType.MIRAGE and not is_mirage_boot: + services = BASE_MIRAGE_COMPOSE_SERVICES + elif node_type == NodeType.SYNC: + services = BASE_SYNC_COMPOSE_SERVICES + else: + services = BASE_SKALE_COMPOSE_SERVICES + + return list(services.values()) + + +def is_base_containers_alive(node_type: NodeType, is_mirage_boot: bool = False) -> bool: + base_container_names = get_expected_container_names(node_type, is_mirage_boot) + dclient = docker.from_env() - containers = dclient.containers.list() - skale_containers = list(filter(lambda c: c.name.startswith('skale_'), containers)) - containers_amount = SYNC_BASE_CONTAINERS_AMOUNT if sync_node else BASE_CONTAINERS_AMOUNT - return len(skale_containers) >= containers_amount + running_container_names = set(container.name for container in dclient.containers.list()) + + for base_container in base_container_names: + if base_container not in running_container_names: + return False + + return True def get_node_info_plain(): @@ -450,6 +492,7 @@ def set_domain_name(domain_name): def run_checks( + node_type: NodeType, network: str = 'mainnet', container_config_path: str = CONTAINER_CONFIG_PATH, disk: Optional[str] = None, @@ -459,13 +502,13 @@ def run_checks( return if disk is None: - env = get_validated_env_config() + env = get_validated_env_config(node_type=node_type) disk = env['DISK_MOUNTPOINT'] - failed_checks = run_host_checks(disk, network, container_config_path) + failed_checks = run_host_checks(disk, node_type, network, container_config_path) if not failed_checks: - print('Requirements checking succesfully finished!') + print('Requirements checking successfully finished!') else: - print('Node is not fully meet the requirements!') + print('Node does not fully meet the requirements!') print_failed_requirements_checks(failed_checks) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 5762ebad..669f8d0f 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -28,6 +28,7 @@ from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml +from node_cli.utils.node_type import NodeType from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH, SNAPSHOTS_SHARED_VOLUME from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, @@ -91,13 +92,17 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = } -def generate_resource_allocation_config(env_file, force=False) -> None: +def generate_resource_allocation_config( + env_file, + node_type: NodeType, + force=False, +) -> None: if not force and os.path.isfile(RESOURCE_ALLOCATION_FILEPATH): msg = 'Resource allocation file already exists' logger.debug(msg) print(msg) return - env_params = get_validated_env_config(env_file) + env_params = get_validated_env_config(node_type=node_type, env_filepath=env_file) if env_params is None: return logger.info('Generating resource allocation file ...') diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 44c6d83a..957e02df 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -13,7 +13,7 @@ NODE_CONFIG_PATH, NODE_CLI_STATUS_FILENAME, SCHAIN_NODE_DATA_PATH, - SCHAINS_MNT_DIR_SYNC, + SCHAINS_MNT_DIR_SINGLE_CHAIN, ) from node_cli.configs.env import get_validated_env_config @@ -27,6 +27,7 @@ ) from node_cli.utils.docker_utils import ensure_volume, is_volume_exists from node_cli.utils.helper import read_json, run_cmd, save_json +from node_cli.utils.node_type import NodeType from lvmpy.src.core import mount, volume_mountpoint @@ -182,10 +183,14 @@ def fillin_snapshot_folder(src_path: str, block_number: int) -> None: def restore_schain_from_snapshot( - schain: str, snapshot_path: str, env_type: Optional[str] = None, schain_type: str = 'medium' + schain: str, + snapshot_path: str, + node_type: NodeType, + env_type: Optional[str] = None, + schain_type: str = 'medium', ) -> None: if env_type is None: - env_config = get_validated_env_config() + env_config = get_validated_env_config(node_type=node_type) env_type = env_config['ENV_TYPE'] ensure_schain_volume(schain, schain_type, env_type) block_number = get_block_number_from_path(snapshot_path) @@ -222,7 +227,7 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: logger.warning('Volume %s already exists', schain) -def cleanup_sync_datadir(schain_name: str, base_path: str = SCHAINS_MNT_DIR_SYNC) -> None: +def cleanup_sync_datadir(schain_name: str, base_path: str = SCHAINS_MNT_DIR_SINGLE_CHAIN) -> None: base_path = os.path.join(base_path, schain_name) regular_folders_pattern = f'{base_path}/[!snapshots]*' logger.info('Removing regular folders') diff --git a/node_cli/main.py b/node_cli/main.py index 8320331f..d1d152ec 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -27,6 +27,7 @@ import click from node_cli.cli import __version__ +from node_cli.cli.exit import exit_cli from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION, TYPE from node_cli.cli.logs import logs_cli @@ -35,13 +36,14 @@ from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli from node_cli.cli.ssl import ssl_cli -from node_cli.cli.exit import exit_cli from node_cli.cli.resources_allocation import resources_allocation_cli from node_cli.cli.sync_node import sync_node_cli - -from node_cli.utils.helper import safe_load_texts, init_default_logger -from node_cli.configs import LONG_LINE +from node_cli.cli.mirage_boot import mirage_boot_cli +from node_cli.cli.mirage_node import mirage_node_cli from node_cli.core.host import init_logs_dir +from node_cli.utils.node_type import NodeType +from node_cli.configs import LONG_LINE +from node_cli.utils.helper import safe_load_texts, init_default_logger from node_cli.utils.helper import error_exit TEXTS = safe_load_texts() @@ -80,8 +82,17 @@ def info(): def get_sources_list() -> List[click.MultiCommand]: - if TYPE == 'sync': + if TYPE == NodeType.SYNC: return [cli, sync_node_cli, ssl_cli] + elif TYPE == NodeType.MIRAGE: + return [ + cli, + logs_cli, + mirage_boot_cli, + mirage_node_cli, + wallet_cli, + ssl_cli, + ] else: return [ cli, diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 5c53ec18..159c0e16 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -21,10 +21,14 @@ update as update_op, init as init_op, init_sync as init_sync_op, + init_mirage_boot as init_mirage_boot_op, + migrate_mirage_boot as migrate_mirage_boot_op, + update_mirage_boot as update_mirage_boot_op, update_sync as update_sync_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op, + restore_mirage as restore_mirage_op, cleanup_sync as cleanup_sync_op, configure_nftables, ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index a4446ef6..c03f7f35 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -24,41 +24,39 @@ import logging from typing import Dict, Optional -from node_cli.cli.info import VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, SKALE_DIR, GLOBAL_SKALE_DIR, ) +from node_cli.core.checks import CheckType, run_checks as run_host_checks +from node_cli.core.docker_config import configure_docker from node_cli.core.host import ( ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host, ) - -from node_cli.core.docker_config import configure_docker -from node_cli.core.nginx import generate_nginx_config from node_cli.core.nftables import configure_nftables +from node_cli.core.nginx import generate_nginx_config from node_cli.core.node_options import NodeOptions from node_cli.core.resources import update_resource_allocation, init_shared_space_volume - -from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive -from node_cli.operations.volume import ( - cleanup_volume_artifacts, - ensure_filestorage_mapping, - prepare_block_device, +from node_cli.core.schains import ( + update_node_cli_schain_status, + cleanup_sync_datadir, ) -from node_cli.operations.docker_lvmpy import lvmpy_install # noqa +from node_cli.cli.info import VERSION, TYPE +from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive +from node_cli.operations.docker_lvmpy import lvmpy_install from node_cli.operations.skale_node import ( download_skale_node, sync_skale_node, update_images, ) -from node_cli.core.checks import CheckType, run_checks as run_host_checks -from node_cli.core.schains import ( - update_node_cli_schain_status, - cleanup_sync_datadir, +from node_cli.operations.volume import ( + cleanup_volume_artifacts, + ensure_filestorage_mapping, + prepare_block_device, ) from node_cli.utils.docker_utils import ( compose_rm, @@ -66,9 +64,10 @@ docker_cleanup, remove_dynamic_containers, ) +from node_cli.utils.helper import str_to_bool, rm_dir from node_cli.utils.meta import get_meta_info, update_meta +from node_cli.utils.node_type import NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks -from node_cli.utils.helper import str_to_bool, rm_dir logger = logging.getLogger(__name__) @@ -80,6 +79,7 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], + TYPE, env['ENV_TYPE'], CONTAINER_CONFIG_TMP_PATH, check_type=CheckType.PREINSTALL, @@ -94,6 +94,7 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], + TYPE, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, @@ -107,8 +108,8 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict) -> bool: - compose_rm(env) +def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: + compose_rm(node_type=node_type, env=env) remove_dynamic_containers() sync_skale_node() @@ -144,12 +145,91 @@ def update(env_filepath: str, env: Dict) -> bool: distro.version(), ) update_images(env=env) - compose_up(env) + compose_up(env=env, node_type=node_type) + return True + + +@checked_host +def migrate_mirage_boot(env_filepath: str, env: Dict) -> bool: + compose_rm(node_type=NodeType.MIRAGE, env=env) + + sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + generate_nginx_config() + + prepare_host(env_filepath, env['ENV_TYPE']) + + current_stream = get_meta_info().config_stream + skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' + if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + logger.info( + 'Stream version was changed from %s to %s', + current_stream, + env['CONTAINER_CONFIGS_STREAM'], + ) + docker_cleanup() + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version(), + ) + update_images(env=env) + compose_up(env=env, node_type=NodeType.MIRAGE) + return True + + +@checked_host +def update_mirage_boot(env_filepath: str, env: Dict) -> bool: + compose_rm(node_type=NodeType.MIRAGE, env=env) + remove_dynamic_containers() + + sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + generate_nginx_config() + + prepare_host(env_filepath, env['ENV_TYPE']) + + current_stream = get_meta_info().config_stream + skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' + if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + logger.info( + 'Stream version was changed from %s to %s', + current_stream, + env['CONTAINER_CONFIGS_STREAM'], + ) + docker_cleanup() + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version(), + ) + update_images(env=env) + compose_up(env=env, node_type=NodeType.MIRAGE) return True @checked_host -def init(env_filepath: str, env: dict) -> bool: +def init(env_filepath: str, env: dict, node_type: NodeType) -> None: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() @@ -179,8 +259,37 @@ def init(env_filepath: str, env: dict) -> bool: update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env=env) - compose_up(env) - return True + compose_up(env=env, node_type=node_type) + + +@checked_host +def init_mirage_boot(env_filepath: str, env: dict) -> None: + sync_skale_node() + + ensure_btrfs_kernel_module_autoloaded() + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + prepare_host(env_filepath, env_type=env['ENV_TYPE']) + link_env_file() + + configure_filebeat() + configure_flask() + generate_nginx_config() + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version(), + ) + update_images(env=env) + + compose_up(env=env, node_type=NodeType.MIRAGE, is_mirage_boot=True) def init_sync( @@ -190,7 +299,7 @@ def init_sync( archive: bool, snapshot: bool, snapshot_from: Optional[str], -) -> bool: +) -> None: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) sync_skale_node() @@ -233,12 +342,11 @@ def init_sync( update_images(env=env, sync_node=True) - compose_up(env, sync_node=True) - return True + compose_up(env=env, node_type=NodeType.SYNC) def update_sync(env_filepath: str, env: Dict) -> bool: - compose_rm(env, sync_node=True) + compose_rm(env=env, node_type=NodeType.SYNC) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node(env['CONTAINER_CONFIGS_STREAM'], env.get('CONTAINER_CONFIGS_DIR')) @@ -266,18 +374,18 @@ def update_sync(env_filepath: str, env: Dict) -> bool: ) update_images(env=env, sync_node=True) - compose_up(env, sync_node=True) + compose_up(env=env, node_type=NodeType.SYNC) return True -def turn_off(env: dict, sync_node: bool = False) -> None: +def turn_off(env: dict, node_type: NodeType) -> None: logger.info('Turning off the node...') - compose_rm(env=env, sync_node=sync_node) + compose_rm(env=env, node_type=node_type) remove_dynamic_containers() logger.info('Node was successfully turned off') -def turn_on(env: dict) -> None: +def turn_on(env: dict, node_type: NodeType) -> None: logger.info('Turning on the node...') update_meta( VERSION, @@ -293,13 +401,14 @@ def turn_on(env: dict) -> None: configure_nftables(enable_monitoring=enable_monitoring) logger.info('Launching containers on the node...') - compose_up(env) + compose_up(env=env, node_type=node_type) -def restore(env, backup_path, config_only=False): +def restore(env, backup_path, node_type: NodeType, config_only=False): unpack_backup_archive(backup_path) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], + TYPE, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, @@ -330,10 +439,58 @@ def restore(env, backup_path, config_only=False): update_resource_allocation(env_type=env['ENV_TYPE']) if not config_only: - compose_up(env) + compose_up(env=env, node_type=node_type) + + failed_checks = run_host_checks( + env['DISK_MOUNTPOINT'], + TYPE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.POSTINSTALL, + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + return True + + +def restore_mirage(env, backup_path, config_only=False): + unpack_backup_archive(backup_path) + failed_checks = run_host_checks( + env['DISK_MOUNTPOINT'], + TYPE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL, + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + + ensure_btrfs_kernel_module_autoloaded() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + link_env_file() + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version(), + ) + + if not config_only: + compose_up(env=env, node_type=NodeType.MIRAGE) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], + TYPE, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, @@ -345,7 +502,7 @@ def restore(env, backup_path, config_only=False): def cleanup_sync(env, schain_name: str) -> None: - turn_off(env, sync_node=True) + turn_off(env, node_type=NodeType.SYNC) cleanup_sync_datadir(schain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index d6d6a966..1595a442 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -30,7 +30,7 @@ DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, SCHAINS_MNT_DIR_REGULAR, - SCHAINS_MNT_DIR_SYNC, + SCHAINS_MNT_DIR_SINGLE_CHAIN, SKALE_STATE_DIR, ) @@ -137,7 +137,7 @@ def prepare_block_device(block_device, force=False): else: logger.info('%s contains %s filesystem', block_device, filesystem) format_as_btrfs(block_device) - mount_device(block_device, SCHAINS_MNT_DIR_SYNC) + mount_device(block_device, SCHAINS_MNT_DIR_SINGLE_CHAIN) def max_resize_btrfs(path): diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 2b75f9a3..60e869f0 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -31,10 +31,12 @@ from node_cli.configs import ( COMPOSE_PATH, SYNC_COMPOSE_PATH, + MIRAGE_COMPOSE_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, NGINX_CONTAINER_NAME, ) +from node_cli.utils.node_type import NodeType logger = logging.getLogger(__name__) @@ -43,21 +45,44 @@ IMA_REMOVE_TIMEOUT = 20 TELEGRAF_REMOVE_TIMEOUT = 20 -MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin') -BASE_COMPOSE_SERVICES = ( - 'transaction-manager', - 'skale-admin', - 'skale-api', - 'bounty', - 'nginx', - 'redis', - 'watchdog', - 'filebeat', -) -MONITORING_COMPOSE_SERVICES = ( - 'node-exporter', - 'advisor', -) +# Services have format : +CORE_COMMON_COMPOSE_SERVICES = { + 'transaction-manager': 'skale_transaction-manager', + 'redis': 'skale_redis', + 'watchdog': 'skale_watchdog', + 'nginx': 'skale_nginx', + 'filebeat': 'skale_filebeat', +} + +BASE_SKALE_COMPOSE_SERVICES = { + **CORE_COMMON_COMPOSE_SERVICES, + 'skale-admin': 'skale_admin', + 'skale-api': 'skale_api', + 'bounty': 'skale_bounty', +} + +CORE_MIRAGE_COMPOSE_SERVICES = { + **CORE_COMMON_COMPOSE_SERVICES, + 'mirage-api': 'mirage_api', +} +BASE_MIRAGE_COMPOSE_SERVICES = { + **CORE_MIRAGE_COMPOSE_SERVICES, + 'mirage-admin': 'mirage_admin', +} +BASE_MIRAGE_BOOT_COMPOSE_SERVICES = { + **CORE_MIRAGE_COMPOSE_SERVICES, + 'mirage-boot': 'mirage_boot_admin', +} + +BASE_SYNC_COMPOSE_SERVICES = { + 'skale-sync-admin': 'skale_sync_admin', + 'nginx': 'skale_nginx', +} + +MONITORING_COMPOSE_SERVICES = { + 'node-exporter': 'monitor_node_exporter', + 'advisor': 'monitor_cadvisor', +} TELEGRAF_SERVICES = ('telegraf',) NOTIFICATION_COMPOSE_SERVICES = ('celery',) COMPOSE_TIMEOUT = 10 @@ -225,9 +250,9 @@ def is_volume_exists(name: str, dutils=None): return True -def compose_rm(env={}, sync_node: bool = False): +def compose_rm(node_type: NodeType, env={}): logger.info('Removing compose containers') - compose_path = get_compose_path(sync_node) + compose_path = get_compose_path(node_type) run_cmd( cmd=( 'docker', @@ -245,47 +270,89 @@ def compose_rm(env={}, sync_node: bool = False): def compose_pull(env: dict, sync_node: bool = False): logger.info('Pulling compose containers') - compose_path = get_compose_path(sync_node) + compose_path = get_compose_path(NodeType.SYNC) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'pull'), env=env) def compose_build(env: dict, sync_node: bool = False): logger.info('Building compose containers') - compose_path = get_compose_path(sync_node) + compose_path = get_compose_path(NodeType.SYNC) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'build'), env=env) -def get_up_compose_cmd(services): - return ('docker', 'compose', '-f', COMPOSE_PATH, 'up', '-d', *services) +def get_compose_path(node_type: NodeType) -> str: + if node_type == NodeType.SYNC: + return SYNC_COMPOSE_PATH + elif node_type == NodeType.MIRAGE: + return MIRAGE_COMPOSE_PATH + else: + return COMPOSE_PATH + + +def get_compose_services(node_type: NodeType) -> list[str]: + if node_type == NodeType.SYNC: + result = list(BASE_SYNC_COMPOSE_SERVICES) + elif node_type == NodeType.MIRAGE: + result = list(BASE_MIRAGE_COMPOSE_SERVICES) + else: + result = list(BASE_SKALE_COMPOSE_SERVICES) + return result -def get_up_compose_sync_cmd(): - return ('docker', 'compose', '-f', SYNC_COMPOSE_PATH, 'up', '-d') +def get_up_compose_cmd(node_type: NodeType, services: Optional[list[str]] = None) -> tuple: + compose_path = get_compose_path(node_type) -def get_compose_path(sync_node: bool) -> str: - return SYNC_COMPOSE_PATH if sync_node else COMPOSE_PATH + if services is None: + services = get_compose_services(node_type) + return ('docker', 'compose', '-f', compose_path, 'up', '-d', *services) -def compose_up(env, sync_node=False): - if sync_node: + +def compose_up(env, node_type: NodeType, is_mirage_boot: bool = False): + if node_type == NodeType.SYNC: logger.info('Running containers for sync node') - run_cmd(cmd=get_up_compose_sync_cmd(), env=env) + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SYNC), env=env) return - logger.info('Running base set of containers') - if 'SGX_CERTIFICATES_DIR_NAME' not in env: env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME - logger.debug('Launching containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(BASE_COMPOSE_SERVICES), env=env) + if node_type == NodeType.MIRAGE: + logger.info('Running mirage base set of containers') + if not is_mirage_boot: + logger.debug('Launching mirage containers with env %s', env) + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.MIRAGE), env=env) + else: + logger.debug('Launching mirage boot containers with env %s', env) + run_cmd( + cmd=get_up_compose_cmd( + node_type=NodeType.MIRAGE, services=list(BASE_MIRAGE_BOOT_COMPOSE_SERVICES) + ), + env=env, + ) + else: + logger.info('Running skale node base set of containers') + logger.debug('Launching skale node containers with env %s', env) + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.REGULAR), env=env) + + if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: + logger.info('Running containers for Telegram notifications') + run_cmd( + cmd=get_up_compose_cmd( + node_type=NodeType.REGULAR, services=list(NOTIFICATION_COMPOSE_SERVICES) + ), + env=env, + ) + if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')): logger.info('Running monitoring containers') - run_cmd(cmd=get_up_compose_cmd(MONITORING_COMPOSE_SERVICES), env=env) - if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: - logger.info('Running containers for Telegram notifications') - run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) + run_cmd( + cmd=get_up_compose_cmd( + node_type=NodeType.REGULAR, services=list(MONITORING_COMPOSE_SERVICES) + ), + env=env, + ) def restart_nginx_container(dutils=None): @@ -322,16 +389,22 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b return False -def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: - return is_container_running(name='skale_admin', dclient=dclient) - +def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) -> bool: + if node_type == NodeType.MIRAGE: + return is_container_running(name='mirage_api', dclient=dclient) + else: + return is_container_running(name='skale_api', dclient=dclient) -def is_api_running(dclient: Optional[DockerClient] = None) -> bool: - return is_container_running(name='skale_api', dclient=dclient) +def is_admin_running(node_type: NodeType, client: Optional[DockerClient] = None) -> bool: + if node_type == NodeType.MIRAGE: + result = is_container_running(name='mirage_admin', dclient=client) + elif node_type == NodeType.SYNC: + result = is_container_running(name='skale_sync_admin', dclient=client) + else: + result = is_container_running(name='skale_admin', dclient=client) -def is_sync_admin_running(dclient: Optional[DockerClient] = None) -> bool: - return is_container_running(name='skale_sync_admin', dclient=dclient) + return result def system_prune(): diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 94e9581b..a8237026 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -1,6 +1,7 @@ import json import os from collections import namedtuple +from typing import Optional from node_cli.configs import META_FILEPATH DEFAULT_VERSION = '1.0.0' @@ -19,7 +20,7 @@ def __new__( cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, + docker_lvmpy_stream: Optional[str] = DEFAULT_DOCKER_LVMPY_STREAM, os_id=DEFAULT_OS_ID, os_version=DEFAULT_OS_VERSION, ): @@ -53,14 +54,18 @@ def compose_default_meta() -> CliMeta: ) -def ensure_meta(meta: CliMeta = None) -> None: +def ensure_meta(meta: Optional[CliMeta] = None) -> None: if not get_meta_info(): meta = meta or compose_default_meta() save_meta(meta) def update_meta( - version: str, config_stream: str, docker_lvmpy_stream: str, os_id: str, os_version: str + version: str, + config_stream: str, + docker_lvmpy_stream: Optional[str], + os_id: str, + os_version: str, ) -> None: ensure_meta() meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) diff --git a/node_cli/utils/node_type.py b/node_cli/utils/node_type.py new file mode 100644 index 00000000..341479f4 --- /dev/null +++ b/node_cli/utils/node_type.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from enum import Enum + + +class NodeType(Enum): + REGULAR = 0 + SYNC = 1 + MIRAGE = 2 diff --git a/scripts/build.sh b/scripts/build.sh index 624fcdf4..d99bb45a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ fi if [ -z "$3" ] then - (>&2 echo 'You should provide type: normal or sync') + (>&2 echo 'You should provide type: normal, sync or mirage') echo $USAGE_MSG exit 1 fi @@ -39,6 +39,8 @@ OS=`uname -s`-`uname -m` if [ "$TYPE" = "sync" ]; then EXECUTABLE_NAME=skale-$VERSION-$OS-sync +elif [ "$TYPE" = "mirage" ]; then + EXECUTABLE_NAME=skale-$VERSION-$OS-mirage else EXECUTABLE_NAME=skale-$VERSION-$OS fi diff --git a/scripts/generate_info.sh b/scripts/generate_info.sh index d554712a..f5933f91 100755 --- a/scripts/generate_info.sh +++ b/scripts/generate_info.sh @@ -3,7 +3,7 @@ set -e VERSION=$1 BRANCH=$2 -TYPE=$3 +TYPE_STR=$3 USAGE_MSG='Usage: generate_info.sh [VERSION] [BRANCH] [TYPE]' @@ -17,8 +17,8 @@ if [ -z "$BRANCH" ]; then echo $USAGE_MSG exit 1 fi -if [ -z "$TYPE" ]; then - (>&2 echo 'You should provide type: normal or sync') +if [ -z "$TYPE_STR" ]; then + (>&2 echo 'You should provide type: normal, sync or mirage') echo $USAGE_MSG exit 1 fi @@ -31,12 +31,31 @@ LATEST_COMMIT=$(git rev-parse HEAD) CURRENT_DATETIME="$(date "+%Y-%m-%d %H:%M:%S")" OS="$(uname -s)-$(uname -m)" +case "$TYPE_STR" in + normal) + TYPE_ENUM="NodeType.REGULAR" + ;; + sync) + TYPE_ENUM="NodeType.SYNC" + ;; + mirage) + TYPE_ENUM="NodeType.MIRAGE" + ;; + *) + (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'normal', 'sync', or 'mirage'") + exit 1 + ;; +esac + rm -f "$DIST_INFO_FILEPATH" touch "$DIST_INFO_FILEPATH" +echo "from node_cli.utils.node_type import NodeType" >> "$DIST_INFO_FILEPATH" +echo "" >> "$DIST_INFO_FILEPATH" + echo "BUILD_DATETIME = '$CURRENT_DATETIME'" >> "$DIST_INFO_FILEPATH" echo "COMMIT = '$LATEST_COMMIT'" >> "$DIST_INFO_FILEPATH" echo "BRANCH = '$BRANCH'" >> "$DIST_INFO_FILEPATH" echo "OS = '$OS'" >> "$DIST_INFO_FILEPATH" echo "VERSION = '$VERSION'" >> "$DIST_INFO_FILEPATH" -echo "TYPE = '$TYPE'" >> "$DIST_INFO_FILEPATH" +echo "TYPE = $TYPE_ENUM" >> "$DIST_INFO_FILEPATH" \ No newline at end of file diff --git a/tests/.skale/config/nginx.conf.j2 b/tests/.skale/config/nginx.conf.j2 deleted file mode 100644 index dc264362..00000000 --- a/tests/.skale/config/nginx.conf.j2 +++ /dev/null @@ -1,47 +0,0 @@ -limit_req_zone $binary_remote_addr zone=one:10m rate=7r/s; - -server { - listen 3009; - - {% if ssl %} - listen 311 ssl; - ssl_certificate /ssl/ssl_cert; - ssl_certificate_key /ssl/ssl_key; - {% endif %} - - proxy_read_timeout 500s; - proxy_connect_timeout 500s; - proxy_send_timeout 500s; - - error_log /var/log/nginx/error.log warn; - client_max_body_size 20m; - - server_name localhost; - limit_req zone=one burst=10; - - location / { - include uwsgi_params; - uwsgi_read_timeout 500s; - uwsgi_socket_keepalive on; - uwsgi_pass 127.0.0.1:3010; - } -} - -server { - listen 80; - - {% if ssl %} - listen 443 ssl; - ssl_certificate /ssl/ssl_cert; - ssl_certificate_key /ssl/ssl_key; - {% endif %} - - error_log /var/log/nginx/error.log warn; - client_max_body_size 20m; - server_name localhost; - limit_req zone=one burst=50; - - location / { - root /filestorage; - } -} \ No newline at end of file diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 5ce570ad..2d0e74c1 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -from node_cli.main import version +from node_cli.main import version, info from tests.helper import run_command @@ -28,3 +28,15 @@ def test_version(): assert result.output == expected result = run_command(version, ['--short']) assert result.output == 'test\n' + + +def test_info_command(): + result = run_command(info, []) + + assert result.exit_code == 0 + + expected_line = 'Full version: test' + assert expected_line in result.output + + assert 'Version:' in result.output + assert 'Build time:' in result.output diff --git a/tests/cli/mirage_cli_test.py b/tests/cli/mirage_cli_test.py new file mode 100644 index 00000000..1a96c22a --- /dev/null +++ b/tests/cli/mirage_cli_test.py @@ -0,0 +1,168 @@ +from click.testing import CliRunner +from unittest import mock +import pathlib + +from node_cli.cli.mirage_node import ( + restore_node, + backup_node, + signature_node, + init_node as init_node_placeholder, + register_node as register_node_placeholder, + update_node as update_node_placeholder, +) +from node_cli.cli.mirage_boot import ( + init_boot, + register_boot, + signature_boot, + migrate_boot, +) + + +@mock.patch('node_cli.cli.mirage_node.restore_mirage') +def test_mirage_node_restore(mock_restore_core, valid_env_file, tmp_path): + runner = CliRunner() + backup_file = tmp_path / 'backup.tar.gz' + backup_file.touch() + backup_path = str(backup_file) + + result = runner.invoke(restore_node, [backup_path, valid_env_file]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_restore_core.assert_called_once_with(backup_path, valid_env_file, False) + + +@mock.patch('node_cli.cli.mirage_node.restore_mirage') +def test_mirage_node_restore_config_only(mock_restore_core, valid_env_file, tmp_path): + runner = CliRunner() + backup_file = tmp_path / 'backup_config.tar.gz' + backup_file.touch() + backup_path = str(backup_file) + + result = runner.invoke(restore_node, [backup_path, valid_env_file, '--config-only']) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_restore_core.assert_called_once_with(backup_path, valid_env_file, True) + + +@mock.patch('node_cli.cli.mirage_node.backup') +def test_mirage_node_backup(mock_backup_core, tmp_path): + runner = CliRunner() + backup_folder = str(tmp_path / 'backups') + pathlib.Path(backup_folder).mkdir(exist_ok=True) + + result = runner.invoke(backup_node, [backup_folder]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_backup_core.assert_called_once_with(backup_folder) + + +@mock.patch('node_cli.cli.mirage_node.get_node_signature') +def test_mirage_node_signature(mock_signature_core): + runner = CliRunner() + validator_id = '42' + signature_val = '0xabc123' + mock_signature_core.return_value = signature_val + + result = runner.invoke(signature_node, [validator_id]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_signature_core.assert_called_once_with(validator_id) + assert f'Signature: {signature_val}' in result.output + + +@mock.patch('node_cli.cli.mirage_node.get_node_signature') +def test_mirage_node_signature_error(mock_signature_core): + runner = CliRunner() + validator_id = '43' + error_msg = 'Core layer error' + mock_signature_core.return_value = {'error': True, 'message': error_msg} + + result = runner.invoke(signature_node, [validator_id]) + + assert result.exit_code != 0, f'Output: {result.output}\nException: {result.exception}' + mock_signature_core.assert_called_once_with(validator_id) + assert error_msg in result.output + + +def test_mirage_node_init_placeholder(): + runner = CliRunner() + result = runner.invoke(init_node_placeholder, []) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + assert "Placeholder: Command 'mirage node init' is not yet implemented." in result.output + + +def test_mirage_node_register_placeholder(): + runner = CliRunner() + result = runner.invoke(register_node_placeholder, []) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + assert "Placeholder: Command 'mirage node register' is not yet implemented." in result.output + + +def test_mirage_node_update_placeholder(valid_env_file): + runner = CliRunner() + result = runner.invoke(update_node_placeholder, ['--yes', valid_env_file]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + assert "Placeholder: Command 'mirage node update' is not yet implemented." in result.output + + +@mock.patch('node_cli.cli.mirage_boot.register') +def test_mirage_boot_register(mock_register_core): + runner = CliRunner() + name = 'test-boot-node' + ip = '1.2.3.4' + port = 10001 + domain = 'boot.skale.test' + + result = runner.invoke( + register_boot, ['--name', name, '--ip', ip, '--port', str(port), '--domain', domain] + ) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_register_core.assert_called_once_with( + name=name, p2p_ip=ip, public_ip=ip, port=port, domain_name=domain + ) + + +@mock.patch('node_cli.cli.mirage_boot.get_node_signature') +def test_mirage_boot_signature(mock_signature_core): + runner = CliRunner() + validator_id = '101' + signature_val = '0xdef456' + mock_signature_core.return_value = signature_val + + result = runner.invoke(signature_boot, [validator_id]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_signature_core.assert_called_once_with(validator_id) + assert f'Signature: {signature_val}' in result.output + + +@mock.patch('node_cli.cli.mirage_boot.init') +def test_mirage_boot_init(mock_init_core, valid_env_file): + runner = CliRunner() + result = runner.invoke(init_boot, [valid_env_file]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_init_core.assert_called_once_with(valid_env_file) + + +@mock.patch('node_cli.cli.mirage_boot.migrate') +def test_mirage_boot_migrate(mock_migrate_core, valid_env_file): + runner = CliRunner() + result = runner.invoke(migrate_boot, ['--yes', valid_env_file]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_migrate_core.assert_called_once_with(valid_env_file, None) + + +@mock.patch('node_cli.cli.mirage_boot.migrate') +def test_mirage_boot_migrate_pull_config(mock_migrate_core, valid_env_file): + runner = CliRunner() + schain_name = 'my-schain-config' + result = runner.invoke(migrate_boot, ['--yes', '--pull-config', schain_name, valid_env_file]) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_migrate_core.assert_called_once_with(valid_env_file, schain_name) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index a94e4157..d1dc82b7 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -54,7 +54,7 @@ init_default_logger() -def test_register_node(resource_alloc, mocked_g_config): +def test_register_node(inited_node, resource_alloc, mocked_g_config): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( @@ -70,7 +70,7 @@ def test_register_node(resource_alloc, mocked_g_config): ) # noqa -def test_register_node_with_error(resource_alloc, mocked_g_config): +def test_register_node_with_error(inited_node, resource_alloc, mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'error', 'payload': ['Strange error']}, @@ -89,7 +89,7 @@ def test_register_node_with_error(resource_alloc, mocked_g_config): ) -def test_register_node_with_prompted_ip(resource_alloc, mocked_g_config): +def test_register_node_with_prompted_ip(inited_node, resource_alloc, mocked_g_config): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( @@ -106,7 +106,7 @@ def test_register_node_with_prompted_ip(resource_alloc, mocked_g_config): ) -def test_register_node_with_default_port(resource_alloc, mocked_g_config): +def test_register_node_with_default_port(inited_node, resource_alloc, mocked_g_config): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( @@ -338,7 +338,7 @@ def test_restore(mocked_g_config): return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.env.validate_env_params', lambda params: None), + patch('node_cli.configs.env.validate_env_params'), ): result = run_command(restore_node, [backup_path, './tests/test-env']) assert result.exit_code == 0 @@ -364,7 +364,7 @@ def test_restore_no_snapshot(mocked_g_config): return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.env.validate_env_params', lambda params: None), + patch('node_cli.configs.env.validate_env_params'), ): result = run_command(restore_node, [backup_path, './tests/test-env', '--no-snapshot']) assert result.exit_code == 0 @@ -403,7 +403,7 @@ def test_turn_off_maintenance_on(mocked_g_config): mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.node.turn_off_op'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - patch('node_cli.configs.env.validate_env_params', lambda params: None), + patch('node_cli.configs.env.validate_env_params'), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -435,7 +435,7 @@ def test_turn_on_maintenance_off(mocked_g_config): mock.patch('node_cli.core.node.turn_on_op'), mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - patch('node_cli.configs.env.validate_env_params', lambda params: None), + patch('node_cli.configs.env.validate_env_params'), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index a169d7ac..b317aad6 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -56,7 +56,7 @@ def test_generate(): resp_mock = response_mock(requests.codes.created) with ( mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, generate, ['./tests/test-env', '--yes'] @@ -71,7 +71,7 @@ def test_generate_already_exists(resource_alloc_config): resp_mock = response_mock(requests.codes.created) with ( mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, generate, ['./tests/test-env', '--yes'] diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index db24799f..9206245a 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -45,7 +45,7 @@ def test_init_sync(mocked_g_config, clean_node_options): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): result = run_command(_init_sync, ['./tests/test-env']) @@ -78,7 +78,7 @@ def test_init_sync_archive(mocked_g_config, clean_node_options): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): result = run_command(_init_sync, ['./tests/test-env', '--archive']) node_options = NodeOptions() @@ -121,7 +121,7 @@ def test_update_sync(mocked_g_config): 'node_cli.core.node.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): result = run_command(_update_sync, ['./tests/test-env', '--yes']) assert result.exit_code == 0 diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index ed778d40..03c955e9 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -2,6 +2,7 @@ from typing import Optional import pytest import requests +import mock from node_cli.configs.env import ( absent_required_params, @@ -12,6 +13,11 @@ validate_env_params, validate_env_type, ALLOWED_ENV_TYPES, + REQUIRED_PARAMS_SKALE, + REQUIRED_PARAMS_SYNC, + REQUIRED_PARAMS_MIRAGE_BOOT, + REQUIRED_PARAMS_MIRAGE, + OPTIONAL_PARAMS, ) from node_cli.configs.alias_address_validation import ( validate_env_alias_or_address, @@ -22,6 +28,7 @@ ContractType, ) from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.node_type import NodeType ENDPOINT = 'http://localhost:8545' @@ -54,22 +61,6 @@ def test_load_env_file_nonexistent(): assert excinfo.value.code == CLIExitCodes.FAILURE.value -def test_load_env_file_not_readable(tmp_path): - # Create a temporary file and remove read permissions - env_file = tmp_path / 'test.env' - env_file.write_text('KEY=value') - os.chmod(env_file, 0o000) - with pytest.raises(PermissionError): - load_env_file(str(env_file)) - os.chmod(env_file, 0o644) # reset permissions - - -@pytest.mark.parametrize('sync_node,has_schain_name', [(True, True), (False, False)]) -def test_build_env_params_sync_and_non_sync(sync_node, has_schain_name): - params = build_env_params(sync_node=sync_node) - assert ('SCHAIN_NAME' in params) == has_schain_name - - def test_populate_env_params_updates_from_environ(monkeypatch): params = {'FOO': ''} monkeypatch.setenv('FOO', 'bar') @@ -77,15 +68,66 @@ def test_populate_env_params_updates_from_environ(monkeypatch): assert params['FOO'] == 'bar' -@pytest.mark.parametrize('env_type', ['mainnet', 'testnet', 'qanet', 'devnet']) -def test_valid_env_types(env_type): - validate_env_type(env_type) - - -def test_invalid_env_type(): - with pytest.raises(SystemExit) as excinfo: - validate_env_type('invalid') - assert excinfo.value.code == CLIExitCodes.FAILURE.value +@pytest.mark.parametrize( + 'node_type, is_mirage_boot, expected_keys, unexpected_keys', + [ + ( + NodeType.REGULAR, + False, + REQUIRED_PARAMS_SKALE.keys(), + {'SCHAIN_NAME'}, + ), + ( + NodeType.SYNC, + False, + REQUIRED_PARAMS_SYNC.keys(), + set(), + ), + ( + NodeType.MIRAGE, + True, + REQUIRED_PARAMS_MIRAGE_BOOT.keys(), + {'DOCKER_LVMPY_STREAM', 'SCHAIN_NAME'}, + ), + ( + NodeType.MIRAGE, + False, + REQUIRED_PARAMS_MIRAGE.keys(), + {'IMA_CONTRACTS', 'DOCKER_LVMPY_STREAM', 'SCHAIN_NAME'}, + ), + ], + ids=['regular', 'sync', 'mirage_boot', 'mirage_regular'], +) +def test_build_env_params_keys(node_type, is_mirage_boot, expected_keys, unexpected_keys): + params = build_env_params(node_type=node_type, is_mirage_boot=is_mirage_boot) + param_keys = set(params.keys()) + + all_expected = set(expected_keys) | set(OPTIONAL_PARAMS.keys()) + missing_expected = all_expected - param_keys + assert not missing_expected, f'Missing expected keys: {missing_expected}' + + found_unexpected = set(unexpected_keys) & param_keys + assert not found_unexpected, f'Found unexpected keys: {found_unexpected}' + + +@pytest.mark.parametrize( + 'env_types, should_fail', + [ + (ALLOWED_ENV_TYPES, False), + (['invalid'], True), + ], + ids=[ + 'correct_env', + 'invalid_env', + ], +) +def test_env_types(env_types, should_fail): + for env_type in env_types: + if should_fail: + with pytest.raises(SystemExit): + validate_env_type(env_type=env_type) + else: + validate_env_type(env_type=env_type) def test_get_chain_id_success(monkeypatch): @@ -95,8 +137,7 @@ def fake_post(url, json): return fake_response monkeypatch.setattr(requests, 'post', fake_post) - chain_id = get_chain_id('http://localhost:8545') - assert chain_id == 1 + assert get_chain_id(ENDPOINT) == 1 def test_get_chain_id_failure(monkeypatch): @@ -106,71 +147,64 @@ def fake_post(url, json): return fake_response monkeypatch.setattr(requests, 'post', fake_post) - with pytest.raises(SystemExit) as excinfo: - get_chain_id('http://localhost:8545') - assert excinfo.value.code == CLIExitCodes.FAILURE.value + with pytest.raises(SystemExit): + get_chain_id(ENDPOINT) -def test_get_network_metadata_success(requests_mock): - metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - requests_mock.get(metadata_url, json=metadata, status_code=200) - result = get_network_metadata() - assert result == metadata - - -def test_get_network_metadata_failure(requests_mock): - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - requests_mock.get(metadata_url, status_code=404) - with pytest.raises(SystemExit) as excinfo: - get_network_metadata() - assert excinfo.value.code == CLIExitCodes.FAILURE.value - - -def test_validate_contract_address_success(requests_mock): - requests_mock.post(ENDPOINT, json={'result': '0x123'}) - validate_contract_address('0x' + 'a' * 40, ENDPOINT) - - -def test_validate_contract_address_no_code(requests_mock): - requests_mock.post(ENDPOINT, json={'result': '0x'}) - with pytest.raises(SystemExit) as excinfo: - validate_contract_address('0x' + 'a' * 40, ENDPOINT) - assert excinfo.value.code == CLIExitCodes.FAILURE.value - - -def test_validate_contract_alias_success(requests_mock): +@pytest.mark.parametrize( + 'metadata,status_code,should_raise', + [ + ({'networks': [{'chainId': 1, 'path': 'mainnet'}]}, 200, False), + (None, 404, True), + ], +) +def test_get_network_metadata(requests_mock, metadata, status_code, should_raise): + metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' + requests_mock.get(metadata_url, json=metadata, status_code=status_code) + + if should_raise: + with pytest.raises(SystemExit): + get_network_metadata() + else: + assert get_network_metadata() == metadata + + +@pytest.mark.parametrize( + 'code,should_raise', + [ + ('0x123', False), + ('0x', True), + ], +) +def test_validate_contract_address(requests_mock, code, should_raise): + requests_mock.post(ENDPOINT, json={'result': code}) + addr = '0x' + 'a' * 40 + if should_raise: + with pytest.raises(SystemExit): + validate_contract_address(addr, ENDPOINT) + else: + validate_contract_address(addr, ENDPOINT) + + +@pytest.mark.parametrize( + 'networks,should_raise', + [ + ([{'chainId': 1, 'path': 'mainnet'}], False), + ([], True), + ], +) +def test_validate_contract_alias(requests_mock, networks, should_raise): requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} - requests_mock.get(metadata_url, json=metadata, status_code=200) - alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/skale-manager/test-alias.json' - ) - requests_mock.get(alias_url, status_code=200) - validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - + metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' + requests_mock.get(metadata_url, json={'networks': networks}, status_code=200) -def test_validate_contract_alias_network_missing(requests_mock): - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - requests_mock.get(metadata_url, json={'networks': []}, status_code=200) - with pytest.raises(SystemExit) as excinfo: + if not should_raise: + alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/skale-manager/test-alias.json' + requests_mock.get(alias_url, status_code=200) validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - assert excinfo.value.code == CLIExitCodes.FAILURE.value + else: + with pytest.raises(SystemExit): + validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) def test_validate_env_alias_or_address_with_address(requests_mock): @@ -181,90 +215,129 @@ def test_validate_env_alias_or_address_with_address(requests_mock): def test_validate_env_alias_or_address_with_alias(requests_mock): requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) + metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} requests_mock.get(metadata_url, json=metadata, status_code=200) - alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' - ) + alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' requests_mock.get(alias_url, status_code=200) validate_env_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) -def test_validate_env_params_missing_key(): - populated_params = { - 'CONTAINER_CONFIGS_STREAM': 'value', - 'ENDPOINT': 'http://localhost:8545', - 'MANAGER_CONTRACTS': '', - 'FILEBEAT_HOST': '127.0.0.1:3010', - 'DISK_MOUNTPOINT': '/dev/sss', - 'SGX_SERVER_URL': 'http://127.0.0.1', - 'DOCKER_LVMPY_STREAM': 'value', - 'ENV_TYPE': 'mainnet', - } - with pytest.raises(SystemExit) as excinfo: - validate_env_params(populated_params) - assert excinfo.value.code == CLIExitCodes.FAILURE.value +@pytest.mark.parametrize('env_type', ALLOWED_ENV_TYPES) +@pytest.mark.parametrize( + 'required_params, key_to_remove, should_fail', + [ + (REQUIRED_PARAMS_MIRAGE_BOOT, None, False), + (REQUIRED_PARAMS_MIRAGE, None, False), + (REQUIRED_PARAMS_MIRAGE_BOOT, 'IMA_CONTRACTS', True), + (REQUIRED_PARAMS_MIRAGE_BOOT, 'FILEBEAT_HOST', True), + (REQUIRED_PARAMS_MIRAGE, 'FILEBEAT_HOST', True), + ], + ids=[ + 'mirage_boot', + 'mirage_regular', + 'mirage_boot_missing_ima', + 'mirage_boot_missing_filebeat', + 'mirage_regular_missing_filebeat', + ], +) +@mock.patch('node_cli.configs.env.validate_env_alias_or_address') +@mock.patch('node_cli.configs.env.validate_env_type') +def test_validate_env_params_mirage( + mock_validate_type, + mock_validate_alias, + required_params, + key_to_remove, + should_fail, + env_type, +): + params = {k: f'{k}_val' for k in required_params} + params['ENV_TYPE'] = env_type + + if key_to_remove: + params[key_to_remove] = '' + + if should_fail: + with pytest.raises(SystemExit): + validate_env_params(params=params) + else: + validate_env_params(params=params) + + +@pytest.mark.parametrize( + 'node_type, is_boot, required_keys_dict', + [ + (NodeType.MIRAGE, True, REQUIRED_PARAMS_MIRAGE_BOOT), + (NodeType.MIRAGE, False, REQUIRED_PARAMS_MIRAGE), + ], + ids=['mirage_boot', 'mirage_regular'], +) +@mock.patch('node_cli.configs.alias_address_validation.validate_env_alias_or_address') +@mock.patch('node_cli.configs.alias_address_validation.get_chain_id', return_value=1) +@mock.patch( + 'node_cli.configs.alias_address_validation.get_network_metadata', + return_value={'networks': [{'chainId': 1, 'path': 'mainnet'}]}, +) +def test_get_validated_env_config_mirage_success( + mock_meta, + mock_chain, + mock_validate_alias, + tmp_path, + monkeypatch, + node_type, + is_boot, + required_keys_dict, +): + env_file = tmp_path / 'mirage.env' + env_content = '' + expected_config = {} + for key in {**required_keys_dict, **OPTIONAL_PARAMS}: + env_value = f'{key}_value' + if key == 'ENDPOINT': + env_value = ENDPOINT + if key == 'ENV_TYPE': + env_value = 'devnet' + if key == 'MANAGER_CONTRACTS': + env_value = '0x' + '1' * 40 + if key == 'IMA_CONTRACTS': + env_value = '0x' + '2' * 40 -def test_validate_env_params_success(valid_env_params, requests_mock): - endpoint = valid_env_params['ENDPOINT'] - requests_mock.post(endpoint, json={'result': '0x1'}) - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} - requests_mock.get(metadata_url, json=metadata, status_code=200) - ima_alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/mainnet-ima/test-ima.json' - ) - requests_mock.get(ima_alias_url, status_code=200) - manager_alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/skale-manager/test-manager.json' - ) - requests_mock.get(manager_alias_url, status_code=200) - validate_env_params(valid_env_params) - - -def test_get_validated_env_config_success( - valid_env_file, mock_chain_response, mock_networks_metadata, requests_mock -): - requests_mock.post(ENDPOINT, json=mock_chain_response) - metadata_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' - ) - requests_mock.get(metadata_url, json=mock_networks_metadata, status_code=200) - ima_alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/mainnet-ima/test-ima.json' - ) - requests_mock.get(ima_alias_url, status_code=200) - manager_alias_url = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/mainnet/skale-manager/test-manager.json' - ) - requests_mock.get(manager_alias_url, status_code=200) - config = get_validated_env_config(valid_env_file) - assert config['ENDPOINT'] == 'http://localhost:8545' - assert config['ENV_TYPE'] in ALLOWED_ENV_TYPES + if key in required_keys_dict: + env_content += f'{key}={env_value}\n' + monkeypatch.setenv(key, env_value) + expected_config[key] = env_value + env_file.write_text(env_content) -def test_get_validated_env_config_missing_file(): - with pytest.raises(SystemExit) as excinfo: - get_validated_env_config('nonexistent.env') - assert excinfo.value.code == CLIExitCodes.FAILURE.value + with mock.patch('node_cli.configs.alias_address_validation.requests.post') as mock_post: + mock_post.return_value = FakeResponse(200, {'result': '0x123'}) + + config = get_validated_env_config( + node_type=node_type, env_filepath=str(env_file), is_mirage_boot=is_boot + ) + assert config is not None + assert set(config.keys()) == set(expected_config.keys()) + for key in expected_config: + assert config[key] == expected_config[key] -def test_get_validated_env_config_unreadable_file(valid_env_file): - os.chmod(valid_env_file, 0o000) - with pytest.raises(PermissionError): - get_validated_env_config(valid_env_file) - os.chmod(valid_env_file, 0o644) + for key in {**required_keys_dict, **OPTIONAL_PARAMS}: + monkeypatch.delenv(key, raising=False) + + +def test_get_validated_env_config_missing_file(): + with pytest.raises(SystemExit): + get_validated_env_config(env_filepath='nonexistent.env', node_type=NodeType.REGULAR) + + +def test_get_validated_env_config_unreadable_file(tmp_path): + env_file = tmp_path / 'unreadable.env' + env_file.touch() + original_mode = env_file.stat().st_mode + try: + os.chmod(env_file, 0o000) + with pytest.raises(PermissionError): + get_validated_env_config(env_filepath=str(env_file), node_type=NodeType.REGULAR) + finally: + os.chmod(env_file, original_mode) diff --git a/tests/conftest.py b/tests/conftest.py index a7c2c0cd..369b2374 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,7 +27,6 @@ import docker import mock import pytest -import yaml from node_cli.configs import ( CONTAINER_CONFIG_TMP_PATH, @@ -36,8 +35,8 @@ META_FILEPATH, NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, - STATIC_PARAMS_FILEPATH, SCHAIN_NODE_DATA_PATH, + NGINX_CONFIG_FILEPATH, ) from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH @@ -45,77 +44,7 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SYNC - - -TEST_ENV_PARAMS = """ -mainnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 2000000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -testnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 200000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -qanet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 200000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -devnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 80000000000 - - packages: - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - docker-compose: 1.1.3 - - docker: - docker-api: 1.1.3 - docker-engine: 1.1.3 -""" - - -@pytest.fixture -def net_params_file(): - with open(STATIC_PARAMS_FILEPATH, 'w') as f: - yaml.dump(yaml.load(TEST_ENV_PARAMS, Loader=yaml.Loader), stream=f, Dumper=yaml.Dumper) - yield STATIC_PARAMS_FILEPATH - os.remove(STATIC_PARAMS_FILEPATH) +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN @pytest.fixture() @@ -191,6 +120,17 @@ def resource_alloc(): os.remove(RESOURCE_ALLOCATION_FILEPATH) +@pytest.fixture +def inited_node(): + path = pathlib.Path(NGINX_CONFIG_FILEPATH) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + try: + yield + finally: + os.remove(NGINX_CONFIG_FILEPATH) + + @pytest.fixture def ssl_folder(): if os.path.isdir(SSL_FOLDER_PATH): @@ -286,11 +226,11 @@ def tmp_schains_dir(): @pytest.fixture def tmp_sync_datadir(): - os.makedirs(TEST_SCHAINS_MNT_DIR_SYNC, exist_ok=True) + os.makedirs(TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN, exist_ok=True) try: - yield TEST_SCHAINS_MNT_DIR_SYNC + yield TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN finally: - shutil.rmtree(TEST_SCHAINS_MNT_DIR_SYNC) + shutil.rmtree(TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN) @pytest.fixture @@ -316,7 +256,6 @@ def valid_env_params(): @pytest.fixture def valid_env_file(valid_env_params): - """Create a temporary .env file whose contents mimic test-env.""" file_name = None try: with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index b490b1ee..cd3a6acb 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -1,7 +1,6 @@ import os import shutil -import time -from pip._internal import main as pipmain +import subprocess import mock import pytest @@ -22,6 +21,8 @@ save_report, ) +from node_cli.utils.node_type import NodeType + @pytest.fixture def requirements_data(): @@ -38,6 +39,13 @@ def requirements_data(): } +@pytest.fixture +def mirage_requirements_data(requirements_data): + reqs = {k: v.copy() for k, v in requirements_data.items()} + reqs['package']['lvm2'] = 'disabled' + return reqs + + @pytest.fixture def server_req(requirements_data): return requirements_data['server'] @@ -195,41 +203,27 @@ def test_checks_docker_api(docker_req): assert r.info['expected_version'] == '111.111.111' -@pytest.fixture -def docker_compose_pkg_1_27_4(): - pipmain(['install', 'docker-compose==1.27.4']) - time.sleep(10) - yield - pipmain(['uninstall', 'docker-compose', '-y']) - - -@pytest.fixture -def docker_compose_pkg_1_24_1(): - pipmain(['install', 'docker-compose==1.24.1']) - time.sleep(10) - yield - pipmain(['uninstall', 'docker-compose', '-y']) - +@mock.patch('node_cli.utils.helper.subprocess.run') +@mock.patch('node_cli.core.checks.shutil.which', return_value='/usr/bin/docker') +def test_checks_docker_compose_version_mocked(mock_shutil_which, mock_subprocess_run, docker_req): + checker = DockerChecker(docker_req) + expected_version = docker_req['docker-compose'] -def test_checks_docker_compose_good_pkg(docker_req, docker_compose_pkg_1_27_4): - checker = DockerChecker(package_req) - r = checker.docker_compose() - r.name == 'docker-compose' - r.status == 'ok' + mock_output = f'Docker Compose version v{expected_version}, build somehash'.encode('utf-8') + mock_result = mock.Mock(spec=subprocess.CompletedProcess) + mock_result.stdout = mock_output + mock_result.stderr = None + mock_result.returncode = 0 + mock_subprocess_run.return_value = mock_result -def test_checks_docker_compose_no_pkg(docker_req): - checker = DockerChecker(package_req) r = checker.docker_compose() - r.name == 'docker-compose' - r.status == 'ok' - -def test_checks_docker_compose_invalid_version(docker_req, docker_compose_pkg_1_24_1): - checker = DockerChecker(docker_req) - r = checker.docker_compose() - r.name == 'docker-compose' - r.status == 'ok' + assert r.name == 'docker' + assert r.status == 'ok', f'Check failed: {r}' + assert isinstance(r.info, str) + assert f'expected docker compose version {expected_version}' in r.info.lower() + assert f'actual v{expected_version}' in r.info.lower() def test_checks_docker_config(docker_req): @@ -344,6 +338,18 @@ def test_get_checks(requirements_data): assert len(checks) == 2 +def test_get_checks_mirage(mirage_requirements_data): + disk = 'test-disk' + mirage_checkers = get_all_checkers(disk, mirage_requirements_data) + + mirage_all_checks = get_checks(mirage_checkers, CheckType.ALL) + mirage_all_names = {f.func.__name__ for f in mirage_all_checks} + assert 'network' in mirage_all_names + assert 'lvm2' not in mirage_all_names + assert 'cpu_total' in mirage_all_names + assert 'btrfs_progs' in mirage_all_names + + def test_get_save_report(tmp_dir_path): path = os.path.join(tmp_dir_path, 'checks.json') report = get_report(path) @@ -373,8 +379,8 @@ def test_merge_report(): def test_get_static_params(tmp_config_dir): - params = get_static_params() + params = get_static_params(NodeType.REGULAR) shutil.copy(STATIC_PARAMS_FILEPATH, tmp_config_dir) - tmp_params = get_static_params(config_path=tmp_config_dir) + tmp_params = get_static_params(NodeType.REGULAR, config_path=tmp_config_dir) assert params['server']['cpu_total'] == 8 assert params == tmp_params diff --git a/tests/core/core_mirage_test.py b/tests/core/core_mirage_test.py new file mode 100644 index 00000000..62d58681 --- /dev/null +++ b/tests/core/core_mirage_test.py @@ -0,0 +1,130 @@ +from unittest import mock + +from node_cli.configs import SKALE_DIR +from node_cli.core.mirage_boot import init as init_boot, migrate, update +from node_cli.core.mirage_node import restore_mirage +from node_cli.utils.node_type import NodeType + + +@mock.patch('node_cli.core.mirage_node.time.sleep') +@mock.patch('node_cli.core.mirage_node.restore_mirage_op') +@mock.patch('node_cli.core.mirage_node.save_env_params') +@mock.patch('node_cli.core.mirage_node.compose_node_env') +def test_restore_mirage( + mock_compose_env, + mock_save_env, + mock_restore_op, + mock_sleep, + valid_env_file, + ensure_meta_removed, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + mock_restore_op.return_value = True + backup_path = '/fake/backup' + + restore_mirage(backup_path, valid_env_file) + + mock_compose_env.assert_called_once_with(valid_env_file, node_type=NodeType.MIRAGE) + mock_save_env.assert_called_once_with(valid_env_file) + expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} + mock_restore_op.assert_called_once_with(expected_env, backup_path, config_only=False) + mock_sleep.assert_called_once() + + +@mock.patch('node_cli.core.mirage_boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.core.mirage_boot.time.sleep') +@mock.patch('node_cli.core.mirage_boot.init_mirage_boot_op') +@mock.patch('node_cli.core.mirage_boot.compose_node_env') +def test_init_mirage_boot( + mock_compose_env, + mock_init_op, + mock_sleep, + mock_is_alive, + valid_env_file, + ensure_meta_removed, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + init_boot(valid_env_file) + + mock_compose_env.assert_called_once_with( + valid_env_file, + node_type=NodeType.MIRAGE, + is_mirage_boot=True, + ) + mock_init_op.assert_called_once_with(valid_env_file, mock_env) + mock_sleep.assert_called_once() + mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE, is_mirage_boot=True) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.core.mirage_boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.core.mirage_boot.time.sleep') +@mock.patch('node_cli.core.mirage_boot.migrate_mirage_boot_op') +@mock.patch('node_cli.core.mirage_boot.compose_node_env') +def test_migrate_mirage_boot( + mock_compose_env, + mock_migrate_op, + mock_sleep, + mock_is_alive, + mock_is_user_valid, + valid_env_file, + inited_node, + resource_alloc, + meta_file_v3, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + mock_migrate_op.return_value = True + pull_config_for_schain = 'mirage' + + migrate(valid_env_file, pull_config_for_schain) + + mock_compose_env.assert_called_once_with( + valid_env_file, + inited_node=True, + sync_schains=False, + pull_config_for_schain=pull_config_for_schain, + node_type=NodeType.MIRAGE, + ) + mock_migrate_op.assert_called_once_with(valid_env_file, mock_env) + mock_sleep.assert_called_once() + mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.core.mirage_boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.core.mirage_boot.time.sleep') +@mock.patch('node_cli.core.mirage_boot.update_mirage_boot_op') +@mock.patch('node_cli.core.mirage_boot.compose_node_env') +def test_update_mirage_boot( + mock_compose_env, + mock_update_op, + mock_sleep, + mock_is_alive, + mock_is_user_valid, + valid_env_file, + inited_node, + resource_alloc, + meta_file_v3, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + mock_update_op.return_value = True + pull_config_for_schain = 'mirage' + + update(valid_env_file, pull_config_for_schain) + + mock_compose_env.assert_called_once_with( + valid_env_file, + inited_node=True, + sync_schains=False, + pull_config_for_schain=pull_config_for_schain, + node_type=NodeType.MIRAGE, + is_mirage_boot=True, + ) + mock_update_op.assert_called_once_with(valid_env_file, mock_env) + mock_sleep.assert_called_once() + mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE, is_mirage_boot=True) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index f45a9be3..c71e4683 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -9,11 +9,19 @@ import pytest import requests -from node_cli.configs import NODE_DATA_PATH +from node_cli.configs import NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH -from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive -from node_cli.core.node import init, pack_dir, update, is_update_safe +from node_cli.core.node import ( + get_expected_container_names, + is_base_containers_alive, + init, + pack_dir, + update, + is_update_safe, + compose_node_env, +) from node_cli.utils.meta import CliMeta +from node_cli.utils.node_type import NodeType from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -21,42 +29,189 @@ dclient = docker.from_env() ALPINE_IMAGE_NAME = 'alpine:3.12' -HELLO_WORLD_IMAGE_NAME = 'hello-world' -CMD = 'sleep 10' +CMD = 'sleep 60' + +WRONG_CONTAINERS = [ + 'WRONG_CONTAINER_1', + 'skale_WRONG_CONTAINER_4', + 'mirage_WRONG_CONTAINER_6', + 'sync_WRONG_CONTAINER_8', +] + +NODE_TYPE_BOOT_COMBINATIONS: list[tuple[NodeType, bool]] = [ + (NodeType.REGULAR, False), + (NodeType.SYNC, False), + (NodeType.MIRAGE, True), + (NodeType.MIRAGE, False), +] + +alive_test_params = [ + pytest.param( + node_type, + is_boot, + get_expected_container_names(node_type, is_boot), + id=f'{node_type.name}-boot_{is_boot}-correct_containers', + ) + for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS +] + +wrong_test_params = [ + pytest.param( + node_type, + is_boot, + WRONG_CONTAINERS, + id=f'{node_type.name}-boot_{is_boot}-wrong_containers', + ) + for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS +] + +missing_test_params = [] +for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS: + expected_names = get_expected_container_names(node_type, is_boot) + containers_to_create = expected_names[1:] + missing_test_params.append( + pytest.param( + node_type, + is_boot, + containers_to_create, + id=f'{node_type.name}-boot_{is_boot}-missing_containers', + ) + ) @pytest.fixture -def skale_base_containers(): - containers = [ - dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) - for i in range(BASE_CONTAINERS_AMOUNT) - ] - yield containers - for c in containers: - c.remove(force=True) - - -@pytest.fixture -def skale_base_containers_without_one(): - containers = [ - dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) - for i in range(BASE_CONTAINERS_AMOUNT - 1) - ] - yield containers - for c in containers: - c.remove(force=True) - +def manage_node_containers(request): + container_names_to_create = request.param + created_containers = [] + try: + for name in container_names_to_create: + try: + existing_container = dclient.containers.get(name) + existing_container.remove(force=True) + except docker.errors.NotFound: + pass + container = dclient.containers.run( + ALPINE_IMAGE_NAME, + detach=True, + name=name, + command=CMD, + ) + created_containers.append(container) + + if created_containers: + time.sleep(2) + + yield created_containers -@pytest.fixture -def skale_base_containers_exited(): - containers = [ - dclient.containers.run(HELLO_WORLD_IMAGE_NAME, detach=True, name=f'skale_test{i}') - for i in range(BASE_CONTAINERS_AMOUNT) - ] - time.sleep(10) - yield containers - for c in containers: - c.remove(force=True) + finally: + all_containers_now = dclient.containers.list(all=True) + cleaned_count = 0 + for container_obj in all_containers_now: + if container_obj.name in container_names_to_create: + try: + container_obj.remove(force=True) + cleaned_count += 1 + except docker.errors.NotFound: + pass + + +@pytest.mark.parametrize( + 'node_type, is_boot, manage_node_containers', + alive_test_params, + indirect=['manage_node_containers'], +) +def test_is_base_containers_alive(manage_node_containers, node_type, is_boot): + assert is_base_containers_alive(node_type=node_type, is_mirage_boot=is_boot) is True + + +@pytest.mark.parametrize( + 'node_type, is_boot, manage_node_containers', + wrong_test_params, + indirect=['manage_node_containers'], +) +def test_is_base_containers_alive_wrong(manage_node_containers, node_type, is_boot): + assert is_base_containers_alive(node_type=node_type, is_mirage_boot=is_boot) is False + + +@pytest.mark.parametrize( + 'node_type, is_boot, manage_node_containers', + missing_test_params, + indirect=['manage_node_containers'], +) +def test_is_base_containers_alive_missing(manage_node_containers, node_type, is_boot): + assert is_base_containers_alive(node_type=node_type, is_mirage_boot=is_boot) is False + + +@pytest.mark.parametrize('node_type, is_boot', NODE_TYPE_BOOT_COMBINATIONS) +def test_is_base_containers_alive_empty(node_type, is_boot): + assert is_base_containers_alive(node_type=node_type, is_mirage_boot=is_boot) is False + + +@pytest.mark.parametrize( + ( + 'node_type, is_boot, inited_node, sync_schains, expected_mnt_dir, ' + 'expect_flask_key, expect_backup_run' + ), + [ + (NodeType.REGULAR, False, True, False, SCHAINS_MNT_DIR_REGULAR, True, False), + (NodeType.REGULAR, False, True, True, SCHAINS_MNT_DIR_REGULAR, True, True), + (NodeType.SYNC, False, False, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, False, False), + (NodeType.MIRAGE, True, True, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, True, False), + (NodeType.MIRAGE, False, True, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, True, False), + ], + ids=[ + 'regular', + 'regular_sync_flag', + 'sync', + 'mirage_boot', + 'mirage_regular', + ], +) +@mock.patch('node_cli.core.node.get_validated_env_config') +@mock.patch('node_cli.core.node.save_env_params') +@mock.patch('node_cli.core.node.get_flask_secret_key', return_value='mock_secret') +def test_compose_node_env( + mock_get_secret, + mock_save_params, + mock_get_validated, + node_type, + is_boot, + inited_node, + sync_schains, + expected_mnt_dir, + expect_flask_key, + expect_backup_run, + valid_env_file, + valid_env_params, +): + mock_get_validated.return_value = valid_env_params.copy() + if node_type == NodeType.SYNC: + mock_get_validated.return_value['ENV_TYPE'] = 'devnet' + else: + mock_get_validated.return_value['ENV_TYPE'] = 'mainnet' + + result_env = compose_node_env( + env_filepath=valid_env_file, + inited_node=inited_node, + sync_schains=sync_schains, + node_type=node_type, + is_mirage_boot=is_boot, + save=True, + ) + + mock_save_params.assert_called_once_with(valid_env_file) + mock_get_validated.assert_called_once_with( + env_filepath=valid_env_file, node_type=node_type, is_mirage_boot=is_boot + ) + assert result_env['SCHAINS_MNT_DIR'] == expected_mnt_dir + assert ( + 'FLASK_SECRET_KEY' in result_env and result_env['FLASK_SECRET_KEY'] is not None + ) == expect_flask_key + if expect_flask_key: + assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' + should_have_backup = sync_schains and node_type != NodeType.SYNC + assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup + assert result_env['ENDPOINT'] == valid_env_params['ENDPOINT'] @pytest.fixture @@ -101,24 +256,6 @@ def test_pack_dir(tmp_dir): pack_dir(backup_dir, cleaned_archive_path, exclude=('trash_data',)) -def test_is_base_containers_alive(skale_base_containers): - cont = skale_base_containers - print([c.name for c in cont]) - assert is_base_containers_alive() - - -def test_is_base_containers_alive_one_failed(skale_base_containers_without_one): - assert not is_base_containers_alive() - - -def test_is_base_containers_alive_exited(skale_base_containers_exited): - assert not is_base_containers_alive() - - -def test_is_base_containers_alive_empty(): - assert not is_base_containers_alive() - - @pytest.fixture def no_resource_file(): try: @@ -151,13 +288,14 @@ def test_init_node(no_resource_file): # todo: write new init node test mock.patch('node_cli.core.node.init_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.utils.helper.post_request', resp_mock), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): - init(env_filepath) + init(env_filepath=env_filepath, node_type=NodeType.REGULAR) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) -def test_update_node(mocked_g_config, resource_file): +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.MIRAGE]) +def test_update_node(node_type, mocked_g_config, resource_file, inited_node): env_filepath = './tests/test-env' resp_mock = response_mock(requests.codes.created) os.makedirs(NODE_DATA_PATH, exist_ok=True) @@ -176,39 +314,78 @@ def test_update_node(mocked_g_config, resource_file): 'node_cli.core.node.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.env.validate_env_params', lambda params: None), + mock.patch('node_cli.configs.env.validate_env_params'), ): with mock.patch( 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() ): # noqa - result = update(env_filepath, pull_config_for_schain=None) + result = update(env_filepath, pull_config_for_schain=None, node_type=node_type) assert result is None -def test_is_update_safe(): - assert is_update_safe() - assert is_update_safe(sync_node=True) - - with mock.patch('node_cli.core.node.is_admin_running', return_value=True): - with mock.patch('node_cli.core.node.is_api_running', return_value=True): - assert not is_update_safe() - assert is_update_safe(sync_node=True) - - with mock.patch('node_cli.core.node.is_sync_admin_running', return_value=True): - assert is_update_safe() - assert not is_update_safe(sync_node=True) - - with mock.patch('node_cli.utils.docker_utils.is_container_running', return_value=True): - with mock.patch( - 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() - ): - assert is_update_safe() - - with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): - assert is_update_safe() - - with mock.patch('node_cli.utils.docker_utils.is_container_running', return_value=True): - with mock.patch( - 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response(safe=False) - ): - assert not is_update_safe() +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.MIRAGE]) +@mock.patch('node_cli.core.node.is_admin_running', return_value=False) +@mock.patch('node_cli.core.node.is_api_running', return_value=False) +@mock.patch('node_cli.utils.helper.requests.get') +def test_is_update_safe_when_admin_and_api_not_running( + mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type +): + assert is_update_safe(node_type=node_type) is True + mock_requests_get.assert_not_called() + + +@mock.patch('node_cli.core.node.is_admin_running', return_value=False) +@mock.patch('node_cli.core.node.is_api_running', return_value=True) +@mock.patch('node_cli.utils.helper.requests.get') +def test_is_update_safe_when_admin_not_running_for_sync( + mock_requests_get, mock_is_api_running, mock_is_admin_running +): + assert is_update_safe(node_type=NodeType.SYNC) is True + mock_requests_get.assert_not_called() + + +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.MIRAGE]) +@pytest.mark.parametrize( + 'api_is_safe, expected_result', + [(True, True), (False, False)], + ids=['api_safe', 'api_unsafe'], +) +@mock.patch('node_cli.core.node.is_admin_running', return_value=True) +@mock.patch('node_cli.utils.helper.requests.get') +def test_is_update_safe_when_admin_running( + mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type +): + mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) + assert is_update_safe(node_type=node_type) is expected_result + mock_requests_get.assert_called_once() + + +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.MIRAGE]) +@pytest.mark.parametrize( + 'api_is_safe, expected_result', + [(True, True), (False, False)], + ids=['api_safe', 'api_unsafe'], +) +@mock.patch('node_cli.core.node.is_admin_running', return_value=False) +@mock.patch('node_cli.core.node.is_api_running', return_value=True) +@mock.patch('node_cli.utils.helper.requests.get') +def test_is_update_safe_when_only_api_running_for_regular( + mock_requests_get, + mock_is_api_running, + mock_is_admin_running, + api_is_safe, + expected_result, + node_type, +): + mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) + assert is_update_safe(node_type=node_type) is expected_result + mock_requests_get.assert_called_once() + + +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.MIRAGE]) +@mock.patch('node_cli.core.node.is_admin_running', return_value=True) +@mock.patch('node_cli.utils.helper.requests.get') +def test_is_update_safe_when_api_call_fails(mock_requests_get, mock_is_admin_running, node_type): + mock_requests_get.side_effect = requests.exceptions.ConnectionError('Test connection error') + assert is_update_safe(node_type=node_type) is False + mock_requests_get.assert_called_once() diff --git a/tests/core/host/docker_config_test.py b/tests/core/host/docker_config_test.py index 87eb391c..4d87cb54 100644 --- a/tests/core/host/docker_config_test.py +++ b/tests/core/host/docker_config_test.py @@ -130,7 +130,7 @@ def container(dclient): c.remove(force=True) -def test_assert_no_contaners(): +def test_assert_no_containers(): assert_no_containers(ignore=('ganache',)) diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py new file mode 100644 index 00000000..b19a21ca --- /dev/null +++ b/tests/core/nginx_test.py @@ -0,0 +1,145 @@ +import os +from pathlib import Path + +import pytest +import mock + +from node_cli.core.nginx import ( + generate_nginx_config, + check_ssl_certs, + is_regular_node_nginx, + SSL_KEY_NAME, + SSL_CRT_NAME, +) +from node_cli.utils.node_type import NodeType +from node_cli.configs import NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, NODE_CERTS_PATH + +TEST_NGINX_TEMPLATE = """ +server { + listen 3009; + {% if ssl %} + listen 311 ssl; + ssl_certificate /ssl/ssl_cert; + ssl_certificate_key /ssl/ssl_key; + {% endif %} +} + +{% if regular_node %} +server { + listen 80; + {% if ssl %} + listen 443 ssl; + ssl_certificate /ssl/ssl_cert; + ssl_certificate_key /ssl/ssl_key; + {% endif %} +} +{% endif %} +""" + +CORE_SSL_SNIPPET = 'listen 311 ssl;' +FILESTORAGE_SNIPPET = 'listen 80;' +FILESTORAGE_SSL_SNIPPET = 'listen 443 ssl;' + + +@pytest.fixture +def nginx_template(): + """Create a temporary nginx template file.""" + os.makedirs(os.path.dirname(NGINX_TEMPLATE_FILEPATH), exist_ok=True) + with open(NGINX_TEMPLATE_FILEPATH, 'w') as f: + f.write(TEST_NGINX_TEMPLATE) + try: + yield + finally: + if os.path.isfile(NGINX_TEMPLATE_FILEPATH): + os.remove(NGINX_TEMPLATE_FILEPATH) + if os.path.isfile(NGINX_CONFIG_FILEPATH): + os.remove(NGINX_CONFIG_FILEPATH) + + +@pytest.mark.parametrize( + 'node_type, ssl_exists, expected_regular_flag, expected_ssl_flag', + [ + (NodeType.REGULAR, True, True, True), + (NodeType.REGULAR, False, True, False), + (NodeType.SYNC, True, True, True), + (NodeType.SYNC, False, True, False), + (NodeType.MIRAGE, True, False, True), + (NodeType.MIRAGE, False, False, False), + ], + ids=[ + 'regular_ssl_on', + 'regular_ssl_off', + 'regular_ssl_on', + 'regular_ssl_off', + 'mirage_ssl_on', + 'mirage_ssl_off', + ], +) +@mock.patch('node_cli.core.nginx.check_ssl_certs') +@mock.patch('node_cli.core.nginx.TYPE') +def test_generate_nginx_config( + mock_type, + mock_check_ssl, + node_type, + ssl_exists, + expected_regular_flag, + expected_ssl_flag, + nginx_template, +): + mock_type.__eq__.side_effect = lambda other: node_type == other + mock_type.__ne__.side_effect = lambda other: node_type != other + mock_check_ssl.return_value = ssl_exists + + generate_nginx_config() + + assert os.path.exists(NGINX_CONFIG_FILEPATH) + with open(NGINX_CONFIG_FILEPATH) as f: + rendered_config = f.read() + + rendered_config = rendered_config.strip() + + if expected_regular_flag: + assert FILESTORAGE_SNIPPET in rendered_config + else: + assert FILESTORAGE_SNIPPET not in rendered_config + + if expected_ssl_flag: + assert CORE_SSL_SNIPPET in rendered_config + else: + assert CORE_SSL_SNIPPET not in rendered_config + + if expected_regular_flag and expected_ssl_flag: + assert FILESTORAGE_SSL_SNIPPET in rendered_config + else: + assert FILESTORAGE_SSL_SNIPPET not in rendered_config + + +def test_check_ssl_certs_exist(ssl_folder): + Path(os.path.join(NODE_CERTS_PATH, SSL_CRT_NAME)).touch() + Path(os.path.join(NODE_CERTS_PATH, SSL_KEY_NAME)).touch() + assert check_ssl_certs() + + +def test_check_ssl_certs_missing_one(ssl_folder): + Path(os.path.join(NODE_CERTS_PATH, SSL_CRT_NAME)).touch() + assert check_ssl_certs() is False + + +def test_check_ssl_certs_missing_both(ssl_folder): + assert check_ssl_certs() is False + + +@pytest.mark.parametrize( + 'node_type, expected_result', + [ + (NodeType.REGULAR, True), + (NodeType.SYNC, True), + (NodeType.MIRAGE, False), + ], +) +@mock.patch('node_cli.core.nginx.TYPE') +def test_is_regular_node_nginx(mock_type, node_type, expected_result): + mock_type.__eq__.side_effect = lambda other: node_type == other + mock_type.__ne__.side_effect = lambda other: node_type != other + + assert is_regular_node_nginx() is expected_result diff --git a/tests/helper.py b/tests/helper.py index f2209088..08d67c20 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -27,7 +27,7 @@ BLOCK_DEVICE = os.getenv('BLOCK_DEVICE') -TEST_SCHAINS_MNT_DIR_SYNC = 'tests/tmp' +TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN = 'tests/tmp' TEST_META_V1 = {'version': '0.1.1', 'config_stream': 'develop'}