diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 145cc3170f..5cd2d338db 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -37,8 +37,8 @@ jobs: uses: ./.github/workflows/runner-selector.yml with: runner_env: ${{ inputs.runner_env }} - overcloud-host-image-build: - name: Build overcloud host images + validate-inputs: + name: Input validation if: github.repository == 'stackhpc/stackhpc-kayobe-config' environment: ${{ inputs.runner_env }} runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} @@ -53,15 +53,18 @@ jobs: exit 1 fi - - name: Install Package - uses: ConorMacBride/install-package@main - with: - apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq - - - name: Start the SSH service - run: | - sudo /etc/init.d/ssh start - + create-tag: + name: Create a tag to be added to resulting images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - runner-selection + - validate-inputs + permissions: {} + outputs: + host_image_tag: ${{ steps.host_image_tag.outputs.host_image_tag }} + steps: - name: Checkout uses: actions/checkout@v4 with: @@ -84,9 +87,34 @@ jobs: run: | echo "host_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + overcloud-host-image-build-rocky: + name: Build Rocky overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.rocky9 + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - create-tag + - runner-selection + - validate-inputs + permissions: {} + steps: - name: Display overcloud host image tag run: | - echo "${{ steps.host_image_tag.outputs.host_image_tag }}" + echo "${{ needs.create-tag.outputs.host_image_tag }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start - name: Install Kayobe run: | @@ -120,7 +148,7 @@ jobs: cat << EOF > terraform.tfvars ssh_public_key = "id_rsa.pub" ssh_username = "ubuntu" - aio_vm_name = "skc-host-image-builder" + aio_vm_name = "skc-host-image-builder-rocky" # Must be an Ubuntu Noble host to successfully build all images # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" @@ -255,11 +283,11 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ -e artifact_path=/opt/kayobe/images/overcloud-rocky-9 \ - -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \ + -e artifact_tag=${{ needs.create-tag.outputs.host_image_tag }} \ -e artifact_type="kayobe-images" \ -e file_regex="*.qcow2" \ -e os_distribution="rocky" \ - -e os_release="9" + -e os_release="9" -vvv env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' @@ -271,7 +299,7 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ - -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} + -e image_name=overcloud-rocky-9-${{ needs.create-tag.outputs.host_image_tag }} env: CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} @@ -285,13 +313,208 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ - -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} + -e image_name=overcloud-rocky-9-${{ needs.create-tag.outputs.host_image_tag }} env: CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() + + - name: Fail if any overcloud host image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_rocky_9.outcome == 'failure' + + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs Rocky + path: ./logs + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() + + overcloud-host-image-build-ubuntu: + name: Build Ubuntu overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.ubuntu-noble + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - create-tag + - runner-selection + - validate-inputs + permissions: {} + steps: + - name: Display overcloud host image tag + run: | + echo "${{ needs.create-tag.outputs.host_image_tag }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-host-image-builder-ubuntu" + # Must be an Ubuntu Noble host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Create bifrost_httpboot Docker volume + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + - name: Build an Ubuntu Noble 24.04 overcloud host image id: build_ubuntu_noble continue-on-error: true @@ -323,11 +546,11 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-noble \ - -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \ + -e artifact_tag=${{ needs.create-tag.outputs.host_image_tag }} \ -e artifact_type="kayobe-images" \ -e file_regex="*.qcow2" \ -e os_distribution="ubuntu" \ - -e os_release="noble" + -e os_release="noble" -vvv env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success' @@ -339,7 +562,7 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble/overcloud-ubuntu-noble.qcow2" \ - -e image_name=overcloud-ubuntu-noble-${{ steps.host_image_tag.outputs.host_image_tag }} + -e image_name=overcloud-ubuntu-noble-${{ needs.create-tag.outputs.host_image_tag }} env: CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} @@ -353,7 +576,7 @@ jobs: kayobe playbook run \ src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble/overcloud-ubuntu-noble.qcow2" \ - -e image_name=overcloud-ubuntu-noble-${{ steps.host_image_tag.outputs.host_image_tag }} + -e image_name=overcloud-ubuntu-noble-${{ needs.create-tag.outputs.host_image_tag }} env: CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} @@ -372,13 +595,538 @@ jobs: run: | echo "Builds failed. See workflow artifacts for details." && exit 1 - if: steps.build_rocky_9.outcome == 'failure' || - steps.build_ubuntu_noble.outcome == 'failure' + if: steps.build_ubuntu_noble.outcome == 'failure' + + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs Ubuntu + path: ./logs + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() + + overcloud-host-image-build-rocky-arm64: + name: Build Rocky ARM64 overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.rocky9 + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - create-tag + - runner-selection + - validate-inputs + permissions: {} + steps: + - name: Display overcloud host image tag + run: | + echo "${{ needs.create-tag.outputs.host_image_tag }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-host-image-builder-rocky-arm64" + # Must be an Ubuntu Noble host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE_ARM64 }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Create bifrost_httpboot Docker volume + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Build a Rocky Linux 9 ARM64 overcloud host image + id: build_rocky_9_arm64 + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution="rocky" \ + -e os_release="9" \ + -e stackhpc_overcloud_dib_name=overcloud-rocky-9-arm64 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky9 + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9-arm64/overcloud-rocky-9-arm64.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_9_arm64.outcome == 'failure' + + - name: Upload Rocky Linux 9 ARM64 overcloud host image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/overcloud-rocky-9-arm64 \ + -e artifact_tag=${{ needs.create-tag.outputs.host_image_tag }} \ + -e artifact_type="kayobe-images" \ + -e file_regex="*.qcow2" \ + -e os_distribution="rocky" \ + -e os_release="9" -vvv + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky9 && steps.build_rocky_9_arm64.outcome == 'success' + + - name: Upload Rocky Linux 9 ARM64 overcloud host image to current Dev Cloud (SMS/Leafcloud) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9-arm64/overcloud-rocky-9-arm64.qcow2" \ + -e image_name=overcloud-rocky-9-arm64-${{ needs.create-tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 && steps.build_rocky_9_arm64.outcome == 'success' + + - name: Upload Rocky Linux 9 ARM64 overcloud host image to other Dev Cloud (Leafcloud/SMS) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9-arm64/overcloud-rocky-9-arm64.qcow2" \ + -e image_name=overcloud-rocky-9-arm64-${{ needs.create-tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} + if: inputs.rocky9 && steps.build_rocky_9_arm64.outcome == 'success' + + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() + + - name: Fail if any overcloud host image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_rocky_9_arm64.outcome == 'failure' + + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs Rocky ARM64 + path: ./logs + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() + + overcloud-host-image-build-ubuntu-arm64: + name: Build Ubuntu ARM64 overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.ubuntu-noble + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - create-tag + - runner-selection + - validate-inputs + permissions: {} + steps: + - name: Display overcloud host image tag + run: | + echo "${{ needs.create-tag.outputs.host_image_tag }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-host-image-builder-ubuntu-arm64" + # Must be an Ubuntu Noble host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE_ARM64 }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Create bifrost_httpboot Docker volume + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Build an Ubuntu Noble 24.04 ARM64 overcloud host image + id: build_ubuntu_noble_arm64 + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution="ubuntu" \ + -e os_release="noble" \ + -e stackhpc_overcloud_dib_name=overcloud-ubuntu-noble-arm64 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-noble + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-noble-arm64/overcloud-ubuntu-noble-arm64.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_noble_arm64.outcome == 'failure' + + - name: Upload Ubuntu Noble 24.04 overcloud host image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-noble-arm64 \ + -e artifact_tag=${{ needs.create-tag.outputs.host_image_tag }} \ + -e artifact_type="kayobe-images" \ + -e file_regex="*.qcow2" \ + -e os_distribution="ubuntu" \ + -e os_release="noble" -vvv + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble_arm64.outcome == 'success' + + - name: Upload Ubuntu Noble ARM64 overcloud host image to current Dev Cloud (SMS/Leafcloud) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble-arm64/overcloud-ubuntu-noble-arm64.qcow2" \ + -e image_name=overcloud-ubuntu-noble-arm64-${{ needs.create-tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble_arm64.outcome == 'success' + + - name: Upload Ubuntu Noble ARM64 overcloud host image to other Dev Cloud (Leafcloud/SMS) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble-arm64/overcloud-ubuntu-noble-arm64.qcow2" \ + -e image_name=overcloud-ubuntu-noble-arm64-${{ needs.create-tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble_arm64.outcome == 'success' + + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() + + - name: Fail if any overcloud host image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_ubuntu_noble_arm64.outcome == 'failure' - name: Upload logs artifact uses: actions/upload-artifact@v4 with: - name: Build logs + name: Build logs Ubuntu ARM64 path: ./logs if: always()