Merge more-images and StackHPC changes to main

.github/workflows/pull-request.yml

@@ -0,0 +1,34 @@
+---
+name: Pull request
+"on":
+  pull_request:
+jobs:
+  lint:
+    if: github.repository == 'stackhpc/openstack-config'
+    runs-on: ubuntu-22.04
+    permissions: {}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # NOTE(upgrade): Keep these in sync with Kayobe's supported Ansible and Python versions (see release notes).
+          - ansible: "2.16"
+            python: "3.12"
+    name: Ansible ${{ matrix.ansible }} lint with Python ${{ matrix.python }}
+    steps:
+      - name: GitHub Checkout 🛎
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }} 🐍
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install dependencies 📦
+        run: |
+          python -m pip install --upgrade pip
+          pip install ansible-core==${{ matrix.ansible }}.* ansible-lint
+
+      - name: Linting code 🧪
+        run: |
+          ansible-lint -v --force-color

.gitignore

@@ -109,8 +109,20 @@ ansible/*.retry
 ansible/roles/*\.*/
 # Galaxy collections
 ansible/collections/**/
+# Image cache
+ansible/openstack-config-image-cache
+# Venvs
+ansible/openstack-config-venv
 
 # Editors
 *~
 .*.swp
 .*sw?
+.vscode/
+
+# Ignore working dirs
+ansible/openstack-config-image-cache
+ansible/openstack-config-venv
+
+# Ignore tmp output from template generation playbook
+generated-magnum-snippets/

README.rst

@@ -3,31 +3,26 @@ OpenStack Configuration
 =============================================
 
 This project contains Ansible playbooks and configuration of infrastructure on
-an existing OpenStack cloud for the OpenStack system. (Supported up to Yoga
-release.)
+an existing OpenStack cloud for the OpenStack system. 
 
 Preparation
 ===========
 
 Ensure that Ansible is installed, either via the system package manager or pip.
-If required, use a virtualenv to avoid interference with the system python
-packages. For example:
+It is recommended that you use a python virtual environment to avoid
+interference with the system python packages. For example:
 
 .. code-block::
 
-   $ virtualenv venv
-   $ source venv/bin/activate
-   $ pip install -U pip
+   $ python3 -m venv openstack-venv
+   $ source openstack-venv/bin/activate
+   $ python -m pip install --upgrade pip
    $ pip install -r requirements.txt
 
 Install Ansible role and collection dependencies from Ansible Galaxy:
 
 .. code-block::
 
-   $ ansible-galaxy role install \
-        -p ansible/roles \
-        -r requirements.yml
-
    $ ansible-galaxy collection install \
        -p ansible/collections \
        -r requirements.yml
@@ -81,3 +76,48 @@ configuration parameter:
 .. code-block::
 
    $ tools/openstack-config -- --vault-password-file config-secret.vault
+
+
+Magnum Cluster Templates
+========================
+
+To generate a new set of Magnum cluster templates and corresponding Glance image
+definitions which utilise the latest stable upstream release tag, set the following
+variables in `etc/openstack-config.yml`
+
+.. code-block:: yaml
+
+   # Chosen flavor on target cloud
+   magnum_default_master_flavor_name:
+   # Chosen flavor on target cloud
+   magnum_default_worker_flavor_name:
+   # External network to use for load balancers etc.
+   magnum_external_net_name:
+   # Optional list of extra labels to add to all generated cluster templates
+   magnum_template_extra_labels:
+
+The load balancer provider defaults to OVN. This can be changed to Amphora, but you
+should only do this if OVN load balancers are unavailable.
+
+.. code-block:: yaml
+
+   magnum_loadbalancer_provider: amphora
+
+Then run the provided playbook with
+
+.. code-block:: bash
+
+   $ tools/openstack-config -p ansible/generate-magnum-capi-templates.yml
+
+This will create a ``generated-magnum-snippets`` directory in the repo root with
+a timestamped sub-directory containing an ``images.yml`` file and a ``templates.yml``
+file. The contents of these two files can then be added to any existing images and
+cluster templates in ``etc/openstack-config.yml``. When deploying the updated config,
+be sure to run the ``openstack-images.yml`` playbook *before* running the
+``openstack-container-clusters.yml`` playbook, otherwise the Magnum API will return
+an error referencing an invalid cluster type with image ``None``. This is handled
+automatically if running the full ``openstack.yml`` playbook.
+
+Note that these templates are a tested set against the specific CAPI management
+cluster release. As such, you should make sure to update your CAPI management
+cluster to the latest release before updating to the latest templates.

ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+bin_ansible_callbacks = True
+callbacks_enabled = ansible.posix.profile_tasks
+stdout_callback = yaml

ansible/generate-magnum-capi-templates.yml

@@ -0,0 +1,65 @@
+---
+- name: Generate cluster templates
+  hosts: localhost
+  vars:
+    root_dir: ../
+  tasks:
+    - name: Check that required variables are defined
+      ansible.builtin.assert:
+        that:
+          - magnum_default_master_flavor_name is defined
+          - magnum_default_worker_flavor_name is defined
+          - magnum_external_net_name is defined
+          - magnum_loadbalancer_provider is defined
+
+    - name: Fetch capi-helm-charts release information
+      ansible.builtin.uri:
+        url: https://api.github.com/repos/azimuth-cloud/capi-helm-charts/releases/latest
+      register: capi_helm_chart_release_data
+
+    - name: Fetch dependencies.json for capi-helm-charts release
+      ansible.builtin.uri:
+        url: https://raw.githubusercontent.com/azimuth-cloud/capi-helm-charts/{{ capi_helm_chart_release_data.json.tag_name }}/dependencies.json
+        return_content: true
+      register: dependencies_response
+
+    - name: Ensure wget packages is installed
+      become: true
+      ansible.builtin.package:
+        name: wget
+        state: present
+
+    - name: Fetch manifest.json for capi-helm-charts images  # noqa command-instead-of-module
+      vars:
+        dependencies: "{{ dependencies_response.content }}"
+    # ansible.builtin.uri:
+    #   url: https://raw.githubusercontent.com/azimuth-cloud/azimuth-images/{{ dependencies_response.json['azimuth-images'] }}/manifest.json
+    # Above URL returns 404 even though similar URL for capi-helm-charts repo works fine
+    # Not sure why but fall back to wget + JSON parsing for now.
+      ansible.builtin.command: >-
+        wget -O - https://github.com/azimuth-cloud/azimuth-images/releases/download/{{ dependencies['azimuth-images'] }}/manifest.json
+      register: manifest_response
+      changed_when: false
+
+    - name: Parse JSON response
+      ansible.builtin.set_fact:
+        new_template_data: "{{ manifest_response.stdout | from_json | dict2items | selectattr('key', 'match', 'kubernetes*') | list }}"
+
+    - name: Ensure output dir exists
+      ansible.builtin.file:
+        path: "{{ [root_dir, 'generated-magnum-snippets', now(utc=true, fmt='%Y-%m-%d-T%H-%M-%S')] | path_join }}"
+        state: directory
+        mode: "0755"
+      register: output_dir
+
+    - name: Write new image config to file
+      ansible.builtin.template:
+        src: "magnum-capi-images.j2"
+        dest: "{{ output_dir.path }}/images.yml"
+        mode: "0644"
+
+    - name: Write new cluster template config to file
+      ansible.builtin.template:
+        src: "magnum-capi-templates.j2"
+        dest: "{{ output_dir.path }}/templates.yml"
+        mode: "0644"

ansible/group_vars/all/openstack

@@ -2,6 +2,10 @@
 ###############################################################################
 # Configuration of OpenStack user environment for OpenStack.
 
+# List of OpenStack domains. Format is as required by the stackhpc.os-projects
+# role.
+openstack_domains: []
+
 # List of OpenStack projects. Format is as required by the stackhpc.os-projects
 # role.
 openstack_projects: []
@@ -21,6 +25,10 @@ openstack_routers: []
 # stackhpc.os-networks role.
 openstack_security_groups: []
 
+# List of RBAC definitions in the openstack projct. Format is as required by the
+# stackhpc.os-networks role.
+openstack_networks_rbac: []
+
 ###############################################################################
 # Configuration of nova flavors for OpenStack.
 
@@ -54,3 +62,26 @@ openstack_host_aggregates: []
 # List of clusters templates in the openstack project. Format is as required by the
 # stackhpc.os-container-clusters role.
 openstack_container_clusters_templates: []
+
+###############################################################################
+# Configuration variables for a CloudKitty ratings service deployment.
+
+# A list where each item is a dictionary mapping the associated fields,
+# with the 'mappings' field also being a list of dictionaries.
+# Example of the mappings and their fields can be found below,
+# however for more information please refer to the README.md file.
+#
+openstack_ratings_hashmap_field_mappings: []
+
+# Much like the field mappings above, the service mappings are a list of
+# dictionaries, however these are not associated with a field.
+#
+openstack_ratings_hashmap_service_mappings: []
+
+###############################################################################
+# Configuration of volumes for OpenStack.
+openstack_volumes_types: []
+
+###############################################################################
+# Configuration of Cloudkitty for OpenStack.
+openstack_cloudkitty_enable: false

ansible/openstack-container-clusters.yml

@@ -4,7 +4,7 @@
   tags:
     - container-clusters-templates
   roles:
-    - role: stackhpc.os-container-clusters
+    - role: stackhpc.openstack.os_container_clusters
       os_container_clusters_venv: "{{ openstack_venv }}"
       os_container_clusters_auth_type: "{{ openstack_auth_type }}"
       os_container_clusters_auth: "{{ openstack_auth }}"

ansible/openstack-flavors.yml

@@ -1,10 +1,10 @@
 ---
-- name: Ensure OpenStack OpenStack nova flavors exist
+- name: Ensure OpenStack nova flavors exist
   hosts: localhost
   tags:
     - flavors
   roles:
-    - role: stackhpc.os-flavors
+    - role: stackhpc.openstack.os_flavors
       os_flavors_venv: "{{ openstack_venv }}"
       os_flavors_auth_type: "{{ openstack_auth_type }}"
       os_flavors_auth: "{{ openstack_auth }}"

ansible/openstack-host-aggregates.yml

@@ -1,10 +1,10 @@
 ---
-- name: Ensure OpenStack OpenStack nova host aggregates exist
+- name: Ensure OpenStack nova host aggregates exist
   hosts: localhost
   tags:
     - host_aggregates
   roles:
-    - role: stackhpc.os_host_aggregates
+    - role: stackhpc.openstack.os_host_aggregates
       os_host_aggregates_venv: "{{ openstack_venv }}"
       os_host_aggregates_auth_type: "{{ openstack_auth_type }}"
       os_host_aggregates_auth: "{{ openstack_auth }}"

ansible/openstack-images.yml

@@ -4,7 +4,7 @@
   tags:
     - images
   roles:
-    - role: stackhpc.os-images
+    - role: stackhpc.openstack.os_images
       os_images_venv: "{{ openstack_venv }}"
       os_images_cache: "{{ ansible_env.PWD }}/openstack-config-image-cache"
       os_images_auth_type: "{{ openstack_auth_type }}"

ansible/openstack-networks.yml

@@ -1,14 +1,15 @@
 ---
-- name: Ensure OpenStack OpenStack networks exist
+- name: Ensure OpenStack networks exist
   hosts: localhost
   tags:
     - networks
   roles:
-    - role: stackhpc.os-networks
+    - role: stackhpc.openstack.os_networks
       os_networks_venv: "{{ openstack_venv }}"
       os_networks_auth_type: "{{ openstack_auth_type }}"
       os_networks_auth: "{{ openstack_auth }}"
       os_networks_cacert: "{{ openstack_cacert }}"
       os_networks: "{{ openstack_networks }}"
       os_networks_routers: "{{ openstack_routers }}"
       os_networks_security_groups: "{{ openstack_security_groups }}"
+      os_networks_rbac: "{{ openstack_networks_rbac }}"

ansible/openstack-project.yml

@@ -1,12 +1,13 @@
 ---
-- name: Ensure OpenStack OpenStack projects exist
+- name: Ensure OpenStack projects exist
   hosts: localhost
   tags:
     - project
   roles:
-    - role: stackhpc.os-projects
+    - role: stackhpc.openstack.os_projects
       os_projects_venv: "{{ openstack_venv }}"
       os_projects_auth_type: "{{ openstack_auth_type }}"
       os_projects_admin_auth: "{{ openstack_auth }}"
       os_projects_cacert: "{{ openstack_cacert }}"
       os_projects: "{{ openstack_projects }}"
+      os_projects_domains: "{{ openstack_domains }}"

ansible/openstack-ratings.yml

@@ -0,0 +1,10 @@
+---
+- name: Generate OpenStack software ratings
+  hosts: localhost
+  tags:
+    - ratings
+  roles:
+    - role: stackhpc.openstack.os_ratings
+      os_ratings_hashmap_field_mappings: "{{ openstack_ratings_hashmap_field_mappings }}"
+      os_ratings_hashmap_service_mappings: "{{ openstack_ratings_hashmap_service_mappings }}"
+      os_ratings_venv: "{{ openstack_venv }}"

ansible/openstack-volume-types.yml

@@ -0,0 +1,12 @@
+---
+- name: Ensure OpenStack volume types exist
+  hosts: localhost
+  tags:
+    - volume-types
+  roles:
+    - role: stackhpc.openstack.os_volumes
+      os_volumes_venv: "{{ openstack_venv }}"
+      os_volumes_auth_type: "{{ openstack_auth_type }}"
+      os_volumes_auth: "{{ openstack_auth }}"
+      os_volumes_cacert: "{{ openstack_cacert }}"
+      os_volumes_types: "{{ openstack_volumes_types }}"

ansible/openstack.yml

@@ -1,8 +1,12 @@
 ---
 # Top level playbook that includes all others.
-- import_playbook: openstack-project.yml
-- import_playbook: openstack-networks.yml
-- import_playbook: openstack-flavors.yml
-- import_playbook: openstack-images.yml
-- import_playbook: openstack-host-aggregates.yml
-- import_playbook: openstack-container-clusters.yml
+
+- import_playbook: openstack-project.yml  # noqa name[play]
+- import_playbook: openstack-networks.yml  # noqa name[play]
+- import_playbook: openstack-flavors.yml  # noqa name[play]
+- import_playbook: openstack-images.yml  # noqa name[play]
+- import_playbook: openstack-host-aggregates.yml  # noqa name[play]
+- import_playbook: openstack-volume-types.yml  # noqa name[play]
+- import_playbook: openstack-container-clusters.yml  # noqa name[play]
+- import_playbook: openstack-ratings.yml  # noqa name[play]
+  when: openstack_cloudkitty_enable | bool