Add SONiC VPP platform support

.github/workflows/base-image.yaml

@@ -16,6 +16,7 @@ jobs:
           - name: 202311
           - name: 202411
           - name: 202505
+          - name: 202511-vpp
 
     steps:
     - name: Log in to the container registry

.github/workflows/integration.yaml

@@ -53,8 +53,8 @@ jobs:
         cache-from: type=registry,ref=${{ env.MINI_LAB_VM_IMAGE }}
         cache-to: type=inline
 
-  build-mini-lab-sonic-image:
-    name: Build mini-lab-sonic image
+  build-mini-lab-sonic-vs-image:
+    name: Build mini-lab-sonic-vs image
     runs-on: ubuntu-latest
 
     steps:
@@ -73,8 +73,8 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
         SHA_TAG=${COMMIT_SHA::8}
 
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic:${SHA_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vs:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vs:${SHA_TAG}" >> $GITHUB_ENV
       env:
         COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
@@ -88,24 +88,68 @@ jobs:
         pull: true
         push: true
         sbom: true
+        file: images/sonic/Dockerfile.vs
         tags: |
           ${{ env.MINI_LAB_SONIC_IMAGE }}
           ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
         cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
         cache-to: type=inline
 
+  build-mini-lab-sonic-vpp-image:
+    name: Build mini-lab-sonic-vpp image
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Log in to the container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ secrets.DOCKER_REGISTRY_USER }}
+        password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Make tag
+      run: |
+        IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
+        SHA_TAG=${COMMIT_SHA::8}
+
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vpp:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vpp:${SHA_TAG}" >> $GITHUB_ENV
+      env:
+        COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build and push mini-lab-sonic container
+      uses: docker/build-push-action@v6
+      with:
+        context: ./images/sonic
+        pull: true
+        push: true
+        sbom: true
+        file: images/sonic/Dockerfile.vpp
+        tags: |
+          ${{ env.MINI_LAB_SONIC_IMAGE }}
+          ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
+        cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
+        cache-to: type=inline
   test:
     name: Run tests
     runs-on: self-hosted
     needs:
       - build-mini-lab-vms-image
-      - build-mini-lab-sonic-image
+      - build-mini-lab-sonic-vs-image
+      - build-mini-lab-sonic-vpp-image
 
     continue-on-error: true
     strategy:
       matrix:
         flavors:
-          - name: sonic
+          - name: sonic_vs
+          - name: sonic_vpp
           - name: gardener
           - name: dell_sonic
 
@@ -140,7 +184,7 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
 
         echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
 
     - name: Run integration tests
       shell: bash

Makefile

@@ -1,18 +1,22 @@
+# runs at parse time on every invocation: ensure certs exist, then (re)generate .env
+$(shell [ -f files/certs/ca.pem ] || ./scripts/gen-certs.sh)
+$(shell ./env.sh 1>&2)
+-include .env
+
 .DEFAULT_GOAL := up
 .EXPORT_ALL_VARIABLES:
 
--include .env
+# disable built-in implicit rules (e.g. the "%: %.sh" rule that silently turns a
+# stray/misspelled prerequisite into a junk executable instead of erroring out)
+MAKEFLAGS += --no-builtin-rules
+.SUFFIXES:
 
 # Commands
 YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4
 
 KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
 KUBECONFIG := $(shell pwd)/.kubeconfig
 
-METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
-METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal)
-METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080)
-
 MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf
 # Default values
 CONTAINERLAB=$(shell which containerlab)
@@ -22,18 +26,28 @@ ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),)
 # do not show skipped ansible tasks
 ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
 
-MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
+MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic_vs)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
-MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
+MINI_LAB_SONIC_IMAGE_TAG := $(or $(MINI_LAB_SONIC_IMAGE_TAG),latest)
+
+MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
+# define this here as well so that kind picks up the network on a clean checkout,
+# where .env does not exist yet at make parse time (-include .env above)
+KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK))
 
 MACHINE_OS=debian-13.0
 MAX_RETRIES := 30
 
 # Machine flavors
-ifeq ($(MINI_LAB_FLAVOR),sonic)
+ifeq ($(MINI_LAB_FLAVOR),sonic_vs)
 LAB_TOPOLOGY=mini-lab.sonic.yaml
 MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
+else ifeq ($(MINI_LAB_FLAVOR),sonic_vpp)
+LAB_TOPOLOGY=mini-lab.sonic.yaml
+MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vpp:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
 LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
 MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
@@ -43,11 +57,13 @@ MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_
 else ifeq ($(MINI_LAB_FLAVOR),kamaji)
 LAB_TOPOLOGY=mini-lab.kamaji.yaml
 KAMAJI_ENABLED=true
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),gardener)
 GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
 K8S_VERSION=1.32.5
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -75,7 +91,7 @@ COMPOSE_ARGS += $(if $(MINI_LAB_HELM_CHARTS),-f compose.dev/helm-charts.yaml)
 endif
 
 .PHONY: up
-up: env gen-certs verify-deployment-image control-plane-bake partition-bake
+up: gen-certs verify-deployment-image control-plane-bake partition-bake
 	@chmod 600 files/ssh/id_ed25519
 	docker compose $(COMPOSE_ARGS) up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition
 	@$(MAKE)	--no-print-directory	start-machines
@@ -93,23 +109,12 @@ restart: down up
 .PHONY: down
 down: cleanup
 
-.PHONY: gen-certs
-gen-certs:
-	@if ! [ -f "files/certs/ca.pem" ]; then \
-		echo "certificate generation required, running cfssl container"; \
-		docker run --rm \
-			--user $$(id -u):$$(id -g) \
-			--entrypoint bash \
-			-v ${PWD}:/work \
-			cfssl/cfssl /work/scripts/roll_certs.sh; fi
-
 .PHONY: roll-certs
 roll-certs:
-	rm files/certs/ca.pem
-	$(MAKE) gen-certs
+	./scripts/gen-certs.sh
 
 .PHONY: control-plane
-control-plane: control-plane-bake env
+control-plane: control-plane-bake
 	docker compose $(COMPOSE_ARGS) up --remove-orphans --force-recreate control-plane
 
 .PHONY: create-proxy-registries
@@ -118,13 +123,16 @@ create-proxy-registries:
 
 .PHONY: control-plane-bake
 control-plane-bake:
+
+	@if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi
 	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
 	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
 		kind create cluster $(KIND_ARGS) \
 			--name metal-control-plane \
 			--config $(KINDCONFIG) \
 			--kubeconfig $(KUBECONFIG); fi
 	$(MAKE) create-proxy-registries
+	docker compose up -d --force-recreate cloud-provider-kind
 
 .PHONY: partition
 partition: partition-bake
@@ -144,7 +152,7 @@ endif
 		./scripts/deactivate_offloading.sh; fi
 
 .PHONY: verify-deployment-image
-verify-deployment-image: env
+verify-deployment-image:
 	@if which cosign 1> /dev/null 2> /dev/null; then \
 		echo -e "\033[0;32mcosign is installed, verifying deployment base image\033[0m" && \
 		. ./.env && cosign verify --key files/cosign.pub ghcr.io/metal-stack/metal-deployment-base:$$DEPLOYMENT_BASE_IMAGE_TAG; \
@@ -170,12 +178,18 @@ external_network:
 		sudo ip -6 route add 2001:db8:0:113::/64 via 2001:db8:0:1::1 dev mini_lab_ext; \
 	fi
 
-.PHONY: env
-env:
-	@./env.sh
+files/certs/ca.pem:
+	@./scripts/gen-certs.sh
+
+.env: files/certs/ca.pem
+	@./env.sh 1>&2
+
+.PHONY: gen-certs                 # keep as a convenience alias
+gen-certs: files/certs/ca.pem
 
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
+	docker network rm --force mini_lab_internal
 
 .PHONY: cleanup-control-plane
 cleanup-control-plane:
@@ -193,7 +207,7 @@ cleanup-partition:
 	docker network rm --force mini_lab_ext
 
 .PHONY: _privatenet
-_privatenet: env
+_privatenet:
 	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network
 
 .PHONY: update-userdata
@@ -236,7 +250,7 @@ public-ipv6:
 	@docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --addressfamily IPv6 -o template --template "{{ .ipaddress }}"
 
 .PHONY: ls
-ls: env
+ls:
 	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine ls
 
 ## SWITCH MANAGEMENT ##
@@ -342,7 +356,7 @@ console-machine04:
 	@$(MAKE) --no-print-directory _console VM=machine04
 
 .PHONY: _password
-_password: env
+_password:
 	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine consolepassword $(MACHINE_UUID)
 
 .PHONY: password-machine01
@@ -433,6 +447,7 @@ dev-env:
 	@echo "export METALCTL_HMAC=${METALCTL_HMAC}"
 	@echo "export METAL_APIV2_URL=${METAL_APIV2_URL}"
 	@echo "export KUBECONFIG=$(KUBECONFIG)"
+	@echo "export METALCTL_CERTIFICATE_AUTHORITY_DATA=$(METALCTL_CERTIFICATE_AUTHORITY_DATA)"
 
 build-dell-sonic:
 	if [ ! -f "sonic-vs.img" ]; then \
@@ -451,7 +466,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

README.md

@@ -238,16 +238,17 @@ make up
 
 All available mini-lab flavors are listed below:
 
-- `sonic`: runs two Community SONiC switches
+- `sonic_vs`: runs two Community SONiC switches with the kernel based virtual switch dataplane
+- `sonic_vpp`: runs two Community SONiC switches with the [VPP](https://fd.io) dataplane
 - `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic)
 - `capms_dell_sonic`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall)
-- `kamaji`: runs a variation of the `sonic` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`.
-- `gardener`: runs the `sonic` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab
+- `kamaji`: runs a variation of the `sonic_vs` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`.
+- `gardener`: runs the `sonic_vs` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab
 
 In order to start specific flavor, you can define the flavor as follows:
 
 ```bash
-export MINI_LAB_FLAVOR=sonic
+export MINI_LAB_FLAVOR=sonic_vs
 make
 ```
 

compose.yaml

@@ -53,6 +53,7 @@ services:
     environment:
       - METALCTL_HMAC=${METALCTL_HMAC}
       - METALCTL_API_URL=${METALCTL_API_URL}
+      - METALCTL_CERTIFICATE_AUTHORITY_DATA=${METALCTL_CERTIFICATE_AUTHORITY_DATA}
       - METALCTL_HMAC_AUTH_TYPE=${METALCTL_HMAC_AUTH_TYPE:-Metal-Admin}
     volumes:
       - ./files/ssh:/root/.ssh:ro
@@ -126,6 +127,20 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+  cloud-provider-kind:
+    image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
+    restart: always
+    networks:
+      - kind
+    environment:
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+    command:
+    # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
+    # using envoy-gateway deployed via roles/gateway instead
+      - --gateway-channel
+      - disabled
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 volumes:
   proxy-docker:
   proxy-gcr:

control-plane/kind.yaml

@@ -2,7 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   apiServerPort: 6443
-  apiServerAddress: 0.0.0.0
+  apiServerAddress: 172.42.0.1
 nodes:
   - role: control-plane
     extraMounts:
@@ -13,10 +13,6 @@ nodes:
         hostPort: 4443
       - containerPort: 8080
         hostPort: 8080
-      - containerPort: 4150
-        hostPort: 4150
-      - containerPort: 50051
-        hostPort: 50051
     # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
       - |

deploy_control_plane.yaml

@@ -6,10 +6,12 @@
   roles:
     - name: ansible-common
       tags: always
-    - name: ingress-controller
-      tags: ingress-controller
     - name: metal-roles/control-plane/roles/prepare
       tags: prepare
+    - name: gateway
+      tags: gateway
+    - name: ingress-controller
+      tags: ingress-controller
     - name: metal-roles/control-plane/roles/nsq
       tags: nsq
     - name: metal-roles/control-plane/roles/metal-db