Attempting to automate testing of armv7 (armhf) images using a Debian 11.3 armhf VM and QEMU

.circleci/config.yml

@@ -10,6 +10,10 @@ executors:
     machine:
       image: ubuntu-2004:current
     resource_class: medium
+  ubuntu2004arm64-large:
+    machine:
+      image: ubuntu-2004:current
+    resource_class: arm-large
 
 jobs:
   build-multi-arch:
@@ -31,13 +35,14 @@ jobs:
             cat $BASH_ENV
             source $BASH_ENV
             echo "Workflow environment variables:"
-            echo $BRANCH
+            echo "BRANCH=$BRANCH"
       - run: uname -a
       - run: docker info
       - checkout
       - run:
           name: "Build Docker images"
           command: |
+            export BRANCH=$CIRCLE_BRANCH
             echo "Branch is $BRANCH"
             NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi
       - run:
@@ -76,16 +81,73 @@ jobs:
       BUILD_DATE: today
     steps:
       - checkout
-      - restore_cache:
-          keys: 
-            - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >>
+      #- restore_cache:
+      #    keys: 
+            # - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >>
+      #      - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >>
+      - run:
+          name: Download VM and install qemu, if armv7l
+          command: |
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                echo "Download armv7l VM..."
+                cd ~
+                curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \
+                  -o debian-11.3-armhf-ci-v1.0.tar.xz
+                echo "Extract VM..."
+                tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz
+                rm debian-11.3-armhf-ci-v1.0.tar.xz
+                echo "Install QEMU binaries"
+                curl -L  https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.0.0-1/xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \
+                  && tar xvfz xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \
+                  && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \
+                  && source ~/.bashrc \
+                  && qemu-system-arm --version \
+                  && qemu-system-aarch64 --version
+            fi
+      - run:
+          name: Start VM if armv7l
+          command: |
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                cd ~/debian-11.3-armhf-ci
+                sed -i 's/&//g' start-vm.sh
+                sed -i 's/-m 1024/-m 3072/' start-vm.sh
+                sed -i 's/,highmem=off//' start-vm.sh
+                sed -i 's/tb-size=256/tb-size=1024/' start-vm.sh
+                #sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh
+                sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh
+                sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh
+                cat start-vm.sh
+                sh start-vm.sh
+            fi
+          background: true
+      - run:
+          name: Wait for VM if armv7l
+          command: |
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                cd ~/debian-11.3-armhf-ci
+                sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh
+                bash wait-for-vm-boot.sh 
+                cd ..
+                echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV
+                source $BASH_ENV
+                ssh -p 8022 [email protected] -N -f -L4444:127.0.0.1:4444 -L5555:127.0.0.1:5555 -L7900:127.0.0.1:7900
+            fi
       - run: uname -a
       - run: docker info
       - run:
-          name: "Load built images from cache into Docker"
+          name: "Load built images into Docker"
           command: |
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock
+            fi
             echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID
-            docker load -i multi-arch-images.tar
+            # docker load -i multi-arch-images.tar
+            docker pull seleniarm/hub:4.4.0-20220814
+            docker pull seleniarm/node-chromium:4.4.0-20220814
+            docker pull seleniarm/node-firefox:4.4.0-20220814
+            docker pull seleniarm/standalone-chromium:4.4.0-20220814
+            docker pull seleniarm/standalone-firefox:4.4.0-20220814
+            docker info
       - run:
           name: "Use Python3 and pip3 instead of Python2.7"
           command: |
@@ -97,9 +159,23 @@ jobs:
           name: "Test Docker images"
           no_output_timeout: 2m
           command: |
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock
+                export GRID_STATUS_MAX_ATTEMPTS=15
+                export PLATFORM=linux/arm/v7
+            fi
             export USE_RANDOM_USER=false
-            export BRANCH=${CIRCLE_BRANCH//\//-}
+            #export BRANCH=${CIRCLE_BRANCH//\//-}
+            export BRANCH=4.3.0
+            export BUILD_DATE=20220624
             USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch
+            # docker run --rm --privileged aptman/qus -- -r
+            # docker run --rm --privileged aptman/qus -s -- -p
+            #docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch
+            docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:4.4.0-20220814 arch
+            if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then
+                sh stop-vm.sh
+            fi
 
   deploy-multi-arch:
     parameters:
@@ -184,24 +260,33 @@ jobs:
 workflows:
   build-and-test-multi-arch:
     jobs:
-      - build-multi-arch:
-          name: build-multi-arch-arm64
-          platforms: linux/arm64
-          machine-type: ubuntu2004arm64
-      - build-multi-arch:
-          name: build-multi-arch-amd64
-          platforms: linux/amd64
-          machine-type: ubuntu2004amd64
-      - test-multi-arch:
-          name: test-multi-arch-arm64
-          requires: [build-multi-arch-arm64]
-          platforms: linux/arm64
-          machine-type: ubuntu2004arm64
+      # - build-multi-arch:
+      #     name: build-multi-arch-arm64
+      #     platforms: linux/arm64
+      #     machine-type: ubuntu2004arm64
+      # - build-multi-arch:
+      #     name: build-multi-arch-amd64
+      #     platforms: linux/amd64
+      #     machine-type: ubuntu2004amd64
+      # - build-multi-arch:
+      #     name: build-multi-arch-armv7l
+      #     platforms: linux/arm/v7
+      #     machine-type: ubuntu2004arm64
+      # - test-multi-arch:
+      #     name: test-multi-arch-arm64
+      #     requires: [build-multi-arch-arm64]
+      #     platforms: linux/arm64
+      #     machine-type: ubuntu2004arm64
+      # - test-multi-arch:
+      #     name: test-multi-arch-amd64
+      #     requires: [build-multi-arch-amd64]
+      #     platforms: linux/amd64
+      #     machine-type: ubuntu2004amd64
       - test-multi-arch:
-          name: test-multi-arch-amd64
-          requires: [build-multi-arch-amd64]
-          platforms: linux/amd64
-          machine-type: ubuntu2004amd64
+          name: test-multi-arch-armv7l
+          # requires: [build-multi-arch-armv7l]
+          platforms: linux/arm/v7
+          machine-type: ubuntu2004arm64-large
 
   deploy-multi-arch:
     jobs:
@@ -214,4 +299,3 @@ workflows:
             branches: 
               only: 
                 - trunk
-

Makefile

@@ -393,7 +393,8 @@ test_firefox_standalone:
 
 
 # Test multi-arch container images
-test_multi_arch: test_chromium_multi \
+test_multi_arch: qemu_user_static \
+ test_chromium_multi \
  test_firefox_multi \
  test_chromium_standalone_multi \
  test_firefox_standalone_multi

tests/test.py

@@ -3,6 +3,7 @@
 import random
 import sys
 import unittest
+import time
 import re
 
 import docker
@@ -23,6 +24,7 @@
 https_proxy = os.environ.get('https_proxy', '')
 no_proxy = os.environ.get('no_proxy', '')
 SKIP_BUILD = os.environ.get('SKIP_BUILD', False)
+PLATFORM = os.environ.get('PLATFORM', '')
 
 IMAGE_NAME_MAP = {
     # Hub
@@ -111,6 +113,12 @@ def prune_networks():
     client.networks.prune()
 
 
+def wait_for_grid(platform):
+    if platform != '':
+        logger.info('Starting container on platform %s so we will wait 10 more seconds since emulation is slower...' % platform)
+        time.sleep(10)
+
+
 def launch_container(container, **kwargs):
     """
     Launch a specific container
@@ -176,6 +184,7 @@ def standalone_browser_container_matches(container):
     use_random_user_id = USE_RANDOM_USER_ID == 'true'
     run_in_docker_compose = RUN_IN_DOCKER_COMPOSE == 'true'
     random_user_id = random.randint(100000, 2147483647)
+    platform = PLATFORM 
 
     if use_random_user_id:
         logger.info("Running tests with a random user ID -> %s" % random_user_id)
@@ -197,9 +206,12 @@ def standalone_browser_container_matches(container):
             """
             ports = {'4444': 4444}
             if use_random_user_id:
-               test_container_id = launch_container(image, ports=ports, user=random_user_id)
+               test_container_id = launch_container(image, ports=ports, user=random_user_id, platform=platform)
             else:
-               test_container_id = launch_container(image, ports=ports)
+               test_container_id = launch_container(image, ports=ports,platform=platform)
+
+            wait_for_grid(platform)
+
         else:
             """
             Hub / Node Configuration
@@ -209,11 +221,13 @@ def standalone_browser_container_matches(container):
             hub_id = launch_hub("grid")
             ports = {'5555': 5555, '7900': 7900}
             if use_random_user_id:
-               test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id)
+               test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id, platform=platform)
             else:
-               test_container_id = launch_container(image, network='grid', ports=ports)
+               test_container_id = launch_container(image, network='grid', ports=ports, platform=platform)
             prune_networks()
 
+            wait_for_grid(platform)
+
         logger.info('========== / Containers ready to go ==========')
 
     try: