From 3e0041a66685d3224bf068990837dd730fc55128 Mon Sep 17 00:00:00 2001 From: Ioannis Bonatakis Date: Mon, 6 Oct 2025 14:01:06 +0200 Subject: [PATCH] Schedule product per livepatch kernel_version Automate the livepatch isos posts. Due to the logic of the script generation this commit inserts a for loop for all the livepatches which will be scrapped from test repository. - the loop always run even if no livepatches found. However it will only append to the isos posts the variables only in the case that actually livepatch versions are available. To get the delta of the increment repo, it checks the datetime. - Use the same uri which is used for rsync. For that, some manipulation is needed to convert the rsync to a regular http URI. The approach is just a bit dirty, as any other approach would require further refactoring and bigger changes. depends-on: https://gitlab.suse.de/openqa/openqa-trigger-from-ibs-plugin/-/merge_requests/263 issue: https://progress.opensuse.org/issues/190134 Signed-off-by: Ioannis Bonatakis --- .circleci/config.yml | 2 +- script/cfg.py | 109 +++++++++++++++++++++- script/scriptgen.py | 7 ++ t/podman/lib/test-in-container-systemd.sh | 3 +- 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a495dacc..ffdeb277 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: - checkout - run: command: | - zypper -n in --no-recommends diffutils make python3 tar git-core + zypper -n in --no-recommends diffutils make python3 tar git-core python3-requests python3-beautifulsoup4 - checkout - run: git clean -df - run: make test_regen_all diff --git a/script/cfg.py b/script/cfg.py index 33d6c82e..377fbfcc 100644 --- a/script/cfg.py +++ b/script/cfg.py @@ -1,4 +1,8 @@ import os +import re +import requests +from bs4 import BeautifulSoup +from datetime import datetime header = '''# GENERATED FILE - DO NOT EDIT set -e @@ -306,10 +310,100 @@ def openqa_call_start_meta_variables(meta_variables): def pre_openqa_call_start(repos): return '' -openqa_call_start = lambda distri, version, archs, staging, news, news_archs, flavor_distri, meta_variables, assets_flavor, repo0folder, openqa_cli: ''' +def _find_latest_livepatches(url): + """ + Scrapes a URL to find all kernel-livepatch-* with the + most recent date. + """ + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.RequestException as e: + print(f"Error fetching URL: {e}") + return [] + + soup = BeautifulSoup(response.text, 'html.parser') + + livepatches_unfiltered = [] + latest_date = None + lp_regex = r'kernel-livepatch-((?:\d+_?)+-\d+)' + #pdb.set_trace() + for link in soup.find_all('a'): + if (link and link.text.startswith('kernel-livepatch-')): + #and '-rt' in link.text): + #pdb.set_trace() + if link.next_sibling and isinstance(link.next_sibling, str): + date_str = link.next_sibling.strip().split()[0] + + try: + # NOTE: The date format on dist.suse.de is YYYY-MM-DD. + current_date = datetime.strptime(date_str, '%Y-%m-%d') + livepatches_unfiltered.append({'name': link.text, 'date': current_date}) + + if latest_date is None or current_date > latest_date: + latest_date = current_date + except (ValueError, IndexError): + # Continue if the date format is unexpected or not present + continue + + if latest_date is None: + return [] + # to list only the recent ones + latest_patches_list = [] + for patch in livepatches_unfiltered: + if patch['date'] == latest_date: + kernel_version = re.search(lp_regex, patch['name']) + latest_patches_list.append(kernel_version.group(1)) + + return latest_patches_list + +def _find_product_folder(p): + """ + Tries to find the product name looking it up in the server's repo dirs + :param p: The local path as defined in ActionGenerator.envdir + :return: The matched product name + """ + uri, prod = os.path.split(p) + target_filename = "files_iso.lst" + iso_file_found = None + + for root, dirs, files in os.walk(p): + if target_filename in files: + iso_file_found = os.path.join(root, target_filename) + break # found + + if iso_file_found: + with open(iso_file_found, 'r') as f: + iso_file = f.readline().strip() + if iso_file.endswith("spdx.json"): + prod_folder_pattern = r'(.*)-(?:aarch64|x86_64|s390x|ppc64le)' + prod_folder = re.match(prod_folder_pattern, iso_file) + if prod_folder: + return prod_folder.group(1) + return "" + +def openqa_fetch_livepatch_list(ag_vars): + kpatches = {} + productpath_local = ag_vars.envdir + prod_build_name = _find_product_folder(productpath_local) + if not prod_build_name: + return # no spdx.json, return and do what you do + repo_path = ag_vars.productrepopath().replace('::', '/', 1) + repo_path = repo_path.replace("repos", ag_vars.brand) + repo_path = f"https://{repo_path}" + for arch in ['x86_64','aarch64','s390x','ppc64le']: + totest_url = f"{repo_path}/{prod_build_name}-{arch}/{arch}/" + latest_kernel_livepatches = _find_latest_livepatches(totest_url) + + if latest_kernel_livepatches: + kpatches.update({arch: latest_kernel_livepatches}) + return kpatches + +openqa_call_start = lambda distri, version, archs, staging, news, news_archs, flavor_distri, meta_variables, assets_flavor, repo0folder, openqa_cli, livepatches={},: ''' archs=(ARCHITECTURS) [ ! -f __envsub/files_repo.lst ] || ! grep -q -- "-POOL-" __envsub/files_repo.lst || additional_repo_suffix=-POOL - +declare -a livepatches +livepatches=(NONE) for flavor in {FLAVORALIASLIST,}; do for arch in "${archs[@]}"; do filter=$flavor @@ -326,6 +420,11 @@ def pre_openqa_call_start(repos): [ -n "$iso" ] || [ "$flavor" != "''' + assets_flavor + r'''" ] || buildex=$(grep -o -E '(Build|Snapshot)[^-]*' __envsub/files_asset.lst | head -n 1) [ -n "$iso$build" ] || build=$(grep -h -o -E '(Build|Snapshot)[^-]*' __envsub/Media1*.lst 2>/dev/null | head -n 1 | grep -o -E '[0-9]\.?[0-9]+(\.[0-9]+)*')|| : [ -n "$build" ] || continue + livepatches_all="''' + str(livepatches) + r'''" + livepatches=($(echo "$livepatches_all" | sed "s/[{}']//g" | sed 's/, /\n/g' | awk -F':' -v arch="$arch" '$1 == arch {print $2}')) + if [ ${#livepatches[@]} -eq 0 ]; then + livepatches=(NONE) + fi buildex=${buildex/.install.iso/} buildex=${buildex/.iso/} buildex=${buildex/.raw.xz/} @@ -342,7 +441,12 @@ def pre_openqa_call_start(repos): } fi # test "$destiso" != "" || continue + for livepatch in "${livepatches[@]}"; do echo "''' + openqa_cli + ''' \\\\\" + if [[ "${livepatch}" != "NONE" ]]; then + echo \" KGRAFT=1 \\\\ + KERNEL_VERSION=${livepatch} \\\\\" + fi ( echo \" DISTRI=$distri \\\\ ARCH=$arch \\\\ @@ -583,6 +687,7 @@ def openqa_call_end(version): echo " FLAVOR=${flavor//Tumbleweed-/} \\\\" ) | LANG=C.UTF-8 sort echo "" + done done done ''' diff --git a/script/scriptgen.py b/script/scriptgen.py index f7603f8e..db26db72 100644 --- a/script/scriptgen.py +++ b/script/scriptgen.py @@ -1008,6 +1008,12 @@ def gen_print_openqa(self, f): "| grep {} | grep $arch | head -n 1".format(self.mask), ) else: + livepatch_updates = cfg.openqa_fetch_livepatch_list(self.ag) + openqa_livepatches = {} + if livepatch_updates: + openqa_livepatches = { + f"{arch}:{' '.join(versions)}" for arch, versions in livepatch_updates.items() if versions + } self.p( cfg.openqa_call_start( self.ag.distri, @@ -1021,6 +1027,7 @@ def gen_print_openqa(self, f): self.assets_flavor, self.repo0folder, self.ag.openqa_cli, + openqa_livepatches, ), f, ) diff --git a/t/podman/lib/test-in-container-systemd.sh b/t/podman/lib/test-in-container-systemd.sh index 5afb1e92..e804bde9 100755 --- a/t/podman/lib/test-in-container-systemd.sh +++ b/t/podman/lib/test-in-container-systemd.sh @@ -58,7 +58,8 @@ until [ $counter -gt 10 ]; do done podman exec "$containername" pwd >& /dev/null || (echo Cannot start container; exit 1 ) >&2 - +# install python dependencies +podman exec "$containername" bash zypper -y in python3-requests python3-beautifulsoup4 podman exec "$containername" bash /opt/init-trigger-from-obs.sh [ -z "$CIRCLE_JOB" ] || echo 'aa-complain /usr/share/openqa/script/openqa' | podman exec "$containername" bash -x