From 94bc41e6d3e9f3b7a2b3b9acbd9915e329935978 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Fri, 23 May 2025 14:32:08 -0400 Subject: [PATCH 1/5] [KRU] Allow rolling release to be used across centos Previously the tooling would find the common RESF tag but when upgrading minor versions it would not search across centos versions. --- rolling-release-update.py | 76 +++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/rolling-release-update.py b/rolling-release-update.py index c99dd14..9a65b4a 100644 --- a/rolling-release-update.py +++ b/rolling-release-update.py @@ -25,7 +25,7 @@ def find_common_tag(old_tags, new_tags): return None -def get_branch_tag_sha_list(repo, branch): +def get_branch_tag_sha_list(repo, branch, minor_version=False): print("[rolling release update] Checking out branch: ", branch) repo.git.checkout(branch) results = subprocess.run( @@ -37,8 +37,17 @@ def get_branch_tag_sha_list(repo, branch): print("[rolling release update] Gathering all the RESF kernel Tags") tags = [] + last_resf_tag = b"" for line in results.stdout.split(b"\n"): if b"tag: resf_kernel" in line: + if DEBUG: + print(line) + tags.append(line.split(b" ")[0]) + if last_resf_tag == b"": + last_resf_tag = line.split(b" ")[0] + if minor_version and b"tag: kernel-" in line: + if DEBUG: + print(line) tags.append(line.split(b" ")[0]) # Print summary instead of all tags @@ -48,7 +57,7 @@ def get_branch_tag_sha_list(repo, branch): for line_tag in tags: print(f" {line_tag.decode()}") - return tags + return tags, last_resf_tag def check_for_fips_protected_changes(repo, branch, common_tag): @@ -142,6 +151,12 @@ def check_for_fips_protected_changes(repo, branch, common_tag): parser.add_argument( "--verbose-git-show", help="When SHAs are detected for removal do the full git show ", action="store_true" ) + parser.add_argument( + "--new-minor-version", + help="Do not stop at the RESF tags, continue down the CENTOS / ROCKY MAIN branch." + " This is used for the new minor version releases", + action="store_true", + ) parser.add_argument( "--demo", help="DEMO mode, will make a new set of branches with demo_ prepended", action="store_true" ) @@ -164,42 +179,51 @@ def check_for_fips_protected_changes(repo, branch, common_tag): rolling_product = args.old_rolling_branch.split("/")[0] print("[rolling release update] Rolling Product: ", rolling_product) - old_rolling_branch_tags = get_branch_tag_sha_list(repo, args.old_rolling_branch) + if args.new_minor_version: + print("[rolling release update] New Minor Version: ", args.new_minor_version) + + old_rolling_branch_tags, old_rolling_resf_tag_sha = get_branch_tag_sha_list( + repo, args.old_rolling_branch, args.new_minor_version + ) if DEBUG: print("[rolling release update] Old Rolling Branch Tags: ", old_rolling_branch_tags) - new_base_branch_tags = get_branch_tag_sha_list(repo, args.new_base_branch) + new_base_branch_tags, new_base_resf_tag_sha = get_branch_tag_sha_list( + repo, args.new_base_branch, args.new_minor_version + ) if DEBUG: print("[rolling release update] New Base Branch Tags: ", new_base_branch_tags) - latest_resf_sha = find_common_tag(old_rolling_branch_tags, new_base_branch_tags) - print("[rolling release update] Latest RESF tag sha: ", latest_resf_sha) - print(repo.git.show('--pretty="%H %s"', "-s", latest_resf_sha.decode())) - - print("[rolling release update] Checking for FIPS protected changes between the common tag and HEAD") - shas_to_check = check_for_fips_protected_changes(repo, args.new_base_branch, latest_resf_sha) - if shas_to_check and args.fips_override is False: - for sha, dir in shas_to_check.items(): - print(f"## Commit {sha.decode()}") - print("'''") - dir_list = [] - for d in dir: - dir_list.append(d.decode()) - print(repo.git.show(sha.decode(), dir_list)) - print("'''") - print("[rolling release update] FIPS protected changes found between the common tag and HEAD") - print("[rolling release update] Please Contact the CIQ FIPS / Security team for further instructions") - print("[rolling release update] Exiting") - exit(1) + common_sha = find_common_tag(old_rolling_branch_tags, new_base_branch_tags) + print("[rolling release update] Common tag sha: ", common_sha) + print(repo.git.show('--pretty="%H %s"', "-s", common_sha.decode())) + + if "fips" in rolling_product: + print("[rolling release update] Checking for FIPS protected changes between the common tag and HEAD") + shas_to_check = check_for_fips_protected_changes(repo, args.new_base_branch, common_sha) + if shas_to_check and args.fips_override is False: + for sha, dir in shas_to_check.items(): + print(f"## Commit {sha.decode()}") + print("'''") + dir_list = [] + for d in dir: + dir_list.append(d.decode()) + print(repo.git.show(sha.decode(), dir_list)) + print("'''") + print("[rolling release update] FIPS protected changes found between the common tag and HEAD") + print("[rolling release update] Please Contact the CIQ FIPS / Security team for further instructions") + print("[rolling release update] Exiting") + exit(1) print("[rolling release update] Checking out old rolling branch: ", args.old_rolling_branch) repo.git.checkout(args.old_rolling_branch) print( "[rolling release update] Finding the CIQ Kernel and Associated Upstream commits between the last resf tag and HEAD" ) + print(f"[rolling release update] Getting SHAS {old_rolling_resf_tag_sha.decode()}..HEAD") rolling_commit_map = {} rollint_commit_map_rev = {} - rolling_commits = repo.git.log(f"{latest_resf_sha.decode()}..HEAD") + rolling_commits = repo.git.log(f"{old_rolling_resf_tag_sha.decode()}..HEAD") for line in rolling_commits.split("\n"): if line.startswith("commit "): ciq_commit = line.split("commit ")[1] @@ -209,7 +233,7 @@ def check_for_fips_protected_changes(repo, branch, common_tag): rolling_commit_map[ciq_commit] = upstream_commit rollint_commit_map_rev[upstream_commit] = ciq_commit - print("[rolling release update] Last RESF tag sha: ", latest_resf_sha) + print("[rolling release update] Last RESF tag sha: ", common_sha) print(f"[rolling release update] Total commits in old branch: {len(rolling_commit_map)}") if DEBUG: @@ -284,7 +308,7 @@ def check_for_fips_protected_changes(repo, branch, common_tag): print("[rolling release update] Creating Map of all new commits from last rolling release fork") new_base_commit_map = {} new_base_commit_map_rev = {} - new_base_commits = repo.git.log(f"{latest_resf_sha.decode()}..HEAD") + new_base_commits = repo.git.log(f"{common_sha.decode()}..HEAD") for line in new_base_commits.split("\n"): if line.startswith("commit "): ciq_commit = line.split("commit ")[1] From c28c53ee62c1cd7f0a69d4419f736f3d88e0bc8c Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Mon, 3 Nov 2025 15:42:33 -0500 Subject: [PATCH 2/5] [RR] Add Interactive settings The new feature allows a user to run in interactive mode where the rolling release will "pause" while the engineer fixes the failed forward-port and commit. It will then attempt to continue. --- rolling-release-update.py | 79 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/rolling-release-update.py b/rolling-release-update.py index 9a65b4a..85abb62 100644 --- a/rolling-release-update.py +++ b/rolling-release-update.py @@ -161,6 +161,9 @@ def check_for_fips_protected_changes(repo, branch, common_tag): "--demo", help="DEMO mode, will make a new set of branches with demo_ prepended", action="store_true" ) parser.add_argument("--debug", help="Enable debug output", action="store_true") + parser.add_argument( + "--interactive", help="Interactive mode - pause on merge conflicts for user resolution", action="store_true" + ) args = parser.parse_args() if args.demo: @@ -366,6 +369,80 @@ def check_for_fips_protected_changes(repo, branch, common_tag): if result.returncode != 0: print(f"[rolling release update] ERROR: Failed to cherry-pick commit {ciq_commit}") print(result.stderr.decode("utf-8")) - exit(1) + + if args.interactive: + print("[rolling release update] ========================================") + print("[rolling release update] INTERACTIVE MODE: Merge conflict detected") + print("[rolling release update] ========================================") + print("[rolling release update] Please resolve or skip the merge conflict manually.") + print("[rolling release update] To resolve:") + print("[rolling release update] 1. Fix merge conflicts in the working directory") + print("[rolling release update] 2. Stage resolved files: git add ") + print("[rolling release update] 3. Complete cherry-pick: git cherry-pick --continue") + print("[rolling release update] (or commit manually if needed)") + print("[rolling release update] To skip:") + print("[rolling release update] 1. To skip this commit: git cherry-pick --skip") + print("[rolling release update] When done:") + print("[rolling release update] Return here and press Enter to continue") + print("[rolling release update] ========================================") + + # Loop until conflict is resolved or user aborts + while True: + user_input = input( + '[rolling release update] Press Enter when resolved (or type "stop"/"abort" to exit): ' + ).strip().lower() + + if user_input in ["stop", "abort"]: + print("[rolling release update] ========================================") + print("[rolling release update] User aborted. Remaining commits to forward port:") + print("[rolling release update] ========================================") + + # Print remaining commits including the current failed one + remaining_commits = list(reversed(rolling_commit_map.items())) + start_idx = remaining_commits.index((ciq_commit, upstream_commit)) + + for remaining_commit, remaining_upstream in remaining_commits[start_idx:]: + short_sha = repo.git.rev_parse("--short", remaining_commit) + summary = repo.git.show("--pretty=%s", "-s", remaining_commit) + print(f" {short_sha} {summary}") + + print("[rolling release update] ========================================") + print(f"[rolling release update] Total remaining: {len(remaining_commits) - start_idx} commits") + exit(1) + + # Verify the cherry-pick was completed successfully + # Check if CHERRY_PICK_HEAD still exists (indicates incomplete cherry-pick) + cherry_pick_head = os.path.join(args.repo, ".git", "CHERRY_PICK_HEAD") + if os.path.exists(cherry_pick_head): + print("[rolling release update] ERROR: Cherry-pick not completed (.git/CHERRY_PICK_HEAD still exists)") + print("[rolling release update] Please complete the cherry-pick with:") + print("[rolling release update] git cherry-pick --continue") + print("[rolling release update] or abort with:") + print("[rolling release update] git cherry-pick --abort") + print('[rolling release update] Type "stop" or "abort" to exit, or press Enter to check again') + continue + + # Check for uncommitted changes + status_result = subprocess.run( + ["git", "status", "--porcelain"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=args.repo + ) + if status_result.returncode != 0: + print("[rolling release update] ERROR: Could not check git status") + print('[rolling release update] Type "stop" or "abort" to exit, or press Enter to check again') + continue + + if status_result.stdout.strip(): + print("[rolling release update] ERROR: There are still uncommitted changes") + print("[rolling release update] Status:") + print(status_result.stdout.decode("utf-8")) + print("[rolling release update] Please commit or stash changes before continuing") + print('[rolling release update] Type "stop" or "abort" to exit, or press Enter to check again') + continue + + # If we got here, everything is resolved + print("[rolling release update] Cherry-pick resolved successfully, continuing...") + break + else: + exit(1) print(f"[rolling release update] Successfully applied all {commits_applied} commits") From 943ed08314582759a61eb5b90501d87d46c281f6 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Wed, 26 Nov 2025 15:00:28 -0500 Subject: [PATCH 3/5] [RR] Fix CentOS rebase check One of the critical items as a fork of RedHat is that our kernel updates out from under us and some of the features/fixes/etc we backport in one version of RLC might need to be dropped on the next update. The latest minor update, to 10.1, rebase missed some items that should have been dropped due being present in in CentOS streams branch but not a part of the RHEL minor version updates. WIWT: Fixed some debugging. --- rolling-release-update.py | 91 ++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/rolling-release-update.py b/rolling-release-update.py index 85abb62..f3ada6f 100644 --- a/rolling-release-update.py +++ b/rolling-release-update.py @@ -6,6 +6,8 @@ import git +from ciq_helpers import get_backport_commit_data + FIPS_PROTECTED_DIRECTORIES = [ b"arch/x86/crypto/", b"crypto/asymmetric_keys/", @@ -25,6 +27,55 @@ def find_common_tag(old_tags, new_tags): return None +def get_commit_maps_from_backport_data(repo_path, branch, common_tag): + """Get commit maps using get_backport_commit_data from ciq_helpers. + + This function properly parses all commits and extracts upstream references + from 'commit ' lines in commit bodies. This ensures we correctly identify + duplicates even when different CIQ commits reference the same upstream commit. + + Returns: + commit_map: dict mapping CIQ commit SHA -> upstream commit SHA (or "" if no upstream) + commit_map_rev: dict mapping upstream commit SHA -> CIQ commit SHA + """ + # get_backport_commit_data returns: + # { "upstream_sha": { "repo_commit": "ciq_sha", "upstream_subject": "...", ... } } + backport_data, success = get_backport_commit_data( + repo_path, + branch, + common_tag.decode() if isinstance(common_tag, bytes) else common_tag, + allow_duplicates=True + ) + + if not success: + print("[rolling release update] WARNING: Duplicate upstream commits detected in backport data") + print("[rolling release update] Continuing with allow_duplicates=True") + + # Transform to the format expected by the rest of the script + commit_map = {} + commit_map_rev = {} + + for upstream_sha, data in backport_data.items(): + ciq_sha = data["repo_commit"] + commit_map[ciq_sha] = upstream_sha + commit_map_rev[upstream_sha] = ciq_sha + + # Also get all commits (including those without upstream references) + # to ensure we have a complete list + repo = git.Repo(repo_path) + repo.git.checkout(branch) + common_tag_str = common_tag.decode() if isinstance(common_tag, bytes) else common_tag + all_commits = repo.git.log("--pretty=%H", f"{common_tag_str}..HEAD").split("\n") + + # Add commits without upstream references (CIQ-specific commits) + for commit_sha in all_commits: + commit_sha = commit_sha.strip() + if commit_sha and commit_sha not in commit_map: + commit_map[commit_sha] = "" + + return commit_map, commit_map_rev + + def get_branch_tag_sha_list(repo, branch, minor_version=False): print("[rolling release update] Checking out branch: ", branch) repo.git.checkout(branch) @@ -224,17 +275,9 @@ def check_for_fips_protected_changes(repo, branch, common_tag): "[rolling release update] Finding the CIQ Kernel and Associated Upstream commits between the last resf tag and HEAD" ) print(f"[rolling release update] Getting SHAS {old_rolling_resf_tag_sha.decode()}..HEAD") - rolling_commit_map = {} - rollint_commit_map_rev = {} - rolling_commits = repo.git.log(f"{old_rolling_resf_tag_sha.decode()}..HEAD") - for line in rolling_commits.split("\n"): - if line.startswith("commit "): - ciq_commit = line.split("commit ")[1] - rolling_commit_map[ciq_commit] = "" - if line.startswith(" commit "): - upstream_commit = line.split(" commit ")[1] - rolling_commit_map[ciq_commit] = upstream_commit - rollint_commit_map_rev[upstream_commit] = ciq_commit + rolling_commit_map, rolling_commit_map_rev = get_commit_maps_from_backport_data( + args.repo, args.old_rolling_branch, old_rolling_resf_tag_sha + ) print("[rolling release update] Last RESF tag sha: ", common_sha) @@ -309,17 +352,9 @@ def check_for_fips_protected_changes(repo, branch, common_tag): exit(1) print("[rolling release update] Creating Map of all new commits from last rolling release fork") - new_base_commit_map = {} - new_base_commit_map_rev = {} - new_base_commits = repo.git.log(f"{common_sha.decode()}..HEAD") - for line in new_base_commits.split("\n"): - if line.startswith("commit "): - ciq_commit = line.split("commit ")[1] - new_base_commit_map[ciq_commit] = "" - if line.startswith(" commit "): - upstream_commit = line.split(" commit ")[1] - new_base_commit_map[ciq_commit] = upstream_commit - new_base_commit_map_rev[upstream_commit] = ciq_commit + new_base_commit_map, new_base_commit_map_rev = get_commit_maps_from_backport_data( + args.repo, f"{os.getlogin()}_{new_rolling_branch_kernel}", common_sha + ) print(f"[rolling release update] Total commits in new branch: {len(new_base_commit_map)}") if DEBUG: @@ -336,14 +371,18 @@ def check_for_fips_protected_changes(repo, branch, common_tag): ) commits_to_remove = {} for ciq_commit, upstream_commit in rolling_commit_map.items(): - if upstream_commit in new_base_commit_map_rev: + if upstream_commit and upstream_commit in new_base_commit_map_rev: + new_base_ciq_commit = new_base_commit_map_rev[upstream_commit] + print( + f"- Old commit {ciq_commit[:12]} backported upstream {upstream_commit[:12]}" + ) print( - f"- Commit {ciq_commit} already present in new base branch: {repo.git.show('--pretty=oneline', '-s', ciq_commit)}" + f" Already in new base as {new_base_ciq_commit[:12]}: {repo.git.show('--pretty=%s', '-s', new_base_ciq_commit)}" ) commits_to_remove[ciq_commit] = upstream_commit - if ciq_commit in new_base_commit_map: + elif ciq_commit in new_base_commit_map: print( - f"- CIQ Commit {ciq_commit} already present in new base branch: {repo.git.show('--pretty=oneline', '-s', ciq_commit)}" + f"- CIQ Commit {ciq_commit[:12]} already present in new base branch: {repo.git.show('--pretty=oneline', '-s', ciq_commit)}" ) commits_to_remove[ciq_commit] = upstream_commit From c4ded0cf3da82063a0d2565657b52ed297e984e3 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Wed, 26 Nov 2025 15:57:41 -0500 Subject: [PATCH 4/5] [RR] RUFF formating updates --- rolling-release-update.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/rolling-release-update.py b/rolling-release-update.py index f3ada6f..0f2e771 100644 --- a/rolling-release-update.py +++ b/rolling-release-update.py @@ -41,10 +41,7 @@ def get_commit_maps_from_backport_data(repo_path, branch, common_tag): # get_backport_commit_data returns: # { "upstream_sha": { "repo_commit": "ciq_sha", "upstream_subject": "...", ... } } backport_data, success = get_backport_commit_data( - repo_path, - branch, - common_tag.decode() if isinstance(common_tag, bytes) else common_tag, - allow_duplicates=True + repo_path, branch, common_tag.decode() if isinstance(common_tag, bytes) else common_tag, allow_duplicates=True ) if not success: @@ -373,9 +370,7 @@ def check_for_fips_protected_changes(repo, branch, common_tag): for ciq_commit, upstream_commit in rolling_commit_map.items(): if upstream_commit and upstream_commit in new_base_commit_map_rev: new_base_ciq_commit = new_base_commit_map_rev[upstream_commit] - print( - f"- Old commit {ciq_commit[:12]} backported upstream {upstream_commit[:12]}" - ) + print(f"- Old commit {ciq_commit[:12]} backported upstream {upstream_commit[:12]}") print( f" Already in new base as {new_base_ciq_commit[:12]}: {repo.git.show('--pretty=%s', '-s', new_base_ciq_commit)}" ) @@ -427,9 +422,11 @@ def check_for_fips_protected_changes(repo, branch, common_tag): # Loop until conflict is resolved or user aborts while True: - user_input = input( - '[rolling release update] Press Enter when resolved (or type "stop"/"abort" to exit): ' - ).strip().lower() + user_input = ( + input('[rolling release update] Press Enter when resolved (or type "stop"/"abort" to exit): ') + .strip() + .lower() + ) if user_input in ["stop", "abort"]: print("[rolling release update] ========================================") @@ -453,7 +450,9 @@ def check_for_fips_protected_changes(repo, branch, common_tag): # Check if CHERRY_PICK_HEAD still exists (indicates incomplete cherry-pick) cherry_pick_head = os.path.join(args.repo, ".git", "CHERRY_PICK_HEAD") if os.path.exists(cherry_pick_head): - print("[rolling release update] ERROR: Cherry-pick not completed (.git/CHERRY_PICK_HEAD still exists)") + print( + "[rolling release update] ERROR: Cherry-pick not completed (.git/CHERRY_PICK_HEAD still exists)" + ) print("[rolling release update] Please complete the cherry-pick with:") print("[rolling release update] git cherry-pick --continue") print("[rolling release update] or abort with:") From 8c9baa6c29318236c73041c545267e4c9926bb02 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Wed, 26 Nov 2025 16:57:04 -0500 Subject: [PATCH 5/5] [RR] Fix up potential index exception --- rolling-release-update.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rolling-release-update.py b/rolling-release-update.py index 0f2e771..7b14ac7 100644 --- a/rolling-release-update.py +++ b/rolling-release-update.py @@ -435,7 +435,13 @@ def check_for_fips_protected_changes(repo, branch, common_tag): # Print remaining commits including the current failed one remaining_commits = list(reversed(rolling_commit_map.items())) - start_idx = remaining_commits.index((ciq_commit, upstream_commit)) + try: + start_idx = remaining_commits.index((ciq_commit, upstream_commit)) + except ValueError: + print("[rolling release update] ERROR: Current commit not found in remaining commits list.") + print("[rolling release update] This may indicate an internal error or unexpected state.") + print("[rolling release update] Aborting.") + exit(1) for remaining_commit, remaining_upstream in remaining_commits[start_idx:]: short_sha = repo.git.rev_parse("--short", remaining_commit)