From 0b762667d261ac2b9256911f1a91f67d423fdd3c Mon Sep 17 00:00:00 2001 From: chiragkyal Date: Fri, 6 Mar 2026 20:21:46 +0530 Subject: [PATCH 1/4] docs: Add RHCOS 10/RHEL 10 compatibility testing guide Add comprehensive testing guide for verifying cert-manager operator compatibility with RHCOS 10 and RHEL 10 on OpenShift 4.20, 4.21, and 4.22. The testing guide provides step-by-step procedures for: - Deployment verification - E2E test execution - Crypto library compatibility verification - Cloud provider integration testing - Troubleshooting and debug information collection The results template provides a standardized format for documenting test outcomes, including cluster information, test results, bugs found, and recommendations. Co-Authored-By: Claude Sonnet 4.5 --- docs/rhcos10-test-results-template.md | 369 ++++++++++++++++++ docs/rhcos10-testing.md | 537 ++++++++++++++++++++++++++ 2 files changed, 906 insertions(+) create mode 100644 docs/rhcos10-test-results-template.md create mode 100644 docs/rhcos10-testing.md diff --git a/docs/rhcos10-test-results-template.md b/docs/rhcos10-test-results-template.md new file mode 100644 index 000000000..95897f077 --- /dev/null +++ b/docs/rhcos10-test-results-template.md @@ -0,0 +1,369 @@ +# RHCOS 10 / RHEL 10 Compatibility Test Results + +> **Template Instructions:** Copy this template and fill in the details for each test run. + +--- + +## Test Information + +**Date:** YYYY-MM-DD +**Tester:** [Your Name] +**Tracking Issue:** [Link to your tracking issue/story] + +--- + +## Cluster Information + +**OpenShift Version:** [e.g., 4.20.1, 4.21.0, 4.22.0] +**RHCOS Version:** [e.g., Red Hat Enterprise Linux CoreOS 410.92.202X...] +**Platform:** [e.g., AWS, GCP, Azure, BareMetal] +**Cluster Name/ID:** [e.g., rhcos10-test-cluster] +**Region/Zone:** [if applicable, e.g., us-east-1] + +**FIPS Mode:** [ ] Enabled / [ ] Disabled + +--- + +## cert-manager Operator Version + +**Operator Version:** [e.g., 1.19.0] +**cert-manager Version:** [e.g., v1.19.2] +**Installation Method:** [ ] OLM / [ ] Manifests +**Subscription Channel:** [if OLM, e.g., stable-v1] + +--- + +## Test Results Summary + +| Test Category | Status | Notes | +|--------------|--------|-------| +| Deployment Verification | [ ] Pass / [ ] Fail | | +| E2E Test Suite | [ ] Pass / [ ] Fail / [ ] Partial | | +| Crypto Library Verification | [ ] Pass / [ ] Fail | | +| Integration Testing | [ ] Pass / [ ] Fail / [ ] N/A | | + +**Overall Status:** [ ] ✅ PASS / [ ] ❌ FAIL / [ ] ⚠️ BLOCKED + +--- + +## Detailed Test Results + +### 1. Deployment Verification + +**Status:** [ ] Pass / [ ] Fail + +#### Operator Deployment + +- **cert-manager-operator namespace:** [ ] Created +- **Operator deployment:** [ ] Available +- **Operator pods:** [ ] Running +- **Operator logs:** [ ] No errors / [ ] Errors found (details below) + +#### Operand Deployment + +- **cert-manager namespace:** [ ] Created +- **cert-manager deployment:** [ ] Available +- **cert-manager-webhook deployment:** [ ] Available +- **cert-manager-cainjector deployment:** [ ] Available +- **All pods running:** [ ] Yes / [ ] No + +#### Issues Found +``` +[Describe any deployment issues here, or write "None"] +``` + +--- + +### 2. E2E Test Suite Results + +**Status:** [ ] Pass / [ ] Fail / [ ] Partial + +**Test Command Used:** +```bash +make test-e2e +# or with filters: +# E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {AWS}' make test-e2e +``` + +**Test Duration:** [e.g., 45 minutes] + +**Test Results:** +- **Total Tests:** [number] +- **Passed:** [number] +- **Failed:** [number] +- **Skipped:** [number] + +#### Failed Tests (if any) + +| Test Name | Failure Reason | JIRA Bug (if filed) | +|-----------|---------------|---------------------| +| [Test 1] | [Reason] | [BUG-XXX] | +| [Test 2] | [Reason] | [BUG-XXX] | + +#### Test Execution Notes +``` +[Add any relevant notes about test execution, e.g., flaky tests, environmental issues, etc.] +``` + +--- + +### 3. Crypto Library Verification + +**Status:** [ ] Pass / [ ] Fail + +**Verification Command:** +```bash +make verify-rhcos10-crypto +``` + +#### OpenSSL Version + +**Node OpenSSL Version:** [e.g., OpenSSL 3.0.7 1 Nov 2022] +**Container OpenSSL Version:** [e.g., OpenSSL 3.0.7 1 Nov 2022] +**Expected:** OpenSSL 3.x for RHCOS 10 ✓ + +#### FIPS Mode Status + +**FIPS Enabled on Nodes:** [ ] Yes / [ ] No +**Value:** [0 or 1] + +If FIPS enabled: +- [ ] cert-manager respects FIPS mode +- [ ] Only FIPS-approved algorithms used +- [ ] No FIPS-related errors in logs + +#### TLS Connectivity + +- **TLS connection to Kubernetes API:** [ ] Success / [ ] Failed +- **Cipher used:** [e.g., TLS_AES_128_GCM_SHA256] +- **Protocol:** [e.g., TLSv1.3] + +#### Certificate Generation Tests + +| Algorithm | Key Size | Status | Notes | +|-----------|----------|--------|-------| +| RSA | 2048 | [ ] Pass / [ ] Fail | | +| RSA | 4096 | [ ] Pass / [ ] Fail | | +| ECDSA | P-256 | [ ] Pass / [ ] Fail | | +| ECDSA | P-384 | [ ] Pass / [ ] Fail | | + +#### Crypto Errors in Logs + +[ ] No crypto-related errors found +[ ] Crypto errors found (details below) + +``` +[Paste any crypto-related errors from logs, or write "None"] +``` + +--- + +### 4. Integration Testing + +**Status:** [ ] Pass / [ ] Fail / [ ] N/A + +#### Cloud Provider Integration (if applicable) + +**Provider:** [ ] AWS Route53 / [ ] GCP Cloud DNS / [ ] Azure DNS / [ ] N/A + +**Test Command:** +```bash +# Example for AWS: +# E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {AWS} && Issuer: isSubsetOf {ACME-DNS01}' make test-e2e +``` + +**Results:** +- **ACME DNS01 Issuer:** [ ] Pass / [ ] Fail / [ ] N/A +- **ACME HTTP01 Issuer:** [ ] Pass / [ ] Fail / [ ] N/A +- **Cloud credentials:** [ ] Working / [ ] Issues + +**Notes:** +``` +[Add notes about cloud provider integration testing] +``` + +#### Vault Integration (if tested) + +- **Vault Issuer:** [ ] Pass / [ ] Fail / [ ] N/A + +**Notes:** +``` +[Add notes about Vault integration testing] +``` + +--- + +## Issues and Bugs + +### Bugs Filed + +| JIRA ID | Summary | Severity | Status | +|---------|---------|----------|--------| +| [BUG-XXX] | [Bug summary] | [ ] Critical / [ ] Major / [ ] Minor | [ ] New / [ ] In Progress | +| [BUG-XXX] | [Bug summary] | [ ] Critical / [ ] Major / [ ] Minor | [ ] New / [ ] In Progress | + +### Known Issues / Workarounds + +``` +[Document any known issues and workarounds here, or write "None"] + +Example: +- Issue: E2E test TestXYZ flakes occasionally + Workaround: Rerun tests; appears to be timing-related + Bug: BUG-XXX +``` + +--- + +## Performance and Resource Usage + +### Pod Resource Usage + +| Pod | CPU (avg) | Memory (avg) | Status | +|-----|-----------|--------------|--------| +| cert-manager-operator-controller-manager | [e.g., 10m] | [e.g., 50Mi] | Normal / High | +| cert-manager | [e.g., 20m] | [e.g., 100Mi] | Normal / High | +| cert-manager-webhook | [e.g., 5m] | [e.g., 30Mi] | Normal / High | +| cert-manager-cainjector | [e.g., 10m] | [e.g., 50Mi] | Normal / High | + +**Resource usage compared to RHCOS 9:** +``` +[Compare if you have baseline from RHCOS 9, or write "N/A - no baseline available"] +``` + +--- + +## Observations and Notes + +### RHCOS 10 Specific Observations + +``` +[Document any RHCOS 10 specific behavior, issues, or improvements observed] + +Examples: +- Faster TLS handshakes with OpenSSL 3.x +- Different default cipher suites +- FIPS mode behavior changes +- etc. +``` + +### Differences from RHCOS 9 (if known) + +``` +[Document any differences observed compared to RHCOS 9, or write "N/A"] +``` + +### Recommendations + +``` +[Add any recommendations for users, documentation updates, or future improvements] +``` + +--- + +## Supporting Evidence + +### Logs and Diagnostics + +**Location of collected diagnostics:** [e.g., _output/diagnostics/] + +**Key files:** +- Cluster version: [file path or attachment] +- Node information: [file path or attachment] +- Operator logs: [file path or attachment] +- Test results: [file path or attachment] +- Crypto verification report: [file path or attachment] + +### Screenshots (optional) + +[Attach screenshots if relevant, e.g., dashboard views, test results, error messages] + +--- + +## Conclusion + +### Summary + +``` +[Provide a 2-3 sentence summary of the test results] + +Example: +cert-manager operator was successfully deployed and tested on OpenShift 4.22 with RHCOS 10. +All e2e tests passed, and crypto library verification confirmed compatibility with OpenSSL 3.x. +No critical issues were found, and the component is ready for OpenShift 4.22 release. +``` + +### Recommendation + +[ ] **Approved for Release** - No blocking issues found +[ ] **Conditional Approval** - Minor issues found, documented with workarounds +[ ] **Not Ready** - Critical issues must be resolved before release +[ ] **Blocked** - Cannot complete testing due to [reason] + +### Sign-off + +**Tested By:** [Your Name] +**Date:** YYYY-MM-DD +**Signature:** [Your signature or approval] + +--- + +## References + +- **Testing Guide:** [docs/rhcos10-testing.md](rhcos10-testing.md) +- **Project Repository:** [cert-manager-operator](https://github.com/openshift/cert-manager-operator) + +--- + +## Appendix: Command Reference + +### Quick Test Commands + +```bash +# Deploy operator +make deploy + +# Wait for stable state +make test-e2e-wait-for-stable-state + +# Run all compatibility tests +make test-rhcos10 + +# Run only crypto verification +make verify-rhcos10-crypto + +# Run e2e tests +make test-e2e + +# Collect diagnostics +oc get nodes -o wide > nodes.txt +oc get pods -n cert-manager-operator > operator-pods.txt +oc get pods -n cert-manager > operand-pods.txt +oc logs deployment/cert-manager -n cert-manager --tail=100 > cert-manager-logs.txt +``` + +### Useful Debug Commands + +```bash +# Check RHCOS version +oc get nodes -o json | jq -r '.items[] | {name: .metadata.name, os: .status.nodeInfo.osImage}' + +# Check FIPS mode +NODE=$(oc get nodes -o name | head -1) +oc debug $NODE -- chroot /host cat /proc/sys/crypto/fips_enabled + +# Check OpenSSL version on node +oc debug $NODE -- chroot /host openssl version -a + +# Check cert-manager pod status +oc get pods -n cert-manager -o wide +oc describe pod -n cert-manager + +# Check for crypto errors in logs +oc logs deployment/cert-manager -n cert-manager | grep -iE "(error|crypto|ssl|tls|fips)" +``` + +--- + +*Template Version: 1.0* +*Last Updated: 2026-03-06* diff --git a/docs/rhcos10-testing.md b/docs/rhcos10-testing.md new file mode 100644 index 000000000..583a0e8fb --- /dev/null +++ b/docs/rhcos10-testing.md @@ -0,0 +1,537 @@ +# RHCOS 10 / RHEL 10 Compatibility Testing Guide + +This guide provides comprehensive instructions for testing cert-manager operator compatibility with RHCOS 10 and RHEL 10 on OpenShift 4.20, 4.21, and 4.22. + +## Overview + +As part of the RHCOS10/RHEL10 readiness effort for OpenShift, the cert-manager operator must be verified to work correctly with: +- Red Hat CoreOS (RHCOS) 10 +- Red Hat Enterprise Linux (RHEL) 10 +- Updated crypto libraries and dependencies +- OpenShift Container Platform versions 4.20, 4.21, and 4.22 + +## Prerequisites + +### Required Access +- OpenShift cluster running RHCOS 10 (versions 4.20, 4.21, or 4.22) +- Cluster administrator privileges + +### Required Tools +- `oc` CLI (OpenShift command-line tool) +- `kubectl` CLI +- `make` (for running automation targets) +- `bash` shell (version 4.0 or higher) +- `jq` (for JSON processing) +- `curl` (for API testing) + +### Verification of RHCOS 10 Environment + +Before beginning testing, verify the cluster is running RHCOS 10: + +```bash +# Check RHCOS version on all nodes +oc get nodes -o json | jq -r '.items[] | {name: .metadata.name, os: .status.nodeInfo.osImage, kernel: .status.nodeInfo.kernelVersion}' + +# Expected output should show RHCOS 10.x +# Example: "Red Hat Enterprise Linux CoreOS 410.92.202X... (CoreOS)" +``` + +Check OpenShift version: + +```bash +oc version +# Should show 4.20.x, 4.21.x, or 4.22.x +``` + +## Testing Scope + +### 1. Deployment Testing +Verify cert-manager operator and operands deploy successfully on RHCOS 10. + +### 2. Functionality Testing +Run comprehensive e2e tests to ensure all cert-manager features work correctly. + +### 3. Crypto Library Testing +Verify compatibility with RHCOS 10 crypto libraries, including: +- OpenSSL version and FIPS mode +- TLS connectivity +- Certificate generation with various algorithms (RSA, ECDSA) +- Certificate validation and verification + +### 4. Integration Testing +Verify cert-manager integrates correctly with: +- OpenShift API server +- OpenShift routing +- Cloud provider APIs (AWS Route53, GCP Cloud DNS, Azure DNS) +- Vault (if applicable) + +## Step-by-Step Testing Procedures + +### Phase 1: Pre-Deployment Verification + +#### 1.1 Verify Cluster Prerequisites + +```bash +# Check cluster operators are healthy +oc get co + +# Verify no degraded operators +oc get co | grep -i degraded + +# Check cluster version +oc get clusterversion +``` + +#### 1.2 Verify RHCOS 10 Node Status + +```bash +# Check all nodes are ready +oc get nodes + +# Verify RHCOS version on each node +for node in $(oc get nodes -o name); do + echo "=== $node ===" + oc debug $node -- chroot /host sh -c "cat /etc/os-release | grep -E '(PRETTY_NAME|VERSION_ID)'" +done +``` + +#### 1.3 Check Crypto Libraries + +Run the automated crypto verification script: + +```bash +make verify-rhcos10-crypto +``` + +Or manually check on a node: + +```bash +# Choose a worker node +NODE=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}') + +# Check OpenSSL version +oc debug node/$NODE -- chroot /host openssl version -a + +# Check FIPS mode status +oc debug node/$NODE -- chroot /host cat /proc/sys/crypto/fips_enabled +``` + +### Phase 2: Deployment Testing + +#### 2.1 Deploy cert-manager Operator + +If using OLM (production method): + +```bash +# Create operator namespace +oc create namespace cert-manager-operator + +# Create subscription (adjust based on your catalog source) +cat <&1 | grep -E "(SSL|TLS|cipher)" + +# Check OpenSSL version in container +oc exec -n cert-manager $CONTROLLER_POD -- openssl version -a +``` + +Verify certificate generation with different algorithms: + +```bash +# Extract and verify RSA certificate +oc get secret test-cert-rhcos10-tls -n default -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -text -noout | grep -A 2 "Public Key Algorithm" + +# Extract and verify ECDSA certificate +oc get secret test-cert-ecdsa-rhcos10-tls -n default -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -text -noout | grep -A 2 "Public Key Algorithm" +``` + +#### 4.3 FIPS Mode Testing + +If cluster is in FIPS mode: + +```bash +# Verify FIPS mode is enabled on nodes +oc debug node/$NODE -- chroot /host cat /proc/sys/crypto/fips_enabled +# Should return 1 if FIPS is enabled + +# Verify cert-manager respects FIPS mode +oc logs deployment/cert-manager -n cert-manager | grep -i fips + +# Test that only FIPS-approved algorithms are used +# Attempt to create certificate with weak algorithm (should fail in FIPS mode) +``` + +### Phase 5: Cloud Provider Integration Testing + +If testing cloud provider integration (AWS Route53, GCP Cloud DNS, Azure DNS): + +#### 5.1 AWS Route53 (if applicable) + +```bash +# Run AWS DNS01 tests +E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {AWS} && Issuer: isSubsetOf {ACME-DNS01}' make test-e2e +``` + +#### 5.2 GCP Cloud DNS (if applicable) + +```bash +# Run GCP DNS01 tests +E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {GCP} && Issuer: isSubsetOf {ACME-DNS01}' make test-e2e +``` + +#### 5.3 Azure DNS (if applicable) + +```bash +# Run Azure DNS01 tests +E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {Azure} && Issuer: isSubsetOf {ACME-DNS01}' make test-e2e +``` + +## Troubleshooting + +### Common Issues + +#### Issue: Pods fail to start with crypto errors + +**Symptoms:** +``` +Error: failed to load private key: crypto/rsa: unsupported key size +``` + +**Investigation:** +```bash +# Check FIPS mode status +oc debug node/$NODE -- chroot /host cat /proc/sys/crypto/fips_enabled + +# Check OpenSSL version +oc exec -n cert-manager $POD -- openssl version -a + +# Check pod logs +oc logs $POD -n cert-manager +``` + +**Resolution:** +- Verify certificates use FIPS-approved key sizes (RSA >= 2048, ECDSA >= 256) +- Check for deprecated algorithms +- Review crypto library compatibility + +#### Issue: TLS handshake failures + +**Symptoms:** +``` +Error: tls: failed to verify certificate: x509: certificate signed by unknown authority +``` + +**Investigation:** +```bash +# Check CA bundle +oc exec -n cert-manager $POD -- ls -la /etc/pki/tls/certs/ + +# Test TLS connection +oc exec -n cert-manager $POD -- curl -v https://kubernetes.default.svc +``` + +**Resolution:** +- Verify CA bundle is properly mounted +- Check proxy configuration +- Verify cluster certificates are valid + +#### Issue: E2E tests fail on RHCOS 10 + +**Investigation:** +```bash +# Check test logs +cat /tmp/report.json | jq '.[] | select(.State == "failed")' + +# Check cluster state +oc get pods -n cert-manager +oc get co + +# Collect debug information +make test-e2e-debug-cluster +``` + +**Resolution:** +- Review test failure details +- Check for RHCOS 10 specific issues +- File bugs with detailed information + +### Debug Information Collection + +When filing bugs, collect the following information: + +```bash +# Cluster version and node information +oc version > debug-info.txt +oc get nodes -o wide >> debug-info.txt +oc get nodes -o json | jq -r '.items[] | {name: .metadata.name, os: .status.nodeInfo.osImage, kernel: .status.nodeInfo.kernelVersion}' >> debug-info.txt + +# Operator status +oc get csv -n cert-manager-operator >> debug-info.txt +oc get deployment -n cert-manager-operator >> debug-info.txt +oc logs deployment/cert-manager-operator-controller-manager -n cert-manager-operator --tail=100 >> debug-info.txt + +# Operand status +oc get certmanager cluster -o yaml >> debug-info.txt +oc get pods -n cert-manager >> debug-info.txt +oc get events -n cert-manager --sort-by='.lastTimestamp' >> debug-info.txt + +# Crypto information +NODE=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}') +oc debug node/$NODE -- chroot /host sh -c "cat /etc/os-release && openssl version -a && cat /proc/sys/crypto/fips_enabled" >> debug-info.txt + +# Test results +cat /tmp/junit.xml >> debug-info.txt +cat /tmp/report.json | jq '.' >> debug-info.txt +``` + +## Results Documentation + +After completing testing, document results using the template: + +```bash +# Generate test report +make report-rhcos10 +``` + +This will create a report in `_output/rhcos10-test-report.md`. + +Alternatively, manually copy and fill out `docs/rhcos10-test-results-template.md`. + +## Reporting Results + +### Document Test Results + +Document your test results using the provided template in `docs/rhcos10-test-results-template.md`: +- Test execution date +- OCP and RHCOS versions tested +- Test results summary +- Links to any bugs filed +- Overall status (Pass/Fail/Blocked) + +### File Bugs + +If issues are discovered: + +1. File bugs in your project's bug tracker +2. Include all debug information collected +3. Tag with appropriate labels (rhcos10, rhel10, crypto) +4. Set appropriate priority based on impact +5. Include reproduction steps and environment details + +### Share Results + +Share test results with your team according to your organization's processes. + +## Appendix: Quick Reference Commands + +```bash +# Deploy operator +make deploy + +# Wait for stable state +make test-e2e-wait-for-stable-state + +# Run e2e tests +make test-e2e + +# Verify crypto libraries +make verify-rhcos10-crypto + +# Run RHCOS 10 compatibility tests +make test-rhcos10 + +# Generate report +make report-rhcos10 + +# Debug cluster state +make test-e2e-debug-cluster + +# Clean up test resources +oc delete certificate,issuer,clusterissuer --all -n default +``` From 41222685947827bda269d07ccbb8a7af279f10ed Mon Sep 17 00:00:00 2001 From: chiragkyal Date: Fri, 6 Mar 2026 20:22:05 +0530 Subject: [PATCH 2/4] feat(test): Add RHCOS 10 crypto verification and testing automation Add automated testing scripts for RHCOS 10 compatibility verification: verify-rhcos10-crypto.sh: - Verifies RHCOS/RHEL version on cluster nodes - Checks OpenSSL versions on nodes and in containers - Verifies FIPS mode status - Tests TLS connectivity from cert-manager pods - Tests certificate generation with various algorithms (RSA, ECDSA) - Checks for crypto-related errors in logs - Generates detailed verification report test-rhcos10-compatibility.sh: - Orchestrates comprehensive compatibility testing - Collects cluster information - Verifies deployment status - Runs E2E test suite - Executes crypto verification - Collects diagnostic information - Generates detailed test report Both scripts support skip flags for selective test execution and provide detailed output for troubleshooting. Co-Authored-By: Claude Sonnet 4.5 --- hack/test-rhcos10-compatibility.sh | 648 +++++++++++++++++++++++++++++ hack/verify-rhcos10-crypto.sh | 639 ++++++++++++++++++++++++++++ 2 files changed, 1287 insertions(+) create mode 100755 hack/test-rhcos10-compatibility.sh create mode 100755 hack/verify-rhcos10-crypto.sh diff --git a/hack/test-rhcos10-compatibility.sh b/hack/test-rhcos10-compatibility.sh new file mode 100755 index 000000000..fa6f01822 --- /dev/null +++ b/hack/test-rhcos10-compatibility.sh @@ -0,0 +1,648 @@ +#!/usr/bin/env bash + +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# test-rhcos10-compatibility.sh - Run comprehensive RHCOS 10 compatibility tests +# +# This script orchestrates comprehensive compatibility testing for cert-manager +# on RHCOS 10, including deployment verification, e2e tests, crypto library checks, +# and results documentation. + +set -euo pipefail + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Configuration +OUTPUT_DIR="${OUTPUT_DIR:-${PROJECT_ROOT}/_output}" +TEST_REPORT="${OUTPUT_DIR}/rhcos10-compatibility-report.md" +E2E_TIMEOUT="${E2E_TIMEOUT:-2h}" +RUN_E2E_TESTS="${RUN_E2E_TESTS:-true}" +RUN_CRYPTO_VERIFICATION="${RUN_CRYPTO_VERIFICATION:-true}" + +# Test results +DEPLOYMENT_PASSED=false +E2E_PASSED=false +CRYPTO_PASSED=false + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" +} + +log_success() { + echo -e "${GREEN}[PASS]${NC} $*" +} + +log_error() { + echo -e "${RED}[FAIL]${NC} $*" +} + +log_warning() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +log_section() { + echo "" + echo -e "${CYAN}========================================${NC}" + echo -e "${CYAN}$*${NC}" + echo -e "${CYAN}========================================${NC}" + echo "" +} + +# Print usage +usage() { + cat < /dev/null; then + missing_tools+=("oc") + fi + + if ! command -v kubectl &> /dev/null; then + missing_tools+=("kubectl") + fi + + if ! command -v jq &> /dev/null; then + missing_tools+=("jq") + fi + + if ! command -v make &> /dev/null; then + missing_tools+=("make") + fi + + if [ ${#missing_tools[@]} -gt 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + # Check cluster connectivity + if ! oc cluster-info &> /dev/null; then + log_error "Cannot connect to cluster. Please ensure you are logged in." + exit 1 + fi + + log_success "All prerequisites met" +} + +# Collect cluster information +collect_cluster_info() { + log_section "Collecting Cluster Information" + + local ocp_version rhcos_version platform + + ocp_version=$(oc version -o json 2>/dev/null | jq -r '.openshiftVersion // "unknown"') + platform=$(oc get infrastructure cluster -o jsonpath='{.status.platformStatus.type}' 2>/dev/null || echo "unknown") + + log_info "OpenShift Version: $ocp_version" + log_info "Platform: $platform" + + # Get RHCOS version from a worker node + local worker_node + worker_node=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$worker_node" ]; then + worker_node=$(oc get nodes -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + fi + + if [ -n "$worker_node" ]; then + rhcos_version=$(oc get node "$worker_node" -o jsonpath='{.status.nodeInfo.osImage}') + log_info "RHCOS Version: $rhcos_version" + + # Verify RHCOS 10 + if echo "$rhcos_version" | grep -qE "(RHCOS|CoreOS|Red Hat Enterprise Linux CoreOS) (10|410)"; then + log_success "Cluster is running RHCOS 10" + else + log_warning "Cluster may not be running RHCOS 10: $rhcos_version" + log_warning "This test suite is designed for RHCOS 10. Results may not be accurate." + fi + else + log_warning "Could not determine RHCOS version" + rhcos_version="unknown" + fi + + # Export for report + export CLUSTER_OCP_VERSION="$ocp_version" + export CLUSTER_RHCOS_VERSION="$rhcos_version" + export CLUSTER_PLATFORM="$platform" + export CLUSTER_NAME="$(oc config current-context)" +} + +# Verify cert-manager deployment +verify_deployment() { + log_section "Verifying cert-manager Deployment" + + # Check if cert-manager-operator namespace exists + if ! oc get namespace cert-manager-operator &> /dev/null; then + log_error "cert-manager-operator namespace does not exist" + log_info "Please deploy cert-manager operator first: make deploy" + return 1 + fi + + # Check operator deployment + log_info "Checking operator deployment..." + if oc get deployment cert-manager-operator-controller-manager -n cert-manager-operator &> /dev/null; then + if oc wait --for=condition=Available=true deployment/cert-manager-operator-controller-manager \ + -n cert-manager-operator --timeout=60s &> /dev/null; then + log_success "Operator deployment is available" + else + log_error "Operator deployment is not available" + return 1 + fi + else + log_error "Operator deployment not found" + return 1 + fi + + # Check if cert-manager namespace exists + if ! oc get namespace cert-manager &> /dev/null; then + log_error "cert-manager namespace does not exist" + return 1 + fi + + # Check operand deployments + log_info "Checking operand deployments..." + + local deployments=("cert-manager" "cert-manager-webhook" "cert-manager-cainjector") + local all_ready=true + + for deployment in "${deployments[@]}"; do + if oc get deployment "$deployment" -n cert-manager &> /dev/null; then + if oc wait --for=condition=Available=true "deployment/$deployment" \ + -n cert-manager --timeout=120s &> /dev/null; then + log_success "Deployment $deployment is available" + else + log_error "Deployment $deployment is not available" + all_ready=false + fi + else + log_error "Deployment $deployment not found" + all_ready=false + fi + done + + if [ "$all_ready" = true ]; then + log_success "All cert-manager deployments are ready" + DEPLOYMENT_PASSED=true + return 0 + else + log_error "Some cert-manager deployments are not ready" + return 1 + fi +} + +# Run e2e tests +run_e2e_tests() { + log_section "Running E2E Test Suite" + + if [ "$RUN_E2E_TESTS" != "true" ]; then + log_info "Skipping e2e tests (--skip-e2e flag set)" + E2E_PASSED=true # Mark as passed if skipped + return 0 + fi + + log_info "Running e2e tests with timeout: $E2E_TIMEOUT" + + if [ -n "${E2E_GINKGO_LABEL_FILTER:-}" ]; then + log_info "Using label filter: $E2E_GINKGO_LABEL_FILTER" + fi + + # Run e2e tests from project root + cd "$PROJECT_ROOT" + + if make test-e2e E2E_TIMEOUT="$E2E_TIMEOUT"; then + log_success "E2E tests passed" + E2E_PASSED=true + return 0 + else + log_error "E2E tests failed" + E2E_PASSED=false + return 1 + fi +} + +# Run crypto verification +run_crypto_verification() { + log_section "Running Crypto Library Verification" + + if [ "$RUN_CRYPTO_VERIFICATION" != "true" ]; then + log_info "Skipping crypto verification (--skip-crypto flag set)" + CRYPTO_PASSED=true # Mark as passed if skipped + return 0 + fi + + # Run crypto verification script + if bash "${SCRIPT_DIR}/verify-rhcos10-crypto.sh"; then + log_success "Crypto verification passed" + CRYPTO_PASSED=true + return 0 + else + log_error "Crypto verification failed" + CRYPTO_PASSED=false + return 1 + fi +} + +# Collect logs and diagnostics +collect_diagnostics() { + log_section "Collecting Diagnostic Information" + + local diag_dir="${OUTPUT_DIR}/diagnostics" + mkdir -p "$diag_dir" + + log_info "Collecting cluster diagnostics to: $diag_dir" + + # Cluster version + oc version > "${diag_dir}/cluster-version.txt" 2>&1 || true + + # Node information + oc get nodes -o wide > "${diag_dir}/nodes.txt" 2>&1 || true + oc get nodes -o json | jq -r '.items[] | {name: .metadata.name, os: .status.nodeInfo.osImage, kernel: .status.nodeInfo.kernelVersion}' \ + > "${diag_dir}/nodes-os-info.json" 2>&1 || true + + # Operator status + oc get csv -n cert-manager-operator > "${diag_dir}/operator-csv.txt" 2>&1 || true + oc get deployment -n cert-manager-operator > "${diag_dir}/operator-deployments.txt" 2>&1 || true + oc get pods -n cert-manager-operator > "${diag_dir}/operator-pods.txt" 2>&1 || true + oc logs deployment/cert-manager-operator-controller-manager -n cert-manager-operator --tail=500 \ + > "${diag_dir}/operator-logs.txt" 2>&1 || true + + # Operand status + oc get certmanager cluster -o yaml > "${diag_dir}/certmanager-cr.yaml" 2>&1 || true + oc get deployment -n cert-manager > "${diag_dir}/operand-deployments.txt" 2>&1 || true + oc get pods -n cert-manager > "${diag_dir}/operand-pods.txt" 2>&1 || true + oc get events -n cert-manager --sort-by='.lastTimestamp' > "${diag_dir}/operand-events.txt" 2>&1 || true + + # Operand logs + for pod in $(oc get pods -n cert-manager -o name 2>/dev/null); do + pod_name=$(basename "$pod") + oc logs "$pod" -n cert-manager --tail=500 > "${diag_dir}/operand-log-${pod_name}.txt" 2>&1 || true + done + + # E2E test results (if available) + if [ -f "${OUTPUT_DIR}/report.json" ]; then + cp "${OUTPUT_DIR}/report.json" "${diag_dir}/" || true + fi + + if [ -f "${OUTPUT_DIR}/junit.xml" ]; then + cp "${OUTPUT_DIR}/junit.xml" "${diag_dir}/" || true + fi + + # Crypto verification report (if available) + if [ -f "${OUTPUT_DIR}/rhcos10-crypto-verification-report.txt" ]; then + cp "${OUTPUT_DIR}/rhcos10-crypto-verification-report.txt" "${diag_dir}/" || true + fi + + log_success "Diagnostics collected" +} + +# Generate test report +generate_report() { + log_section "Generating Test Report" + + mkdir -p "$OUTPUT_DIR" + + local overall_status="FAIL" + if [ "$DEPLOYMENT_PASSED" = true ] && [ "$E2E_PASSED" = true ] && [ "$CRYPTO_PASSED" = true ]; then + overall_status="PASS" + fi + + cat > "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" <> "$TEST_REPORT" < "${REPORT_FILE}" </dev/null || echo "unknown") + +EOF +} + +# Add to report +add_to_report() { + echo "$*" >> "${REPORT_FILE}" +} + +# Check prerequisites +check_prerequisites() { + log_info "Checking prerequisites..." + + local missing_tools=() + + if ! command -v oc &> /dev/null; then + missing_tools+=("oc") + fi + + if ! command -v jq &> /dev/null; then + missing_tools+=("jq") + fi + + if ! command -v openssl &> /dev/null; then + missing_tools+=("openssl") + fi + + if [ ${#missing_tools[@]} -gt 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + # Check cluster connectivity + if ! oc cluster-info &> /dev/null; then + log_error "Cannot connect to cluster. Please ensure you are logged in." + exit 1 + fi + + log_success "All prerequisites met" +} + +# Check RHCOS version +check_rhcos_version() { + log_info "Checking RHCOS version on cluster nodes..." + add_to_report "" + add_to_report "================================================================================ +Node OS Versions +================================================================================" + + local nodes_json + nodes_json=$(oc get nodes -o json) + + local rhcos10_nodes=0 + local total_nodes=0 + + while IFS= read -r node; do + ((total_nodes++)) + local name os_image kernel + name=$(echo "$node" | jq -r '.name') + os_image=$(echo "$node" | jq -r '.os') + kernel=$(echo "$node" | jq -r '.kernel') + + add_to_report "Node: $name" + add_to_report " OS Image: $os_image" + add_to_report " Kernel: $kernel" + + # Check if RHCOS 10 or RHEL 10 + if echo "$os_image" | grep -qE "(RHCOS|CoreOS|Red Hat Enterprise Linux CoreOS) (10|410)"; then + ((rhcos10_nodes++)) + log_success "Node $name is running RHCOS 10" + else + log_warning "Node $name may not be running RHCOS 10: $os_image" + fi + done < <(echo "$nodes_json" | jq -c '.items[] | {name: .metadata.name, os: .status.nodeInfo.osImage, kernel: .status.nodeInfo.kernelVersion}') + + add_to_report "" + add_to_report "Total nodes: $total_nodes" + add_to_report "RHCOS 10 nodes: $rhcos10_nodes" + + if [ "$rhcos10_nodes" -eq "$total_nodes" ]; then + log_success "All $total_nodes nodes are running RHCOS 10" + elif [ "$rhcos10_nodes" -gt 0 ]; then + log_warning "Only $rhcos10_nodes of $total_nodes nodes are running RHCOS 10" + else + log_error "No nodes detected as running RHCOS 10" + fi +} + +# Check OpenSSL version on nodes +check_node_openssl() { + log_info "Checking OpenSSL version on nodes..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Node OpenSSL Versions" + add_to_report "================================================================================" + + local worker_node + worker_node=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$worker_node" ]; then + # Try master node if no worker + worker_node=$(oc get nodes -l node-role.kubernetes.io/master -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + fi + + if [ -z "$worker_node" ]; then + log_error "No nodes found to check OpenSSL version" + return + fi + + log_info "Checking OpenSSL on node: $worker_node" + + local openssl_output + if openssl_output=$(oc debug "node/$worker_node" -- chroot /host openssl version -a 2>&1); then + add_to_report "Node: $worker_node" + add_to_report "$openssl_output" + + # Extract version + local version + version=$(echo "$openssl_output" | grep -E "^OpenSSL" | awk '{print $2}') + + if [ -n "$version" ]; then + log_success "OpenSSL version on node: $version" + + # Check for OpenSSL 3.x (expected in RHCOS 10 / RHEL 10) + if echo "$version" | grep -qE "^3\."; then + log_success "OpenSSL 3.x detected (expected for RHCOS 10)" + else + log_warning "OpenSSL version is not 3.x: $version" + fi + fi + else + log_error "Failed to check OpenSSL version on node" + add_to_report "Error: Failed to retrieve OpenSSL version" + fi +} + +# Check FIPS mode +check_fips_mode() { + log_info "Checking FIPS mode status..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "FIPS Mode Status" + add_to_report "================================================================================" + + local worker_node + worker_node=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$worker_node" ]; then + worker_node=$(oc get nodes -l node-role.kubernetes.io/master -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + fi + + if [ -z "$worker_node" ]; then + log_error "No nodes found to check FIPS mode" + return + fi + + local fips_status + if fips_status=$(oc debug "node/$worker_node" -- chroot /host cat /proc/sys/crypto/fips_enabled 2>&1); then + add_to_report "Node: $worker_node" + add_to_report "FIPS enabled: $fips_status" + + if [ "$fips_status" = "1" ]; then + log_info "FIPS mode is ENABLED on node" + else + log_info "FIPS mode is DISABLED on node" + fi + else + log_warning "Could not determine FIPS mode status" + add_to_report "Error: Could not retrieve FIPS status" + fi +} + +# Check cert-manager pods +check_certmanager_pods() { + log_info "Checking cert-manager pod status..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Cert-Manager Pod Status" + add_to_report "================================================================================" + + if ! oc get namespace cert-manager &> /dev/null; then + log_error "cert-manager namespace does not exist" + return + fi + + local pods + pods=$(oc get pods -n cert-manager -o json) + + local pod_count + pod_count=$(echo "$pods" | jq '.items | length') + + if [ "$pod_count" -eq 0 ]; then + log_error "No cert-manager pods found" + return + fi + + log_info "Found $pod_count cert-manager pods" + + local running_pods=0 + while IFS= read -r pod_name; do + local status + status=$(oc get pod "$pod_name" -n cert-manager -o jsonpath='{.status.phase}') + + add_to_report "Pod: $pod_name - Status: $status" + + if [ "$status" = "Running" ]; then + ((running_pods++)) + log_success "Pod $pod_name is running" + else + log_error "Pod $pod_name is not running: $status" + fi + done < <(echo "$pods" | jq -r '.items[].metadata.name') + + if [ "$running_pods" -eq "$pod_count" ]; then + log_success "All $pod_count cert-manager pods are running" + else + log_error "Only $running_pods of $pod_count pods are running" + fi +} + +# Check OpenSSL in cert-manager containers +check_container_openssl() { + log_info "Checking OpenSSL version in cert-manager containers..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Container OpenSSL Versions" + add_to_report "================================================================================" + + if ! oc get namespace cert-manager &> /dev/null; then + log_error "cert-manager namespace does not exist" + return + fi + + # Check controller pod + local controller_pod + controller_pod=$(oc get pods -n cert-manager -l app=cert-manager,app.kubernetes.io/component=controller -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$controller_pod" ]; then + log_warning "cert-manager controller pod not found" + return + fi + + log_info "Checking OpenSSL in controller pod: $controller_pod" + + local openssl_version + if openssl_version=$(oc exec -n cert-manager "$controller_pod" -- openssl version -a 2>&1); then + add_to_report "Controller Pod: $controller_pod" + add_to_report "$openssl_version" + + local version + version=$(echo "$openssl_version" | grep -E "^OpenSSL" | awk '{print $2}') + log_success "Container OpenSSL version: $version" + else + log_warning "Could not retrieve OpenSSL version from container (may not have openssl binary)" + add_to_report "Note: OpenSSL binary may not be available in container" + fi +} + +# Test TLS connectivity from cert-manager +check_tls_connectivity() { + log_info "Testing TLS connectivity from cert-manager controller..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "TLS Connectivity Tests" + add_to_report "================================================================================" + + if ! oc get namespace cert-manager &> /dev/null; then + log_error "cert-manager namespace does not exist" + return + fi + + local controller_pod + controller_pod=$(oc get pods -n cert-manager -l app=cert-manager,app.kubernetes.io/component=controller -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$controller_pod" ]; then + log_error "cert-manager controller pod not found" + return + fi + + log_info "Testing TLS connection to Kubernetes API from pod: $controller_pod" + + local tls_output + if tls_output=$(oc exec -n cert-manager "$controller_pod" -- curl -v -k https://kubernetes.default.svc 2>&1 | head -30); then + add_to_report "TLS connection test to kubernetes.default.svc:" + add_to_report "$tls_output" + + if echo "$tls_output" | grep -q "SSL connection using"; then + local cipher + cipher=$(echo "$tls_output" | grep "SSL connection using" || echo "") + log_success "TLS connection successful: $cipher" + else + log_warning "TLS connection test completed but cipher information not found" + fi + else + log_error "TLS connectivity test failed" + fi +} + +# Check for crypto-related errors in logs +check_crypto_errors_in_logs() { + log_info "Checking cert-manager logs for crypto-related errors..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Crypto-Related Errors in Logs" + add_to_report "================================================================================" + + if ! oc get namespace cert-manager &> /dev/null; then + log_error "cert-manager namespace does not exist" + return + fi + + local error_found=false + + while IFS= read -r pod_name; do + log_info "Checking logs for pod: $pod_name" + + local logs + if logs=$(oc logs "$pod_name" -n cert-manager --tail=500 2>&1 | grep -iE "(crypto|ssl|tls|fips|cipher|certificate.*error|x509.*error)" || true); then + if [ -n "$logs" ]; then + add_to_report "Pod: $pod_name" + add_to_report "$logs" + log_warning "Found crypto-related messages in $pod_name logs (review needed)" + error_found=true + fi + fi + done < <(oc get pods -n cert-manager -o jsonpath='{.items[*].metadata.name}') + + if [ "$error_found" = false ]; then + log_success "No crypto-related errors found in logs" + add_to_report "No crypto-related errors found" + fi +} + +# Test certificate generation with different algorithms +test_certificate_generation() { + log_info "Testing certificate generation with different algorithms..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Certificate Generation Tests" + add_to_report "================================================================================" + + local test_ns="cert-manager-rhcos10-test" + + # Create test namespace + if ! oc get namespace "$test_ns" &> /dev/null; then + oc create namespace "$test_ns" > /dev/null 2>&1 || true + fi + + # Cleanup function + cleanup_test_resources() { + log_info "Cleaning up test resources..." + oc delete certificate --all -n "$test_ns" &> /dev/null || true + oc delete issuer --all -n "$test_ns" &> /dev/null || true + oc delete namespace "$test_ns" &> /dev/null || true + } + + # Ensure cleanup on exit + trap cleanup_test_resources EXIT + + # Create self-signed issuer + log_info "Creating self-signed issuer..." + oc apply -f - > /dev/null 2>&1 < /dev/null 2>&1 < /dev/null; then + log_success "RSA 2048 certificate generated successfully" + add_to_report "✓ RSA 2048: SUCCESS" + else + log_error "Failed to generate RSA 2048 certificate" + add_to_report "✗ RSA 2048: FAILED" + fi + + # Test RSA 4096 + log_info "Testing RSA 4096 certificate generation..." + oc apply -f - > /dev/null 2>&1 < /dev/null; then + log_success "RSA 4096 certificate generated successfully" + add_to_report "✓ RSA 4096: SUCCESS" + else + log_error "Failed to generate RSA 4096 certificate" + add_to_report "✗ RSA 4096: FAILED" + fi + + # Test ECDSA P-256 + log_info "Testing ECDSA P-256 certificate generation..." + oc apply -f - > /dev/null 2>&1 < /dev/null; then + log_success "ECDSA P-256 certificate generated successfully" + add_to_report "✓ ECDSA P-256: SUCCESS" + else + log_error "Failed to generate ECDSA P-256 certificate" + add_to_report "✗ ECDSA P-256: FAILED" + fi + + # Test ECDSA P-384 + log_info "Testing ECDSA P-384 certificate generation..." + oc apply -f - > /dev/null 2>&1 < /dev/null; then + log_success "ECDSA P-384 certificate generated successfully" + add_to_report "✓ ECDSA P-384: SUCCESS" + else + log_error "Failed to generate ECDSA P-384 certificate" + add_to_report "✗ ECDSA P-384: FAILED" + fi +} + +# Generate summary +generate_summary() { + log_info "Generating verification summary..." + add_to_report "" + add_to_report "================================================================================" + add_to_report "Summary" + add_to_report "================================================================================" + add_to_report "Checks Passed: $CHECKS_PASSED" + add_to_report "Checks Failed: $CHECKS_FAILED" + add_to_report "Checks Warning: $CHECKS_WARNING" + add_to_report "" + + local status + if [ "$CHECKS_FAILED" -eq 0 ]; then + status="PASS" + add_to_report "Overall Status: ✓ PASS" + log_success "Overall verification: PASS" + else + status="FAIL" + add_to_report "Overall Status: ✗ FAIL" + log_error "Overall verification: FAIL" + fi + + if [ "$CHECKS_WARNING" -gt 0 ]; then + add_to_report "Warnings: Review recommended" + log_warning "$CHECKS_WARNING warnings require review" + fi + + add_to_report "" + add_to_report "Report saved to: $REPORT_FILE" + add_to_report "================================================================================" + + echo "" + log_info "Detailed report saved to: $REPORT_FILE" + echo "" + + return $([ "$status" = "PASS" ] && echo 0 || echo 1) +} + +# Main execution +main() { + echo "================================================================================" + echo "RHCOS 10 Crypto Library Verification" + echo "================================================================================" + echo "" + + init_report + check_prerequisites + check_rhcos_version + check_node_openssl + check_fips_mode + check_certmanager_pods + check_container_openssl + check_tls_connectivity + check_crypto_errors_in_logs + test_certificate_generation + + echo "" + generate_summary +} + +# Run main function +main "$@" From 56aa80737db4cf646ce724b8541a5bee4e06d3d7 Mon Sep 17 00:00:00 2001 From: chiragkyal Date: Fri, 6 Mar 2026 20:22:18 +0530 Subject: [PATCH 3/4] feat(make): Add RHCOS 10 testing targets Add Makefile targets for RHCOS 10 compatibility testing: - test-rhcos10: Run comprehensive RHCOS 10 compatibility test suite - verify-rhcos10-crypto: Verify RHCOS 10 crypto library compatibility - report-rhcos10: Generate RHCOS 10 test report from results These targets provide convenient access to the RHCOS 10 testing automation scripts and follow existing project conventions for test execution. Co-Authored-By: Claude Sonnet 4.5 --- Makefile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Makefile b/Makefile index 81e612e96..2aec30a14 100644 --- a/Makefile +++ b/Makefile @@ -286,6 +286,25 @@ test-e2e-debug-cluster: -oc logs deployment/cert-manager-operator -n cert-manager-operator @echo "---- /Debugging the current state ----" +.PHONY: test-rhcos10 +test-rhcos10: ## Run RHCOS 10 compatibility test suite. + hack/test-rhcos10-compatibility.sh + +.PHONY: verify-rhcos10-crypto +verify-rhcos10-crypto: ## Verify RHCOS 10 crypto library compatibility. + hack/verify-rhcos10-crypto.sh + +.PHONY: report-rhcos10 +report-rhcos10: ## Generate RHCOS 10 test report from results. + @echo "RHCOS 10 Test Report" + @echo "====================" + @if [ -f "$(OUTPUT_DIR)/rhcos10-compatibility-report.md" ]; then \ + cat "$(OUTPUT_DIR)/rhcos10-compatibility-report.md"; \ + else \ + echo "No test report found. Run 'make test-rhcos10' first."; \ + exit 1; \ + fi + .PHONY: lint lint: $(GOLANGCI_LINT) ## Run golangci-lint linter. $(GOLANGCI_LINT) run --verbose --config $(PROJECT_ROOT)/.golangci.yaml $(PROJECT_ROOT)/... From d3d13312d11cfc01b59a9c3e493057fbc00d3c96 Mon Sep 17 00:00:00 2001 From: chiragkyal Date: Fri, 6 Mar 2026 20:50:15 +0530 Subject: [PATCH 4/4] fix: Address CodeRabbit review comments - Fix critical set -e bug with post-increment in verify-rhcos10-crypto.sh (change ((var++)) to ((++var)) to prevent unexpected exits) - Fix OUTPUT_DIR propagation to child crypto verification script - Add text language identifiers to markdown fenced code blocks (MD040) Fixes identified by CodeRabbit code review. Co-Authored-By: Claude Sonnet 4.5 --- docs/rhcos10-test-results-template.md | 22 +++++++++++----------- docs/rhcos10-testing.md | 4 ++-- hack/test-rhcos10-compatibility.sh | 2 +- hack/verify-rhcos10-crypto.sh | 12 ++++++------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/rhcos10-test-results-template.md b/docs/rhcos10-test-results-template.md index 95897f077..2252a944a 100644 --- a/docs/rhcos10-test-results-template.md +++ b/docs/rhcos10-test-results-template.md @@ -68,7 +68,7 @@ - **All pods running:** [ ] Yes / [ ] No #### Issues Found -``` +```text [Describe any deployment issues here, or write "None"] ``` @@ -101,7 +101,7 @@ make test-e2e | [Test 2] | [Reason] | [BUG-XXX] | #### Test Execution Notes -``` +```text [Add any relevant notes about test execution, e.g., flaky tests, environmental issues, etc.] ``` @@ -152,7 +152,7 @@ If FIPS enabled: [ ] No crypto-related errors found [ ] Crypto errors found (details below) -``` +```text [Paste any crypto-related errors from logs, or write "None"] ``` @@ -178,7 +178,7 @@ If FIPS enabled: - **Cloud credentials:** [ ] Working / [ ] Issues **Notes:** -``` +```text [Add notes about cloud provider integration testing] ``` @@ -187,7 +187,7 @@ If FIPS enabled: - **Vault Issuer:** [ ] Pass / [ ] Fail / [ ] N/A **Notes:** -``` +```text [Add notes about Vault integration testing] ``` @@ -204,7 +204,7 @@ If FIPS enabled: ### Known Issues / Workarounds -``` +```text [Document any known issues and workarounds here, or write "None"] Example: @@ -227,7 +227,7 @@ Example: | cert-manager-cainjector | [e.g., 10m] | [e.g., 50Mi] | Normal / High | **Resource usage compared to RHCOS 9:** -``` +```text [Compare if you have baseline from RHCOS 9, or write "N/A - no baseline available"] ``` @@ -237,7 +237,7 @@ Example: ### RHCOS 10 Specific Observations -``` +```text [Document any RHCOS 10 specific behavior, issues, or improvements observed] Examples: @@ -249,13 +249,13 @@ Examples: ### Differences from RHCOS 9 (if known) -``` +```text [Document any differences observed compared to RHCOS 9, or write "N/A"] ``` ### Recommendations -``` +```text [Add any recommendations for users, documentation updates, or future improvements] ``` @@ -284,7 +284,7 @@ Examples: ### Summary -``` +```text [Provide a 2-3 sentence summary of the test results] Example: diff --git a/docs/rhcos10-testing.md b/docs/rhcos10-testing.md index 583a0e8fb..366110435 100644 --- a/docs/rhcos10-testing.md +++ b/docs/rhcos10-testing.md @@ -379,7 +379,7 @@ E2E_GINKGO_LABEL_FILTER='Platform: isSubsetOf {Azure} && Issuer: isSubsetOf {ACM #### Issue: Pods fail to start with crypto errors **Symptoms:** -``` +```text Error: failed to load private key: crypto/rsa: unsupported key size ``` @@ -403,7 +403,7 @@ oc logs $POD -n cert-manager #### Issue: TLS handshake failures **Symptoms:** -``` +```text Error: tls: failed to verify certificate: x509: certificate signed by unknown authority ``` diff --git a/hack/test-rhcos10-compatibility.sh b/hack/test-rhcos10-compatibility.sh index fa6f01822..d92349316 100755 --- a/hack/test-rhcos10-compatibility.sh +++ b/hack/test-rhcos10-compatibility.sh @@ -323,7 +323,7 @@ run_crypto_verification() { fi # Run crypto verification script - if bash "${SCRIPT_DIR}/verify-rhcos10-crypto.sh"; then + if OUTPUT_DIR="$OUTPUT_DIR" bash "${SCRIPT_DIR}/verify-rhcos10-crypto.sh"; then log_success "Crypto verification passed" CRYPTO_PASSED=true return 0 diff --git a/hack/verify-rhcos10-crypto.sh b/hack/verify-rhcos10-crypto.sh index a2175acb4..3572af9da 100755 --- a/hack/verify-rhcos10-crypto.sh +++ b/hack/verify-rhcos10-crypto.sh @@ -45,17 +45,17 @@ log_info() { log_success() { echo -e "${GREEN}[PASS]${NC} $*" - ((CHECKS_PASSED++)) + ((++CHECKS_PASSED)) } log_error() { echo -e "${RED}[FAIL]${NC} $*" - ((CHECKS_FAILED++)) + ((++CHECKS_FAILED)) } log_warning() { echo -e "${YELLOW}[WARN]${NC} $*" - ((CHECKS_WARNING++)) + ((++CHECKS_WARNING)) } # Initialize report @@ -123,7 +123,7 @@ Node OS Versions local total_nodes=0 while IFS= read -r node; do - ((total_nodes++)) + ((++total_nodes)) local name os_image kernel name=$(echo "$node" | jq -r '.name') os_image=$(echo "$node" | jq -r '.os') @@ -135,7 +135,7 @@ Node OS Versions # Check if RHCOS 10 or RHEL 10 if echo "$os_image" | grep -qE "(RHCOS|CoreOS|Red Hat Enterprise Linux CoreOS) (10|410)"; then - ((rhcos10_nodes++)) + ((++rhcos10_nodes)) log_success "Node $name is running RHCOS 10" else log_warning "Node $name may not be running RHCOS 10: $os_image" @@ -273,7 +273,7 @@ check_certmanager_pods() { add_to_report "Pod: $pod_name - Status: $status" if [ "$status" = "Running" ]; then - ((running_pods++)) + ((++running_pods)) log_success "Pod $pod_name is running" else log_error "Pod $pod_name is not running: $status"