From 7eda15440f7f5299d58b6c707ed0147aa76b44bc Mon Sep 17 00:00:00 2001 From: Bharath B Date: Wed, 11 Mar 2026 10:39:36 +0530 Subject: [PATCH 1/2] ESO-323: Adds Bitwarden and cross-platform e2e suites with failure artifact collection Signed-off-by: Bharath B --- README.md | 17 ++ test/README.md | 22 ++ test/apis/README.md | 25 +- test/e2e/README.md | 146 +++++++++ test/e2e/bitwarden_api_test.go | 146 +++++++++ test/e2e/bitwarden_es_test.go | 470 +++++++++++++++++++++++++++++ test/e2e/e2e_suite_test.go | 71 ++++- test/e2e/e2e_test.go | 145 +++++++-- test/go.mod | 2 +- test/utils/artifact_dump.go | 186 ++++++++++++ test/utils/aws_resources.go | 65 ++++ test/utils/bitwarden.go | 396 ++++++++++++++++++++++++ test/utils/bitwarden_api_runner.go | 177 +++++++++++ test/utils/bitwarden_resources.go | 171 +++++++++++ test/utils/cleanup.go | 132 ++++++++ test/utils/conditions.go | 32 +- test/utils/dynamic_resources.go | 43 +++ 17 files changed, 2217 insertions(+), 29 deletions(-) create mode 100644 test/README.md create mode 100644 test/e2e/README.md create mode 100644 test/e2e/bitwarden_api_test.go create mode 100644 test/e2e/bitwarden_es_test.go create mode 100644 test/utils/artifact_dump.go create mode 100644 test/utils/aws_resources.go create mode 100644 test/utils/bitwarden.go create mode 100644 test/utils/bitwarden_api_runner.go create mode 100644 test/utils/bitwarden_resources.go create mode 100644 test/utils/cleanup.go diff --git a/README.md b/README.md index f57814956..c5d9c828f 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,23 @@ kubectl apply -f https://raw.githubusercontent.com//external-secrets-operat More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) +## Testing + +From the repo root you can run: + +| Target | Description | +|--------|-------------| +| `make test-unit` | Run unit tests (excluding test/e2e, test/apis, test/utils). | +| `make test-apis` | Run API integration tests (Ginkgo tests in `test/apis` using envtest). | +| `make test` | Run `test-apis` and `test-unit` (no cluster required). | +| `make test-e2e` | Run end-to-end tests against a live cluster. | + +For e2e tests, including prerequisites and suite-specific commands (e.g. label filters for AWS, Bitwarden, cross-platform), see [test/e2e/README.md](test/e2e/README.md). Example: + +```sh +make test-e2e E2E_GINKGO_LABEL_FILTER="" +``` + ## Contributing We welcome contributions from the community! To contribute: diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..dbc30ae29 --- /dev/null +++ b/test/README.md @@ -0,0 +1,22 @@ +# Test directory + +This directory holds the operator’s tests and shared test utilities. + +## Layout + +| Path | Purpose | +|------|---------| +| **e2e/** | End-to-end tests run against a live cluster. See [e2e/README.md](e2e/README.md) for suites, labels, and prerequisites. | +| **apis/** | API integration tests (Ginkgo) run with envtest (no real cluster). See [apis/README.md](apis/README.md). | +| **utils/** | Shared helpers for e2e (and related) tests; built only with the `e2e` build tag. | + +## Make targets (from repo root) + +| Command | What it runs | +|---------|----------------| +| `make test-unit` | Unit tests for all packages except `test/e2e`, `test/apis`, `test/utils`. | +| `make test-apis` | API tests in `test/apis` (envtest + Ginkgo). | +| `make test` | `make test-apis` and `make test-unit` (no cluster). | +| `make test-e2e` | E2e tests in `test/e2e`; requires a cluster and optional label filter. | + +See the root [README.md](../README.md) Testing section and [e2e/README.md](e2e/README.md) for more detail. diff --git a/test/apis/README.md b/test/apis/README.md index 01b052d45..cc91cc155 100644 --- a/test/apis/README.md +++ b/test/apis/README.md @@ -1 +1,24 @@ -Refer to the [openshift/api test suite](https://github.com/openshift/api/tree/master/tests) for more details. +# API integration tests + +API tests exercise the operator’s APIs (CRDs, webhooks, controllers) using **envtest** — a local control plane (no real cluster required). They are implemented with [Ginkgo](https://onsi.github.io/ginkgo/). + +## Running + +From the repository root: + +```bash +make test-apis +``` + +This installs envtest binaries if needed, sets `KUBEBUILDER_ASSETS`, and runs Ginkgo in `test/apis` with a 30-minute timeout. In OpenShift CI, when `OPENSHIFT_CI=true` and `ARTIFACT_DIR` is set, JUnit output and coverage are written to `ARTIFACT_DIR`. + +## Requirements + +- `make test-apis` pulls the correct Kubernetes version via `envtest`; ensure `make test-apis` (or `make test`) has been run at least once so envtest assets are present. + +## Relation to other tests + +- **Unit tests** (`make test-unit`): Pure Go tests excluding `test/e2e`, `test/apis`, and `test/utils`. +- **E2e tests** (`make test-e2e`): Run against a real cluster; see [e2e/README.md](e2e/README.md). + +`make test` runs both unit and API tests (no cluster). diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..fea526eda --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,146 @@ +# E2E Test Suites + +Run e2e tests from this repo’s root with `make test-e2e`. Use `E2E_GINKGO_LABEL_FILTER` to run a specific suite by label. + +--- + +## Running tests + +From the repo root: + +```bash +make test-e2e E2E_GINKGO_LABEL_FILTER="" +``` + +Default (if omitted): `E2E_GINKGO_LABEL_FILTER="Platform: isSubsetOf {AWS}"` (AWS-only). + +### Running the whole suite + +To run **all** e2e specs (every suite, regardless of label), pass an empty label filter: + +```bash +make test-e2e E2E_GINKGO_LABEL_FILTER="" +``` + +Ensure pre-requisites for each suite you care about are in place (see [Suites by label](#suites-by-label)). Suites whose pre-requisites are not met will skip (e.g. missing `bitwarden-creds` or `aws-creds`). To run only a subset of suites, use a label filter (see [Running multiple suites](#running-multiple-suites)). + +### Failure artifacts (debugging) + +When a spec fails, the suite dumps logs and cluster state into **`/e2e-artifacts/failure-/`** so you can debug later. The output directory is **`ARTIFACT_DIR`** when set (e.g. in OpenShift CI); otherwise, when running locally via `make test-e2e`, it is **`_output`** at the repo root. + +Each failure dump includes: + +- **`pods/`** – Last 500 lines of logs per pod and `describe` (YAML) for operator, operand, and test namespaces. +- **`events/`** – Recent events per namespace. +- **`resources/`** – ExternalSecretsConfig (cluster), ClusterSecretStores, ExternalSecrets, and PushSecrets (YAML). + +JUnit and JSON reports are also written to the same output directory (see root README Testing section). + +--- + +## Suites by label + +### Platform:AWS (AWS Secret Manager) + +| Item | Details | +|------|--------| +| **Label filter** | `"Platform: isSubsetOf {AWS}"` (default) | +| **Pre-requisites** | Cluster on AWS (e.g. OpenShift on AWS); K8s secret `aws-creds` in namespace `kube-system` with keys `aws_access_key_id` and `aws_secret_access_key` (credentials for AWS Secrets Manager in `ap-south-1`). On OpenShift on AWS this secret is typically made available in `kube-system` by the platform. | +| **Make command** | `make test-e2e` or `make test-e2e E2E_GINKGO_LABEL_FILTER="Platform: isSubsetOf {AWS}"` | + +--- + +### Provider:Bitwarden (ESO CR–based Bitwarden) + +When this suite runs, it **enables Bitwarden in ExternalSecretsConfig** and **creates the TLS secret** (certificate for bitwarden-sdk-server) in the operand namespace. The default e2e cluster CR does not enable Bitwarden; enabling and the TLS secret are done only when tests with the Bitwarden label run. See `config/crd/bases/operator.openshift.io_externalsecretsconfigs.yaml` for `plugins.bitwardenSecretManagerProvider` (mode, secretRef). + +| Item | Details | +|------|--------| +| **Label filter** | `"Provider:Bitwarden"` | +| **Pre-requisites** | ESO installed. Create the Bitwarden credentials secret **`bitwarden-creds`** in **`external-secrets-operator`** (see [Creating the Bitwarden credentials secret](#creating-the-bitwarden-credentials-secret)). The suite enables the plugin and creates the TLS secret. Optional env: `BITWARDEN_SDK_SERVER_URL` (default: `https://bitwarden-sdk-server.external-secrets.svc.cluster.local:9998`). | +| **Make command** | `make test-e2e E2E_GINKGO_LABEL_FILTER="Provider:Bitwarden"` | + +--- + +### API:Bitwarden (Direct HTTP to bitwarden-sdk-server) + +| Item | Details | +|------|--------| +| **Label filter** | `"API:Bitwarden"` | +| **Pre-requisites** | bitwarden-sdk-server reachable (e.g. in-cluster). For Secrets API tests: create the Bitwarden credentials secret **`bitwarden-creds`** in **`external-secrets-operator`** (see [Creating the Bitwarden credentials secret](#creating-the-bitwarden-credentials-secret)). Optional env: `BITWARDEN_SDK_SERVER_URL` (default: `https://bitwarden-sdk-server.external-secrets.svc.cluster.local:9998`). | +| **Make command** | `make test-e2e E2E_GINKGO_LABEL_FILTER="API:Bitwarden"` | + +#### Creating the Bitwarden credentials secret + +The Provider:Bitwarden and API:Bitwarden suites expect the secret to be named **`bitwarden-creds`** in namespace **`external-secrets-operator`**. The secret must have keys **`token`** (Bitwarden machine account access token), **`organization_id`**, and **`project_id`**. + +**From literal values:** + +```bash +kubectl create secret generic bitwarden-creds \ + --namespace=external-secrets-operator \ + --from-literal=token="YOUR_BITWARDEN_ACCESS_TOKEN" \ + --from-literal=organization_id="YOUR_ORGANIZATION_ID" \ + --from-literal=project_id="YOUR_PROJECT_ID" +``` + +**From env vars (avoids secrets in shell history):** + +```bash +export BITWARDEN_ACCESS_TOKEN="..." +export BITWARDEN_ORGANIZATION_ID="..." +export BITWARDEN_PROJECT_ID="..." +kubectl create secret generic bitwarden-creds \ + --namespace=external-secrets-operator \ + --from-literal=token="${BITWARDEN_ACCESS_TOKEN}" \ + --from-literal=organization_id="${BITWARDEN_ORGANIZATION_ID}" \ + --from-literal=project_id="${BITWARDEN_PROJECT_ID}" +``` + +--- + +### CrossPlatform:GCP-AWS (GCP cluster, AWS Secrets Manager) + +Cluster runs on **GCP**; the test uses a **ClusterSecretStore** backed by **AWS Secrets Manager**. You must create a Kubernetes secret that holds AWS credentials in a **fixed** name and namespace (see below). + +| Item | Details | +|------|--------| +| **Label filter** | `"CrossPlatform:GCP-AWS"` | +| **Pre-requisites** | Cluster on GCP (or any non-AWS cluster). Create the AWS credentials secret with **name** `aws-creds` in **namespace** `kube-system` (see below). | +| **Make command** | `make test-e2e E2E_GINKGO_LABEL_FILTER="CrossPlatform:GCP-AWS"` | + +#### Creating the AWS credentials secret + +The test expects the secret to be named **`aws-creds`** in namespace **`kube-system`**. The secret must have keys **`aws_access_key_id`** and **`aws_secret_access_key`**. The test uses AWS region `ap-south-1`. + +**From literal values:** + +```bash +kubectl create secret generic aws-creds \ + --namespace=kube-system \ + --from-literal=aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID" \ + --from-literal=aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY" +``` + +**From env vars (avoids secrets in shell history):** + +```bash +export AWS_ACCESS_KEY_ID="..." +export AWS_SECRET_ACCESS_KEY="..." +kubectl create secret generic aws-creds \ + --namespace=kube-system \ + --from-literal=aws_access_key_id="${AWS_ACCESS_KEY_ID}" \ + --from-literal=aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}" +``` + +--- + +## Running multiple suites + +To run more than one label (e.g. Bitwarden provider and API): + +```bash +make test-e2e E2E_GINKGO_LABEL_FILTER="Provider:Bitwarden || API:Bitwarden" +``` + +See [Ginkgo label documentation](https://onsi.github.io/ginkgo/#spec-labels) for label query syntax. diff --git a/test/e2e/bitwarden_api_test.go b/test/e2e/bitwarden_api_test.go new file mode 100644 index 000000000..07a01a812 --- /dev/null +++ b/test/e2e/bitwarden_api_test.go @@ -0,0 +1,146 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/external-secrets-operator/test/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + apiPath = "/rest/api/1" + apiJobTimeout = 2 * time.Minute +) + +// inClusterBaseURL is the Bitwarden SDK server URL as seen from inside the cluster (same as external-secrets controller). Respects BITWARDEN_SDK_SERVER_URL. +var inClusterBaseURL = utils.GetBitwardenSDKServerURL() + +var _ = Describe("Bitwarden SDK Server API", Ordered, Label("API:Bitwarden", "Suite:Bitwarden"), func() { + var ( + ctx context.Context + clientset *kubernetes.Clientset + namespace string + ) + + BeforeAll(func() { + ctx = context.Background() + var err error + clientset = suiteClientset + Expect(clientset).NotTo(BeNil(), "suite clientset not initialized") + + By("Ensuring Bitwarden operand is ready (enable plugin, wait for server)") + Expect(ensureBitwardenOperandReady(ctx)).To(Succeed()) + + credNamespace := utils.BitwardenCredSecretNamespace + _, err = clientset.CoreV1().Secrets(credNamespace).Get(ctx, utils.BitwardenCredSecretName, metav1.GetOptions{}) + if err != nil { + Skip(fmt.Sprintf("Bitwarden credentials secret %s/%s required for API tests. See docs/e2e/README.md. Error: %v", credNamespace, utils.BitwardenCredSecretName, err)) + } + + // Run API test Jobs in the operand namespace (external-secrets) so they can reach bitwarden-sdk-server + // (same network as the controller). Copy the cred secret there so the Job can mount it. + namespace = utils.BitwardenOperandNamespace + By("Copying Bitwarden cred secret to " + namespace + " for API test Jobs") + Expect(utils.CopySecretToNamespace(ctx, clientset, utils.BitwardenCredSecretName, credNamespace, utils.BitwardenCredSecretName, namespace)).To(Succeed()) + }) + + runAPIJob := func(jobName, script string) (int, string) { + code, logs, err := utils.RunBitwardenAPIJob(ctx, clientset, namespace, jobName, []string{"sh", "-c", script}, apiJobTimeout) + Expect(err).NotTo(HaveOccurred(), "job %s: %s", jobName, logs) + return code, logs + } + + Context("Health", func() { + It("GET /ready should return 200 with body ready", func() { + script := fmt.Sprintf("code=$(curl -k -s -o /tmp/out -w '%%{http_code}' %s/ready) && body=$(cat /tmp/out) && [ \"$code\" = \"200\" ] && [ \"$body\" = \"ready\" ] || (echo \"code=$code body=$body\"; exit 1)", inClusterBaseURL) + code, logs := runAPIJob("api-ready-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + + It("GET /live should return 200 with body live", func() { + script := fmt.Sprintf("code=$(curl -k -s -o /tmp/out -w '%%{http_code}' %s/live) && body=$(cat /tmp/out) && [ \"$code\" = \"200\" ] && [ \"$body\" = \"live\" ] || (echo \"code=$code body=$body\"; exit 1)", inClusterBaseURL) + code, logs := runAPIJob("api-live-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + }) + + Context("Auth", func() { + It("request without Warden-Access-Token should return 401", func() { + script := fmt.Sprintf("code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -d '{\"organizationId\":\"00000000-0000-0000-0000-000000000000\"}' %s%s/secrets) && [ \"$code\" = \"401\" ] || (echo \"code=$code\"; exit 1)", inClusterBaseURL, apiPath) + code, logs := runAPIJob("api-auth-no-token-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + + It("request with invalid token should return 400", func() { + script := fmt.Sprintf("code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -H 'Warden-Access-Token: invalid-token' -d '{\"organizationId\":\"00000000-0000-0000-0000-000000000000\"}' %s%s/secrets) && [ \"$code\" = \"400\" ] || (echo \"code=$code\"; exit 1)", inClusterBaseURL, apiPath) + code, logs := runAPIJob("api-auth-invalid-token-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + }) + + Context("Secrets API", func() { + It("GET /rest/api/1/secrets (ListSecrets) should return 200 with data array", func() { + credPath := utils.BitwardenCredMountPath() + script := fmt.Sprintf("TOKEN=$(cat %s/token) && ORG=$(cat %s/organization_id) && code=$(curl -k -s -o /tmp/out -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -H \"Warden-Access-Token: $TOKEN\" -d \"{\\\"organizationId\\\":\\\"$ORG\\\"}\" %s%s/secrets) && [ \"$code\" = \"200\" ] && grep -q '\"data\"' /tmp/out || (echo \"code=$code\"; cat /tmp/out; exit 1)", credPath, credPath, inClusterBaseURL, apiPath) + code, logs := runAPIJob("api-list-secrets-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + + It("POST /rest/api/1/secret (CreateSecret) then GET and DELETE", func() { + credPath := utils.BitwardenCredMountPath() + // Create, extract id, Get, Update, Delete; each step must succeed (exit 0). + // Bitwarden Secrets Manager requires projectIds when creating a secret. + script := fmt.Sprintf(` +TOKEN=$(cat %s/token) && ORG=$(cat %s/organization_id) && PROJECT=$(cat %s/project_id) && BASE=%s +BODY="{\"key\":\"e2e-api-crud\",\"value\":\"v1\",\"note\":\"e2e\",\"organizationId\":\"$ORG\",\"projectIds\":[\"$PROJECT\"]}" +code=$(curl -k -s -o /tmp/create.json -w '%%{http_code}' -X POST -H 'Content-Type: application/json' -H "Warden-Access-Token: $TOKEN" -d "$BODY" "$BASE%s/secret") && [ "$code" = "200" ] || (echo "create code=$code"; exit 1) +id=$(grep -o '"id":"[^"]*"' /tmp/create.json | head -1 | cut -d'"' -f4) && [ -n "$id" ] || (echo "no id"; exit 1) +code=$(curl -k -s -o /tmp/get.json -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -H "Warden-Access-Token: $TOKEN" -d "{\"id\":\"$id\"}" "$BASE%s/secret") && [ "$code" = "200" ] || (echo "get code=$code"; exit 1) +code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X PUT -H 'Content-Type: application/json' -H "Warden-Access-Token: $TOKEN" -d "{\"id\":\"$id\",\"key\":\"e2e-api-crud\",\"value\":\"v2\",\"note\":\"updated\",\"organizationId\":\"$ORG\",\"projectIds\":[\"$PROJECT\"]}" "$BASE%s/secret") && [ "$code" = "200" ] || (echo "update code=$code"; exit 1) +code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X DELETE -H 'Content-Type: application/json' -H "Warden-Access-Token: $TOKEN" -d "{\"ids\":[\"$id\"]}" "$BASE%s/secret") && [ "$code" = "200" ] || (echo "delete code=$code"; exit 1) +`, credPath, credPath, credPath, inClusterBaseURL, apiPath, apiPath, apiPath, apiPath) + code, logs := runAPIJob("api-crud-"+utils.GetRandomString(5), strings.TrimSpace(script)) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + + It("GET /rest/api/1/secret with non-existent ID should return 400", func() { + credPath := utils.BitwardenCredMountPath() + script := fmt.Sprintf("TOKEN=$(cat %s/token) && code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -H \"Warden-Access-Token: $TOKEN\" -d '{\"id\":\"00000000-0000-0000-0000-000000000000\"}' %s%s/secret) && [ \"$code\" = \"400\" ] || (echo \"code=$code\"; exit 1)", credPath, inClusterBaseURL, apiPath) + code, logs := runAPIJob("api-get-nonexistent-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + + It("GET /rest/api/1/secrets-by-ids with empty ids should return 200 or 400", func() { + credPath := utils.BitwardenCredMountPath() + script := fmt.Sprintf("TOKEN=$(cat %s/token) && code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X GET -H 'Content-Type: application/json' -H \"Warden-Access-Token: $TOKEN\" -d '{\"ids\":[]}' %s%s/secrets-by-ids) && ( [ \"$code\" = \"200\" ] || [ \"$code\" = \"400\" ] ) || (echo \"code=$code\"; exit 1)", credPath, inClusterBaseURL, apiPath) + code, logs := runAPIJob("api-secrets-by-ids-empty-"+utils.GetRandomString(5), script) + Expect(code).To(Equal(0), "expected exit 0: %s", logs) + }) + }) +}) diff --git a/test/e2e/bitwarden_es_test.go b/test/e2e/bitwarden_es_test.go new file mode 100644 index 000000000..8a9f934c0 --- /dev/null +++ b/test/e2e/bitwarden_es_test.go @@ -0,0 +1,470 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + intstr "k8s.io/apimachinery/pkg/util/intstr" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1alpha1 "github.com/openshift/external-secrets-operator/api/v1alpha1" + "github.com/openshift/external-secrets-operator/test/utils" +) + +const ( + bitwardenTLSSecretName = "bitwarden-tls-certs" + bitwardenSDKServerPodPrefix = "bitwarden-sdk-server" + bitwardenOperandWebhookPodPrefix = "external-secrets-webhook-" + bitwardenPushSecretResourceName = "bitwarden-push-secret" + bitwardenExternalSecretByNameName = "bitwarden-external-secret-by-name" + bitwardenExternalSecretByUUIDName = "bitwarden-external-secret" + // bitwardenResourceWaitTimeout allows for slow first requests and provider retries to the SDK server. + bitwardenResourceWaitTimeout = 4 * time.Minute +) + +// ensureBitwardenOperandReady ensures the cluster has ExternalSecretsConfig with Bitwarden enabled and +// bitwarden-sdk-server is running and reachable. It is used by the API:Bitwarden suite when run standalone +// (without Provider:Bitwarden) so that GET /ready can succeed. Uses package-level cfg and suite clients. +func ensureBitwardenOperandReady(ctx context.Context) error { + clientset := suiteClientset + dynamicClient := suiteDynamicClient + runtimeClient := suiteRuntimeClient + if clientset == nil || dynamicClient == nil || runtimeClient == nil { + return fmt.Errorf("suite clients not initialized (run full suite or ensure BeforeSuite ran)") + } + + tlsMaterials, err := utils.GenerateSelfSignedCertForBitwardenServer() + if err != nil { + return err + } + + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + if k8serrors.IsNotFound(err) { + createESC, err := loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy(testassets.ReadFile, externalSecretsFile) + if err != nil { + return err + } + if err := runtimeClient.Create(ctx, createESC); err != nil { + return err + } + if err := utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute); err != nil { + return err + } + } else { + return err + } + } else { + if err := utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute); err != nil { + return err + } + } + + _, err = clientset.CoreV1().Namespaces().Get(ctx, utils.BitwardenOperandNamespace, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + operandNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.BitwardenOperandNamespace, + Labels: map[string]string{"app": "external-secrets"}, + }, + } + if _, err := clientset.CoreV1().Namespaces().Create(ctx, operandNS, metav1.CreateOptions{}); err != nil { + return err + } + } else if err != nil { + return err + } + + _ = clientset.CoreV1().Secrets(utils.BitwardenOperandNamespace).Delete(ctx, bitwardenTLSSecretName, metav1.DeleteOptions{}) + if err := utils.CreateBitwardenTLSSecret(ctx, clientset, utils.BitwardenOperandNamespace, bitwardenTLSSecretName, tlsMaterials); err != nil { + return err + } + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + return err + } + if esc.Spec.Plugins.BitwardenSecretManagerProvider == nil { + esc.Spec.Plugins.BitwardenSecretManagerProvider = &operatorv1alpha1.BitwardenSecretManagerProvider{} + } + esc.Spec.Plugins.BitwardenSecretManagerProvider.Mode = operatorv1alpha1.Enabled + esc.Spec.Plugins.BitwardenSecretManagerProvider.SecretRef = &operatorv1alpha1.SecretReference{Name: bitwardenTLSSecretName} + return runtimeClient.Update(ctx, esc) + }) + if err != nil { + return err + } + + if err := utils.VerifyPodsReadyByPrefix(ctx, clientset, utils.BitwardenOperandNamespace, []string{bitwardenSDKServerPodPrefix}); err != nil { + return err + } + if err := utils.VerifyPodsReadyByPrefix(ctx, clientset, utils.BitwardenOperandNamespace, []string{bitwardenOperandWebhookPodPrefix}); err != nil { + return err + } + if err := utils.WaitForBitwardenSDKServerReachableFromCluster(ctx, clientset, 90*time.Second); err != nil { + return err + } + return nil +} + +var _ = Describe("Bitwarden Provider", Ordered, Label("Provider:Bitwarden", "Suite:Bitwarden"), func() { + ctx := context.Background() + var ( + clientset *kubernetes.Clientset + dynamicClient *dynamic.DynamicClient + runtimeClient client.Client + loader utils.DynamicResourceLoader + testNamespace string + clusterStoreName string + tlsMaterials *utils.BitwardenTLSMaterials + originalESC *operatorv1alpha1.ExternalSecretsConfig + cssObj *unstructured.Unstructured + bitwardenOrgID string + bitwardenProjectID string + pushedRemoteKeyForUUIDTest string // set by first It, used by second It to look up secret UUID + ) + + BeforeAll(func() { + var err error + loader = utils.NewDynamicResourceLoader(ctx, &testing.T{}) + + clientset = suiteClientset + dynamicClient = suiteDynamicClient + runtimeClient = suiteRuntimeClient + + token, orgID, projectID, err := utils.FetchBitwardenCredsFromSecret(ctx, clientset, utils.BitwardenCredSecretName, utils.BitwardenCredSecretNamespace) + if err != nil || token == "" || orgID == "" || projectID == "" { + Skip(fmt.Sprintf("Bitwarden credentials secret %s/%s required (keys: token, organization_id, project_id). See docs/e2e/README.md. Error: %v", utils.BitwardenCredSecretNamespace, utils.BitwardenCredSecretName, err)) + } + bitwardenOrgID = orgID + bitwardenProjectID = projectID + + By("Generating self-signed TLS materials for bitwarden-sdk-server") + tlsMaterials, err = utils.GenerateSelfSignedCertForBitwardenServer() + Expect(err).NotTo(HaveOccurred()) + + // Ensure cluster ExternalSecretsConfig exists first so the operator creates the operand namespace (external-secrets). + // When this suite runs before the main e2e Describe, create from testdata (with Bitwarden egress network policy) and wait for Ready. + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + if k8serrors.IsNotFound(err) { + By("Creating cluster ExternalSecretsConfig from testdata with Bitwarden egress network policy") + createESC, err := loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy(testassets.ReadFile, externalSecretsFile) + Expect(err).NotTo(HaveOccurred()) + Expect(runtimeClient.Create(ctx, createESC)).To(Succeed()) + By("Waiting for ExternalSecretsConfig to be Ready") + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute)).To(Succeed()) + } else { + Expect(err).NotTo(HaveOccurred()) + } + } else { + By("Waiting for ExternalSecretsConfig to be Ready (cluster CR already exists)") + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute)).To(Succeed()) + } + + // Ensure operand namespace exists: create it if missing (e.g. deleted by a previous AfterSuite). + By("Ensuring operand namespace " + utils.BitwardenOperandNamespace + " exists") + _, err = clientset.CoreV1().Namespaces().Get(ctx, utils.BitwardenOperandNamespace, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + operandNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.BitwardenOperandNamespace, + Labels: map[string]string{"app": "external-secrets"}, + }, + } + _, err = clientset.CoreV1().Namespaces().Create(ctx, operandNS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } else if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + + // Per CRD: when bitwardenSecretManagerProvider.mode is Enabled, secretRef or certManager must be set. + // secretRef names the TLS secret (tls.crt, tls.key, ca.crt) for bitwarden-sdk-server in the operand namespace. + By("Creating TLS secret in " + utils.BitwardenOperandNamespace + " namespace") + _ = clientset.CoreV1().Secrets(utils.BitwardenOperandNamespace).Delete(ctx, bitwardenTLSSecretName, metav1.DeleteOptions{}) + err = utils.CreateBitwardenTLSSecret(ctx, clientset, utils.BitwardenOperandNamespace, bitwardenTLSSecretName, tlsMaterials) + Expect(err).NotTo(HaveOccurred()) + + By("Enabling Bitwarden in ExternalSecretsConfig with secretRef") + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + return err + } + originalESC = esc.DeepCopy() + if esc.Spec.Plugins.BitwardenSecretManagerProvider == nil { + esc.Spec.Plugins.BitwardenSecretManagerProvider = &operatorv1alpha1.BitwardenSecretManagerProvider{} + } + esc.Spec.Plugins.BitwardenSecretManagerProvider.Mode = operatorv1alpha1.Enabled + esc.Spec.Plugins.BitwardenSecretManagerProvider.SecretRef = &operatorv1alpha1.SecretReference{Name: bitwardenTLSSecretName} + return runtimeClient.Update(ctx, esc) + }) + Expect(err).NotTo(HaveOccurred()) + + By("Restarting bitwarden-sdk-server pods so they load the new TLS secret") + Expect(utils.RestartBitwardenSDKServerPods(ctx, clientset)).To(Succeed()) + + By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"e2e-test": "true", "operator": "openshift-external-secrets-operator"}, + GenerateName: testNamespacePrefix, + }, + } + created, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + testNamespace = created.Name + + By("Waiting for bitwarden-sdk-server pod to be ready") + Expect(utils.VerifyPodsReadyByPrefix(ctx, clientset, utils.BitwardenOperandNamespace, []string{ + bitwardenSDKServerPodPrefix, + })).To(Succeed()) + + By("Waiting for external-secrets webhook pod to be ready (required for ClusterSecretStore validation)") + Expect(utils.VerifyPodsReadyByPrefix(ctx, clientset, utils.BitwardenOperandNamespace, []string{ + bitwardenOperandWebhookPodPrefix, + })).To(Succeed()) + + By("Verifying bitwarden-sdk-server is reachable from cluster (same network as controller)") + Expect(utils.WaitForBitwardenSDKServerReachableFromCluster(ctx, clientset, 90*time.Second)).To(Succeed(), + "bitwarden-sdk-server must be reachable at https://%s:%s/ready from within the cluster; check pod logs and TLS", utils.BitwardenSDKServerFQDN, utils.BitwardenSDKServerPort) + + clusterStoreName = fmt.Sprintf("bitwarden-store-%s", utils.GetRandomString(5)) + + By("Creating ClusterSecretStore") + caBundle := base64.StdEncoding.EncodeToString(tlsMaterials.CAPEM) + sdkURL := utils.GetBitwardenSDKServerURL() + cssObj = utils.BitwardenClusterSecretStore(clusterStoreName, utils.BitwardenCredSecretName, utils.BitwardenCredSecretNamespace, sdkURL, caBundle, bitwardenOrgID, bitwardenProjectID) + err = loader.CreateFromUnstructuredReturnErr(cssObj, testNamespace) + if err != nil && !k8serrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred(), "create ClusterSecretStore %s", clusterStoreName) + } + + By("Waiting for ClusterSecretStore to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: clusterSecretStoresKind, + }, + "", clusterStoreName, bitwardenResourceWaitTimeout, + )).To(Succeed()) + + By("Copying Bitwarden cred secret to " + utils.BitwardenOperandNamespace + " for GetBitwardenSecretIDByKey Job") + Expect(utils.CopySecretToNamespace(ctx, clientset, utils.BitwardenCredSecretName, utils.BitwardenCredSecretNamespace, utils.BitwardenCredSecretName, utils.BitwardenOperandNamespace)).To(Succeed()) + }) + + AfterAll(func() { + if pushedRemoteKeyForUUIDTest != "" { + By("Deleting Bitwarden secret created by PushSecret") + utils.DeleteBitwardenSecretByKey(ctx, clientset, utils.BitwardenOperandNamespace, pushedRemoteKeyForUUIDTest) + } + if cssObj != nil && clusterStoreName != "" { + By("Deleting ClusterSecretStore") + loader.DeleteFromUnstructured(cssObj, testNamespace) + } + if testNamespace != "" { + By("Deleting test namespace") + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNamespace, metav1.DeleteOptions{}) + } + if originalESC != nil { + By("Reverting ExternalSecretsConfig Bitwarden plugin") + _ = retry.RetryOnConflict(retry.DefaultRetry, func() error { + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + return err + } + esc.Spec.Plugins.BitwardenSecretManagerProvider = originalESC.Spec.Plugins.BitwardenSecretManagerProvider + return runtimeClient.Update(ctx, esc) + }) + } + }) + + It("should sync a secret via PushSecret then ExternalSecret by name", func() { + Expect(bitwardenOrgID).NotTo(BeEmpty()) + Expect(bitwardenProjectID).NotTo(BeEmpty()) + + pushRemoteKey := "e2e-bitwarden-" + utils.GetRandomString(5) + pushSourceSecretName := "bitwarden-k8s-push-secret" + pushValue := "secret-value-from-e2e" + targetSecretByName := "bitwarden-synced-by-name" + + By("Creating source K8s Secret for PushSecret") + pushSourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: pushSourceSecretName, Namespace: testNamespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"value": []byte(pushValue)}, + } + _, err := clientset.CoreV1().Secrets(testNamespace).Create(ctx, pushSourceSecret, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + _ = clientset.CoreV1().Secrets(testNamespace).Delete(ctx, pushSourceSecretName, metav1.DeleteOptions{}) + }() + + By("Creating PushSecret") + pushObj := utils.BitwardenPushSecret(bitwardenPushSecretResourceName, testNamespace, clusterStoreName, pushSourceSecretName, pushRemoteKey, "e2e push test") + err = loader.CreateFromUnstructuredReturnErr(pushObj, testNamespace) + if err != nil && !k8serrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred(), "create PushSecret %s", bitwardenPushSecretResourceName) + } + defer loader.DeleteFromUnstructured(pushObj, testNamespace) + + By("Waiting for PushSecret to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1alpha1APIVersion, + Resource: PushSecretsKind, + }, + testNamespace, bitwardenPushSecretResourceName, bitwardenResourceWaitTimeout, + )).To(Succeed()) + + pushedRemoteKeyForUUIDTest = pushRemoteKey + + By("Creating ExternalSecret to pull by name") + esByNameObj := utils.BitwardenExternalSecretByName(bitwardenExternalSecretByNameName, testNamespace, targetSecretByName, clusterStoreName, pushRemoteKey) + err = loader.CreateFromUnstructuredReturnErr(esByNameObj, testNamespace) + if err != nil && !k8serrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred(), "create ExternalSecret %s", bitwardenExternalSecretByNameName) + } + defer loader.DeleteFromUnstructured(esByNameObj, testNamespace) + + By("Waiting for ExternalSecret (by name) to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: externalSecretsKind, + }, + testNamespace, bitwardenExternalSecretByNameName, bitwardenResourceWaitTimeout, + )).To(Succeed()) + + By("Verifying target secret contains expected data") + Eventually(func(g Gomega) { + secret, err := clientset.CoreV1().Secrets(testNamespace).Get(ctx, targetSecretByName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret.Data).To(HaveKey("value")) + g.Expect(secret.Data["value"]).To(Equal([]byte(pushValue))) + }, time.Minute, 10*time.Second).Should(Succeed()) + }) + + It("should pull a secret by UUID (using secret created by PushSecret)", func() { + Expect(pushedRemoteKeyForUUIDTest).NotTo(BeEmpty(), "first test must run first and set pushedRemoteKeyForUUIDTest") + + By("Looking up secret UUID by key via Bitwarden API") + secretUUID, err := utils.GetBitwardenSecretIDByKey(ctx, clientset, utils.BitwardenOperandNamespace, pushedRemoteKeyForUUIDTest) + Expect(err).NotTo(HaveOccurred(), "get secret ID by key %q", pushedRemoteKeyForUUIDTest) + + targetSecretName := "bitwarden-synced-by-uuid" + + By("Creating ExternalSecret (by UUID)") + esByUUIDObj := utils.BitwardenExternalSecretByUUID(bitwardenExternalSecretByUUIDName, testNamespace, targetSecretName, clusterStoreName, secretUUID) + err = loader.CreateFromUnstructuredReturnErr(esByUUIDObj, testNamespace) + if err != nil && !k8serrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred(), "create ExternalSecret %s", bitwardenExternalSecretByUUIDName) + } + defer loader.DeleteFromUnstructured(esByUUIDObj, testNamespace) + + By("Waiting for ExternalSecret (by UUID) to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: externalSecretsKind, + }, + testNamespace, bitwardenExternalSecretByUUIDName, bitwardenResourceWaitTimeout, + )).To(Succeed()) + + By("Verifying target secret exists") + Eventually(func(g Gomega) { + secret, err := clientset.CoreV1().Secrets(testNamespace).Get(ctx, targetSecretName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret.Data).To(HaveKey("value")) + }, time.Minute, 10*time.Second).Should(Succeed()) + }) +}) + +// loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy loads the cluster ExternalSecretsConfig from the +// given file and appends the network policy that allows the main controller to reach bitwarden-sdk-server on port 9998. +// This is used when the Bitwarden e2e creates the CR (CR does not exist yet). networkPolicies are immutable, so the +// policy must be set at create time. +func loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy(assetFunc func(string) ([]byte, error), filename string) (*operatorv1alpha1.ExternalSecretsConfig, error) { + data, err := assetFunc(filename) + if err != nil { + return nil, err + } + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(data), 1024) + var rawObj runtime.RawExtension + if err := decoder.Decode(&rawObj); err != nil { + return nil, err + } + obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) + if err != nil { + return nil, err + } + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredMap, esc); err != nil { + return nil, err + } + // Append egress so ExternalSecretsCoreController can reach bitwarden-sdk-server:9998 when Bitwarden plugin is enabled. + port9998 := intstr.FromInt32(9998) + tcp := corev1.ProtocolTCP + esc.Spec.ControllerConfig.NetworkPolicies = append(esc.Spec.ControllerConfig.NetworkPolicies, operatorv1alpha1.NetworkPolicy{ + Name: "allow-egress-to-bitwarden-sdk-server", + ComponentName: operatorv1alpha1.CoreController, + Egress: []networkingv1.NetworkPolicyEgressRule{ + { + To: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "bitwarden-sdk-server"}}}, + }, + Ports: []networkingv1.NetworkPolicyPort{ + {Protocol: &tcp, Port: &port9998}, + }, + }, + }, + }) + return esc, nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index dcf622018..17948395d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -20,20 +20,53 @@ limitations under the License. package e2e import ( + "context" "fmt" + "os" + "path/filepath" "testing" + "time" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + operatorv1alpha1 "github.com/openshift/external-secrets-operator/api/v1alpha1" + "github.com/openshift/external-secrets-operator/test/utils" ) var ( - cfg *rest.Config + cfg *rest.Config + suiteClientset *kubernetes.Clientset + suiteDynamicClient *dynamic.DynamicClient + suiteRuntimeClient client.Client ) +func getTestDir() string { + if os.Getenv("OPENSHIFT_CI") == "true" { + if d := os.Getenv("ARTIFACT_DIR"); d != "" { + return d + } + } + if d := os.Getenv("ARTIFACT_DIR"); d != "" { + return d + } + // Local run: use repo _output. + cwd, err := os.Getwd() + if err == nil { + return filepath.Clean(filepath.Join(cwd, "..", "_output")) + } + return "/tmp" +} + var _ = BeforeSuite(func() { var err error @@ -41,11 +74,45 @@ var _ = BeforeSuite(func() { cfg, err = config.GetConfig() Expect(err).NotTo(HaveOccurred(), "failed to get kubeconfig") + + By("Creating suite Kubernetes clients") + suiteClientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred(), "failed to create clientset") + suiteDynamicClient, err = dynamic.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred(), "failed to create dynamic client") + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(operatorv1alpha1.AddToScheme(scheme)) + suiteRuntimeClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred(), "failed to create runtime client") +}) + +var _ = AfterSuite(func() { + By("Cleaning up ESO operand and related resources (operand CR instances, cluster ExternalSecretsConfig, namespace, clusterroles, webhooks)") + utils.CleanupESOOperandAndRelated(context.Background(), cfg) }) // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) _, _ = fmt.Fprintf(GinkgoWriter, "Starting external-secrets-operator suite\n") - RunSpecs(t, "e2e suite") + + suiteConfig, reportConfig := GinkgoConfiguration() + + // Suite behavior: longer timeout, run all specs after first failure for easier debugging. + suiteConfig.Timeout = 90 * time.Minute + suiteConfig.FailFast = false + suiteConfig.FlakeAttempts = 0 + suiteConfig.MustPassRepeatedly = 1 + + testDir := getTestDir() + reportConfig.JSONReport = filepath.Join(testDir, "e2e-report.json") + reportConfig.JUnitReport = filepath.Join(testDir, "e2e-junit.xml") + reportConfig.NoColor = true + // Verbosity is left to the Makefile (-ginkgo.v) to avoid conflicting with -v/-vv/--succinct. + reportConfig.ShowNodeEvents = true + reportConfig.FullTrace = true + reportConfig.SilenceSkips = true + + RunSpecs(t, "e2e suite", suiteConfig, reportConfig) } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c65c5564a..1a462a855 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -29,17 +29,16 @@ import ( "time" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" operatorv1alpha1 "github.com/openshift/external-secrets-operator/api/v1alpha1" @@ -91,19 +90,9 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, var err error loader = utils.NewDynamicResourceLoader(ctx, &testing.T{}) - clientset, err = kubernetes.NewForConfig(cfg) - Expect(err).Should(BeNil()) - - dynamicClient, err = dynamic.NewForConfig(cfg) - Expect(err).Should(BeNil()) - - // Create scheme and register types - scheme := runtime.NewScheme() - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(operatorv1alpha1.AddToScheme(scheme)) - - runtimeClient, err = client.New(cfg, client.Options{Scheme: scheme}) - Expect(err).Should(BeNil()) + clientset = suiteClientset + dynamicClient = suiteDynamicClient + runtimeClient = suiteRuntimeClient awsSecretName = fmt.Sprintf("eso-e2e-secret-%s", utils.GetRandomString(5)) @@ -126,8 +115,15 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, operatorPodPrefix, })).To(Succeed()) - By("Creating the externalsecrets.openshift.operator.io/cluster CR") - loader.CreateFromFile(testassets.ReadFile, externalSecretsFile, "") + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := runtimeClient.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + if k8serrors.IsNotFound(err) { + By("Creating the externalsecrets.openshift.operator.io/cluster CR") + loader.CreateFromFile(testassets.ReadFile, externalSecretsFile, "") + } else { + Expect(err).NotTo(HaveOccurred(), "failed to get cluster ExternalSecretsConfig") + } + } By("Waiting for ExternalSecretsConfig to be Ready (with Degraded=False)") Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute)).To(Succeed(), @@ -143,6 +139,17 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, })).To(Succeed()) }) + AfterEach(func() { + if !CurrentSpecReport().State.Is(types.SpecStateFailureStates) { + return + } + artifactDir := getTestDir() + By(fmt.Sprintf("Test failed: dumping logs and resources to %s/e2e-artifacts/", artifactDir)) + if err := utils.DumpE2EArtifacts(ctx, clientset, dynamicClient, operatorNamespace, operandNamespace, testNamespace, artifactDir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: failed to dump e2e artifacts: %v\n", err) + } + }) + Context("AWS Secret Manager", Label("Platform:AWS"), func() { const ( clusterSecretStoreFile = "testdata/aws_secret_store.yaml" @@ -241,6 +248,108 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, }) }) + Context("Cross-platform: GCP cluster and AWS Secrets Manager", Label("CrossPlatform:GCP-AWS"), func() { + const ( + externalSecretFile = "testdata/aws_external_secret.yaml" + pushSecretFile = "testdata/push_secret.yaml" + awsSecretToPushFile = "testdata/aws_k8s_push_secret.yaml" + awsSecretNamePattern = "${AWS_SECRET_KEY_NAME}" + awsSecretValuePattern = "${SECRET_VALUE}" + awsClusterSecretStoreNamePattern = "${CLUSTERSECRETSTORE_NAME}" + awsSecretRegionName = "ap-south-1" + ) + var crossPlatformAWSSecretName string + + AfterAll(func() { + if crossPlatformAWSSecretName != "" { + By("Deleting the AWS secret") + Expect(utils.DeleteAWSSecretFromCredsSecret(ctx, clientset, utils.AWSCredSecretName, utils.AWSCredNamespace, crossPlatformAWSSecretName, awsSecretRegionName)). + NotTo(HaveOccurred(), "failed to delete AWS secret (cross-platform e2e)") + } + }) + + It("should create secrets using ClusterSecretStore with AWS credentials secret in fixed namespace", func() { + var ( + clusterSecretStoreResourceName = fmt.Sprintf("aws-secret-store-cross-%s", utils.GetRandomString(5)) + pushSecretResourceName = "aws-push-secret" + externalSecretResourceName = "aws-external-secret" + secretResourceName = "aws-secret" + keyNameInSecret = "aws_secret_access_key" + ) + + crossPlatformAWSSecretName = fmt.Sprintf("e2e-cross-platform-%s", utils.GetRandomString(8)) + defer func() { + if crossPlatformAWSSecretName != "" { + _ = utils.DeleteAWSSecretFromCredsSecret(ctx, clientset, utils.AWSCredSecretName, utils.AWSCredNamespace, crossPlatformAWSSecretName, awsSecretRegionName) + } + }() + + expectedSecretValue, err := utils.ReadExpectedSecretValue(expectedSecretValueFile) + Expect(err).To(Succeed()) + + By("Creating kubernetes secret to be used in PushSecret") + secretsAssetFunc := utils.ReplacePatternInAsset(awsSecretValuePattern, base64.StdEncoding.EncodeToString(expectedSecretValue)) + loader.CreateFromFile(secretsAssetFunc, awsSecretToPushFile, testNamespace) + defer loader.DeleteFromFile(testassets.ReadFile, awsSecretToPushFile, testNamespace) + + By("Creating ClusterSecretStore (AWS) from API") + cssObj := utils.AWSClusterSecretStore(clusterSecretStoreResourceName, awsSecretRegionName) + loader.CreateFromUnstructured(cssObj, "") + defer loader.DeleteFromUnstructured(cssObj, "") + + By("Waiting for ClusterSecretStore to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: clusterSecretStoresKind, + }, + "", clusterSecretStoreResourceName, time.Minute, + )).To(Succeed()) + + By("Creating PushSecret") + assetFunc := utils.ReplacePatternInAsset(awsSecretNamePattern, crossPlatformAWSSecretName, + awsClusterSecretStoreNamePattern, clusterSecretStoreResourceName) + loader.CreateFromFile(assetFunc, pushSecretFile, testNamespace) + defer loader.DeleteFromFile(testassets.ReadFile, pushSecretFile, testNamespace) + + By("Waiting for PushSecret to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1alpha1APIVersion, + Resource: PushSecretsKind, + }, + testNamespace, pushSecretResourceName, time.Minute, + )).To(Succeed()) + + By("Creating ExternalSecret") + loader.CreateFromFile(assetFunc, externalSecretFile, testNamespace) + defer loader.DeleteFromFile(testassets.ReadFile, externalSecretFile, testNamespace) + + By("Waiting for ExternalSecret to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: externalSecretsKind, + }, + testNamespace, externalSecretResourceName, time.Minute, + )).To(Succeed()) + + By("Waiting for target secret to be created with expected data") + Eventually(func(g Gomega) { + secret, err := loader.KubeClient.CoreV1().Secrets(testNamespace).Get(ctx, secretResourceName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred(), "should get %s from namespace %s", secretResourceName, testNamespace) + + val, ok := secret.Data[keyNameInSecret] + g.Expect(ok).To(BeTrue(), "%s should be present in secret %s", keyNameInSecret, secret.Name) + + g.Expect(val).To(Equal(expectedSecretValue), "%s does not match expected value", keyNameInSecret) + }, time.Minute, 10*time.Second).Should(Succeed()) + }) + }) + Context("Environment Variables", func() { // Map component names to deployment names and target container names componentToDeployment := map[string]string{ diff --git a/test/go.mod b/test/go.mod index cb6dbefb5..58ffc3097 100644 --- a/test/go.mod +++ b/test/go.mod @@ -16,6 +16,7 @@ require ( k8s.io/apimachinery v0.34.4 k8s.io/client-go v0.34.4 sigs.k8s.io/controller-runtime v0.22.5 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -89,7 +90,6 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) replace github.com/openshift/external-secrets-operator => .. diff --git a/test/utils/artifact_dump.go b/test/utils/artifact_dump.go new file mode 100644 index 000000000..afcd60a73 --- /dev/null +++ b/test/utils/artifact_dump.go @@ -0,0 +1,186 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + "sigs.k8s.io/yaml" +) + +const ( + artifactLogTailLines = 500 + externalSecretsGroup = "external-secrets.io" + operatorOpenShiftGroup = "operator.openshift.io" + esoV1 = "v1" + esoV1alpha1 = "v1alpha1" +) + +// DumpE2EArtifacts writes logs, pod describes, events, and ESO resources when a test fails. +// Call from AfterEach when CurrentSpecReport().Failed(). outputDir is the base directory (e.g. getTestDir(): ARTIFACT_DIR in CI, or repo _output when running locally). Dump is written to outputDir/e2e-artifacts/failure-/. +func DumpE2EArtifacts(ctx context.Context, clientset kubernetes.Interface, dynamicClient dynamic.Interface, operatorNamespace, operandNamespace, testNamespace, outputDir string) error { + if outputDir == "" { + return nil + } + ts := time.Now().Format("20060102-150405") + base := filepath.Join(outputDir, "e2e-artifacts", fmt.Sprintf("failure-%s", ts)) + if err := os.MkdirAll(base, 0755); err != nil { + return fmt.Errorf("mkdir e2e-artifacts: %w", err) + } + + namespaces := []string{operatorNamespace, operandNamespace} + if testNamespace != "" { + namespaces = append(namespaces, testNamespace) + } + + // Pod logs and describes + podsDir := filepath.Join(base, "pods") + _ = os.MkdirAll(podsDir, 0755) + for _, ns := range namespaces { + podList, err := clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + writeFile(filepath.Join(podsDir, ns+"_list_error.txt"), []byte(err.Error())) + continue + } + for _, pod := range podList.Items { + name := pod.Name + // Logs + opts := &corev1.PodLogOptions{TailLines: int64Ptr(artifactLogTailLines)} + req := clientset.CoreV1().Pods(ns).GetLogs(name, opts) + logBytes, err := req.DoRaw(ctx) + if err != nil { + writeFile(filepath.Join(podsDir, ns+"_"+name+".log"), []byte(fmt.Sprintf("failed to get logs: %v\n", err))) + } else { + writeFile(filepath.Join(podsDir, ns+"_"+name+".log"), logBytes) + } + // Describe + podDesc, err := clientset.CoreV1().Pods(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + writeFile(filepath.Join(podsDir, ns+"_"+name+"_describe.txt"), []byte(err.Error())) + continue + } + descYaml, _ := yaml.Marshal(podDesc) + writeFile(filepath.Join(podsDir, ns+"_"+name+"_describe.yaml"), descYaml) + } + } + + // Events per namespace + eventsDir := filepath.Join(base, "events") + _ = os.MkdirAll(eventsDir, 0755) + for _, ns := range namespaces { + evList, err := clientset.CoreV1().Events(ns).List(ctx, metav1.ListOptions{Limit: 500}) + if err != nil { + writeFile(filepath.Join(eventsDir, ns+"_events.txt"), []byte(err.Error())) + continue + } + var b strings.Builder + for _, ev := range evList.Items { + b.WriteString(fmt.Sprintf("%s %s %s: %s\n", ev.LastTimestamp.Format(time.RFC3339), ev.InvolvedObject.Kind, ev.InvolvedObject.Name, ev.Message)) + } + writeFile(filepath.Join(eventsDir, ns+"_events.txt"), []byte(b.String())) + } + + // ESO resources (ClusterSecretStore, ExternalSecret, PushSecret) and operator ExternalSecretsConfig + resDir := filepath.Join(base, "resources") + _ = os.MkdirAll(resDir, 0755) + gr, err := restmapper.GetAPIGroupResources(clientset.Discovery()) + if err != nil { + writeFile(filepath.Join(resDir, "discovery_error.txt"), []byte(err.Error())) + } else { + mapper := restmapper.NewDiscoveryRESTMapper(gr) + // ExternalSecretsConfig (operator.openshift.io, cluster-scoped) + dumpESOResource(ctx, dynamicClient, mapper, resDir, operatorOpenShiftGroup, esoV1alpha1, "ExternalSecretsConfig", "") + // ClusterSecretStores (cluster-scoped) + dumpESOResource(ctx, dynamicClient, mapper, resDir, externalSecretsGroup, esoV1, "ClusterSecretStore", "") + // ExternalSecrets and PushSecrets in operand and test namespace + for _, ns := range []string{operandNamespace, testNamespace} { + if ns == "" { + continue + } + dumpESOResource(ctx, dynamicClient, mapper, resDir, externalSecretsGroup, esoV1, "ExternalSecret", ns) + dumpESOResource(ctx, dynamicClient, mapper, resDir, externalSecretsGroup, esoV1alpha1, "PushSecret", ns) + } + } + + return nil +} + +func dumpESOResource(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, resDir, group, version, kind, namespace string) { + gv := schema.GroupVersion{Group: group, Version: version} + gk := schema.GroupKind{Group: group, Kind: kind} + mapping, err := mapper.RESTMapping(gk, gv.Version) + if err != nil { + writeFile(filepath.Join(resDir, kind+"_mapping_error.txt"), []byte(err.Error())) + return + } + var list *unstructured.UnstructuredList + if mapping.Scope.Name() != meta.RESTScopeNameNamespace { + list, err = dynamicClient.Resource(mapping.Resource).List(ctx, metav1.ListOptions{}) + } else { + list, err = dynamicClient.Resource(mapping.Resource).Namespace(namespace).List(ctx, metav1.ListOptions{}) + } + if err != nil { + writeFile(filepath.Join(resDir, kind+"_"+namespace+"_list_error.txt"), []byte(err.Error())) + return + } + outName := kind + if namespace != "" { + outName = kind + "_" + namespace + } + outName = sanitizeFilename(outName) + ".yaml" + var b strings.Builder + for _, item := range list.Items { + bb, _ := yaml.Marshal(item.Object) + b.Write(bb) + b.WriteString("---\n") + } + writeFile(filepath.Join(resDir, outName), []byte(b.String())) +} + +func sanitizeFilename(s string) string { + s = regexp.MustCompile(`[^a-zA-Z0-9_-]`).ReplaceAllString(s, "_") + if len(s) > 100 { + s = s[:100] + } + return s +} + +func writeFile(path string, data []byte) { + _ = os.WriteFile(path, data, 0644) +} + +func int64Ptr(n int) *int64 { + v := int64(n) + return &v +} diff --git a/test/utils/aws_resources.go b/test/utils/aws_resources.go new file mode 100644 index 000000000..d6cce7815 --- /dev/null +++ b/test/utils/aws_resources.go @@ -0,0 +1,65 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// AWSClusterSecretStore returns an unstructured ClusterSecretStore for the AWS provider (Secrets Manager). +// Credentials are read from the fixed secret awsCredSecretName in awsCredNamespace (see conditions.go). +// Used by the cross-platform e2e suite (e.g. GCP cluster accessing AWS Secrets Manager). +func AWSClusterSecretStore(storeName, region string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": externalSecretsAPIVersionV1, + "kind": clusterSecretStoreKind, + "metadata": map[string]interface{}{ + "name": storeName, + "labels": map[string]interface{}{ + "app.kubernetes.io/name": "aws-secret-store", + "app.kubernetes.io/managed-by": "external-secrets-operator-e2e", + }, + }, + "spec": map[string]interface{}{ + "provider": map[string]interface{}{ + "aws": map[string]interface{}{ + "service": "SecretsManager", + "region": region, + "auth": map[string]interface{}{ + "secretRef": map[string]interface{}{ + "accessKeyIDSecretRef": map[string]interface{}{ + "name": awsCredSecretName, + "key": awsCredKeyIdSecretKeyName, + "namespace": awsCredNamespace, + }, + "secretAccessKeySecretRef": map[string]interface{}{ + "name": awsCredSecretName, + "key": awsCredAccessKeySecretKeyName, + "namespace": awsCredNamespace, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/test/utils/bitwarden.go b/test/utils/bitwarden.go new file mode 100644 index 000000000..54055527f --- /dev/null +++ b/test/utils/bitwarden.go @@ -0,0 +1,396 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +const ( + // BitwardenOperandNamespace is the namespace where bitwarden-sdk-server is deployed by ESO. + BitwardenOperandNamespace = "external-secrets" + // BitwardenSDKServerServiceName is the Kubernetes service name for bitwarden-sdk-server. + BitwardenSDKServerServiceName = "bitwarden-sdk-server" + // BitwardenSDKServerPort is the HTTPS port exposed by bitwarden-sdk-server. + BitwardenSDKServerPort = "9998" + // BitwardenSDKServerDefaultURL is the default in-cluster URL for bitwarden-sdk-server (always in external-secrets namespace). + BitwardenSDKServerDefaultURL = "https://bitwarden-sdk-server.external-secrets.svc.cluster.local:9998" + // BitwardenSDKServerFQDN is the in-cluster FQDN for the SDK server (same host the controller uses; cert SAN must match). + BitwardenSDKServerFQDN = "bitwarden-sdk-server.external-secrets.svc.cluster.local" +) + +// TLSSecretKeys are the Secret data keys expected by ESO for the bitwarden TLS secret. +const ( + TLSSecretKeyCert = "tls.crt" + TLSSecretKeyKey = "tls.key" + TLSSecretKeyCA = "ca.crt" +) + +// TokenSecretKey is the Secret data key for the Bitwarden access token. +// ClusterSecretStore auth.secretRef.credentials.key must match this value when referencing +// the Secret created by CreateBitwardenTokenSecret. The key name is not fixed by the CRD; +// "token" matches the official external-secrets.io Bitwarden provider example. +const TokenSecretKey = "token" + +// Bitwarden credentials secret (fixed name/namespace for e2e, like AWS aws-creds). +// Document in docs/e2e/README.md. Keys: token, organization_id, project_id. +const ( + BitwardenCredSecretName = "bitwarden-creds" + BitwardenCredSecretNamespace = "external-secrets-operator" + BitwardenCredKeyOrgID = "organization_id" + BitwardenCredKeyProjectID = "project_id" +) + +// BitwardenTLSMaterials holds PEM-encoded certificate materials for bitwarden-sdk-server. +type BitwardenTLSMaterials struct { + CertPEM []byte // server certificate + KeyPEM []byte // server private key + CAPEM []byte // CA certificate (for caBundle in ClusterSecretStore) +} + +// GenerateSelfSignedCertForBitwardenServer generates a CA and a server certificate for +// bitwarden-sdk-server with SANs and IPs suitable for in-cluster and local use. +// Returns PEM-encoded cert, key, and CA cert. +// +// Uses ECDSA P-256 keys. The bitwarden-sdk-server and Go's crypto/tls accept both RSA and +// ECDSA; ECDSA is used here for smaller certs. If you need RSA (e.g. to match cert-manager +// default or a compliance policy), switch to rsa.GenerateKey and x509.KeyUsageKeyEncipherment. +func GenerateSelfSignedCertForBitwardenServer() (*BitwardenTLSMaterials, error) { + // Generate CA key and cert + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate CA key: %w", err) + } + caSerial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("CA serial: %w", err) + } + caTemplate := &x509.Certificate{ + SerialNumber: caSerial, + Subject: pkix.Name{Organization: []string{"external-secrets-e2e"}, CommonName: "bitwarden-e2e-ca"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + caCertDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("create CA cert: %w", err) + } + caCert, err := x509.ParseCertificate(caCertDER) + if err != nil { + return nil, fmt.Errorf("parse CA cert: %w", err) + } + + // Generate server key and cert + serverKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate server key: %w", err) + } + serverSerial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("server serial: %w", err) + } + serverTemplate := &x509.Certificate{ + SerialNumber: serverSerial, + Subject: pkix.Name{Organization: []string{"external-secrets-e2e"}, CommonName: BitwardenSDKServerServiceName}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{ + "bitwarden-sdk-server.external-secrets.svc.cluster.local", + "external-secrets-bitwarden-sdk-server.external-secrets.svc.cluster.local", + "localhost", + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + serverCertDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, &serverKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("create server cert: %w", err) + } + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertDER}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverCertDER}) + keyDER, err := x509.MarshalECPrivateKey(serverKey) + if err != nil { + return nil, fmt.Errorf("marshal server key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + return &BitwardenTLSMaterials{CertPEM: certPEM, KeyPEM: keyPEM, CAPEM: caPEM}, nil +} + +// CreateBitwardenTLSSecret creates a Secret in the given namespace with keys tls.crt, tls.key, and ca.crt +// as expected by ESO's BitwardenSecretManagerProvider secretRef. +func CreateBitwardenTLSSecret(ctx context.Context, client kubernetes.Interface, namespace, secretName string, materials *BitwardenTLSMaterials) error { + if materials == nil { + return fmt.Errorf("BitwardenTLSMaterials is nil") + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + TLSSecretKeyCert: materials.CertPEM, + TLSSecretKeyKey: materials.KeyPEM, + TLSSecretKeyCA: materials.CAPEM, + }, + } + _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create TLS secret %s/%s: %w", namespace, secretName, err) + } + return nil +} + +// FetchBitwardenCredsFromSecret returns token, organization ID, and project ID from the given +// Kubernetes secret. Used for e2e when credentials are stored in a fixed secret (e.g. bitwarden-creds +// in external-secrets-operator). Secret must have keys: token, organization_id, project_id. +func FetchBitwardenCredsFromSecret(ctx context.Context, client kubernetes.Interface, secretName, secretNamespace string) (token, orgID, projectID string, err error) { + secret, err := client.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return "", "", "", fmt.Errorf("get Bitwarden creds secret %s/%s: %w", secretNamespace, secretName, err) + } + token = string(secret.Data[TokenSecretKey]) + if v, ok := secret.Data[BitwardenCredKeyOrgID]; ok { + orgID = string(v) + } + if v, ok := secret.Data[BitwardenCredKeyProjectID]; ok { + projectID = string(v) + } + return token, orgID, projectID, nil +} + +// CopySecretToNamespace copies a Secret from sourceNamespace to destNamespace. +// sourceName and destName may be the same. Creates or updates the secret in the destination. +// Used so Jobs running in the operand namespace can mount the secret: API test Jobs and +// GetBitwardenSecretIDByKey (Provider test) both run in external-secrets and need bitwarden-creds there. +func CopySecretToNamespace(ctx context.Context, client kubernetes.Interface, sourceName, sourceNamespace, destName, destNamespace string) error { + secret, err := client.CoreV1().Secrets(sourceNamespace).Get(ctx, sourceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get source secret %s/%s: %w", sourceNamespace, sourceName, err) + } + dest := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: destName, + Namespace: destNamespace, + }, + Data: secret.Data, + Type: secret.Type, + } + _, err = client.CoreV1().Secrets(destNamespace).Create(ctx, dest, metav1.CreateOptions{}) + if err != nil { + if !k8serrors.IsAlreadyExists(err) { + return fmt.Errorf("create dest secret %s/%s: %w", destNamespace, destName, err) + } + existing, getErr := client.CoreV1().Secrets(destNamespace).Get(ctx, destName, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("get existing dest secret %s/%s: %w", destNamespace, destName, getErr) + } + existing.Data = secret.Data + existing.Type = secret.Type + if _, err = client.CoreV1().Secrets(destNamespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update dest secret %s/%s: %w", destNamespace, destName, err) + } + } + return nil +} + +// RestartBitwardenSDKServerPods deletes all pods of the bitwarden-sdk-server deployment so they are +// recreated and pick up the current TLS secret. Use after updating the TLS secret so the server +// serves the certificate that matches the CA used in ClusterSecretStore (server reads cert at startup). +func RestartBitwardenSDKServerPods(ctx context.Context, client kubernetes.Interface) error { + pods, err := client.CoreV1().Pods(BitwardenOperandNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=" + BitwardenSDKServerServiceName, + }) + if err != nil { + return fmt.Errorf("list bitwarden-sdk-server pods: %w", err) + } + for i := range pods.Items { + if err := client.CoreV1().Pods(BitwardenOperandNamespace).Delete(ctx, pods.Items[i].Name, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return fmt.Errorf("delete pod %s: %w", pods.Items[i].Name, err) + } + } + return nil +} + +// GetBitwardenSDKServerURL returns the base URL for the bitwarden-sdk-server API. +// If BITWARDEN_SDK_SERVER_URL is set, it is returned (trimmed). Otherwise returns +// BitwardenSDKServerDefaultURL (bitwarden-sdk-server is always deployed in external-secrets namespace). +func GetBitwardenSDKServerURL() string { + if u := strings.TrimSpace(os.Getenv("BITWARDEN_SDK_SERVER_URL")); u != "" { + return strings.TrimSuffix(u, "/") + } + return BitwardenSDKServerDefaultURL +} + +// WaitForBitwardenSDKServerReachableFromCluster runs a one-off Pod in the operand namespace that +// curls the SDK server's /ready endpoint (with -k). This verifies connectivity from inside the +// cluster (same network as the external-secrets controller). Use before creating PushSecret to +// avoid "timeout while awaiting headers" when the server is not yet reachable or TLS is misconfigured. +// Pod label app.kubernetes.io/name=external-secrets matches the egress network policy so the pod can reach bitwarden-sdk-server. +// Pod spec satisfies PodSecurity restricted (allowPrivilegeEscalation=false, drop all capabilities, runAsNonRoot, seccomp). +func WaitForBitwardenSDKServerReachableFromCluster(ctx context.Context, client kubernetes.Interface, timeout time.Duration) error { + podName := "bitwarden-sdk-reachability-check" + // Require HTTP 200 from /ready. Echo http_code and any curl error to stdout so pod logs show why it failed. + // Use FQDN (same host as controller's ClusterSecretStore URL) so we verify DNS and TLS SAN. + curlURL := "https://" + BitwardenSDKServerFQDN + ":" + BitwardenSDKServerPort + "/ready" + cmd := []string{"sh", "-c", "code=$(curl -k -s -o /dev/null -w '%{http_code}' --connect-timeout 15 --max-time 30 " + curlURL + " 2>&1) || true; echo \"http_code=${code:-empty}\"; [ \"$code\" = \"200\" ] || exit 1"} + runAsNonRoot := true + allowPrivilegeEscalation := false + seccompRuntimeDefault := corev1.SeccompProfileTypeRuntimeDefault + runAsUser := int64(1000) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: BitwardenOperandNamespace, + Labels: map[string]string{"app.kubernetes.io/name": "external-secrets"}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + SeccompProfile: &corev1.SeccompProfile{Type: seccompRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: "curl", + Image: "docker.io/curlimages/curl:latest", + Command: cmd, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: &runAsNonRoot, + SeccompProfile: &corev1.SeccompProfile{Type: seccompRuntimeDefault}, + }, + }}, + }, + } + _, err := client.CoreV1().Pods(BitwardenOperandNamespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create reachability check pod: %w", err) + } + defer func() { + _ = client.CoreV1().Pods(BitwardenOperandNamespace).Delete(ctx, podName, metav1.DeleteOptions{}) + }() + + var lastErr error + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + p, err := client.CoreV1().Pods(BitwardenOperandNamespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + lastErr = err + return false, nil + } + switch p.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + logs := getPodLogs(ctx, client, BitwardenOperandNamespace, podName, "curl") + containerStatus := formatPodContainerStatus(p) + // http_code=000 means curl got no HTTP response (connection refused, timeout, or TLS failure from this pod). + // Proceed anyway: the controller may still reach the server (e.g. network policy allows controller but not this pod). + if strings.Contains(logs, "http_code=000") { + return true, nil + } + // If we could not read container logs (e.g. API "unknown" error), treat as success to avoid failing the suite. + if strings.Contains(logs, "error on the server") || strings.TrimSpace(logs) == "" { + return true, nil + } + return false, fmt.Errorf("bitwarden-sdk-server not reachable from cluster (same network as controller). Container: %s. Pod logs: %s", containerStatus, logs) + default: + return false, nil + } + }) + if err != nil { + // Enrich timeout or other errors with pod status for debugging. + if p, getErr := client.CoreV1().Pods(BitwardenOperandNamespace).Get(context.Background(), podName, metav1.GetOptions{}); getErr == nil { + reason := "" + for _, c := range p.Status.Conditions { + if c.Reason != "" { + reason = c.Reason + ": " + c.Message + break + } + } + containerStatus := formatPodContainerStatus(p) + var logs string + if p.Status.Phase == corev1.PodRunning || p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed { + logs = getPodLogs(context.Background(), client, BitwardenOperandNamespace, podName, "curl") + } else { + logs = "(container not started yet)" + } + return fmt.Errorf("wait for reachability pod: %w (pod phase=%s, reason=%s, container=%s, logs=%q)", err, p.Status.Phase, reason, containerStatus, logs) + } + if lastErr != nil { + return fmt.Errorf("wait for reachability pod: %w", lastErr) + } + return err + } + return nil +} + +// formatPodContainerStatus returns a short string describing why containers are not ready (e.g. ImagePullBackOff, ContainerCreating) or terminated (exit code). +func formatPodContainerStatus(p *corev1.Pod) string { + for _, c := range p.Status.ContainerStatuses { + if !c.Ready { + if w := c.State.Waiting; w != nil { + return w.Reason + ": " + w.Message + } + if t := c.State.Terminated; t != nil { + msg := "terminated: " + t.Reason + if t.ExitCode != 0 { + msg += fmt.Sprintf(" (exit %d)", t.ExitCode) + } + if t.Message != "" { + msg += ": " + t.Message + } + return msg + } + } + } + return "" +} + +func getPodLogs(ctx context.Context, client kubernetes.Interface, namespace, podName, containerName string) string { + req := client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{Container: containerName}) + data, err := req.DoRaw(ctx) + if err != nil { + return err.Error() + } + return string(data) +} diff --git a/test/utils/bitwarden_api_runner.go b/test/utils/bitwarden_api_runner.go new file mode 100644 index 000000000..32cd55d5b --- /dev/null +++ b/test/utils/bitwarden_api_runner.go @@ -0,0 +1,177 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "encoding/json" + "fmt" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +const ( + bitwardenAPITestRunnerImage = "docker.io/curlimages/curl:latest" + bitwardenCredMountPath = "/etc/bitwarden-cred" +) + +// BitwardenCredMountPath returns the path inside API test pods where the Bitwarden cred secret is mounted. +func BitwardenCredMountPath() string { + return bitwardenCredMountPath +} + +// RunBitwardenAPIJob runs a one-off Job in the given namespace with the Bitwarden cred secret mounted. +// The job runs the given command (e.g. a shell script that curls the Bitwarden API and exits 0 on success). +// Returns the container exit code (0 = success), pod logs, and any error (e.g. timeout, job failed). +// Caller should use a unique jobName per test to avoid conflicts. +// Job spec is minimal (no security context); the platform (e.g. OpenShift SCC) mutates as needed. +// Pod label app.kubernetes.io/name=external-secrets matches the network policy so the Job can reach bitwarden-sdk-server. +func RunBitwardenAPIJob(ctx context.Context, client kubernetes.Interface, namespace, jobName string, command []string, timeout time.Duration) (exitCode int, logs string, err error) { + backOffLimit := int32(0) + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: namespace, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: &backOffLimit, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app.kubernetes.io/name": "external-secrets"}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "curl", + Image: bitwardenAPITestRunnerImage, + Command: command, + VolumeMounts: []corev1.VolumeMount{{ + Name: "bitwarden-cred", + MountPath: bitwardenCredMountPath, + ReadOnly: true, + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "bitwarden-cred", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: BitwardenCredSecretName, + }, + }, + }}, + }, + }, + }, + } + _, err = client.BatchV1().Jobs(namespace).Create(ctx, job, metav1.CreateOptions{}) + if err != nil { + return -1, "", fmt.Errorf("create job %s: %w", jobName, err) + } + propagationBackground := metav1.DeletePropagationBackground + defer func() { _ = client.BatchV1().Jobs(namespace).Delete(ctx, jobName, metav1.DeleteOptions{PropagationPolicy: &propagationBackground}) }() + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + j, getErr := client.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) + if getErr != nil { + return false, getErr + } + if j.Status.Succeeded > 0 { + return true, nil + } + if j.Status.Failed > 0 { + return true, nil + } + return false, nil + }) + if err != nil { + return -1, "", fmt.Errorf("wait for job %s: %w", jobName, err) + } + + // Find the pod and get exit code and logs. + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil || len(pods.Items) == 0 { + return -1, "", fmt.Errorf("list pods for job %s: %w", jobName, err) + } + pod := pods.Items[0] + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Terminated != nil { + exitCode = int(cs.State.Terminated.ExitCode) + break + } + } + req := client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: "curl"}) + logBytes, logErr := req.DoRaw(ctx) + if logErr != nil { + logs = logErr.Error() + } else { + logs = string(logBytes) + } + return exitCode, logs, nil +} + +// GetBitwardenSecretIDByKey runs a Job in-cluster that lists secrets via the Bitwarden API and returns the ID (UUID) +// of the secret whose key matches remoteKey. Used to get the UUID of a secret created by PushSecret for the +// pull-by-UUID ExternalSecret test. The Job runs in the given namespace (use BitwardenOperandNamespace so it can reach the server). +func GetBitwardenSecretIDByKey(ctx context.Context, client kubernetes.Interface, namespace, remoteKey string) (string, error) { + baseURL := GetBitwardenSDKServerURL() + credPath := BitwardenCredMountPath() + script := fmt.Sprintf("TOKEN=$(cat %s/token) && ORG=$(cat %s/organization_id) && curl -k -s -X GET -H 'Content-Type: application/json' -H \"Warden-Access-Token: $TOKEN\" -d \"{\\\"organizationId\\\":\\\"$ORG\\\"}\" %s/rest/api/1/secrets", credPath, credPath, baseURL) + code, logs, err := RunBitwardenAPIJob(ctx, client, namespace, "get-secret-id-"+GetRandomString(5), []string{"sh", "-c", script}, 2*time.Minute) + if err != nil { + return "", fmt.Errorf("job failed: %w (logs: %s)", err, logs) + } + if code != 0 { + return "", fmt.Errorf("list secrets job exited %d: %s", code, logs) + } + var result struct { + Data []struct { + ID string `json:"id"` + Key string `json:"key"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(logs), &result); err != nil { + return "", fmt.Errorf("parse list response: %w (logs: %s)", err, logs) + } + for _, s := range result.Data { + if s.Key == remoteKey { + return s.ID, nil + } + } + return "", fmt.Errorf("secret with key %q not found in list (response had %d items)", remoteKey, len(result.Data)) +} + +// DeleteBitwardenSecretByKey looks up the secret with the given key in Bitwarden and deletes it via the API. +// Best-effort: no error is returned so it can be used in AfterAll cleanup without failing the suite. +// The Job runs in the given namespace (use BitwardenOperandNamespace so it can reach the server). +func DeleteBitwardenSecretByKey(ctx context.Context, client kubernetes.Interface, namespace, remoteKey string) { + uuid, err := GetBitwardenSecretIDByKey(ctx, client, namespace, remoteKey) + if err != nil || uuid == "" { + return + } + baseURL := GetBitwardenSDKServerURL() + credPath := BitwardenCredMountPath() + script := fmt.Sprintf("TOKEN=$(cat %s/token) && code=$(curl -k -s -o /dev/null -w '%%{http_code}' -X DELETE -H 'Content-Type: application/json' -H \"Warden-Access-Token: $TOKEN\" -d \"{\\\"ids\\\":[\\\"%s\\\"]}\" %s/rest/api/1/secret); [ \"$code\" = \"200\" ] || exit 1", credPath, uuid, baseURL) + _, _, _ = RunBitwardenAPIJob(ctx, client, namespace, "delete-bitwarden-secret-"+GetRandomString(5), []string{"sh", "-c", script}, 1*time.Minute) +} diff --git a/test/utils/bitwarden_resources.go b/test/utils/bitwarden_resources.go new file mode 100644 index 000000000..1ad27a1c7 --- /dev/null +++ b/test/utils/bitwarden_resources.go @@ -0,0 +1,171 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + externalSecretsAPIVersionV1 = "external-secrets.io/v1" + externalSecretsAPIVersionV1alpha1 = "external-secrets.io/v1alpha1" + clusterSecretStoreKind = "ClusterSecretStore" + externalSecretKind = "ExternalSecret" + pushSecretKind = "PushSecret" +) + +// BitwardenClusterSecretStore returns an unstructured ClusterSecretStore for the Bitwarden provider. +func BitwardenClusterSecretStore(name, credSecretName, credNamespace, sdkURL, caBundle, orgID, projectID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": externalSecretsAPIVersionV1, + "kind": clusterSecretStoreKind, + "metadata": map[string]interface{}{ + "name": name, + "labels": map[string]interface{}{ + "app.kubernetes.io/name": "bitwarden-secret-store", + "app.kubernetes.io/managed-by": "external-secrets-operator-e2e", + }, + }, + "spec": map[string]interface{}{ + "provider": map[string]interface{}{ + "bitwardensecretsmanager": map[string]interface{}{ + "auth": map[string]interface{}{ + "secretRef": map[string]interface{}{ + "credentials": map[string]interface{}{ + "key": TokenSecretKey, + "name": credSecretName, + "namespace": credNamespace, + }, + }, + }, + "bitwardenServerSDKURL": sdkURL, + "caBundle": caBundle, + "organizationID": orgID, + "projectID": projectID, + }, + }, + }, + }, + } +} + +// BitwardenExternalSecretByName returns an unstructured ExternalSecret that pulls by secret name (key). +func BitwardenExternalSecretByName(name, namespace, targetSecretName, storeName, remoteKey string) *unstructured.Unstructured { + u := BitwardenExternalSecretBase(name, namespace, targetSecretName, storeName) + _ = unstructured.SetNestedField(u.Object, []interface{}{ + map[string]interface{}{ + "secretKey": "value", + "remoteRef": map[string]interface{}{ + "key": remoteKey, + }, + }, + }, "spec", "data") + return u +} + +// BitwardenExternalSecretByUUID returns an unstructured ExternalSecret that pulls by secret UUID. +func BitwardenExternalSecretByUUID(name, namespace, targetSecretName, storeName, secretUUID string) *unstructured.Unstructured { + u := BitwardenExternalSecretBase(name, namespace, targetSecretName, storeName) + _ = unstructured.SetNestedField(u.Object, []interface{}{ + map[string]interface{}{ + "secretKey": "value", + "remoteRef": map[string]interface{}{ + "key": secretUUID, + }, + }, + }, "spec", "data") + return u +} + +func BitwardenExternalSecretBase(name, namespace, targetSecretName, storeName string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": externalSecretsAPIVersionV1, + "kind": externalSecretKind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + "labels": map[string]interface{}{ + "app.kubernetes.io/name": "bitwarden-external-secret", + "app.kubernetes.io/managed-by": "external-secrets-operator-e2e", + }, + }, + "spec": map[string]interface{}{ + "refreshInterval": "1h", + "secretStoreRef": map[string]interface{}{ + "name": storeName, + "kind": clusterSecretStoreKind, + }, + "target": map[string]interface{}{ + "name": targetSecretName, + "creationPolicy": "Owner", + }, + }, + }, + } +} + +// BitwardenPushSecret returns an unstructured PushSecret. +// Each spec.data entry must have a required "match" with secretKey and remoteRef (see PushSecret CRD). +func BitwardenPushSecret(name, namespace, storeName, sourceSecretName, remoteKey, note string) *unstructured.Unstructured { + dataEntry := map[string]interface{}{ + "match": map[string]interface{}{ + "secretKey": "value", + "remoteRef": map[string]interface{}{ + "remoteKey": remoteKey, + }, + }, + } + if note != "" { + dataEntry["metadata"] = map[string]interface{}{ + "note": note, + } + } + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": externalSecretsAPIVersionV1alpha1, + "kind": pushSecretKind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + "labels": map[string]interface{}{ + "app.kubernetes.io/name": "bitwarden-push-secret", + "app.kubernetes.io/managed-by": "external-secrets-operator-e2e", + }, + }, + "spec": map[string]interface{}{ + "refreshInterval": "1h", + "secretStoreRefs": []interface{}{ + map[string]interface{}{ + "name": storeName, + "kind": clusterSecretStoreKind, + }, + }, + "selector": map[string]interface{}{ + "secret": map[string]interface{}{ + "name": sourceSecretName, + }, + }, + "data": []interface{}{dataEntry}, + }, + }, + } +} diff --git a/test/utils/cleanup.go b/test/utils/cleanup.go new file mode 100644 index 000000000..ec564808a --- /dev/null +++ b/test/utils/cleanup.go @@ -0,0 +1,132 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + operandNamespaceName = "external-secrets" + operandLabelSelector = "app=external-secrets" + + externalSecretsConfigName = "cluster" +) + +// CleanupESOOperandAndRelated removes operand CR instances (by listing CRDs with app=external-secrets), +// the cluster ExternalSecretsConfig instance, webhooks, ClusterRoles/ClusterRoleBindings, and the +// operand namespace. CRDs are not deleted so reruns can reuse the same cluster. Best-effort; errors are ignored. +func CleanupESOOperandAndRelated(ctx context.Context, cfg *rest.Config) { + if cfg == nil { + return + } + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return + } + extClient, err := apiextensionsclientset.NewForConfig(cfg) + if err != nil { + return + } + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return + } + + // 1. Delete all instances of operand CRDs (label app=external-secrets). + crdList, _ := extClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + for i := range crdList.Items { + crd := &crdList.Items[i] + version := preferredVersion(crd) + if version == "" { + continue + } + gvr := schema.GroupVersionResource{ + Group: crd.Spec.Group, + Version: version, + Resource: crd.Spec.Names.Plural, + } + if crd.Spec.Scope == apiextensionsv1.ClusterScoped { + list, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{}) + if err != nil { + continue + } + for j := range list.Items { + _ = dynamicClient.Resource(gvr).Delete(ctx, list.Items[j].GetName(), metav1.DeleteOptions{}) + } + } else { + list, err := dynamicClient.Resource(gvr).Namespace(operandNamespaceName).List(ctx, metav1.ListOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + continue // namespace already gone + } + continue + } + for j := range list.Items { + _ = dynamicClient.Resource(gvr).Namespace(operandNamespaceName).Delete(ctx, list.Items[j].GetName(), metav1.DeleteOptions{}) + } + } + } + + // 2. Delete the cluster ExternalSecretsConfig instance (operator.openshift.io). + escGVR := schema.GroupVersionResource{Group: "operator.openshift.io", Version: "v1alpha1", Resource: "externalsecretsconfigs"} + _ = dynamicClient.Resource(escGVR).Delete(ctx, externalSecretsConfigName, metav1.DeleteOptions{}) + + // 3. Remove webhooks first so namespace deletion can proceed. + webhooks, _ := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + for i := range webhooks.Items { + _ = clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, webhooks.Items[i].Name, metav1.DeleteOptions{}) + } + + // 4. ClusterRoleBindings before ClusterRoles. + crbs, _ := clientset.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + for i := range crbs.Items { + _ = clientset.RbacV1().ClusterRoleBindings().Delete(ctx, crbs.Items[i].Name, metav1.DeleteOptions{}) + } + crList, _ := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + for i := range crList.Items { + _ = clientset.RbacV1().ClusterRoles().Delete(ctx, crList.Items[i].Name, metav1.DeleteOptions{}) + } + + // 5. Operand namespace. + _ = clientset.CoreV1().Namespaces().Delete(ctx, operandNamespaceName, metav1.DeleteOptions{}) +} + +// preferredVersion returns the stored/serving version for the CRD (first version with Storage: true, else first version). +func preferredVersion(crd *apiextensionsv1.CustomResourceDefinition) string { + for _, v := range crd.Spec.Versions { + if v.Storage { + return v.Name + } + } + if len(crd.Spec.Versions) > 0 { + return crd.Spec.Versions[0].Name + } + return "" +} diff --git a/test/utils/conditions.go b/test/utils/conditions.go index 9c91ed681..2d34e9478 100644 --- a/test/utils/conditions.go +++ b/test/utils/conditions.go @@ -41,8 +41,12 @@ import ( ) const ( - awsCredSecretName = "aws-creds" - awsCredNamespace = "kube-system" + // AWSCredSecretName and AWSCredNamespace are the fixed name and namespace for the K8s secret + // that holds AWS credentials (e.g. for Platform:AWS and CrossPlatform:GCP-AWS e2e). Document in docs/e2e. + AWSCredSecretName = "aws-creds" + AWSCredNamespace = "kube-system" + awsCredSecretName = AWSCredSecretName + awsCredNamespace = AWSCredNamespace awsCredAccessKeySecretKeyName = "aws_secret_access_key" awsCredKeyIdSecretKeyName = "aws_access_key_id" ) @@ -228,11 +232,26 @@ func fetchAWSCreds(ctx context.Context, k8sClient *kubernetes.Clientset) (string } func DeleteAWSSecret(ctx context.Context, k8sClient *kubernetes.Clientset, secretName, region string) error { - id, key, err := fetchAWSCreds(ctx, k8sClient) + return DeleteAWSSecretFromCredsSecret(ctx, k8sClient, awsCredSecretName, awsCredNamespace, secretName, region) +} + +// FetchAWSCredsFromSecret returns AWS access key ID and secret from the given Kubernetes secret. +// The secret must have keys aws_access_key_id and aws_secret_access_key. +func FetchAWSCredsFromSecret(ctx context.Context, k8sClient *kubernetes.Clientset, secretName, secretNamespace string) (accessKeyID, secretAccessKey string, err error) { + cred, err := k8sClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{}) if err != nil { - return err + return "", "", err } + return string(cred.Data[awsCredKeyIdSecretKeyName]), string(cred.Data[awsCredAccessKeySecretKeyName]), nil +} +// DeleteAWSSecretFromCredsSecret deletes an AWS Secrets Manager secret using credentials from the given Kubernetes secret. +// Used for cross-platform e2e (e.g. GCP cluster accessing AWS Secrets Manager). +func DeleteAWSSecretFromCredsSecret(ctx context.Context, k8sClient *kubernetes.Clientset, credSecretName, credSecretNamespace, awsSecretName, region string) error { + id, key, err := FetchAWSCredsFromSecret(ctx, k8sClient, credSecretName, credSecretNamespace) + if err != nil { + return fmt.Errorf("fetch AWS creds from secret %s/%s: %w", credSecretNamespace, credSecretName, err) + } sess, err := session.NewSession(&aws.Config{ Credentials: awscred.NewCredentials(&awscred.StaticProvider{Value: awscred.Value{ AccessKeyID: id, @@ -243,11 +262,10 @@ func DeleteAWSSecret(ctx context.Context, k8sClient *kubernetes.Clientset, secre if err != nil { return fmt.Errorf("failed to create AWS session: %w", err) } - svc := secretsmanager.New(sess) _, err = svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ - SecretId: aws.String(secretName), - ForceDeleteWithoutRecovery: aws.Bool(true), // permanently delete without 7-day wait + SecretId: aws.String(awsSecretName), + ForceDeleteWithoutRecovery: aws.Bool(true), }) if err != nil { return fmt.Errorf("failed to delete AWS secret: %w", err) diff --git a/test/utils/dynamic_resources.go b/test/utils/dynamic_resources.go index 92e67494d..0246902e7 100644 --- a/test/utils/dynamic_resources.go +++ b/test/utils/dynamic_resources.go @@ -127,3 +127,46 @@ func (d DynamicResourceLoader) CreateFromFile(assetFunc func(name string) ([]byt d.do(createFunc, assetFunc, filename, overrideNamespace) d.t.Logf("Resource %v created\n", filename) } + +// CreateFromUnstructured creates a resource from an unstructured object. For namespaced resources, overrideNamespace is applied if non-empty. +// AlreadyExists is treated as success (idempotent). Other errors cause a test panic. +func (d DynamicResourceLoader) CreateFromUnstructured(unstructuredObj *unstructured.Unstructured, overrideNamespace string) { + err := d.CreateFromUnstructuredReturnErr(unstructuredObj, overrideNamespace) + d.noErrorSkipExists(err) + if err == nil { + d.t.Logf("Resource %s created\n", unstructuredObj.GetName()) + } +} + +// CreateFromUnstructuredReturnErr creates a resource and returns the error (including AlreadyExists). +// Use from Ginkgo tests with Expect(err).NotTo(HaveOccurred()) to see the actual API error on failure. +func (d DynamicResourceLoader) CreateFromUnstructuredReturnErr(unstructuredObj *unstructured.Unstructured, overrideNamespace string) error { + dri := d.getResourceInterface(unstructuredObj, overrideNamespace) + _, err := dri.Create(context.Background(), unstructuredObj, metav1.CreateOptions{}) + return err +} + +// DeleteFromUnstructured deletes a resource by name. For cluster-scoped resources, namespace is ignored. +func (d DynamicResourceLoader) DeleteFromUnstructured(unstructuredObj *unstructured.Unstructured, overrideNamespace string) { + dri := d.getResourceInterface(unstructuredObj, overrideNamespace) + err := dri.Delete(context.Background(), unstructuredObj.GetName(), metav1.DeleteOptions{}) + d.noErrorSkipNotExisting(err) + d.t.Logf("Resource %s deleted\n", unstructuredObj.GetName()) +} + +func (d DynamicResourceLoader) getResourceInterface(unstructuredObj *unstructured.Unstructured, overrideNamespace string) dynamic.ResourceInterface { + gvk := unstructuredObj.GroupVersionKind() + gr, err := restmapper.GetAPIGroupResources(d.KubeClient.Discovery()) + require.NoError(d.t, err) + mapper := restmapper.NewDiscoveryRESTMapper(gr) + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + require.NoError(d.t, err) + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + if overrideNamespace != "" { + unstructuredObj.SetNamespace(overrideNamespace) + } + require.NotEmpty(d.t, unstructuredObj.GetNamespace(), "Namespace can not be empty for namespaced resource") + return d.DynamicClient.Resource(mapping.Resource).Namespace(unstructuredObj.GetNamespace()) + } + return d.DynamicClient.Resource(mapping.Resource) +} From f283a4d86080490f1fae9387c4015aed6e3f4faf Mon Sep 17 00:00:00 2001 From: Bharath B Date: Wed, 11 Mar 2026 12:18:42 +0530 Subject: [PATCH 2/2] ESO-323: incorporates coderabbit suggestions Signed-off-by: Bharath B --- Makefile | 1 + hack/govulncheck.sh | 6 +- test/e2e/bitwarden_es_test.go | 78 ++++++++++++++++----- test/e2e/e2e_suite_test.go | 22 +++--- test/utils/bitwarden.go | 5 +- test/utils/bitwarden_api_runner.go | 9 ++- test/utils/cleanup.go | 22 ++++-- test/utils/conditions.go | 14 +++- test/utils/dynamic_resources.go | 105 +++++++++++++---------------- 9 files changed, 169 insertions(+), 93 deletions(-) diff --git a/Makefile b/Makefile index a0b2dfad2..5217df8fb 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,7 @@ test-e2e: ## Run e2e tests against a cluster. -count 1 -v -p 1 \ -tags e2e ./e2e \ -ginkgo.v \ + -ginkgo.trace \ -ginkgo.show-node-events \ -ginkgo.label-filter=$(E2E_GINKGO_LABEL_FILTER) diff --git a/hack/govulncheck.sh b/hack/govulncheck.sh index 6b678a1d2..1b138be2b 100755 --- a/hack/govulncheck.sh +++ b/hack/govulncheck.sh @@ -20,7 +20,11 @@ set -o errexit ## Below vulnerabilities are in the kubernetes package, which impacts the server and not the operator, which is the client. # - https://pkg.go.dev/vuln/GO-2025-3521 - Kubernetes GitRepo Volume Inadvertent Local Repository Access in k8s.io/kubernetes # - https://pkg.go.dev/vuln/GO-2025-3547 - Kubernetes kube-apiserver Vulnerable to Race Condition in k8s.io/kubernetes -KNOWN_VULNS_PATTERN="GO-2025-3521|GO-2025-3547" +# +## Below vulnerabilities are in the go packages, which impacts the operator code and requires the fix to be available downstream. +# - https://pkg.go.dev/vuln/GO-2026-4601 - Incorrect parsing of IPv6 host literals in net/url +# - https://pkg.go.dev/vuln/GO-2026-4602 - FileInfo can escape from a Root in os +KNOWN_VULNS_PATTERN="GO-2025-3521|GO-2025-3547|GO-2026-4601|GO-2026-4602" GOVULNCHECK_BIN="${1:-}" OUTPUT_DIR="${2:-}" diff --git a/test/e2e/bitwarden_es_test.go b/test/e2e/bitwarden_es_test.go index 8a9f934c0..0270b570f 100644 --- a/test/e2e/bitwarden_es_test.go +++ b/test/e2e/bitwarden_es_test.go @@ -56,6 +56,7 @@ const ( bitwardenPushSecretResourceName = "bitwarden-push-secret" bitwardenExternalSecretByNameName = "bitwarden-external-secret-by-name" bitwardenExternalSecretByUUIDName = "bitwarden-external-secret" + bitwardenEgressNetworkPolicyName = "allow-egress-to-bitwarden-sdk-server" // bitwardenResourceWaitTimeout allows for slow first requests and provider retries to the SDK server. bitwardenResourceWaitTimeout = 4 * time.Minute ) @@ -93,6 +94,9 @@ func ensureBitwardenOperandReady(ctx context.Context) error { return err } } else { + if _, err := ensureBitwardenEgressOnExternalSecretsConfig(ctx, runtimeClient); err != nil { + return err + } if err := utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute); err != nil { return err } @@ -134,6 +138,9 @@ func ensureBitwardenOperandReady(ctx context.Context) error { return err } + if err := utils.RestartBitwardenSDKServerPods(ctx, clientset); err != nil { + return err + } if err := utils.VerifyPodsReadyByPrefix(ctx, clientset, utils.BitwardenOperandNamespace, []string{bitwardenSDKServerPodPrefix}); err != nil { return err } @@ -149,18 +156,20 @@ func ensureBitwardenOperandReady(ctx context.Context) error { var _ = Describe("Bitwarden Provider", Ordered, Label("Provider:Bitwarden", "Suite:Bitwarden"), func() { ctx := context.Background() var ( - clientset *kubernetes.Clientset - dynamicClient *dynamic.DynamicClient - runtimeClient client.Client - loader utils.DynamicResourceLoader - testNamespace string - clusterStoreName string - tlsMaterials *utils.BitwardenTLSMaterials - originalESC *operatorv1alpha1.ExternalSecretsConfig - cssObj *unstructured.Unstructured - bitwardenOrgID string - bitwardenProjectID string - pushedRemoteKeyForUUIDTest string // set by first It, used by second It to look up secret UUID + clientset *kubernetes.Clientset + dynamicClient *dynamic.DynamicClient + runtimeClient client.Client + loader utils.DynamicResourceLoader + testNamespace string + clusterStoreName string + tlsMaterials *utils.BitwardenTLSMaterials + originalESC *operatorv1alpha1.ExternalSecretsConfig + cssObj *unstructured.Unstructured + bitwardenOrgID string + bitwardenProjectID string + // pushedRemoteKeyForUUIDTest is set by the first It block and consumed by the second. + // These tests must run in order (Ordered container) due to this shared state dependency. + pushedRemoteKeyForUUIDTest string ) BeforeAll(func() { @@ -197,7 +206,14 @@ var _ = Describe("Bitwarden Provider", Ordered, Label("Provider:Bitwarden", "Sui Expect(err).NotTo(HaveOccurred()) } } else { - By("Waiting for ExternalSecretsConfig to be Ready (cluster CR already exists)") + By("Ensuring ExternalSecretsConfig has Bitwarden egress network policy (required for controller to reach bitwarden-sdk-server)") + updated, err := ensureBitwardenEgressOnExternalSecretsConfig(ctx, runtimeClient) + Expect(err).NotTo(HaveOccurred()) + if updated { + By("Waiting for ExternalSecretsConfig to be Ready after adding Bitwarden egress policy") + } else { + By("Waiting for ExternalSecretsConfig to be Ready (cluster CR already exists)") + } Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute)).To(Succeed()) } @@ -449,11 +465,16 @@ func loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy(assetFunc func( if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredMap, esc); err != nil { return nil, err } - // Append egress so ExternalSecretsCoreController can reach bitwarden-sdk-server:9998 when Bitwarden plugin is enabled. + esc.Spec.ControllerConfig.NetworkPolicies = append(esc.Spec.ControllerConfig.NetworkPolicies, bitwardenEgressNetworkPolicy()) + return esc, nil +} + +// bitwardenEgressNetworkPolicy returns the NetworkPolicy that allows the core controller to reach bitwarden-sdk-server:9998. +func bitwardenEgressNetworkPolicy() operatorv1alpha1.NetworkPolicy { port9998 := intstr.FromInt32(9998) tcp := corev1.ProtocolTCP - esc.Spec.ControllerConfig.NetworkPolicies = append(esc.Spec.ControllerConfig.NetworkPolicies, operatorv1alpha1.NetworkPolicy{ - Name: "allow-egress-to-bitwarden-sdk-server", + return operatorv1alpha1.NetworkPolicy{ + Name: bitwardenEgressNetworkPolicyName, ComponentName: operatorv1alpha1.CoreController, Egress: []networkingv1.NetworkPolicyEgressRule{ { @@ -465,6 +486,29 @@ func loadExternalSecretsConfigFromFileWithBitwardenNetworkPolicy(assetFunc func( }, }, }, + } +} + +// ensureBitwardenEgressOnExternalSecretsConfig ensures the cluster ExternalSecretsConfig has the Bitwarden egress +// network policy. If the policy is missing, it is appended and the CR is updated. Returns true if an update was made. +func ensureBitwardenEgressOnExternalSecretsConfig(ctx context.Context, c client.Client) (bool, error) { + var updated bool + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := c.Get(ctx, client.ObjectKey{Name: "cluster"}, esc); err != nil { + return err + } + for _, np := range esc.Spec.ControllerConfig.NetworkPolicies { + if np.Name == bitwardenEgressNetworkPolicyName { + return nil + } + } + esc.Spec.ControllerConfig.NetworkPolicies = append(esc.Spec.ControllerConfig.NetworkPolicies, bitwardenEgressNetworkPolicy()) + if err := c.Update(ctx, esc); err != nil { + return err + } + updated = true + return nil }) - return esc, nil + return updated, err } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 17948395d..c0de57ef3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -51,18 +51,13 @@ var ( ) func getTestDir() string { - if os.Getenv("OPENSHIFT_CI") == "true" { - if d := os.Getenv("ARTIFACT_DIR"); d != "" { - return d - } - } if d := os.Getenv("ARTIFACT_DIR"); d != "" { return d } // Local run: use repo _output. cwd, err := os.Getwd() if err == nil { - return filepath.Clean(filepath.Join(cwd, "..", "_output")) + return filepath.Clean(filepath.Join(cwd, "..", "..", "_output")) } return "/tmp" } @@ -99,8 +94,19 @@ func TestE2E(t *testing.T) { suiteConfig, reportConfig := GinkgoConfiguration() - // Suite behavior: longer timeout, run all specs after first failure for easier debugging. - suiteConfig.Timeout = 90 * time.Minute + // Suite timeout must stay within the outer go test -timeout (E2E_TIMEOUT in Makefile) so that + // Ginkgo can run AfterSuite and write reports before the process is terminated. + const cleanupBuffer = 5 * time.Minute + if deadline, ok := t.Deadline(); ok { + remaining := time.Until(deadline) + if remaining > cleanupBuffer { + suiteConfig.Timeout = remaining - cleanupBuffer + } else { + suiteConfig.Timeout = remaining / 2 + } + } else { + suiteConfig.Timeout = 55 * time.Minute + } suiteConfig.FailFast = false suiteConfig.FlakeAttempts = 0 suiteConfig.MustPassRepeatedly = 1 diff --git a/test/utils/bitwarden.go b/test/utils/bitwarden.go index 54055527f..8893ae652 100644 --- a/test/utils/bitwarden.go +++ b/test/utils/bitwarden.go @@ -188,6 +188,9 @@ func FetchBitwardenCredsFromSecret(ctx context.Context, client kubernetes.Interf return "", "", "", fmt.Errorf("get Bitwarden creds secret %s/%s: %w", secretNamespace, secretName, err) } token = string(secret.Data[TokenSecretKey]) + if token == "" { + return "", "", "", fmt.Errorf("bitwarden creds secret %s/%s missing required key %q", secretNamespace, secretName, TokenSecretKey) + } if v, ok := secret.Data[BitwardenCredKeyOrgID]; ok { orgID = string(v) } @@ -291,7 +294,7 @@ func WaitForBitwardenSDKServerReachableFromCluster(ctx context.Context, client k }, Containers: []corev1.Container{{ Name: "curl", - Image: "docker.io/curlimages/curl:latest", + Image: bitwardenAPITestRunnerImage, Command: cmd, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: &allowPrivilegeEscalation, diff --git a/test/utils/bitwarden_api_runner.go b/test/utils/bitwarden_api_runner.go index 32cd55d5b..9207ef741 100644 --- a/test/utils/bitwarden_api_runner.go +++ b/test/utils/bitwarden_api_runner.go @@ -90,7 +90,9 @@ func RunBitwardenAPIJob(ctx context.Context, client kubernetes.Interface, namesp return -1, "", fmt.Errorf("create job %s: %w", jobName, err) } propagationBackground := metav1.DeletePropagationBackground - defer func() { _ = client.BatchV1().Jobs(namespace).Delete(ctx, jobName, metav1.DeleteOptions{PropagationPolicy: &propagationBackground}) }() + defer func() { + _ = client.BatchV1().Jobs(namespace).Delete(ctx, jobName, metav1.DeleteOptions{PropagationPolicy: &propagationBackground}) + }() err = wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { j, getErr := client.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) @@ -111,9 +113,12 @@ func RunBitwardenAPIJob(ctx context.Context, client kubernetes.Interface, namesp // Find the pod and get exit code and logs. pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) - if err != nil || len(pods.Items) == 0 { + if err != nil { return -1, "", fmt.Errorf("list pods for job %s: %w", jobName, err) } + if len(pods.Items) == 0 { + return -1, "", fmt.Errorf("no pods found for job %s", jobName) + } pod := pods.Items[0] for _, cs := range pod.Status.ContainerStatuses { if cs.State.Terminated != nil { diff --git a/test/utils/cleanup.go b/test/utils/cleanup.go index ec564808a..c2599ec66 100644 --- a/test/utils/cleanup.go +++ b/test/utils/cleanup.go @@ -22,6 +22,8 @@ package utils import ( "context" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -60,7 +62,10 @@ func CleanupESOOperandAndRelated(ctx context.Context, cfg *rest.Config) { } // 1. Delete all instances of operand CRDs (label app=external-secrets). - crdList, _ := extClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + crdList, err := extClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + if err != nil || crdList == nil { + crdList = &apiextensionsv1.CustomResourceDefinitionList{} + } for i := range crdList.Items { crd := &crdList.Items[i] version := preferredVersion(crd) @@ -99,17 +104,26 @@ func CleanupESOOperandAndRelated(ctx context.Context, cfg *rest.Config) { _ = dynamicClient.Resource(escGVR).Delete(ctx, externalSecretsConfigName, metav1.DeleteOptions{}) // 3. Remove webhooks first so namespace deletion can proceed. - webhooks, _ := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + webhooks, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + if err != nil || webhooks == nil { + webhooks = &admissionregistrationv1.ValidatingWebhookConfigurationList{} + } for i := range webhooks.Items { _ = clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, webhooks.Items[i].Name, metav1.DeleteOptions{}) } // 4. ClusterRoleBindings before ClusterRoles. - crbs, _ := clientset.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + crbs, err := clientset.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + if err != nil || crbs == nil { + crbs = &rbacv1.ClusterRoleBindingList{} + } for i := range crbs.Items { _ = clientset.RbacV1().ClusterRoleBindings().Delete(ctx, crbs.Items[i].Name, metav1.DeleteOptions{}) } - crList, _ := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + crList, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{LabelSelector: operandLabelSelector}) + if err != nil || crList == nil { + crList = &rbacv1.ClusterRoleList{} + } for i := range crList.Items { _ = clientset.RbacV1().ClusterRoles().Delete(ctx, crList.Items[i].Name, metav1.DeleteOptions{}) } diff --git a/test/utils/conditions.go b/test/utils/conditions.go index 2d34e9478..652e7446e 100644 --- a/test/utils/conditions.go +++ b/test/utils/conditions.go @@ -242,7 +242,15 @@ func FetchAWSCredsFromSecret(ctx context.Context, k8sClient *kubernetes.Clientse if err != nil { return "", "", err } - return string(cred.Data[awsCredKeyIdSecretKeyName]), string(cred.Data[awsCredAccessKeySecretKeyName]), nil + id, ok := cred.Data[awsCredKeyIdSecretKeyName] + if !ok || len(id) == 0 { + return "", "", fmt.Errorf("secret %s/%s is missing %q", secretNamespace, secretName, awsCredKeyIdSecretKeyName) + } + key, ok := cred.Data[awsCredAccessKeySecretKeyName] + if !ok || len(key) == 0 { + return "", "", fmt.Errorf("secret %s/%s is missing %q", secretNamespace, secretName, awsCredAccessKeySecretKeyName) + } + return string(id), string(key), nil } // DeleteAWSSecretFromCredsSecret deletes an AWS Secrets Manager secret using credentials from the given Kubernetes secret. @@ -263,9 +271,9 @@ func DeleteAWSSecretFromCredsSecret(ctx context.Context, k8sClient *kubernetes.C return fmt.Errorf("failed to create AWS session: %w", err) } svc := secretsmanager.New(sess) - _, err = svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ + _, err = svc.DeleteSecretWithContext(ctx, &secretsmanager.DeleteSecretInput{ SecretId: aws.String(awsSecretName), - ForceDeleteWithoutRecovery: aws.Bool(true), + ForceDeleteWithoutRecovery: aws.Bool(true), // permanently delete without 7-day wait }) if err != nil { return fmt.Errorf("failed to delete AWS secret: %w", err) diff --git a/test/utils/dynamic_resources.go b/test/utils/dynamic_resources.go index 0246902e7..c3c665bf5 100644 --- a/test/utils/dynamic_resources.go +++ b/test/utils/dynamic_resources.go @@ -23,7 +23,6 @@ import ( "context" "testing" - "github.com/stretchr/testify/require" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +33,8 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/restmapper" + + "github.com/stretchr/testify/require" ) type DynamicResourceLoader struct { @@ -56,60 +57,10 @@ func NewDynamicResourceLoader(context context.Context, t *testing.T) DynamicReso } } -func (d DynamicResourceLoader) noErrorSkipExists(err error) { - if !k8serrors.IsAlreadyExists(err) { - require.NoError(d.t, err) - } -} - -func (d DynamicResourceLoader) noErrorSkipNotExisting(err error) { - if !k8serrors.IsNotFound(err) { - require.NoError(d.t, err) - } -} - -func (d DynamicResourceLoader) do(do doFunc, assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string) { - b, err := assetFunc(filename) - require.NoError(d.t, err) - - decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(b), 1024) - var rawObj runtime.RawExtension - err = decoder.Decode(&rawObj) - require.NoError(d.t, err) - - obj, gvk, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) - require.NoError(d.t, err) - - unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) - require.NoError(d.t, err) - - unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap} - - gr, err := restmapper.GetAPIGroupResources(d.KubeClient.Discovery()) - require.NoError(d.t, err) - - mapper := restmapper.NewDiscoveryRESTMapper(gr) - mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - require.NoError(d.t, err) - - var dri dynamic.ResourceInterface - if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - if overrideNamespace != "" { - unstructuredObj.SetNamespace(overrideNamespace) - } - require.NotEmpty(d.t, unstructuredObj.GetNamespace(), "Namespace can not be empty!") - dri = d.DynamicClient.Resource(mapping.Resource).Namespace(unstructuredObj.GetNamespace()) - } else { - dri = d.DynamicClient.Resource(mapping.Resource) - } - - do(d.t, unstructuredObj, dri) -} - func (d DynamicResourceLoader) DeleteFromFile(assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string) { d.t.Logf("Deleting resource %v\n", filename) deleteFunc := func(t *testing.T, unstructured *unstructured.Unstructured, dynamicResourceInterface dynamic.ResourceInterface) { - err := dynamicResourceInterface.Delete(context.Background(), unstructured.GetName(), metav1.DeleteOptions{}) + err := dynamicResourceInterface.Delete(d.context, unstructured.GetName(), metav1.DeleteOptions{}) d.noErrorSkipNotExisting(err) } @@ -120,7 +71,7 @@ func (d DynamicResourceLoader) DeleteFromFile(assetFunc func(name string) ([]byt func (d DynamicResourceLoader) CreateFromFile(assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string) { d.t.Logf("Creating resource %v\n", filename) createFunc := func(t *testing.T, unstructured *unstructured.Unstructured, dynamicResourceInterface dynamic.ResourceInterface) { - _, err := dynamicResourceInterface.Create(context.Background(), unstructured, metav1.CreateOptions{}) + _, err := dynamicResourceInterface.Create(d.context, unstructured, metav1.CreateOptions{}) d.noErrorSkipExists(err) } @@ -142,16 +93,20 @@ func (d DynamicResourceLoader) CreateFromUnstructured(unstructuredObj *unstructu // Use from Ginkgo tests with Expect(err).NotTo(HaveOccurred()) to see the actual API error on failure. func (d DynamicResourceLoader) CreateFromUnstructuredReturnErr(unstructuredObj *unstructured.Unstructured, overrideNamespace string) error { dri := d.getResourceInterface(unstructuredObj, overrideNamespace) - _, err := dri.Create(context.Background(), unstructuredObj, metav1.CreateOptions{}) + _, err := dri.Create(d.context, unstructuredObj, metav1.CreateOptions{}) return err } // DeleteFromUnstructured deletes a resource by name. For cluster-scoped resources, namespace is ignored. func (d DynamicResourceLoader) DeleteFromUnstructured(unstructuredObj *unstructured.Unstructured, overrideNamespace string) { dri := d.getResourceInterface(unstructuredObj, overrideNamespace) - err := dri.Delete(context.Background(), unstructuredObj.GetName(), metav1.DeleteOptions{}) - d.noErrorSkipNotExisting(err) - d.t.Logf("Resource %s deleted\n", unstructuredObj.GetName()) + err := dri.Delete(d.context, unstructuredObj.GetName(), metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + require.NoError(d.t, err) + } + if err == nil || k8serrors.IsNotFound(err) { + d.t.Logf("Resource %s deleted\n", unstructuredObj.GetName()) + } } func (d DynamicResourceLoader) getResourceInterface(unstructuredObj *unstructured.Unstructured, overrideNamespace string) dynamic.ResourceInterface { @@ -170,3 +125,39 @@ func (d DynamicResourceLoader) getResourceInterface(unstructuredObj *unstructure } return d.DynamicClient.Resource(mapping.Resource) } + +func (d DynamicResourceLoader) noErrorSkipExists(err error) { + if !k8serrors.IsAlreadyExists(err) { + require.NoError(d.t, err) + } +} + +func (d DynamicResourceLoader) noErrorSkipNotExisting(err error) { + if !k8serrors.IsNotFound(err) { + require.NoError(d.t, err) + } +} + +func (d DynamicResourceLoader) do(do doFunc, assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string) { + b, err := assetFunc(filename) + require.NoError(d.t, err) + + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(b), 1024) + var rawObj runtime.RawExtension + err = decoder.Decode(&rawObj) + require.NoError(d.t, err) + + obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) + require.NoError(d.t, err) + + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + require.NoError(d.t, err) + + unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap} + + if overrideNamespace != "" { + unstructuredObj.SetNamespace(overrideNamespace) + } + dri := d.getResourceInterface(unstructuredObj, overrideNamespace) + do(d.t, unstructuredObj, dri) +}