fix(ci): re-enable XGBoost E2E test (kubeflow#3323)

* test(e2e): re-enable XGBoost E2E test after runtime image publication

Signed-off-by: krishna-kg732 <krishnagupta.kg2k6@gmail.com>

* Chore:using test image instead of latest

Signed-off-by: krishna-kg732 <krishnagupta.kg2k6@gmail.com>

---------

Signed-off-by: krishna-kg732 <krishnagupta.kg2k6@gmail.com>

Author: Krishna-kg732

.github/workflows/test-e2e.yaml

@@ -60,8 +60,7 @@ jobs:
           make test-e2e-notebook NOTEBOOK_INPUT=./examples/pytorch/audio-classification/audio-classification.ipynb NOTEBOOK_OUTPUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}_audio-classification.ipynb PAPERMILL_TIMEOUT=1800
           make test-e2e-notebook NOTEBOOK_INPUT=./examples/local/local-training-mnist.ipynb NOTEBOOK_OUTPUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}_local-training-mnist.ipynb PAPERMILL_TIMEOUT=1800
           make test-e2e-notebook NOTEBOOK_INPUT=./examples/local/local-container-mnist.ipynb NOTEBOOK_OUTPUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}_local-container-mnist.ipynb PAPERMILL_TIMEOUT=1800
-        #TODO(Krishna-kg732): uncomment once the xgboost-runtime image is published
-        #make test-e2e-notebook NOTEBOOK_INPUT=./examples/xgboost/distributed-training/xgboost-distributed.ipynb NOTEBOOK_OUTPUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}_xgboost-distributed.ipynb PAPERMILL_TIMEOUT=1800
+          make test-e2e-notebook NOTEBOOK_INPUT=./examples/xgboost/distributed-training/xgboost-distributed.ipynb NOTEBOOK_OUTPUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}_xgboost-distributed.ipynb PAPERMILL_TIMEOUT=1800
 
       # TODO (andreyvelich): Discuss how we can upload artifacts for multiple Notebooks.
       - name: Upload Artifacts to GitHub

hack/e2e-setup-cluster.sh

@@ -97,7 +97,20 @@ print_cluster_info() {
 
 # TODO (andreyvelich): Currently, we print manager logs due to flaky test.
 echo "Deploy Kubeflow Trainer runtimes"
-kubectl apply --server-side -k manifests/overlays/runtimes || (
+E2E_RUNTIMES_DIR="artifacts/e2e/runtimes"
+mkdir -p "${E2E_RUNTIMES_DIR}"
+XGBOOST_RUNTIME_CI_IMAGE_NAME="ghcr.io/kubeflow/trainer/xgboost-runtime"
+cat <<EOF >"${E2E_RUNTIMES_DIR}/kustomization.yaml"
+  apiVersion: kustomize.config.k8s.io/v1beta1
+  kind: Kustomization
+  resources:
+  - ../../../manifests/overlays/runtimes
+  images:
+  - name: "${XGBOOST_RUNTIME_CI_IMAGE_NAME}"
+    newTag: "${CI_IMAGE_TAG}"
+EOF
+
+kubectl apply --server-side -k "${E2E_RUNTIMES_DIR}" || (
   kubectl logs -n ${NAMESPACE} -l app.kubernetes.io/name=trainer &&
     print_cluster_info &&
     exit 1
@@ -122,7 +135,7 @@ kubectl apply --server-side -k manifests/overlays/runtimes || (
 # load_image_to_kind ${JAX_RUNTIME_IMAGE}
 
 # Build and load custom runtime images that are not available in public registries.
-XGBOOST_RUNTIME_CI_IMAGE="ghcr.io/kubeflow/trainer/xgboost-runtime:${CI_IMAGE_TAG}"
+XGBOOST_RUNTIME_CI_IMAGE="${XGBOOST_RUNTIME_CI_IMAGE_NAME}:${CI_IMAGE_TAG}"
 echo "Build XGBoost runtime image"
 ${CONTAINER_RUNTIME} build . -f cmd/runtimes/xgboost/Dockerfile -t ${XGBOOST_RUNTIME_CI_IMAGE}
 load_image_to_kind ${XGBOOST_RUNTIME_CI_IMAGE}

test/e2e/e2e_test.go

@@ -247,8 +247,6 @@ var _ = ginkgo.Describe("TrainJob e2e", func() {
 	ginkgo.When("Creating TrainJob to perform XGBoost workload", func() {
 		// Verify the `xgboost-distributed` ClusterTrainingRuntime.
 		ginkgo.It("should create TrainJob with XGBoost runtime reference", func() {
-			// TODO (krishna-kg732): Remove this skip once the xgboost-runtime image is published to GHCR.
-			ginkgo.Skip("xgboost-runtime image not yet published to GHCR")
 			// Create a TrainJob.
 			trainJob := testingutil.MakeTrainJobWrapper(ns.Name, "e2e-test-xgboost").
 				RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), xgboostRuntime).