Enable to Switch a DB in ScalarDB Cluster test (#169)

Author: yito88

.github/workflows/cluster-test.yml

@@ -94,7 +94,7 @@ jobs:
     - name: Setup Java
       uses: actions/setup-java@v5
       with:
-        java-version: '11'
+        java-version: '21'
         distribution: 'temurin'
 
     - name: Install leiningen
@@ -194,7 +194,7 @@ jobs:
 
         lein with-profile cluster run test \
           --nodes localhost \
-          --db cluster \
+          --db postgres \
           --concurrency 5 \
           --username runner \
           --ssh-private-key ~/.ssh/id_rsa \

.github/workflows/daily.yml

@@ -191,15 +191,15 @@ jobs:
         if [ "${{ matrix.tests.enable_group_commit }}" = "true" ]; then
           OPTS+=" --enable-group-commit"
         fi
+        DOCKER_USERNAME=${{ github.repository_owner }} \
+        DOCKER_ACCESS_TOKEN=${{ secrets.CR_PAT }} \
         lein with-profile cluster run test \
           --time-limit ${{ matrix.tests.time_limit }} \
           --nodes localhost \
-          --db cluster \
+          --db postgres \
           --concurrency 5 \
           --username runner \
           --ssh-private-key ~/.ssh/id_rsa \
-          --docker-username ${{ github.repository_owner }} \
-          --docker-access-token ${{ secrets.CR_PAT }} \
           ${OPTS} > /dev/null 2>&1
 
     - name: Record result

.github/workflows/test.yml

@@ -121,7 +121,7 @@ jobs:
     - name: ScalarDB Cluster test
       run: |
         cd scalardb
-        lein with-profile cluster run test --workload transfer --nodes localhost --db cluster --username runner --ssh-private-key ~/.ssh/id_rsa --docker-username ${{ github.repository_owner }} --docker-access-token ${{ secrets.CR_PAT }} --nemesis crash
+        DOCKER_USERNAME=${{ github.repository_owner }} DOCKER_ACCESS_TOKEN=${{ secrets.CR_PAT }} lein with-profile cluster run test --workload transfer --nodes localhost --db postgres --username runner --ssh-private-key ~/.ssh/id_rsa --nemesis crash
 
     - name: Check test result
       run: |

README.md

@@ -68,11 +68,11 @@ The test will attempt to SSH into a machine where the Kubernetes cluster is depl
 
 Below is an example command to run the ScalarDB Cluster test:
 ```sh
-lein with-profile cluster run test --workload transfer --db cluster \
+DOCKER_USERNAME=${GITHUB_USER} \
+DOCKER_ACCESS_TOKEN=${GITHUB_ACCESS_TOKEN} \
+lein with-profile cluster run test --workload transfer --db postgres \
     --nodes ${KUBERNETES_CLUSTER_HOST} \
-    --username ${USER} \
-    --docker-username ${GITHUB_USER} \
-    --docker-access-token ${GITHUB_ACCESS_TOKEN}
+    --username ${USER}
 ```
   - `KUBERNETES_CLUSTER_HOST`: The IP address or hostname of the machine running the Kubernetes cluster.
   - `USER`: The SSH username used to connect to the Kubernetes cluster host. If needed, `--password` or `--ssh-private-key` can be added.

scalardb/project.clj

@@ -36,7 +36,8 @@
                                                     com.azure/azure-cosmos
                                                     com.datastax.cassandra/cassandra-driver-core
                                                     com.google.cloud/alloydb-jdbc-connector]]]
-                       :env {:scalardb-cluster-version "4.0.0-SNAPSHOT"
+                       :env {:cluster? "true"
+                             :scalardb-cluster-version "4.0.0-SNAPSHOT"
                              :helm-chart-version "1.7.2"}}
              :use-jars {:dependencies [[com.google.guava/guava "31.1-jre"]
                                        [org.apache.commons/commons-text "1.13.0"]]

scalardb/src/scalardb/db/cassandra.clj

@@ -11,7 +11,6 @@
 
 (defrecord ExtCassandra []
   ext/DbExtension
-  (get-db-type [_] :cassandra)
   (live-nodes [_ test] (cassandra/live-nodes test))
   (wait-for-recovery [_ test] (cassandra/wait-rf-nodes test))
   (create-table-opts
@@ -37,7 +36,7 @@
   (create-storage-properties [this test] (ext/create-properties this test)))
 
 (defn gen-db
-  [faults admin]
+  [faults admin & _]
   (let [db (ext/extend-db (cassandra/db) (->ExtCassandra))
         ;; replace :kill nemesis with :crash for Cassandra
         faults (mapv #(if (= % :kill) :crash %) faults)]

scalardb/src/scalardb/db/cluster.clj

@@ -53,6 +53,10 @@
 
     :imagePullSecrets [{:name "scalardb-ghcr-secret"}]}})
 
+(defn- throw-unsupported-db-error
+  [db-type]
+  (throw (ex-info "Unsupported DB for ScalarDB Cluster test" {:db db-type})))
+
 (defn- update-cluster-values
   [test values]
   (let [path [:scalardbCluster :scalardbClusterNodeProperties]
@@ -87,9 +91,11 @@
 (defn- install!
   "Install prerequisites.
   You should already have installed kind (or similar tool), kubectl and helm."
-  []
-  ;; postgre
-  (c/exec :helm :repo :add "bitnami" "https://charts.bitnami.com/bitnami")
+  [db-type]
+  (case db-type
+    :postgres (c/exec :helm :repo :add
+                      "bitnami" "https://charts.bitnami.com/bitnami")
+    (throw-unsupported-db-error db-type))
   ;; ScalarDB cluster
   (c/exec :helm :repo :add
           "scalar-labs" "https://scalar-labs.github.io/helm-charts")
@@ -99,17 +105,17 @@
   (c/exec :helm :repo :update))
 
 (defn- configure!
-  [test]
-  (binding [c/*dir* (System/getProperty "user.dir")]
-    (try
-      (c/exec :kubectl :delete :secret "scalardb-ghcr-secret")
+  []
+  (when (and (seq (env :docker-username)) (seq (env :docker-access-token)))
+    (binding [c/*dir* (System/getProperty "user.dir")]
+      (try
+        (c/exec :kubectl :delete :secret "scalardb-ghcr-secret")
       ;; ignore the failure when the secret doesn't exist
-      (catch Exception _))
-    (when-let [docker-username (:docker-username test)]
+        (catch Exception _))
       (c/exec :kubectl :create :secret :docker-registry "scalardb-ghcr-secret"
               "--docker-server=ghcr.io"
-              (str "--docker-username=" docker-username)
-              (str "--docker-password=" (:docker-access-token test)))))
+              (str "--docker-username=" (env :docker-username))
+              (str "--docker-password=" (env :docker-access-token)))))
 
   ;; Chaos Mesh
   (try
@@ -118,21 +124,22 @@
       (c/exec :kubectl :create :ns "chaos-mesh"))))
 
 (defn- start!
-  [test]
-  ;; postgresql
-  (c/exec
-   :helm :install POSTGRESQL_NAME "bitnami/postgresql"
-   :--set "auth.postgresPassword=postgres"
-   :--set "primary.persistence.enabled=true"
-   ;; Need an external IP for storage APIs
-   :--set "service.type=LoadBalancer"
-   :--set "primary.service.type=LoadBalancer"
-   ;; Use legacy images
-   :--set "image.repository=bitnamilegacy/postgresql"
-   :--set "volumePermissions.image.repository=bitnamilegacy/os-shell"
-   :--set "metrics.image.repository=bitnamilegacy/postgres-exporter"
-   :--set "global.security.allowInsecureImages=true"
-   :--version "16.7.0")
+  [test db-type]
+  (case db-type
+    :postgres (c/exec
+               :helm :install POSTGRESQL_NAME "bitnami/postgresql"
+               :--set "auth.postgresPassword=postgres"
+               :--set "primary.persistence.enabled=true"
+               ;; Need an external IP for storage APIs
+               :--set "service.type=LoadBalancer"
+               :--set "primary.service.type=LoadBalancer"
+               ;; Use legacy images
+               :--set "image.repository=bitnamilegacy/postgresql"
+               :--set "volumePermissions.image.repository=bitnamilegacy/os-shell"
+               :--set "metrics.image.repository=bitnamilegacy/postgres-exporter"
+               :--set "global.security.allowInsecureImages=true"
+               :--version "16.7.0")
+    (throw-unsupported-db-error db-type))
 
   ;; ScalarDB Cluster
   (let [chart-version (or (some-> (env :helm-chart-version) not-empty)
@@ -168,14 +175,13 @@
                (apply c/exec :rm :-f))
       (catch Exception _ nil)))
   (info "wiping the pods...")
-  (try (c/exec :helm :uninstall POSTGRESQL_NAME) (catch Exception _ nil))
-  (try (c/exec :kubectl :delete
-               :pvc :-l "app.kubernetes.io/instance=postgresql-scalardb-cluster")
-       (catch Exception _ nil))
-  (try (c/exec :helm :uninstall CLUSTER_NAME) (catch Exception _ nil))
-  (try (c/exec :helm :uninstall CLUSTER2_NAME) (catch Exception _ nil))
-  (try (c/exec :helm :uninstall "chaos-mesh" :-n "chaos-mesh")
-       (catch Exception _ nil)))
+  (doseq [cmd [[:helm :uninstall POSTGRESQL_NAME]
+               [:kubectl :delete
+                :pvc :-l "app.kubernetes.io/instance=postgresql-scalardb-cluster"]
+               [:helm :uninstall CLUSTER_NAME]
+               [:helm :uninstall CLUSTER2_NAME]
+               [:helm :uninstall "chaos-mesh" :-n "chaos-mesh"]]]
+    (try (apply c/exec cmd) (catch Exception _ nil))))
 
 (defn- get-pod-list
   [name]
@@ -245,15 +251,15 @@
 
 (defn db
   "Setup ScalarDB Cluster."
-  []
+  [db-type]
   (reify
     db/DB
     (setup! [_ test _]
       (when-not (:leave-db-running? test)
         (wipe!))
-      (install!)
-      (configure! test)
-      (start! test)
+      (install! db-type)
+      (configure!)
+      (start! test db-type)
       ;; wait for the pods
       (wait-for-recovery test))
 
@@ -281,9 +287,8 @@
     (log-files [_ test _]
       (get-logs test))))
 
-(defrecord ExtCluster []
+(defrecord ExtCluster [db-type]
   ext/DbExtension
-  (get-db-type [_] :cluster)
   (live-nodes [_ test] (running-pods? test))
   (wait-for-recovery [_ test] (wait-for-recovery test))
   (create-table-opts [_ _] {})
@@ -303,17 +308,20 @@
             (mapv create-fn [ip ip2])
             (create-fn ip)))))
   (create-storage-properties [_ _]
-    (let [node (-> test :nodes first)
-          ip (c/on node (get-load-balancer-ip POSTGRESQL_NAME))]
-      (doto (Properties.)
-        (.setProperty "scalar.db.storage" "jdbc")
-        (.setProperty "scalar.db.contact_points"
-                      (str "jdbc:postgresql://" ip ":5432/postgres"))
-        (.setProperty "scalar.db.username" "postgres")
-        (.setProperty "scalar.db.password" "postgres")))))
+    (let [node (-> test :nodes first)]
+      (case db-type
+        :postgres (let [ip (c/on node (get-load-balancer-ip POSTGRESQL_NAME))]
+                    (doto (Properties.)
+                      (.setProperty "scalar.db.storage" "jdbc")
+                      (.setProperty "scalar.db.contact_points"
+                                    (str "jdbc:postgresql://" ip ":5432/postgres"))
+                      (.setProperty "scalar.db.username" "postgres")
+                      (.setProperty "scalar.db.password" "postgres")))
+        (throw-unsupported-db-error db-type)))))
 
 (defn gen-db
-  [faults admin]
+  [faults admin db-type]
   (when (seq admin)
     (warn "The admin operations are ignored: " admin))
-  [(ext/extend-db (db) (->ExtCluster)) (n/nemesis-package db 60 faults) 1])
+  (let [db (ext/extend-db (db db-type) (->ExtCluster db-type))]
+    [db (n/nemesis-package db 60 faults) 1]))

scalardb/src/scalardb/db/postgres.clj

@@ -140,7 +140,6 @@
 
 (defrecord ExtPostgres []
   ext/DbExtension
-  (get-db-type [_] :postgres)
   (live-nodes [_ test] (live-node? test))
   (wait-for-recovery [_ test] (wait-for-recovery test))
   (create-table-opts [_ _] {})
@@ -159,7 +158,7 @@
   (create-storage-properties [this test] (ext/create-properties this test)))
 
 (defn gen-db
-  [faults admin]
+  [faults admin & _]
   (when (seq admin)
     (warn "The admin operations are ignored: " admin))
   [(ext/extend-db (db) (->ExtPostgres))

scalardb/src/scalardb/db_extend.clj

@@ -28,7 +28,6 @@
     (.setProperty "scalar.db.consensus_commit.include_metadata.enabled" "true")))
 
 (defprotocol DbExtension
-  (get-db-type [this])
   (live-nodes [this test])
   (wait-for-recovery [this test])
   (create-table-opts [this test])
@@ -53,7 +52,6 @@
     db/LogFiles
     (log-files [_ test node] (db/log-files db test node))
     DbExtension
-    (get-db-type [_] (get-db-type ext-db))
     (live-nodes [_ test] (live-nodes ext-db test))
     (wait-for-recovery [_ test] (wait-for-recovery ext-db test))
     (create-table-opts [_ test] (create-table-opts ext-db test))

scalardb/src/scalardb/runner.clj

@@ -19,11 +19,10 @@
              [elle-write-read-2pc]]
             [clojure.core :as c]))
 
-(def db-keys
+(def supported-dbs
   "The map of test DBs."
   {"cassandra" :cassandra
-   "postgres" :postgres
-   "cluster" :cluster})
+   "postgres" :postgres})
 
 (def workload-keys
   "A map of test workload keys."
@@ -58,7 +57,7 @@
 
 (def test-opt-spec
   [(cli/repeated-opt nil "--db NAME" "DB(s) on which the test is run"
-                     [:cassandra] db-keys)
+                     [:cassandra] supported-dbs)
 
    (cli/repeated-opt nil "--workload NAME" "Test(s) to run" [] workload-keys)
 
@@ -84,15 +83,7 @@
 
    [nil "--config-file CONFIG_FILE"
     "ScalarDB config file. When this is given, other configuration options are ignored."
-    :default ""]
-
-   [nil "--docker-username DOCKER_USERNAME"
-    "Username to pull ScalarDB Cluster node image in ghcr.io."
-    :default nil]
-
-   [nil "--docker-access-token DOCKER_ACCESS_TOKEN"
-    "Access token to pull ScalarDB Cluster node image in ghcr.io."
-    :default nil]])
+    :default ""]])
 
 (defn- test-name
   [workload-key faults admin]
@@ -101,21 +92,20 @@
       (into (map name admin))
       (->> (remove nil?) (string/join "-"))))
 
-(defn- load-module
-  [db-key]
-  (case db-key
-    :cassandra (require 'scalardb.db.cassandra)
-    :postgres  (require 'scalardb.db.postgres)
-    :cluster  (require 'scalardb.db.cluster)
-    (throw (ex-info "Unsupported DB" {:db db-key}))))
-
 (defn- gen-db
   "Returns [extended-db constructed-nemesis num-max-nodes]."
-  [db-key faults admin]
-  (load-module db-key)
-  (let [gen-db-sym (symbol (str "scalardb.db." (name db-key)) "gen-db")
+  [db-type faults admin]
+  (let [gen-db-sym (if (= (env :cluster?) "true")
+                     (do (require 'scalardb.db.cluster)
+                         'scalardb.db.cluster/gen-db)
+                     (case db-type
+                       :cassandra (do (require 'scalardb.db.cassandra)
+                                      'scalardb.db.cassandra/gen-db)
+                       :postgres  (do (require 'scalardb.db.postgres)
+                                      'scalardb.db.postgres/gen-db)
+                       (throw (ex-info "Unsupported DB" {:db db-type}))))
         gen-db-fn (resolve gen-db-sym)]
-    (gen-db-fn faults admin)))
+    (gen-db-fn faults admin db-type)))
 
 (defn- gen-test-opt-spec
   []
@@ -136,8 +126,8 @@
    :decommissioned (atom #{})})
 
 (defn scalardb-test
-  [base-opts db-key workload-key faults admin]
-  (let [[db nemesis max-nodes] (gen-db db-key faults admin)
+  [base-opts db-type workload-key faults admin]
+  (let [[db nemesis max-nodes] (gen-db db-type faults admin)
         consistency-model (->> base-opts :consistency-model (mapv keyword))
         workload-opts (merge base-opts
                              scalardb-opts
@@ -169,12 +159,12 @@
            :usage (cli/test-usage)
            :run (fn [{:keys [options]}]
                   (doseq [_ (range (:test-count options))
-                          db-key (:db options)
+                          db-type (:db options)
                           workload-key (:workload options)
                           faults (:nemesis options)
                           admin (or (:admin options) [[]])]
                     (let [test (-> options
-                                   (scalardb-test db-key
+                                   (scalardb-test db-type
                                                   workload-key
                                                   faults
                                                   admin)