Skip to content

Commit 645cfcf

Browse files
GUIGUI
committed
Use our db-syncer Ruby gem for providing db dump/restore tasks.
While this is a bit funky to shoehorn this into this Python app, we have other components of our deployment process that use this library to provide easy database restores, so by leveraging this library, this is the easiest way to integrate that process with this app.
1 parent 86a8e5d commit 645cfcf

22 files changed

Lines changed: 822 additions & 2 deletions

‎.github/workflows/db-sync-dump.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: DB Sync Dump
2+
3+
on:
4+
schedule:
5+
- cron: "40 5 * * *" # Every day at 10:40 PM MST / 11:40 PM MDT
6+
workflow_dispatch:
7+
push:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
vault-nonsensitive-secrets:
15+
name: Vault Non-Sensitive Secrets
16+
runs-on: self-hosted
17+
outputs:
18+
nonsensitive-secrets: ${{ steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets }}
19+
steps:
20+
- name: Import vault nonsensitive secrets
21+
id: vault-nonsensitive-secrets
22+
uses: TADA/vault-action/nonsensitive-secrets@v1
23+
with:
24+
template: |
25+
{{ with (datasource "vault" "reopt-api/ci/deploy").data }}
26+
{{ $secrets = coll.Merge (coll.Dict
27+
"container_registry" .container_registry
28+
"staging_rancher_project_id" .staging_rancher_project_id
29+
) $secrets }}
30+
{{ end }}
31+
vault-role-id: ${{ secrets.VAULT_ROLE_ID }}
32+
vault-secret-id: ${{ secrets.VAULT_SECRET_ID }}
33+
34+
dump:
35+
name: Dump
36+
uses: TADA/deploy-action/.github/workflows/run-job.yml@v2
37+
needs:
38+
- vault-nonsensitive-secrets
39+
with:
40+
job-name: reopt-api-db-sync-dump
41+
job-command: "DB_SYNCER_PERFORM_UPLOAD=true rails db:data:dump --trace"
42+
deploy-env: production
43+
render-config-command: "DB_OWNER_AUTH=true DB_SYNCER=true RENDER_JOB=run-job ./config/deploy/render"
44+
rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }}
45+
registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }}
46+
vault-registry-credentials-path: secret/data/deploy/common/aws-ecr
47+
vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test
48+
force-run: true
49+
secrets:
50+
vault-role-id: ${{ secrets.VAULT_ROLE_ID }}
51+
vault-secret-id: ${{ secrets.VAULT_SECRET_ID }}

‎.github/workflows/deploy.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ on:
1010
- reopened
1111
- unlabeled
1212
workflow_dispatch:
13+
inputs:
14+
db-sync-restore:
15+
description: "Staging Only: Restore this branch's database with a recent snapshot from production."
16+
type: boolean
1317

1418
concurrency:
1519
# Concurrency group is more complicated in this case because:

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,9 @@ compare_run_*.json
127127

128128
/config/deploy/external
129129
/config/deploy/tmp
130+
docker-compose.override.yml
131+
132+
!/db-syncer/lib/
133+
/db-syncer/db/structure.sql
134+
/db-syncer/log
135+
/db-syncer/tmp

‎config/deploy/AppKbldConfig.pkl‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,22 @@ sources {
4141
}
4242
}
4343
}
44+
45+
new {
46+
image = "db-syncer"
47+
path = "db-syncer"
48+
docker {
49+
build {
50+
file = "Dockerfile"
51+
pull = true
52+
buildkit = true
53+
rawOptions {
54+
"--build-arg"
55+
"NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)"
56+
}
57+
}
58+
}
59+
}
4460
}
4561

4662
destinations {
@@ -53,4 +69,9 @@ destinations {
5369
["image"] = "julia-api"
5470
["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api"
5571
}
72+
73+
new {
74+
["image"] = "db-syncer"
75+
["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api"
76+
}
5677
}

‎config/deploy/RunJob.pkl‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import "./AppKbldConfig.pkl"
2+
import "@k8s/K8sResource.pkl"
3+
import "@tadaSharedKube/TadaAppNamespace.pkl"
4+
import "@tadaSharedKube/TadaEcrLoginRenewConfigMap.pkl"
5+
import "@tadaSharedKube/TadaEcrLoginRenewCronJob.pkl"
6+
import "@tadaSharedKube/TadaEcrLoginRenewInitialSetupJob.pkl"
7+
import "@tadaSharedKube/TadaEcrLoginRenewRole.pkl"
8+
import "@tadaSharedKube/TadaEcrLoginRenewRoleBinding.pkl"
9+
import "@tadaSharedKube/TadaEcrLoginRenewSecrets.pkl"
10+
import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl"
11+
import "@tadaSharedKube/TadaWebConfigMap.pkl"
12+
import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl"
13+
import "@tadaSharedKube/TadaWebRunJob.pkl"
14+
import "@tadaSharedKube/TadaWebSecrets.pkl"
15+
16+
local containerImage = "reopt-api"
17+
18+
resources: Listing<K8sResource> = new {
19+
new TadaAppNamespace {
20+
tadaRancherResourceQuotaPodLimit = 1
21+
}
22+
23+
new AppKbldConfig {}
24+
25+
// To be able to pull images from our ECR repos in our on-premise (non-AWS)
26+
// clusters.
27+
local ecrLoginRenewConfigMapResource = new TadaEcrLoginRenewConfigMap {}
28+
ecrLoginRenewConfigMapResource
29+
new TadaEcrLoginRenewCronJob {}
30+
new TadaEcrLoginRenewInitialSetupJob {}
31+
new TadaEcrLoginRenewRoleBinding {}
32+
new TadaEcrLoginRenewRole {}
33+
new TadaEcrLoginRenewSecrets {}
34+
new TadaEcrLoginRenewServiceAccount {}
35+
36+
new TadaWebConfigMap {}
37+
38+
local secretResource = new TadaWebSecrets {}
39+
secretResource
40+
41+
new TadaWebRunJob {
42+
tadaContainerImage = containerImage
43+
tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName
44+
tadaJobCommand = read("env:RUN_JOB_COMMAND")
45+
tadaSecretName = secretResource.metadata.name
46+
}
47+
}
48+
49+
output {
50+
value = resources
51+
renderer = (K8sResource.output.renderer as YamlRenderer) {
52+
isStream = true
53+
}
54+
}

‎config/deploy/WebDeployment.pkl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl"
1313
import "@tadaSharedKube/TadaWebConfigMap.pkl"
1414
import "@tadaSharedKube/TadaWebDbMigrateJob.pkl"
1515
import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl"
16+
import "@tadaSharedKube/TadaWebDbSyncRestoreJob.pkl"
1617
import "@tadaSharedKube/TadaWebDeployment.pkl"
1718
import "@tadaSharedKube/TadaWebIngress.pkl"
1819
import "@tadaSharedKube/TadaWebPodDisruptionBudget.pkl"
@@ -71,6 +72,13 @@ resources: Listing<K8sResource> = new {
7172

7273
new TadaWebDbMigrateSecrets {}
7374

75+
when (deployMetadata.`perform-db-sync-restore` == "true") {
76+
new TadaWebDbSyncRestoreJob {
77+
tadaContainerImage = "db-syncer"
78+
tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName
79+
}
80+
}
81+
7482
new TadaWebDbMigrateJob {
7583
tadaContainerImage = containerImage
7684
tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName

‎db-syncer/.gomplate.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
datasources:
2+
vault:
3+
url: "vault:///secret/data"

‎db-syncer/Dockerfile‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
FROM public.ecr.aws/docker/library/ruby:4.0-trixie
2+
3+
# Install NLR root certs for machines running on NLR's network.
4+
ARG NREL_ROOT_CERT_URL_ROOT=""
5+
RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" && update-ca-certificates; fi
6+
7+
# Postgresql
8+
ARG POSTGRESQL_MAJOR_VERSION=18
9+
RUN set -x && \
10+
distro=$(. /etc/os-release && echo "$VERSION_CODENAME") && \
11+
curl -fsSL -o /usr/share/keyrings/pgdg.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
12+
echo "deb [signed-by=/usr/share/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ ${distro}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
13+
apt-get update && \
14+
apt-get -y --no-install-recommends install "postgresql-client-${POSTGRESQL_MAJOR_VERSION}" rclone zstd && \
15+
rm -rf /var/lib/apt/lists/* /var/lib/dpkg/*-old /var/cache/* /var/log/*
16+
17+
ARG TARGETARCH
18+
19+
# Config templates.
20+
ARG GOMPLATE_VERSION=3.11.8
21+
RUN curl -fsSL -o /usr/local/bin/gomplate "https://github.com/hairyhenderson/gomplate/releases/download/v${GOMPLATE_VERSION}/gomplate_linux-${TARGETARCH}" && \
22+
chmod +x /usr/local/bin/gomplate
23+
24+
WORKDIR /app/db-syncer
25+
26+
# Install gems
27+
COPY Gemfile Gemfile.lock /app/db-syncer/
28+
ARG BUNDLE_FROZEN="true"
29+
ENV BUNDLE_FROZEN=$BUNDLE_FROZEN
30+
RUN bundle install

‎db-syncer/Gemfile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
source "https://rubygems.org"
2+
3+
# Syncing production/staging data back to local environments.
4+
gem "db-syncer", :require => false, :git => "https://github.nrel.gov/TADA/db-syncer.git", :branch => "main"
5+
6+
# Vault secrets as environment variables for development.
7+
gem "vault_env_secrets", "~> 2.0.0"
8+
9+
# This isn't really a full Rails app, we're just using it for the db-syncer gem
10+
# above to handle database dumps/restores for staging and development
11+
# environments.
12+
gem "rails", "~> 8.1.3"
13+
gem "pg", "~> 1.6.3"

0 commit comments

Comments
 (0)