Improve FERC EQR sensor, deployment, testability, & notifications

.devcontainer/devcontainer.json

@@ -19,6 +19,8 @@
     "containerEnv": {
         "PUDL_INPUT": "/workspaces/pudl-input",
         "PUDL_OUTPUT": "/workspaces/pudl-output",
+        "PUDL_ROOT_PATH": "/workspaces/pudl",
+        "PUDL_FERCEQR_ARCHIVE_PATH": "/workspaces/pudl-input/archives.catalyst.coop/ferceqr/",
         "DAGSTER_HOME": "/workspaces/dagster-home",
         "ANTHROPIC_API_KEY": "${localEnv:ANTHROPIC_API_KEY}",
         "GEMINI_API_KEY": "${localEnv:GEMINI_API_KEY}",
@@ -52,6 +54,7 @@
                 "chat.agent.sandbox.enabled": "off",
                 "chat.tools.global.autoApprove": true,
                 "chat.agent.sandbox.allowUnsandboxedCommands": true,
+                "chat.permissions.default": "autoApprove",
                 "claudeCode.allowDangerouslySkipPermissions": true,
                 "claudeCode.initialPermissionMode": "bypassPermissions",
                 "evenBetterToml.taplo.path": "/home/ubuntu/.pixi/bin/taplo",

.env.example

@@ -16,12 +16,12 @@ PUDL_FORCE_FERC_TO_SQLITE=true
 
 # Optional: nonstandard path to FERC EQR input data archive.
 # Useful for working with local EQR data in development
-FERCEQR_ARCHIVE_PATH=/abs/path/to/ferceqr/zipfile/dir
+PUDL_FERCEQR_ARCHIVE_PATH=/abs/path/to/ferceqr/zipfile/dir
 
-# Optional: If set to any value, FERCEQR_BUILD triggers deployment of the EQR data after
-# the end of the ferceqr job. Read by src/pudl/dagster/sensors.py and
-# src/pudl/dagster/assets/__init__.py
-FERCEQR_BUILD=true
+# Optional: path to a file that specifies FERC EQR deployment targets.
+# If unset, local development will skip FERC EQR deployment.
+# Production batch runs should point this at the packaged config file inside the repo.
+PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH=/absolute/path/to/ferceqr_deployment_targets.yml
 
 # Optional: required for devcontainer to work correctly.
 PUDL_ROOT_PATH=/absolute/path/to/pudl/repo

.github/workflows/build-deploy-ferceqr.yml

@@ -2,21 +2,39 @@
 name: build-deploy-ferceqr
 on:
   workflow_dispatch:
+    inputs:
+      start_partition:
+        description: First partition (e.g. 2013q3). Optional. If set, requires end_partition. If both unset, process all available data.
+        required: false
+        type: string
+      end_partition:
+        description: Last partition (e.g. 2013q4). Optional. If set, requires start_partition. If both unset, process all available data.
+        required: false
+        type: string
+      deployment_mode:
+        description: Deployment target. If none, do not deploy the data.
+        required: false
+        default: test
+        type: choice
+        options:
+          - test
+          - production
+          - none
   schedule:
     - cron: "48 0 3 * *" # 00:48 UTC on the third of every month
 
 env:
   GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }}
-  GCS_OUTPUT_BUCKET: gs://pudl.catalyst.coop/ferceqr
   GCS_LOGS_BUCKET: gs://builds.catalyst.coop/ferceqr_logs
-  S3_OUTPUT_BUCKET: s3://pudl.catalyst.coop/ferceqr
   BATCH_JOB_JSON: batch_job.json
   BUILD_ID: ""
   BATCH_JOB_ID: ""
+  PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH: ""
+  emoji: ":warning:"
 
 jobs:
-  build_and_deploy_eqr:
-    name: Build Docker image, push to Docker Hub and deploy to Google Batch
+  build_and_deploy_ferceqr:
+    name: Kick off a FERC EQR build and deployment with Google Batch
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -27,6 +45,43 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Set environment variables for the FERC EQR batch job
+        run: |
+          batch_job_id="$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)"
+          build_id="$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)-${{ github.ref_name }}"
+          case "${{ github.event_name == 'workflow_dispatch' && inputs.deployment_mode || 'production' }}" in
+            test)
+              ferceqr_config_path='/home/ubuntu/pudl/src/pudl/package_data/settings/ferceqr_test_deployment_targets.yml'
+              ;;
+            none)
+              ferceqr_config_path=''
+              ;;
+            *)
+              ferceqr_config_path='/home/ubuntu/pudl/src/pudl/package_data/settings/ferceqr_deployment_targets.yml'
+              ;;
+          esac
+          {
+              echo "BATCH_JOB_ID=$batch_job_id"
+              echo "BUILD_ID=$build_id"
+              echo "PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH=$ferceqr_config_path"
+          } >> "$GITHUB_ENV"
+          echo "BUILD_ID: $build_id"
+          echo "BATCH_JOB_ID: $batch_job_id"
+          echo "PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH: $ferceqr_config_path"
+
+      - name: Notify Zulip of FERC EQR build launch
+        if: ${{ always() }}
+        uses: zulip/github-actions-zulip/send-message@v2
+        with:
+          api-key: ${{ secrets.ZULIP_API_KEY }}
+          email: "build-status-bot@catalyst-cooperative.zulipchat.com"
+          organization-url: "https://catalyst-cooperative.zulipchat.com"
+          type: "stream"
+          to: "pudl-deployments"
+          topic: "build-deploy-ferceqr"
+          content: >
+            :rocket: Launching FERC EQR Build: ${{ env.BUILD_ID }}; :bulls_eye: Target: ${{ inputs.deployment_mode }}
+
       - name: Docker Metadata
         id: docker_metadata
         uses: docker/metadata-action@v6
@@ -59,30 +114,17 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - id: "auth"
+      - name: Authenticate with Google Cloud
+        id: "gcloud-auth"
         uses: "google-github-actions/auth@v3"
         with:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
           service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
 
-      - name: Set action environment variables
-        run: |
-          {
-              echo "BATCH_JOB_ID=$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)";
-              echo "BUILD_ID=$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)-${{ github.ref_name }}";
-          } >> "$GITHUB_ENV"
-
-      - name: Show freshly set envvars
-        run: |
-          echo "BUILD_ID: $BUILD_ID"
-          echo "BATCH_JOB_ID: $BATCH_JOB_ID"
-
-      # Setup gcloud CLI
-      - name: Set up Cloud SDK
+      - name: Set up Google Cloud SDK
+        id: "gcloud-sdk"
         uses: google-github-actions/setup-gcloud@v3
 
-      # Deploy PUDL image to GCE
-      # Dagster Postgres connection configured in builds/dagster.yaml - otherwise we get a str for port num
       - name: Make GCP Batch config file
         run: |-
           ./devtools/generate_batch_config.py \
@@ -96,32 +138,22 @@ jobs:
             --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
             --container-env BUILD_REF=${{ github.ref_name }} \
             --container-env BUILD_ID=${{ env.BUILD_ID }} \
+            --container-env FERCEQR_START_PARTITION=${{ github.event_name == 'workflow_dispatch' && inputs.start_partition || '' }} \
+            --container-env FERCEQR_END_PARTITION=${{ github.event_name == 'workflow_dispatch' && inputs.end_partition || '' }} \
             --container-env FERCEQR_BUILD=true \
+            --container-env PUDL_FERCEQR_ARCHIVE_PATH=gs://archives.catalyst.coop/ferceqr/published \
+            --container-env PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH="${{ env.PUDL_FERCEQR_DEPLOYMENT_CONFIG_PATH }}" \
             --container-env ZULIP_API_KEY=${{ secrets.ZULIP_API_KEY }} \
             --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \
-            --container-env GCS_OUTPUT_BUCKET=${{ env.GCS_OUTPUT_BUCKET }} \
             --container-env GCS_LOGS_BUCKET=${{ env.GCS_LOGS_BUCKET }} \
-            --container-env S3_OUTPUT_BUCKET=${{ env.S3_OUTPUT_BUCKET }} \
             --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \
-            --container-env OMP_NUM_THREADS=4 \
+            --container-env OMP_NUM_THREADS=8 \
             --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \
+            --vcpu 16 \
+            --mem-gb 127 \
             --disk-gb 1000 \
             --output ${{ env.BATCH_JOB_JSON }}
 
-      - name: Send notification to Zulip build status stream
-        if: ${{ always() }}
-        uses: zulip/github-actions-zulip/send-message@v2
-        with:
-          api-key: ${{ secrets.ZULIP_API_KEY }}
-          email: "build-status-bot@catalyst-cooperative.zulipchat.com"
-          organization-url: "https://catalyst-cooperative.zulipchat.com"
-          type: "stream"
-          to: "pudl-deployments"
-          topic: "FERC EQR Builds"
-          content: >
-            :rocket: Launching FERC EQR Build: ${{ env.BUILD_ID }}
-
-      # Start the batch job
       - name: Launch Google Batch job
         run: gcloud batch jobs submit run-ferceqr-etl-${{ env.BATCH_JOB_ID }} --config ${{ env.BATCH_JOB_JSON }} --location us-east1
 
@@ -143,6 +175,6 @@ jobs:
           organization-url: "https://catalyst-cooperative.zulipchat.com"
           type: "stream"
           to: "pudl-deployments"
-          topic: "FERC EQR Builds"
+          topic: "build-deploy-ferceqr"
           content: >
             ${{ env.emoji }} Google Batch job for ${{ env.BUILD_ID }} launched with status: ${{ job.status }}.

builds/Dockerfile

@@ -45,8 +45,10 @@ ENV DAGSTER_HOME=${CONTAINER_PUDL_WORKSPACE}/dagster_home
 
 RUN mkdir -p ${PUDL_INPUT} ${PUDL_OUTPUT} ${DAGSTER_HOME} ${PUDL_ROOT_PATH}
 
-# Copy dagster configuration file
-COPY builds/dagster.yaml ${DAGSTER_HOME}/dagster.yaml
+# Copy job-specific dagster configuration files; each batch script selects
+# the appropriate one at startup by copying it to $DAGSTER_HOME/dagster.yaml.
+COPY builds/dagster-pudl.yaml ${DAGSTER_HOME}/dagster-pudl.yaml
+COPY builds/dagster-ferceqr.yaml ${DAGSTER_HOME}/dagster-ferceqr.yaml
 
 # Copy the entire PUDL repo into the image.
 # We can't install the PUDL package without .git/ because the version is dynamically

builds/dagster-ferceqr.yaml

@@ -0,0 +1,5 @@
+run_coordinator:
+  module: dagster
+  class: QueuedRunCoordinator
+  config:
+    max_concurrent_runs: 10