feat: upgrade preflight toolkit + entrypoint connectivity diagnostics

bin/docker-entrypoint

@@ -3,15 +3,137 @@
 # Remove stale PID file if present (prevents "server is already running" errors)
 rm -f /rails/tmp/pids/server.pid
 
-# If running the rails server then create or migrate existing database
-# db:prepare is idempotent: creates DB if missing, runs pending migrations if exists
-# This also triggers admin:bootstrap (hooked into db:prepare)
+# ─── Pre-boot database connectivity check ─────────────────────────────
+# Runs BEFORE Rails boots. Uses pg_isready (no Ruby, no Rails) so it
+# works even when the Rails environment itself can't load due to a
+# misconfigured database connection. On failure, prints diagnostics
+# with exact fix commands instead of a raw PG::ConnectionBad backtrace.
 #
-# IMPORTANT: db:prepare does NOT need DISABLE_DATABASE_ENVIRONMENT_CHECK=1
-# Unlike db:schema:load (rake task), db:prepare calls load_schema() directly
-# as a Ruby method, bypassing the protected environment check entirely.
-# This is by design — see Rails source: DatabaseTasks#initialize_database
-if [[ "$@" == *"server"* ]]; then
+# This is the "gas and go" safety net: if the container can't reach the
+# database, the operator sees actionable output within 10 seconds
+# instead of a cryptic Rails crash.
+# ──────────────────────────────────────────────────────────────────────
+check_database_connectivity() {
+  # Build connection params from the same env vars that database.yml reads
+  if [ -n "$DATABASE_URL" ]; then
+    DB_CHECK_URL="$DATABASE_URL"
+  else
+    local host="${DATABASE_HOST:-127.0.0.1}"
+    local port="${DATABASE_PORT:-5432}"
+    local user="${POSTGRES_USER:-postgres}"
+    local dbname="${POSTGRES_DB:-vulcan_postgres_production}"
+    DB_CHECK_URL="postgres://${user}@${host}:${port}/${dbname}"
+  fi
+
+  # Extract host:port for pg_isready (handles both URL and individual vars)
+  local check_host check_port check_user
+  if [ -n "$DATABASE_URL" ]; then
+    check_host=$(echo "$DATABASE_URL" | sed -E 's|.*@([^:/]+).*|\1|')
+    check_port=$(echo "$DATABASE_URL" | sed -E 's|.*:([0-9]+)/.*|\1|')
+    check_user=$(echo "$DATABASE_URL" | sed -E 's|.*://([^:@]+).*|\1|')
+    [ -z "$check_port" ] && check_port=5432
+  else
+    check_host="${DATABASE_HOST:-127.0.0.1}"
+    check_port="${DATABASE_PORT:-5432}"
+    check_user="${POSTGRES_USER:-postgres}"
+  fi
+
+  echo "Checking database connectivity (${check_host}:${check_port})..."
+
+  # pg_isready is a lightweight TCP + PG protocol check — no auth needed
+  if command -v pg_isready &>/dev/null; then
+    if pg_isready -h "$check_host" -p "$check_port" -U "$check_user" -t 10 &>/dev/null; then
+      echo "  ✓ Database is reachable"
+      return 0
+    fi
+  fi
+
+  # pg_isready failed or not available — try a psql connection test
+  if command -v psql &>/dev/null; then
+    if psql "$DB_CHECK_URL" -c "SELECT 1" &>/dev/null 2>&1; then
+      echo "  ✓ Database is reachable"
+      return 0
+    fi
+  fi
+
+  # ── Connection failed — print diagnostics ──
+  echo ""
+  echo "======================================================================"
+  echo "  ✗ DATABASE CONNECTION FAILED"
+  echo "======================================================================"
+  echo ""
+  echo "  Vulcan cannot reach the database at ${check_host}:${check_port}"
+  echo ""
+  echo "  ── Check these first ──"
+  echo ""
+  if [ -n "$DATABASE_URL" ]; then
+    echo "  DATABASE_URL is set: ${check_host}:${check_port}"
+  else
+    echo "  DATABASE_URL is NOT set (using individual env vars)"
+    echo "    DATABASE_HOST=${DATABASE_HOST:-(not set, defaulting to 127.0.0.1)}"
+    echo "    DATABASE_PORT=${DATABASE_PORT:-(not set, defaulting to 5432)}"
+    echo "    POSTGRES_USER=${POSTGRES_USER:-(not set, defaulting to postgres)}"
+    echo "    POSTGRES_DB=${POSTGRES_DB:-(not set, defaulting to vulcan_postgres_production)}"
+  fi
+  echo ""
+  echo "  ── Common fixes ──"
+  echo ""
+  echo "  1. Is the database host correct?"
+  echo "     • Docker Compose: use 'db' (the service name), not 'localhost'"
+  echo "     • Aurora RDS: use the CLUSTER endpoint, not the instance endpoint"
+  echo "     • Kubernetes: use the service DNS name"
+  echo ""
+  echo "  2. Is SSL required?"
+  echo "     • Aurora/RDS/Cloud SQL require SSL by default"
+  echo "     • Add ?sslmode=require to DATABASE_URL:"
+  echo "       DATABASE_URL=postgres://user:pass@host:5432/dbname?sslmode=require"
+  echo ""
+  echo "  3. Is GSSAPI causing issues?"
+  echo "     • Aurora does not support GSSAPI authentication"
+  echo "     • Set: DATABASE_GSSENCMODE=disable"
+  echo ""
+  echo "  4. Is the database server running?"
+  echo "     • Docker Compose: check 'docker compose ps db'"
+  echo "     • Aurora: check the RDS console for cluster status"
+  echo "     • Firewall/security group blocking port ${check_port}?"
+  echo ""
+  echo "  5. Are credentials correct?"
+  echo "     • Check POSTGRES_USER and POSTGRES_PASSWORD in .env"
+  echo "     • Aurora IAM auth requires a different connection method"
+  echo ""
+  echo "  ── Quick test from this container ──"
+  echo ""
+  echo "  docker compose exec web bash"
+  if [ -n "$DATABASE_URL" ]; then
+    echo "  pg_isready -h ${check_host} -p ${check_port}"
+  else
+    echo "  pg_isready -h \$DATABASE_HOST -p \$DATABASE_PORT"
+  fi
+  echo ""
+  echo "  ── Standalone diagnostic (no Rails needed) ──"
+  echo ""
+  echo "  curl -fsSL https://raw.githubusercontent.com/mitre/vulcan/master/bin/upgrade-check.sh | bash -s -- \"\$DATABASE_URL\""
+  echo ""
+  echo "======================================================================"
+  echo ""
+  return 1
+}
+
+if [[ "$*" == *"server"* ]]; then
+  # Check connectivity BEFORE Rails boots — gives actionable diagnostics
+  # instead of a raw PG::ConnectionBad backtrace
+  if ! check_database_connectivity; then
+    echo "Exiting. Fix the database connection and restart the container."
+    exit 1
+  fi
+
+  # db:prepare is idempotent: creates DB if missing, runs pending migrations if exists
+  # This also triggers admin:bootstrap (hooked into db:prepare)
+  #
+  # IMPORTANT: db:prepare does NOT need DISABLE_DATABASE_ENVIRONMENT_CHECK=1
+  # Unlike db:schema:load (rake task), db:prepare calls load_schema() directly
+  # as a Ruby method, bypassing the protected environment check entirely.
+  # This is by design — see Rails source: DatabaseTasks#initialize_database
   ./bin/rails db:prepare
 fi
 

bin/upgrade-check.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Vulcan Upgrade Diagnostic — standalone shell script
+#
+# Checks database connectivity and schema state WITHOUT requiring Rails.
+# Use this when you can't install the rake task into a running container.
+#
+# Usage:
+#   ./upgrade-check.sh postgres://user:pass@host:5432/vulcan_production
+#   ./upgrade-check.sh  # uses DATABASE_URL from environment
+#
+# Requirements: psql (PostgreSQL client)
+
+DB_URL="${1:-${DATABASE_URL:-}}"
+
+if [ -z "$DB_URL" ]; then
+  echo "Usage: $0 <DATABASE_URL>"
+  echo "   or: DATABASE_URL=postgres://... $0"
+  echo
+  echo "Example:"
+  echo "  $0 postgres://user:pass@your-cluster.rds.amazonaws.com:5432/vulcan_production?sslmode=require"
+  exit 1
+fi
+
+if ! command -v psql &>/dev/null; then
+  echo "ERROR: psql not found. Install postgresql-client."
+  exit 1
+fi
+
+echo "======================================================================"
+echo "  Vulcan Upgrade Diagnostic (standalone)"
+echo "======================================================================"
+echo
+
+# ── Phase 1: Connection ──
+echo "── Phase 1: Connection ──"
+echo
+
+PG_VERSION=$(psql "$DB_URL" -t -A -c "SELECT version()" 2>&1) || {
+  echo "  ✗ Cannot connect to database"
+  echo
+  echo "  Error: $PG_VERSION"
+  echo
+  echo "  Checklist:"
+  echo "    - Is the hostname correct? (use cluster endpoint for Aurora)"
+  echo "    - Is sslmode=require in the URL? (required for Aurora/RDS)"
+  echo "    - Is the port correct? (default: 5432)"
+  echo "    - Can this machine reach the host? (security group / firewall)"
+  echo "    - Try: psql '$DB_URL' -c 'SELECT 1'"
+  exit 1
+}
+
+echo "  ✓ Connected"
+echo "    $PG_VERSION"
+
+if echo "$PG_VERSION" | grep -qi aurora; then
+  echo "    Runtime: Amazon Aurora"
+fi
+
+# SSL
+SSL_USED=$(psql "$DB_URL" -t -A -c "SELECT CASE WHEN ssl THEN 'yes' ELSE 'no' END FROM pg_stat_ssl WHERE pid = pg_backend_pid()" 2>/dev/null || echo "unknown")
+if [ "$SSL_USED" = "yes" ]; then
+  echo "  ✓ SSL connection active"
+elif [ "$SSL_USED" = "no" ]; then
+  echo "  ⚠ SSL NOT active — add ?sslmode=require for cloud databases"
+else
+  echo "  ℹ SSL status unknown (pg_stat_ssl not available)"
+fi
+
+# Read replica
+IS_REPLICA=$(psql "$DB_URL" -t -A -c "SELECT pg_is_in_recovery()" 2>/dev/null || echo "unknown")
+if [ "$IS_REPLICA" = "t" ]; then
+  echo "  ✗ Database is a READ REPLICA — migrations cannot run"
+  echo "    Use the writer/primary endpoint instead"
+elif [ "$IS_REPLICA" = "f" ]; then
+  echo "  ✓ Database is primary (writable)"
+fi
+
+# Encoding
+ENCODING=$(psql "$DB_URL" -t -A -c "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()")
+if [ "$ENCODING" = "UTF8" ]; then
+  echo "  ✓ Encoding: $ENCODING"
+else
+  echo "  ⚠ Encoding: $ENCODING (expected UTF8)"
+fi
+
+# pg_trgm
+if psql "$DB_URL" -t -A -c "SELECT 'test' % 'test'" &>/dev/null; then
+  echo "  ✓ pg_trgm extension available"
+else
+  echo "  ⚠ pg_trgm extension not available (needed for search)"
+  echo "    Aurora: enable in DB parameter group"
+  echo "    Vanilla PG: CREATE EXTENSION IF NOT EXISTS pg_trgm;"
+fi
+
+# ── Phase 2: Schema ──
+echo
+echo "── Phase 2: Schema ──"
+echo
+
+SCHEMA_VERSION=$(psql "$DB_URL" -t -A -c "SELECT MAX(version) FROM schema_migrations" 2>/dev/null || echo "none")
+echo "  Current schema version: $SCHEMA_VERSION"
+
+MIGRATION_COUNT=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM schema_migrations" 2>/dev/null || echo "0")
+echo "  Applied migrations: $MIGRATION_COUNT"
+
+# ── Phase 3: Data Integrity ──
+echo
+echo "── Phase 3: Data Integrity ──"
+echo
+
+# Check if reviews table exists
+HAS_REVIEWS=$(psql "$DB_URL" -t -A -c "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='reviews')")
+if [ "$HAS_REVIEWS" = "t" ]; then
+  ORPHAN_USERS=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM reviews WHERE user_id IS NOT NULL AND user_id NOT IN (SELECT id FROM users)")
+  if [ "$ORPHAN_USERS" = "0" ]; then
+    echo "  ✓ No orphaned review.user_id"
+  else
+    echo "  ⚠ $ORPHAN_USERS review(s) with orphaned user_id (migration will nullify)"
+  fi
+
+  ORPHAN_RULES=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM reviews WHERE rule_id IS NOT NULL AND rule_id NOT IN (SELECT id FROM base_rules)")
+  if [ "$ORPHAN_RULES" = "0" ]; then
+    echo "  ✓ No orphaned review.rule_id"
+  else
+    echo "  ⚠ $ORPHAN_RULES review(s) with orphaned rule_id (migration will delete)"
+  fi
+
+  REVIEW_COUNT=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM reviews")
+  USER_COUNT=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM users")
+  RULE_COUNT=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM base_rules")
+  echo
+  echo "  Table sizes:"
+  echo "    reviews:    $REVIEW_COUNT"
+  echo "    users:      $USER_COUNT"
+  echo "    base_rules: $RULE_COUNT"
+else
+  echo "  ℹ reviews table does not exist (fresh database)"
+fi
+
+AUDIT_COUNT=$(psql "$DB_URL" -t -A -c "SELECT COUNT(*) FROM audits" 2>/dev/null || echo "0")
+echo "    audits:     $AUDIT_COUNT"
+
+# ── Summary ──
+echo
+echo "── Summary ──"
+echo
+echo "  If all checks passed: proceed with the upgrade."
+echo "  If connection failed: fix DATABASE_URL and re-run."
+echo
+echo "  Next steps:"
+echo "    1. Back up: pg_dump -Fc \$DATABASE_URL > backup.dump"
+echo "    2. Upgrade the container image"
+echo "    3. Start the container (db:prepare runs automatically)"
+echo "    4. Verify: rails upgrade:verify (inside the new container)"
+
+if echo "$PG_VERSION" | grep -qi aurora; then
+  echo
+  echo "── Aurora Notes ──"
+  echo "  • Use cluster endpoint (not instance) for DATABASE_URL"
+  echo "  • Add ?sslmode=require to DATABASE_URL"
+  echo "  • Set DATABASE_GSSENCMODE=disable in environment"
+  echo "  • Enable pg_trgm in DB parameter group"
+fi
+
+echo
+echo "======================================================================"

docs/.vitepress/config.js

@@ -189,6 +189,7 @@ export default defineConfig({
             { text: "Bare Metal", link: "/deployment/bare-metal" },
             { text: "Heroku", link: "/deployment/heroku" },
             { text: "Kubernetes", link: "/deployment/kubernetes" },
+            { text: "Upgrade Guide", link: "/deployment/upgrade-guide" },
           ],
         },
         {

docs/deployment/docker.md

@@ -165,6 +165,20 @@ docker compose run --rm web bundle exec rails db:migrate
 
 > **Why `db:prepare` and not `db:migrate`?** `db:prepare` handles both fresh and existing databases. It is the Rails 8 standard pattern for Docker entrypoints. Unlike `db:schema:load`, `db:prepare` does NOT need `DISABLE_DATABASE_ENVIRONMENT_CHECK` — it calls `load_schema()` as a Ruby method, bypassing the protected environment check by design. See [Rails source: DatabaseTasks#initialize_database](https://github.com/rails/rails/blob/main/activerecord/lib/active_record/tasks/database_tasks.rb) for details.
 
+### Upgrading Between Versions
+
+The upgrade toolkit is built into every image from v2.3.6+:
+
+```bash
+docker compose pull                                  # Get new image
+docker compose run --rm web rails upgrade:preflight   # Check before upgrading
+docker compose run --rm web rails upgrade:fix         # Fix any issues
+docker compose up -d                                  # Start (runs db:prepare)
+docker compose exec web rails upgrade:verify          # Confirm success
+```
+
+See the full [Upgrade Guide](upgrade-guide) for Aurora RDS notes, Kubernetes/ECS paths, and troubleshooting.
+
 ## Monitoring
 
 ### Health Check Endpoints

docs/deployment/index.md

@@ -23,6 +23,8 @@ Choose the deployment method that fits your infrastructure and team.
 
 **Enterprise / multi-tenant?** [Kubernetes](kubernetes) with Helm chart for scaling and isolation.
 
+**Upgrading from an older version?** See the [Upgrade Guide](upgrade-guide) — includes a preflight diagnostic, auto-fix for common data issues, and Aurora RDS guidance.
+
 **Air-gapped / classified network?** [Bare Metal](bare-metal) for full control without container dependencies.
 
 ## Common Requirements (All Deployments)