[Infra] Dockerize AstroML Environment

.env.example

@@ -1,7 +1,5 @@
 # AstroML Environment Configuration
 # Copy this file to .env and customize for your environment
-# See docker-env-guide.md for detailed configuration information
-
 # ============================================================================
 # Database Configuration
 # ============================================================================
@@ -43,12 +41,14 @@ STELLAR_SECRET_KEY=your_stellar_secret_key_here
 # ============================================================================
 # Application Configuration
 # ============================================================================
+APP_ENV=development
+ASTROML_ENV=development
+PYTHONPATH=/app
+DEBUG=true
+
 LOG_LEVEL=INFO
 LOG_FORMAT=json
 LOG_FILE=./logs/astroml.log
-PYTHONPATH=/app
-APP_ENV=development
-DEBUG=False
 
 # ============================================================================
 # API Configuration
@@ -88,7 +88,7 @@ MLFLOW_TRACKING_URI=http://localhost:5000
 MLFLOW_EXPERIMENT_NAME=astroml
 
 # ============================================================================
-# Jupyter Configuration (for development)
+# Jupyter Configuration
 # ============================================================================
 JUPYTER_TOKEN=astroml_dev
 JUPYTER_PASSWORD=astroml_dev
@@ -149,10 +149,6 @@ SOROBAN_NETWORK=public
 SOROBAN_RPC_URL=https://soroban-testnet.stellar.org
 SOROBAN_SECRET_KEY=your_soroban_secret_key_here
 SOROBAN_FEE=10000
-
-# ============================================================================
-# Performance Configuration
-# ============================================================================
 MAX_WORKERS=4
 BATCH_SIZE=1000
 MEMORY_LIMIT=8GB
@@ -165,6 +161,14 @@ NETWORK_TIMEOUT=30
 RETRY_COUNT=3
 RETRY_DELAY=1
 
+# ============================================================================
+# Feature Store Advanced Configuration
+# ============================================================================
+FEATURE_STORE_CACHE_STRATEGY=LRU
+FEATURE_STORE_STORAGE_FORMAT=PARQUET
+FEATURE_STORAGE_COMPRESSION=snappy
+FEATURE_STORE_VERSIONING=true
+
 # ============================================================================
 # Development Configuration
 # ============================================================================
@@ -174,7 +178,10 @@ MOCK_SERVICES=false
 
 # ============================================================================
 # Production Configuration
+# ============================================================================# ============================================================================
+# Production Configuration
 # ============================================================================
+# Production Configuration
 PROD_MODE=false
 MONITORING_ENABLED=false
 ALERTING_ENABLED=false
@@ -186,3 +193,25 @@ ENABLE_STREAMING=True
 ENABLE_MONITORING=True
 ENABLE_GPU_TRAINING=True
 ENABLE_SOROBAN_CONTRACTS=True
+
+# ============================================================================
+# Docker Configuration
+# ============================================================================
+DOCKER_REGISTRY=astroml
+DOCKER_TAG=latest
+DOCKER_BUILDKIT=1
+
+# ============================================================================
+# Data Configuration
+# ============================================================================
+DATA_PATH=./data
+MODELS_PATH=./models
+LOGS_PATH=./logs
+CONFIG_PATH=./config
+
+# ============================================================================
+# Network Configuration
+# ============================================================================
+NETWORK_TIMEOUT=30
+RETRY_COUNT=3
+RETRY_DELAY=1

.github/workflows/docker-ci-cd.yml

@@ -0,0 +1,187 @@
+name: Docker CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+  release:
+    types: [ created ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest pytest-cov
+
+      - name: Run tests
+        run: |
+          pytest tests/ -v --cov=astroml --cov-report=xml
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+
+  build-docker-images:
+    runs-on: ubuntu-latest
+    needs: build-and-test
+    strategy:
+      matrix:
+        stage: [base, development, feature-store, ingestion, training-cpu, production]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: ${{ matrix.stage }}
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.stage }}-${{ github.sha }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.stage }}-latest
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  security-scan:
+    runs-on: ubuntu-latest
+    needs: build-docker-images
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production-latest
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  deploy-kubernetes:
+    runs-on: ubuntu-latest
+    needs: [build-docker-images, security-scan]
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: 'v1.28.0'
+
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CONFIG }}" | base64 -d > kubeconfig
+          export KUBECONFIG=kubeconfig
+
+      - name: Install kustomize
+        run: |
+          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
+          sudo mv kustomize /usr/local/bin/
+
+      - name: Deploy to Kubernetes
+        run: |
+          kustomize build k8s/ | kubectl apply -f -
+
+      - name: Verify deployment
+        run: |
+          kubectl rollout status deployment/feature-store -n astroml
+          kubectl rollout status deployment/astroml-ingestion -n astroml
+          kubectl rollout status deployment/postgres -n astroml
+          kubectl rollout status deployment/redis -n astroml
+
+  deploy-staging:
+    runs-on: ubuntu-latest
+    needs: [build-docker-images, security-scan]
+    if: github.ref == 'refs/heads/develop' && github.event_name == 'push'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: 'v1.28.0'
+
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > kubeconfig
+          export KUBECONFIG=kubeconfig
+
+      - name: Install kustomize
+        run: |
+          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
+          sudo mv kustomize /usr/local/bin/
+
+      - name: Deploy to Staging
+        run: |
+          kustomize build k8s/overlays/staging | kubectl apply -f -
+
+      - name: Verify deployment
+        run: |
+          kubectl rollout status deployment/feature-store -n astroml-staging
+          kubectl rollout status deployment/astroml-ingestion -n astroml-staging
+
+  notify:
+    runs-on: ubuntu-latest
+    needs: [deploy-kubernetes]
+    if: always()
+    steps:
+      - name: Send notification
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          text: |
+            Deployment Status: ${{ job.status }}
+            Branch: ${{ github.ref }}
+            Commit: ${{ github.sha }}
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/pre-commit.yml

@@ -0,0 +1,28 @@
+name: pre-commit
+
+on:
+  push:
+    branches: ["main", "master"]
+  pull_request:
+    branches: ["main", "master"]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          # Issue #203: cache pip wheels so pre-commit hooks install faster
+          cache: pip
+
+      - name: Install pre-commit
+        env:
+          PIP_CACHE_DIR: ~/.cache/pip
+        run: pip install pre-commit
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files