Integrate AWS Parameter Store for Zappa/Infrastructure deployments (#2551)

* Use AWS SSM Parameter Store to handle environment variables

* Use focused policy for read access

* Update documentation

* Add flag for create_rds_proxy

* set default value of create_rds_proxy to false

* Populate Zappa/Lambda environment variables from ssm/parameter store

* Update documentation

* Update example

* add default configurations

* add security group db from lambda

Author: rudransh-shrivastava

backend/settings/staging.py

@@ -17,8 +17,6 @@ class Staging(Base):
         traces_sample_rate=0.5,
     )
 
-    AWS_ACCESS_KEY_ID = values.SecretValue()
-    AWS_SECRET_ACCESS_KEY = values.SecretValue()
     AWS_STORAGE_BUCKET_NAME = "owasp-nest"
     AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
     AWS_S3_OBJECT_PARAMETERS = {

backend/wsgi.py

@@ -2,9 +2,30 @@
 
 import os
 
+
+def _populate_environ_from_ssm():
+    ssm_param_path = os.getenv("AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH")
+    if not ssm_param_path:
+        return
+
+    from pathlib import Path
+
+    import boto3
+
+    client = boto3.client("ssm")
+    paginator = client.get_paginator("get_parameters_by_path")
+    response_iterator = paginator.paginate(Path=ssm_param_path, WithDecryption=True)
+
+    for page in response_iterator:
+        for param in page["Parameters"]:
+            os.environ[Path(param["Name"]).name] = param["Value"]
+
+
+_populate_environ_from_ssm()
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.local")
 os.environ.setdefault("DJANGO_CONFIGURATION", "Local")
 
-from configurations.wsgi import get_wsgi_application
+from configurations.wsgi import get_wsgi_application  # noqa: E402
 
 application = get_wsgi_application()

backend/zappa_settings.example.json

@@ -1,27 +1,23 @@
 {
   "staging": {
     "app_function": "wsgi.application",
-    "django_settings": "settings.staging",
-    "environment_variables": {
-      "DJANGO_ALGOLIA_APPLICATION_ID": "${DJANGO_ALGOLIA_APPLICATION_ID}",
-      "DJANGO_ALGOLIA_WRITE_API_KEY": "${DJANGO_ALGOLIA_WRITE_API_KEY}",
-      "DJANGO_ALLOWED_HOSTS": "${DJANGO_ALLOWED_HOSTS}",
-      "DJANGO_AWS_ACCESS_KEY_ID": "${DJANGO_AWS_ACCESS_KEY_ID}",
-      "DJANGO_AWS_SECRET_ACCESS_KEY": "${DJANGO_AWS_SECRET_ACCESS_KEY}",
-      "DJANGO_CONFIGURATION": "Staging",
-      "DJANGO_DB_HOST": "${DJANGO_DB_HOST}",
-      "DJANGO_DB_NAME": "${DJANGO_DB_NAME}",
-      "DJANGO_DB_USER": "${DJANGO_DB_USER}",
-      "DJANGO_DB_PORT": "${DJANGO_DB_PORT}",
-      "DJANGO_DB_PASSWORD": "${DJANGO_DB_PASSWORD}",
-      "DJANGO_OPEN_AI_SECRET_KEY": "${DJANGO_OPEN_AI_SECRET_KEY}",
-      "DJANGO_REDIS_HOST": "${DJANGO_REDIS_HOST}",
-      "DJANGO_REDIS_PASSWORD": "${DJANGO_REDIS_PASSWORD}",
-      "DJANGO_SECRET_KEY": "${DJANGO_SECRET_KEY}",
-      "DJANGO_SENTRY_DSN": "${DJANGO_SENTRY_DSN}",
-      "DJANGO_SLACK_BOT_TOKEN": "${DJANGO_SLACK_BOT_TOKEN}",
-      "DJANGO_SLACK_SIGNING_SECRET": "${DJANGO_SLACK_SIGNING_SECRET}"
+    "aws_environment_variables": {
+      "AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH": "/owasp-nest/staging"
     },
+    "django_settings": "settings.staging",
+    "extra_permissions": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "ssm:GetParametersByPath",
+          "ssm:GetParameter"
+        ],
+        "Resource": [
+          "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/owasp-nest/staging",
+          "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/owasp-nest/staging/*"
+        ]
+      }
+    ],
     "manage_roles": true,
     "project_name": "nest-backend",
     "runtime": "python3.13",

infrastructure/README.md

@@ -39,8 +39,6 @@ Follow these steps to set up the infrastructure:
      cat terraform.tfvars.example > terraform.tfvars
      ```
 
-   - Update the default `django_` prefixed variables. (database/redis credentials will be added later)
-
 3. **Apply Changes**:
 
    - Init terraform if needed:
@@ -55,51 +53,11 @@ Follow these steps to set up the infrastructure:
      terraform apply
      ```
 
-4. **Copy Outputs**:
-
-   - Run the following command to view all outputs. Use the `-raw` flag for sensitive outputs.
-   - Copy required outputs (i.e. `database_endpoint`, `db_password`, `redis_auth_token`, and `redis_endpoint`)
-     to the previously created `terraform.tfvars`:
+4. **Populate Secrets**:
 
-     ```bash
-     terraform output
-     ```
+   - Visit the AWS Console > Systems Manager > Parameter Store.
+   - Populate all `DJANGO_*` secrets that have `to-be-set-in-aws-console` value.
 
-     Example Output:
-
-     ```bash
-     database_endpoint = "owasp-nest-staging-proxy.proxy-000000000000.us-east-2.rds.amazonaws.com"
-     db_password = <sensitive>
-     ecr_repository_url = "000000000000.dkr.ecr.us-east-2.amazonaws.com/owasp-nest-staging-backend"
-     lambda_security_group_id = "sg-00000000000000000"
-     private_subnet_ids = [
-          "subnet-00000000000000000",
-          "subnet-11111111111111111",
-          "subnet-22222222222222222",
-     ]
-     redis_auth_token = <sensitive>
-     redis_endpoint = "master.owasp-nest-staging-cache.aaaaaa.region1.cache.amazonaws.com"
-     zappa_s3_bucket = "owasp-nest-zappa-deployments"
-     ```
-
-     ```bash
-     terraform output -raw db_password
-     ```
-
-     ```bash
-     terraform output -raw redis_auth_token
-     ```
-
-5. **Apply The Changes Again**:
-
-   - Apply the changes again using the following command:
-
-     ```bash
-     terraform apply
-     ```
-
-*Note*: Step 4 and 5 ensure that ECS/Fargate tasks have proper environment variables.
-These two steps will be removed when AWS Secrets Manager is integrated.
 
 ## Setting up Zappa
 
@@ -145,11 +103,15 @@ The Django backend deployment is managed by Zappa. This includes the API Gateway
 
 5. **Deploy**:
 
+    - *Note*: Make sure to populate all `DJANGO_*` secrets that are set as `to-be-set-in-aws-console`
+      in the Parameter Store. The deployment might fail with no logs if secrets such as
+      `DJANGO_SLACK_BOT_TOKEN` are invalid.
+
     ```bash
     zappa deploy staging
     ```
 
-Once deployed, Zappa will provide you with a URL. You can use this URL to test the API.
+Once deployed, use the URL provided by Zappa to test the API.
 
 ## Setup Database
 
@@ -224,6 +186,12 @@ Migrate and load data into the new database.
 
 ## Helpful Commands
 
+- To view logs for a `staging` deployment run:
+
+  ```bash
+  zappa tail staging
+  ```
+
 - To update a Zappa `staging` deployment run:
 
   ```bash

infrastructure/main.tf

@@ -19,26 +19,6 @@ locals {
     ManagedBy   = "Terraform"
     Project     = var.project_name
   }
-  django_environment_variables = {
-    DJANGO_ALGOLIA_APPLICATION_ID = var.django_algolia_application_id
-    DJANGO_ALGOLIA_WRITE_API_KEY  = var.django_algolia_write_api_key
-    DJANGO_ALLOWED_HOSTS          = var.django_allowed_hosts
-    DJANGO_AWS_ACCESS_KEY_ID      = var.django_aws_access_key_id
-    DJANGO_AWS_SECRET_ACCESS_KEY  = var.django_aws_secret_access_key
-    DJANGO_CONFIGURATION          = var.django_configuration
-    DJANGO_DB_HOST                = var.django_db_host
-    DJANGO_DB_NAME                = var.django_db_name
-    DJANGO_DB_USER                = var.django_db_user
-    DJANGO_DB_PORT                = var.django_db_port
-    DJANGO_DB_PASSWORD            = var.django_db_password
-    DJANGO_OPEN_AI_SECRET_KEY     = var.django_open_ai_secret_key
-    DJANGO_REDIS_HOST             = var.django_redis_host
-    DJANGO_REDIS_PASSWORD         = var.django_redis_password
-    DJANGO_SECRET_KEY             = var.django_secret_key
-    DJANGO_SENTRY_DSN             = var.django_sentry_dsn
-    DJANGO_SLACK_BOT_TOKEN        = var.django_slack_bot_token
-    DJANGO_SLACK_SIGNING_SECRET   = var.django_slack_signing_secret
-  }
 }
 
 module "cache" {
@@ -47,7 +27,6 @@ module "cache" {
   common_tags           = local.common_tags
   environment           = var.environment
   project_name          = var.project_name
-  redis_auth_token      = var.redis_auth_token
   redis_engine_version  = var.redis_engine_version
   redis_node_type       = var.redis_node_type
   redis_num_cache_nodes = var.redis_num_cache_nodes
@@ -60,6 +39,7 @@ module "database" {
   source = "./modules/database"
 
   common_tags                = local.common_tags
+  create_rds_proxy           = var.create_rds_proxy
   db_allocated_storage       = var.db_allocated_storage
   db_backup_retention_period = var.db_backup_retention_period
   db_engine_version          = var.db_engine_version
@@ -68,7 +48,7 @@ module "database" {
   db_password                = var.db_password
   db_storage_type            = var.db_storage_type
   db_subnet_ids              = module.networking.private_subnet_ids
-  db_username                = var.db_username
+  db_user                    = var.db_user
   environment                = var.environment
   project_name               = var.project_name
   proxy_security_group_ids   = [module.security.rds_proxy_sg_id]
@@ -80,7 +60,7 @@ module "ecs" {
 
   aws_region                    = var.aws_region
   common_tags                   = local.common_tags
-  django_environment_variables  = local.django_environment_variables
+  container_parameters_arns     = module.parameters.ssm_parameter_arns
   environment                   = var.environment
   fixtures_read_only_policy_arn = module.storage.fixtures_read_only_policy_arn
   fixtures_s3_bucket            = var.fixtures_s3_bucket
@@ -101,15 +81,31 @@ module "networking" {
   vpc_cidr             = var.vpc_cidr
 }
 
+module "parameters" {
+  source = "./modules/parameters"
+
+  common_tags    = local.common_tags
+  db_host        = module.database.db_proxy_endpoint
+  db_name        = var.db_name
+  db_password    = module.database.db_password
+  db_port        = var.db_port
+  db_user        = var.db_user
+  environment    = var.environment
+  project_name   = var.project_name
+  redis_host     = module.cache.redis_primary_endpoint
+  redis_password = module.cache.redis_auth_token
+}
+
 module "security" {
   source = "./modules/security"
 
-  common_tags  = local.common_tags
-  db_port      = var.db_port
-  environment  = var.environment
-  project_name = var.project_name
-  redis_port   = var.redis_port
-  vpc_id       = module.networking.vpc_id
+  common_tags      = local.common_tags
+  create_rds_proxy = var.create_rds_proxy
+  db_port          = var.db_port
+  environment      = var.environment
+  project_name     = var.project_name
+  redis_port       = var.redis_port
+  vpc_id           = module.networking.vpc_id
 }
 
 module "storage" {

infrastructure/modules/cache/main.tf

@@ -14,10 +14,9 @@ terraform {
 }
 
 locals {
-  generate_redis_auth_token = var.redis_auth_token == null || var.redis_auth_token == ""
-  parameter_group_name      = "default.redis${local.redis_major_version}"
-  redis_auth_token          = local.generate_redis_auth_token ? random_password.redis_auth_token[0].result : var.redis_auth_token
-  redis_major_version       = split(".", var.redis_engine_version)[0]
+  parameter_group_name = "default.redis${local.redis_major_version}"
+  redis_auth_token     = random_password.redis_auth_token[0].result
+  redis_major_version  = split(".", var.redis_engine_version)[0]
 }
 
 resource "aws_elasticache_subnet_group" "main" {
@@ -29,7 +28,7 @@ resource "aws_elasticache_subnet_group" "main" {
 }
 
 resource "random_password" "redis_auth_token" {
-  count  = local.generate_redis_auth_token ? 1 : 0
+  count  = 1
   length = 32
   # Redis auth token has specific requirements for special characters.
   override_special = "!&#$^<>-"

infrastructure/modules/cache/variables.tf

@@ -26,13 +26,6 @@ variable "project_name" {
   type        = string
 }
 
-variable "redis_auth_token" {
-  description = "The auth token for Redis"
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
 variable "redis_engine_version" {
   description = "The version of the Redis engine"
   type        = string