diff --git a/.copier-answers.resonant.yml b/.copier-answers.resonant.yml index 5479bef5..acfa3e7e 100644 --- a/.copier-answers.resonant.yml +++ b/.copier-answers.resonant.yml @@ -1,4 +1,4 @@ -_commit: v0.48.1 +_commit: v0.50.2 _src_path: https://github.com/kitware-resonant/cookiecutter-resonant core_app_name: core include_example_code: false diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..c568abee --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,83 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose +{ + "name": "GeoDatalytics", + "dockerComposeFile": [ + "../docker-compose.yml", + "../docker-compose.override.yml", + "./docker-compose.devcontainer.yml" + ], + "service": "django", + "overrideCommand": true, + // The "vscode" user and remoteUser are set by the base image label (devcontainers/base). + "workspaceFolder": "/home/vscode/geodatalytics", + "features": { + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/rails/devcontainer/features/postgres-client:1": { + "version": 18 + }, + "ghcr.io/devcontainers/features/terraform:1": {}, + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers-extra/features/heroku-cli:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + // Python + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "ms-python.mypy-type-checker", + "charliermarsh.ruff", + // Django + "batisteo.vscode-django", + "augustocdias.tasks-shell-input", + // Other file formats + "editorconfig.editorconfig", + "mikestead.dotenv", + "tamasfe.even-better-toml", + "timonwong.shellcheck", + // Infrastructure + "ms-azuretools.vscode-containers", + "hashicorp.terraform", + "github.vscode-github-actions", + // Remove AWS extension, as only the CLI is wanted; see: https://github.com/devcontainers/features/issues/1228 + "-AmazonWebServices.aws-toolkit-vscode" + ], + "settings": { + "containers.containerClient": "com.microsoft.visualstudio.containers.docker", + // Container-specific Python paths + "python.defaultInterpreterPath": "/home/vscode/venv/bin/python", + // Ensure that `envFile` from any user settings is ignored; Docker Compose provides it. + "python.envFile": "", + // Reduce file watcher overhead for generated/cache directories. + "files.watcherExclude": { + "**/__pycache__/**": true, + "**/.pytest_cache/**": true, + "**/node_modules/**": true + } + } + } + }, + // Prevent a prompt every time the debugger opens a port or Django auto-restarts. + "otherPortsAttributes": { + "onAutoForward": "silent" + }, + "portsAttributes": { + "8000": { + "label": "Django", + // Show a dialog if the port isn't free. + "requireLocalPort": true, + "onAutoForward": "silent" + } + }, + // Install a global Python and create a venv before VSCode extensions start, + // to prevent prompts and ensure test discovery works on first load. + "onCreateCommand": { + "python": ["uv", "python", "install", "--default"], + "venv": ["uv", "sync", "--all-extras", "--all-groups"] + }, + // Ensure it is re-synced on restarts. + "updateContentCommand": ["uv", "sync", "--all-extras", "--all-groups"] +} diff --git a/.devcontainer/docker-compose.devcontainer.yml b/.devcontainer/docker-compose.devcontainer.yml new file mode 100644 index 00000000..55b21a82 --- /dev/null +++ b/.devcontainer/docker-compose.devcontainer.yml @@ -0,0 +1,14 @@ +services: + django: + # Don't expose ports, devcontainer forwarding is superior, since we can just bind to localhost. + ports: !reset [] + # Don't auto-run the default command, launch.json or the terminal will be used. + command: !reset [] + + celery: + # Celery will be started via launch.json or the terminal. + profiles: ["celery"] + + web: + # npm will be started via launch.json or the terminal. + profiles: ["web"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68f48b7f..1401fc44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: contents: read jobs: lint-client: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Install npm @@ -23,7 +23,7 @@ jobs: working-directory: web test-server: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 services: postgres: image: postgis/postgis:18-3.6 diff --git a/.github/workflows/nightly_ci.yml b/.github/workflows/nightly_ci.yml index 5445396b..7a616c30 100644 --- a/.github/workflows/nightly_ci.yml +++ b/.github/workflows/nightly_ci.yml @@ -8,7 +8,7 @@ permissions: contents: read jobs: test-server-slow: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 services: postgres: image: postgis/postgis:18-3.6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4bbbc3d5..7d827177 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ permissions: contents: write jobs: version-and-release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v6 diff --git a/README.md b/README.md index 1a04d336..1392ab06 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Urban areas, particularly those with underserved populations and critical assets GeoDatalytics is designed to assist urban planners, policymakers, logistics engineers, and infrastructure developers/owners in managing increasingly large, complex, and diverse datasets. By integrating climate data with other critical data types such as infrastructure, sensor, and demographic information using our advanced analytic and visualization tools, users will be empowered to make data-driven decisions. -GeoDatalytics is built with Kitware's [Resonant][girder-4-cookiecutter-link] technology stack. It consists of a series of container services, managed by `docker-compose`. These services include a Django Python server, PostgreSQL/PostGIS database, Minio object store, Vue web application, and Celery task service. +GeoDatalytics is built with Kitware's [Resonant][resonant-cookiecutter-link] technology stack. This stack includes a Django Python server, PostgreSQL/PostGIS database, Minio object store, Vue web application, and Celery task service. ## Getting Started with GeoDatalytics @@ -42,4 +42,4 @@ To run GeoDatalytics locally with `docker-compose`, follow the instructions in t [kitware-link]: https://kitware.com [sds-lab-link]: https://sdslab.io [mass-mapper-link]: https://maps.massgis.digital.mass.gov/MassMapper/MassMapper.html -[girder-4-cookiecutter-link]: https://github.com/girder/cookiecutter-girder-4 +[resonant-cookiecutter-link]: https://github.com/kitware-resonant/cookiecutter-resonant diff --git a/dev/django.Dockerfile b/dev/django.Dockerfile index d94c6b59..f62c265c 100644 --- a/dev/django.Dockerfile +++ b/dev/django.Dockerfile @@ -1,13 +1,33 @@ -FROM ghcr.io/astral-sh/uv:debian - -# Make Python more friendly to running in containers -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 - -# Make uv install content in well-known locations -ENV UV_PROJECT_ENVIRONMENT=/var/lib/venv \ - UV_CACHE_DIR=/var/cache/uv/cache \ - UV_PYTHON_INSTALL_DIR=/var/cache/uv/bin \ - # The uv cache and environment are expected to be mounted on different volumes, - # so hardlinks won't work - UV_LINK_MODE=symlink +FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Ensure Python output appears immediately in container logs. +ENV PYTHONUNBUFFERED=1 + +# Put the uv and npm caches in a separate location, +# where they can persist and be shared across containers. +# The uv cache and virtual environment are on different volumes, so hardlinks won't work. +ENV UV_CACHE_DIR=/home/vscode/pkg-cache/uv \ + UV_PYTHON_INSTALL_DIR=/home/vscode/pkg-cache/uv-python \ + UV_LINK_MODE=symlink \ + NPM_CONFIG_CACHE=/home/vscode/pkg-cache/npm + +# Put the virtual environment outside the project directory, +# to improve performance on macOS and prevent accidental usage from the host machine. +# Activate it, so `uv run` doesn't need to be prefixed. +ENV UV_PROJECT_ENVIRONMENT=/home/vscode/venv \ + PATH="/home/vscode/venv/bin:$PATH" + +# Put tool scratch files outside the project directory too. +ENV TOX_WORK_DIR=/home/vscode/tox \ + RUFF_CACHE_DIR=/home/vscode/.cache/ruff \ + MYPY_CACHE_DIR=/home/vscode/.cache/mypy + +RUN ["chsh", "-s", "/usr/bin/zsh", "vscode"] + +USER vscode + +# Pre-create named volume mount points, so the new volume inherits `vscode` user ownership: +# https://docs.docker.com/engine/storage/volumes/#populate-a-volume-using-a-container +RUN ["mkdir", "/home/vscode/pkg-cache"] diff --git a/dev/docker-development.md b/dev/docker-development.md new file mode 100644 index 00000000..a5faba3c --- /dev/null +++ b/dev/docker-development.md @@ -0,0 +1,23 @@ +# Docker Compose Development (without VS Code) + +An alternative to the recommended [dev container](../README.md) workflow. + +## Setup +1. `docker compose run --rm django ./manage.py migrate` +1. `docker compose run --rm django ./manage.py createsuperuser` + +## Run +1. `docker compose up` +1. Access http://localhost:8000/ +1. `Ctrl+C` to stop + +To include the Celery worker: `docker compose --profile celery up` + +## Update +1. `docker compose down` +1. `docker compose pull` +1. `docker compose build --pull` +1. `docker compose run --rm django ./manage.py migrate` + +## Reset +Remove all data and volumes: `docker compose down -v` diff --git a/dev/export-env.sh b/dev/export-env.sh index ee0e3b0e..4ea1678e 100644 --- a/dev/export-env.sh +++ b/dev/export-env.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash # Export environment variables from the .env file in the first argument. # If no argument is given, default to "dev/.env.docker-compose-native". # This file must be sourced, not run. @@ -21,6 +22,7 @@ fi # Using "set -a" allows .env files with spaces or comments to work seamlessly # https://stackoverflow.com/a/45971167 set -a +# shellcheck source=.env.docker-compose-native . "$_dotenv_file" set +a diff --git a/dev/native-development.md b/dev/native-development.md new file mode 100644 index 00000000..9c410410 --- /dev/null +++ b/dev/native-development.md @@ -0,0 +1,19 @@ +# Native Development (advanced) + +Runs Python on the host while using Docker Compose for services. + +## Setup +1. [Install `uv`](https://docs.astral.sh/uv/getting-started/installation/) +1. Start services: `docker compose -f ./docker-compose.yml up -d` +1. Load environment: `source ./dev/export-env.sh` +1. `./manage.py migrate` +1. `./manage.py createsuperuser` + +## Run +1. Ensure services are running: `docker compose -f ./docker-compose.yml up -d` +1. `source ./dev/export-env.sh` +1. `./manage.py runserver` +1. In a separate terminal: `celery --app uvdat.celery worker --loglevel INFO --without-heartbeat` +1. Access http://localhost:8000/ + +Stop services when done: `docker compose stop` diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 73edc89f..7d69d56d 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -9,12 +9,11 @@ services: ] # Log printing is enhanced by a TTY tty: true - environment: - UV_ENV_FILE: ./dev/.env.docker-compose - working_dir: /opt/django-project + working_dir: /home/vscode/geodatalytics + env_file: ./dev/.env.docker-compose volumes: - - .:/opt/django-project - - uv_cache:/var/cache/uv + - .:/home/vscode/geodatalytics + - pkg_cache:/home/vscode/pkg-cache ports: - 8000:8000 depends_on: @@ -41,14 +40,13 @@ services: "--loglevel", "INFO", "--without-heartbeat" ] - # Docker Compose does not set the TTY width, which causes Celery errors + # uv progress doesn't display properly with a Docker TTY tty: false - environment: - UV_ENV_FILE: ./dev/.env.docker-compose - working_dir: /opt/django-project + working_dir: /home/vscode/geodatalytics + env_file: ./dev/.env.docker-compose volumes: - - .:/opt/django-project - - uv_cache:/var/cache/uv + - .:/home/vscode/geodatalytics + - pkg_cache:/home/vscode/pkg-cache depends_on: postgres: condition: service_healthy @@ -80,8 +78,8 @@ services: UV_ENV_FILE: ./dev/.env.docker-compose working_dir: /opt/django-project volumes: - - .:/opt/django-project - - uv_cache:/var/cache/uv + - .:/home/vscode/geodatalytics + - pkg_cache:/home/vscode/pkg-cache shm_size: 1gb profiles: ["gpu"] # Only runs with --profile gpu deploy: @@ -113,5 +111,5 @@ services: - 8080:8080 volumes: - uv_cache: + pkg_cache: web_node_modules: diff --git a/docs/setup.md b/docs/setup.md index b4c19d27..c131cdb5 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,60 +1,60 @@ # Setup Guide +This guide walks you through setting up GeoDatalytics for local development using Dev Containers. + +## Setup +1. Install [VS Code with dev container support](https://code.visualstudio.com/docs/devcontainers/containers#_installation). +1. Open the project in VS Code, then run `Dev Containers: Reopen in Container` + from the Command Palette (`Ctrl+Shift+P`). +1. Once the container is ready, open a terminal and run: + ```sh + ./manage.py migrate + ./manage.py createsuperuser + ``` -This guide walks you through setting up GeoDatalytics for local development using Docker Compose. +### Load Sample Data (Optional) +The ingest command loads datasets, charts, and project configuration from an ingestion file: -## Prerequisites +```sh +./manage.py ingest {JSON_FILE} +``` -- [Docker](https://docs.docker.com/get-docker/) (v20.10+) -- [Docker Compose](https://docs.docker.com/compose/install/) (v2.0+) -- [Node.js](https://nodejs.org/) (v22+ recommended) -- [npm](https://www.npmjs.com/) +Available ingest options (paths relative to `sample_data/`): ---- +- `boston_floods/data.json` +- `multiframe_test.json` +- `la_wildfires.json` +- `new_york_energy/data.json` -## Initial Setup +## Run +Open the **Run and Debug** panel (`Ctrl+Shift+D`) and select a launch configuration: -### 1. Build and Start Docker Containers +* **Django: Server** — Starts the development server at http://localhost:8000/ +* **Django: Server (eager Celery)** — Same, but Celery tasks run synchronously + in the web process (useful for debugging task code without a worker) +* **Celery: Worker** — Starts only the Celery worker +* **Django + Celery** — Starts both the server and a Celery worker +* **Django: Management Command** — Pick and run any management command -```bash -docker compose up -``` +## Test +Run the full test suite from a terminal: `tox` -> **Note:** Ensure all containers start and stay running before continuing. Check the logs for any errors. +Auto-format code: `tox -e format` -### 2. Initialize the Database +Run and debug individual tests from the **Testing** panel (`Ctrl+Shift+;`). -While the containers are running, open a **separate terminal** and run: +## Rebuild +After changes to the Dockerfile, Docker Compose files, or `devcontainer.json`, +run `Dev Containers: Rebuild Container` from the Command Palette (`Ctrl+Shift+P`). -```bash -# Apply database migrations -docker compose run --rm django ./manage.py migrate +For dependency changes in `pyproject.toml`, just run `uv sync --all-extras --all-groups`. -# Create an admin user (you will be prompted for email and password) -docker compose run --rm -it django ./manage.py createsuperuser -``` -> **Note:** The `createsuperuser` command prompts you to create login credentials (email and password). Use these credentials to sign into both the Admin Panel and User Interface. If you forget your password, run `createsuperuser` again to create a new admin account. -> **Windows Users:** If the `createsuperuser` command hangs or doesn't show prompts, prefix the command with `winpty`: -> -> ```bash -> winpty docker compose run --rm -it django ./manage.py createsuperuser -> ``` -### 3. Load Sample Data (Optional) -The ingest command loads datasets, charts, and project configuration from an ingestion file: -```bash -docker compose run --rm django ./manage.py ingest {JSON_FILE} -``` -Available ingest options (paths relative to `sample_data/`): -- `boston_floods/data.json` -- `multiframe_test.json` -- `la_wildfires.json` -- `new_york_energy/data.json` --- @@ -94,51 +94,8 @@ Press `Ctrl+C` in the terminal running `docker compose up`, or run: docker compose stop ``` ---- - -## Application Maintenance - -When new package dependencies or database schema changes occur, update your development environment: - -```bash -# Pull latest base images -docker compose pull - -# Rebuild containers (no cache) -docker compose build --pull --no-cache - -# Apply any new migrations -docker compose run --rm django ./manage.py migrate -``` - ---- - ## Troubleshooting -### Container Build Failures - -If you encounter build errors related to Python packages: - -1. **Clear Docker build cache:** - - ```bash - docker compose build --no-cache - ``` - -2. **Prune unused Docker resources:** - ```bash - docker system prune -a - ``` - -### Database Connection Issues - -Ensure PostgreSQL is running and healthy: - -```bash -docker compose ps -docker compose logs postgres -``` - ### Port Conflicts If ports 8000, 8080, 5432, or 9000 are in use, modify the port mappings in `docker-compose.override.yml`. @@ -158,23 +115,3 @@ docker compose up ``` GPU acceleration is optional and only needed for accelerated inferencing. - -### Windows-Specific Issues - -**Interactive commands don't work or hang:** - -On Windows (especially Git Bash/MINGW), interactive Docker commands like `createsuperuser` may hang or fail to display prompts. Prefix the command with `winpty`: - -```bash -winpty docker compose run --rm -it django ./manage.py createsuperuser -``` - -**"No such container" errors:** - -If you see errors like `No such container: django` when trying to attach to a running container, use `docker compose run` instead: - -```bash -docker compose run --rm -it django ./manage.py -``` - -This creates a new container instance rather than attaching to an existing one. diff --git a/pyproject.toml b/pyproject.toml index 6d62347a..3113d24a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "django-guardian==3.3.0", "django-large-image==0.10.3", "django-oauth-toolkit==3.2.0", - "django-resonant-settings[allauth,celery]==0.48.1", + "django-resonant-settings[allauth,celery]==0.50.2", "django-resonant-utils[allauth,s3_storage]==0.19.0", "django-stubs-ext==6.0.1", "djangorestframework==3.17.0", @@ -203,6 +203,10 @@ extend-immutable-calls = ["ninja.Query"] [tool.ruff.lint.flake8-self] extend-ignore-names = ["_base_manager", "_default_manager", "_meta"] +[tool.ruff.lint.flake8-type-checking] +runtime-evaluated-base-classes = ["pydantic.BaseModel"] +runtime-evaluated-decorators = ["pydantic.validate_call"] + [tool.ruff.lint.isort] # Sort by name, don't cluster "from" vs "import" force-sort-within-sections = true diff --git a/tox.ini b/tox.ini index ee053f3a..b322a949 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,8 @@ env_list = runner = uv-venv-lock-runner pass_env = DJANGO_* + RUFF_CACHE_DIR + MYPY_CACHE_DIR extras = development tasks diff --git a/uv.lock b/uv.lock index 8878d6b1..bcefbdc7 100644 --- a/uv.lock +++ b/uv.lock @@ -1323,15 +1323,15 @@ wheels = [ [[package]] name = "django-resonant-settings" -version = "0.48.1" +version = "0.50.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, { name = "django-environ" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/93/36/70d9e2441a204e138c50c97b7791efa8e8bc15e17dfea2c2413733d2e576/django_resonant_settings-0.48.1.tar.gz", hash = "sha256:c101ccc60e6922e875140eeb3c2f089aac2bf0ebca63c512f826f91286d16983", size = 19231, upload-time = "2026-03-06T05:01:10.797Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/e2/ab46f4f23546d24202e081216ddcadae49e8c24bba8d5109f2b00514e122/django_resonant_settings-0.50.2.tar.gz", hash = "sha256:aada679d9843a298910fc9b0037e285e49246850db6aca820d4e1688e13f9db3", size = 19231, upload-time = "2026-03-29T17:28:58.927Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/f2/15b8afe28ed7b76e8e7f7d6bac8b49cc977691142659d5468671c0889536/django_resonant_settings-0.48.1-py3-none-any.whl", hash = "sha256:01e494132d05828c4a79b54d1e5013052b4e40cc034e7dd52cf8b4c93a57e4e7", size = 26102, upload-time = "2026-03-06T05:01:09.754Z" }, + { url = "https://files.pythonhosted.org/packages/8c/31/b642b8c15fd221ce4b717daef5a85180f77e1b75a6be7784dca3f2eb3f87/django_resonant_settings-0.50.2-py3-none-any.whl", hash = "sha256:8c063f98052aa1827302c0c2440dc5e37c23d46cde65a884e6f249562334f5f8", size = 26047, upload-time = "2026-03-29T17:28:57.905Z" }, ] [package.optional-dependencies] @@ -2047,7 +2047,7 @@ requires-dist = [ { name = "django-large-image", specifier = "==0.10.3" }, { name = "django-minio-storage", marker = "extra == 'development'", specifier = "==0.5.9" }, { name = "django-oauth-toolkit", specifier = "==3.2.0" }, - { name = "django-resonant-settings", extras = ["allauth", "celery"], specifier = "==0.48.1" }, + { name = "django-resonant-settings", extras = ["allauth", "celery"], specifier = "==0.50.2" }, { name = "django-resonant-utils", extras = ["allauth", "s3-storage"], specifier = "==0.19.0" }, { name = "django-resonant-utils", extras = ["minio-storage"], marker = "extra == 'development'", specifier = "==0.19.0" }, { name = "django-s3-file-field", extras = ["minio"], marker = "extra == 'development'", specifier = "==1.1.0" },