From bad68822cd6a9435800109252e52c75b71ae7093 Mon Sep 17 00:00:00 2001 From: Vadym Mariiechko Date: Sat, 30 May 2026 07:56:34 +0200 Subject: [PATCH] Add monitoring-sql-warehouse asset Drop-in DABs SQL warehouse tuned for scheduled Databricks Alerts and monitoring queries: 2X-Small serverless PRO with auto_stop_mins=1, the serverless minimum verified end-to-end via DABs deploy. Keeps cost proportional to actual query time instead of warm-up dwell. The asset installs one resource YAML under /.sql_warehouse.yml (default resources/, picked up by the conventional resources/*.yml glob without any databricks.yml change) and an in-bundle usage doc at docs/monitoring-sql-warehouse/README.md covering the serverless 1-minute nuance, cross-resource reference pattern, and CLI/API paths for editing warehouses created outside DABs (with citations to the Create / Edit Warehouse API references and a worked curl example). Five prompts with safe defaults (target_dir, warehouse_resource_key, warehouse_name, cluster_size, auto_stop_mins); pattern-validated against the official cluster sizes (2X-Small through 5X-Large) and integer-only auto-stop values. Resource ships with channel pinned to CHANNEL_NAME_CURRENT so Databricks SQL Preview-channel rollouts cannot drift query behavior, and two cost-attribution tags (workload=monitoring-alerts, created_by=dabs-asset/monitoring-sql-warehouse) for traceability in usage reports. Asset-specific tests assert the exact installed file set, YAML parse, the proven serverless-PRO config (warehouse_type=PRO, enable_serverless_compute=true, auto_stop_mins integer-serialized, channel pinned to CHANNEL_NAME_CURRENT), both cost-attribution tags, and that custom warehouse_resource_key / target_dir / auto_stop_mins values flow into both the filename and the rendered YAML. Tagged for release as v1.8.0 in CHANGELOG.md. --- ASSETS.md | 1 + CHANGELOG.md | 10 + ROADMAP.md | 2 + assets/monitoring-sql-warehouse/README.md | 33 ++++ .../databricks_template_schema.json | 55 ++++++ .../docs/monitoring-sql-warehouse/README.md | 133 +++++++++++++ ...ouse_resource_key}}.sql_warehouse.yml.tmpl | 40 ++++ tests/assets/test_monitoring_sql_warehouse.py | 174 ++++++++++++++++++ .../assets/monitoring_sql_warehouse.json | 7 + 9 files changed, 455 insertions(+) create mode 100644 assets/monitoring-sql-warehouse/README.md create mode 100644 assets/monitoring-sql-warehouse/databricks_template_schema.json create mode 100644 assets/monitoring-sql-warehouse/template/docs/monitoring-sql-warehouse/README.md create mode 100644 assets/monitoring-sql-warehouse/template/{{.target_dir}}/{{.warehouse_resource_key}}.sql_warehouse.yml.tmpl create mode 100644 tests/assets/test_monitoring_sql_warehouse.py create mode 100644 tests/configs/assets/monitoring_sql_warehouse.json diff --git a/ASSETS.md b/ASSETS.md index c7dd057..f3f1536 100644 --- a/ASSETS.md +++ b/ASSETS.md @@ -19,6 +19,7 @@ Run the command from the root of your bundle project. The CLI will prompt for an |---|---|---|---| | `sdp-checkpoint-recovery` | Reset checkpoint selection on a Lakeflow Spark Declarative Pipeline after a source table has been dropped and recreated. | Stable | [README](assets/sdp-checkpoint-recovery/README.md) | | `dbx-ro-query` | Dependency-free Python wrapper around `databricks experimental aitools tools query` that gives LLM agents a guarded read-only SQL window into a Databricks workspace. Ships a `SKILL.md` for agent integration. | Stable | [README](assets/dbx-ro-query/README.md) | +| `monitoring-sql-warehouse` | Small, dedicated serverless SQL warehouse (2X-Small, `auto_stop_mins: 1`) for scheduled Databricks Alerts and monitoring queries. Keeps cost proportional to actual query time instead of idle warm-up. | Stable | [README](assets/monitoring-sql-warehouse/README.md) | ## What an asset is not diff --git a/CHANGELOG.md b/CHANGELOG.md index 601b6d9..d84dff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [1.8.0] - 2026-05-30 + +### Added +- **Asset `monitoring-sql-warehouse`**: Declarative Automation Bundle resource defining a small, dedicated serverless SQL warehouse tuned for bursty workloads (scheduled Databricks Alerts, monitoring queries, ad-hoc health checks). + - Defaults to `cluster_size: 2X-Small`, `warehouse_type: PRO`, `enable_serverless_compute: true`, and `auto_stop_mins: 1` (the serverless minimum verified end-to-end via DABs deploy; sub-minute values silently revert to the platform default). + - Channel pinned to `CHANNEL_NAME_CURRENT` so Databricks SQL Preview-channel rollouts cannot change query behavior under monitoring workloads. Cost-attribution tags `workload=monitoring-alerts` and `created_by=dabs-asset/monitoring-sql-warehouse` ship by default for traceability in usage reports. + - Five prompts (`target_dir`, `warehouse_resource_key`, `warehouse_name`, `cluster_size`, `auto_stop_mins`) with safe defaults; the resource file lands at `/.sql_warehouse.yml` and is picked up by the conventional `resources/*.yml` include glob without any `databricks.yml` change. + - In-bundle usage doc at `docs/monitoring-sql-warehouse/README.md` covers the auto-stop nuance, cross-resource ID reference pattern, and `databricks warehouses edit --auto-stop-mins` / REST API paths for editing warehouses created outside DABs. + ## [1.7.1] - 2026-05-13 ### Fixed @@ -176,6 +185,7 @@ Initial public release. - L2 tests: YAML syntax, environment targets, content validation - CI/CD tests: pipeline generation, auth patterns, branch references +[1.8.0]: https://github.com/vmariiechko/databricks-bundle-template/releases/tag/v1.8.0 [1.7.0]: https://github.com/vmariiechko/databricks-bundle-template/releases/tag/v1.7.0 [1.6.0]: https://github.com/vmariiechko/databricks-bundle-template/releases/tag/v1.6.0 [1.5.0]: https://github.com/vmariiechko/databricks-bundle-template/releases/tag/v1.5.0 diff --git a/ROADMAP.md b/ROADMAP.md index 11a49e2..cf00756 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -47,6 +47,8 @@ Modular sub-templates installable via `databricks bundle init --templ **Shipped:** - `sdp-checkpoint-recovery`: Reset checkpoint selection on a Lakeflow Spark Declarative Pipeline after a source table has been dropped and recreated +- `dbx-ro-query`: Dependency-free Python wrapper around `databricks experimental aitools tools query` that gives LLM agents a guarded read-only SQL window into a Databricks workspace +- `monitoring-sql-warehouse`: Dedicated serverless SQL warehouse tuned for bursty workloads (scheduled Alerts, monitoring queries) with `auto_stop_mins: 1` **Possible future assets** (ideas for contributors, not commitments): - `etl-pipeline`: Medallion-layered Declarative Pipeline with Bronze/Silver layers and DLT expectations diff --git a/assets/monitoring-sql-warehouse/README.md b/assets/monitoring-sql-warehouse/README.md new file mode 100644 index 0000000..dd1937e --- /dev/null +++ b/assets/monitoring-sql-warehouse/README.md @@ -0,0 +1,33 @@ +# monitoring-sql-warehouse + +A small, dedicated serverless SQL warehouse tuned for bursty workloads: scheduled Databricks Alerts, monitoring queries, and ad-hoc health checks. Defaults to `2X-Small` with `auto_stop_mins: 1` so cost stays proportional to actual query time. + +## Install + +```bash +databricks bundle init https://github.com/vmariiechko/databricks-bundle-template \ + --template-dir assets/monitoring-sql-warehouse +``` + +You will be prompted for: + +| Prompt | Default | Notes | +|---|---|---| +| `target_dir` | `resources` | Where the resource YAML lands. Default matches the `resources/*.yml` glob most bundles already include. | +| `warehouse_resource_key` | `monitoring_sql_warehouse` | DABs key under `resources.sql_warehouses.`. Also the resource filename. | +| `warehouse_name` | `Monitoring SQL Warehouse` | Display name in the workspace SQL Warehouses list. | +| `cluster_size` | `2X-Small` | One of: 2X-Small, X-Small, Small, Medium, Large, X-Large, 2X-Large, 3X-Large, 4X-Large. | +| `auto_stop_mins` | `1` | Integer. `1` is the serverless minimum; `0` disables auto-stop; `10+` for Pro/Classic. | + +Two files are installed: + +- `/.sql_warehouse.yml`: the DABs resource definition (serverless PRO, single cluster, channel pinned to `CHANNEL_NAME_CURRENT`, tagged `workload=monitoring-alerts` and `created_by=dabs-asset/monitoring-sql-warehouse`). +- `docs/monitoring-sql-warehouse/README.md`: usage notes, the `auto_stop_mins: 1` serverless nuance, and CLI/API paths for editing existing warehouses outside DABs. + +## Usage + +After install, open `docs/monitoring-sql-warehouse/README.md` in your project for deploy steps, the cross-resource reference pattern (`${resources.sql_warehouses..id}`), and the auto-stop behavior reference table. + +## What this asset is + +A standalone sub-template in the [databricks-bundle-template](https://github.com/vmariiechko/databricks-bundle-template) asset library. It does not depend on the core template; it can be installed into any Databricks bundle. See [ASSETS.md](../../ASSETS.md) for the full catalog. diff --git a/assets/monitoring-sql-warehouse/databricks_template_schema.json b/assets/monitoring-sql-warehouse/databricks_template_schema.json new file mode 100644 index 0000000..422e6fd --- /dev/null +++ b/assets/monitoring-sql-warehouse/databricks_template_schema.json @@ -0,0 +1,55 @@ +{ + "welcome_message": "\nMonitoring SQL Warehouse: Asset Installer\n\nInstalls a single Declarative Automation Bundle resource file that\ndefines a small, dedicated serverless SQL warehouse tuned for bursty\nworkloads (scheduled Databricks Alerts, monitoring queries, ad-hoc\nhealth checks).\n\nWhy it exists: Alerts and monitoring queries are sub-second, but a\ngeneral-purpose warehouse on a long auto-stop window stays warm for\nminutes after each run. Idle time dominates the bill. A dedicated\n2X-Small serverless warehouse with auto_stop_mins=1 keeps the cost\nproportional to actual query time.\n\nThe asset is opinionated about serverless PRO: the 1-minute auto-stop\nis only valid on serverless. Pro/Classic stay at the 10-minute floor.\n\nLet's pick the install settings...\n", + + "properties": { + "target_dir": { + "type": "string", + "default": "resources", + "description": "\nTarget directory for the warehouse resource file (relative to your bundle root).\nMost bundles already include `resources/*.yml` from databricks.yml, so the\ndefault `resources` lands the file where the bundle will auto-pick it up.\n\nIf your bundle uses a different convention (e.g., `resources/sql_warehouses/`),\nset that here. The directory will be created if it does not exist.\ntarget_dir", + "order": 1, + "pattern": "^[A-Za-z0-9_][A-Za-z0-9_./-]*$", + "pattern_match_failure_message": "Target directory must start with a letter, number, or underscore and contain only letters, numbers, underscores, slashes, hyphens, or dots." + }, + + "warehouse_resource_key": { + "type": "string", + "default": "monitoring_sql_warehouse", + "description": "\n========================================\n\nDABs resource key for the warehouse.\n\nThis key appears in the bundle as `resources.sql_warehouses.` and is\nalso the filename: `/.sql_warehouse.yml`. Other bundle\nresources reference this warehouse by `${resources.sql_warehouses..id}`.\n\nDefault `monitoring_sql_warehouse` is safe for most projects. Pick another\nname only if your bundle already defines a warehouse with this key.\nMust be lowercase snake_case (letters, numbers, underscores).\nwarehouse_resource_key", + "order": 2, + "pattern": "^[a-z][a-z0-9_]*$", + "pattern_match_failure_message": "Resource key must be lowercase, start with a letter, and contain only letters, numbers, and underscores." + }, + + "warehouse_name": { + "type": "string", + "default": "Monitoring SQL Warehouse", + "description": "\n========================================\n\nDisplay name shown in the Databricks workspace SQL Warehouses list.\n\nThis is what users see in the UI when picking a warehouse for an Alert\nor query. Free-form; spaces and mixed case are fine.\nwarehouse_name", + "order": 3, + "pattern": "^.+$", + "pattern_match_failure_message": "Warehouse name cannot be empty." + }, + + "cluster_size": { + "type": "string", + "default": "2X-Small", + "description": "\n========================================\n\nCluster size for the warehouse.\n\nValid Databricks SQL warehouse sizes (smallest to largest):\n 2X-Small, X-Small, Small, Medium, Large, X-Large, 2X-Large, 3X-Large, 4X-Large, 5X-Large\n\nDefault `2X-Small` is the smallest and cheapest. It is sufficient for\nDatabricks Alerts and monitoring queries that complete in under a second.\nScale up only if your queries scan large tables.\ncluster_size", + "order": 4, + "pattern": "^(2X-Small|X-Small|Small|Medium|Large|X-Large|2X-Large|3X-Large|4X-Large|5X-Large)$", + "pattern_match_failure_message": "Cluster size must be one of: 2X-Small, X-Small, Small, Medium, Large, X-Large, 2X-Large, 3X-Large, 4X-Large, 5X-Large." + }, + + "auto_stop_mins": { + "type": "string", + "default": "1", + "description": "\n========================================\n\nIdle minutes before the warehouse auto-stops (integer).\n\nDefault `1` is the serverless minimum verified via DABs deploy: the UI\nshows 'After 1 minute of inactivity'. Sub-minute values are NOT supported\n(the field is modeled as an integer).\n\nSpecial values:\n 0 = disable auto-stop entirely (warehouse runs until manually stopped)\n 1 = serverless minimum (recommended for Alerts / bursty workloads)\n 10+ = required floor for Pro/Classic warehouses; safe upper bound for serverless too\n\nKeep at 1 unless you have a reason to override.\nauto_stop_mins", + "order": 5, + "pattern": "^(0|[1-9][0-9]*)$", + "pattern_match_failure_message": "auto_stop_mins must be a non-negative integer (0 to disable, 1 for serverless minimum, 10+ for Pro/Classic)." + } + }, + + "success_message": "\n========================================\n\nMonitoring SQL warehouse installed:\n - Resource: {{.target_dir}}/{{.warehouse_resource_key}}.sql_warehouse.yml\n - Usage doc: docs/monitoring-sql-warehouse/README.md\n\nNext steps:\n\n1. Confirm your `databricks.yml` includes the resource file. Most bundles\n already include `resources/*.yml` by default; verify:\n include:\n - resources/*.yml\n If you installed to a custom subdirectory, add the matching glob.\n\n2. Validate and deploy:\n databricks bundle validate -t \n databricks bundle deploy -t \n\n3. After deploy, the warehouse appears in the workspace SQL Warehouses list\n as '{{.warehouse_name}}'. Point your Alerts / monitoring queries at it.\n\n4. Reference the warehouse ID from other bundle resources via:\n ${resources.sql_warehouses.{{.warehouse_resource_key}}.id}\n\nUsage notes, the auto_stop_mins=1 nuance, and the CLI/API paths for\nexisting warehouses live in 'docs/monitoring-sql-warehouse/README.md'.\n", + + "min_databricks_cli_version": "v0.296.0", + "version": 1 +} diff --git a/assets/monitoring-sql-warehouse/template/docs/monitoring-sql-warehouse/README.md b/assets/monitoring-sql-warehouse/template/docs/monitoring-sql-warehouse/README.md new file mode 100644 index 0000000..8c27d67 --- /dev/null +++ b/assets/monitoring-sql-warehouse/template/docs/monitoring-sql-warehouse/README.md @@ -0,0 +1,133 @@ +# Monitoring SQL Warehouse + +A small, dedicated serverless SQL warehouse for scheduled Databricks Alerts, monitoring queries, and ad-hoc health checks. Optimized for bursty, sub-second workloads where idle time would otherwise dominate the bill. + +## When to use this + +Use a dedicated warehouse like this when: + +- You run scheduled Databricks Alerts (every minute, every five minutes, every hour) whose underlying SQL completes in well under a second. +- The default warehouse those queries land on has a long auto-stop window (10 minutes is the platform default for Pro/Classic; serverless allows 1 minute). +- You see the warehouse staying warm for the full auto-stop window after a single sub-second query, inflating monthly cost. + +A separate `2X-Small` serverless warehouse with `auto_stop_mins: 1` keeps cost proportional to actual query time: the warehouse starts on the Alert run, executes in under a second, and stops one minute after the last query. + +## What this asset installs + +| Path | Purpose | +|---|---| +| `/.sql_warehouse.yml` | The DABs SQL warehouse resource definition. | +| `docs/monitoring-sql-warehouse/README.md` | This file. | + +Default install paths: + +- Resource: `resources/monitoring_sql_warehouse.sql_warehouse.yml` +- Docs: `docs/monitoring-sql-warehouse/README.md` + +## Bundle integration + +Most generated bundles already include `resources/*.yml` from `databricks.yml`: + +```yaml +include: + - resources/*.yml +``` + +If your bundle uses that pattern and you accepted the default `target_dir`, the warehouse is picked up automatically. No `databricks.yml` change is needed. + +If you installed to a custom subdirectory (e.g., `resources/sql_warehouses/`), add the matching glob to `databricks.yml`: + +```yaml +include: + - resources/sql_warehouses/*.yml +``` + +## Deploy + +```bash +databricks bundle validate -t +databricks bundle deploy -t +``` + +After deploy, the warehouse appears in the workspace SQL Warehouses list under the display name you chose at install time. + +## Referencing the warehouse from other bundle resources + +Other resources (Alerts, jobs, dashboards) can pin to this warehouse by ID: + +```yaml +${resources.sql_warehouses..id} +``` + +For example, an Alert resource: + +```yaml +resources: + alerts: + my_alert: + warehouse_id: ${resources.sql_warehouses.monitoring_sql_warehouse.id} + # ... rest of the alert definition +``` + +## The `auto_stop_mins: 1` nuance + +The proven configuration was verified by deploying this asset to a real workspace (Databricks SQL channel v2026.10). The notes below capture behavior that contradicts or refines parts of the public API reference. + +| Setting | Behavior | +|---|---| +| `auto_stop_mins: 0` | Disables auto-stop entirely. The warehouse runs until manually stopped. Does NOT mean "stop immediately." | +| `auto_stop_mins: 1` | Valid and effective on serverless via DABs/API. The workspace UI shows "After 1 minute of inactivity." This is the recommended value for Alerts and monitoring workloads. | +| `auto_stop_mins: 0.5` (or any sub-minute value) | Not supported. The field is modeled as an integer; the bundle CLI emits an integer-coercion warning and the workspace silently reverts to the platform default (120 minutes). | +| `auto_stop_mins: 10+` | Per the public [Warehouses API reference][create-warehouse-api], the documented contract is `auto_stop_mins` must be `0` or `>= 10`. This floor applies to Pro/Classic warehouses. Safe upper bound for serverless too. | + +The public [Warehouses API reference][create-warehouse-api] states that `auto_stop_mins` must be `0` or `>= 10`. That floor applies to Pro/Classic warehouses; the serverless path accepts `1` end-to-end via DABs and via the CLI (confirmed by deploying this resource and inspecting the workspace UI). + +The manual UI editor for an existing warehouse may enforce a higher floor on serverless than 1 minute. If you see that, fall back to the DABs path or the `databricks warehouses edit` CLI path, both of which apply `1` on serverless. + +[create-warehouse-api]: https://docs.databricks.com/api/workspace/warehouses/create +[edit-warehouse-api]: https://docs.databricks.com/api/workspace/warehouses/edit + +## Applying the same tuning to a warehouse created outside DABs + +If you have an existing warehouse that you do not want to recreate as a DABs resource, you can edit it in place to the same auto-stop behavior. + +### Via the Databricks CLI + +```bash +databricks warehouses edit --auto-stop-mins 1 +``` + +The full flag set is visible via `databricks warehouses edit --help`. For serverless, also ensure `--enable-serverless-compute` is on and `--warehouse-type PRO` is set. + +### Via the REST API + +Use the [Edit Warehouse endpoint][edit-warehouse-api] (`POST /api/2.0/sql/warehouses/{id}/edit`) with `auto_stop_mins: 1`. For serverless, include `enable_serverless_compute: true` and `warehouse_type: PRO` in the request body. The endpoint reference still documents the `0 or >= 10` contract; the serverless `1` value is accepted in practice and is what DABs and the CLI emit. + +```bash +curl -X POST \ + -H "Authorization: Bearer $DATABRICKS_TOKEN" \ + -H "Content-Type: application/json" \ + https://$DATABRICKS_HOST/api/2.0/sql/warehouses//edit \ + -d '{ + "auto_stop_mins": 1, + "enable_serverless_compute": true, + "warehouse_type": "PRO" + }' +``` + +### Via the workspace UI + +The "Edit warehouse" form may enforce a higher minimum than 1 minute for serverless. If the UI rejects `1`, use the CLI or DABs path. + +## Cost intuition + +A `2X-Small` serverless warehouse billed per second of running time, kept warm by a 1-minute auto-stop after each sub-second query, costs a small constant per scheduled run plus the query duration itself. Compared to a 10-minute auto-stop window on the same size warehouse, the per-run dwell time drops by an order of magnitude. Apply this to dozens of scheduled Alerts and the monthly delta is meaningful. + +## References + +1. [Databricks Docs: Configure SQL warehouse settings (auto-stop)](https://docs.databricks.com/aws/en/compute/sql-warehouse/warehouse-behavior) +2. [Declarative Automation Bundles: `sql_warehouses` resource reference](https://docs.databricks.com/aws/en/dev-tools/bundles/resources.html#sql_warehouse) +3. [Databricks API: Create Warehouse][create-warehouse-api] (`auto_stop_mins` contract) +4. [Databricks API: Edit Warehouse][edit-warehouse-api] (in-place edits to an existing warehouse) +5. [Databricks CLI: `databricks warehouses edit`](https://docs.databricks.com/aws/en/dev-tools/cli/reference/warehouses-commands.html#edit) +6. [Databricks Alerts overview](https://docs.databricks.com/aws/en/sql/user/alerts/) diff --git a/assets/monitoring-sql-warehouse/template/{{.target_dir}}/{{.warehouse_resource_key}}.sql_warehouse.yml.tmpl b/assets/monitoring-sql-warehouse/template/{{.target_dir}}/{{.warehouse_resource_key}}.sql_warehouse.yml.tmpl new file mode 100644 index 0000000..533e0d7 --- /dev/null +++ b/assets/monitoring-sql-warehouse/template/{{.target_dir}}/{{.warehouse_resource_key}}.sql_warehouse.yml.tmpl @@ -0,0 +1,40 @@ +# Monitoring SQL Warehouse +# +# Small, dedicated serverless SQL warehouse tuned for bursty workloads: +# scheduled Databricks Alerts, monitoring queries, ad-hoc health checks. +# +# Why a dedicated warehouse: a general-purpose warehouse on a long +# auto-stop window stays warm for minutes after a sub-second Alert +# query, and idle time dominates the bill. A 2X-Small serverless +# warehouse with auto_stop_mins=1 keeps cost proportional to actual +# query time. +# +# Reference this warehouse from other bundle resources via: +# ${resources.sql_warehouses.{{.warehouse_resource_key}}.id} +# +# See docs/monitoring-sql-warehouse/README.md for usage notes, +# the auto_stop_mins=1 serverless nuance, and CLI/API alternatives +# for warehouses created outside DABs. + +resources: + sql_warehouses: + {{.warehouse_resource_key}}: + name: {{.warehouse_name}} + cluster_size: {{.cluster_size}} + warehouse_type: PRO + enable_serverless_compute: true + auto_stop_mins: {{.auto_stop_mins}} + min_num_clusters: 1 + max_num_clusters: 1 + # Pin to the stable Databricks SQL channel. Prevents Preview-channel + # feature flags from changing query behavior under monitoring queries + # that are assumed-stable. Override to CHANNEL_NAME_PREVIEW only if + # you are intentionally validating an upcoming release. + channel: + name: CHANNEL_NAME_CURRENT + tags: + custom_tags: + - key: workload + value: monitoring-alerts + - key: created_by + value: dabs-asset/monitoring-sql-warehouse diff --git a/tests/assets/test_monitoring_sql_warehouse.py b/tests/assets/test_monitoring_sql_warehouse.py new file mode 100644 index 0000000..376237a --- /dev/null +++ b/tests/assets/test_monitoring_sql_warehouse.py @@ -0,0 +1,174 @@ +"""Asset-specific tests for `assets/monitoring-sql-warehouse`. + +Framework-level smoke checks (no .tmpl leftovers, schema validity, etc.) +live in `tests/assets/test_framework.py`. This file asserts behavior +unique to this asset: exact files installed at the chosen paths, YAML +parses and matches the proven serverless-PRO configuration, and the +custom `warehouse_resource_key` flows into both the filename and the +DABs resource key. +""" + +from pathlib import Path + +import pytest +import yaml + +ASSET_NAME = "monitoring-sql-warehouse" +DEFAULT_CONFIG = "monitoring_sql_warehouse" +DEFAULT_TARGET_DIR = "resources" +DEFAULT_RESOURCE_KEY = "monitoring_sql_warehouse" +DEFAULT_WAREHOUSE_NAME = "Monitoring SQL Warehouse" +DEFAULT_CLUSTER_SIZE = "2X-Small" +DEFAULT_AUTO_STOP_MINS = 1 +DOCS_PATH = "docs/monitoring-sql-warehouse/README.md" + + +def _resource_path(target_dir: str, resource_key: str) -> str: + return f"{target_dir}/{resource_key}.sql_warehouse.yml" + + +@pytest.fixture(scope="module") +def installed(install_asset) -> Path: + """Install the asset once per module using the default config.""" + return install_asset(ASSET_NAME, config=DEFAULT_CONFIG) + + +def test_installs_expected_files(installed: Path): + expected_yaml = installed / _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY) + expected_doc = installed / DOCS_PATH + assert expected_yaml.is_file(), f"missing warehouse YAML: {expected_yaml}" + assert expected_doc.is_file(), f"missing usage doc: {expected_doc}" + + +def test_no_extra_files_installed(installed: Path): + """Asset should install exactly two files: the resource YAML and the usage doc.""" + installed_files = sorted( + p.relative_to(installed).as_posix() for p in installed.rglob("*") if p.is_file() + ) + expected = sorted([ + _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY), + DOCS_PATH, + ]) + assert installed_files == expected, ( + f"unexpected files: {set(installed_files) - set(expected)}" + ) + + +def test_warehouse_yaml_parses(installed: Path): + yaml_path = installed / _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY) + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + assert "resources" in data, "top-level `resources:` key missing" + assert "sql_warehouses" in data["resources"], "missing sql_warehouses block" + assert DEFAULT_RESOURCE_KEY in data["resources"]["sql_warehouses"], ( + f"warehouse key {DEFAULT_RESOURCE_KEY!r} missing" + ) + + +def test_warehouse_matches_proven_config(installed: Path): + """Assert the YAML matches the values Vadim verified against a live workspace.""" + yaml_path = installed / _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY) + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + wh = data["resources"]["sql_warehouses"][DEFAULT_RESOURCE_KEY] + + assert wh["name"] == DEFAULT_WAREHOUSE_NAME + assert wh["cluster_size"] == DEFAULT_CLUSTER_SIZE + assert wh["warehouse_type"] == "PRO" + assert wh["enable_serverless_compute"] is True + assert wh["auto_stop_mins"] == DEFAULT_AUTO_STOP_MINS + assert isinstance(wh["auto_stop_mins"], int), ( + "auto_stop_mins must serialize as an integer; the field is integer-only" + ) + assert wh["min_num_clusters"] == 1 + assert wh["max_num_clusters"] == 1 + + +def test_warehouse_has_cost_attribution_tags(installed: Path): + """The proven config tags the warehouse so cost reporting can attribute it. + + `workload=monitoring-alerts` -> cost dashboards group by workload type. + `created_by=dabs-asset/...` -> traceability back to the originating asset. + """ + yaml_path = installed / _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY) + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + wh = data["resources"]["sql_warehouses"][DEFAULT_RESOURCE_KEY] + + tags = wh.get("tags", {}).get("custom_tags", []) + assert {"key": "workload", "value": "monitoring-alerts"} in tags, ( + f"missing workload tag in custom_tags: {tags}" + ) + assert {"key": "created_by", "value": "dabs-asset/monitoring-sql-warehouse"} in tags, ( + f"missing created_by tag in custom_tags: {tags}" + ) + + +def test_warehouse_pins_stable_channel(installed: Path): + """Channel is pinned to CHANNEL_NAME_CURRENT so Preview rollouts don't drift behavior.""" + yaml_path = installed / _resource_path(DEFAULT_TARGET_DIR, DEFAULT_RESOURCE_KEY) + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + wh = data["resources"]["sql_warehouses"][DEFAULT_RESOURCE_KEY] + assert wh.get("channel", {}).get("name") == "CHANNEL_NAME_CURRENT", ( + f"channel.name should be CHANNEL_NAME_CURRENT, got: {wh.get('channel')}" + ) + + +def test_custom_resource_key_flows_into_filename_and_yaml(install_asset): + """A user-supplied warehouse_resource_key must appear in both the + filename and the DABs resource key inside the YAML.""" + custom_key = "alerts_warehouse_x" + out = install_asset( + ASSET_NAME, + overrides={ + "target_dir": "resources", + "warehouse_resource_key": custom_key, + "warehouse_name": "Alerts Warehouse X", + "cluster_size": "X-Small", + "auto_stop_mins": "5", + }, + ) + + yaml_path = out / _resource_path("resources", custom_key) + assert yaml_path.is_file(), f"missing renamed YAML: {yaml_path}" + + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + assert custom_key in data["resources"]["sql_warehouses"], ( + f"resource key {custom_key!r} did not flow into YAML" + ) + wh = data["resources"]["sql_warehouses"][custom_key] + assert wh["name"] == "Alerts Warehouse X" + assert wh["cluster_size"] == "X-Small" + assert wh["auto_stop_mins"] == 5 + + +def test_custom_target_dir_is_honored(install_asset): + """A user-supplied target_dir must place the YAML under that subdirectory.""" + out = install_asset( + ASSET_NAME, + overrides={ + "target_dir": "resources/sql_warehouses", + "warehouse_resource_key": DEFAULT_RESOURCE_KEY, + "warehouse_name": DEFAULT_WAREHOUSE_NAME, + "cluster_size": DEFAULT_CLUSTER_SIZE, + "auto_stop_mins": "1", + }, + ) + yaml_path = out / _resource_path("resources/sql_warehouses", DEFAULT_RESOURCE_KEY) + assert yaml_path.is_file(), f"missing YAML at custom target_dir: {yaml_path}" + + +def test_auto_stop_zero_serializes_as_int(install_asset): + """auto_stop_mins=0 (disable auto-stop) must render as the integer 0, not '0'.""" + out = install_asset( + ASSET_NAME, + overrides={ + "target_dir": "resources", + "warehouse_resource_key": DEFAULT_RESOURCE_KEY, + "warehouse_name": DEFAULT_WAREHOUSE_NAME, + "cluster_size": DEFAULT_CLUSTER_SIZE, + "auto_stop_mins": "0", + }, + ) + yaml_path = out / _resource_path("resources", DEFAULT_RESOURCE_KEY) + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + wh = data["resources"]["sql_warehouses"][DEFAULT_RESOURCE_KEY] + assert wh["auto_stop_mins"] == 0 + assert isinstance(wh["auto_stop_mins"], int) diff --git a/tests/configs/assets/monitoring_sql_warehouse.json b/tests/configs/assets/monitoring_sql_warehouse.json new file mode 100644 index 0000000..91d3ceb --- /dev/null +++ b/tests/configs/assets/monitoring_sql_warehouse.json @@ -0,0 +1,7 @@ +{ + "target_dir": "resources", + "warehouse_resource_key": "monitoring_sql_warehouse", + "warehouse_name": "Monitoring SQL Warehouse", + "cluster_size": "2X-Small", + "auto_stop_mins": "1" +}