diff --git a/manifest.json b/manifest.json index 94d04391..b19ce7db 100644 --- a/manifest.json +++ b/manifest.json @@ -46,7 +46,7 @@ "version": "0.1.0" }, "databricks-apps": { - "description": "Build apps on Databricks Apps platform.", + "description": "Build AppKit apps on Databricks: compose analytics, Lakebase OLTP/synced reads, Genie, serving, files, jobs, and custom endpoints. Capability-based scaffolding \u2014 invoke BEFORE implementation.", "files": [ "SKILL.md", "agents/openai.yaml", @@ -54,16 +54,22 @@ "assets/databricks.svg", "references/appkit/appkit-sdk.md", "references/appkit/custom-endpoints.md", + "references/appkit/data-patterns.md", + "references/appkit/environments.md", "references/appkit/files.md", "references/appkit/frontend.md", "references/appkit/genie.md", "references/appkit/jobs.md", + "references/appkit/lakebase-oltp.md", + "references/appkit/lakebase-synced-reads.md", "references/appkit/lakebase.md", + "references/appkit/lifecycle.md", "references/appkit/model-serving.md", "references/appkit/overview.md", "references/appkit/proto-contracts.md", "references/appkit/proto-first.md", "references/appkit/sql-queries.md", + "references/appkit/warehouse-mutations.md", "references/other-frameworks.md", "references/platform-guide.md", "references/testing.md" diff --git a/skills/databricks-apps/SKILL.md b/skills/databricks-apps/SKILL.md index e7d60134..b265c612 100644 --- a/skills/databricks-apps/SKILL.md +++ b/skills/databricks-apps/SKILL.md @@ -1,6 +1,6 @@ --- name: databricks-apps -description: "Build apps on Databricks Apps platform. Use when asked to create dashboards, data apps, analytics tools, or visualizations. Evaluates data access patterns (analytics vs Lakebase synced tables) before scaffolding. Invoke BEFORE starting implementation." +description: "Build AppKit apps on Databricks: compose analytics, Lakebase OLTP/synced reads, Genie, serving, files, jobs, and custom endpoints. Capability-based scaffolding — invoke BEFORE implementation." compatibility: Requires databricks CLI (>= v0.294.0) metadata: version: "0.1.2" @@ -11,216 +11,116 @@ parent: databricks-core **FIRST**: Use the parent `databricks-core` skill for CLI basics, authentication, and profile selection. -Build apps that deploy to Databricks Apps platform. - -## Required Reading by Phase - -| Phase | READ BEFORE proceeding | -|-------|------------------------| -| Scaffolding | **⚠️ STOP — review the State Storage Guidance and complete the Data Access Decision Gate below before scaffolding.** Parent `databricks-core` skill (auth, warehouse discovery); then run `databricks apps manifest` + `databricks apps init` with `--features` and `--set` (see AppKit section below) | -| Writing SQL queries | [SQL Queries Guide](references/appkit/sql-queries.md) | -| Writing UI components | [Frontend Guide](references/appkit/frontend.md) | -| Using `useAnalyticsQuery` | [AppKit SDK](references/appkit/appkit-sdk.md) | -| Adding API endpoints | [Custom Endpoints Guide](references/appkit/custom-endpoints.md) | -| Using Lakebase (OLTP database) | [Lakebase Guide](references/appkit/lakebase.md) | -| Adding Genie chat / Genie-powered apps | [Genie Guide](references/appkit/genie.md) — follow the Genie agent workflow below | -| Using Model Serving (ML inference) | [Model Serving Guide](references/appkit/model-serving.md) | -| Typed data contracts (proto-first design) | [Proto-First Guide](references/appkit/proto-first.md) and [Plugin Contracts](references/appkit/proto-contracts.md) | -| Managing files in UC Volumes | [Files Guide](references/appkit/files.md) | -| Triggering / monitoring Lakeflow Jobs from the app | [Jobs Guide](references/appkit/jobs.md) | -| Platform rules (permissions, deployment, limits) | [Platform Guide](references/platform-guide.md) — READ for ALL apps including AppKit | -| Non-AppKit app (Streamlit, FastAPI, Flask, Gradio, Next.js, etc.) | [Other Frameworks](references/other-frameworks.md) | - -## Generic Guidelines - -- **App name**: ≤26 characters, lowercase letters/numbers/hyphens only (no underscores). dev- prefix adds 4 chars, max 30 total. -- **Validation**: `databricks apps validate --profile ` before deploying. -- **Smoke tests** (AppKit only): ALWAYS update `tests/smoke.spec.ts` selectors BEFORE running validation. Default template checks for "Minimal Databricks App" heading and "hello world" text — these WILL fail in your custom app. See [testing guide](references/testing.md). -- **Smoke test selectors**: use only Playwright locator APIs — `getByRole`, `getByText`, `getByPlaceholder`, `getByLabel`. `getByLabelText` does not exist in Playwright (it is a React Testing Library method) and throws `TypeError` at runtime. See [testing guide](references/testing.md) or `npx playwright codegen`. -- **Smoke test data**: keep result sets under the 1 MB analytics-event payload cap. Queries returning thousands of rows cause `INVALID_REQUEST: Event exceeds max size of 1048576 bytes` and `net::ERR_ABORTED`, leaving every asserted UI element absent. Use `LIMIT` or an aggregated query (e.g. `COUNT(*) GROUP BY status`) — never raw row dumps. -- **AppKit version**: never override the `@databricks/appkit` or `@databricks/appkit-ui` version in `package.json` — `databricks apps init` sets the correct version. Do not run `npm install @databricks/appkit@` unless explicitly asked by the user. If you need a different version, re-scaffold with `databricks apps init --version `. -- **Authentication**: covered by parent `databricks-core` skill. -- **AppKit API surface**: before writing code that calls AppKit APIs (`createApp`, plugin shapes, `useAnalyticsQuery`, etc.), run `npx @databricks/appkit docs
` and use the actual signature. Training data has stale shapes; a single invented signature fails `tsc --noEmit` during validate. The docs ship with the installed AppKit and are the authoritative source. -- **TypeScript casts**: never use `as unknown as ` double-assertions — `appkit lint` enforces `no-double-type-assertion` and one violation fails the entire validate step. Instead: narrow with Zod (`z.infer`), use a runtime type guard, or write a typed mapper function. If a query result needs reshaping, type the row schema via queryKey types rather than casting. - -## Project Structure (after `databricks apps init --features analytics`) -- `client/src/App.tsx` — main React component (start here) -- `config/queries/*.sql` — SQL query files (queryKey = filename without .sql) -- `server/server.ts` — backend entry (`onPluginsReady` + Express routes) -- `tests/smoke.spec.ts` — smoke test (⚠️ MUST UPDATE selectors for your app) -- `client/src/appKitTypes.d.ts` — auto-generated types (`npm run typegen`) - -## Project Structure (after `databricks apps init --features lakebase`) -- `server/server.ts` — backend with Lakebase pool + Express routes -- `client/src/App.tsx` — React frontend -- `app.yaml` — manifest with `database` resource declaration -- `package.json` — includes `@databricks/lakebase` dependency -- Note: **No `config/queries/`** — Lakebase apps use `appkit.lakebase.query()` in Express routes, not SQL files - -## Data Discovery - -Before writing any SQL, use the parent `databricks-core` skill for data exploration — search `information_schema` by keyword, then batch `discover-schema` for the tables you need. Do NOT skip this step. - -**State Storage Guidance (evaluate BEFORE the Decision Gate):** +Build TypeScript/React apps on the Databricks Apps platform (AppKit). Apps are **compositions of capabilities** — not single archetypes. See [Data Patterns Guide](references/appkit/data-patterns.md) for the full model. -If the user's app description involves storing or persisting data — forms, CRUD operations, user submissions, orders, todos, or other user-generated content — the app likely needs a Lakebase database. +## Agent workflow (follow in order) -1. **Ask the user** whether the app needs persistent storage (Lakebase) before scaffolding. Do not silently add Lakebase. -2. If confirmed, use the **`databricks-lakebase`** skill to create a Lakebase project and obtain the branch and database resource names. -3. Scaffold with `--features lakebase` and pass `--set lakebase.postgres.branch= --set lakebase.postgres.database=`. -4. If the app **also** reads from Unity Catalog tables, proceed to the Data Access Decision Gate below to determine whether to add `--features analytics` or use Lakebase synced tables. +### 0. Detect environment + prerequisites -Do NOT add Lakebase to analytics, dashboard, or visualization apps unless the user explicitly requests persistent write-back storage. Read-only data display, filters, and preferences do not require a database. +**Check the environment first — the workflow differs.** → [Environments](references/appkit/environments.md) -## Development Workflow (FOLLOW THIS ORDER) - -**Data Access Decision Gate (REQUIRED before scaffolding):** - -If the app reads from Unity Catalog / lakehouse tables, you MUST show the comparison below to the user and ask them to choose. Do not skip this. Do not choose for them. +```bash +echo "${DATABRICKS_APPS_AGENTIC_MODE:-}" +``` -| | **(A) Lakebase synced tables** | **(B) Analytics** | -|--|---|---| -| Speed | Sub-second responses | Takes a few seconds | -| Best for | Full-text search, typeahead, autocomplete, real-time lookups, operational apps | Dashboards, charts, aggregations, KPIs, filtered queries, browsing | -| How it works | Data synced from Delta into Lakebase Postgres | Queries run on SQL warehouse at read time | +- **`true` → Agentic mode:** the app is already scaffolded and every resource is provisioned. Auth is ambient — never select a profile, omit `--profile`. **Skip** step 3 and the deploy in step 5; in step 2 run **only** the design + discovery gates (write_path, read_path, data_discovery); read wired plugins from `appkit.plugins.json` / `app.yaml` instead of inferring; no smoke tests. Still do step 1 (read, don't infer), step 4 (write code), `npm run dev`, and `databricks apps validate`. → [Environments](references/appkit/environments.md) +- **else → Local:** select a profile via `databricks-core` (never auto-select) and follow every step below. -After showing the table, add a brief recommendation. Default to recommending Analytics (B) for most read-only apps — dashboards, charts, filtered queries, browsing, and aggregations. Recommend Lakebase synced tables (A) only when the app needs sub-second latency for full-text search, typeahead/autocomplete, real-time lookups by ID, or operational data serving. Note: "search" or "filter" in a prompt usually means SQL WHERE clauses (Analytics), not full-text search (Lakebase). Always let the user make the final call. +### 1. Infer capabilities -After the user chooses: -- (A) Lakebase synced tables → scaffold with `--features lakebase`. See [Lakebase Guide](references/appkit/lakebase.md) for full workflow. -- (B) Analytics → scaffold with `--features analytics`. -- Both → scaffold with `--features analytics,lakebase` if the app needs both patterns. -- If the app does NOT read Unity Catalog data (pure CRUD, Genie, Model Serving), skip this gate and scaffold with the appropriate `--features` flag. +From the user request, build a capability set (`reads_warehouse`, `writes_oltp`, `genie`, `files`, etc.). Include only what was asked for — do not add Lakebase or analytics by default. -**Analytics apps** (`--features analytics`): +**Agentic mode:** do **not** infer — read the enabled plugins from `appkit.plugins.json` / `app.yaml`. If the request needs a plugin that isn't wired, **stop and tell the user**; never provision it yourself. -1. Create SQL files in `config/queries/` -2. Run `npm run typegen` — verify all queries show ✓ -3. Read `client/src/appKitTypes.d.ts` to see generated types -4. **THEN** write `App.tsx` using the generated types -5. Update `tests/smoke.spec.ts` selectors -6. Run `databricks apps validate --profile ` +→ [Data Patterns: Capability catalog](references/appkit/data-patterns.md#capability-catalog) -**DO NOT** write UI code before running typegen — types won't exist and you'll waste time on compilation errors. +### 2. Run conditional gates -**Lakebase apps** (`--features lakebase`): No SQL files or typegen. See [Lakebase Guide](references/appkit/lakebase.md) for the `onPluginsReady` pattern: initialize schema at startup, register Express routes in `server/server.ts`, then build the React frontend. +Run gates **only** for capabilities in the set (write path, read path, Genie space, Lakebase resources, data discovery). -## When to Use What +**Agentic mode:** run **design + discovery** gates only (write_path, read_path, data_discovery). Skip provisioning gates (Lakebase resources, Genie space) — they already exist. -After completing the decision gate above, use this routing table: +→ [Data Patterns: Conditional gates](references/appkit/data-patterns.md#conditional-gates) -- **Read analytics data → display in chart/table**: Use visualization components with `queryKey` prop -- **Read analytics data → custom display (KPIs, cards)**: Use `useAnalyticsQuery` hook -- **Read analytics data → need computation before display**: Still use `useAnalyticsQuery`, transform client-side -- **Read lakehouse data at low latency (lookups, search, catalogs)**: Use Lakebase synced tables — see [Lakebase Guide](references/appkit/lakebase.md) -- **Read/write persistent data (users, orders, CRUD state)**: Use Lakebase via Express routes in `onPluginsReady` — see [Lakebase Guide](references/appkit/lakebase.md) -- **Natural language query interface over tables (Genie)**: Use `genie()` plugin — see [Genie Guide](references/appkit/genie.md) -- **Call ML model endpoint**: Use `serving()` plugin — see [Model Serving Guide](references/appkit/model-serving.md) -- **Trigger or monitor a Lakeflow Job from the app**: Use the `jobs()` plugin — see [Jobs Guide](references/appkit/jobs.md) -- **⚠️ NEVER add custom endpoints to run SELECT queries against the warehouse** — always use SQL files in `config/queries/` -- **⚠️ NEVER use `useAnalyticsQuery` for Lakebase data** — it queries the SQL warehouse only +### 3. Scaffold -## Frameworks +**Local only — skip entirely in agentic mode** (the app is already scaffolded). -### AppKit (Recommended) +`databricks apps manifest` → derive `--features` (union of plugins) + all `--set` → `databricks apps init --run none`. -TypeScript/React framework with type-safe SQL queries and built-in components. +Run `apps init` **from your current working directory** — it creates the app under a new `/`. Do **not** `mkdir`/`cd` into an app directory first and do not re-run init, or the app nests at `//`. After init, confirm `/package.json` exists; if you see a doubled `//`, move the inner app up one level before continuing. -**Official Documentation** — the source of truth for all API details: +Apply manifest scaffolding rules silently; STOP only on `must` vs `never` conflict. -```bash -npx @databricks/appkit docs # ← ALWAYS start here to see available pages -npx @databricks/appkit docs # view a section by name or doc path -npx @databricks/appkit docs --full # full index with all API entries -npx @databricks/appkit docs "appkit-ui API reference" # example: section by name -npx @databricks/appkit docs ./docs/plugins/analytics.md # example: specific doc file -``` +→ [Data Patterns: Scaffolding](references/appkit/data-patterns.md#scaffolding) -**DO NOT guess doc paths.** Run without args first, pick from the index. The `` argument accepts both section names (from the index) and file paths. Docs are the authority on component props, hook signatures, and server APIs — skill files only cover anti-patterns and gotchas. +### 4. Execute checklist slices -**App Manifest and Scaffolding** +Union the slice checklists for each capability. Follow [Lifecycle](references/appkit/lifecycle.md) for phase order (Genie space → Lakebase deploy → typegen → UI). -**Agent workflow for scaffolding: get the manifest first, then build the init command.** +→ [Data Patterns: Checklist slices](references/appkit/data-patterns.md#checklist-slices) -1. **Get the manifest** (JSON schema describing plugins and their resources): - ```bash - databricks apps manifest --profile - # See plugins available in a specific AppKit version: - databricks apps manifest --version --profile - # Custom template: - databricks apps manifest --template --profile - ``` - The output defines: - - **Plugins**: each has a key (plugin ID for `--features`), plus `requiredByTemplate`, and `resources`. - - **requiredByTemplate**: If **true**, that plugin is **mandatory** for this template — do **not** add it to `--features` (it is included automatically); you must still supply all of its required resources via `--set`. If **false** or absent, the plugin is **optional** — add it to `--features` only when the user's prompt indicates they want that capability (e.g. analytics/SQL), and then supply its required resources via `--set`. - - **Resources**: Each plugin has `resources.required` and `resources.optional` (arrays). Each item has `resourceKey` and `fields` (object: field name → description/env). Use `--set ..=` for each required resource field of every plugin you include. +**First action after init (local):** update `tests/smoke.spec.ts` before the first `databricks apps validate`. **Agentic mode has no smoke tests — skip this.** -2. **Scaffold** (DO NOT use `npx`; use the CLI only): - ```bash - databricks apps init --name --features , \ - --set ..= \ - --set ..= \ - --description "" --run none --profile - # --run none: skip auto-run after scaffolding (review code first) - # With custom template: - databricks apps init --template --name --features ... --set ... --profile - ``` - Optionally use `--version ` to target a specific AppKit version. - - **Required**: `--name`, `--profile`. Name: ≤26 chars, lowercase letters/numbers/hyphens only. Use `--features` only for **optional** plugins the user wants (plugins with `requiredByTemplate: false` or absent); mandatory plugins must not be listed in `--features`. - - **Resources**: Pass `--set` for every required resource (each field in `resources.required`) for (1) all plugins with `requiredByTemplate: true`, and (2) any optional plugins you added to `--features`. Add `--set` for `resources.optional` only when the user requests them. - - **Discovery**: Use the parent `databricks-core` skill to resolve IDs (e.g. warehouse: `databricks warehouses list --profile ` or `databricks experimental aitools tools get-default-warehouse --profile `). +### 5. Validate and deploy -**DO NOT guess** plugin names, resource keys, or property names — always derive them from `databricks apps manifest` output. Example: if the manifest shows plugin `analytics` with a required resource `resourceKey: "sql-warehouse"` and `fields: { "id": ... }`, include `--set analytics.sql-warehouse.id=`. +**Local:** validate, then deploy with user consent. First deploy: `bundle deploy` then `apps deploy`. -**Scaffolding Rules Protocol** — `databricks apps manifest` may emit `scaffolding.rules` at the template level (top-level `scaffolding.rules`) and on individual plugins (`plugins[].scaffolding.rules`). Each block has `must` / `should` / `never` arrays of short directive strings. Consume them as follows: +**Agentic mode:** run `databricks apps validate` (no `--profile`, no smoke) as your done-check. **Never deploy** — it's handled externally. -1. **Gather** — for every plugin in your final `--features` list AND every plugin with `requiredByTemplate: true`, read `plugins[].scaffolding.rules`. Union those with the top-level template `scaffolding.rules` into one working set, tagged by source (template vs ``). -2. **Precedence** — manifest rules override the directives baked into this skill. Where the manifest is silent on a topic, this skill's content is the floor. -3. **Phase ordering** — rules whose text begins with `Before init` MUST be executed before `databricks apps init`. Rules beginning with `After init` MUST be executed after init completes (e.g. migrations, typegen, connectivity checks). Rules without a phase prefix apply throughout the scaffold/develop loop. -4. **Conflict detection** — if a plugin `must` rule contradicts a template `never` rule on the same target (or vice versa), STOP and ask the user which to follow before proceeding. Do not silently pick one. Treat `must` vs `never` on the same action as a conflict; `should` is advisory and does not block. -5. **Reporting** — before running `databricks apps init`, surface the merged working set to the user grouped by phase (Before init / After init / Always) and by severity (must / should / never), so the active guardrails are explicit. +→ [Lifecycle](references/appkit/lifecycle.md) -**READ [AppKit Overview](references/appkit/overview.md)** for project structure, workflow, and pre-implementation checklist. +## Generic guidelines -**Genie Agent Workflow** — when the user wants a Genie-powered app, do **not** start by asking for a Genie Space ID. Instead: +- **App name**: ≤26 chars, lowercase letters/numbers/hyphens only. +- **Validation**: `databricks apps validate --profile ` before deploying. +- **Smoke tests**: Update selectors before validate — default template expects "Minimal Databricks App". Use Playwright `getByRole`, `getByText`, `getByLabel` — not `getByLabelText`. See [testing guide](references/testing.md). +- **Smoke test data**: Keep analytics payloads under 1 MB — use `LIMIT` or aggregates. +- **AppKit version**: Do not override `@databricks/appkit` in `package.json`; re-scaffold with `--version` if needed. +- **AppKit API surface**: Before first use of an API shape, run `npx @databricks/appkit docs
` — do not guess signatures. +- **TypeScript**: No `as unknown as ` — use Zod or typed mappers (`appkit lint` enforces this). + +## Deep-dive references + +| Topic | Guide | +|-------|-------| +| Local vs agentic mode | [Environments](references/appkit/environments.md) | +| Capabilities, gates, recipes, scaffolding | [Data Patterns](references/appkit/data-patterns.md) | +| Dev / validate / deploy order | [Lifecycle](references/appkit/lifecycle.md) | +| Project structure, visualizations | [Overview](references/appkit/overview.md) | +| Warehouse SELECT queries | [SQL Queries](references/appkit/sql-queries.md) | +| Custom routes | [Custom Endpoints](references/appkit/custom-endpoints.md) | +| Delta/UC DML | [Warehouse Mutations](references/appkit/warehouse-mutations.md) | +| Lakebase OLTP + synced reads | [Lakebase](references/appkit/lakebase.md) → [OLTP](references/appkit/lakebase-oltp.md) / [Synced Reads](references/appkit/lakebase-synced-reads.md) | +| Genie | [Genie](references/appkit/genie.md) | +| Model Serving | [Model Serving](references/appkit/model-serving.md) | +| Files | [Files](references/appkit/files.md) | +| Jobs from app | [Jobs](references/appkit/jobs.md) | +| UI components | [Frontend](references/appkit/frontend.md) | +| Platform permissions, resources | [Platform Guide](references/platform-guide.md) | +| Non-AppKit (Streamlit, FastAPI, …) | [Other Frameworks](references/other-frameworks.md) | +| Proto-first / multi-plugin contracts | [Proto-First](references/appkit/proto-first.md) (advanced, optional) | + +## AppKit docs (source of truth) -1. Ask which Unity Catalog tables the app should query (fully qualified: `catalog.schema.table`). -2. Ask whether to reuse an existing Genie space or create a new one. -3. If creating: discover the warehouse, then create the space with `databricks genie create-space` (see [Genie Guide](references/appkit/genie.md) for syntax and serialized space format). -4. If reusing: discover existing spaces with `databricks genie list-spaces --profile ` and let the user pick. -5. Scaffold or wire the space ID into the app — derive `--set` keys from `databricks apps manifest`. +```bash +npx @databricks/appkit docs # index — start here +npx @databricks/appkit docs # section name or doc path +``` -Read the [Genie Guide](references/appkit/genie.md) for configuration, SSE endpoints, and frontend integration. +Skill files cover anti-patterns and Databricks-specific workflow; AppKit docs cover API signatures. -### Common Scaffolding Mistakes +## Common scaffolding mistakes ```bash -# ❌ WRONG: name is NOT a positional argument +# ❌ WRONG — name is not positional databricks apps init --features analytics my-app-name -# → "unknown command" error -# ✅ CORRECT: use --name flag -databricks apps init --name my-app-name --features analytics --set "..." --profile +# ✅ CORRECT +databricks apps init --name my-app-name --features analytics \ + --set analytics.sql-warehouse.id= --run none --profile ``` -### Directory Naming - -`databricks apps init` creates directories in kebab-case matching the app name. -App names must be lowercase with hyphens only (≤26 chars). - -### Other Frameworks (Streamlit, FastAPI, Flask, Gradio, Dash, Next.js, etc.) - -Databricks Apps supports any framework that runs as an HTTP server. LLMs already know these frameworks — the challenge is Databricks platform integration. - -**READ [Other Frameworks Guide](references/other-frameworks.md) BEFORE building any non-AppKit app.** It covers port/host configuration, `app.yaml` and `databricks.yml` setup, dependency management, networking, and framework-specific gotchas. - -### Post-Deploy Verification - -After deploying, verify the app is running: - -```bash -databricks apps get --profile -o json # Check app_status.state: RUNNING -databricks apps logs --follow --profile # Stream live logs (Ctrl+C to stop) -``` +## Leave this skill when… -> **Note:** `databricks apps logs` requires OAuth authentication and does not work with PAT. Use `databricks apps get` for status checks if using PAT auth. +Creating Lakebase **projects** / synced tables → **`databricks-lakebase`**. Creating serving **endpoints** → **`databricks-model-serving`**. Authoring Lakeflow **jobs** → **`databricks-jobs`**. Wiring plugins into an app → stay here. diff --git a/skills/databricks-apps/references/appkit/custom-endpoints.md b/skills/databricks-apps/references/appkit/custom-endpoints.md index 5e06ea79..4011aa23 100644 --- a/skills/databricks-apps/references/appkit/custom-endpoints.md +++ b/skills/databricks-apps/references/appkit/custom-endpoints.md @@ -1,14 +1,18 @@ # Custom API Endpoints -**CRITICAL**: Do NOT add custom endpoints for SQL queries or warehouse data retrieval. Use `config/queries/` + `useAnalyticsQuery` instead. +**CRITICAL**: Do NOT add custom endpoints for warehouse **SELECT** queries or read-only data retrieval. Use `config/queries/` + `useAnalyticsQuery` instead. **CRITICAL**: Do NOT add custom endpoints for Unity Catalog file operations. Use the Files plugin instead. When you need server-side logic that no plugin covers, extend the AppKit server in `onPluginsReady` and register Express routes with `appkit.server.extend()`. +**Writes are allowed via custom endpoints**, but **which backend** (Postgres app state / Delta DML / Jobs) is decided in **[Data Patterns: Write path](data-patterns.md#write-path)** — the canonical table. Don't re-decide it here. + +> **Agentic mode:** never run `databricks apps manifest` or pass `--profile`. For check 2 below, read the enabled plugins from `appkit.plugins.json` / `app.yaml` and the `createApp({ plugins: [...] })` array in `server/server.ts` instead. Checks 1 and 3 apply unchanged. See [Environments](environments.md). + Use custom endpoints ONLY for: -- **Mutations**: Creating, updating, or deleting data (INSERT, UPDATE, DELETE) +- **Data mutations** — Express routes in `onPluginsReady` using Lakebase or warehouse DML as above (not `useAnalyticsQuery`) - **External APIs**: Calling Databricks APIs not covered by a dedicated plugin (MLflow, Workspace API, etc.) - **Complex business logic**: Multi-step operations that cannot be expressed in SQL - **File processing**: Uploads, processing, transformations (when not covered by the Files plugin) @@ -44,7 +48,7 @@ databricks apps manifest --profile **Key plugins to check for:** -- **analytics** — provides SQL warehouse query execution (do NOT reimplement with custom endpoints) +- **analytics** — provides SQL warehouse execution: **reads** via `config/queries/` + `useAnalyticsQuery`; **writes** via `appkit.analytics.query()` inside custom mutation routes (see [Warehouse Mutations](warehouse-mutations.md)). Do NOT reimplement SELECT retrieval with custom endpoints. - **lakebase** — provides Lakebase plugin for PostgreSQL CRUD (use plugin in routes, don't create raw connections) - **genie** — provides Genie AI-powered data exploration (check before building custom natural-language-to-SQL routes) - **files** — provides file storage and retrieval helpers (check before writing custom file upload/download routes) @@ -61,13 +65,23 @@ Read `server/server.ts` to see what routes already exist. Add new handlers insid ## Server-side Pattern -Register routes inside `onPluginsReady` so plugins are initialized before the server accepts requests: +Register routes inside `onPluginsReady` so plugins are initialized before the server accepts requests. + +**Include the plugins your routes need** — `server()` alone is not enough for warehouse or Lakebase mutations: + +| Route purpose | Plugins (minimum) | +|---------------|-------------------| +| External Databricks APIs (MLflow, Workspace) | `[server()]` | +| Delta / UC DML | `[server(), analytics({})]` — see [Warehouse Mutations](warehouse-mutations.md) | +| Postgres CRUD | `[server(), lakebase()]` — see [Lakebase OLTP](lakebase-oltp.md) | +| Trigger Lakeflow Jobs | `[server(), jobs()]` — see [Jobs](jobs.md) | + +### Example: external API (no warehouse or Lakebase) ```typescript // server/server.ts import { createApp, server } from "@databricks/appkit"; import { getExecutionContext } from "@databricks/appkit"; -import { z } from "zod"; await createApp({ plugins: [server()], @@ -82,23 +96,16 @@ await createApp({ }); res.json(response); }); - - // Example: Mutation - app.post("/api/records", async (req, res) => { - const parsed = z.object({ name: z.string() }).safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: "Invalid input" }); - return; - } - // Custom logic here - res.status(201).json({ success: true, id: 123 }); - }); }); }, }); ``` -For Lakebase CRUD routes, schema initialization, and chat persistence, see [Lakebase Guide](lakebase.md). +Do **not** copy a placeholder mutation that returns fake IDs — real writes must use `appkit.lakebase.query()` or `appkit.analytics.query()` as in [Lakebase OLTP](lakebase-oltp.md) and [Warehouse Mutations Guide](warehouse-mutations.md). + +For Lakebase CRUD routes, schema initialization, and chat persistence, see [Lakebase OLTP](lakebase-oltp.md). + +For Delta / Unity Catalog writes (`INSERT`, `UPDATE`, `DELETE`, `MERGE`), see [Warehouse Mutations Guide](warehouse-mutations.md). ## Client-side Pattern @@ -119,10 +126,11 @@ function MyComponent() { }, []); const handleCreate = async () => { - await fetch("/api/records", { + // POST to your mutation route — see Lakebase or Warehouse Mutations guides for server handlers + await fetch("/api/books", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ name: "test" }), + body: JSON.stringify({ title: "Example" }), }); }; @@ -130,32 +138,15 @@ function MyComponent() { } ``` -## Decision Tree for Data Operations - -1. **Need to display data from SQL?** - - **Chart or Table?** → Use visualization components (`BarChart`, `LineChart`, `DataTable`, etc.) - - **Custom display (KPIs, cards, lists)?** → Use `useAnalyticsQuery` hook - - **Never** add custom endpoints for SQL SELECT statements against the warehouse - -2. **Need to call a Databricks API?** - - Serving endpoints → use `serving()` plugin (see [Model Serving Guide](model-serving.md)) - - Jobs → use `jobs()` plugin (see [Jobs Guide](jobs.md)) - - MLflow, Workspace API, other APIs → custom endpoint via `onPluginsReady` - -3. **Need to modify data?** → Custom endpoint in `onPluginsReady` - - INSERT, UPDATE, DELETE operations - - Multi-step transactions - - Business logic with side effects +## Route patterns and anti-patterns -4. **Need non-SQL custom logic?** → Custom endpoint in `onPluginsReady` - - File processing - - External API calls - - Complex computations in TypeScript +For **which backend** to use (reads, writes, plugins), see [Data Patterns](data-patterns.md) — do not duplicate that decision tree here. -**Summary:** +**Summary for custom routes:** -- ✅ SQL queries → Visualization components or `useAnalyticsQuery` -- ✅ Databricks APIs without a plugin → custom endpoint via `onPluginsReady` -- ✅ Data mutations → custom endpoint via `onPluginsReady` -- ❌ SQL warehouse queries → custom endpoints (NEVER do this) -- ❌ Files operations → custom endpoints (NEVER do this — use Files plugin) +- ✅ SQL **reads** → `config/queries/` / `useAnalyticsQuery` (not custom endpoints) +- ✅ Postgres mutations → [Lakebase OLTP](lakebase-oltp.md) +- ✅ Delta / UC DML → [Warehouse Mutations](warehouse-mutations.md) +- ✅ Databricks APIs without a plugin → custom endpoint + `onPluginsReady` +- ❌ Warehouse SELECT in custom endpoints +- ❌ Files operations in custom endpoints — use [Files](files.md) plugin diff --git a/skills/databricks-apps/references/appkit/data-patterns.md b/skills/databricks-apps/references/appkit/data-patterns.md new file mode 100644 index 00000000..86ddadf5 --- /dev/null +++ b/skills/databricks-apps/references/appkit/data-patterns.md @@ -0,0 +1,237 @@ +# AppKit Data Patterns & Capabilities + +**Canonical reference** for choosing plugins, running gates, and composing checklists. Plugin guides (`genie.md`, `files.md`, etc.) cover setup only — pattern selection lives here. + +Apps are **compositions of capabilities**, not single archetypes. Derive `--features` as the **union** of capability flags below. + +## Capability catalog + +| Flag | Plugin | Owns | Deep guide | +|------|--------|------|------------| +| `reads_warehouse` | `analytics` | `config/queries/` (SELECT), charts, `useAnalyticsQuery` | [SQL Queries](sql-queries.md) | +| `reads_synced` | `lakebase` | Read-only queries on Lakebase synced tables | [Lakebase Synced Reads](lakebase-synced-reads.md) | +| `writes_oltp` | `lakebase` | Postgres CRUD, schema init, deploy-first | [Lakebase OLTP](lakebase-oltp.md) | +| `writes_delta` | `analytics` | Warehouse DML via custom routes | [Warehouse Mutations](warehouse-mutations.md) | +| `writes_via_job` | `jobs` | Trigger/monitor Lakeflow Jobs | [Jobs](jobs.md) | +| `genie` | `genie` | NL Q&A over UC tables (SSE) | [Genie](genie.md) | +| `files` | `files` | UC Volume upload/download/browse | [Files](files.md) | +| `serving` | `serving` | Model inference / chat endpoints | [Model Serving](model-serving.md) | + +> **These are concepts, not CLI flags.** Never pass a capability name to `--features`. The CLI value is the **Plugin** column. E.g. `{ writes_oltp, reads_warehouse }` → `--features lakebase,analytics` (not `--features writes_oltp,reads_warehouse`). + +**Route mechanics** (all mutation paths): [Custom Endpoints](custom-endpoints.md) — `onPluginsReady` + `appkit.server.extend()`. + +### Infer capabilities from the user request + +Include a flag **only when the user asked for it** (or it is clearly required). Do not add Lakebase or analytics "just in case." + +| User intent | Typical flags | +|-------------|---------------| +| Dashboard, KPIs, charts over UC tables | `reads_warehouse` | +| Form, todos, sessions, app-owned CRUD | `writes_oltp` | +| Save into existing Delta/UC table on submit | `writes_delta` | +| Large batch write, ETL from app action | `writes_via_job` | +| Sub-second lookup on synced lakehouse data | `reads_synced` | +| "Ask questions about my data" chat | `genie` | +| Upload/download files in volumes | `files` | +| LLM / model endpoint in app | `serving` | + +Map flags → plugins → `--features` (comma-separated union). Derive every `--set` from `databricks apps manifest`. + +## Conditional gates + +Run **only** gates whose capability is in the set. Skip the rest. + +> **Agentic mode:** the capability set comes from `appkit.plugins.json` / `app.yaml`, not inference. Run **design + discovery** gates only (`write_path`, `read_path`, `data_discovery`). Skip the **provisioning** gates (`lakebase_resources`, `genie_space`) — those resources already exist. See [Environments](environments.md). + +| Gate | Run when | Action | +|------|----------|--------| +| **write_path** | `writes_oltp`, `writes_delta`, or `writes_via_job` | Use [Write path](#write-path) table; ask if unclear | +| **read_path** | App reads UC/lakehouse data **and** you must choose synced vs warehouse SQL | Use [Read path](#read-path); see skip rules below | +| **genie_space** | `genie` | Tables + create/reuse space — [Genie workflow](#genie-workflow) | +| **lakebase_resources** | `writes_oltp` or `reads_synced` | Three `--set lakebase.postgres.{project,branch,database}` from manifest | +| **data_discovery** | `reads_warehouse` or `writes_delta` | Parent `databricks-core` skill — schema search before SQL | + +**Skip read_path when:** +- App has no UC/lakehouse reads (pure CRUD, serving-only, files-only). +- User already chose warehouse SQL (`reads_warehouse`) **and** does not need synced-table latency. +- App uses `genie` + fixed dashboards — Genie complements analytics; both use the warehouse (different UX). + +**write_path question (if unclear):** *Should this data live in Postgres (app state), in Delta/UC (lakehouse), or run as a background job?* + +## Write path + +When the app **persists or mutates** data: + +| Need | Write path | Read next | +|------|------------|-----------| +| App-owned state (forms, CRUD, sessions) — Postgres is system of record | Custom route + `appkit.lakebase.query()` | [Lakebase OLTP](lakebase-oltp.md) | +| User action updates **existing Delta/UC table now** (small scoped DML) | Custom route + `appkit.analytics.query()` + Zod | [Warehouse Mutations](warehouse-mutations.md) | +| Large / async lakehouse write | Custom route → `jobs()` plugin | [Jobs](jobs.md) | +| Postgres now; curated data in Delta **later** (async OK) | Lakebase OLTP + [Lakehouse Sync](../../../databricks-lakebase/references/lakehouse-sync.md) (UI-only) | **`databricks-lakebase`** skill | +| Read-only dashboard / KPI | No write path | [SQL Queries](sql-queries.md) | + +**Defaults:** App-owned CRUD → `writes_oltp`. Curated lakehouse table the user explicitly named → consider `writes_delta` or `writes_via_job`. + +## Read path + +When the app **reads** Unity Catalog / lakehouse data and synced vs warehouse SQL is not already decided: + +| | **(A) Lakebase synced reads** | **(B) Analytics (warehouse SQL)** | +|--|---|---| +| Speed | Sub-second | Few seconds | +| Best for | Typeahead, ID lookups, operational serving from synced gold | Dashboards, charts, KPIs, SQL filters | +| How | Delta synced into Lakebase Postgres (read-only) | `config/queries/` + warehouse at read time | + +Recommend **(B)** for most dashboards. Recommend **(A)** only for sub-second lookup/typeahead on synced tables. + +**Do not** use synced reads **and** warehouse SQL for the **same dataset** without an explicit reason. + +After choice: (A) → `--features lakebase` (+ analytics if also dashboarding different data). (B) → `--features analytics`. + +## Composition rules + +- **Genie + analytics:** OK — SQL files for fixed KPIs; Genie for NL. Flags: `reads_warehouse`, `genie`. +- **Lakebase OLTP + analytics:** Common hybrid — reads via SQL files; writes via Postgres routes. Flags: `writes_oltp`, `reads_warehouse`. +- **Files / serving / jobs:** Stack freely with any read/write combo. +- **Never write to synced tables** — read-only replicas; see [Lakebase Synced Reads](lakebase-synced-reads.md). +- **Never** warehouse SELECT in custom endpoints — use `config/queries/`. +- **Never** `useAnalyticsQuery` for Lakebase data. + +## Named recipes (examples) + +Recipes illustrate common **capability unions** — not exclusive app types. + +### Analytics dashboard + +`{ reads_warehouse }` → `--features analytics` + +Gates: data_discovery. Slices: analytics only. + +### Lakebase CRUD + +`{ writes_oltp }` → `--features lakebase` + +Gates: write_path, lakebase_resources. Slices: lakebase_oltp. Deploy before local dev. + +### AI analyst + +`{ reads_warehouse, genie }` → `--features analytics,genie` + +Gates: genie_space; skip read_path. Slices: genie → analytics. + +### Ops console (multi-plugin) + +`{ reads_warehouse, writes_oltp, files, genie }` → `--features analytics,lakebase,files,genie` + +Gates: write_path, genie_space, lakebase_resources; skip read_path if user wants warehouse dashboards. + +Slice order: genie space → lakebase scaffold replace → SQL/typegen → files volumes → UI. + +**UI:** Separate surfaces — dashboard | chat | files | CRUD (do not merge into one data hook). + +### Serving chatbot + +`{ serving }` (+ optional `writes_oltp` for chat history) → `--features serving` or `serving,lakebase` + +Use **`databricks-model-serving`** skill to create the endpoint first. + +## Checklist slices + +Union slices for every flag in the capability set. See [Lifecycle](lifecycle.md) for ordering. + +### Slice: `reads_warehouse` + +- [ ] `config/queries/*.sql` (SELECT only — not DML) +- [ ] `npm run typegen` — verify types in `appKitTypes.d.ts` +- [ ] UI with `queryKey` / `useAnalyticsQuery` +- [ ] → [SQL Queries](sql-queries.md), [Frontend](frontend.md) + +### Slice: `reads_synced` + +- [ ] Synced table exists; SP granted SELECT — **`databricks-lakebase`** skill +- [ ] Read-only Express routes — never write to synced tables +- [ ] → [Lakebase Synced Reads](lakebase-synced-reads.md) + +### Slice: `writes_oltp` + +- [ ] Replace scaffold todo boilerplate; use `setupXRoutes(appkit)` — no `AppKitWithLakebase` +- [ ] Schema + CRUD in `onPluginsReady` +- [ ] Frontend uses `fetch('/api/...')` — not `useAnalyticsQuery` +- [ ] Deploy before local dev +- [ ] → [Lakebase OLTP](lakebase-oltp.md) + +### Slice: `writes_delta` + +- [ ] `[server(), analytics({})]` plugins +- [ ] One route per mutation; fixed SQL + Zod — never client SQL +- [ ] → [Warehouse Mutations](warehouse-mutations.md) + +### Slice: `writes_via_job` + +- [ ] Job exists — **`databricks-jobs`** skill to author; app only triggers +- [ ] → [Jobs](jobs.md) + +### Slice: `genie` + +- [ ] Space + tables wired before or during init +- [ ] → [Genie workflow](#genie-workflow), [Genie](genie.md) + +### Slice: `files` / `serving` + +- [ ] Manifest `--set` + volume or endpoint env vars +- [ ] → [Files](files.md) or [Model Serving](model-serving.md) + +### All AppKit apps + +- [ ] Update `tests/smoke.spec.ts` **before first** `databricks apps validate` +- [ ] → [Testing](../testing.md) + +## Genie workflow + +Do **not** start by asking for a Space ID. + +1. Ask which UC tables (`catalog.schema.table`). +2. Ask: reuse existing space or create new? +3. If creating: warehouse + `databricks genie create-space` — see [Genie](genie.md). +4. If reusing: `databricks genie list-spaces`. +5. Scaffold with `--features genie` (+ others) — derive `--set` from manifest. + +## Scaffolding + +1. `databricks apps manifest --profile ` (or `--version`, `--template`). +2. Build `--features` from capability union; add `--set` for every required resource field. +3. **Manifest rules:** Gather rules for selected plugins only. Apply automatically. **STOP and ask user only** if a plugin `must` contradicts a template `never`. Do not dump the full rule list unless there is a conflict. +4. Init: + +```bash +databricks apps init --name --features , \ + --set ..= \ + --description "" --run none --profile +``` + + Run this **from the working directory** — `apps init` creates the app in a new `/` subdirectory. Do **not** create or `cd` into `/` beforehand, and do not run `apps init` more than once, or the app nests at `//`. +5. **Verify layout:** confirm `/package.json` exists directly under the working directory. If init produced a doubled `//`, lift the inner app up one level before continuing (`mv //{.,}* / 2>/dev/null; rmdir /`) — the app must live at `/`, never nested. + +**DO NOT guess** plugin keys or `--set` paths — derive from manifest. + +**Common mistake:** `databricks apps init --features analytics my-app` — name must be `--name my-app`. + +## Leave this skill when… + +| Task | Skill | +|------|-------| +| Create Lakebase **project**, synced table pipeline, SP grants | **`databricks-lakebase`** | +| Create model serving **endpoint** | **`databricks-model-serving`** | +| Author Lakeflow **job** definition | **`databricks-jobs`** | +| Wire endpoints/routes into AppKit app | Stay here | + +## State storage (Lakebase OLTP) + +When `writes_oltp` is in the set: + +1. Use **`databricks-lakebase`** skill to create/obtain project, branch, database. +2. All three `--set lakebase.postgres.{project,branch,database}` from manifest. +3. If also `reads_warehouse` or `reads_synced`, complete [Read path](#read-path) for the lakehouse read side. + +Do **not** add Lakebase to read-only dashboards unless the user requests persistent storage. diff --git a/skills/databricks-apps/references/appkit/environments.md b/skills/databricks-apps/references/appkit/environments.md new file mode 100644 index 00000000..9000bf34 --- /dev/null +++ b/skills/databricks-apps/references/appkit/environments.md @@ -0,0 +1,50 @@ +# Environments: Local vs Agentic Mode + +AppKit apps are built in **two environments**. Detect which one you are in **before doing anything else** — the workflow differs. + +## Detect first + +```bash +echo "${DATABRICKS_APPS_AGENTIC_MODE:-}" +``` + +- `true` → **Agentic mode**. The app has already been initialized and every resource the wired plugins need is provisioned for you. Follow the **Agentic** column below. +- empty / anything else → **Local**. You are on a user machine and must discover, scaffold, and deploy yourself. Follow the **Local** column (the rest of this skill's default guidance). + +## What changes + +| Step | Local | Agentic mode (`DATABRICKS_APPS_AGENTIC_MODE=true`) | +|------|-------|----------------------------------------------------| +| **Auth / profile** | Select a profile via `databricks-core`; pass `--profile` on every CLI call. Never auto-select. | **Ambient — handled by the environment.** Never select a profile; **omit `--profile`** on every CLI call. | +| **Capabilities** | *Infer* from the request, then choose `--features`. | **Pre-wired. Do not infer or choose.** Read the enabled plugins from `appkit.plugins.json` / `app.yaml` (see below). | +| **Scaffold** | `databricks apps manifest` → `databricks apps init --features … --set …`. | **Already done.** Never run `manifest` or `init`. The project already exists on disk. | +| **Resources / `--set`** | Discover IDs (`list-projects`, warehouse id, …) and pass `--set` flags. | **Pre-provisioned.** Targets are injected as env vars (see below). Never create, select, or `--set` resources. | +| **Provisioning gates** | Run `lakebase_resources`, `genie_space` creation, etc. | **Skip.** Resources exist. | +| **Design + discovery gates** | Run `write_path`, `read_path`, `data_discovery`. | **Still run** — architecture and table selection are still your job. | +| **Resource-creation handoffs** | Use `databricks-lakebase` / `databricks-model-serving` / `databricks-jobs` and the Genie create/reuse-space flow to create infra. | **Skip all handoffs.** The space / endpoint / project / job already exist. | +| **Lakebase deploy-first** | OLTP requires deploy-before-dev (SP must own the schema). | **Suppressed** — deploy and schema ownership are handled externally. | +| **Dev / preview** | `npm run dev`. | `npm run dev` — connects to the **live** injected resources. | +| **Smoke tests** | Update `tests/smoke.spec.ts` before validate. | **Removed in agentic mode — do not write or update smoke tests.** | +| **Validate** | `databricks apps validate --profile `. | `databricks apps validate` (no `--profile`; build/typecheck/lint only — no smoke). | +| **Deploy** | `bundle deploy` → `apps deploy` (user consent). | **None.** Deploy is handled externally. Never run deploy commands. | + +## Reading what's wired (agentic mode) + +The app is already scaffolded, so discover its shape from files instead of the CLI: + +- **`appkit.plugins.json`** — which plugins are enabled (your capability set). +- **`app.yaml`** — the injected env vars (resource targets), e.g. `DATABRICKS_GENIE_SPACE_ID`, `DATABRICKS_JOB_*`, `DATABRICKS_SERVING_ENDPOINT_NAME`, `LAKEBASE_ENDPOINT` / `PG*`, `DATABRICKS_VOLUME_*`, and the warehouse id. These env var names align with what the plugins declare. +- `server/server.ts` — the `createApp({ plugins: [...] })` array confirms the same. + +Read these env vars at runtime; **never** hardcode or re-provision the values behind them. + +## If a needed capability is not wired + +If the request requires a plugin that is **not** in `appkit.plugins.json`, **stop and tell the user** the app does not have that capability wired. **Do not** run `apps init`, provision resources, or otherwise try to add it yourself — provisioning is handled externally, not by the agent. + +## Still your job in agentic mode + +- **Data discovery** — which `catalog.schema.table` the analytics SQL should hit (resources existing ≠ knowing the tables). Use `databricks-core` (ambient auth, no `--profile`). +- **All application code** — `config/queries/*.sql`, custom routes, Lakebase schema init in `onPluginsReady`, React UI. +- **Design choices** — write path vs read path, route architecture, composition. +- **Run `npm run dev`** and **`databricks apps validate`** as your done-check. diff --git a/skills/databricks-apps/references/appkit/files.md b/skills/databricks-apps/references/appkit/files.md index 27ca2106..d51a4988 100644 --- a/skills/databricks-apps/references/appkit/files.md +++ b/skills/databricks-apps/references/appkit/files.md @@ -1,17 +1,12 @@ # Files: Unity Catalog Volume Operations -**For full Files plugin API (routes, types, config options)**: run `npx @databricks/appkit docs ./docs/plugins/files.md`. +**Pattern selection:** [Data Patterns](data-patterns.md). This guide covers **`files()` plugin setup only.** -Use the `files()` plugin when your app needs to **browse, upload, download, or manage files** in Databricks Unity Catalog Volumes. For analytics dashboards reading from a SQL warehouse, use `config/queries/` instead. For persistent CRUD storage, use Lakebase. +**For full Files plugin API**: run `npx @databricks/appkit docs ./docs/plugins/files.md`. -## When to Use Files vs Other Patterns +Use the `files()` plugin to **browse, upload, download, or manage files** in Unity Catalog Volumes. -| Pattern | Use Case | Data Source | -| ----------------- | ------------------------------------------- | ------------------------ | -| Analytics | Read-only dashboards, charts, KPIs | Databricks SQL Warehouse | -| Lakebase | CRUD operations, persistent state, forms | PostgreSQL (Lakebase) | -| Files | File uploads, downloads, browsing, previews | Unity Catalog Volumes | -| Files + Analytics | Upload CSVs then query warehouse tables | Volumes + SQL Warehouse | +> **Agentic mode:** the volumes already exist and the `DATABRICKS_VOLUME_*` env vars are injected. **Skip** *Scaffolding*; do not create volumes or set `--set` flags. Just call `files()` (it discovers the volumes from the env vars). See [Environments](environments.md). ## Scaffolding diff --git a/skills/databricks-apps/references/appkit/genie.md b/skills/databricks-apps/references/appkit/genie.md index 21734d1c..b41f4f3c 100644 --- a/skills/databricks-apps/references/appkit/genie.md +++ b/skills/databricks-apps/references/appkit/genie.md @@ -1,16 +1,10 @@ # AppKit Genie Guide -Use Genie when your app needs a **natural language query interface** over Unity Catalog tables. For analytics dashboards, use `config/queries/` instead. For persistent storage, use Lakebase. +**Pattern selection:** [Data Patterns](data-patterns.md). This guide covers **`genie()` plugin setup only.** -## When to Use +Use Genie for a **natural language query interface** over Unity Catalog tables. Fixed KPIs still use `config/queries/` (`reads_warehouse`). App-owned CRUD uses Lakebase (`writes_oltp`). -| Pattern | Use Case | Data Source | -|---------|----------|-------------| -| Analytics | Read-only dashboards, charts, KPIs | SQL Warehouse | -| Lakebase | CRUD operations, persistent state, forms | PostgreSQL (Lakebase) | -| Model Serving | Chat, AI features, model inference | Serving Endpoint | -| Genie | Natural language queries over tables | Genie Space → SQL Warehouse | -| Multiple | Combine plugins as needed | Mix of the above | +> **Agentic mode:** the Genie space already exists and `DATABRICKS_GENIE_SPACE_ID` is injected. **Skip** *Genie Space Creation*, *Scaffolding*, and *Adding Genie to an Existing App* — do not run `genie create-space` / `list-spaces` or ask which tables. Just call `genie()` (it reads the env var) and build the chat UI. See [Environments](environments.md). ## Architecture @@ -116,9 +110,9 @@ env: ```typescript import { createApp, server, analytics, genie } from "@databricks/appkit"; -createApp({ +await createApp({ plugins: [server(), analytics(), genie()], -}).catch(console.error); +}); ``` Preserve existing plugins and add `genie()` to the array. @@ -153,6 +147,8 @@ For the `spaces` map API, `GenieChat alias` prop, and `useGenieChat` hook, see ` This section covers the **deployment-specific patterns** for multi-space Genie apps (databricks.yml, app.yaml, stale conversation cleanup). +> **Agentic mode:** skip the `databricks.yml` / `app.yaml` subsections below — spaces are pre-wired; use the injected `DATABRICKS_GENIE_SPACE_*` env vars, and if a space the user needs is not wired, **stop and tell the user**. The client-side patterns (spaces map, build version stamp, stale-conversation cleanup) still apply. See [Environments](environments.md). + **databricks.yml** — add one variable + resource per space, plus target-level values: ```yaml diff --git a/skills/databricks-apps/references/appkit/jobs.md b/skills/databricks-apps/references/appkit/jobs.md index 48b632ab..f58ac017 100644 --- a/skills/databricks-apps/references/appkit/jobs.md +++ b/skills/databricks-apps/references/appkit/jobs.md @@ -1,10 +1,12 @@ # Jobs: Trigger Lakeflow Jobs from Apps -**For full Jobs plugin API (routes, types, config options)**: run `npx @databricks/appkit docs` → Jobs plugin. +**Pattern selection:** [Data Patterns](data-patterns.md). This guide covers **`jobs()` plugin setup only.** -Use the `jobs()` plugin when your app needs to **trigger or monitor pre-existing Databricks Lakeflow Jobs** (notebooks, Python scripts, SQL, dbt, JARs) and surface their status to users. The jobs themselves still live as regular Lakeflow Jobs in the workspace — the plugin is the typed, resource-scoped accessor that lets app code start runs, poll status, and stream completion events. +**For full Jobs plugin API**: run `npx @databricks/appkit docs` → Jobs plugin. -The plugin is **resource-scoped**: only jobs declared via config or discovered from `DATABRICKS_JOB_*` env vars are accessible. It is not a generic Jobs SDK wrapper — to author or schedule jobs, use the `databricks-jobs` (Lakeflow) skill instead. See [`overview.md`](./overview.md) for the cross-plugin data-pattern selector. +Use the `jobs()` plugin to **trigger or monitor pre-existing Lakeflow Jobs** from the app. To **author** jobs, use the **`databricks-jobs`** skill. + +> **Agentic mode:** the job(s) are already wired and the `DATABRICKS_JOB_*` env vars are injected. **Skip** *Scaffolding* and `--set`; do not author or create jobs. Just call `jobs()` (it discovers jobs from the env vars) and build the trigger/monitor UI. See [Environments](environments.md). ## Scaffolding diff --git a/skills/databricks-apps/references/appkit/lakebase-oltp.md b/skills/databricks-apps/references/appkit/lakebase-oltp.md new file mode 100644 index 00000000..0dc0260e --- /dev/null +++ b/skills/databricks-apps/references/appkit/lakebase-oltp.md @@ -0,0 +1,509 @@ +# Lakebase OLTP (App-owned CRUD) + +**Capability:** `writes_oltp` — Postgres as system of record for forms, CRUD, sessions, app state. + +**Pattern selection:** [Data Patterns](data-patterns.md). **Synced lakehouse reads** (read-only Delta replicas) are a different pattern — [Lakebase Synced Reads](lakebase-synced-reads.md). + +For warehouse dashboards, use `config/queries/` (`reads_warehouse`). Never use `useAnalyticsQuery` for Lakebase data. + +> **Agentic mode:** the Lakebase project/branch/database already exist and `LAKEBASE_ENDPOINT` / `PG*` are injected. **Skip** the *Scaffolding*, *Adding Lakebase to an Existing App*, and `server/.env` sections, and the **deploy-first** rule — deploy and schema ownership are handled externally, and `npm run dev` hits the live database. Just write the schema init + CRUD routes in `onPluginsReady`. See [Environments](environments.md). + +## Scaffolding + +**Scaffolding is the fastest way to get started.** If you already have an app, see *Adding Lakebase to an Existing App* below. + +**Always derive `--set` keys from the manifest** — do not guess field names: + +```bash +databricks apps manifest --profile +``` + +For the `lakebase` plugin, the required `postgres` resource has **three** user-supplied fields: `project`, `branch`, and `database`. All three must appear as `--set` flags — omitting `lakebase.postgres.project` causes init to fail. + +Discover values (after creating a project via the `databricks-lakebase` skill): + +```bash +databricks postgres list-projects --profile # → lakebase.postgres.project +databricks postgres list-branches projects/ --profile # → lakebase.postgres.branch +databricks postgres list-databases projects//branches/ --profile # → lakebase.postgres.database +``` + +Use the `.name` field from each list command as the `--set` value. + +**Lakebase only** (no analytics SQL warehouse): +```bash +databricks apps init --name --features lakebase \ + --set "lakebase.postgres.project=" \ + --set "lakebase.postgres.branch=" \ + --set "lakebase.postgres.database=" \ + --run none --profile +``` + +**Both Lakebase and analytics**: +```bash +databricks apps init --name --features analytics,lakebase \ + --set "analytics.sql-warehouse.id=" \ + --set "lakebase.postgres.project=" \ + --set "lakebase.postgres.branch=" \ + --set "lakebase.postgres.database=" \ + --run none --profile +``` + +Where ``, ``, and `` are full resource paths (e.g. `projects/`, `projects//branches/`, `projects//branches//databases/`). + +> For multi-environment deployments (dev/prod), use `variables:` and `targets:` blocks in `databricks.yml` — see the **`databricks-dabs`** skill for patterns. + +**Naming conventions:** Use domain names for user-facing code (`ItemsPage.tsx`, `/api/items`, `item-routes.ts`). Keep `lakebase` naming only for infrastructure config (`lakebase()` plugin, `LAKEBASE_ENDPOINT`, `postgres` app resource). + +**Get resource names** (if you have an existing project): +```bash +# List projects → use the name field +databricks postgres list-projects --profile +# List branches → use the name field of a READY branch +databricks postgres list-branches projects/ --profile +# List databases → use the name field +databricks postgres list-databases projects//branches/ --profile +``` + +## Adding Lakebase to an Existing App + +**`databricks.yml`** — add Lakebase variables and resource: + +```yaml +variables: + lakebase_branch: + description: Lakebase branch resource name + lakebase_database: + description: Lakebase database resource name + +resources: + apps: + app: + resources: + # ... existing resources ... + - name: postgres + postgres: + branch: ${var.lakebase_branch} + database: ${var.lakebase_database} + +targets: + default: + variables: + lakebase_branch: projects//branches/ + lakebase_database: projects//branches//databases/ +``` + +Use the `databricks-lakebase` skill to create a Lakebase project and discover branch/database resource names. + +For per-user connections (OBO/RLS), also add `postgres` to `user_api_scopes` — see `npx @databricks/appkit docs ./docs/plugins/lakebase.md` for OBO setup. + +**`app.yaml`** — add env injection: + +```yaml +env: + # ... existing env vars ... + - name: LAKEBASE_ENDPOINT + valueFrom: postgres +``` + +Other Lakebase env vars (`PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, `PGSSLMODE`) are auto-injected by the platform when the `postgres` resource is configured. Only `LAKEBASE_ENDPOINT` must be set explicitly. + +**`server/server.ts`** — register the plugin: + +```typescript +import { createApp, server, analytics, lakebase } from "@databricks/appkit"; + +await createApp({ + plugins: [server(), analytics(), lakebase()], +}); +``` + +Preserve existing plugins and add `lakebase()` to the array. + +**`server/.env`** — for local development: + +```dotenv +PGHOST= +PGPORT=5432 +PGDATABASE= +PGSSLMODE=require +LAKEBASE_ENDPOINT=projects//branches//endpoints/ +``` + +Get connection details from `databricks postgres get-endpoint`. See *Local Development* below for the full workflow. + +Deploy the app before local development — see *Local Development > Prerequisites* below. Update smoke tests if headings or routes changed, then `databricks apps validate`. + +## Project Structure (after `databricks apps init --features lakebase`) + +``` +my-app/ +├── server/ +│ ├── server.ts # Entry — calls setup from onPluginsReady +│ └── routes/lakebase/ +│ └── todo-routes.ts # ⚠️ Starter CRUD — replace for your domain +├── client/src/ +│ ├── App.tsx # Layout + nav (may include /lakebase route) +│ └── pages/lakebase/ +│ └── LakebasePage.tsx # ⚠️ Starter todo UI — replace for your domain +├── tests/smoke.spec.ts # ⚠️ Asserts "Todo List" — update for your UI +├── databricks.yml # Lakebase postgres resource wiring +├── app.yaml # Manifest with database resource declaration +└── package.json # Includes @databricks/lakebase dependency +``` + +Note: **No `config/queries/` directory** — Lakebase apps use server-side `appkit.lakebase.query()` calls, not SQL files. + +## What the scaffold gives you (replace, don't keep) + +`databricks apps init --features lakebase` generates a **working todo list demo**, not your final app. Treat every file below as **starter code to replace** once you know your domain (reading log, inventory, registrations, etc.). + +| Scaffold artifact | What it is | What to do | +|-------------------|------------|------------| +| `server/routes/lakebase/todo-routes.ts` | Sample CRUD for `app.todos` + hand-written `AppKitWithLakebase` | Rename/replace (e.g. `book-routes.ts`), change table/schema/API paths to your domain | +| `client/src/pages/lakebase/LakebasePage.tsx` | Todo list UI wired to `/api/lakebase/todos` | Replace with your page (e.g. `ReadingLogPage.tsx`) and your API paths | +| `client/src/App.tsx` | Generic home page + nav link to `/lakebase` | Simplify or rewire — many CRUD apps use the domain page as `/` | +| `tests/smoke.spec.ts` | Expects headings like "Todo List" and todo-specific selectors | Update **every** assertion to match your UI before `databricks apps validate` | +| `AppKitWithLakebase` interface | Local typing workaround in `*-routes.ts` | **Not official AppKit API** — see *Lakebase route modules — typing* below | + +**Agent checklist after init:** + +1. **Decide your domain** (table name, API prefix, page title) — use domain names in routes and UI (`/api/books`, `BooksPage.tsx`), not "todo" or generic "lakebase" labels in user-facing code. +2. **Replace backend** — schema (`app.`), Zod validators, Express routes under a domain path (e.g. `/api/books`, not `/api/lakebase/todos`). +3. **Replace frontend** — form, list, filters; point `fetch()` at your new API routes. +4. **Update smoke tests** — assert your headings, buttons, and stable empty-state copy. Avoid asserting dynamic Lakebase content (e.g. "Your shelf is empty") if validate runs without a live database. +5. **Remove dead scaffold files** — delete `todo-routes.ts` / `LakebasePage.tsx` after replacement so agents don't copy leftover patterns. + +Do **not** ship a custom app that still exposes "Todo List" in the UI or `/api/lakebase/todos` unless the user explicitly asked for a todo app. + +## Lakebase route modules — typing + +Inside `server.ts`, `onPluginsReady(appkit)` is already fully typed via AppKit's internal `PluginMap` — no manual interface needed: + +```typescript +await createApp({ + plugins: [lakebase(), server()], + async onPluginsReady(appkit) { + // appkit.lakebase.query(...) and appkit.server.extend(...) are typed here + await setupBookRoutes(appkit); + }, +}); +``` + +The scaffold's `AppKitWithLakebase` in `todo-routes.ts` is **not** exported by `@databricks/appkit`. It exists because route setup was split into a separate file and `PluginMap` is not part of the public package exports. Agents often copy it and treat it as required AppKit surface area — **don't**. + +### Recommended patterns + +**Option A — Inline in `server.ts` (simplest)** + +Register schema init and routes directly in `onPluginsReady`. No shared type, no duplicate interface. + +**Option B — Extract routes with generic inference (recommended for larger apps)** + +Pass `appkit` from `onPluginsReady` into a setup function. TypeScript infers `T` from the call site — no `appkit-types.ts`, no duplicated plugin tuple, no hand-trimmed interface: + +```typescript +// server/routes/lakebase/book-routes.ts +import type { Application } from "express"; + +export async function setupBookRoutes< + T extends { + lakebase: { + query(text: string, params?: unknown[]): Promise<{ rows: Record[] }>; + }; + server: { extend(fn: (app: Application) => void): void }; + }, +>(appkit: T) { + await appkit.lakebase.query("CREATE SCHEMA IF NOT EXISTS app"); + appkit.server.extend((app) => { /* ... */ }); +} +``` + +```typescript +// server/server.ts — plugins inline; T inferred at the call site +await createApp({ + plugins: [lakebase(), server()], + async onPluginsReady(appkit) { + await setupBookRoutes(appkit); // appkit is fully typed here + }, +}); +``` + +Keep `plugins: [lakebase(), server()]` inline in `server.ts` (same array as production). If AppKit later exports `PluginMap` / `AppKitHandle`, prefer that over widening the generic constraint. + +**Option C — Minimal local interface (fallback only)** + +If generics are not viable, a narrow app-local type is acceptable — rename it (e.g. `RouteSetupContext`) and comment that it is **not** AppKit API. Do **not** add a separate `appkit-types.ts` that re-declares the plugin list. + +### Anti-patterns + +```typescript +// ❌ Copy scaffold's AppKitWithLakebase — looks like official API +interface AppKitWithLakebase { /* ... */ } + +// ❌ appkit-types.ts + exported appPlugins tuple — duplicates server.ts, drifts when plugins change +export const appPlugins = [lakebase(), server()] as const; + +// ❌ Double-assert to satisfy a local interface — appkit lint forbids this +setupBookRoutes(appkit as unknown as AppKitWithLakebase); +``` + +Prefer Option A or B. When replacing scaffold routes, **rename** any local interface if you keep Option C — do not leave `AppKitWithLakebase` in a non-todo app. + +## Lakebase Plugin API + +Scaffolding with `--features lakebase` (see above) generates this pattern. Access Lakebase through the plugin handle returned by `createApp()`: + +```typescript +import { createApp, lakebase } from "@databricks/appkit"; + +const appkit = await createApp({ + plugins: [lakebase()], +}); + +// Query via the plugin handle — handles pooling and token refresh automatically +const result = await appkit.lakebase.query("SELECT * FROM users WHERE id = $1", [userId]); +``` + +The `lakebase()` plugin auto-configures from platform-injected env vars at deploy time. No manual pool setup needed. + +## Environment Variables (auto-set when deployed with database resource) + +| Variable | Description | +|----------|-------------| +| `PGHOST` | Lakebase hostname | +| `PGPORT` | Port (default 5432) | +| `PGDATABASE` | Database name | +| `PGUSER` | Service principal client ID | +| `PGSSLMODE` | SSL mode (`require`) | +| `LAKEBASE_ENDPOINT` | Endpoint resource path | + +## CRUD Routes Pattern + +Always use server-side routes for Lakebase operations — do NOT call `appkit.lakebase.query()` from the client. Use `onPluginsReady` to initialize the schema and register Express routes: + +```typescript +// server/server.ts +import { createApp, server, lakebase } from "@databricks/appkit"; +import { z } from 'zod'; + +await createApp({ + plugins: [server(), lakebase()], + async onPluginsReady(appkit) { + // Schema init (runs once before server accepts requests) + await appkit.lakebase.query(` + CREATE SCHEMA IF NOT EXISTS app_data; + CREATE TABLE IF NOT EXISTS app_data.items ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() + ); + `); + + // CRUD routes via Express + appkit.server.extend((app) => { + app.get('/api/items', async (_req, res) => { + const { rows } = await appkit.lakebase.query( + "SELECT * FROM app_data.items ORDER BY created_at DESC LIMIT 100" + ); + res.json(rows); + }); + + app.post('/api/items', async (req, res) => { + const parsed = z.object({ name: z.string().min(1) }).safeParse(req.body); + if (!parsed.success) { res.status(400).json({ error: 'Invalid input' }); return; } + const { rows } = await appkit.lakebase.query( + "INSERT INTO app_data.items (name) VALUES ($1) RETURNING *", + [parsed.data.name] + ); + res.status(201).json(rows[0]); + }); + + app.delete('/api/items/:id', async (req, res) => { + const id = parseInt(req.params.id, 10); + if (isNaN(id)) { res.status(400).json({ error: 'Invalid id' }); return; } + await appkit.lakebase.query("DELETE FROM app_data.items WHERE id = $1", [id]); + res.status(204).send(); + }); + }); + }, +}); +``` + +> **Deploy first!** The Service Principal must create and own the schema before local development. See [Lifecycle: First deploy](lifecycle.md#first-deploy) and **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps**. + +## Schema Initialization + +**Always create a custom schema** — the Service Principal cannot access any existing schemas (including `public`). It must create the schema itself to become its owner. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for the full permission model and deploy-first workflow. Initialize tables inside the `onPluginsReady` callback before registering routes (see CRUD pattern above): + +```typescript +// Inside onPluginsReady — runs once at startup before handling requests +await appkit.lakebase.query(` + CREATE SCHEMA IF NOT EXISTS app_data; + CREATE TABLE IF NOT EXISTS app_data.items ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() + ); +`); +``` + +## ORM Integration (Optional) + +The plugin exposes the raw `pg.Pool` via `appkit.lakebase.pool` — works with any PostgreSQL library: + +```typescript +// Drizzle ORM +import { drizzle } from "drizzle-orm/node-postgres"; +const db = drizzle(appkit.lakebase.pool); + +// Prisma (with @prisma/adapter-pg) +import { PrismaPg } from "@prisma/adapter-pg"; +const adapter = new PrismaPg(appkit.lakebase.pool); +const prisma = new PrismaClient({ adapter }); +``` + +For ORM-compatible config: `appkit.lakebase.getOrmConfig()`. + +## Chat Persistence Pattern + +Save AI chat conversations to Lakebase so users can resume sessions and scroll full message history. + +**Schema** — create in a separate `chat` schema (not `app`) so the deploy-first ownership model stays clean: + +```sql +CREATE SCHEMA IF NOT EXISTS chat; + +CREATE TABLE IF NOT EXISTS chat.chats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id TEXT NOT NULL, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat.messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chat.chats(id) ON DELETE CASCADE, + role TEXT NOT NULL CHECK (role IN ('system', 'user', 'assistant', 'tool')), + content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at + ON chat.messages(chat_id, created_at); +``` + +**Bootstrap** — run setup in `onPluginsReady` so tables exist before the server accepts requests: + +```typescript +await createApp({ + plugins: [server(), lakebase()], + async onPluginsReady(appkit) { + await setupChatTables(appkit); + // then register routes via appkit.server.extend(...) + }, +}); +``` + +**Persistence helpers** — use parameterized queries: + +```typescript +export async function createChat(appkit, input: { userId: string; title: string }) { + const result = await appkit.lakebase.query( + `INSERT INTO chat.chats (user_id, title) VALUES ($1, $2) + RETURNING id, user_id, title, created_at, updated_at`, + [input.userId, input.title], + ); + return result.rows[0]; +} + +export async function appendMessage(appkit, input: { chatId: string; role: string; content: string }) { + const result = await appkit.lakebase.query( + `INSERT INTO chat.messages (chat_id, role, content) VALUES ($1, $2, $3) + RETURNING id, chat_id, role, content, created_at`, + [input.chatId, input.role, input.content], + ); + return result.rows[0]; +} +``` + +**User identity**: In deployed apps, use `req.header("x-forwarded-email")` (injected by the Databricks Apps platform proxy; for off-platform deployments, use your own auth middleware). For local dev, hardcode a test user ID. + +**History endpoints**: +- `GET /api/chats` — list chats for current user +- `GET /api/chats/:chatId/messages` — load ordered history +- `DELETE /api/chats/:chatId` — delete chat (messages cascade) + +**AI SDK v6 integration**: Use `setMessages()` from `useChat` return value for history loading (NOT `initialMessages`). To read response headers like `X-Chat-Id`, pass a custom `fetch` wrapper on the `TextStreamChatTransport` constructor. + +## Key Differences from Analytics Pattern + +| | Analytics | Lakebase | +|--|-----------|---------| +| SQL dialect | Databricks SQL (Spark SQL) | Standard PostgreSQL | +| Query location | `config/queries/*.sql` files | `appkit.lakebase.query()` in Express routes | +| Data retrieval | `useAnalyticsQuery` hook | Express route via `server.extend()` | +| Date functions | `CURRENT_TIMESTAMP()`, `DATEDIFF(DAY, ...)` | `NOW()`, `AGE(...)` | +| Auto-increment | N/A | `SERIAL` or `GENERATED ALWAYS AS IDENTITY` | +| Insert pattern | N/A | `INSERT ... VALUES ($1) RETURNING *` | +| Params | Named (`:param`) | Positional (`$1, $2, ...`) | + +**NEVER use `useAnalyticsQuery` for Lakebase data** — it queries the SQL warehouse, not Lakebase. +**NEVER put Lakebase SQL in `config/queries/`** — those files are only for warehouse queries. + +## Local Development + +### Prerequisites (MUST verify before local development) + +**This applies when your Databricks App uses Lakebase.** Run this check before any local development: + +```bash +databricks apps get --profile +``` + +Check the response for the `active_deployment` field. If it exists with `status.state` of `SUCCEEDED`, the app has been deployed. If `active_deployment` is missing, the app has never been deployed: +1. **STOP** — do not proceed with local development +2. **First deploy** (app not in workspace): `databricks bundle deploy -t --profile `, then `databricks apps deploy -t --profile ` +3. Wait for deployment to complete, then continue + +For apps that already have `active_deployment`, use `databricks apps deploy` only. See [Lifecycle](lifecycle.md). + +If you skip this step, the Service Principal won't own the database schema. You'll create schemas under your credentials that the SP **cannot access** after deployment. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for the full workflow and recovery steps. + +Lakebase project creators already have database access after the first deploy. Collaborators need `databricks_superuser` granted by the project creator via Branch Overview. + +> **Project-owner note:** If you are the Lakebase project owner, `databricks_create_role` may fail with "role already exists" and `GRANT databricks_superuser` may fail with "permission denied to grant role" — both errors are safe to ignore; the project owner already has the necessary access. + +The Lakebase env vars (`PGHOST`, `PGDATABASE`, etc.) are auto-set only when deployed. For local development, get the connection details from your endpoint and set them manually: + +```bash +# Get endpoint connection details +databricks postgres get-endpoint \ + projects//branches//endpoints/ \ + --profile +``` + +Then create `server/.env` with the values from the endpoint response: + +``` +PGHOST= +PGPORT=5432 +PGDATABASE= +PGUSER= +PGSSLMODE=require +LAKEBASE_ENDPOINT=projects//branches//endpoints/ +``` + +Load `server/.env` in your dev server (e.g. via `dotenv` or `node --env-file=server/.env`). Never commit `.env` files — add `server/.env` to `.gitignore`. + +## Troubleshooting + +| Error | Cause | Solution | +|-------|-------|---------| +| `permission denied for schema public` | SP cannot access `public` schema | Create custom schema: `CREATE SCHEMA IF NOT EXISTS app_data` and qualify all table names with `app_data.` | +| `permission denied for schema ` | Schema was created by another role (e.g. you ran locally before deploying) | Schema owned by wrong role. To preserve data: export first (`pg_dump` or temp schema copy). **Ask the user before dropping.** Then drop + redeploy. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for full steps. | +| Works locally but `permission denied` after deploy | Local credentials created the schema; the SP cannot access schemas it does not own | Schema owned by wrong role — see row above for export + drop + redeploy steps | +| `connection refused` | Pool not connected or wrong env vars | Check `PGHOST`, `PGPORT`, `LAKEBASE_ENDPOINT` are set | +| `relation "X" does not exist` | Tables not initialized | Run `CREATE TABLE IF NOT EXISTS` at startup | +| App builds but pool fails at runtime | Env vars not set locally | Set vars in `server/.env` — see Local Development above | diff --git a/skills/databricks-apps/references/appkit/lakebase-synced-reads.md b/skills/databricks-apps/references/appkit/lakebase-synced-reads.md new file mode 100644 index 00000000..36684894 --- /dev/null +++ b/skills/databricks-apps/references/appkit/lakebase-synced-reads.md @@ -0,0 +1,63 @@ +# Lakebase Synced Reads (Read-only) + +**Capability:** `reads_synced` — low-latency reads of Unity Catalog / Delta data materialized into Lakebase Postgres. + +**Not OLTP CRUD.** App-owned writes use [Lakebase OLTP](lakebase-oltp.md) (`writes_oltp`). Warehouse dashboards use [SQL Queries](sql-queries.md) (`reads_warehouse`). + +**Pattern selection:** [Data Patterns](data-patterns.md). Create synced tables and grant SP access via the **`databricks-lakebase`** skill — [synced-tables.md](../../../databricks-lakebase/references/synced-tables.md). + +> **Agentic mode:** the synced tables, SP grants, and `PG*` env vars already exist / are injected. **Skip** *Scaffolding* and all `databricks-lakebase` handoffs — do not create synced tables or grant access. Just write the read-only routes. See [Environments](environments.md). + +## Architecture + +``` +Delta gold tables → Synced tables (read-only) → App reads via appkit.lakebase.query() +App writes → Lakebase OLTP tables → optional Lakehouse Sync → Delta +``` + +Use synced reads when data is curated in Delta, changes relatively slowly, and must be served at OLTP latency — lookups, catalogs, operational consoles on gold tables. + +> **Security:** Synced tables do not propagate Unity Catalog fine-grained access control (row/column masks). If UC FGAC is critical, use warehouse SQL with user authorization instead. + +## Scaffolding + +Same plugin and `--set` flags as OLTP — `--features lakebase` + three `lakebase.postgres.{project,branch,database}` values from manifest. **No deploy-first for schema init** — synced tables already exist after the sync pipeline runs. + +Hybrid apps often combine `reads_synced` + `writes_oltp` + `reads_warehouse` — use separate tables/routes for each; never write to synced tables. + +## How it works + +Synced tables (via `databricks postgres create-synced-table`) appear as regular Postgres tables. Use `appkit.lakebase.query()` in Express routes — **read-only**. + +| | OLTP CRUD tables | Synced tables | +|--|------------------|---------------| +| Created by | App SP (`CREATE TABLE`) | Sync pipeline | +| Owned by | SP role | System role (`databricks_writer_*`) | +| Operations | Read + Write | **Read-only** | +| Schema init | App in `onPluginsReady` | Exists after sync | +| Deploy-first | **Yes** (SP must own schema) | No | + +**Permission grant:** App SP has `CAN_CONNECT_AND_CREATE` but not `pg_read_all_data`. Project owner must grant SELECT on synced tables — see **`databricks-lakebase`** skill (Grant app SP access to synced tables). + +## Example route + +```typescript +// Inside onPluginsReady → appkit.server.extend((app) => { ... }) +app.get("/api/top-pickups", async (_req, res) => { + const { rows } = await appkit.lakebase.query(` + SELECT pickup_zip, COUNT(*) AS trip_count, AVG(fare_amount) AS avg_fare + FROM public.nyc_trips + GROUP BY pickup_zip + ORDER BY trip_count DESC + LIMIT 10 + `); + res.json(rows); +}); +``` + +## Rules + +- **Never write to synced tables** — corrupts sync state. +- **Never** put synced-table SELECT in `config/queries/` — those files are warehouse-only. +- **Never** `useAnalyticsQuery` for Lakebase data. +- Mixed patterns: read synced tables; write to separate app-owned OLTP tables — see [Lakebase OLTP](lakebase-oltp.md). diff --git a/skills/databricks-apps/references/appkit/lakebase.md b/skills/databricks-apps/references/appkit/lakebase.md index 1f18c45a..1009ebd0 100644 --- a/skills/databricks-apps/references/appkit/lakebase.md +++ b/skills/databricks-apps/references/appkit/lakebase.md @@ -1,438 +1,12 @@ -# Lakebase: OLTP Database for Apps +# Lakebase in AppKit -Use Lakebase when your app needs **persistent read/write storage** — forms, CRUD operations, user-generated data. For analytics dashboards reading from a SQL warehouse, use `config/queries/` instead. +Same `--features lakebase` plugin — **two different patterns**. Do not conflate them. -## When to Use Lakebase vs Analytics +| Pattern | Capability | Guide | +|---------|------------|-------| +| **OLTP CRUD** — app-owned Postgres (forms, sessions, todos) | `writes_oltp` | [Lakebase OLTP](lakebase-oltp.md) | +| **Synced reads** — read-only Delta replicas in Postgres | `reads_synced` | [Lakebase Synced Reads](lakebase-synced-reads.md) | -| Pattern | Use Case | Data Source | -|---------|----------|-------------| -| Analytics | Read-only dashboards, charts, KPIs | Databricks SQL Warehouse | -| Lakebase | CRUD operations, persistent state, forms, low-latency reads of synced lakehouse data | PostgreSQL (Lakebase Autoscaling) | -| Both | Dashboard with user preferences/saved state | Warehouse + Lakebase | +**Pattern selection and gates:** [Data Patterns](data-patterns.md). -> **Serving lakehouse data to apps?** If your app needs low-latency reads of Delta/UC tables (entity lookups, product catalogs, feature serving), use **Lakebase synced tables** to materialize them into Lakebase instead of querying a SQL warehouse (which takes seconds to minutes). See *Reading from Synced Tables* below. - -## Scaffolding - -**Scaffolding is the fastest way to get started.** If you already have an app, see *Adding Lakebase to an Existing App* below. - -**Lakebase only** (no analytics SQL warehouse): -```bash -databricks apps init --name --features lakebase \ - --set "lakebase.postgres.branch=" \ - --set "lakebase.postgres.database=" \ - --run none --profile -``` - -**Both Lakebase and analytics**: -```bash -databricks apps init --name --features analytics,lakebase \ - --set "analytics.sql-warehouse.id=" \ - --set "lakebase.postgres.branch=" \ - --set "lakebase.postgres.database=" \ - --run none --profile -``` - -Where `` and `` are full resource names (e.g. `projects//branches/` and `projects//branches//databases/`). - -Use the `databricks-lakebase` skill to create a Lakebase project and discover branch/database resource names before running this command. - -> For multi-environment deployments (dev/prod), use `variables:` and `targets:` blocks in `databricks.yml` — see the **`databricks-dabs`** skill for patterns. - -**Naming conventions:** Use domain names for user-facing code (`ItemsPage.tsx`, `/api/items`, `item-routes.ts`). Keep `lakebase` naming only for infrastructure config (`lakebase()` plugin, `LAKEBASE_ENDPOINT`, `postgres` app resource). - -**Get resource names** (if you have an existing project): -```bash -# List branches → use the name field of a READY branch -databricks postgres list-branches projects/ --profile -# List databases → use the name field -databricks postgres list-databases projects//branches/ --profile -``` - -## Adding Lakebase to an Existing App - -**`databricks.yml`** — add Lakebase variables and resource: - -```yaml -variables: - lakebase_branch: - description: Lakebase branch resource name - lakebase_database: - description: Lakebase database resource name - -resources: - apps: - app: - resources: - # ... existing resources ... - - name: postgres - postgres: - branch: ${var.lakebase_branch} - database: ${var.lakebase_database} - -targets: - default: - variables: - lakebase_branch: projects//branches/ - lakebase_database: projects//branches//databases/ -``` - -Use the `databricks-lakebase` skill to create a Lakebase project and discover branch/database resource names. - -For per-user connections (OBO/RLS), also add `postgres` to `user_api_scopes` — see `npx @databricks/appkit docs ./docs/plugins/lakebase.md` for OBO setup. - -**`app.yaml`** — add env injection: - -```yaml -env: - # ... existing env vars ... - - name: LAKEBASE_ENDPOINT - valueFrom: postgres -``` - -Other Lakebase env vars (`PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, `PGSSLMODE`) are auto-injected by the platform when the `postgres` resource is configured. Only `LAKEBASE_ENDPOINT` must be set explicitly. - -**`server/server.ts`** — register the plugin: - -```typescript -import { createApp, server, analytics, lakebase } from "@databricks/appkit"; - -createApp({ - plugins: [server(), analytics(), lakebase()], -}).catch(console.error); -``` - -Preserve existing plugins and add `lakebase()` to the array. - -**`server/.env`** — for local development: - -```dotenv -PGHOST= -PGPORT=5432 -PGDATABASE= -PGSSLMODE=require -LAKEBASE_ENDPOINT=projects//branches//endpoints/ -``` - -Get connection details from `databricks postgres get-endpoint`. See *Local Development* below for the full workflow. - -Deploy the app before local development — see *Local Development > Prerequisites* below. Update smoke tests if headings or routes changed, then `databricks apps validate`. - -## Project Structure (after `databricks apps init --features lakebase`) - -``` -my-app/ -├── server/ -│ └── server.ts # Backend with Lakebase plugin + Express routes -├── client/ -│ └── src/ -│ └── App.tsx # React frontend -├── app.yaml # Manifest with database resource declaration -└── package.json # Includes @databricks/lakebase dependency -``` - -Note: **No `config/queries/` directory** — Lakebase apps use server-side `appkit.lakebase.query()` calls, not SQL files. - -## Lakebase Plugin API - -Scaffolding with `--features lakebase` (see above) generates this pattern. Access Lakebase through the plugin handle returned by `createApp()`: - -```typescript -import { createApp, lakebase } from "@databricks/appkit"; - -const appkit = await createApp({ - plugins: [lakebase()], -}); - -// Query via the plugin handle — handles pooling and token refresh automatically -const result = await appkit.lakebase.query("SELECT * FROM users WHERE id = $1", [userId]); -``` - -The `lakebase()` plugin auto-configures from platform-injected env vars at deploy time. No manual pool setup needed. - -## Environment Variables (auto-set when deployed with database resource) - -| Variable | Description | -|----------|-------------| -| `PGHOST` | Lakebase hostname | -| `PGPORT` | Port (default 5432) | -| `PGDATABASE` | Database name | -| `PGUSER` | Service principal client ID | -| `PGSSLMODE` | SSL mode (`require`) | -| `LAKEBASE_ENDPOINT` | Endpoint resource path | - -## CRUD Routes Pattern - -Always use server-side routes for Lakebase operations — do NOT call `appkit.lakebase.query()` from the client. Use `onPluginsReady` to initialize the schema and register Express routes: - -```typescript -// server/server.ts -import { createApp, server, lakebase } from "@databricks/appkit"; -import { z } from 'zod'; - -await createApp({ - plugins: [server(), lakebase()], - async onPluginsReady(appkit) { - // Schema init (runs once before server accepts requests) - await appkit.lakebase.query(` - CREATE SCHEMA IF NOT EXISTS app_data; - CREATE TABLE IF NOT EXISTS app_data.items ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW() - ); - `); - - // CRUD routes via Express - appkit.server.extend((app) => { - app.get('/api/items', async (_req, res) => { - const { rows } = await appkit.lakebase.query( - "SELECT * FROM app_data.items ORDER BY created_at DESC LIMIT 100" - ); - res.json(rows); - }); - - app.post('/api/items', async (req, res) => { - const parsed = z.object({ name: z.string().min(1) }).safeParse(req.body); - if (!parsed.success) { res.status(400).json({ error: 'Invalid input' }); return; } - const { rows } = await appkit.lakebase.query( - "INSERT INTO app_data.items (name) VALUES ($1) RETURNING *", - [parsed.data.name] - ); - res.status(201).json(rows[0]); - }); - - app.delete('/api/items/:id', async (req, res) => { - const id = parseInt(req.params.id, 10); - if (isNaN(id)) { res.status(400).json({ error: 'Invalid id' }); return; } - await appkit.lakebase.query("DELETE FROM app_data.items WHERE id = $1", [id]); - res.status(204).send(); - }); - }); - }, -}); -``` - -> **Deploy first (App + Lakebase only)!** When your Databricks App uses Lakebase, the Service Principal must create and own the schema. Run `databricks apps deploy` before any local development. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for details. - -## Schema Initialization - -**Always create a custom schema** — the Service Principal cannot access any existing schemas (including `public`). It must create the schema itself to become its owner. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for the full permission model and deploy-first workflow. Initialize tables inside the `onPluginsReady` callback before registering routes (see CRUD pattern above): - -```typescript -// Inside onPluginsReady — runs once at startup before handling requests -await appkit.lakebase.query(` - CREATE SCHEMA IF NOT EXISTS app_data; - CREATE TABLE IF NOT EXISTS app_data.items ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW() - ); -`); -``` - -## ORM Integration (Optional) - -The plugin exposes the raw `pg.Pool` via `appkit.lakebase.pool` — works with any PostgreSQL library: - -```typescript -// Drizzle ORM -import { drizzle } from "drizzle-orm/node-postgres"; -const db = drizzle(appkit.lakebase.pool); - -// Prisma (with @prisma/adapter-pg) -import { PrismaPg } from "@prisma/adapter-pg"; -const adapter = new PrismaPg(appkit.lakebase.pool); -const prisma = new PrismaClient({ adapter }); -``` - -For ORM-compatible config: `appkit.lakebase.getOrmConfig()`. - -## Chat Persistence Pattern - -Save AI chat conversations to Lakebase so users can resume sessions and scroll full message history. - -**Schema** — create in a separate `chat` schema (not `app`) so the deploy-first ownership model stays clean: - -```sql -CREATE SCHEMA IF NOT EXISTS chat; - -CREATE TABLE IF NOT EXISTS chat.chats ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id TEXT NOT NULL, - title TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE IF NOT EXISTS chat.messages ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - chat_id UUID NOT NULL REFERENCES chat.chats(id) ON DELETE CASCADE, - role TEXT NOT NULL CHECK (role IN ('system', 'user', 'assistant', 'tool')), - content TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at - ON chat.messages(chat_id, created_at); -``` - -**Bootstrap** — run setup in `onPluginsReady` so tables exist before the server accepts requests: - -```typescript -await createApp({ - plugins: [server(), lakebase()], - async onPluginsReady(appkit) { - await setupChatTables(appkit); - // then register routes via appkit.server.extend(...) - }, -}); -``` - -**Persistence helpers** — use parameterized queries: - -```typescript -export async function createChat(appkit, input: { userId: string; title: string }) { - const result = await appkit.lakebase.query( - `INSERT INTO chat.chats (user_id, title) VALUES ($1, $2) - RETURNING id, user_id, title, created_at, updated_at`, - [input.userId, input.title], - ); - return result.rows[0]; -} - -export async function appendMessage(appkit, input: { chatId: string; role: string; content: string }) { - const result = await appkit.lakebase.query( - `INSERT INTO chat.messages (chat_id, role, content) VALUES ($1, $2, $3) - RETURNING id, chat_id, role, content, created_at`, - [input.chatId, input.role, input.content], - ); - return result.rows[0]; -} -``` - -**User identity**: In deployed apps, use `req.header("x-forwarded-email")` (injected by the Databricks Apps platform proxy; for off-platform deployments, use your own auth middleware). For local dev, hardcode a test user ID. - -**History endpoints**: -- `GET /api/chats` — list chats for current user -- `GET /api/chats/:chatId/messages` — load ordered history -- `DELETE /api/chats/:chatId` — delete chat (messages cascade) - -**AI SDK v6 integration**: Use `setMessages()` from `useChat` return value for history loading (NOT `initialMessages`). To read response headers like `X-Chat-Id`, pass a custom `fetch` wrapper on the `TextStreamChatTransport` constructor. - -## Reading from Lakebase synced tables - -Lakebase synced tables materialize Delta/UC tables into Lakebase Postgres for low-latency app reads. The lakehouse remains the source of truth; Lakebase serves as a read-optimized index. - -**Architecture:** -``` -Delta gold tables → Synced tables (read-only) → App reads via appkit.lakebase.query() -App writes → Lakebase OLTP tables → optional Lakehouse Sync → Delta -``` - -**Use synced tables when** data is curated in Delta, changes relatively slowly, and must be served at OLTP latency — operational consoles, user-facing apps on gold tables, feature serving, or hybrid read/write patterns. See the **`databricks-lakebase`** skill's [synced-tables.md](../../../databricks-lakebase/references/synced-tables.md) for the full decision checklist. - -> **Security note:** Synced tables do not propagate Unity Catalog fine-grained access control (row filters, column masks). If UC FGAC is critical, use DBSQL with user authorization instead. - -### How It Works - -Synced tables (created via `databricks postgres create-synced-table`) appear as regular Postgres tables. From the app's perspective, use the same `appkit.lakebase.query()` pattern but **read-only**. - -**Key differences from CRUD tables:** - -| | CRUD tables | Lakebase synced tables | -|--|-------------|---------------| -| Created by | App SP (via `CREATE TABLE`) | Sync pipeline (DLT) | -| Owned by | SP role | System role (`databricks_writer_*`) | -| Operations | Read + Write | **Read-only** (writes corrupt sync) | -| Schema init | App must `CREATE SCHEMA/TABLE` | Already exists after sync | -| Deploy-first | Required (SP must own schema) | Not required | - -**Permission grant required:** The app's SP has `CAN_CONNECT_AND_CREATE` but does **not** have `pg_read_all_data`. To read synced tables, the project owner must grant access — see the **`databricks-lakebase`** skill's SKILL.md "Grant app SP access to synced tables" section for the SQL commands and psql connection steps. - -**Example Express route reading synced taxi data:** - -```typescript -// Inside onPluginsReady → appkit.server.extend((app) => { ... }) -app.get('/api/top-pickups', async (_req, res) => { - const { rows } = await appkit.lakebase.query(` - SELECT pickup_zip, COUNT(*) AS trip_count, AVG(fare_amount) AS avg_fare - FROM public.nyc_trips - GROUP BY pickup_zip - ORDER BY trip_count DESC - LIMIT 10 - `); - res.json(rows); -}); -``` - -> **Do not write to synced tables.** The sync pipeline manages the data — direct writes corrupt the sync state. For mixed read/write patterns, read from synced tables and write to separate app-owned tables. To create synced tables and grant the app's SP read access, see the **`databricks-lakebase`** skill's [synced-tables.md](../../../databricks-lakebase/references/synced-tables.md) and the "Grant app SP access to synced tables" section in its SKILL.md. - -## Key Differences from Analytics Pattern - -| | Analytics | Lakebase | -|--|-----------|---------| -| SQL dialect | Databricks SQL (Spark SQL) | Standard PostgreSQL | -| Query location | `config/queries/*.sql` files | `appkit.lakebase.query()` in Express routes | -| Data retrieval | `useAnalyticsQuery` hook | Express route via `server.extend()` | -| Date functions | `CURRENT_TIMESTAMP()`, `DATEDIFF(DAY, ...)` | `NOW()`, `AGE(...)` | -| Auto-increment | N/A | `SERIAL` or `GENERATED ALWAYS AS IDENTITY` | -| Insert pattern | N/A | `INSERT ... VALUES ($1) RETURNING *` | -| Params | Named (`:param`) | Positional (`$1, $2, ...`) | - -**NEVER use `useAnalyticsQuery` for Lakebase data** — it queries the SQL warehouse, not Lakebase. -**NEVER put Lakebase SQL in `config/queries/`** — those files are only for warehouse queries. - -## Local Development - -### Prerequisites (MUST verify before local development) - -**This applies when your Databricks App uses Lakebase.** Run this check before any local development: - -```bash -databricks apps get --profile -``` - -Check the response for the `active_deployment` field. If it exists with `status.state` of `SUCCEEDED`, the app has been deployed. If `active_deployment` is missing, the app has never been deployed: -1. **STOP** — do not proceed with local development -2. Deploy first: `databricks apps deploy --profile ` -3. Wait for deployment to complete, then continue - -If you skip this step, the Service Principal won't own the database schema. You'll create schemas under your credentials that the SP **cannot access** after deployment. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for the full workflow and recovery steps. - -Lakebase project creators already have database access after the first deploy. Collaborators need `databricks_superuser` granted by the project creator via Branch Overview. - -> **Project-owner note:** If you are the Lakebase project owner, `databricks_create_role` may fail with "role already exists" and `GRANT databricks_superuser` may fail with "permission denied to grant role" — both errors are safe to ignore; the project owner already has the necessary access. - -The Lakebase env vars (`PGHOST`, `PGDATABASE`, etc.) are auto-set only when deployed. For local development, get the connection details from your endpoint and set them manually: - -```bash -# Get endpoint connection details -databricks postgres get-endpoint \ - projects//branches//endpoints/ \ - --profile -``` - -Then create `server/.env` with the values from the endpoint response: - -``` -PGHOST= -PGPORT=5432 -PGDATABASE= -PGUSER= -PGSSLMODE=require -LAKEBASE_ENDPOINT=projects//branches//endpoints/ -``` - -Load `server/.env` in your dev server (e.g. via `dotenv` or `node --env-file=server/.env`). Never commit `.env` files — add `server/.env` to `.gitignore`. - -## Troubleshooting - -| Error | Cause | Solution | -|-------|-------|---------| -| `permission denied for schema public` | SP cannot access `public` schema | Create custom schema: `CREATE SCHEMA IF NOT EXISTS app_data` and qualify all table names with `app_data.` | -| `permission denied for schema ` | Schema was created by another role (e.g. you ran locally before deploying) | Schema owned by wrong role. To preserve data: export first (`pg_dump` or temp schema copy). **Ask the user before dropping.** Then drop + redeploy. See **`databricks-lakebase`** skill's **Schema Permissions for Deployed Apps** for full steps. | -| Works locally but `permission denied` after deploy | Local credentials created the schema; the SP cannot access schemas it does not own | Schema owned by wrong role — see row above for export + drop + redeploy steps | -| `connection refused` | Pool not connected or wrong env vars | Check `PGHOST`, `PGPORT`, `LAKEBASE_ENDPOINT` are set | -| `relation "X" does not exist` | Tables not initialized | Run `CREATE TABLE IF NOT EXISTS` at startup | -| App builds but pool fails at runtime | Env vars not set locally | Set vars in `server/.env` — see Local Development above | +**Infrastructure** (create project, synced table pipeline, SP grants): **`databricks-lakebase`** skill. diff --git a/skills/databricks-apps/references/appkit/lifecycle.md b/skills/databricks-apps/references/appkit/lifecycle.md new file mode 100644 index 00000000..e1f66359 --- /dev/null +++ b/skills/databricks-apps/references/appkit/lifecycle.md @@ -0,0 +1,65 @@ +# AppKit Development Lifecycle + +Ordering for scaffold → develop → validate → deploy. Capability-specific steps — see [Data Patterns](data-patterns.md) for which apply. + +> **Agentic mode:** scaffold and deploy are handled externally — skip both. No smoke tests. `npm run dev` runs against **live** injected resources, so the Lakebase deploy-first rule and "don't assert Lakebase rows locally" caveat below **do not apply**. You still run `databricks apps validate` (no `--profile`). See [Environments](environments.md). + +## Phase order by capability + +**All apps:** Prerequisites (profile + CLI via `databricks-core`) → Gates (conditional — [Data Patterns](data-patterns.md)) → Scaffold (`manifest` → `apps init --run none`) → First code (**update smoke tests first**) → Validate (`databricks apps validate`) → Deploy (user consent). + +Capability-specific ordering layered on top: + +- **`reads_warehouse`** — SQL files + `npm run typegen` before building UI; deploy is optional before `npm run dev`. +- **`writes_oltp`** — replace the scaffold and put schema init + routes in `onPluginsReady`; **deploy before local dev** (the SP must own the schema first); don't assert Lakebase rows in local validate. +- **`genie`** — create or reuse the space before/with init. + +**Hybrid apps** (e.g. analytics + lakebase + genie): follow the **strictest** rule — e.g. deploy-before-dev, because OLTP requires it. + +## Recommended slice order (multi-plugin) + +After init, when several capabilities are active: + +1. Genie space (if `genie`) +2. Lakebase schema/routes + first deploy (if `writes_oltp`) +3. Analytics SQL + typegen (if `reads_warehouse`) +4. Files / serving plugin config (if present) +5. Frontend — **separate UI surfaces** per capability +6. Smoke tests → validate + +## First deploy + +⚠️ **USER CONSENT REQUIRED** before any deploy. See [Platform Guide](../platform-guide.md). + +A new scaffold has bundle config but **no workspace app** until bundle deploy. `databricks apps deploy` alone often fails with **app does not exist**. + +1. `databricks bundle deploy -t --profile ` +2. `databricks apps deploy -t --profile ` (or `bundle run `) + +Check: `databricks apps get --profile ` — missing `active_deployment` means first-deploy path. + +**Lakebase OLTP:** do not run `npm run dev` against Lakebase until step 2 succeeds — SP must own the schema first. + +## Subsequent deploys + +1. `databricks apps validate --profile ` +2. `databricks apps deploy -t --profile ` + +`bundle deploy` alone does not restart the app — follow with `apps deploy` when config/code changed. + +## Deploy before local dev? + +| Mix | Deploy before `npm run dev`? | +|-----|------------------------------| +| Analytics only | No | +| Lakebase synced reads only | No (grant SP after deploy) | +| Lakebase OLTP CRUD | **Yes** | +| Hybrid with OLTP writes | **Yes** | +| Genie / serving / files only (no OLTP) | No | + +## Post-deploy verification + +```bash +databricks apps get --profile -o json # app_status.state: RUNNING +databricks apps logs --follow --profile # OAuth required; not PAT +``` diff --git a/skills/databricks-apps/references/appkit/model-serving.md b/skills/databricks-apps/references/appkit/model-serving.md index ed8a0a18..2d787b5a 100644 --- a/skills/databricks-apps/references/appkit/model-serving.md +++ b/skills/databricks-apps/references/appkit/model-serving.md @@ -1,15 +1,10 @@ # Model Serving: Calling ML Endpoints from Apps -Use Model Serving when your app needs **AI features** — chat, inference, embeddings, or predictions from a Databricks Model Serving endpoint. For analytics dashboards, use `config/queries/` instead. For persistent storage, use Lakebase. +**Pattern selection:** [Data Patterns](data-patterns.md). This guide covers **`serving()` plugin setup only.** -## When to Use +Use Model Serving for **AI features** — chat, inference, embeddings, predictions from a Databricks serving endpoint. -| Pattern | Use Case | Data Source | -|---------|----------|-------------| -| Analytics | Read-only dashboards, charts, KPIs | SQL Warehouse | -| Lakebase | CRUD operations, persistent state, forms | PostgreSQL (Lakebase) | -| Model Serving | Chat, AI features, model inference | Serving Endpoint | -| Multiple | Dashboard with AI features or persistent state | Combine as needed | +> **Agentic mode:** the serving endpoint already exists and `DATABRICKS_SERVING_ENDPOINT_NAME` is injected. **Skip** *Scaffolding* and *Adding Model Serving to an Existing App*; do not use the `databricks-model-serving` skill to create an endpoint. Just call the `serving()` plugin (it reads the env var). See [Environments](environments.md). ## Scaffolding @@ -66,9 +61,9 @@ The injected value is the endpoint **name** (not a URL). Use it in server-side c ```typescript import { createApp, server, analytics, serving } from "@databricks/appkit"; -createApp({ +await createApp({ plugins: [server(), analytics(), serving()], -}).catch(console.error); +}); ``` Preserve existing plugins and add `serving()` to the array. diff --git a/skills/databricks-apps/references/appkit/overview.md b/skills/databricks-apps/references/appkit/overview.md index 3dbfc09b..0049fd8f 100644 --- a/skills/databricks-apps/references/appkit/overview.md +++ b/skills/databricks-apps/references/appkit/overview.md @@ -1,149 +1,120 @@ # AppKit Overview -AppKit is the recommended way to build Databricks Apps - provides type-safe SQL queries, React components, and seamless deployment. +AppKit is the recommended way to build Databricks Apps — type-safe SQL queries, React components, and seamless deployment. -## Choose Your Data Pattern FIRST +**Local vs agentic mode:** [Environments](environments.md) — detect with `DATABRICKS_APPS_AGENTIC_MODE` first; in agentic mode, scaffold/deploy/smoke-tests are skipped. -Before scaffolding, decide which data pattern the app needs: +**Pattern selection, gates, and capability composition:** [Data Patterns](data-patterns.md) (canonical). -| Pattern | When to use | Init command | -|---------|-------------|-------------| -| **Analytics** (read-only) | Dashboards, charts, KPIs from warehouse | `--features analytics --set analytics.sql-warehouse.id=` | -| **Lakebase synced tables** (low-latency reads) | Point lookups, entity search, catalogs from lakehouse data | `--features lakebase` (no `--set` flags needed) + sync Delta table via `databricks-lakebase` skill | -| **Lakebase (OLTP)** (read/write) | CRUD forms, persistent state, user data | `--features lakebase --set lakebase.postgres.branch= --set lakebase.postgres.database=` | -| **Genie** (NL queries) | Chat interface over Unity Catalog tables | `--features genie --set genie..=` (check manifest) | -| **Model Serving** (ML inference) | Chat, AI features, model predictions | `--features serving --set serving.serving-endpoint.name=` (check manifest) | -| **Jobs** (trigger Lakeflow Jobs) | Kick off and monitor pre-existing notebooks / Python / SQL / dbt jobs | `--features jobs --set jobs..=` (check manifest) | -| **Multiple** | Combine plugins as needed (e.g. dashboard + CRUD, analytics + Genie) | `--features analytics,lakebase,genie,...` with all required `--set` flags per plugin | +**Scaffold → dev → validate → deploy order:** [Lifecycle](lifecycle.md). -See [Lakebase Guide](lakebase.md) for full Lakebase scaffolding and app-code patterns. -See [Genie Guide](genie.md) for space creation, plugin setup, and frontend components. +## Workflow (summary) -## Workflow +1. **Classify capabilities** → [Data Patterns](data-patterns.md) +2. **Scaffold**: `databricks apps manifest` → `databricks apps init --run none` +3. **Develop**: follow checklist slices + [Lifecycle](lifecycle.md) +4. **Validate**: `databricks apps validate` (after updating smoke tests) +5. **Deploy**: [Lifecycle: First deploy](lifecycle.md#first-deploy) (⚠️ user consent) -1. **Scaffold**: Run `databricks apps manifest`, then `databricks apps init` with `--features` and `--set` as in parent SKILL.md (App Manifest and Scaffolding) -2. **Develop**: `cd && npm install && npm run dev` -3. **Validate**: `databricks apps validate` -4. **Deploy**: `databricks apps deploy --profile ` (⚠️ USER CONSENT REQUIRED) +## Data discovery -## Data Discovery (Before Writing SQL) +When `reads_warehouse` or `writes_delta` is in the capability set, use the parent **`databricks-core`** skill before writing SQL. -**Use the parent `databricks-core` skill for data discovery** (table search, schema exploration, query execution). +## Pre-implementation checklists -## Pre-Implementation Checklist +Use the checklist slices for your capability set — [Data Patterns: Checklist slices](data-patterns.md#checklist-slices). -Before writing App.tsx, complete these steps: +Quick reference: -1. ✅ Create SQL files in `config/queries/` -2. ✅ Run `npm run typegen` to generate query types -3. ✅ Read `client/src/appKitTypes.d.ts` to see available query result types -4. ✅ Verify component props via `npx @databricks/appkit docs` (check the relevant component page) -5. ✅ Plan smoke test updates (default expects "Minimal Databricks App") +| Capabilities | Before `App.tsx` | +|--------------|------------------| +| `reads_warehouse` | SQL files + typegen | +| `writes_oltp` | Replace scaffold; plan `onPluginsReady` routes; deploy before dev | +| Hybrid | Union both; warehouse reads ≠ Lakebase writes | -**DO NOT** write UI code until types are generated and verified. +## Post-implementation -## Post-Implementation Checklist +Before `databricks apps validate`: -Before running `databricks apps validate`: +1. Update `tests/smoke.spec.ts` selectors — **local only; agentic mode has no smoke tests** +2. Remove default "hello world" assertions — **local only** +3. Run typegen if analytics reads changed +4. Convert numeric SQL display with `Number()` -1. ✅ Update `tests/smoke.spec.ts` heading selector to match your app title -2. ✅ Update or remove the 'hello world' text assertion -3. ✅ Verify `npm run typegen` has been run after all SQL files are finalized -4. ✅ Ensure all numeric SQL values use `Number()` conversion in display code +## Project structure -## Project Structure +**Analytics reads** (`reads_warehouse`): ``` my-app/ -├── server/ -│ ├── server.ts # Backend entry point (AppKit) -│ └── .env # Optional local dev env vars (do not commit) -├── client/ -│ ├── index.html -│ ├── vite.config.ts -│ └── src/ -│ ├── main.tsx -│ └── App.tsx # <- Main app component (start here) -├── config/ -│ └── queries/ -│ └── my_query.sql # -> queryKey: "my_query" -├── app.yaml # Deployment config -├── package.json -└── tsconfig.json +├── config/queries/*.sql # SELECT only → queryKey +├── client/src/App.tsx +├── server/server.ts # onPluginsReady + routes (if mutations/APIs) +└── tests/smoke.spec.ts ``` -**Key files to modify:** +**Lakebase OLTP** (`writes_oltp`) — no `config/queries/`; CRUD via Express routes. See [Lakebase OLTP](lakebase-oltp.md). + +**Key files:** + | Task | File | |------|------| | Build UI | `client/src/App.tsx` | -| Add SQL query | `config/queries/.sql` | -| Add API endpoint | `server/server.ts` (`onPluginsReady` + `server.extend`) | -| Add shared helpers (optional) | create `shared/types.ts` or `client/src/lib/formatters.ts` | +| Add warehouse read query | `config/queries/.sql` | +| Add API / mutation route | `server/server.ts` (`onPluginsReady` + `server.extend`) | | Fix smoke test | `tests/smoke.spec.ts` | -## Type Safety - -For type generation details, see: `npx @databricks/appkit docs ./docs/development/type-generation.md` +## Type safety (analytics reads) -**Quick workflow:** 1. Add/modify SQL in `config/queries/` -2. Types auto-generate during dev via the Vite plugin (or run `npm run typegen` manually) -3. Types appear in `client/src/appKitTypes.d.ts` +2. Run `npm run typegen` (or auto during dev) +3. Types in `client/src/appKitTypes.d.ts` -## Adding Visualizations +Details: `npx @databricks/appkit docs ./docs/development/type-generation.md` + +## Adding visualizations -**Step 1**: Create SQL file `config/queries/my_data.sql` ```sql +-- config/queries/my_data.sql SELECT category, COUNT(*) as count FROM my_table GROUP BY category ``` -**Step 2**: Use component (types auto-generated!) ```typescript import { BarChart } from '@databricks/appkit-ui/react'; -// Query mode: fetches data automatically - -// Data mode: pass static data directly (no queryKey/parameters needed) - ``` -## AppKit Official Documentation - -**Always use AppKit docs as the source of truth for API details.** +## AppKit official documentation ```bash -npx @databricks/appkit docs # show the docs index (start here) -npx @databricks/appkit docs # look up a section by name or doc path +npx @databricks/appkit docs +npx @databricks/appkit docs ``` -Do not guess paths — run without args first, then pick from the index. - -## References - -| When you're about to... | Read | -|-------------------------|------| -| Write SQL files | [SQL Queries](sql-queries.md) — parameterization, dialect, sql.* helpers | -| Use `useAnalyticsQuery` | [AppKit SDK](appkit-sdk.md) — memoization, conditional queries | -| Add chart/table components | [Frontend](frontend.md) — component quick reference, anti-patterns | -| Add API mutation endpoints | [Custom Endpoints](custom-endpoints.md) — only if you need server-side logic | -| Use Lakebase for CRUD / persistent state | [Lakebase](lakebase.md) — Lakebase plugin API, `onPluginsReady` patterns, schema init | -| Add Genie chat | [Genie](genie.md) — space creation, plugin setup, frontend components | -| Call ML model serving endpoints | [Model Serving](model-serving.md) — serving plugin, frontend hooks | -| Trigger / monitor Lakeflow Jobs from the app | [Jobs](jobs.md) — env discovery, JobHandle API, SSE streaming | - -## Critical Rules - -1. **SQL for data retrieval**: Use `config/queries/` + visualization components. Never custom endpoints for warehouse SELECT. -2. **Numeric types**: SQL numbers may return as strings. Always convert: `Number(row.amount)` -3. **Type imports**: Use `import type { ... }` (verbatimModuleSyntax enabled). -4. **Charts are ECharts**: No Recharts children — use props (`xKey`, `yKey`, `colors`). `xKey`/`yKey` auto-detect from schema if omitted. -5. **Two data modes**: Charts/tables support query mode (`queryKey` + `parameters`) and data mode (static `data` prop). -6. **Conditional queries**: Use `autoStart: false` option or conditional rendering to control query execution. - -## Decision Tree - -- **Display data from SQL?** - - Chart/Table → `BarChart`, `LineChart`, `DataTable` components - - Custom layout (KPIs, cards) → `useAnalyticsQuery` hook -- **Call Databricks API?** → Dedicated plugin (serving, jobs, files) or custom endpoint via `onPluginsReady` -- **Modify data?** → Express routes in `onPluginsReady` +## Plugin setup guides + +Pattern selection → [Data Patterns](data-patterns.md). These docs are **setup only**: + +| Plugin | Guide | +|--------|-------| +| SQL reads | [SQL Queries](sql-queries.md) | +| Custom routes | [Custom Endpoints](custom-endpoints.md) | +| Delta DML | [Warehouse Mutations](warehouse-mutations.md) | +| Lakebase | [Lakebase](lakebase.md) → [OLTP](lakebase-oltp.md) / [Synced Reads](lakebase-synced-reads.md) | +| Genie | [Genie](genie.md) | +| Serving | [Model Serving](model-serving.md) | +| Files | [Files](files.md) | +| Jobs | [Jobs](jobs.md) | +| UI | [Frontend](frontend.md), [AppKit SDK](appkit-sdk.md) | + +## Critical rules + +1. Warehouse **reads** → `config/queries/` — never custom endpoints for SELECT. +2. **Writes** → pick path in [Data Patterns: Write path](data-patterns.md#write-path). +3. SQL numbers may be strings — use `Number(row.amount)`. +4. Charts are ECharts — use `xKey`/`yKey` props, not Recharts children. +5. Never `useAnalyticsQuery` for Lakebase data. + +## Decision tree + +→ [Data Patterns](data-patterns.md) — capability catalog, gates, write/read paths, recipes. diff --git a/skills/databricks-apps/references/appkit/proto-first.md b/skills/databricks-apps/references/appkit/proto-first.md index 5d158bba..a5f4d6ea 100644 --- a/skills/databricks-apps/references/appkit/proto-first.md +++ b/skills/databricks-apps/references/appkit/proto-first.md @@ -1,6 +1,8 @@ # Proto-First App Design -Schema-first approach for AppKit apps using protobuf data contracts. Define contracts BEFORE implementation — derive TypeScript types, Lakebase DDL, and Volume paths from `.proto` files. +**Advanced / optional** — use only for multi-plugin apps with strict typed boundaries. For most apps, skip this and use [Data Patterns](data-patterns.md) instead. + +Schema-first approach for AppKit apps using protobuf data contracts. **When to use:** New apps with multiple plugins (files + lakebase + jobs), or adding typed boundaries to existing apps. Skip for quick prototypes. @@ -14,7 +16,8 @@ Define protobuf data contracts FIRST, then derive everything else (TypeScript ty | Scenario | Use this skill | |----------|---------------| -| Creating a new Databricks app | YES — define contracts before `databricks apps init` | +| Creating a new **multi-plugin** app (e.g. files + lakebase + jobs) | YES — define contracts before `databricks apps init` | +| Single-plugin app (dashboard, simple CRUD) | NO — use [Data Patterns](data-patterns.md) | | Adding a new data boundary to an existing app | YES — add proto before implementation | | Quick prototype / hackathon | NO — skip contracts, move fast | | Modifying existing typed code | NO — contracts already exist | @@ -41,7 +44,7 @@ Every Databricks app decomposes into a combination of these plugin modules: | **Database** | lakebase | Postgres tables | Structured records, queries, migrations | | **Compute** | jobs | Databricks Jobs API | Job runs, task results, cluster configs | | **Analytics** | analytics | SQL Warehouse | Read-only queries, dashboards | -| **Serving** | server | HTTP routes | API endpoints, SSE streams | +| **HTTP API** | server | Custom routes, SSE | API endpoints, streams | ### Decomposition Rules @@ -244,7 +247,9 @@ Example migration: ```sql -- migrations/001_create_runs.sql -CREATE TABLE IF NOT EXISTS runs ( +-- Schema-qualified: the app SP cannot use `public` — see lakebase-oltp.md. +CREATE SCHEMA IF NOT EXISTS app_data; +CREATE TABLE IF NOT EXISTS app_data.runs ( run_id TEXT NOT NULL, app_name TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'RUN_STATUS_PENDING', @@ -256,6 +261,8 @@ CREATE TABLE IF NOT EXISTS runs ( ); ``` +Execute migrations in `onPluginsReady` via `appkit.lakebase.query()` — the Lakebase OLTP **deploy-first** rule ([lakebase-oltp.md](lakebase-oltp.md)) still applies. + ### 3d. Validate ```bash @@ -303,4 +310,4 @@ Before writing implementation code: ## References -- [Plugin Contract Details](references/plugin-contracts.md) — proto↔plugin type mappings for files, lakebase, jobs +- [Plugin Contract Details](proto-contracts.md) — proto↔plugin type mappings for files, lakebase, jobs diff --git a/skills/databricks-apps/references/appkit/sql-queries.md b/skills/databricks-apps/references/appkit/sql-queries.md index c1143911..c5d0985a 100644 --- a/skills/databricks-apps/references/appkit/sql-queries.md +++ b/skills/databricks-apps/references/appkit/sql-queries.md @@ -1,13 +1,17 @@ # SQL Query Files -**IMPORTANT**: ALWAYS use SQL files in `config/queries/` for data retrieval. NEVER add custom endpoints for warehouse SQL queries. +**IMPORTANT**: ALWAYS use SQL files in `config/queries/` for **data retrieval (SELECT)**. NEVER add custom endpoints for warehouse SELECT queries. -- Store ALL SQL queries in `config/queries/` directory +For **writes** to Delta / Unity Catalog (`INSERT`, `UPDATE`, `DELETE`, `MERGE`), use custom mutation routes with `appkit.analytics.query()` — see [Warehouse Mutations](warehouse-mutations.md). Do not put DML in `config/queries/` for client-side execution. + +- Store all **read** (SELECT) queries in `config/queries/` directory - Name files descriptively: `trip_statistics.sql`, `user_metrics.sql`, `sales_by_region.sql` - Reference by filename (without extension) in `useAnalyticsQuery` or directly in a visualization component passing it as `queryKey` - App Kit automatically executes queries against configured Databricks warehouse - Benefits: Built-in caching, proper connection pooling, better performance +> **Agentic mode:** the warehouse is already wired — no scaffold or `--set`. Writing the SQL files is identical; you still confirm the target `catalog.schema.table` via data discovery (ambient auth, no `--profile`). See [Environments](environments.md). + ## Type Generation For full type generation details, see: `npx @databricks/appkit docs ./docs/development/type-generation.md` diff --git a/skills/databricks-apps/references/appkit/warehouse-mutations.md b/skills/databricks-apps/references/appkit/warehouse-mutations.md new file mode 100644 index 00000000..3d899810 --- /dev/null +++ b/skills/databricks-apps/references/appkit/warehouse-mutations.md @@ -0,0 +1,224 @@ +# Warehouse Mutations (Delta / Unity Catalog) + +Use this guide when an AppKit app must **write to Unity Catalog Delta tables** via the SQL warehouse — `INSERT`, `UPDATE`, `DELETE`, or `MERGE` — in response to a user action. + +For **reads** from the warehouse, use [SQL Queries](sql-queries.md) (`config/queries/` + `useAnalyticsQuery`). **Never** add custom endpoints for SELECT. + +For **app-owned operational state** (forms, CRUD, session data), prefer [Lakebase OLTP](lakebase-oltp.md) instead of writing Delta directly from the app. + +**Pattern selection and gates:** [Data Patterns](data-patterns.md). + +> **Agentic mode:** the warehouse is already wired (id injected). **Skip** the `databricks apps init … --set` scaffold block below; just write the mutation route. You still confirm the target `catalog.schema.table` via data discovery. See [Environments](environments.md). + +## When this guide applies + +You're in the right place **only if** a user action must land in an existing Delta / Unity Catalog table **now**, as small scoped DML (`INSERT` / `UPDATE` / `DELETE` / `MERGE`). + +Anything else — app-owned CRUD, async/batch writes, or reads — is a different path. Choose it in **[Data Patterns: Write path](data-patterns.md#write-path)** (the canonical decision table); don't re-decide it here. + +**Never write to Lakebase synced tables** — they are read-only replicas of Delta; app writes corrupt sync. See [Lakebase Synced Reads](lakebase-synced-reads.md). + +## How it works + +The analytics plugin exposes **server-side** SQL execution via `appkit.analytics.query()` (and `appkit.analytics.asUser(req).query(...)` for on-behalf-of-user). This is separate from the client hook `useAnalyticsQuery`, which is for **read-only** display. + +``` +Browser → POST /api/your-mutation → Express route (Zod validate) + → appkit.analytics.query(fixed SQL, params) + → SQL warehouse → Delta table +``` + +**Scaffold with analytics** so the warehouse resource is wired: + +```bash +databricks apps init --name --features analytics \ + --set "analytics.sql-warehouse.id=" \ + --run none --profile +``` + +Hybrids need only the plugins actually involved: read warehouse + write Delta is `--features analytics` alone; read warehouse + write Lakebase is `--features analytics,lakebase` — in all cases with the required `--set` flags from `databricks apps manifest`. + +## Canonical pattern + +Register **one named route per mutation**. Use **fixed SQL** with `:param` placeholders and `sql.*` helpers. Validate input with Zod. **Never** accept arbitrary SQL from the client. + +Before writing code, run `npx @databricks/appkit docs ./docs/plugins/analytics.md` on the installed AppKit version and verify `query()` signature and parameter types. + +**Start inline in `onPluginsReady`** — `appkit` is already fully typed there, so no extra interfaces or generics are needed: + +```typescript +// server/server.ts +import { createApp, analytics, server, sql } from "@databricks/appkit"; +import { z } from "zod"; + +const FeedbackBody = z.object({ + userId: z.string().min(1), + rating: z.number().int().min(1).max(5), + comment: z.string().max(2000).optional(), +}); + +await createApp({ + plugins: [server(), analytics({})], + async onPluginsReady(appkit) { + appkit.server.extend((app) => { + app.post("/api/feedback", async (req, res) => { + const parsed = FeedbackBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: "Invalid input" }); + return; + } + + try { + await appkit.analytics.query( + `INSERT INTO catalog.schema.feedback (user_id, rating, comment, created_at) + VALUES (:user_id, :rating, :comment, current_timestamp())`, + { + user_id: sql.string(parsed.data.userId), + rating: sql.number(parsed.data.rating), + comment: sql.string(parsed.data.comment ?? ""), + }, + ); + res.status(201).json({ ok: true }); + } catch (err) { + console.error("Failed to insert feedback:", err); + res.status(500).json({ error: "Failed to save feedback" }); + } + }); + }); + }, +}); +``` + +### Extracting routes (larger apps, optional) + +For bigger apps, move the handler into a `setupFeedbackRoutes(appkit)` function and call it from `onPluginsReady`. Let TypeScript **infer** the type from the call site — do **not** hand-write an `AppKitWith*` interface or add `appkit-types.ts`: + +```typescript +// server/routes/warehouse/feedback-routes.ts +import { sql } from "@databricks/appkit"; +import type { Application } from "express"; + +export function setupFeedbackRoutes< + T extends { + analytics: { + query( + statement: string, + parameters?: Record | ReturnType>, + ): Promise; + }; + server: { extend(fn: (app: Application) => void): void }; + }, +>(appkit: T) { + appkit.server.extend((app) => { + // same /api/feedback handler as above + }); +} +``` + +```typescript +// server/server.ts +await createApp({ + plugins: [server(), analytics({})], + async onPluginsReady(appkit) { + setupFeedbackRoutes(appkit); // T inferred here — fully typed, no casts + }, +}); +``` + +For on-behalf-of-user calls (`appkit.analytics.asUser(req).query(...)`), see *Service principal vs on-behalf-of-user* below. Same generic-inference pattern as [Lakebase OLTP](lakebase-oltp.md) *Lakebase route modules — typing*. + +## Service principal vs on-behalf-of-user + +| Call | Credentials | When to use | +|------|-------------|-------------| +| `appkit.analytics.query(...)` | App service principal | System writes, batch ops, trusted server-side validation | +| `appkit.analytics.asUser(req).query(...)` | End user's Databricks identity | UC row/column policies must apply; user-scoped audit | + +OBO requires the deployed app proxy headers (`x-forwarded-user`, `x-forwarded-access-token`). In local dev without those headers, OBO may fall back to SP — verify behavior with `npx @databricks/appkit docs` for your AppKit version. + +Grant UC privileges to whichever identity executes the statement (SP client ID from `databricks apps get ` or the signed-in user). + +## Unity Catalog permissions + +The app SP (or user, for OBO) needs at minimum: + +- `USE CATALOG` on the target catalog +- `USE SCHEMA` on the target schema +- `MODIFY` (or table-appropriate write privilege) on the target table + +`CAN_USE` on the SQL warehouse is wired via the analytics plugin resource in `databricks.yml`. **Warehouse access does not imply table write access** — verify UC grants separately. + +Confirm access **without mutating real data** before wiring the route: + +```bash +# Preferred: verify the identity can read the target (no write side effect) +databricks experimental aitools tools query \ + "SELECT 1 FROM catalog.schema.feedback LIMIT 1" --profile +``` + +If you must exercise the write path itself, insert into a disposable table (e.g. `catalog.schema._appkit_smoke`) and drop it afterward — **do not** write throwaway rows into the real target table. + +## Client-side pattern + +Mutations use `fetch` to your custom route — **not** `useAnalyticsQuery`: + +```typescript +await fetch("/api/feedback", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ userId, rating, comment }), +}); +``` + +Keep reads on `useAnalyticsQuery` / visualization components; keep writes on POST/PUT/PATCH/DELETE to your mutation routes. + +## Anti-patterns + +```typescript +// ❌ NEVER — arbitrary SQL from the client (injection, privilege escalation) +app.post("/api/sql", async (req, res) => { + await appkit.analytics.query(req.body.sql); +}); + +// ❌ NEVER — SELECT in a custom endpoint (use config/queries/) +app.get("/api/items", async (_req, res) => { + const rows = await appkit.analytics.query("SELECT * FROM catalog.schema.items"); + res.json(rows); +}); + +// ❌ NEVER — string concatenation with user input +await appkit.analytics.query( + `INSERT INTO t VALUES ('${req.body.name}')`, +); + +// ❌ NEVER — heavy MERGE / large batch in a synchronous request handler +// Use jobs() plugin instead +``` + +## When to use Jobs instead + +Prefer the [Jobs](jobs.md) plugin when the write: + +- Touches many rows or large partitions +- Runs multi-statement ETL +- Should be retried/monitored asynchronously +- Must not block the HTTP response + +Pattern: custom endpoint validates input → `appkit.jobs("").runNow(...)` with parameters → job notebook/SQL task writes Delta. See [Jobs](jobs.md) for the `JobHandle` API. + +## Agents plugin note + +If you enable the **agents** plugin, built-in `analytics.query` **agent tools** are **read-only** (SELECT-only classifier). That restriction applies to LLM-invoked tools, **not** to your own server routes calling `appkit.analytics.query()` directly. Do not confuse agent tool safety with app mutation routes. + +## Validation and testing + +- Update `tests/smoke.spec.ts` to assert mutation **UI** (buttons, forms) — not that a Delta row landed (validate runs without live warehouse writes in many setups). +- `databricks apps validate` checks build/typecheck/lint/smoke — it does **not** prove UC write permissions. Verify grants after deploy. +- After deploy, confirm with `databricks apps logs --follow` if mutations fail at runtime. + +## Related guides + +- [Custom Endpoints](custom-endpoints.md) — when to add routes vs use plugins +- [SQL Queries](sql-queries.md) — read path only (`config/queries/`) +- [Lakebase OLTP](lakebase-oltp.md) — Postgres CRUD; [Lakebase Synced Reads](lakebase-synced-reads.md) — read-only Delta replicas +- [Jobs](jobs.md) — async lakehouse writes diff --git a/skills/databricks-apps/references/platform-guide.md b/skills/databricks-apps/references/platform-guide.md index 446ea3b8..43bfe737 100644 --- a/skills/databricks-apps/references/platform-guide.md +++ b/skills/databricks-apps/references/platform-guide.md @@ -105,17 +105,25 @@ env: ⚠️ **USER CONSENT REQUIRED** — always confirm with the user before deploying. +**Canonical flow** is documented in the parent **`databricks-apps`** skill *Deployment Workflow* (first deploy vs updates, Lakebase deploy-first rules). Summary: + ```bash -# Option A: single command (recommended) — validates, deploys, and runs +# First deploy (app not in workspace yet) — bundle deploy registers the app resource +databricks bundle deploy -t --profile +databricks apps deploy -t --profile + +# Subsequent deploys (app already exists) databricks apps deploy -t --profile -# Option B: step by step +# Alternative step-by-step (updates or first deploy after bundle deploy) databricks apps validate --profile databricks bundle deploy -t --profile databricks bundle run -t --profile ``` -❌ **Common mistake:** Running only `bundle deploy` and expecting the app to update. Deploy uploads code but does NOT apply config changes or restart the app. Use `databricks apps deploy` or add `bundle run` after `bundle deploy`. +❌ **Common mistakes:** +- Running `databricks apps deploy` on a freshly scaffolded app before `bundle deploy` → **app does not exist** +- Running only `bundle deploy` and expecting the app to update → also run `databricks apps deploy` or `bundle run` ### ⚠️ Destructive Updates Warning diff --git a/skills/databricks-apps/references/testing.md b/skills/databricks-apps/references/testing.md index bf1eb4d6..6d4f9dc7 100644 --- a/skills/databricks-apps/references/testing.md +++ b/skills/databricks-apps/references/testing.md @@ -31,6 +31,8 @@ describe('Feature Name', () => { ## Smoke Test (Playwright) +**First action after `apps init`:** update `tests/smoke.spec.ts` before the first `databricks apps validate` — regardless of app capabilities. + The template includes a smoke test at `tests/smoke.spec.ts` that verifies the app loads correctly. **⚠️ MUST UPDATE after customizing the app:** diff --git a/skills/databricks-lakebase/SKILL.md b/skills/databricks-lakebase/SKILL.md index db539e77..75e621ff 100644 --- a/skills/databricks-lakebase/SKILL.md +++ b/skills/databricks-lakebase/SKILL.md @@ -99,9 +99,11 @@ databricks postgres list-databases projects//branches/ -- | Value | JSON path | Used for | |-------|-----------|----------| +| Project resource path | `name` (from `list-projects`) | `lakebase.postgres.project` | +| Branch resource path | `name` (from `list-branches`) | `lakebase.postgres.branch` | +| Database resource path | `name` (from `list-databases`) | `lakebase.postgres.database` | | Endpoint host | `status.hosts.host` | `PGHOST`, `lakebase.postgres.host` | | Endpoint resource path | `name` | `LAKEBASE_ENDPOINT`, `lakebase.postgres.endpointPath` | -| Database resource path | `name` | `lakebase.postgres.database` | | PostgreSQL database name | `status.postgres_database` | `PGDATABASE`, `lakebase.postgres.databaseName` | ### Updating a Project @@ -178,34 +180,43 @@ databricks postgres reset-branch projects//branches/ --pr ### Build a Databricks App -After creating a project, scaffold a connected Databricks App: +After creating a project, scaffold a connected Databricks App. **Derive all three `--set` fields from `databricks apps manifest`** — the `lakebase` plugin requires `project`, `branch`, and `database` (omitting `project` fails init): ```bash -# 1. Get branch name +# 0. Confirm required fields (do not guess keys) +databricks apps manifest --profile + +# 1. Project resource path +databricks postgres list-projects --profile + +# 2. Branch resource path (use production after create-project) databricks postgres list-branches projects/ --profile -# 2. Get database name +# 3. Database resource path databricks postgres list-databases projects//branches/ --profile -# 3. Scaffold with lakebase feature +# 4. Scaffold — use .name from each list command databricks apps init --name --features lakebase \ + --set "lakebase.postgres.project=" \ --set "lakebase.postgres.branch=" \ --set "lakebase.postgres.database=" \ --run none --profile ``` -For the full app workflow, use the **`databricks-apps`** skill. +For the full app workflow (deploy, local dev, CRUD patterns), use the **`databricks-apps`** skill. ### Schema Permissions for Deployed Apps The app's Service Principal has `CAN_CONNECT_AND_CREATE` -- it can create new objects but **cannot access existing schemas**. The SP must create the schema to become its owner. -**ALWAYS deploy the app before running it locally.** This is the #1 source of Lakebase permission errors. +**ALWAYS deploy the app before running it locally (Lakebase OLTP CRUD apps).** This is the #1 source of Lakebase permission errors. + +**Correct workflow** — see **`databricks-apps`** skill *Deployment Workflow* for the full picture: -**Correct workflow:** -1. **Deploy first**: `databricks apps deploy --profile ` -2. **Grant local access** *(if needed)*: assign `databricks_superuser` via UI (project creators already have access) -3. **Develop locally**: your credentials get DML access to SP-owned schemas +1. **First deploy** (app never existed in workspace): `databricks bundle deploy -t --profile `, then `databricks apps deploy -t --profile `. `apps deploy` alone on a new scaffold often returns **app does not exist**. +2. **Subsequent deploys**: `databricks apps deploy -t --profile ` +3. **Grant local access** *(if needed)*: assign `databricks_superuser` via UI (project creators already have access) +4. **Develop locally**: your credentials get DML access to SP-owned schemas **If you already ran locally first** and hit `permission denied`: the schema is owned by your credentials, not the SP. **Do NOT drop the schema without asking the user** -- dropping it deletes all data. diff --git a/skills/databricks-lakebase/references/connectivity.md b/skills/databricks-lakebase/references/connectivity.md index 30bdae6e..69f17e42 100644 --- a/skills/databricks-lakebase/references/connectivity.md +++ b/skills/databricks-lakebase/references/connectivity.md @@ -11,7 +11,7 @@ ## Connection Patterns (Python) -> **JavaScript/TypeScript Databricks Apps** using AppKit get Lakebase connectivity via the `lakebase()` plugin — see the **`databricks-apps`** skill's [Lakebase guide](../../databricks-apps/references/appkit/lakebase.md). +> **JavaScript/TypeScript Databricks Apps** using AppKit get Lakebase connectivity via the `lakebase()` plugin — see the **`databricks-apps`** skill's [Lakebase guides](../../databricks-apps/references/appkit/lakebase.md) ([OLTP](../../databricks-apps/references/appkit/lakebase-oltp.md) / [synced reads](../../databricks-apps/references/appkit/lakebase-synced-reads.md)). ### Pattern 1: Direct Connection (Scripts/Notebooks) diff --git a/skills/databricks-lakebase/references/off-platform.md b/skills/databricks-lakebase/references/off-platform.md index 80b1f984..cea69e83 100644 --- a/skills/databricks-lakebase/references/off-platform.md +++ b/skills/databricks-lakebase/references/off-platform.md @@ -200,4 +200,4 @@ export default defineConfig({ - For on-platform connection patterns, see [connectivity.md](connectivity.md) - For vector similarity search with pgvector, see [pgvector.md](pgvector.md) -- For AppKit-based Lakebase integration, see the `databricks-apps` skill's [lakebase.md](../../databricks-apps/references/appkit/lakebase.md) +- For AppKit-based Lakebase integration, see the `databricks-apps` skill's [Lakebase router](../../databricks-apps/references/appkit/lakebase.md) ([OLTP](../../databricks-apps/references/appkit/lakebase-oltp.md), [synced reads](../../databricks-apps/references/appkit/lakebase-synced-reads.md)) diff --git a/skills/databricks-lakebase/references/pgvector.md b/skills/databricks-lakebase/references/pgvector.md index f6f7314d..d2b97780 100644 --- a/skills/databricks-lakebase/references/pgvector.md +++ b/skills/databricks-lakebase/references/pgvector.md @@ -47,21 +47,12 @@ If using a different model (768d or 1536d), change `VECTOR(1024)` to match. ## Vector Store Module -Create `server/lib/vector-store.ts`: +Create `server/lib/vector-store.ts`. For typing `appkit` in extracted modules, see the **`databricks-apps`** skill's [Lakebase OLTP](../../databricks-apps/references/appkit/lakebase-oltp.md) section *Lakebase route modules — typing* — use generic `setupXRoutes(appkit: T)` called from `onPluginsReady`; do not copy scaffold's `AppKitWithLakebase` or add `appkit-types.ts`. ```typescript -import type { Application } from "express"; - -interface AppKitWithLakebase { - lakebase: { - query(text: string, params?: unknown[]): Promise<{ rows: Record[] }>; - }; - server: { - extend(fn: (app: Application) => void): void; - }; -} - -export async function setupVectorTables(appkit: AppKitWithLakebase) { +export async function setupVectorTables< + T extends { lakebase: { query(text: string, params?: unknown[]): Promise } }, +>(appkit: T) { try { await appkit.lakebase.query("CREATE EXTENSION IF NOT EXISTS vector"); } catch (err: unknown) { @@ -90,7 +81,7 @@ export async function setupVectorTables(appkit: AppKitWithLakebase) { } export async function insertDocument( - appkit: AppKitWithLakebase, + appkit: AppKit, input: { content: string; embedding: number[]; metadata?: Record }, ) { const result = await appkit.lakebase.query( @@ -103,7 +94,7 @@ export async function insertDocument( } export async function retrieveSimilar( - appkit: AppKitWithLakebase, + appkit: AppKit, queryEmbedding: number[], limit = 5, ) {