diff --git a/web/package.json b/web/package.json index 877c7a779..05d04caec 100644 --- a/web/package.json +++ b/web/package.json @@ -22,9 +22,9 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.828.0", + "@aws-sdk/client-sts": "^3.907.0", "@aws-sdk/lib-storage": "^3.828.0", "@aws-sdk/s3-request-presigner": "^3.828.0", - "@aws-sdk/client-sts": "^3.907.0", "@babel/runtime": "7.26.0", "@codemirror/autocomplete": "6.18.3", "@codemirror/lang-json": "6.0.1", diff --git a/web/public/duckdb-extensions/v1.4.3/wasm_eh/avro.duckdb_extension.wasm b/web/public/duckdb-extensions/v1.4.3/wasm_eh/avro.duckdb_extension.wasm new file mode 100644 index 000000000..f54b573bb Binary files /dev/null and b/web/public/duckdb-extensions/v1.4.3/wasm_eh/avro.duckdb_extension.wasm differ diff --git a/web/public/duckdb-extensions/v1.4.3/wasm_eh/iceberg.duckdb_extension.wasm b/web/public/duckdb-extensions/v1.4.3/wasm_eh/iceberg.duckdb_extension.wasm new file mode 100644 index 000000000..a6f1062c7 Binary files /dev/null and b/web/public/duckdb-extensions/v1.4.3/wasm_eh/iceberg.duckdb_extension.wasm differ diff --git a/web/scripts/update-duckdb-extensions.ts b/web/scripts/update-duckdb-extensions.ts index 3068bcbca..3cec83283 100644 --- a/web/scripts/update-duckdb-extensions.ts +++ b/web/scripts/update-duckdb-extensions.ts @@ -5,7 +5,7 @@ import https from "https"; const DUCKDB_ENGINE_VERSION = "v1.4.3"; -const DEFAULT_EXTS = ["parquet", "json", "httpfs"]; +const DEFAULT_EXTS = ["parquet", "json", "httpfs", "iceberg", "avro"]; const OUTPUT_DIR = path.join( process.cwd(), diff --git a/web/src/core/adapters/icebergApi/icebergApi.ts b/web/src/core/adapters/icebergApi/icebergApi.ts new file mode 100644 index 000000000..270be558f --- /dev/null +++ b/web/src/core/adapters/icebergApi/icebergApi.ts @@ -0,0 +1,174 @@ +import type { IcebergApi } from "core/ports/IcebergApi"; +import type { SqlOlap } from "core/ports/SqlOlap"; +import { id } from "tsafe/id"; + +export type IcebergCatalogConfig = { + name: string; + warehouse: string; + endpoint: string; + /** + * Returns a fresh bearer token before each request. + * Each catalog can have its own auth provider (different OIDC clients, + * different realms, etc.). Returns undefined for public catalogs. + */ + getAccessToken: () => Promise; +}; + +/** + * Creates one IcebergApi instance that manages multiple catalogs. + * All catalog configs (endpoint, warehouse, token provider) are fixed at + * creation time — callers select which catalog to use via the `catalog` param. + */ +export function createDuckDbIcebergApi(params: { + sqlOlap: SqlOlap; + catalogs: IcebergCatalogConfig[]; +}): IcebergApi { + const { sqlOlap, catalogs } = params; + + function secretName(catalogName: string): string { + return `iceberg_${catalogName}`; + } + + // Eagerly install the iceberg extension, create secrets and attach all + // catalogs in a single connection so everything is ready before the first query. + const prDb = (async () => { + const { db } = await sqlOlap.getConfiguredAsyncDuckDb(); + + const conn = await db.connect(); + try { + await conn.query("INSTALL iceberg;\nLOAD iceberg;"); + + for (const catalogConfig of catalogs) { + const token = await catalogConfig.getAccessToken(); + + if (token !== undefined) { + await conn.query( + [ + `CREATE OR REPLACE SECRET "${secretName(catalogConfig.name)}" (`, + ` TYPE iceberg,`, + ` TOKEN '${token}'`, + ");" + ].join("\n") + ); + } + + const attachLines = [ + `ATTACH '${catalogConfig.warehouse}' AS "${catalogConfig.name}" (`, + ` TYPE iceberg,`, + ...(token !== undefined + ? [` SECRET '${secretName(catalogConfig.name)}',`] + : []), + ` ENDPOINT '${catalogConfig.endpoint}'`, + ");" + ]; + await conn.query(attachLines.join("\n")); + } + } finally { + await conn.close(); + } + + return db; + })(); + + return { + listAllTables: async () => { + let db: import("@duckdb/duckdb-wasm").AsyncDuckDB; + try { + db = await prDb; + } catch { + return id({ + errorCause: "network error" + }); + } + + const conn = await db.connect(); + try { + const result = await conn.query( + `SELECT table_catalog AS database, table_schema AS schema, table_name AS name FROM information_schema.tables;` + ); + + const tables: IcebergApi.TableEntry[] = result.toArray().map(row => ({ + catalog: String(row["database"]), + namespace: String(row["schema"]), + name: String(row["name"]) + })); + + return id({ tables }); + } catch (e) { + const cause = classifyError(e); + return id({ + errorCause: + cause === "unauthorized" ? "unauthorized" : "network error" + }); + } finally { + await conn.close(); + } + }, + + fetchTablePreview: async ({ catalog: catalogName, namespace, table, limit }) => { + const catalogConfig = catalogs.find(c => c.name === catalogName); + + if (catalogConfig === undefined) { + return id({ + errorCause: "network error" + }); + } + + let db: import("@duckdb/duckdb-wasm").AsyncDuckDB; + try { + db = await prDb; + } catch { + return id({ + errorCause: "network error" + }); + } + + const conn = await db.connect(); + try { + const result = await conn.query( + `SELECT * FROM "${catalogName}"."${namespace}"."${table}" LIMIT ${limit};` + ); + + const columns: IcebergApi.Column[] = ( + result.schema.fields as { + name: string; + type: { toString(): string }; + nullable: boolean; + }[] + ).map((field, index) => ({ + fieldId: index, + name: field.name, + rawType: field.type.toString(), + isRequired: !field.nullable + })); + + const rows: Record[] = result + .toArray() + .map(row => Object.fromEntries(Object.entries(row))); + + return id({ columns, rows }); + } catch (e) { + return id({ + errorCause: classifyError(e) + }); + } finally { + await conn.close(); + } + } + }; +} + +// --------------------------------------------------------------------------- +// Error classification +// --------------------------------------------------------------------------- + +function classifyError(e: unknown): "unauthorized" | "table not found" | "network error" { + const msg = (e instanceof Error ? e.message : String(e)).toLowerCase(); + if (msg.includes("401") || msg.includes("unauthorized") || msg.includes("403")) { + return "unauthorized"; + } + if (msg.includes("not found") || msg.includes("does not exist")) { + return "table not found"; + } + return "network error"; +} diff --git a/web/src/core/adapters/icebergApi/index.ts b/web/src/core/adapters/icebergApi/index.ts new file mode 100644 index 000000000..784849fc7 --- /dev/null +++ b/web/src/core/adapters/icebergApi/index.ts @@ -0,0 +1 @@ +export * from "./icebergApi"; diff --git a/web/src/core/adapters/onyxiaApi/ApiTypes.ts b/web/src/core/adapters/onyxiaApi/ApiTypes.ts index 1e1464a2d..fd33f72ba 100644 --- a/web/src/core/adapters/onyxiaApi/ApiTypes.ts +++ b/web/src/core/adapters/onyxiaApi/ApiTypes.ts @@ -126,6 +126,12 @@ export type ApiTypes = { } ))[]; }>; + iceberg?: ArrayOrNot<{ + warehouse: string; + endpoint: string; + catalog: string; + oidcConfiguration?: Partial; + }>; }; vault?: { URL: string; diff --git a/web/src/core/adapters/onyxiaApi/onyxiaApi.ts b/web/src/core/adapters/onyxiaApi/onyxiaApi.ts index 02c9892c2..c2333ec0f 100644 --- a/web/src/core/adapters/onyxiaApi/onyxiaApi.ts +++ b/web/src/core/adapters/onyxiaApi/onyxiaApi.ts @@ -315,6 +315,30 @@ export function createOnyxiaApi(params: { s3ConfigCreationFormDefaults }; })(), + iceberg: (() => { + const icebergConfig_api = (() => { + const value = apiRegion.data?.iceberg; + + if (value === undefined) { + return []; + } + + if (value instanceof Array) { + return value; + } + + return [value]; + })(); + return icebergConfig_api.map(icebergConfig => ({ + warehouse: icebergConfig.warehouse, + endpoint: icebergConfig.endpoint, + catalog: icebergConfig.catalog, + oidcParams: + apiTypesOidcConfigurationToOidcParams_Partial( + icebergConfig.oidcConfiguration + ) + })); + })(), allowedURIPatternForUserDefinedInitScript: apiRegion.services.allowedURIPattern, kafka: (() => { diff --git a/web/src/core/bootstrap.ts b/web/src/core/bootstrap.ts index 7dd134067..a5c142d9c 100644 --- a/web/src/core/bootstrap.ts +++ b/web/src/core/bootstrap.ts @@ -6,6 +6,7 @@ import { } from "clean-architecture"; import type { OnyxiaApi } from "core/ports/OnyxiaApi"; import type { SqlOlap } from "core/ports/SqlOlap"; +import type { IcebergApi } from "core/ports/IcebergApi"; import { usecases } from "./usecases"; import type { SecretsManager } from "core/ports/SecretsManager"; import type { Oidc } from "core/ports/Oidc"; @@ -15,6 +16,7 @@ import { pluginSystemInitCore } from "pluginSystem"; import { createOnyxiaApi } from "core/adapters/onyxiaApi"; import { assert } from "tsafe/assert"; import { fnv1aHashToHex } from "core/tools/fnv1aHashToHex"; +import { createDuckDbIcebergApi } from "./adapters/icebergApi"; export type ParamsOfBootstrapCore = { apiUrl: string; @@ -38,6 +40,8 @@ export type Context = { onyxiaApi: OnyxiaApi; secretsManager: SecretsManager; sqlOlap: SqlOlap; + icebergApi: IcebergApi; + icebergCatalogConfigs: { name: string; warehouse: string; endpoint: string }[]; }; export type Core = GenericCore; @@ -140,6 +144,37 @@ export async function bootstrapCore( // NOTE: Never reached } + const sqlOlap = createDuckDbSqlOlap({ + getS3Client: async () => { + if (!oidc.isUserLoggedIn) { + return { + errorCause: "need login" + }; + } + + const result = await dispatch( + usecases.s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() + ); + + if (result === undefined) { + return { + errorCause: "no s3 client" + }; + } + + const { s3Config, s3Client } = result; + + return { + s3Client, + s3_endpoint: s3Config.paramsOfCreateS3Client.url, + s3_url_style: s3Config.paramsOfCreateS3Client.pathStyleAccess + ? "path" + : "vhost", + s3_region: s3Config.region + }; + } + }); + const context: Context = { paramsOfBootstrapCore: params, oidc, @@ -148,36 +183,12 @@ export async function bootstrapCore( debugMessage: "SecretsManager not initialized, probably because user is not logged in." }), - sqlOlap: createDuckDbSqlOlap({ - getS3Client: async () => { - if (!oidc.isUserLoggedIn) { - return { - errorCause: "need login" - }; - } - - const result = await dispatch( - usecases.s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ); - - if (result === undefined) { - return { - errorCause: "no s3 client" - }; - } - - const { s3Config, s3Client } = result; - - return { - s3Client, - s3_endpoint: s3Config.paramsOfCreateS3Client.url, - s3_url_style: s3Config.paramsOfCreateS3Client.pathStyleAccess - ? "path" - : "vhost", - s3_region: s3Config.region - }; - } - }) + sqlOlap, + icebergApi: createObjectThatThrowsIfAccessed({ + debugMessage: + "IcebergApi not initialized, probably because user is not logged in or because iceberg is not configured for the current deployment region." + }), + icebergCatalogConfigs: [] }; const { core, dispatch, getState } = createCore({ @@ -255,6 +266,60 @@ export async function bootstrapCore( }); } + init_iceberg_api: { + if (!oidc.isUserLoggedIn) { + break init_iceberg_api; + } + + const deploymentRegion = + usecases.deploymentRegionManagement.selectors.currentDeploymentRegion( + getState() + ); + + if (deploymentRegion.iceberg.length === 0) { + break init_iceberg_api; + } + + const [{ createOidc, mergeOidcParams }, { oidcParams }] = await Promise.all([ + import("core/adapters/oidc"), + onyxiaApi.getAvailableRegionsAndOidcParams() + ]); + + assert(oidcParams !== undefined); + + const catalogs = await Promise.all( + deploymentRegion.iceberg.map(async warehouseConfig => { + const oidc_iceberg = await createOidc({ + ...mergeOidcParams({ + oidcParams, + oidcParams_partial: warehouseConfig.oidcParams + }), + transformBeforeRedirectForKeycloakTheme, + getCurrentLang, + autoLogin: true, + enableDebugLogs: enableOidcDebugLogs + }); + + return { + name: warehouseConfig.catalog, + warehouse: warehouseConfig.warehouse, + endpoint: warehouseConfig.endpoint, + getAccessToken: async (): Promise => { + if (!oidc_iceberg.isUserLoggedIn) return undefined; + return (await oidc_iceberg.getTokens()).accessToken; + } + }; + }) + ); + + context.icebergApi = createDuckDbIcebergApi({ sqlOlap, catalogs }); + context.icebergCatalogConfigs = deploymentRegion.iceberg.map(wc => ({ + name: wc.catalog, + warehouse: wc.warehouse, + endpoint: wc.endpoint + })); + } + if (oidc.isUserLoggedIn) { await dispatch(usecases.userConfigs.protectedThunks.initialize()); } diff --git a/web/src/core/ports/IcebergApi.ts b/web/src/core/ports/IcebergApi.ts new file mode 100644 index 000000000..1fd6852ac --- /dev/null +++ b/web/src/core/ports/IcebergApi.ts @@ -0,0 +1,96 @@ +/** + * Port for the Iceberg REST Catalog API (e.g. Apache Polaris). + * + * One IcebergApi instance manages several catalogs whose endpoints, warehouses + * and token providers are all fixed at creation time. + * Callers identify which catalog to target via the `catalog` parameter. + */ +export type IcebergApi = { + /** + * Returns the flat list of all (namespace, table) pairs for a given catalog. + * + * Equivalent to: SHOW ALL TABLES + */ + listAllTables: () => Promise; + + /** + * Fetches the first `limit` rows of a table together with its column schema + * in a single query. The column schema is derived from the Arrow result schema + * so no separate DESCRIBE round-trip is needed. + * + * Equivalent to: SELECT * FROM .. LIMIT + */ + fetchTablePreview: (params: { + catalog: string; + namespace: string; + table: string; + limit: number; + }) => Promise; +}; + +export namespace IcebergApi { + // ---------------------------------------------------------------- + // Shared domain types + // ---------------------------------------------------------------- + + /** A row from SHOW ALL TABLES */ + export type TableEntry = { + catalog: string; + namespace: string; + name: string; + }; + + /** + * A column derived from the Arrow result schema. + * + * `rawType` is the Arrow type string (e.g. "Int<32, true>", "Utf8"). + */ + export type Column = { + fieldId: number; + name: string; + rawType: string; + isRequired: boolean; + }; + + // ---------------------------------------------------------------- + // listAllTables + // ---------------------------------------------------------------- + + export type ListAllTablesResult = + | ListAllTablesResult.Success + | ListAllTablesResult.Failed; + + export namespace ListAllTablesResult { + export type Success = { + tables: TableEntry[]; + errorCause?: never; + }; + + export type Failed = { + tables?: never; + errorCause: "unauthorized" | "network error"; + }; + } + + // ---------------------------------------------------------------- + // fetchTablePreview + // ---------------------------------------------------------------- + + export type FetchTablePreviewResult = + | FetchTablePreviewResult.Success + | FetchTablePreviewResult.Failed; + + export namespace FetchTablePreviewResult { + export type Success = { + columns: Column[]; + rows: Record[]; + errorCause?: never; + }; + + export type Failed = { + columns?: never; + rows?: never; + errorCause: "unauthorized" | "table not found" | "network error"; + }; + } +} diff --git a/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts b/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts index 8d8453a3b..ac81cbefa 100644 --- a/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts +++ b/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts @@ -26,6 +26,12 @@ export type DeploymentRegion = { }) | undefined; allowedURIPatternForUserDefinedInitScript: string; + iceberg: { + warehouse: string; + endpoint: string; + catalog: string; + oidcParams: OidcParams_Partial; + }[]; kafka: | { url: string; diff --git a/web/src/core/usecases/icebergCatalog/index.ts b/web/src/core/usecases/icebergCatalog/index.ts new file mode 100644 index 000000000..6e655c5cd --- /dev/null +++ b/web/src/core/usecases/icebergCatalog/index.ts @@ -0,0 +1,3 @@ +export * from "./state"; +export * from "./thunks"; +export * from "./selectors"; diff --git a/web/src/core/usecases/icebergCatalog/selectors.ts b/web/src/core/usecases/icebergCatalog/selectors.ts new file mode 100644 index 000000000..68d381260 --- /dev/null +++ b/web/src/core/usecases/icebergCatalog/selectors.ts @@ -0,0 +1,60 @@ +import type { State as RootState } from "core/bootstrap"; +import { createSelector } from "clean-architecture"; +import { name } from "./state"; +import type { IcebergApi } from "core/ports/IcebergApi"; + +const state = (rootState: RootState) => rootState[name]; + +export type CatalogView = { + name: string; + warehouse: string; + endpoint: string; + namespaces: { + name: string; + tables: { name: string }[]; + }[]; +}; + +export type SelectedTableView = { + catalog: string; + namespace: string; + table: string; + isLoading: boolean; + columns: IcebergApi.Column[]; + rows: Record[]; +}; + +export type View = { + isLoading: boolean; + catalogs: CatalogView[]; + selectedTable: SelectedTableView | undefined; +}; + +const view = createSelector( + state, + (state): View => ({ + isLoading: state.stateDescription === "loading", + catalogs: state.catalogs.map(catalog => ({ + name: catalog.name, + warehouse: catalog.warehouse, + endpoint: catalog.endpoint, + namespaces: catalog.namespaces.map(ns => ({ + name: ns.name, + tables: ns.tables.map(tableName => ({ name: tableName })) + })) + })), + selectedTable: + state.selectedTable === undefined + ? undefined + : { + catalog: state.selectedTable.catalog, + namespace: state.selectedTable.namespace, + table: state.selectedTable.table, + isLoading: state.selectedTable.isLoading, + columns: state.selectedTable.columns, + rows: state.selectedTable.rows + } + }) +); + +export const selectors = { view }; diff --git a/web/src/core/usecases/icebergCatalog/state.ts b/web/src/core/usecases/icebergCatalog/state.ts new file mode 100644 index 000000000..4b61d174a --- /dev/null +++ b/web/src/core/usecases/icebergCatalog/state.ts @@ -0,0 +1,129 @@ +import { createUsecaseActions } from "clean-architecture"; +import { id } from "tsafe/id"; +import type { IcebergApi } from "core/ports/IcebergApi"; + +export const name = "icebergCatalog" as const; + +/** Hierarchical structure built from listAllTables (table names only). */ +export type NamespaceMetadata = { + name: string; + tables: string[]; +}; + +export type CatalogMetadata = { + name: string; + warehouse: string; + endpoint: string; + namespaces: NamespaceMetadata[]; +}; + +/** Preview of a table selected by the user — schema + first rows in one query. */ +export type SelectedTable = { + catalog: string; + namespace: string; + table: string; + isLoading: boolean; + columns: IcebergApi.Column[]; + rows: Record[]; +}; + +export type TablePreviewCacheEntry = { + columns: IcebergApi.Column[]; + rows: Record[]; +}; + +export type State = { + stateDescription: "not loaded" | "loading" | "ready"; + catalogs: CatalogMetadata[]; + selectedTable: SelectedTable | undefined; + /** Keyed by `catalog\0namespace\0table` */ + previewCache: Record; +}; + +export function previewCacheKey( + catalog: string, + namespace: string, + table: string +): string { + return `${catalog}\0${namespace}\0${table}`; +} + +export const { reducer, actions } = createUsecaseActions({ + name, + initialState: id({ + stateDescription: "not loaded", + catalogs: [], + selectedTable: undefined, + previewCache: {} + }), + reducers: { + loadingStarted: state => { + state.stateDescription = "loading"; + }, + catalogsLoaded: ( + state, + { payload }: { payload: { catalogs: CatalogMetadata[] } } + ) => { + state.stateDescription = "ready"; + state.catalogs = payload.catalogs; + }, + tableSelectionStarted: ( + state, + { + payload + }: { payload: { catalog: string; namespace: string; table: string } } + ) => { + state.selectedTable = { + ...payload, + isLoading: true, + columns: [], + rows: [] + }; + }, + tablePreviewLoaded: ( + state, + { + payload + }: { + payload: { + columns: IcebergApi.Column[]; + rows: Record[]; + }; + } + ) => { + if (state.selectedTable === undefined) return; + state.selectedTable.isLoading = false; + state.selectedTable.columns = payload.columns; + state.selectedTable.rows = payload.rows; + const key = previewCacheKey( + state.selectedTable.catalog, + state.selectedTable.namespace, + state.selectedTable.table + ); + state.previewCache[key] = { columns: payload.columns, rows: payload.rows }; + }, + tableSelectedFromCache: ( + state, + { + payload + }: { + payload: { + catalog: string; + namespace: string; + table: string; + columns: IcebergApi.Column[]; + rows: Record[]; + }; + } + ) => { + state.selectedTable = { + catalog: payload.catalog, + namespace: payload.namespace, + table: payload.table, + isLoading: false, + columns: payload.columns, + rows: payload.rows + }; + } + } +}); diff --git a/web/src/core/usecases/icebergCatalog/thunks.ts b/web/src/core/usecases/icebergCatalog/thunks.ts new file mode 100644 index 000000000..63df73c7f --- /dev/null +++ b/web/src/core/usecases/icebergCatalog/thunks.ts @@ -0,0 +1,101 @@ +import type { Thunks } from "core/bootstrap"; +import { actions, previewCacheKey } from "./state"; +import type { CatalogMetadata, NamespaceMetadata } from "./state"; +import { id } from "tsafe/id"; + +export const thunks = { + /** + * Fetches the flat table list from all configured catalogs and groups + * them into a hierarchical catalog → namespace → tables structure. + */ + initialize: + () => + async (...args) => { + const [dispatch, , { icebergApi, icebergCatalogConfigs }] = args; + + dispatch(actions.loadingStarted()); + + const listResult = await icebergApi.listAllTables(); + const allTables = + listResult.errorCause !== undefined ? [] : listResult.tables; + + const catalogs: CatalogMetadata[] = icebergCatalogConfigs.map(config => { + const namespaceMap = new Map(); + + for (const entry of allTables.filter(t => t.catalog === config.name)) { + const tables = namespaceMap.get(entry.namespace) ?? []; + tables.push(entry.name); + namespaceMap.set(entry.namespace, tables); + } + + const namespaces: NamespaceMetadata[] = Array.from( + namespaceMap.entries() + ).map(([name, tables]) => ({ name, tables })); + + return id({ + name: config.name, + warehouse: config.warehouse, + endpoint: config.endpoint, + namespaces + }); + }); + + dispatch(actions.catalogsLoaded({ catalogs })); + }, + + /** + * Fetches schema + preview rows for a table in a single DuckDB query. + * No-ops if the same table is already selected. + * Serves from cache if previously fetched. + */ + selectTable: + (params: { catalog: string; namespace: string; table: string }) => + async (...args) => { + const { catalog, namespace, table } = params; + const [dispatch, getState, { icebergApi }] = args; + + const state = getState().icebergCatalog; + + // No-op if already the active table + if ( + state.selectedTable !== undefined && + !state.selectedTable.isLoading && + state.selectedTable.catalog === catalog && + state.selectedTable.namespace === namespace && + state.selectedTable.table === table + ) { + return; + } + + // Serve from cache if available + const cached = state.previewCache[previewCacheKey(catalog, namespace, table)]; + if (cached !== undefined) { + dispatch( + actions.tableSelectedFromCache({ + catalog, + namespace, + table, + columns: cached.columns, + rows: cached.rows + }) + ); + return; + } + + dispatch(actions.tableSelectionStarted({ catalog, namespace, table })); + + const result = await icebergApi.fetchTablePreview({ + catalog, + namespace, + table, + limit: 10 + }); + + dispatch( + actions.tablePreviewLoaded({ + columns: result.errorCause === undefined ? result.columns : [], + rows: result.errorCause === undefined ? result.rows : [] + }) + ); + } +} satisfies Thunks; diff --git a/web/src/core/usecases/index.ts b/web/src/core/usecases/index.ts index bd292355c..ee9ca4b5e 100644 --- a/web/src/core/usecases/index.ts +++ b/web/src/core/usecases/index.ts @@ -24,6 +24,7 @@ import * as dataExplorer from "./dataExplorer"; import * as projectManagement from "./projectManagement"; import * as viewQuotas from "./viewQuotas"; import * as dataCollection from "./dataCollection"; +import * as icebergCatalog from "./icebergCatalog"; export const usecases = { autoLogoutCountdown, @@ -51,5 +52,6 @@ export const usecases = { dataExplorer, projectManagement, viewQuotas, - dataCollection + dataCollection, + icebergCatalog }; diff --git a/web/src/ui/App/LeftBar.tsx b/web/src/ui/App/LeftBar.tsx index a6992b5cf..961a6ab69 100644 --- a/web/src/ui/App/LeftBar.tsx +++ b/web/src/ui/App/LeftBar.tsx @@ -114,6 +114,13 @@ export const LeftBar = memo((props: Props) => { link: routes.dataCollection().link, availability: isDevModeEnabled ? "available" : "not visible" }, + { + itemId: "icebergCatalog", + icon: getIconUrlByName("Storage"), + label: "Catalog Iceberg", + link: routes.icebergCatalog().link, + availability: isDevModeEnabled ? "available" : "not visible" + }, { itemId: "sqlOlapShell", icon: getIconUrlByName("Terminal"), @@ -154,6 +161,8 @@ export const LeftBar = memo((props: Props) => { case "catalog": case "launcher": return "catalog"; + case "icebergCatalog": + return "icebergCatalog"; case "myServices": case "myService": return "myServices"; diff --git a/web/src/ui/pages/icebergCatalog/IcebergCatalogs.tsx b/web/src/ui/pages/icebergCatalog/IcebergCatalogs.tsx new file mode 100644 index 000000000..024a78c84 --- /dev/null +++ b/web/src/ui/pages/icebergCatalog/IcebergCatalogs.tsx @@ -0,0 +1,492 @@ +import { useMemo, useState } from "react"; +import { tss } from "tss"; +import { Text } from "onyxia-ui/Text"; +import { Icon } from "onyxia-ui/Icon"; +import { getIconUrlByName } from "lazy-icons"; +import { CircularProgress } from "onyxia-ui/CircularProgress"; +import Tooltip from "@mui/material/Tooltip"; +import Collapse from "@mui/material/Collapse"; +import { type GridColDef } from "@mui/x-data-grid"; +import { CustomDataGrid, autosizeOptions } from "ui/shared/Datagrid/CustomDataGrid"; +import type { CatalogView, SelectedTableView } from "core/usecases/icebergCatalog"; + +export type Props = { + className?: string; + catalogs: CatalogView[]; + isLoading: boolean; + selectedTable: SelectedTableView | undefined; + onSelectTable: (params: { + catalog: string; + namespace: string; + table: string; + }) => void; +}; + +export function IcebergCatalogs(props: Props) { + const { className, catalogs, isLoading, selectedTable, onSelectTable } = props; + const { classes, cx } = useStyles(); + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (catalogs.length === 0) { + return ( +
+ + No catalog available + + No Iceberg catalog is configured for this region. + +
+ ); + } + + return ( +
+ {catalogs.map(catalog => ( + + ))} +
+ ); +} + +function shortenUrl(url: string): string { + try { + const { hostname, pathname } = new URL(url); + return hostname + (pathname !== "/" ? pathname : ""); + } catch { + return url; + } +} + +function CatalogCard(props: { + catalog: CatalogView; + selectedTable: SelectedTableView | undefined; + onSelectTable: Props["onSelectTable"]; +}) { + const { catalog, selectedTable, onSelectTable } = props; + const { classes } = useStyles(); + const totalTables = catalog.namespaces.reduce((sum, ns) => sum + ns.tables.length, 0); + + return ( +
+
+
+ +
+
+ {catalog.name} +
+ + {catalog.warehouse} + + · + + + {shortenUrl(catalog.endpoint)} + + +
+
+
+ + {catalog.namespaces.length} namespace + {catalog.namespaces.length !== 1 ? "s" : ""} + + + {totalTables} table{totalTables !== 1 ? "s" : ""} + +
+
+
+ {catalog.namespaces.length === 0 ? ( +
+ + No namespaces found. + +
+ ) : ( + catalog.namespaces.map(ns => ( + + )) + )} +
+
+ ); +} + +function NamespaceSection(props: { + namespace: CatalogView["namespaces"][number]; + catalogName: string; + selectedTable: SelectedTableView | undefined; + onSelectTable: Props["onSelectTable"]; +}) { + const { namespace, catalogName, selectedTable, onSelectTable } = props; + const [open, setOpen] = useState(true); + const { classes, cx } = useStyles(); + + return ( +
+ + +
+ {namespace.tables.length === 0 ? ( +
+ + No tables in this namespace. + +
+ ) : ( + namespace.tables.map(table => ( + + )) + )} +
+ +
+ ); +} + +function TableRow(props: { + table: CatalogView["namespaces"][number]["tables"][number]; + catalogName: string; + namespaceName: string; + selectedTable: SelectedTableView | undefined; + onSelectTable: Props["onSelectTable"]; +}) { + const { table, catalogName, namespaceName, selectedTable, onSelectTable } = props; + const { classes, cx } = useStyles(); + + const isSelected = + selectedTable?.catalog === catalogName && + selectedTable.namespace === namespaceName && + selectedTable.table === table.name; + + return ( + <> +
+ onSelectTable({ + catalog: catalogName, + namespace: namespaceName, + table: table.name + }) + } + > + + + {table.name} + +
+ + {isSelected && selectedTable !== undefined && ( + + )} + + + ); +} + +function TablePreviewPanel(props: { selectedTable: SelectedTableView }) { + const { selectedTable } = props; + const { classes } = useStyles(); + + const columns = useMemo( + () => + selectedTable.columns.map(col => ({ + field: col.name, + headerName: col.name, + description: col.rawType, + flex: 1, + minWidth: 100, + sortable: false + })), + [selectedTable.columns] + ); + + const rows = useMemo( + () => selectedTable.rows.map((row, i) => ({ ...row, _id: i })), + [selectedTable.rows] + ); + + return ( +
+ {selectedTable.isLoading ? ( +
+ +
+ ) : ( + row._id as number} + hideFooter + disableColumnMenu + autoHeight + autosizeOnMount + autosizeOptions={autosizeOptions} + /> + )} +
+ ); +} + +const useStyles = tss.withName({ IcebergCatalogs }).create(({ theme }) => ({ + root: { + display: "flex", + flexDirection: "column", + gap: theme.spacing(3) + }, + emptyState: { + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: theme.spacing(2), + padding: theme.spacing(10), + opacity: 0.6, + textAlign: "center" + }, + emptyIcon: { + fontSize: "3rem", + marginBottom: theme.spacing(1) + }, + // ── Catalog card ────────────────────────────────────────────────────── + catalogCard: { + borderRadius: 8, + overflow: "hidden", + backgroundColor: theme.colors.useCases.surfaces.surface1, + boxShadow: theme.shadows[1] + }, + catalogHeader: { + display: "flex", + alignItems: "center", + gap: theme.spacing(3), + padding: theme.spacing({ topBottom: 3, rightLeft: 4 }), + borderBottom: `1px solid ${theme.colors.useCases.typography.textTertiary}` + }, + catalogIconWrapper: { + width: 40, + height: 40, + borderRadius: 8, + backgroundColor: theme.colors.useCases.surfaces.surface2, + display: "flex", + alignItems: "center", + justifyContent: "center", + flexShrink: 0 + }, + catalogIcon: { + fontSize: "1.3rem", + color: theme.colors.useCases.typography.textSecondary + }, + catalogTitleGroup: { + flex: 1, + minWidth: 0, + display: "flex", + flexDirection: "column", + gap: theme.spacing(1) + }, + catalogMeta: { + display: "flex", + alignItems: "center", + gap: theme.spacing(1), + flexWrap: "wrap" + }, + metaText: { + fontFamily: "monospace", + fontSize: "0.72rem", + color: theme.colors.useCases.typography.textSecondary, + backgroundColor: theme.colors.useCases.surfaces.surface2, + padding: "2px 8px", + borderRadius: 4, + maxWidth: 280, + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", + cursor: "default" + }, + metaSep: { + color: theme.colors.useCases.typography.textDisabled, + fontSize: "0.75rem", + userSelect: "none" + }, + catalogBadges: { + display: "flex", + gap: theme.spacing(1), + flexShrink: 0 + }, + badge: { + fontSize: "0.72rem", + fontWeight: 500, + padding: "3px 10px", + borderRadius: 20, + backgroundColor: theme.colors.useCases.surfaces.surface2, + color: theme.colors.useCases.typography.textSecondary, + whiteSpace: "nowrap" + }, + // ── Namespace sections ──────────────────────────────────────────────── + namespaceList: {}, + emptyInner: { + padding: `${theme.spacing(3)} ${theme.spacing(4)}` + }, + namespaceSection: { + borderBottom: `1px solid ${theme.colors.useCases.typography.textTertiary}`, + "&:last-child": { borderBottom: "none" } + }, + namespaceHeader: { + display: "flex", + alignItems: "center", + gap: theme.spacing(2), + width: "100%", + padding: `${theme.spacing(2)} ${theme.spacing(4)}`, + background: "none", + border: "none", + cursor: "pointer", + textAlign: "left", + color: "inherit", + transition: "background-color 0.15s", + "&:hover": { + backgroundColor: theme.colors.useCases.surfaces.surface2 + } + }, + nsIcon: { + fontSize: "1.1rem", + color: theme.colors.useCases.typography.textSecondary, + flexShrink: 0 + }, + nsCountBadge: { + fontSize: "0.68rem", + fontWeight: 600, + padding: "1px 7px", + borderRadius: 20, + backgroundColor: theme.colors.useCases.surfaces.surface2, + color: theme.colors.useCases.typography.textSecondary + }, + nsSpacer: { flex: 1 }, + nsChevron: { + fontSize: "1rem", + color: theme.colors.useCases.typography.textDisabled, + transition: "transform 0.2s ease", + flexShrink: 0 + }, + nsChevronOpen: { + transform: "rotate(90deg)" + }, + // ── Table rows ──────────────────────────────────────────────────────── + tableList: { + marginLeft: `calc(${theme.spacing(4)} + 8px)`, + paddingLeft: theme.spacing(3), + paddingBottom: theme.spacing(1), + borderLeft: `2px solid ${theme.colors.useCases.typography.textTertiary}` + }, + emptyTables: { + padding: `${theme.spacing(2)} 0` + }, + tableRow: { + position: "relative", + display: "flex", + alignItems: "center", + gap: theme.spacing(2), + padding: `${theme.spacing(1)} ${theme.spacing(2)} ${theme.spacing(1)} 0`, + cursor: "pointer", + borderRadius: 6, + transition: "background-color 0.15s", + "&::before": { + content: '""', + position: "absolute", + left: `calc(-1 * ${theme.spacing(3)})`, + top: "50%", + transform: "translateY(-50%)", + width: `calc(${theme.spacing(3)} - 2px)`, + height: 2, + backgroundColor: theme.colors.useCases.typography.textTertiary + }, + "&:hover": { + backgroundColor: theme.colors.useCases.surfaces.surface2 + } + }, + tableRowSelected: { + backgroundColor: `${theme.colors.useCases.typography.textFocus}11`, + "&:hover": { + backgroundColor: `${theme.colors.useCases.typography.textFocus}1a` + } + }, + tableIcon: { + fontSize: "1rem", + color: theme.colors.useCases.typography.textSecondary, + flexShrink: 0 + }, + tableName: { + flex: 1, + minWidth: 0, + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap" + }, + // ── Inline preview panel ────────────────────────────────────────────── + previewPanel: { + marginLeft: `-${theme.spacing(3)}`, + borderTop: `1px solid ${theme.colors.useCases.typography.textTertiary}`, + borderBottom: `1px solid ${theme.colors.useCases.typography.textTertiary}`, + marginBottom: theme.spacing(1) + }, + previewLoading: { + display: "flex", + justifyContent: "center", + padding: theme.spacing(3) + }, + previewGrid: { + border: "none", + borderRadius: 0, + backgroundColor: theme.colors.useCases.surfaces.surface1, + fontSize: "0.8rem" + } +})); diff --git a/web/src/ui/pages/icebergCatalog/Page.tsx b/web/src/ui/pages/icebergCatalog/Page.tsx new file mode 100644 index 000000000..94d6af78b --- /dev/null +++ b/web/src/ui/pages/icebergCatalog/Page.tsx @@ -0,0 +1,65 @@ +import { useEffect } from "react"; +import { PageHeader } from "onyxia-ui/PageHeader"; +import { getIconUrlByName } from "lazy-icons"; +import { Text } from "onyxia-ui/Text"; +import { tss } from "tss"; +import { withLoader } from "ui/tools/withLoader"; +import { enforceLogin } from "ui/shared/enforceLogin"; +import { useCoreState, getCoreSync } from "core"; +import { IcebergCatalogs } from "./IcebergCatalogs"; + +const Page = withLoader({ + loader: enforceLogin, + Component: IcebergCatalogPage +}); + +function IcebergCatalogPage() { + const { classes } = useStyles(); + const { catalogs, isLoading, selectedTable } = useCoreState("icebergCatalog", "view"); + + const { + functions: { icebergCatalog } + } = getCoreSync(); + + useEffect(() => { + icebergCatalog.initialize(); + }, []); + + return ( +
+ + Browse catalogs, their namespaces, and Iceberg tables, then open + any table in the data explorer. + + } + /> +
+ +
+
+ ); +} + +export default Page; + +const useStyles = tss.withName({ IcebergCatalogPage }).create(({ theme }) => ({ + root: { + height: "100%", + display: "flex", + flexDirection: "column" + }, + content: { + flex: 1, + overflow: "auto", + paddingBottom: theme.spacing(4) + } +})); diff --git a/web/src/ui/pages/icebergCatalog/index.ts b/web/src/ui/pages/icebergCatalog/index.ts new file mode 100644 index 000000000..9cf4bc637 --- /dev/null +++ b/web/src/ui/pages/icebergCatalog/index.ts @@ -0,0 +1,3 @@ +import { lazy, memo } from "react"; +export * from "./route"; +export const LazyComponent = memo(lazy(() => import("./Page"))); diff --git a/web/src/ui/pages/icebergCatalog/route.ts b/web/src/ui/pages/icebergCatalog/route.ts new file mode 100644 index 000000000..ae76a9431 --- /dev/null +++ b/web/src/ui/pages/icebergCatalog/route.ts @@ -0,0 +1,7 @@ +import { createGroup, defineRoute } from "type-route"; + +export const routeDefs = { + icebergCatalog: defineRoute(`/iceberg-catalog`) +}; + +export const routeGroup = createGroup(routeDefs); diff --git a/web/src/ui/pages/index.ts b/web/src/ui/pages/index.ts index eb5118f4f..df1b85196 100644 --- a/web/src/ui/pages/index.ts +++ b/web/src/ui/pages/index.ts @@ -15,6 +15,7 @@ import * as sqlOlapShell from "./sqlOlapShell"; import * as dataExplorer from "./dataExplorer"; import * as fileExplorer from "./fileExplorerEntry"; import * as dataCollection from "./dataCollection"; +import * as icebergCatalog from "./icebergCatalog"; export const pages = { account, @@ -31,7 +32,8 @@ export const pages = { sqlOlapShell, dataExplorer, fileExplorer, - dataCollection + dataCollection, + icebergCatalog }; export const { routeDefs } = mergeRouteDefs({ pages }); diff --git a/web/yarn.lock b/web/yarn.lock index 558870e28..9861af5c7 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -7852,16 +7852,8 @@ string-argv@~0.3.2: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: + name string-width-cjs version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -7903,14 +7895,8 @@ stringify-entities@^4.0.0: character-entities-html4 "^2.0.0" character-entities-legacy "^3.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: + name strip-ansi-cjs version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==