diff --git a/.changeset/orange-mayflies-lie.md b/.changeset/orange-mayflies-lie.md deleted file mode 100644 index 37667abb0..000000000 --- a/.changeset/orange-mayflies-lie.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: disable json filters for now diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index a15689dff..156b9b5bc 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -666,7 +666,12 @@ const DBSearchPageFiltersComponent = ({ ...chartConfig, dateRange, }, - keys: [key], + keys: [ + key + .split('.') + .map(v => `\`${v}\``) + .join('.'), + ], limit: 200, disableRowLimit: true, }); diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index efcb88391..f448d5626 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -347,8 +347,6 @@ export class Metadata { column: string; maxKeys?: number; } & TableConnection) { - // HDX-2480 delete line below to reenable json filters - return []; // Need to disable JSON keys for the time being. const cacheKey = metricName ? `${databaseName}.${tableName}.${column}.${metricName}.keys` : `${databaseName}.${tableName}.${column}.keys`; @@ -359,47 +357,53 @@ export class Metadata { const where = metricName ? chSql`WHERE MetricName=${{ String: metricName }}` : ''; - const sql = chSql`WITH all_paths AS - ( - SELECT DISTINCT JSONDynamicPathsWithTypes(${{ Identifier: column }}) as paths - FROM ${tableExpr({ database: databaseName, table: tableName })} ${where} - LIMIT ${{ Int32: maxKeys }} - SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2 - ) - SELECT groupUniqArrayMap(paths) as pathMap - FROM all_paths;`; - const keys = await this.clickhouseClient - .query<'JSON'>({ - query: sql.sql, - query_params: sql.params, - connectionId, - clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', - ...this.getClickHouseSettings(), - }, - }) - .then(res => res.json<{ pathMap: Record }>()) - .then(d => { - const keys: { key: string; chType: string }[] = []; - for (const [key, typeArr] of Object.entries(d.data[0].pathMap)) { - if (!key || !typeArr || !Array.isArray(typeArr)) { - throw new Error( - `Error fetching keys for filters (key: ${key}, typeArr: ${typeArr})`, - ); + try { + // First try to use the Paths column + const sql = chSql`SELECT DISTINCT arrayJoin(${{ Identifier: `${column}Keys` }}) as keys + FROM ${tableExpr({ database: databaseName, table: tableName })} ${where} + LIMIT ${{ Int32: maxKeys }} + SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2`; + + const keys = await this.clickhouseClient + .query({ + query: sql.sql, + query_params: sql.params, + connectionId, + format: 'JSONColumnsWithMetadata', + clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', + ...this.getClickHouseSettings(), + }, + }) + .then(res => res.json()) + .then(d => { + const keys: { key: string; chType: string }[] = []; + for (const key of (d.data as unknown as { keys: string[] }) + .keys) { + if (!key) { + continue; + } + keys.push({ + key: key, + chType: 'String', // Default to String type for path entries + }); } - keys.push({ - key: key, - chType: typeArr[0], - }); - } - return keys; - }); - return keys; + return keys; + }); + return keys; + } catch (error) { + // If the Paths column doesn't exist, return empty array + console.warn( + `Column ${column}Paths not found in table ${databaseName}.${tableName}:`, + error, + ); + return []; + } }, ); } @@ -487,8 +491,6 @@ export class Metadata { }); for (const c of columns) { - // HDX-2480 delete condition below to reenable json filters - if (c.type === 'JSON') continue; fields.push({ path: [c.name], type: c.type,