perf: Implement query chunking for charts

.changeset/soft-donkeys-fetch.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: Implement query chunking for charts

packages/app/src/components/DBTimeChart.tsx

@@ -65,8 +65,9 @@ function DBTimeChartComponent({
   const { data, isLoading, isError, error, isPlaceholderData, isSuccess } =
     useQueriedChartConfig(queriedConfig, {
       placeholderData: (prev: any) => prev,
-      queryKey: [queryKeyPrefix, queriedConfig],
+      queryKey: [queryKeyPrefix, queriedConfig, 'chunked'],
       enabled,
+      enableQueryChunking: true,
     });
 
   useEffect(() => {
@@ -75,7 +76,8 @@ function DBTimeChartComponent({
     }
   }, [isError, isErrorExpanded, errorExpansion]);
 
-  const isLoadingOrPlaceholder = isLoading || isPlaceholderData;
+  const isLoadingOrPlaceholder =
+    isLoading || !data?.isComplete || isPlaceholderData;
   const { data: source } = useSource({ id: sourceId });
 
   const { graphResults, timestampColumn, groupKeys, lineNames, lineColors } =

packages/app/src/components/PatternTable.tsx

@@ -29,10 +29,11 @@ export default function PatternTable({
 
   const [selectedPattern, setSelectedPattern] = useState<Pattern | null>(null);
 
-  const { totalCount, isLoading: isTotalCountLoading } = useSearchTotalCount(
-    totalCountConfig,
-    totalCountQueryKeyPrefix,
-  );
+  const {
+    totalCount,
+    isLoading: isTotalCountLoading,
+    isTotalCountComplete,
+  } = useSearchTotalCount(totalCountConfig, totalCountQueryKeyPrefix);
 
   const {
     data: groupedResults,
@@ -46,7 +47,8 @@ export default function PatternTable({
     totalCount,
   });
 
-  const isLoading = isTotalCountLoading || isGroupedPatternsLoading;
+  const isLoading =
+    isTotalCountLoading || !isTotalCountComplete || isGroupedPatternsLoading;
 
   const sortedGroupedResults = useMemo(() => {
     return Object.values(groupedResults).sort(

packages/app/src/components/SearchTotalCountChart.tsx

@@ -10,7 +10,7 @@ export function useSearchTotalCount(
   config: ChartConfigWithDateRange,
   queryKeyPrefix: string,
 ) {
-  // copied from DBTimeChart
+  // queriedConfig, queryKey, and enableQueryChunking match DBTimeChart so that react query can de-dupe these queries.
   const { granularity } = useTimeChartSettings(config);
   const queriedConfig = {
     ...config,
@@ -22,12 +22,15 @@ export function useSearchTotalCount(
     isLoading,
     isError,
   } = useQueriedChartConfig(queriedConfig, {
-    queryKey: [queryKeyPrefix, queriedConfig],
+    queryKey: [queryKeyPrefix, queriedConfig, 'chunked'],
     staleTime: 1000 * 60 * 5,
     refetchOnWindowFocus: false,
     placeholderData: keepPreviousData, // no need to flash loading state when in live tail
+    enableQueryChunking: true,
   });
 
+  const isTotalCountComplete = !!totalCountData?.isComplete;
+
   const totalCount = useMemo(() => {
     return totalCountData?.data?.reduce(
       (p: number, v: any) => p + Number.parseInt(v['count()']),
@@ -39,6 +42,7 @@ export function useSearchTotalCount(
     totalCount,
     isLoading,
     isError,
+    isTotalCountComplete,
   };
 }