Create evals from UI (#597)

Author: chelojimenez

client/src/components/EvalsTab.tsx

@@ -3,17 +3,26 @@ import { useAuth } from "@workos-inc/authkit-react";
 import { useConvexAuth, useQuery } from "convex/react";
 import { FlaskConical } from "lucide-react";
 import { EmptyState } from "@/components/ui/empty-state";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import type { EvalCase, EvalIteration, EvalSuite } from "./evals/types";
 import { aggregateSuite } from "./evals/helpers";
 import { SuitesOverview } from "./evals/suites-overview";
 import { SuiteIterationsView } from "./evals/suite-iterations-view";
+import { EvalRunner } from "./evals/eval-runner";
+import { useChat } from "@/hooks/use-chat";
 
 export function EvalsTab() {
   const { isAuthenticated, isLoading } = useConvexAuth();
   const { user } = useAuth();
 
   const [selectedSuiteId, setSelectedSuiteId] = useState<string | null>(null);
 
+  const { availableModels } = useChat({
+    systemPrompt: "",
+    temperature: 1,
+    selectedServers: [],
+  });
+
   // Fetch overview data for authenticated user - only suites with metadata
   const enableOverviewQuery = isAuthenticated && !!user;
   const overviewData = useQuery(
@@ -106,40 +115,53 @@ export function EvalsTab() {
   return (
     <div className="p-6 space-y-6">
       <div className="flex items-center justify-between">
-        <div className="flex items-center space-x-2">
-          <h1 className="text-2xl font-bold">Evals</h1>
-          {overviewData?.metadata && (
-            <div className="text-sm text-muted-foreground">
-              {overviewData.metadata.iterationsPassed} passed ·{" "}
-              {overviewData.metadata.iterationsFailed} failed
+        <h1 className="text-2xl font-bold">Evals</h1>
+        {overviewData?.metadata && (
+          <div className="text-sm text-muted-foreground">
+            {overviewData.metadata.iterationsPassed} passed ·{" "}
+            {overviewData.metadata.iterationsFailed} failed
+          </div>
+        )}
+      </div>
+
+      <Tabs defaultValue="results" className="w-full">
+        <TabsList>
+          <TabsTrigger value="results">Results</TabsTrigger>
+          <TabsTrigger value="create">Create Run</TabsTrigger>
+        </TabsList>
+
+        <TabsContent value="results" className="mt-6">
+          {!selectedSuite ? (
+            <SuitesOverview
+              suites={suites || []}
+              onSelectSuite={setSelectedSuiteId}
+            />
+          ) : isSuiteDetailsLoading ? (
+            <div className="flex items-center justify-center h-64">
+              <div className="text-center">
+                <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary mx-auto" />
+                <p className="mt-4 text-muted-foreground">
+                  Loading suite details...
+                </p>
+              </div>
             </div>
+          ) : (
+            <SuiteIterationsView
+              suite={selectedSuite}
+              cases={suiteDetails?.testCases || []}
+              iterations={iterationsForSelectedSuite}
+              aggregate={suiteAggregate}
+              onBack={() => setSelectedSuiteId(null)}
+            />
           )}
-        </div>
-      </div>
+        </TabsContent>
 
-      {!selectedSuite ? (
-        <SuitesOverview
-          suites={suites || []}
-          onSelectSuite={setSelectedSuiteId}
-        />
-      ) : isSuiteDetailsLoading ? (
-        <div className="flex items-center justify-center h-64">
-          <div className="text-center">
-            <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary mx-auto" />
-            <p className="mt-4 text-muted-foreground">
-              Loading suite details...
-            </p>
+        <TabsContent value="create" className="mt-6">
+          <div className="max-w-4xl">
+            <EvalRunner availableModels={availableModels} inline={true} />
           </div>
-        </div>
-      ) : (
-        <SuiteIterationsView
-          suite={selectedSuite}
-          cases={suiteDetails?.testCases || []}
-          iterations={iterationsForSelectedSuite}
-          aggregate={suiteAggregate}
-          onBack={() => setSelectedSuiteId(null)}
-        />
-      )}
+        </TabsContent>
+      </Tabs>
     </div>
   );
 }

client/src/components/chat/model-selector.tsx

@@ -20,6 +20,7 @@ interface ModelSelectorProps {
   onModelChange: (model: ModelDefinition) => void;
   disabled?: boolean;
   isLoading?: boolean;
+  hideProvidedModels?: boolean;
 }
 
 // Helper function to group models by provider
@@ -62,6 +63,7 @@ export function ModelSelector({
   onModelChange,
   disabled,
   isLoading,
+  hideProvidedModels = false,
 }: ModelSelectorProps) {
   const [isModelSelectorOpen, setIsModelSelectorOpen] = useState(false);
   const currentModelData = currentModel;
@@ -73,9 +75,9 @@ export function ModelSelector({
   // Get sorted provider keys for consistent ordering
   const sortedProviders = Array.from(groupedModels.keys()).sort();
   const MCPJAM_PROVIDERS: ModelProvider[] = ["meta"];
-  const mcpjamProviders = sortedProviders.filter((p) =>
-    MCPJAM_PROVIDERS.includes(p),
-  );
+  const mcpjamProviders = hideProvidedModels
+    ? []
+    : sortedProviders.filter((p) => MCPJAM_PROVIDERS.includes(p));
   const otherProviders = sortedProviders.filter(
     (p) => !MCPJAM_PROVIDERS.includes(p),
   );

client/src/components/evals/SuiteRow.tsx

@@ -99,7 +99,7 @@ export function SuiteRow({ suite, onSelectSuite }: SuiteRowProps) {
             passed={aggregate.totals.passed}
             failed={aggregate.totals.failed}
             cancelled={aggregate.totals.cancelled}
-            pending={0}
+            pending={aggregate.totals.pending}
           />
         ) : (
           <span className="text-xs text-muted-foreground">Loading...</span>