diff --git a/src/chunking/merger.ts b/src/chunking/merger.ts index c3a23a9..3a4c067 100644 --- a/src/chunking/merger.ts +++ b/src/chunking/merger.ts @@ -5,13 +5,13 @@ export function mergeViolations( ): SemiObjectiveItem[] { const all = chunkViolations.flat(); - // Deduplicate using composite key (quoted_text + description + analysis) + // Deduplicate using composite key (quoted_text + description + message) const seen = new Set(); return all.filter((v) => { const key = [ v.quoted_text?.toLowerCase().trim() || "", v.description?.toLowerCase().trim() || "", - v.analysis?.toLowerCase().trim() || "", + v.message?.toLowerCase().trim() || "", ].join("|"); if (seen.has(key)) return false; seen.add(key); diff --git a/src/cli/commands.ts b/src/cli/commands.ts index c8f0b8f..ae02d41 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -1,26 +1,31 @@ -import type { Command } from 'commander'; -import { existsSync } from 'fs'; -import * as path from 'path'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; -import { createProvider } from '../providers/provider-factory'; -import { PerplexitySearchProvider } from '../providers/perplexity-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import { loadConfig } from '../boundaries/config-loader'; -import { loadStyleGuide } from '../boundaries/style-guide-loader'; -import { loadRuleFile, type PromptFile } from '../prompts/prompt-loader'; -import { RulePackLoader } from '../boundaries/rule-pack-loader'; -import { PresetLoader } from '../config/preset-loader'; -import { printGlobalSummary, printTokenUsage } from '../output/reporter'; -import { DefaultRequestBuilder } from '../providers/request-builder'; -import { loadDirective } from '../prompts/directive-loader'; -import { resolveTargets } from '../scan/file-resolver'; -import { parseCliOptions, parseEnvironment } from '../boundaries/index'; -import { handleUnknownError } from '../errors/index'; -import { evaluateFiles } from './orchestrator'; -import { OutputFormat } from './types'; -import { DEFAULT_CONFIG_FILENAME, STYLE_GUIDE_FILENAME, ZERO_CONFIG_PACK_NAME, ZERO_CONFIG_PROMPT_ID } from '../config/constants'; -import { Severity, Type } from '../evaluators/types'; +import type { Command } from "commander"; +import { existsSync } from "fs"; +import * as path from "path"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; +import { createProvider } from "../providers/provider-factory"; +import { PerplexitySearchProvider } from "../providers/perplexity-provider"; +import type { SearchProvider } from "../providers/search-provider"; +import { loadConfig } from "../boundaries/config-loader"; +import { loadStyleGuide } from "../boundaries/style-guide-loader"; +import { loadRuleFile, type PromptFile } from "../prompts/prompt-loader"; +import { RulePackLoader } from "../boundaries/rule-pack-loader"; +import { PresetLoader } from "../config/preset-loader"; +import { printGlobalSummary, printTokenUsage } from "../output/reporter"; +import { DefaultRequestBuilder } from "../providers/request-builder"; +import { loadDirective } from "../prompts/directive-loader"; +import { resolveTargets } from "../scan/file-resolver"; +import { parseCliOptions, parseEnvironment } from "../boundaries/index"; +import { handleUnknownError } from "../errors/index"; +import { evaluateFiles } from "./orchestrator"; +import { OutputFormat } from "./types"; +import { + DEFAULT_CONFIG_FILENAME, + STYLE_GUIDE_FILENAME, + ZERO_CONFIG_PACK_NAME, + ZERO_CONFIG_PROMPT_ID, +} from "../config/constants"; +import { Severity, Type } from "../evaluators/types"; // eslint-disable-next-line @typescript-eslint/naming-convention const __filename = fileURLToPath(import.meta.url); @@ -33,12 +38,23 @@ const __dirname = dirname(__filename); */ export function registerMainCommand(program: Command): void { program - .option('-v, --verbose', 'Enable verbose logging') - .option('--show-prompt', 'Print full prompt and injected content') - .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') - .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') - .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) - .argument('[paths...]', 'files or directories to check (required)') + .option("-v, --verbose", "Enable verbose logging") + .option("--show-prompt", "Print full prompt and injected content") + .option( + "--show-prompt-trunc", + "Print truncated prompt/content previews (500 chars)" + ) + .option("--suggest", "Show fix suggestions for each issue") + .option( + "--output ", + "Output format: line (default), json, or vale-json, rdjson", + "line" + ) + .option( + "--config ", + `Path to custom ${DEFAULT_CONFIG_FILENAME} config file` + ) + .argument("[paths...]", "files or directories to check (required)") .action(async (paths: string[] = []) => { // Require explicit paths to prevent accidental full directory scans // Users must provide specific files, directories, or wildcards (e.g., `vectorlint *`) @@ -52,7 +68,7 @@ export function registerMainCommand(program: Command): void { try { cliOptions = parseCliOptions(program.opts()); } catch (e: unknown) { - const err = handleUnknownError(e, 'Parsing CLI options'); + const err = handleUnknownError(e, "Parsing CLI options"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -62,9 +78,9 @@ export function registerMainCommand(program: Command): void { try { env = parseEnvironment(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Validating environment variables'); + const err = handleUnknownError(e, "Validating environment variables"); console.error(`Error: ${err.message}`); - console.error('Please set these in your .env file or environment.'); + console.error("Please set these in your .env file or environment."); process.exit(1); } @@ -73,7 +89,7 @@ export function registerMainCommand(program: Command): void { try { directive = loadDirective(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading directive'); + const err = handleUnknownError(e, "Loading directive"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -81,7 +97,9 @@ export function registerMainCommand(program: Command): void { // Load style guide (VECTORLINT.md) const styleGuide = loadStyleGuide(process.cwd()); if (styleGuide.content && cliOptions.verbose) { - console.log(`[vectorlint] Loaded style guide from ${STYLE_GUIDE_FILENAME} (${styleGuide.tokenEstimate} estimated tokens)`); + console.log( + `[vectorlint] Loaded style guide from ${STYLE_GUIDE_FILENAME} (${styleGuide.tokenEstimate} estimated tokens)` + ); } const provider = createProvider( @@ -104,7 +122,7 @@ export function registerMainCommand(program: Command): void { try { config = loadConfig(process.cwd(), cliOptions.config); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading configuration'); + const err = handleUnknownError(e, "Loading configuration"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -118,15 +136,19 @@ export function registerMainCommand(program: Command): void { const prompts: PromptFile[] = []; try { - const presetsDir = path.resolve(__dirname, '../presets'); + const presetsDir = path.resolve(__dirname, "../presets"); const presetLoader = new PresetLoader(presetsDir); const loader = new RulePackLoader(presetLoader); const packs = await loader.listAllPacks(rulesPath); if (packs.length === 0 && cliOptions.verbose) { - console.warn(`[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath} or presets.`); - console.warn(`[vectorlint] Please organize your rules into subdirectories or use a valid preset.`); + console.warn( + `[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath} or presets.` + ); + console.warn( + `[vectorlint] Please organize your rules into subdirectories or use a valid preset.` + ); } for (const pack of packs) { @@ -136,7 +158,8 @@ export function registerMainCommand(program: Command): void { for (const filePath of rulePaths) { const result = loadRuleFile(filePath, pack.name); if (result.warning) { - if (cliOptions.verbose) console.warn(`[vectorlint] ${result.warning}`); + if (cliOptions.verbose) + console.warn(`[vectorlint] ${result.warning}`); } if (result.prompt) { prompts.push(result.prompt); @@ -147,21 +170,32 @@ export function registerMainCommand(program: Command): void { if (prompts.length === 0) { if (styleGuide.content) { if (cliOptions.verbose) { - console.log('[vectorlint] No rules found, but VECTORLINT.md exists. Running in zero-config mode.'); + console.log( + "[vectorlint] No rules found, but VECTORLINT.md exists. Running in zero-config mode." + ); } - prompts.push(createStyleGuidePrompt(styleGuide.path || path.resolve(process.cwd(), STYLE_GUIDE_FILENAME))); + prompts.push( + createStyleGuidePrompt( + styleGuide.path || + path.resolve(process.cwd(), STYLE_GUIDE_FILENAME) + ) + ); } else { if (rulesPath) { - console.error(`Error: no .md rules found in ${rulesPath} or presets.`); + console.error( + `Error: no .md rules found in ${rulesPath} or presets.` + ); } else { - console.error('Error: no rules found. Either set RulesPath in config or configure RunRules with a valid preset.'); + console.error( + "Error: no rules found. Either set RulesPath in config or configure RunRules with a valid preset." + ); } process.exit(1); } } } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading prompts'); + const err = handleUnknownError(e, "Loading prompts"); console.error(`Error: failed to load prompts: ${err.message}`); process.exit(1); } @@ -177,26 +211,27 @@ export function registerMainCommand(program: Command): void { configDir: config.configDir, }); } catch (e: unknown) { - const err = handleUnknownError(e, 'Resolving target files'); + const err = handleUnknownError(e, "Resolving target files"); console.error(`Error: failed to resolve target files: ${err.message}`); process.exit(1); } if (targets.length === 0) { - console.error('Error: no target files found to evaluate.'); + console.error("Error: no target files found to evaluate."); process.exit(1); } - - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY + const searchProvider: SearchProvider | undefined = process.env + .PERPLEXITY_API_KEY ? new PerplexitySearchProvider({ debug: false }) : undefined; const outputFormat = cliOptions.output as OutputFormat; if (!Object.values(OutputFormat).includes(outputFormat)) { - console.error(`Error: Invalid output format '${cliOptions.output}'. Valid options: line, json, vale-json, rdjson`); + console.error( + `Error: Invalid output format '${cliOptions.output}'. Valid options: line, json, vale-json, rdjson` + ); process.exit(1); } @@ -210,6 +245,7 @@ export function registerMainCommand(program: Command): void { verbose: cliOptions.verbose, outputFormat: outputFormat, scanPaths: config.scanPaths, + suggest: cliOptions.suggest, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, outputPricePerMillion: env.OUTPUT_PRICE_PER_MILLION, @@ -217,7 +253,7 @@ export function registerMainCommand(program: Command): void { }); // Print global summary (only for line format) - if (cliOptions.output === 'line') { + if (cliOptions.output === "line") { if (result.tokenUsage) { printTokenUsage(result.tokenUsage); } @@ -230,7 +266,9 @@ export function registerMainCommand(program: Command): void { } // Exit with appropriate code - process.exit(result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0); + process.exit( + result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0 + ); }); } @@ -239,11 +277,11 @@ function createStyleGuidePrompt(fullPath: string): PromptFile { id: ZERO_CONFIG_PROMPT_ID, filename: STYLE_GUIDE_FILENAME, fullPath, - body: 'Evaluate the provided content against the attached Global Style Guide. Report any violations of the rules defined in the style guide.', + body: "Evaluate the provided content against the attached Global Style Guide. Report any violations of the rules defined in the style guide.", pack: ZERO_CONFIG_PACK_NAME, meta: { id: ZERO_CONFIG_PROMPT_ID, - name: 'Style Guide Compliance', + name: "Style Guide Compliance", evaluator: Type.BASE, severity: Severity.WARNING, }, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 7c660c8..fa5c5ee 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,31 +1,48 @@ -import { readFileSync } from 'fs'; -import * as path from 'path'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; -import { RdJsonFormatter } from '../output/rdjson-formatter'; -import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; -import { isSubjectiveResult } from '../prompts/schema'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; -import { OutputFormat } from './types'; -import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess -} from './types'; +import { readFileSync } from "fs"; +import * as path from "path"; +import type { PromptFile } from "../prompts/prompt-loader"; +import { ScanPathResolver } from "../boundaries/scan-path-resolver"; +import { + ValeJsonFormatter, + type JsonIssue, +} from "../output/vale-json-formatter"; +import { + JsonFormatter, + type Issue, + type ScoreComponent, +} from "../output/json-formatter"; +import { RdJsonFormatter } from "../output/rdjson-formatter"; import { - calculateCost, - TokenUsageStats -} from '../providers/token-usage'; + printFileHeader, + printIssueRow, + printEvaluationSummaries, + type EvaluationSummary, +} from "../output/reporter"; +import { checkTarget } from "../prompts/target"; +import { isSubjectiveResult } from "../prompts/schema"; +import { handleUnknownError, MissingDependencyError } from "../errors/index"; +import { createEvaluator } from "../evaluators/index"; +import { Type, Severity } from "../evaluators/types"; +import { OutputFormat } from "./types"; +import type { + EvaluationOptions, + EvaluationResult, + ErrorTrackingResult, + ReportIssueParams, + ProcessViolationsParams, + ProcessCriterionParams, + ProcessCriterionResult, + ValidationParams, + ProcessPromptResultParams, + RunPromptEvaluationParams, + RunPromptEvaluationResult, + EvaluateFileParams, + EvaluateFileResult, + RunPromptEvaluationResultSuccess, +} from "./types"; +import { calculateCost, TokenUsageStats } from "../providers/token-usage"; import { locateQuotedText } from "../output/location"; - /* * Returns the evaluator type, defaulting to 'base' if not specified. */ @@ -47,7 +64,7 @@ function buildRuleName( if (criterionId) { parts.push(criterionId); } - return parts.join('.'); + return parts.join("."); } /* @@ -157,6 +174,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { outputFormat, jsonFormatter, verbose, + suggest, } = params; let hadOperationalErrors = false; @@ -175,7 +193,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { for (const v of violations) { if (!v) continue; - const rowSummary = (v.analysis || "").trim(); + const rowSummary = (v.message || "").trim(); try { const locWithMatch = locateQuotedText( @@ -224,6 +242,10 @@ function locateAndReportViolations(params: ProcessViolationsParams): { } // Report only verified, unique violations + // For line output, only show suggestions when --suggest flag is used + // For JSON formats, always include suggestions (machine consumption) + const includeSuggestion = outputFormat !== OutputFormat.Line || suggest; + for (const { v, line, @@ -240,7 +262,8 @@ function locateAndReportViolations(params: ProcessViolationsParams): { ruleName, outputFormat, jsonFormatter, - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), + ...(includeSuggestion && + v.suggestion !== undefined && { suggestion: v.suggestion }), scoreText, match: matchedText, }); @@ -268,6 +291,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose, + suggest, } = params; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -276,13 +300,13 @@ function extractAndReportCriterion( const criterionId = exp.id ? String(exp.id) : exp.name - ? String(exp.name) + ? String(exp.name) .replace(/[^A-Za-z0-9]+/g, " ") .split(" ") .filter(Boolean) .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) .join("") - : ""; + : ""; const ruleName = buildRuleName(packName, promptId, criterionId); const weightNum = exp.weight || 1; @@ -302,6 +326,8 @@ function extractAndReportCriterion( expTargetSpec?.suggestion || metaTargetSpec?.suggestion || "Add the required target section."; + // For line output, only show suggestions when --suggest flag is used + const includeSuggestion = outputFormat !== OutputFormat.Line || suggest; reportIssue({ file: relFile, line: 1, @@ -311,7 +337,7 @@ function extractAndReportCriterion( ruleName, outputFormat, jsonFormatter, - suggestion, + ...(includeSuggestion && { suggestion }), scoreText: "nil", match: "", }); @@ -400,7 +426,8 @@ function extractAndReportCriterion( quoted_text?: string; context_before?: string; context_after?: string; - analysis?: string; + issue?: string; + message?: string; suggestion?: string; }>, content, @@ -411,6 +438,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; @@ -552,6 +580,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose, + suggest, } = params; const meta = promptFile.meta; const promptId = (meta.id || "").toString(); @@ -567,7 +596,10 @@ function routePromptResult( const violationCount = result.violations.length; // Group violations by criterionName - const violationsByCriterion = new Map(); + const violationsByCriterion = new Map< + string | undefined, + typeof result.violations + >(); for (const v of result.violations) { const criterionName = v.criterionName; if (!violationsByCriterion.has(criterionName)) { @@ -584,7 +616,7 @@ function routePromptResult( // Find criterion ID from meta let criterionId: string | undefined; if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); + const criterion = meta.criteria.find((c) => c.name === criterionName); criterionId = criterion?.id; } @@ -597,12 +629,14 @@ function routePromptResult( relFile, severity, ruleName, - scoreText: '', + scoreText: "", outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + hadOperationalErrors = + hadOperationalErrors || violationResult.hadOperationalErrors; if (severity === Severity.ERROR) { totalErrors += violations.length; @@ -613,7 +647,12 @@ function routePromptResult( } // If no violations but we have a message (JSON output), report it - if (violationCount === 0 && (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson) && result.message) { + if ( + violationCount === 0 && + (outputFormat === OutputFormat.Json || + outputFormat === OutputFormat.ValeJson) && + result.message + ) { const ruleName = buildRuleName(promptFile.pack, promptId, undefined); reportIssue({ file: relFile, @@ -671,6 +710,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); promptErrors += criterionResult.errors; @@ -741,7 +781,6 @@ async function runPromptEvaluation( ); const result = await evaluator.evaluate(relFile, content); - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; return resultObj; @@ -766,6 +805,7 @@ async function evaluateFile( scanPaths, outputFormat = OutputFormat.Line, verbose, + suggest, } = options; let hadOperationalErrors = false; @@ -873,6 +913,7 @@ async function evaluateFile( outputFormat, jsonFormatter, verbose, + suggest: !!suggest, }); totalErrors += promptResult.errors; totalWarnings += promptResult.warnings; @@ -902,7 +943,7 @@ async function evaluateFile( requestFailures, hadOperationalErrors, hadSeverityErrors, - tokenUsage: tokenUsageStats + tokenUsage: tokenUsageStats, }; } @@ -975,10 +1016,13 @@ export async function evaluateFiles( }; // Calculate cost if pricing is configured - const cost = calculateCost({ - inputTokens: totalInputTokens, - outputTokens: totalOutputTokens - }, options.pricing); + const cost = calculateCost( + { + inputTokens: totalInputTokens, + outputTokens: totalOutputTokens, + }, + options.pricing + ); if (cost !== undefined) { tokenUsage.totalCost = cost; } diff --git a/src/cli/types.ts b/src/cli/types.ts index 9666d5e..0948981 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,139 +1,148 @@ -import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import type { EvaluationSummary } from '../output/reporter'; -import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; -import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; -import { Severity } from '../evaluators/types'; -import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { PromptFile } from "../prompts/prompt-loader"; +import type { LLMProvider } from "../providers/llm-provider"; +import type { SearchProvider } from "../providers/search-provider"; +import type { + PromptMeta, + PromptCriterionSpec, +} from "../schemas/prompt-schemas"; +import type { FilePatternConfig } from "../boundaries/file-section-parser"; +import type { EvaluationSummary } from "../output/reporter"; +import { ValeJsonFormatter } from "../output/vale-json-formatter"; +import { JsonFormatter, type ScoreComponent } from "../output/json-formatter"; +import { RdJsonFormatter } from "../output/rdjson-formatter"; +import type { + EvaluationResult as PromptEvaluationResult, + SubjectiveResult, +} from "../prompts/schema"; +import { Severity } from "../evaluators/types"; +import type { TokenUsageStats, PricingConfig } from "../providers/token-usage"; export enum OutputFormat { - Line = "line", - Json = "json", - ValeJson = "vale-json", - RdJson = "rdjson", + Line = "line", + Json = "json", + ValeJson = "vale-json", + RdJson = "rdjson", } export interface EvaluationOptions { - prompts: PromptFile[]; - rulesPath: string | undefined; - provider: LLMProvider; - searchProvider?: SearchProvider; - concurrency: number; - verbose: boolean; - scanPaths: FilePatternConfig[]; - outputFormat?: OutputFormat; - pricing?: PricingConfig; + prompts: PromptFile[]; + rulesPath: string | undefined; + provider: LLMProvider; + searchProvider?: SearchProvider; + concurrency: number; + verbose: boolean; + scanPaths: FilePatternConfig[]; + outputFormat?: OutputFormat; + pricing?: PricingConfig; + suggest?: boolean; } export interface EvaluationResult { - totalFiles: number; - totalErrors: number; - totalWarnings: number; - requestFailures: number; - hadOperationalErrors: boolean; - hadSeverityErrors: boolean; - tokenUsage?: TokenUsageStats; + totalFiles: number; + totalErrors: number; + totalWarnings: number; + requestFailures: number; + hadOperationalErrors: boolean; + hadSeverityErrors: boolean; + tokenUsage?: TokenUsageStats; } export interface ErrorTrackingResult { - errors: number; - warnings: number; - hadOperationalErrors: boolean; - hadSeverityErrors: boolean; - scoreEntries?: EvaluationSummary[]; + errors: number; + warnings: number; + hadOperationalErrors: boolean; + hadSeverityErrors: boolean; + scoreEntries?: EvaluationSummary[]; } export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; + content: string; + relFile: string; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + verbose?: boolean; + suggest?: boolean; } export interface ReportIssueParams { - file: string; - line: number; - column: number; - severity: Severity; - summary: string; - ruleName: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - suggestion?: string; - scoreText?: string; - match?: string; + file: string; + line: number; + column: number; + severity: Severity; + summary: string; + ruleName: string; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + suggestion?: string; + scoreText?: string; + match?: string; } export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; + violations: Array<{ + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + issue?: string; + message?: string; + suggestion?: string; + }>; + severity: Severity; + ruleName: string; + scoreText: string; } export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: SubjectiveResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; + exp: PromptCriterionSpec; + result: SubjectiveResult; + packName: string; + promptId: string; + promptFilename: string; + meta: PromptMeta; } export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; + userScore: number; + maxScore: number; + scoreEntry: { id: string; scoreText: string; score?: number }; + scoreComponent?: ScoreComponent; } export interface ValidationParams { - meta: PromptMeta; - result: SubjectiveResult; + meta: PromptMeta; + result: SubjectiveResult; } export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; + promptFile: PromptFile; + result: PromptEvaluationResult; } export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; + promptFile: PromptFile; + relFile: string; + content: string; + provider: LLMProvider; + searchProvider?: SearchProvider; } export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; + ok: true; + result: PromptEvaluationResult; } export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; + | RunPromptEvaluationResultSuccess + | { ok: false; error: Error }; export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + file: string; + options: EvaluationOptions; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface EvaluateFileResult extends ErrorTrackingResult { - requestFailures: number; - tokenUsage?: TokenUsageStats; + requestFailures: number; + tokenUsage?: TokenUsageStats; } diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index 64834ba..2122da6 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -10,7 +10,7 @@ import path from "path"; */ const DEFAULT_DIRECTIVE = ` List every finding you detect. -- If the issue occurs within a sentence, quote the offending word or short phrase as evidence (example: "leverage" is an AI buzzword). +- If the issue occurs within a sentence, quote the offending word or short phrase as evidence. - If a sentence contains multiple issues, report each as a separate violation. **IMPORTANT**: The input has line numbers prepended (format: "123\\ttext"). Use these line numbers when reporting issues. @@ -21,10 +21,28 @@ For each finding, provide: The text must exist verbatim in 'Input' as a direct substring (excluding the line number prefix). - context_before: 10–20 exact characters immediately before quoted_text (or empty string if at start) - context_after: 10–20 exact characters immediately after quoted_text (or empty string if at end) -- analysis: a specific, concrete explanation of the issue (respect word limit) -- suggestion: a succinct, imperative fix (max 15 words) +- issue: The problem (NEVER include the quoted_text here). Aim for 5-12 words. +- message: The user-facing output. Format based on scope: + - WORD-level (buzzwords, specific terms): "quoted_text" + issue → e.g., "ensure" is a common AI buzzword + - SENTENCE/SECTION/DOCUMENT-level: issue only → e.g., Opens with meta-commentary +- suggestion: a succinct, imperative fix - When a criterion has no findings, provide one short positive remark describing compliance. +**MESSAGE EXAMPLES** (follow this style exactly): + +WORD-level (quoted_text is 1-4 words) — message INCLUDES quoted_text: +✓ quoted_text: "ensure" → message: "ensure" is a common AI buzzword +✓ quoted_text: "game-changer" → message: "game-changer" is a buzzword phrase + +SENTENCE-level (quoted_text is >4 words) — message does NOT include quoted_text: +✓ quoted_text: "LLM-based chunking uses a large language model..." → message: Buzzword-heavy sentence +✓ quoted_text: "In this article, we will explore..." → message: Opens with meta-commentary + +SECTION-level — message references section names: +✓ message: Overlaps with "What is Chunking?" on context + +**CRITICAL: If quoted_text is longer than 4 words, do NOT include it in the message.** + ***CRITICAL RULES*** 1. Go through the Input before anything else, and show your step-by-step reasoning or the approach you'll take to accomplish the task. @@ -32,7 +50,8 @@ For each finding, provide: 3. If you cannot find a verbatim match in 'Input', do NOT report it - skip that finding entirely. 4. Do NOT infer or hypothesize issues. Only report what you can directly quote from 'Input'. 5. Fabricating quotes that don't exist in 'Input' is equivalent to failure. -6. The line number you report must match the prepended number on that line in Input.`; +6. The line number you report must match the prepended number on that line in Input. +7. When comparing/contrasting sections, ALWAYS name the sections. NEVER say "both sections" without naming them.`; export function loadDirective(cwd: string = process.cwd()): string { // 1) Project override diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 96374f5..d3a20a1 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -33,14 +33,16 @@ export function buildSubjectiveLLMSchema() { quoted_text: { type: "string" }, context_before: { type: "string" }, context_after: { type: "string" }, - analysis: { type: "string" }, + issue: { type: "string" }, + message: { type: "string" }, suggestion: { type: "string" }, }, required: [ "quoted_text", "context_before", "context_after", - "analysis", + "issue", + "message", "suggestion", ], }, @@ -74,7 +76,8 @@ export function buildSemiObjectiveLLMSchema() { context_before: { type: "string" }, context_after: { type: "string" }, description: { type: "string" }, - analysis: { type: "string" }, + issue: { type: "string" }, + message: { type: "string" }, suggestion: { type: "string" }, }, required: [ @@ -82,7 +85,8 @@ export function buildSemiObjectiveLLMSchema() { "context_before", "context_after", "description", - "analysis", + "issue", + "message", "suggestion", ], }, @@ -103,7 +107,8 @@ export type SubjectiveLLMResult = { quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; }>; @@ -112,8 +117,9 @@ export type SubjectiveLLMResult = { export type SemiObjectiveLLMResult = { violations: Array<{ description: string; - analysis: string; - suggestion?: string; + issue: string; + message: string; + suggestion: string; quoted_text?: string; context_before?: string; context_after?: string; @@ -135,7 +141,8 @@ export type SubjectiveResult = { quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; }>; @@ -144,7 +151,8 @@ export type SubjectiveResult = { export type SemiObjectiveItem = { description: string; - analysis: string; + issue: string; + message: string; suggestion?: string; quoted_text?: string; context_before?: string; @@ -160,7 +168,8 @@ export type SemiObjectiveResult = { severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; violations: Array<{ - analysis: string; + issue: string; + message: string; suggestion?: string; quoted_text?: string; context_before?: string; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 7e4f74c..73ecb1c 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,11 +1,12 @@ -import { z } from 'zod'; +import { z } from "zod"; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ verbose: z.boolean().default(false), showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), - output: z.enum(['line', 'json', 'vale-json', 'rdjson']).default('line'), + suggest: z.boolean().default(false), + output: z.enum(["line", "json", "vale-json", "rdjson"]).default("line"), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index 4d9c93e..07123fe 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -54,7 +54,8 @@ export function calculateSemiObjectiveScore( // Map items to violation format const mappedViolations = violations.map((item) => ({ - analysis: item.analysis, + issue: item.issue, + message: item.message, ...(item.suggestion && { suggestion: item.suggestion }), ...(item.quoted_text && { quoted_text: item.quoted_text }), ...(item.context_before && { context_before: item.context_before }), @@ -177,7 +178,8 @@ export function averageSubjectiveScores( quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; summaries: string[]; @@ -214,7 +216,8 @@ export function averageSubjectiveScores( quoted_text: v.quoted_text || "", context_before: v.context_before || "", context_after: v.context_after || "", - analysis: v.analysis || "", + issue: v.issue || "", + message: v.message || "", suggestion: v.suggestion || "", }); } @@ -252,7 +255,7 @@ export function averageSubjectiveScores( const uniqueViolations = entry.violations.filter((v) => { const key = [ v.quoted_text?.toLowerCase().trim() || "", - v.analysis?.toLowerCase().trim() || "", + v.message?.toLowerCase().trim() || "", ].join("|"); if (seen.has(key)) return false; seen.add(key); diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts index b7d5ae5..a16783c 100644 --- a/tests/scoring-types.test.ts +++ b/tests/scoring-types.test.ts @@ -20,7 +20,7 @@ describe("Scoring Types", () => { filename: "test.md", fullPath: "/test.md", body: "Evaluate this.", - pack: "test", + pack: "TestPack", meta: { id: "test-judge", name: "Test Judge", @@ -83,7 +83,7 @@ describe("Scoring Types", () => { filename: "test.md", fullPath: "/test.md", body: "Count things.", - pack: "test", + pack: "TestPack", meta: { id: "test-semi", name: "Test Semi", @@ -100,7 +100,8 @@ describe("Scoring Types", () => { violations: [ { description: "Issue 1", - analysis: "First issue found", + issue: "First issue found", + message: "First issue found", suggestion: "", quoted_text: "", context_before: "", @@ -108,7 +109,8 @@ describe("Scoring Types", () => { }, { description: "Issue 2", - analysis: "Second issue found", + issue: "Second issue found", + message: "Second issue found", suggestion: "", quoted_text: "", context_before: "", @@ -182,7 +184,7 @@ describe("Scoring Types", () => { filename: "tech.md", fullPath: "/tech.md", body: "Check accuracy", - pack: "test", + pack: "TestPack", meta: { id: "tech-acc", name: "Tech Acc", type: "check" }, };