From 86f159e924c2c47d8be0cafc7bca66f2dac3933f Mon Sep 17 00:00:00 2001 From: Jahtofunmi Osho Date: Tue, 13 Jan 2026 08:46:09 +0100 Subject: [PATCH 1/2] refactor: replace analysis field with issue/message pair - Rename `analysis` field to `issue` and add `message` field - Add `--suggest` CLI flag to cli-schemas - Update subjective and semi-objective LLM schemas - Add prompt guidance for scope-aware message formatting - Update scoring logic to use new fields for deduplication - Add rule for naming sections in comparisons --- src/chunking/merger.ts | 4 +- src/cli/commands.ts | 118 +++++++++++-------- src/cli/orchestrator.ts | 130 ++++++++++++++------- src/cli/types.ts | 193 +++++++++++++++++--------------- src/prompts/directive-loader.ts | 27 ++++- src/prompts/schema.ts | 27 +++-- src/schemas/cli-schemas.ts | 5 +- src/scoring/scorer.ts | 11 +- tests/scoring-types.test.ts | 9 +- 9 files changed, 319 insertions(+), 205 deletions(-) diff --git a/src/chunking/merger.ts b/src/chunking/merger.ts index c3a23a9..3a4c067 100644 --- a/src/chunking/merger.ts +++ b/src/chunking/merger.ts @@ -5,13 +5,13 @@ export function mergeViolations( ): SemiObjectiveItem[] { const all = chunkViolations.flat(); - // Deduplicate using composite key (quoted_text + description + analysis) + // Deduplicate using composite key (quoted_text + description + message) const seen = new Set(); return all.filter((v) => { const key = [ v.quoted_text?.toLowerCase().trim() || "", v.description?.toLowerCase().trim() || "", - v.analysis?.toLowerCase().trim() || "", + v.message?.toLowerCase().trim() || "", ].join("|"); if (seen.has(key)) return false; seen.add(key); diff --git a/src/cli/commands.ts b/src/cli/commands.ts index b5ec2d1..21e77b7 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -1,24 +1,24 @@ -import type { Command } from 'commander'; -import { existsSync } from 'fs'; -import * as path from 'path'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; -import { createProvider } from '../providers/provider-factory'; -import { PerplexitySearchProvider } from '../providers/perplexity-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import { loadConfig } from '../boundaries/config-loader'; -import { loadRuleFile, type PromptFile } from '../prompts/prompt-loader'; -import { RulePackLoader } from '../boundaries/rule-pack-loader'; -import { PresetLoader } from '../config/preset-loader'; -import { printGlobalSummary, printTokenUsage } from '../output/reporter'; -import { DefaultRequestBuilder } from '../providers/request-builder'; -import { loadDirective } from '../prompts/directive-loader'; -import { resolveTargets } from '../scan/file-resolver'; -import { parseCliOptions, parseEnvironment } from '../boundaries/index'; -import { handleUnknownError } from '../errors/index'; -import { evaluateFiles } from './orchestrator'; -import { OutputFormat } from './types'; -import { DEFAULT_CONFIG_FILENAME } from '../config/constants'; +import type { Command } from "commander"; +import { existsSync } from "fs"; +import * as path from "path"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; +import { createProvider } from "../providers/provider-factory"; +import { PerplexitySearchProvider } from "../providers/perplexity-provider"; +import type { SearchProvider } from "../providers/search-provider"; +import { loadConfig } from "../boundaries/config-loader"; +import { loadRuleFile, type PromptFile } from "../prompts/prompt-loader"; +import { RulePackLoader } from "../boundaries/rule-pack-loader"; +import { PresetLoader } from "../config/preset-loader"; +import { printGlobalSummary, printTokenUsage } from "../output/reporter"; +import { DefaultRequestBuilder } from "../providers/request-builder"; +import { loadDirective } from "../prompts/directive-loader"; +import { resolveTargets } from "../scan/file-resolver"; +import { parseCliOptions, parseEnvironment } from "../boundaries/index"; +import { handleUnknownError } from "../errors/index"; +import { evaluateFiles } from "./orchestrator"; +import { OutputFormat } from "./types"; +import { DEFAULT_CONFIG_FILENAME } from "../config/constants"; // eslint-disable-next-line @typescript-eslint/naming-convention const __filename = fileURLToPath(import.meta.url); @@ -31,12 +31,23 @@ const __dirname = dirname(__filename); */ export function registerMainCommand(program: Command): void { program - .option('-v, --verbose', 'Enable verbose logging') - .option('--show-prompt', 'Print full prompt and injected content') - .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') - .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') - .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) - .argument('[paths...]', 'files or directories to check (required)') + .option("-v, --verbose", "Enable verbose logging") + .option("--show-prompt", "Print full prompt and injected content") + .option( + "--show-prompt-trunc", + "Print truncated prompt/content previews (500 chars)" + ) + .option("--suggest", "Show fix suggestions for each issue") + .option( + "--output ", + "Output format: line (default), json, or vale-json, rdjson", + "line" + ) + .option( + "--config ", + `Path to custom ${DEFAULT_CONFIG_FILENAME} config file` + ) + .argument("[paths...]", "files or directories to check (required)") .action(async (paths: string[] = []) => { // Require explicit paths to prevent accidental full directory scans // Users must provide specific files, directories, or wildcards (e.g., `vectorlint *`) @@ -50,7 +61,7 @@ export function registerMainCommand(program: Command): void { try { cliOptions = parseCliOptions(program.opts()); } catch (e: unknown) { - const err = handleUnknownError(e, 'Parsing CLI options'); + const err = handleUnknownError(e, "Parsing CLI options"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -60,9 +71,9 @@ export function registerMainCommand(program: Command): void { try { env = parseEnvironment(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Validating environment variables'); + const err = handleUnknownError(e, "Validating environment variables"); console.error(`Error: ${err.message}`); - console.error('Please set these in your .env file or environment.'); + console.error("Please set these in your .env file or environment."); process.exit(1); } @@ -71,7 +82,7 @@ export function registerMainCommand(program: Command): void { try { directive = loadDirective(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading directive'); + const err = handleUnknownError(e, "Loading directive"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -96,7 +107,7 @@ export function registerMainCommand(program: Command): void { try { config = loadConfig(process.cwd(), cliOptions.config); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading configuration'); + const err = handleUnknownError(e, "Loading configuration"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -110,15 +121,19 @@ export function registerMainCommand(program: Command): void { const prompts: PromptFile[] = []; try { - const presetsDir = path.resolve(__dirname, '../presets'); + const presetsDir = path.resolve(__dirname, "../presets"); const presetLoader = new PresetLoader(presetsDir); const loader = new RulePackLoader(presetLoader); const packs = await loader.listAllPacks(rulesPath); if (packs.length === 0 && cliOptions.verbose) { - console.warn(`[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath} or presets.`); - console.warn(`[vectorlint] Please organize your rules into subdirectories or use a valid preset.`); + console.warn( + `[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath} or presets.` + ); + console.warn( + `[vectorlint] Please organize your rules into subdirectories or use a valid preset.` + ); } for (const pack of packs) { @@ -128,7 +143,8 @@ export function registerMainCommand(program: Command): void { for (const filePath of rulePaths) { const result = loadRuleFile(filePath, pack.name); if (result.warning) { - if (cliOptions.verbose) console.warn(`[vectorlint] ${result.warning}`); + if (cliOptions.verbose) + console.warn(`[vectorlint] ${result.warning}`); } if (result.prompt) { prompts.push(result.prompt); @@ -138,14 +154,18 @@ export function registerMainCommand(program: Command): void { if (prompts.length === 0) { if (!rulesPath) { - console.error('Error: no rules found. Either set RulesPath in config or configure RunRules with a valid preset.'); + console.error( + "Error: no rules found. Either set RulesPath in config or configure RunRules with a valid preset." + ); } else { - console.error(`Error: no .md rules found in ${rulesPath} or presets.`); + console.error( + `Error: no .md rules found in ${rulesPath} or presets.` + ); } process.exit(1); } } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading prompts'); + const err = handleUnknownError(e, "Loading prompts"); console.error(`Error: failed to load prompts: ${err.message}`); process.exit(1); } @@ -161,26 +181,27 @@ export function registerMainCommand(program: Command): void { configDir: config.configDir, }); } catch (e: unknown) { - const err = handleUnknownError(e, 'Resolving target files'); + const err = handleUnknownError(e, "Resolving target files"); console.error(`Error: failed to resolve target files: ${err.message}`); process.exit(1); } if (targets.length === 0) { - console.error('Error: no target files found to evaluate.'); + console.error("Error: no target files found to evaluate."); process.exit(1); } - - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY + const searchProvider: SearchProvider | undefined = process.env + .PERPLEXITY_API_KEY ? new PerplexitySearchProvider({ debug: false }) : undefined; const outputFormat = cliOptions.output as OutputFormat; if (!Object.values(OutputFormat).includes(outputFormat)) { - console.error(`Error: Invalid output format '${cliOptions.output}'. Valid options: line, json, vale-json, rdjson`); + console.error( + `Error: Invalid output format '${cliOptions.output}'. Valid options: line, json, vale-json, rdjson` + ); process.exit(1); } @@ -194,6 +215,7 @@ export function registerMainCommand(program: Command): void { verbose: cliOptions.verbose, outputFormat: outputFormat, scanPaths: config.scanPaths, + suggest: cliOptions.suggest, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, outputPricePerMillion: env.OUTPUT_PRICE_PER_MILLION, @@ -201,7 +223,7 @@ export function registerMainCommand(program: Command): void { }); // Print global summary (only for line format) - if (cliOptions.output === 'line') { + if (cliOptions.output === "line") { if (result.tokenUsage) { printTokenUsage(result.tokenUsage); } @@ -214,6 +236,8 @@ export function registerMainCommand(program: Command): void { } // Exit with appropriate code - process.exit(result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0); + process.exit( + result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0 + ); }); } diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 7c660c8..fa5c5ee 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,31 +1,48 @@ -import { readFileSync } from 'fs'; -import * as path from 'path'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; -import { RdJsonFormatter } from '../output/rdjson-formatter'; -import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; -import { isSubjectiveResult } from '../prompts/schema'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; -import { OutputFormat } from './types'; -import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess -} from './types'; +import { readFileSync } from "fs"; +import * as path from "path"; +import type { PromptFile } from "../prompts/prompt-loader"; +import { ScanPathResolver } from "../boundaries/scan-path-resolver"; +import { + ValeJsonFormatter, + type JsonIssue, +} from "../output/vale-json-formatter"; +import { + JsonFormatter, + type Issue, + type ScoreComponent, +} from "../output/json-formatter"; +import { RdJsonFormatter } from "../output/rdjson-formatter"; import { - calculateCost, - TokenUsageStats -} from '../providers/token-usage'; + printFileHeader, + printIssueRow, + printEvaluationSummaries, + type EvaluationSummary, +} from "../output/reporter"; +import { checkTarget } from "../prompts/target"; +import { isSubjectiveResult } from "../prompts/schema"; +import { handleUnknownError, MissingDependencyError } from "../errors/index"; +import { createEvaluator } from "../evaluators/index"; +import { Type, Severity } from "../evaluators/types"; +import { OutputFormat } from "./types"; +import type { + EvaluationOptions, + EvaluationResult, + ErrorTrackingResult, + ReportIssueParams, + ProcessViolationsParams, + ProcessCriterionParams, + ProcessCriterionResult, + ValidationParams, + ProcessPromptResultParams, + RunPromptEvaluationParams, + RunPromptEvaluationResult, + EvaluateFileParams, + EvaluateFileResult, + RunPromptEvaluationResultSuccess, +} from "./types"; +import { calculateCost, TokenUsageStats } from "../providers/token-usage"; import { locateQuotedText } from "../output/location"; - /* * Returns the evaluator type, defaulting to 'base' if not specified. */ @@ -47,7 +64,7 @@ function buildRuleName( if (criterionId) { parts.push(criterionId); } - return parts.join('.'); + return parts.join("."); } /* @@ -157,6 +174,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { outputFormat, jsonFormatter, verbose, + suggest, } = params; let hadOperationalErrors = false; @@ -175,7 +193,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { for (const v of violations) { if (!v) continue; - const rowSummary = (v.analysis || "").trim(); + const rowSummary = (v.message || "").trim(); try { const locWithMatch = locateQuotedText( @@ -224,6 +242,10 @@ function locateAndReportViolations(params: ProcessViolationsParams): { } // Report only verified, unique violations + // For line output, only show suggestions when --suggest flag is used + // For JSON formats, always include suggestions (machine consumption) + const includeSuggestion = outputFormat !== OutputFormat.Line || suggest; + for (const { v, line, @@ -240,7 +262,8 @@ function locateAndReportViolations(params: ProcessViolationsParams): { ruleName, outputFormat, jsonFormatter, - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), + ...(includeSuggestion && + v.suggestion !== undefined && { suggestion: v.suggestion }), scoreText, match: matchedText, }); @@ -268,6 +291,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose, + suggest, } = params; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -276,13 +300,13 @@ function extractAndReportCriterion( const criterionId = exp.id ? String(exp.id) : exp.name - ? String(exp.name) + ? String(exp.name) .replace(/[^A-Za-z0-9]+/g, " ") .split(" ") .filter(Boolean) .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) .join("") - : ""; + : ""; const ruleName = buildRuleName(packName, promptId, criterionId); const weightNum = exp.weight || 1; @@ -302,6 +326,8 @@ function extractAndReportCriterion( expTargetSpec?.suggestion || metaTargetSpec?.suggestion || "Add the required target section."; + // For line output, only show suggestions when --suggest flag is used + const includeSuggestion = outputFormat !== OutputFormat.Line || suggest; reportIssue({ file: relFile, line: 1, @@ -311,7 +337,7 @@ function extractAndReportCriterion( ruleName, outputFormat, jsonFormatter, - suggestion, + ...(includeSuggestion && { suggestion }), scoreText: "nil", match: "", }); @@ -400,7 +426,8 @@ function extractAndReportCriterion( quoted_text?: string; context_before?: string; context_after?: string; - analysis?: string; + issue?: string; + message?: string; suggestion?: string; }>, content, @@ -411,6 +438,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; @@ -552,6 +580,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose, + suggest, } = params; const meta = promptFile.meta; const promptId = (meta.id || "").toString(); @@ -567,7 +596,10 @@ function routePromptResult( const violationCount = result.violations.length; // Group violations by criterionName - const violationsByCriterion = new Map(); + const violationsByCriterion = new Map< + string | undefined, + typeof result.violations + >(); for (const v of result.violations) { const criterionName = v.criterionName; if (!violationsByCriterion.has(criterionName)) { @@ -584,7 +616,7 @@ function routePromptResult( // Find criterion ID from meta let criterionId: string | undefined; if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); + const criterion = meta.criteria.find((c) => c.name === criterionName); criterionId = criterion?.id; } @@ -597,12 +629,14 @@ function routePromptResult( relFile, severity, ruleName, - scoreText: '', + scoreText: "", outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + hadOperationalErrors = + hadOperationalErrors || violationResult.hadOperationalErrors; if (severity === Severity.ERROR) { totalErrors += violations.length; @@ -613,7 +647,12 @@ function routePromptResult( } // If no violations but we have a message (JSON output), report it - if (violationCount === 0 && (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson) && result.message) { + if ( + violationCount === 0 && + (outputFormat === OutputFormat.Json || + outputFormat === OutputFormat.ValeJson) && + result.message + ) { const ruleName = buildRuleName(promptFile.pack, promptId, undefined); reportIssue({ file: relFile, @@ -671,6 +710,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose: !!verbose, + suggest: !!suggest, }); promptErrors += criterionResult.errors; @@ -741,7 +781,6 @@ async function runPromptEvaluation( ); const result = await evaluator.evaluate(relFile, content); - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; return resultObj; @@ -766,6 +805,7 @@ async function evaluateFile( scanPaths, outputFormat = OutputFormat.Line, verbose, + suggest, } = options; let hadOperationalErrors = false; @@ -873,6 +913,7 @@ async function evaluateFile( outputFormat, jsonFormatter, verbose, + suggest: !!suggest, }); totalErrors += promptResult.errors; totalWarnings += promptResult.warnings; @@ -902,7 +943,7 @@ async function evaluateFile( requestFailures, hadOperationalErrors, hadSeverityErrors, - tokenUsage: tokenUsageStats + tokenUsage: tokenUsageStats, }; } @@ -975,10 +1016,13 @@ export async function evaluateFiles( }; // Calculate cost if pricing is configured - const cost = calculateCost({ - inputTokens: totalInputTokens, - outputTokens: totalOutputTokens - }, options.pricing); + const cost = calculateCost( + { + inputTokens: totalInputTokens, + outputTokens: totalOutputTokens, + }, + options.pricing + ); if (cost !== undefined) { tokenUsage.totalCost = cost; } diff --git a/src/cli/types.ts b/src/cli/types.ts index 9666d5e..0948981 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,139 +1,148 @@ -import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import type { EvaluationSummary } from '../output/reporter'; -import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; -import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; -import { Severity } from '../evaluators/types'; -import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { PromptFile } from "../prompts/prompt-loader"; +import type { LLMProvider } from "../providers/llm-provider"; +import type { SearchProvider } from "../providers/search-provider"; +import type { + PromptMeta, + PromptCriterionSpec, +} from "../schemas/prompt-schemas"; +import type { FilePatternConfig } from "../boundaries/file-section-parser"; +import type { EvaluationSummary } from "../output/reporter"; +import { ValeJsonFormatter } from "../output/vale-json-formatter"; +import { JsonFormatter, type ScoreComponent } from "../output/json-formatter"; +import { RdJsonFormatter } from "../output/rdjson-formatter"; +import type { + EvaluationResult as PromptEvaluationResult, + SubjectiveResult, +} from "../prompts/schema"; +import { Severity } from "../evaluators/types"; +import type { TokenUsageStats, PricingConfig } from "../providers/token-usage"; export enum OutputFormat { - Line = "line", - Json = "json", - ValeJson = "vale-json", - RdJson = "rdjson", + Line = "line", + Json = "json", + ValeJson = "vale-json", + RdJson = "rdjson", } export interface EvaluationOptions { - prompts: PromptFile[]; - rulesPath: string | undefined; - provider: LLMProvider; - searchProvider?: SearchProvider; - concurrency: number; - verbose: boolean; - scanPaths: FilePatternConfig[]; - outputFormat?: OutputFormat; - pricing?: PricingConfig; + prompts: PromptFile[]; + rulesPath: string | undefined; + provider: LLMProvider; + searchProvider?: SearchProvider; + concurrency: number; + verbose: boolean; + scanPaths: FilePatternConfig[]; + outputFormat?: OutputFormat; + pricing?: PricingConfig; + suggest?: boolean; } export interface EvaluationResult { - totalFiles: number; - totalErrors: number; - totalWarnings: number; - requestFailures: number; - hadOperationalErrors: boolean; - hadSeverityErrors: boolean; - tokenUsage?: TokenUsageStats; + totalFiles: number; + totalErrors: number; + totalWarnings: number; + requestFailures: number; + hadOperationalErrors: boolean; + hadSeverityErrors: boolean; + tokenUsage?: TokenUsageStats; } export interface ErrorTrackingResult { - errors: number; - warnings: number; - hadOperationalErrors: boolean; - hadSeverityErrors: boolean; - scoreEntries?: EvaluationSummary[]; + errors: number; + warnings: number; + hadOperationalErrors: boolean; + hadSeverityErrors: boolean; + scoreEntries?: EvaluationSummary[]; } export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; + content: string; + relFile: string; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + verbose?: boolean; + suggest?: boolean; } export interface ReportIssueParams { - file: string; - line: number; - column: number; - severity: Severity; - summary: string; - ruleName: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - suggestion?: string; - scoreText?: string; - match?: string; + file: string; + line: number; + column: number; + severity: Severity; + summary: string; + ruleName: string; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + suggestion?: string; + scoreText?: string; + match?: string; } export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; + violations: Array<{ + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + issue?: string; + message?: string; + suggestion?: string; + }>; + severity: Severity; + ruleName: string; + scoreText: string; } export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: SubjectiveResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; + exp: PromptCriterionSpec; + result: SubjectiveResult; + packName: string; + promptId: string; + promptFilename: string; + meta: PromptMeta; } export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; + userScore: number; + maxScore: number; + scoreEntry: { id: string; scoreText: string; score?: number }; + scoreComponent?: ScoreComponent; } export interface ValidationParams { - meta: PromptMeta; - result: SubjectiveResult; + meta: PromptMeta; + result: SubjectiveResult; } export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; + promptFile: PromptFile; + result: PromptEvaluationResult; } export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; + promptFile: PromptFile; + relFile: string; + content: string; + provider: LLMProvider; + searchProvider?: SearchProvider; } export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; + ok: true; + result: PromptEvaluationResult; } export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; + | RunPromptEvaluationResultSuccess + | { ok: false; error: Error }; export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + file: string; + options: EvaluationOptions; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface EvaluateFileResult extends ErrorTrackingResult { - requestFailures: number; - tokenUsage?: TokenUsageStats; + requestFailures: number; + tokenUsage?: TokenUsageStats; } diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index 64834ba..de5ba7f 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -10,7 +10,7 @@ import path from "path"; */ const DEFAULT_DIRECTIVE = ` List every finding you detect. -- If the issue occurs within a sentence, quote the offending word or short phrase as evidence (example: "leverage" is an AI buzzword). +- If the issue occurs within a sentence, quote the offending word or short phrase as evidence. - If a sentence contains multiple issues, report each as a separate violation. **IMPORTANT**: The input has line numbers prepended (format: "123\\ttext"). Use these line numbers when reporting issues. @@ -21,10 +21,28 @@ For each finding, provide: The text must exist verbatim in 'Input' as a direct substring (excluding the line number prefix). - context_before: 10–20 exact characters immediately before quoted_text (or empty string if at start) - context_after: 10–20 exact characters immediately after quoted_text (or empty string if at end) -- analysis: a specific, concrete explanation of the issue (respect word limit) -- suggestion: a succinct, imperative fix (max 15 words) +- issue: The problem (NEVER include the quoted_text here). Aim for 5-12 words. +- message: The user-facing output. Format based on scope: + - WORD-level (buzzwords, specific terms): "quoted_text" + issue → e.g., "ensure" is a common AI buzzword + - SENTENCE/SECTION/DOCUMENT-level: issue only → e.g., Opens with meta-commentary +- suggestion: a succinct, imperative fix - When a criterion has no findings, provide one short positive remark describing compliance. +**MESSAGE EXAMPLES** (follow this style exactly): + +WORD-level (quoted_text is 1-4 words) — message INCLUDES quoted_text: +✓ quoted_text: "ensure" → message: "ensure" is a common AI buzzword +✓ quoted_text: "game-changer" → message: "game-changer" is a buzzword phrase + +SENTENCE-level (quoted_text is >5 words) — message does NOT include quoted_text: +✓ quoted_text: "LLM-based chunking uses a large language model..." → message: Buzzword-heavy sentence +✓ quoted_text: "In this article, we will explore..." → message: Opens with meta-commentary + +SECTION-level — message references section names: +✓ message: Overlaps with "What is Chunking?" on context + +**CRITICAL: If quoted_text is longer than 4 words, do NOT include it in the message.** + ***CRITICAL RULES*** 1. Go through the Input before anything else, and show your step-by-step reasoning or the approach you'll take to accomplish the task. @@ -32,7 +50,8 @@ For each finding, provide: 3. If you cannot find a verbatim match in 'Input', do NOT report it - skip that finding entirely. 4. Do NOT infer or hypothesize issues. Only report what you can directly quote from 'Input'. 5. Fabricating quotes that don't exist in 'Input' is equivalent to failure. -6. The line number you report must match the prepended number on that line in Input.`; +6. The line number you report must match the prepended number on that line in Input. +7. When comparing/contrasting sections, ALWAYS name the sections. NEVER say "both sections" without naming them.`; export function loadDirective(cwd: string = process.cwd()): string { // 1) Project override diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 1606fd5..d539761 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -33,14 +33,16 @@ export function buildSubjectiveLLMSchema() { quoted_text: { type: "string" }, context_before: { type: "string" }, context_after: { type: "string" }, - analysis: { type: "string" }, + issue: { type: "string" }, + message: { type: "string" }, suggestion: { type: "string" }, }, required: [ "quoted_text", "context_before", "context_after", - "analysis", + "issue", + "message", "suggestion", ], }, @@ -74,7 +76,8 @@ export function buildSemiObjectiveLLMSchema() { context_before: { type: "string" }, context_after: { type: "string" }, description: { type: "string" }, - analysis: { type: "string" }, + issue: { type: "string" }, + message: { type: "string" }, suggestion: { type: "string" }, }, required: [ @@ -82,7 +85,8 @@ export function buildSemiObjectiveLLMSchema() { "context_before", "context_after", "description", - "analysis", + "issue", + "message", "suggestion", ], }, @@ -103,7 +107,8 @@ export type SubjectiveLLMResult = { quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; }>; @@ -112,7 +117,8 @@ export type SubjectiveLLMResult = { export type SemiObjectiveLLMResult = { violations: Array<{ description: string; - analysis: string; + issue: string; + message: string; suggestion?: string; quoted_text?: string; context_before?: string; @@ -135,7 +141,8 @@ export type SubjectiveResult = { quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; }>; @@ -144,7 +151,8 @@ export type SubjectiveResult = { export type SemiObjectiveItem = { description: string; - analysis: string; + issue: string; + message: string; suggestion?: string; quoted_text?: string; context_before?: string; @@ -160,7 +168,8 @@ export type SemiObjectiveResult = { severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; violations: Array<{ - analysis: string; + issue: string; + message: string; suggestion?: string; quoted_text?: string; context_before?: string; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 7e4f74c..73ecb1c 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,11 +1,12 @@ -import { z } from 'zod'; +import { z } from "zod"; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ verbose: z.boolean().default(false), showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), - output: z.enum(['line', 'json', 'vale-json', 'rdjson']).default('line'), + suggest: z.boolean().default(false), + output: z.enum(["line", "json", "vale-json", "rdjson"]).default("line"), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index 4ab4ff1..f909209 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -54,7 +54,8 @@ export function calculateSemiObjectiveScore( // Map items to violation format const mappedViolations = violations.map((item) => ({ - analysis: item.analysis, + issue: item.issue, + message: item.message, ...(item.suggestion && { suggestion: item.suggestion }), ...(item.quoted_text && { quoted_text: item.quoted_text }), ...(item.context_before && { context_before: item.context_before }), @@ -178,7 +179,8 @@ export function averageSubjectiveScores( quoted_text: string; context_before: string; context_after: string; - analysis: string; + issue: string; + message: string; suggestion: string; }>; summaries: string[]; @@ -215,7 +217,8 @@ export function averageSubjectiveScores( quoted_text: v.quoted_text || "", context_before: v.context_before || "", context_after: v.context_after || "", - analysis: v.analysis || "", + issue: v.issue || "", + message: v.message || "", suggestion: v.suggestion || "", }); } @@ -253,7 +256,7 @@ export function averageSubjectiveScores( const uniqueViolations = entry.violations.filter((v) => { const key = [ v.quoted_text?.toLowerCase().trim() || "", - v.analysis?.toLowerCase().trim() || "", + v.message?.toLowerCase().trim() || "", ].join("|"); if (seen.has(key)) return false; seen.add(key); diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts index e892ea5..1ac2068 100644 --- a/tests/scoring-types.test.ts +++ b/tests/scoring-types.test.ts @@ -29,6 +29,7 @@ describe("Scoring Types", () => { { id: "c2", name: "Criterion 2", weight: 50 }, ], }, + pack: "TestPack", }; it("should calculate weighted average correctly", async () => { @@ -87,6 +88,7 @@ describe("Scoring Types", () => { name: "Test Semi", type: "semi-objective", }, + pack: "TestPack", }; it("should calculate score correctly based on violation count", async () => { @@ -98,7 +100,8 @@ describe("Scoring Types", () => { violations: [ { description: "Issue 1", - analysis: "First issue found", + issue: "First issue found", + message: "First issue found", suggestion: "", quoted_text: "", context_before: "", @@ -106,7 +109,8 @@ describe("Scoring Types", () => { }, { description: "Issue 2", - analysis: "Second issue found", + issue: "Second issue found", + message: "Second issue found", suggestion: "", quoted_text: "", context_before: "", @@ -181,6 +185,7 @@ describe("Scoring Types", () => { fullPath: "/tech.md", body: "Check accuracy", meta: { id: "tech-acc", name: "Tech Acc", type: "semi-objective" }, + pack: "TestPack", }; const evaluator = new TechnicalAccuracyEvaluator( From 9d7abf3d52d6eb8534efafc2e97d1eb6ee85b12a Mon Sep 17 00:00:00 2001 From: Jahtofunmi Osho Date: Tue, 13 Jan 2026 09:41:04 +0100 Subject: [PATCH 2/2] fix: align directive threshold and schema types --- src/prompts/directive-loader.ts | 2 +- src/prompts/schema.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index de5ba7f..2122da6 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -34,7 +34,7 @@ WORD-level (quoted_text is 1-4 words) — message INCLUDES quoted_text: ✓ quoted_text: "ensure" → message: "ensure" is a common AI buzzword ✓ quoted_text: "game-changer" → message: "game-changer" is a buzzword phrase -SENTENCE-level (quoted_text is >5 words) — message does NOT include quoted_text: +SENTENCE-level (quoted_text is >4 words) — message does NOT include quoted_text: ✓ quoted_text: "LLM-based chunking uses a large language model..." → message: Buzzword-heavy sentence ✓ quoted_text: "In this article, we will explore..." → message: Opens with meta-commentary diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 406f5c4..d3a20a1 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -119,7 +119,7 @@ export type SemiObjectiveLLMResult = { description: string; issue: string; message: string; - suggestion?: string; + suggestion: string; quoted_text?: string; context_before?: string; context_after?: string;