diff --git a/.changeset/clear-lines-report.md b/.changeset/clear-lines-report.md new file mode 100644 index 00000000..dae2dfc3 --- /dev/null +++ b/.changeset/clear-lines-report.md @@ -0,0 +1,5 @@ +--- +"evalite": patch +--- + +Show scorer names and scores when CLI threshold checks fail. diff --git a/packages/evalite-tests/tests/threshold.test.ts b/packages/evalite-tests/tests/threshold.test.ts index 771a03b1..c05fde0b 100644 --- a/packages/evalite-tests/tests/threshold.test.ts +++ b/packages/evalite-tests/tests/threshold.test.ts @@ -12,7 +12,11 @@ it("Should set exitCode to 1 if the score is below the threshold", async () => { scoreThreshold: 50, }); - expect(fixture.getOutput()).toContain("Threshold 50% (failed)"); + const output = fixture.getOutput(); + + expect(output).toContain("Threshold 50% (failed)"); + expect(output).toContain("XYZ"); + expect(output).toContain("20%"); expect(exit).toHaveBeenCalledWith(1); }); diff --git a/packages/evalite/src/reporter.ts b/packages/evalite/src/reporter.ts index fb3fb22b..e31eb310 100644 --- a/packages/evalite/src/reporter.ts +++ b/packages/evalite/src/reporter.ts @@ -30,6 +30,26 @@ import { average, max } from "./utils.js"; const F_POINTER = "❯"; const separator = c.dim(" > "); +function getScorersBelowThreshold( + scores: Evalite.Score[], + scoreThreshold: number +) { + const scoresByName = new Map(); + + for (const score of scores) { + const existingScores = scoresByName.get(score.name) ?? []; + existingScores.push(score.score ?? 0); + scoresByName.set(score.name, existingScores); + } + + return Array.from(scoresByName, ([name, scorerScores]) => ({ + name, + score: average(scorerScores, (score) => score), + })) + .filter((scorer) => scorer.score * 100 < scoreThreshold) + .sort((a, b) => a.name.localeCompare(b.name)); +} + export interface EvaliteReporterOptions { isWatching: boolean; port: number; @@ -244,7 +264,12 @@ export default class EvaliteReporter implements Reporter { ); if (typeof this.opts.scoreThreshold === "number") { - renderThreshold(this.ctx.logger, this.opts.scoreThreshold, averageScore); + renderThreshold( + this.ctx.logger, + this.opts.scoreThreshold, + averageScore, + getScorersBelowThreshold(scores, this.opts.scoreThreshold) + ); } renderSummaryStats(this.ctx.logger, { diff --git a/packages/evalite/src/reporter/rendering.ts b/packages/evalite/src/reporter/rendering.ts index 96c6b712..278f2e0e 100644 --- a/packages/evalite/src/reporter/rendering.ts +++ b/packages/evalite/src/reporter/rendering.ts @@ -285,7 +285,8 @@ export function renderScoreDisplay( export function renderThreshold( logger: { log: (msg: string) => void }, scoreThreshold: number, - averageScore: number | null + averageScore: number | null, + failingScorers: { name: string; score: number }[] = [] ): "passed" | "failed" { let thresholdScoreSuffix = ""; let result: "passed" | "failed"; @@ -308,6 +309,19 @@ export function renderThreshold( ].join("") ); + if (result === "failed" && failingScorers.length > 0) { + logger.log( + [ + " ", + c.dim("Scorers"), + " ", + failingScorers + .map((scorer) => `${scorer.name} ${displayScore(scorer.score)}`) + .join(c.dim(", ")), + ].join("") + ); + } + return result; }