feat: show failing scorers for threshold failures

.changeset/clear-lines-report.md

@@ -0,0 +1,5 @@
+---
+"evalite": patch
+---
+
+Show scorer names and scores when CLI threshold checks fail.

packages/evalite-tests/tests/threshold.test.ts

@@ -12,7 +12,11 @@ it("Should set exitCode to 1 if the score is below the threshold", async () => {
     scoreThreshold: 50,
   });
 
-  expect(fixture.getOutput()).toContain("Threshold  50% (failed)");
+  const output = fixture.getOutput();
+
+  expect(output).toContain("Threshold  50% (failed)");
+  expect(output).toContain("XYZ");
+  expect(output).toContain("20%");
   expect(exit).toHaveBeenCalledWith(1);
 });
 

packages/evalite/src/reporter.ts

@@ -30,6 +30,26 @@ import { average, max } from "./utils.js";
 const F_POINTER = "❯";
 const separator = c.dim(" > ");
 
+function getScorersBelowThreshold(
+  scores: Evalite.Score[],
+  scoreThreshold: number
+) {
+  const scoresByName = new Map<string, number[]>();
+
+  for (const score of scores) {
+    const existingScores = scoresByName.get(score.name) ?? [];
+    existingScores.push(score.score ?? 0);
+    scoresByName.set(score.name, existingScores);
+  }
+
+  return Array.from(scoresByName, ([name, scorerScores]) => ({
+    name,
+    score: average(scorerScores, (score) => score),
+  }))
+    .filter((scorer) => scorer.score * 100 < scoreThreshold)
+    .sort((a, b) => a.name.localeCompare(b.name));
+}
+
 export interface EvaliteReporterOptions {
   isWatching: boolean;
   port: number;
@@ -244,7 +264,12 @@ export default class EvaliteReporter implements Reporter {
     );
 
     if (typeof this.opts.scoreThreshold === "number") {
-      renderThreshold(this.ctx.logger, this.opts.scoreThreshold, averageScore);
+      renderThreshold(
+        this.ctx.logger,
+        this.opts.scoreThreshold,
+        averageScore,
+        getScorersBelowThreshold(scores, this.opts.scoreThreshold)
+      );
     }
 
     renderSummaryStats(this.ctx.logger, {

packages/evalite/src/reporter/rendering.ts

@@ -285,7 +285,8 @@ export function renderScoreDisplay(
 export function renderThreshold(
   logger: { log: (msg: string) => void },
   scoreThreshold: number,
-  averageScore: number | null
+  averageScore: number | null,
+  failingScorers: { name: string; score: number }[] = []
 ): "passed" | "failed" {
   let thresholdScoreSuffix = "";
   let result: "passed" | "failed";
@@ -308,6 +309,19 @@ export function renderThreshold(
     ].join("")
   );
 
+  if (result === "failed" && failingScorers.length > 0) {
+    logger.log(
+      [
+        "    ",
+        c.dim("Scorers"),
+        "  ",
+        failingScorers
+          .map((scorer) => `${scorer.name} ${displayScore(scorer.score)}`)
+          .join(c.dim(", ")),
+      ].join("")
+    );
+  }
+
   return result;
 }