Skip to content

Commit c61715a

Browse files
realarkrealark
authored
Merge pull request #140 from braintrustdata/ark/eval-span-attributes
update eval span attributes to render input/output like we do for datasets
2 parents d5018ef + d951c61 commit c61715a

4 files changed

Lines changed: 89 additions & 65 deletions

File tree

‎braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -461,11 +461,6 @@ private <I, O> void handleStreamingEval(
461461
taskSpan.end();
462462
evalSpan.setStatus(
463463
StatusCode.ERROR, e.getMessage());
464-
evalSpan.setAttribute(
465-
"braintrust.output_json",
466-
toJson(
467-
Collections.singletonMap(
468-
"output", null)));
469464
log.debug(
470465
"Task threw exception for input: "
471466
+ datasetCase.input(),
@@ -681,7 +676,7 @@ private void setEvalSpanAttributes(
681676
}
682677
evalSpan.setAttribute(PARENT, braintrustParent.toParentValue())
683678
.setAttribute("braintrust.span_attributes", toJson(spanAttrs))
684-
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
679+
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
685680
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()));
686681

687682
if (datasetCase.origin().isPresent()) {
@@ -694,8 +689,7 @@ private void setEvalSpanAttributes(
694689
if (!datasetCase.metadata().isEmpty()) {
695690
evalSpan.setAttribute("braintrust.metadata", toJson(datasetCase.metadata()));
696691
}
697-
evalSpan.setAttribute(
698-
"braintrust.output_json", toJson(Map.of("output", taskResult.result())));
692+
evalSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
699693
}
700694

701695
/**
@@ -715,7 +709,7 @@ private void setEvalSpanAttributesForError(
715709
}
716710
evalSpan.setAttribute(PARENT, braintrustParent.toParentValue())
717711
.setAttribute("braintrust.span_attributes", toJson(spanAttrs))
718-
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
712+
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
719713
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()));
720714

721715
if (datasetCase.origin().isPresent()) {
@@ -745,9 +739,9 @@ private void setTaskSpanAttributes(
745739

746740
taskSpan.setAttribute(PARENT, braintrustParent.toParentValue())
747741
.setAttribute("braintrust.span_attributes", toJson(taskSpanAttrs))
748-
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
749-
.setAttribute(
750-
"braintrust.output_json", toJson(Map.of("output", taskResult.result())));
742+
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
743+
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()))
744+
.setAttribute("braintrust.output_json", toJson(taskResult.result()));
751745
}
752746

753747
private void setScoreSpanAttributes(

‎braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,8 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
120120
.setSpanKind(SpanKind.CLIENT)
121121
.setAttribute(PARENT, "experiment_id:" + experimentId)
122122
.setAttribute("braintrust.span_attributes", toJson(Map.of("type", "eval")))
123-
.setAttribute(
124-
"braintrust.input_json",
125-
toJson(Map.of("input", datasetCase.input())))
126-
.setAttribute("braintrust.expected", toJson(datasetCase.expected()))
123+
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
124+
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()))
127125
.startSpan();
128126
if (datasetCase.origin().isPresent()) {
129127
rootSpan.setAttribute("braintrust.origin", toJson(datasetCase.origin().get()));
@@ -146,22 +144,21 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
146144
.setAttribute(
147145
"braintrust.span_attributes",
148146
toJson(Map.of("type", "task")))
147+
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
148+
.setAttribute(
149+
"braintrust.expected_json", toJson(datasetCase.expected()))
149150
.startSpan();
150151
taskSpanId = taskSpan.getSpanContext().getSpanId();
151152
try (var unused =
152153
BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
153154
taskResult = task.apply(datasetCase, parameters);
154-
rootSpan.setAttribute(
155-
"braintrust.output_json",
156-
toJson(Map.of("output", taskResult.result())));
155+
taskSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
156+
rootSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
157157
} catch (Exception e) {
158158
taskSpan.setStatus(StatusCode.ERROR, e.getMessage());
159159
taskSpan.recordException(e);
160160
taskSpan.end();
161161
rootSpan.setStatus(StatusCode.ERROR, e.getMessage());
162-
rootSpan.setAttribute(
163-
"braintrust.output_json",
164-
toJson(Collections.singletonMap("output", null)));
165162
log.debug("Task threw exception for input: " + datasetCase.input(), e);
166163
// run scoreForTaskException on each scorer
167164
for (var scorer : scorers) {

‎braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java‎

Lines changed: 36 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -534,13 +534,21 @@ void testStreamingEval() throws Exception {
534534
io.opentelemetry.api.common.AttributeKey.stringKey(
535535
"braintrust.input_json"));
536536
assertNotNull(inputJson, "Eval span should have input_json");
537+
JsonNode input = JSON_MAPPER.readTree(inputJson);
538+
assertTrue(
539+
input.isTextual(),
540+
"Eval span input_json should be the raw input value, not a wrapper object");
537541

538542
String expectedJson =
539543
evalSpan.getAttributes()
540544
.get(
541545
io.opentelemetry.api.common.AttributeKey.stringKey(
542546
"braintrust.expected_json"));
543547
assertNotNull(expectedJson, "Eval span should have expected_json");
548+
JsonNode expected = JSON_MAPPER.readTree(expectedJson);
549+
assertTrue(
550+
expected.isTextual(),
551+
"Eval span expected_json should be the raw expected value");
544552

545553
String outputJson =
546554
evalSpan.getAttributes()
@@ -549,7 +557,7 @@ void testStreamingEval() throws Exception {
549557
"braintrust.output_json"));
550558
assertNotNull(outputJson, "Eval span should have output_json");
551559
JsonNode output = JSON_MAPPER.readTree(outputJson);
552-
assertEquals("java-fruit", output.get("output").asText());
560+
assertEquals("java-fruit", output.asText());
553561
}
554562

555563
for (SpanData taskSpan : taskSpans) {
@@ -578,6 +586,21 @@ void testStreamingEval() throws Exception {
578586
io.opentelemetry.api.common.AttributeKey.stringKey(
579587
"braintrust.input_json"));
580588
assertNotNull(inputJson, "Task span should have input_json");
589+
JsonNode input = JSON_MAPPER.readTree(inputJson);
590+
assertTrue(
591+
input.isTextual(),
592+
"Task span input_json should be the raw input value, not a wrapper object");
593+
594+
String expectedJson =
595+
taskSpan.getAttributes()
596+
.get(
597+
io.opentelemetry.api.common.AttributeKey.stringKey(
598+
"braintrust.expected_json"));
599+
assertNotNull(expectedJson, "Task span should have expected_json");
600+
JsonNode expected = JSON_MAPPER.readTree(expectedJson);
601+
assertTrue(
602+
expected.isTextual(),
603+
"Task span expected_json should be the raw expected value");
581604

582605
String outputJson =
583606
taskSpan.getAttributes()
@@ -586,7 +609,7 @@ void testStreamingEval() throws Exception {
586609
"braintrust.output_json"));
587610
assertNotNull(outputJson, "Task span should have output_json");
588611
JsonNode output = JSON_MAPPER.readTree(outputJson);
589-
assertEquals("java-fruit", output.get("output").asText());
612+
assertEquals("java-fruit", output.asText());
590613
}
591614

592615
for (SpanData scoreSpan : scoreSpans) {
@@ -762,8 +785,8 @@ void testExperimentEval() throws Exception {
762785
assertFalse(
763786
spanAttrs.has("generation"), "experiment spans should not carry generation");
764787
assertNotNull(
765-
evalSpan.getAttributes().get(AttributeKey.stringKey("braintrust.expected")),
766-
"standard Eval decorator should set braintrust.expected");
788+
evalSpan.getAttributes()
789+
.get(AttributeKey.stringKey("braintrust.expected_json")));
767790
}
768791
}
769792

@@ -1039,15 +1062,11 @@ void testTaskErrorHandling() throws Exception {
10391062
erroredEvalSpan.getStatus().getDescription().contains("task failed on bad-input"),
10401063
"eval span error should contain the exception message");
10411064

1042-
// The errored eval span should have output: null
1043-
@SuppressWarnings("unchecked")
1044-
Map<String, Object> erroredOutputJson =
1045-
fromJson(
1046-
erroredEvalSpan
1047-
.getAttributes()
1048-
.get(AttributeKey.stringKey("braintrust.output_json")),
1049-
Map.class);
1050-
assertNull(erroredOutputJson.get("output"), "errored case output should be null");
1065+
assertNull(
1066+
erroredEvalSpan
1067+
.getAttributes()
1068+
.get(AttributeKey.stringKey("braintrust.output_json")),
1069+
"errored eval span should not have output_json");
10511070

10521071
// Find the task span for the errored case (should have ERROR status)
10531072
var erroredTaskSpan =
@@ -1105,15 +1124,12 @@ void testTaskErrorHandling() throws Exception {
11051124
.filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
11061125
.findFirst()
11071126
.orElseThrow(() -> new AssertionError("expected a successful eval span"));
1108-
@SuppressWarnings("unchecked")
1109-
Map<String, Object> successOutputJson =
1110-
fromJson(
1127+
JsonNode successOutputJson =
1128+
JSON_MAPPER.readTree(
11111129
successEvalSpan
11121130
.getAttributes()
1113-
.get(AttributeKey.stringKey("braintrust.output_json")),
1114-
Map.class);
1115-
assertEquals(
1116-
"result", successOutputJson.get("output"), "successful case should have output");
1131+
.get(AttributeKey.stringKey("braintrust.output_json")));
1132+
assertEquals("result", successOutputJson.asText(), "successful case should have output");
11171133
}
11181134

11191135
@Test

‎braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java‎

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -86,28 +86,45 @@ public void evalOtelTraceWithProperAttributes() {
8686
"all eval spans must set the parent to the experiment id");
8787
if (span.getParentSpanId().equals(SpanId.getInvalid())) {
8888
numRootSpans.incrementAndGet();
89-
var inputJson =
89+
var input =
9090
fromJson(
9191
span.getAttributes()
9292
.get(AttributeKey.stringKey("braintrust.input_json")),
93-
Map.class);
94-
assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson);
93+
String.class);
94+
assertTrue(
95+
input.equals("strawberry") || input.equals("asparagus"),
96+
"invalid input: " + input);
9597

9698
var expected =
9799
fromJson(
98100
span.getAttributes()
99-
.get(AttributeKey.stringKey("braintrust.expected")),
101+
.get(AttributeKey.stringKey("braintrust.expected_json")),
100102
String.class);
101103
assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected);
102104

103-
var outputJson =
105+
var output =
104106
fromJson(
105107
span.getAttributes()
106108
.get(AttributeKey.stringKey("braintrust.output_json")),
107-
Map.class);
108-
var output = outputJson.get("output");
109-
assertNotNull(output, "invlaid output: " + outputJson);
110-
assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output);
109+
String.class);
110+
assertNotNull(output, "invalid output: " + output);
111+
assertTrue(isFruitOrVegetable(output), "invalid output: " + output);
112+
} else if ("task".equals(span.getName())) {
113+
// task span carries its own bare input/output
114+
var input =
115+
fromJson(
116+
span.getAttributes()
117+
.get(AttributeKey.stringKey("braintrust.input_json")),
118+
String.class);
119+
assertTrue(
120+
input.equals("strawberry") || input.equals("asparagus"),
121+
"invalid task input: " + input);
122+
var output =
123+
fromJson(
124+
span.getAttributes()
125+
.get(AttributeKey.stringKey("braintrust.output_json")),
126+
String.class);
127+
assertTrue(isFruitOrVegetable(output), "invalid task output: " + output);
111128
}
112129
}
113130
assertEquals(2, numRootSpans.get(), "each case should make a root span");
@@ -315,8 +332,7 @@ public void evalRootSpanPassesOriginIfPresent() {
315332
var inputJson =
316333
span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"));
317334
assertNotNull(inputJson);
318-
fromJson(inputJson, Map.class);
319-
var input = (String) (fromJson(inputJson, Map.class)).get("input");
335+
var input = fromJson(inputJson, String.class);
320336
assertNotNull(input);
321337
var origin = span.getAttributes().get(AttributeKey.stringKey("braintrust.origin"));
322338
switch (input) {
@@ -481,14 +497,11 @@ void evalContinuesWhenTaskThrows() {
481497
erroredRootSpan.getStatus().getDescription().contains("task failed on bad-input"),
482498
"root span error should contain the exception message");
483499

484-
// The errored root span should have output: null
485-
var erroredOutputJson =
486-
fromJson(
487-
erroredRootSpan
488-
.getAttributes()
489-
.get(AttributeKey.stringKey("braintrust.output_json")),
490-
Map.class);
491-
assertNull(erroredOutputJson.get("output"), "errored case output should be null");
500+
assertNull(
501+
erroredRootSpan
502+
.getAttributes()
503+
.get(AttributeKey.stringKey("braintrust.output_json")),
504+
"errors should not set output json");
492505

493506
// Find the task span for the errored case (child of errored root, type=task, status=ERROR)
494507
var erroredTaskSpan =
@@ -519,6 +532,11 @@ void evalContinuesWhenTaskThrows() {
519532
assertTrue(
520533
erroredTaskSpan.getEvents().stream().anyMatch(e -> e.getName().equals("exception")),
521534
"task span should have an exception event");
535+
assertNull(
536+
erroredTaskSpan
537+
.getAttributes()
538+
.get(AttributeKey.stringKey("braintrust.output_json")),
539+
"errors should not set output json");
522540

523541
// The errored case should still have score spans (from scoreForTaskException default = 0.0)
524542
var erroredScoreSpans =
@@ -560,14 +578,13 @@ void evalContinuesWhenTaskThrows() {
560578
.filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
561579
.findFirst()
562580
.orElseThrow(() -> new AssertionError("expected a successful root span"));
563-
var successOutputJson =
581+
var successOutput =
564582
fromJson(
565583
successRootSpan
566584
.getAttributes()
567585
.get(AttributeKey.stringKey("braintrust.output_json")),
568-
Map.class);
569-
assertEquals(
570-
"result", successOutputJson.get("output"), "successful case should have output");
586+
String.class);
587+
assertEquals("result", successOutput, "successful case should have output");
571588
}
572589

573590
@Test

0 commit comments

Comments
 (0)