Skip to content

Commit b212a77

Browse files
authored
fix: conform AI SDK evaluate traces to question spans
1 parent 8a9b440 commit b212a77

11 files changed

Lines changed: 446 additions & 320 deletions

‎e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json‎

Lines changed: 43 additions & 37 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt‎

Lines changed: 42 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -11,57 +11,61 @@ span_tree:
1111
│ "operation": "evaluate",
1212
│ "testRunId": "<run:1>"
1313
│ }
14-
│ └── evaluate [llm]
14+
│ └── evaluate [question]
1515
│ input: {
16-
│ "questions": {
17-
│ "category": {
16+
│ "questions": [
17+
│ {
1818
│ "criteria": {
1919
│ "billing": "Payments and refunds",
2020
│ "technical": "Software problems"
2121
│ },
2222
│ "instructions": "Which team should handle this request?",
2323
│ "type": "choice"
2424
│ },
25-
│ "duplicate": {
26-
│ "instructions": "Does the customer report a duplicate charge?",
27-
│ "type": "boolean"
28-
│ },
29-
│ "urgency": {
25+
│ {
3026
│ "criteria": [
3127
│ "Low",
3228
│ "Medium",
3329
│ "High"
3430
│ ],
3531
│ "instructions": "How urgent is this request?",
3632
│ "type": "score"
33+
│ },
34+
│ {
35+
│ "instructions": "Does the customer report a duplicate charge?",
36+
│ "type": "boolean"
3737
│ }
38-
│ },
38+
│ ],
3939
│ "state": {
4040
│ "message": "I was charged twice for my subscription. Please refund the duplicate charge."
4141
│ }
4242
│ }
4343
│ output: {
44-
│ "category": {
45-
│ "choice": "billing",
46-
│ "probabilities": {
47-
│ "billing": 1,
48-
│ "technical": 0
44+
│ "answers": [
45+
│ {
46+
│ "choice": "billing",
47+
│ "confidence": 1,
48+
│ "probabilities": {
49+
│ "billing": 1,
50+
│ "technical": 0
51+
│ },
52+
│ "type": "choice"
4953
│ },
50-
│ "type": "choice"
51-
│ },
52-
│ "duplicate": {
53-
│ "probability": 0.99,
54-
│ "type": "boolean"
55-
│ },
56-
│ "urgency": {
57-
│ "probabilities": {
58-
│ "0": 0.03,
59-
│ "1": 0.84,
60-
│ "2": 0.13
54+
│ {
55+
│ "probability": 0.99,
56+
│ "type": "boolean"
6157
│ },
62-
│ "score": 1.1,
63-
│ "type": "score"
64-
│ }
58+
│ {
59+
│ "confidence": 0.76,
60+
│ "probabilities": {
61+
│ "0": 0.03,
62+
│ "1": 0.84,
63+
│ "2": 0.13
64+
│ },
65+
│ "score": 1.1,
66+
│ "type": "score"
67+
│ }
68+
│ ]
6569
│ }
6670
│ metadata: {
6771
│ "braintrust": {
@@ -89,21 +93,23 @@ span_tree:
8993
│ "operation": "evaluate-string",
9094
│ "testRunId": "<run:1>"
9195
│ }
92-
│ └── evaluate [llm]
96+
│ └── evaluate [question]
9397
│ input: {
94-
│ "questions": {
95-
│ "positive": {
98+
│ "questions": [
99+
│ {
96100
│ "instructions": "Was the delivery successful?",
97101
│ "type": "boolean"
98102
│ }
99-
│ },
103+
│ ],
100104
│ "state": "The package arrived intact and on time."
101105
│ }
102106
│ output: {
103-
│ "positive": {
104-
│ "probability": 0.98,
105-
│ "type": "boolean"
106-
│ }
107+
│ "answers": [
108+
│ {
109+
│ "probability": 0.98,
110+
│ "type": "boolean"
111+
│ }
112+
│ ]
107113
│ }
108114
│ metadata: {
109115
│ "braintrust": {

‎e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json‎

Lines changed: 43 additions & 37 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)