Skip to content

Commit d4ad8d3

Browse files
committed
prompt fetching impl
1 parent 5a7dc09 commit d4ad8d3

19 files changed

Lines changed: 1052 additions & 70 deletions

‎README.md‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ dependencies {
3131
### Evals
3232

3333
```java
34-
var config = BraintrustConfig.fromEnvironment();
35-
var openTelemetry = BraintrustTracing.of(config, true);
34+
var braintrust = Braintrust.get();
35+
var openTelemetry = braintrust.openTelemetryCreate();
3636
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
3737

3838
Function<String, String> getFoodType =
@@ -49,11 +49,8 @@ Function<String, String> getFoodType =
4949
return response.choices().get(0).message().content().orElse("").toLowerCase();
5050
};
5151

52-
var eval =
53-
Eval.<String, String>builder()
52+
var eval = braintrust.<String, String>evalBuilder()
5453
.name("java-eval-x-" + System.currentTimeMillis())
55-
.tracer(BraintrustTracing.getTracer(openTelemetry))
56-
.config(config)
5754
.cases(
5855
EvalCase.of("asparagus", "vegetable"),
5956
EvalCase.of("banana", "fruit"))
@@ -73,8 +70,8 @@ System.out.println("\n\n" + result.createReportString());
7370
### OpenAI Tracing
7471

7572
```java
76-
var braintrustConfig = BraintrustConfig.fromEnvironment();
77-
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
73+
var braintrust = Braintrust.get();
74+
var openTelemetry = braintrust.openTelemetryCreate();
7875
OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
7976

8077
var request =

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ jar {
219219
'Implementation-Title': 'Braintrust Java SDK',
220220
'Implementation-Version': version,
221221
'Implementation-Vendor': 'Braintrust',
222-
'Main-Class': 'dev.braintrust.trace.SDKMain'
222+
'Main-Class': 'dev.braintrust.SDKMain'
223223
)
224224
}
225225
}

‎examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@
44
import com.anthropic.client.okhttp.AnthropicOkHttpClient;
55
import com.anthropic.models.messages.MessageCreateParams;
66
import com.anthropic.models.messages.Model;
7-
import dev.braintrust.config.BraintrustConfig;
7+
import dev.braintrust.Braintrust;
88
import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic;
9-
import dev.braintrust.trace.BraintrustTracing;
109

1110
/** Basic OTel + Anthropic instrumentation example */
1211
public class AnthropicInstrumentationExample {
@@ -18,9 +17,9 @@ public static void main(String[] args) throws Exception {
1817
+ " fail.\n");
1918
}
2019

21-
var braintrustConfig = BraintrustConfig.fromEnvironment();
22-
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
23-
var tracer = BraintrustTracing.getTracer(openTelemetry);
20+
var braintrust = Braintrust.get();
21+
var openTelemetry = braintrust.openTelemetryCreate();
22+
var tracer = openTelemetry.getTracer("my-instrumentation");
2423

2524
// Wrap Anthropic client with Braintrust instrumentation
2625
AnthropicClient anthropicClient =
@@ -35,7 +34,7 @@ public static void main(String[] args) throws Exception {
3534
}
3635

3736
var url =
38-
braintrustConfig.fetchProjectURI()
37+
braintrust.projectUri()
3938
+ "/logs?r=%s&s=%s"
4039
.formatted(
4140
rootSpan.getSpanContext().getTraceId(),

‎examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package dev.braintrust.examples;
22

3-
import dev.braintrust.config.BraintrustConfig;
4-
import dev.braintrust.trace.BraintrustTracing;
3+
import dev.braintrust.Braintrust;
54
import io.opentelemetry.api.GlobalOpenTelemetry;
65
import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
76
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
@@ -51,8 +50,8 @@ public static void main(String[] args) throws Exception {
5150
// NOTE: there are many ways to set up otel builders, etc.
5251
// The important line is here: call enable with your otel builders and braintrust will
5352
// export open telemetry data in addition to your existing setup
54-
var braintrustConfig = BraintrustConfig.fromEnvironment();
55-
BraintrustTracing.enable(braintrustConfig, tracerBuilder, loggerBuilder, meterBuilder);
53+
var braintrust = Braintrust.get();
54+
braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder);
5655

5756
var openTelemetry =
5857
OpenTelemetrySdk.builder()
@@ -74,7 +73,7 @@ public static void main(String[] args) throws Exception {
7473
span.end();
7574
}
7675
var url =
77-
braintrustConfig.fetchProjectURI()
76+
braintrust.projectUri()
7877
+ "/logs?r=%s&s=%s"
7978
.formatted(
8079
span.getSpanContext().getTraceId(),

‎examples/src/main/java/dev/braintrust/examples/ExperimentExample.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,16 @@
33
import com.openai.client.okhttp.OpenAIOkHttpClient;
44
import com.openai.models.ChatModel;
55
import com.openai.models.chat.completions.ChatCompletionCreateParams;
6-
import dev.braintrust.config.BraintrustConfig;
7-
import dev.braintrust.eval.Eval;
6+
import dev.braintrust.Braintrust;
87
import dev.braintrust.eval.EvalCase;
98
import dev.braintrust.eval.Scorer;
109
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
11-
import dev.braintrust.trace.BraintrustTracing;
1210
import java.util.function.Function;
1311

1412
public class ExperimentExample {
1513
public static void main(String[] args) throws Exception {
16-
var config = BraintrustConfig.fromEnvironment();
17-
var openTelemetry = BraintrustTracing.of(config, true);
14+
var braintrust = Braintrust.get();
15+
var openTelemetry = braintrust.openTelemetryCreate();
1816
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
1917

2018
Function<String, String> getFoodType =
@@ -32,13 +30,12 @@ public static void main(String[] args) throws Exception {
3230
};
3331

3432
var eval =
35-
Eval.<String, String>builder()
33+
braintrust
34+
.<String, String>evalBuilder()
3635
.name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a
3736
// constant, additional runs
3837
// will append new cases to
3938
// the same experiment
40-
.tracer(BraintrustTracing.getTracer(openTelemetry))
41-
.config(config)
4239
.cases(
4340
EvalCase.of("strawberry", "fruit"),
4441
EvalCase.of("asparagus", "vegetable"),

‎examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
/** Basic OTel + OpenAI instrumentation example */
1212
public class OpenAIInstrumentationExample {
13+
1314
public static void main(String[] args) throws Exception {
1415
if (null == System.getenv("OPENAI_API_KEY")) {
1516
System.err.println(

‎examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import com.openai.client.okhttp.OpenAIOkHttpClient;
88
import dev.braintrust.Braintrust;
99
import dev.braintrust.prompt.BraintrustPromptLoader.PromptLoadRequest;
10+
import java.util.Map;
1011

1112
public class PromptFetchingExample {
1213
public static void main(String... args) {
@@ -20,24 +21,34 @@ public static void main(String... args) {
2021
OpenAIClient openAIClient = wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
2122

2223
{ // simple example
23-
var prompt = braintrust.promptLoader().load("my-prompt-slug");
24+
var prompt = braintrust.promptLoader().load("kind-greeter-69d2");
2425
var response =
25-
openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt));
26-
System.out.println("got response: %s".formatted(response));
26+
openAIClient
27+
.chat()
28+
.completions()
29+
.create(
30+
buildChatCompletionsPrompt(
31+
prompt, Map.of("name", "Sam Malone")));
32+
System.out.println("got response: %s".formatted(response.choices().get(0).message()));
2733
}
2834
{ // more complex prompt options
2935
var prompt =
3036
braintrust
3137
.promptLoader()
3238
.load(
3339
PromptLoadRequest.builder()
34-
.projectName("my-project")
35-
.promptSlug("my-prompt-slug")
36-
.version("5878bd218351fb8e")
40+
.projectName("andrew-misc")
41+
.promptSlug("unkind-greeter-fd4c")
42+
.version("cbbc711da9f7d445")
3743
.defaults("model", "gpt-3.5-turbo")
3844
.build());
3945
var response =
40-
openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt));
46+
openAIClient
47+
.chat()
48+
.completions()
49+
.create(
50+
buildChatCompletionsPrompt(
51+
prompt, Map.of("name", "Frasier Crane")));
4152
System.out.println("got response: %s".formatted(response));
4253
}
4354

‎examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
package dev.braintrust.examples;
22

3-
import dev.braintrust.config.BraintrustConfig;
4-
import dev.braintrust.trace.BraintrustTracing;
3+
import dev.braintrust.Braintrust;
54

65
public class SimpleOpenTelemetryExample {
76
public static void main(String[] args) throws Exception {
8-
var braintrustConfig = BraintrustConfig.fromEnvironment();
9-
var openTelemetry = BraintrustTracing.quickstart();
10-
var tracer = BraintrustTracing.getTracer(openTelemetry);
7+
var braintrust = Braintrust.get();
8+
var openTelemetry = braintrust.openTelemetryCreate();
9+
var tracer = openTelemetry.getTracer("my-instrumentation");
1110

1211
var span = tracer.spanBuilder("hello-java").startSpan();
1312
try (var ignored = span.makeCurrent()) {
@@ -18,7 +17,7 @@ public static void main(String[] args) throws Exception {
1817
span.end();
1918
}
2019
var url =
21-
braintrustConfig.fetchProjectURI()
20+
braintrust.projectUri()
2221
+ "/logs?r=%s&s=%s"
2322
.formatted(
2423
span.getSpanContext().getTraceId(),

‎src/main/java/dev/braintrust/Braintrust.java‎

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@
22

33
import dev.braintrust.api.BraintrustApiClient;
44
import dev.braintrust.config.BraintrustConfig;
5+
import dev.braintrust.eval.Eval;
56
import dev.braintrust.prompt.BraintrustPromptLoader;
7+
import dev.braintrust.trace.BraintrustTracing;
68
import io.opentelemetry.api.OpenTelemetry;
79
import io.opentelemetry.sdk.logs.SdkLoggerProviderBuilder;
810
import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
911
import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
1012
import java.net.URI;
13+
import java.util.concurrent.atomic.AtomicReference;
1114
import javax.annotation.Nonnull;
1215
import lombok.Getter;
1316
import lombok.experimental.Accessors;
17+
import lombok.extern.slf4j.Slf4j;
1418

1519
/**
1620
* Main entry point for the Braintrust SDK.
@@ -30,23 +34,43 @@
3034
* @see #openTelemetryEnable(SdkTracerProviderBuilder, SdkLoggerProviderBuilder,
3135
* SdkMeterProviderBuilder)
3236
*/
37+
@Slf4j
3338
public class Braintrust {
39+
private static final String SDK_VERSION = SDKMain.loadVersionFromProperties();
40+
private static final AtomicReference<Braintrust> instance = new AtomicReference<>();
41+
3442
/**
3543
* get or create the global braintrust instance. Most users will want to use this method to
3644
* access the Braintrust SDK.
3745
*/
3846
public static Braintrust get() {
39-
throw new RuntimeException("TODO");
47+
var current = instance.get();
48+
if (null == current) {
49+
return get(BraintrustConfig.fromEnvironment());
50+
} else {
51+
return current;
52+
}
4053
}
4154

4255
/** get or create the global braintrust instance from the given config */
4356
public static Braintrust get(BraintrustConfig config) {
44-
throw new RuntimeException("TODO");
57+
var current = instance.get();
58+
if (null == current) {
59+
var success = instance.compareAndSet(null, of(config));
60+
if (success) {
61+
log.info("initialized global Braintrust sdk {}", SDK_VERSION);
62+
}
63+
return instance.get();
64+
} else {
65+
return current;
66+
}
4567
}
4668

4769
/** Create a new Braintrust instance from the given config */
4870
public static Braintrust of(BraintrustConfig config) {
49-
throw new RuntimeException("TODO");
71+
BraintrustApiClient apiClient = BraintrustApiClient.of(config);
72+
BraintrustPromptLoader promptLoader = BraintrustPromptLoader.of(config, apiClient);
73+
return new Braintrust(config, apiClient, promptLoader);
5074
}
5175

5276
@Getter
@@ -71,21 +95,53 @@ private Braintrust(
7195
}
7296

7397
public URI projectUri() {
74-
throw new RuntimeException("TODO");
98+
// TODO cache?
99+
return config.fetchProjectURI();
75100
}
76101

102+
/**
103+
* Quick start method that sets up global OpenTelemetry with this Braintrust. <br>
104+
* <br>
105+
* If you're looking for more options for configuring Braintrust/OpenTelemetry, consult the
106+
* `enable` method.
107+
*/
77108
public OpenTelemetry openTelemetryCreate() {
78-
throw new RuntimeException("TODO");
109+
return openTelemetryCreate(true);
79110
}
80111

112+
/**
113+
* Quick start method that sets up OpenTelemetry with this Braintrust. <br>
114+
* <br>
115+
* If you're looking for more options for configuring Braintrust and OpenTelemetry, consult the
116+
* `enable` method.
117+
*/
81118
public OpenTelemetry openTelemetryCreate(boolean registerGlobal) {
82-
throw new RuntimeException("TODO");
119+
return BraintrustTracing.of(this.config, registerGlobal);
83120
}
84121

122+
/**
123+
* Add braintrust to existing open telemetry builders <br>
124+
* <br>
125+
* This method provides the most options for configuring Braintrust and OpenTelemetry. If you're
126+
* looking for a more user-friendly setup, consult the `quickstart` methods. <br>
127+
* <br>
128+
* NOTE: if your otel setup does not have any particular builder, pass an instance of the
129+
* default provider builder. E.g. `SdkMeterProvider.builder()` <br>
130+
* <br>
131+
* NOTE: This method should only be invoked once. Enabling Braintrust multiple times is
132+
* unsupported and may lead to undesired behavior
133+
*/
85134
public void openTelemetryEnable(
86135
@Nonnull SdkTracerProviderBuilder tracerProviderBuilder,
87136
@Nonnull SdkLoggerProviderBuilder loggerProviderBuilder,
88137
@Nonnull SdkMeterProviderBuilder meterProviderBuilder) {
89-
throw new RuntimeException("TODO");
138+
BraintrustTracing.enable(
139+
this.config, tracerProviderBuilder, loggerProviderBuilder, meterProviderBuilder);
140+
}
141+
142+
/** Create a new eval builder */
143+
public <INPUT, OUTPUT> Eval.Builder<INPUT, OUTPUT> evalBuilder() {
144+
return (Eval.Builder<INPUT, OUTPUT>)
145+
Eval.builder().config(this.config).apiClient(this.apiClient);
90146
}
91147
}

src/main/java/dev/braintrust/trace/SDKMain.java renamed to src/main/java/dev/braintrust/SDKMain.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.braintrust.trace;
1+
package dev.braintrust;
22

33
import java.util.Properties;
44

0 commit comments

Comments
 (0)