From 9d260c4e8f7a5d8f04f04510a9ca013bef2ae401 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 11 Nov 2025 17:20:12 -0800 Subject: [PATCH 1/2] feat: deprecate v0 package structure --- .../java/examples/simple_chat/SimpleChat.java | 10 +- .../judgeval-instrumentation-openai/pom.xml | 151 ----- .../openai/OpenAITelemetry.java | 17 - judgeval-java/pom.xml | 2 +- .../com/judgmentlabs/judgeval/Judgeval.java | 111 +++- .../judgeval/data/APIScorerType.java | 10 +- .../judgeval/data/EvaluationRunBuilder.java | 83 --- .../judgmentlabs/judgeval/data/Example.java | 32 +- .../judgeval/data/ScorerData.java | 88 ++- .../judgeval/data/ScoringResult.java | 50 +- .../data/TraceEvaluationRunBuilder.java | 77 --- .../{v1 => }/evaluation/Evaluation.java | 2 +- .../evaluation/EvaluationFactory.java | 2 +- .../judgeval/exceptions/JudgmentAPIError.java | 20 - .../exceptions/JudgmentRuntimeError.java | 11 - .../exceptions/JudgmentTestError.java | 11 - .../judgeval/{v1 => }/package-info.java | 0 .../judgeval/scorers/APIScorer.java | 21 +- .../judgeval/scorers/BaseScorer.java | 11 +- .../{v1 => }/scorers/ScorersFactory.java | 8 +- .../api_scorers/AnswerCorrectnessScorer.java | 42 -- .../api_scorers/AnswerRelevancyScorer.java | 42 -- .../api_scorers/FaithfulnessScorer.java | 42 -- .../InstructionAdherenceScorer.java | 43 -- .../custom_scorer/CustomScorer.java | 72 --- .../prompt_scorer/BasePromptScorer.java | 143 ----- .../prompt_scorer/PromptScorer.java | 95 --- .../prompt_scorer/TracePromptScorer.java | 96 --- .../built_in/AnswerCorrectnessScorer.java | 6 +- .../built_in/AnswerRelevancyScorer.java | 6 +- .../built_in/BuiltInScorersFactory.java | 2 +- .../scorers/built_in/FaithfulnessScorer.java | 6 +- .../scorers/custom_scorer/CustomScorer.java | 6 +- .../custom_scorer/CustomScorerFactory.java | 2 +- .../scorers/prompt_scorer/PromptScorer.java | 6 +- .../prompt_scorer/PromptScorerFactory.java | 2 +- .../judgeval/tracer/BaseTracer.java | 351 +++++------ .../judgeval/tracer/ISerializer.java | 20 +- .../judgeval/tracer/JudgevalTraceKeys.java | 51 -- .../judgmentlabs/judgeval/tracer/Tracer.java | 229 ++++--- .../judgeval/tracer/TracerConfiguration.java | 148 ----- .../{v1 => }/tracer/TracerFactory.java | 2 +- .../judgmentlabs/judgeval/v1/Judgeval.java | 117 ---- .../judgeval/v1/data/APIScorerType.java | 28 - .../judgeval/v1/data/Example.java | 74 --- .../judgeval/v1/data/ScorerData.java | 165 ----- .../judgeval/v1/data/ScoringResult.java | 94 --- .../judgeval/v1/scorers/APIScorer.java | 153 ----- .../judgeval/v1/scorers/BaseScorer.java | 22 - .../judgeval/v1/tracer/BaseTracer.java | 563 ------------------ .../judgeval/v1/tracer/ISerializer.java | 30 - .../judgeval/v1/tracer/Tracer.java | 211 ------- .../exporters/JudgmentSpanExporter.java | 179 ------ .../v1/tracer/exporters/NoOpSpanExporter.java | 45 -- .../judgeval/{v1 => }/JudgevalTest.java | 6 +- .../judgeval/{v1 => }/data/ExampleTest.java | 2 +- .../evaluation/EvaluationFactoryTest.java | 2 +- .../{v1 => }/scorers/ScorersFactoryTest.java | 8 +- .../built_in/BuiltInScorersFactoryTest.java | 2 +- .../{v1 => }/tracer/BaseTracerTest.java | 6 +- .../{v1 => }/tracer/TracerFactoryTest.java | 2 +- .../judgeval/{v1 => }/tracer/TracerTest.java | 2 +- package.json | 20 +- 63 files changed, 651 insertions(+), 3209 deletions(-) delete mode 100644 instrumentation/judgeval-instrumentation-openai/pom.xml delete mode 100644 instrumentation/judgeval-instrumentation-openai/src/main/java/com/judgmentlabs/judgeval/instrumentation/openai/OpenAITelemetry.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/EvaluationRunBuilder.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRunBuilder.java rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/evaluation/Evaluation.java (95%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/evaluation/EvaluationFactory.java (91%) delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentRuntimeError.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentTestError.java rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/package-info.java (100%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/ScorersFactory.java (81%) delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/custom_scorer/CustomScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/built_in/AnswerCorrectnessScorer.java (91%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/built_in/AnswerRelevancyScorer.java (91%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/built_in/BuiltInScorersFactory.java (93%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/built_in/FaithfulnessScorer.java (91%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/custom_scorer/CustomScorer.java (89%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/custom_scorer/CustomScorerFactory.java (93%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/prompt_scorer/PromptScorer.java (95%) rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/scorers/prompt_scorer/PromptScorerFactory.java (98%) delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java rename judgeval-java/src/main/java/com/judgmentlabs/judgeval/{v1 => }/tracer/TracerFactory.java (92%) delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/Judgeval.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/APIScorerType.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/Example.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScorerData.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScoringResult.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/APIScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/BaseScorer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/ISerializer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/Tracer.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/JudgmentSpanExporter.java delete mode 100644 judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/NoOpSpanExporter.java rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/JudgevalTest.java (91%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/data/ExampleTest.java (96%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/evaluation/EvaluationFactoryTest.java (95%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/scorers/ScorersFactoryTest.java (82%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/scorers/built_in/BuiltInScorersFactoryTest.java (93%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/tracer/BaseTracerTest.java (96%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/tracer/TracerFactoryTest.java (94%) rename judgeval-java/src/test/java/com/judgmentlabs/judgeval/{v1 => }/tracer/TracerTest.java (97%) diff --git a/examples/src/main/java/examples/simple_chat/SimpleChat.java b/examples/src/main/java/examples/simple_chat/SimpleChat.java index d0306c9..3198025 100644 --- a/examples/src/main/java/examples/simple_chat/SimpleChat.java +++ b/examples/src/main/java/examples/simple_chat/SimpleChat.java @@ -2,16 +2,13 @@ import java.time.Duration; -import com.judgmentlabs.judgeval.instrumentation.openai.OpenAITelemetry; -import com.judgmentlabs.judgeval.v1.Judgeval; -import com.judgmentlabs.judgeval.v1.data.Example; +import com.judgmentlabs.judgeval.Judgeval; +import com.judgmentlabs.judgeval.data.Example; import com.openai.client.OpenAIClient; import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; -import io.opentelemetry.api.GlobalOpenTelemetry; - public class SimpleChat { public static void main(String[] args) { var client = Judgeval.builder() @@ -20,8 +17,7 @@ public static void main(String[] args) { .build(); var tracer = client.tracer().create().projectName("SimpleChat-Java").build(); - OpenAIClient baseClient = OpenAIOkHttpClient.fromEnv(); - var otelClient = OpenAITelemetry.builder(GlobalOpenTelemetry.get()).build().wrap(baseClient); + OpenAIClient otelClient = OpenAIOkHttpClient.fromEnv(); tracer.span("chat.session", () -> { diff --git a/instrumentation/judgeval-instrumentation-openai/pom.xml b/instrumentation/judgeval-instrumentation-openai/pom.xml deleted file mode 100644 index 00c971f..0000000 --- a/instrumentation/judgeval-instrumentation-openai/pom.xml +++ /dev/null @@ -1,151 +0,0 @@ - - - 4.0.0 - com.judgmentlabs - judgeval-instrumentation-openai - 0.0.2 - jar - Judgeval Instrumentation OpenAI - OpenAI instrumentation for Judgeval Java - https://github.com/judgmentlabs/judgeval-java - - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - judgmentlabs - Judgment Labs - contact@judgmentlabs.com - Judgment Labs - https://judgmentlabs.ai - - - - - scm:git:git://github.com/judgmentlabs/judgeval-java.git - scm:git:ssh://github.com:judgmentlabs/judgeval-java.git - https://github.com/judgmentlabs/judgeval-java/tree/main - - - - 21 - 21 - UTF-8 - - - - - io.opentelemetry.instrumentation - opentelemetry-openai-java-1.1 - 2.21.0-alpha - - - - - - com.diffplug.spotless - spotless-maven-plugin - 2.43.0 - - - - 4.28 - ../../.vscode/java-formatter.xml - - - - java,javax,org,com - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - - 21 - 21 - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.8.0 - true - - central - true - uploaded - 600 - 10 - - - - - org.apache.maven.plugins - maven-source-plugin - 3.3.0 - - - attach-sources - - jar-no-fork - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.6.3 - - - attach-javadocs - - jar - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.1.0 - - - sign-artifacts - verify - - sign - - - - --pinentry-mode - loopback - - - - - - - - - - - central - Maven Central Repository - https://central.sonatype.com/api/v1/publisher/deployments - - - diff --git a/instrumentation/judgeval-instrumentation-openai/src/main/java/com/judgmentlabs/judgeval/instrumentation/openai/OpenAITelemetry.java b/instrumentation/judgeval-instrumentation-openai/src/main/java/com/judgmentlabs/judgeval/instrumentation/openai/OpenAITelemetry.java deleted file mode 100644 index e50daa8..0000000 --- a/instrumentation/judgeval-instrumentation-openai/src/main/java/com/judgmentlabs/judgeval/instrumentation/openai/OpenAITelemetry.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.judgmentlabs.judgeval.instrumentation.openai; - -import io.opentelemetry.api.OpenTelemetry; - -public final class OpenAITelemetry { - private OpenAITelemetry() { - } - - public static io.opentelemetry.instrumentation.openai.v1_1.OpenAITelemetryBuilder builder( - OpenTelemetry openTelemetry) { - return io.opentelemetry.instrumentation.openai.v1_1.OpenAITelemetry.builder(openTelemetry); - } - - public static io.opentelemetry.instrumentation.openai.v1_1.OpenAITelemetry create(OpenTelemetry openTelemetry) { - return io.opentelemetry.instrumentation.openai.v1_1.OpenAITelemetry.create(openTelemetry); - } -} diff --git a/judgeval-java/pom.xml b/judgeval-java/pom.xml index e7a99d0..7c070ff 100644 --- a/judgeval-java/pom.xml +++ b/judgeval-java/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.judgmentlabs judgeval-java - 0.3.0 + 0.4.0 jar Judgeval Java Java SDK for Judgeval diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/Judgeval.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/Judgeval.java index 926ca88..53e0bed 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/Judgeval.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/Judgeval.java @@ -1,16 +1,115 @@ -package com.judgmentlabs.judgeval; +package com.judgmentlabs.judgeval.v1; -public final class Judgeval extends com.judgmentlabs.judgeval.v1.Judgeval { - private Judgeval(Builder builder) { - super(builder); +import java.util.Objects; + +import com.judgmentlabs.judgeval.Env; +import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; +import com.judgmentlabs.judgeval.evaluation.EvaluationFactory; +import com.judgmentlabs.judgeval.scorers.ScorersFactory; +import com.judgmentlabs.judgeval.tracer.TracerFactory; + +/** + * Main entry point for the Judgment SDK. Provides access to tracer, scorer, and + * evaluation factories. + */ +public class Judgeval { + private final String apiKey; + private final String organizationId; + private final String apiUrl; + private final JudgmentSyncClient internalClient; + + protected Judgeval(Builder builder) { + this.apiKey = Objects.requireNonNull(builder.apiKey, "apiKey required"); + this.organizationId = Objects.requireNonNull(builder.organizationId, "organizationId required"); + this.apiUrl = builder.apiUrl != null ? builder.apiUrl : Env.JUDGMENT_API_URL; + this.internalClient = new JudgmentSyncClient(apiUrl, apiKey, organizationId); + } + + /** + * Returns a factory for creating tracers. + * + * @return the tracer factory + */ + public TracerFactory tracer() { + return new TracerFactory(internalClient); + } + + /** + * Returns a factory for creating scorers. + * + * @return the scorer factory + */ + public ScorersFactory scorers() { + return new ScorersFactory(internalClient); + } + + /** + * Returns a factory for creating evaluations. + * + * @return the evaluation factory + */ + public EvaluationFactory evaluation() { + return new EvaluationFactory(internalClient); } + /** + * Creates a new builder for configuring a Judgeval. + * + * @return a new builder instance + */ public static Builder builder() { return new Builder(); } - public static final class Builder extends com.judgmentlabs.judgeval.v1.Judgeval.Builder { - @Override + /** + * Builder for configuring and creating Judgeval instances. + */ + public static class Builder { + private String apiKey = Env.JUDGMENT_API_KEY; + private String organizationId = Env.JUDGMENT_ORG_ID; + private String apiUrl = Env.JUDGMENT_API_URL; + + /** + * Sets the API key for authentication. + * + * @param apiKey + * the API key + * @return this builder + */ + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + /** + * Sets the organization ID. + * + * @param organizationId + * the organization ID + * @return this builder + */ + public Builder organizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + + /** + * Sets the API URL. + * + * @param apiUrl + * the API URL + * @return this builder + */ + public Builder apiUrl(String apiUrl) { + this.apiUrl = apiUrl; + return this; + } + + /** + * Builds and returns a new Judgeval instance. + * + * @return the configured Judgeval + */ public Judgeval build() { return new Judgeval(this); } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java index e3765f7..a978424 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java @@ -1,22 +1,14 @@ package com.judgmentlabs.judgeval.data; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.data.APIScorerType} - * instead. + * Available types of API-based scorers. */ -@Deprecated public enum APIScorerType { PROMPT_SCORER("Prompt Scorer"), TRACE_PROMPT_SCORER("Trace Prompt Scorer"), FAITHFULNESS("Faithfulness"), ANSWER_RELEVANCY("Answer Relevancy"), ANSWER_CORRECTNESS("Answer Correctness"), - INSTRUCTION_ADHERENCE("Instruction Adherence"), - EXECUTION_ORDER("Execution Order"), - DERAILMENT("Derailment"), - TOOL_ORDER("Tool Order"), - CLASSIFIER("Classifier"), - TOOL_DEPENDENCY("Tool Dependency"), CUSTOM("Custom"); private final String value; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/EvaluationRunBuilder.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/EvaluationRunBuilder.java deleted file mode 100644 index f6cc75e..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/EvaluationRunBuilder.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.scorers.BaseScorer; -import com.judgmentlabs.judgeval.scorers.api_scorers.custom_scorer.CustomScorer; - -/** - * @deprecated Replaced by - * com.judgmentlabs.judgeval.v1.data.EvaluationRunBuilder - */ -@Deprecated -public class EvaluationRunBuilder { - private String projectName; - private String evalName; - private String model; - private com.judgmentlabs.judgeval.internal.api.models.Example example; - private String traceId; - private String spanId; - private final List scorers = new ArrayList<>(); - - public EvaluationRunBuilder projectName(String v) { - this.projectName = v; - return this; - } - - public EvaluationRunBuilder evalName(String v) { - this.evalName = v; - return this; - } - - public EvaluationRunBuilder model(String v) { - this.model = v; - return this; - } - - public EvaluationRunBuilder example(Example v) { - this.example = (com.judgmentlabs.judgeval.internal.api.models.Example) v; - return this; - } - - public EvaluationRunBuilder addScorer(BaseScorer v) { - this.scorers.add(v); - return this; - } - - public EvaluationRunBuilder trace(String traceId, String spanId) { - this.traceId = traceId; - this.spanId = spanId; - return this; - } - - public ExampleEvaluationRun build() { - ExampleEvaluationRun run = new ExampleEvaluationRun(); - run.setProjectName(projectName); - run.setEvalName(evalName); - run.setModel(model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL); - List judgment = new ArrayList<>(); - List custom = new ArrayList<>(); - for (BaseScorer s : scorers) { - if (s instanceof CustomScorer) - custom.add((com.judgmentlabs.judgeval.internal.api.models.BaseScorer) s); - else - judgment.add(s.getScorerConfig()); - } - run.setJudgmentScorers(judgment); - run.setCustomScorers(custom); - run.setExamples(List.of(example)); - run.setTraceId(traceId); - run.setTraceSpanId(spanId); - run.setId(UUID.randomUUID().toString()); - run.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - return run; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/Example.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/Example.java index ff0f53c..113a437 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/Example.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/Example.java @@ -4,9 +4,8 @@ import java.util.UUID; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.data.Example} instead. + * Represents an evaluation example with arbitrary properties. */ -@Deprecated public class Example extends com.judgmentlabs.judgeval.internal.api.models.Example { public Example() { @@ -18,10 +17,18 @@ public Example() { setName(null); } + /** + * Creates a new builder for configuring an Example. + * + * @return a new builder instance + */ public static Builder builder() { return new Builder(); } + /** + * Builder for configuring and creating Example instances. + */ public static final class Builder { private final Example example; @@ -29,16 +36,37 @@ private Builder() { this.example = new Example(); } + /** + * Sets a custom property on the example. + * + * @param key + * the property key + * @param value + * the property value + * @return this builder + */ public Builder property(String key, Object value) { example.setAdditionalProperty(key, value); return this; } + /** + * Sets the name of the example. + * + * @param name + * the example name + * @return this builder + */ public Builder name(String name) { example.setName(name); return this; } + /** + * Builds and returns the configured Example. + * + * @return the configured Example + */ public Example build() { return example; } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScorerData.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScorerData.java index aa94afb..e51c78b 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScorerData.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScorerData.java @@ -3,15 +3,22 @@ import java.util.Map; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.data.ScorerData} instead. + * Represents the result of a single scorer evaluation. */ -@Deprecated public class ScorerData extends com.judgmentlabs.judgeval.internal.api.models.ScorerData { + /** + * Creates a new builder for configuring ScorerData. + * + * @return a new builder instance + */ public static Builder builder() { return new Builder(); } + /** + * Builder for configuring and creating ScorerData instances. + */ public static final class Builder { private final ScorerData scorerData; @@ -19,51 +26,123 @@ private Builder() { this.scorerData = new ScorerData(); } + /** + * Sets the scorer name. + * + * @param name + * the scorer name + * @return this builder + */ public Builder name(String name) { scorerData.setName(name); return this; } + /** + * Sets the evaluation score. + * + * @param score + * the score value + * @return this builder + */ public Builder score(Double score) { scorerData.setScore(score); return this; } + /** + * Sets whether the evaluation succeeded. + * + * @param success + * true if evaluation succeeded + * @return this builder + */ public Builder success(Boolean success) { scorerData.setSuccess(success); return this; } + /** + * Sets the reason for the evaluation result. + * + * @param reason + * the evaluation reason + * @return this builder + */ public Builder reason(String reason) { scorerData.setReason(reason); return this; } + /** + * Sets the evaluation threshold. + * + * @param threshold + * the threshold value + * @return this builder + */ public Builder threshold(Double threshold) { scorerData.setThreshold(threshold); return this; } + /** + * Sets strict mode for evaluation. + * + * @param strictMode + * true for strict mode + * @return this builder + */ public Builder strictMode(Boolean strictMode) { scorerData.setStrictMode(strictMode); return this; } + /** + * Sets the model used for evaluation. + * + * @param evaluationModel + * the model name + * @return this builder + */ public Builder evaluationModel(String evaluationModel) { scorerData.setEvaluationModel(evaluationModel); return this; } + /** + * Sets an error message if evaluation failed. + * + * @param error + * the error message + * @return this builder + */ public Builder error(String error) { scorerData.setError(error); return this; } + /** + * Sets additional metadata for the evaluation. + * + * @param additionalMetadata + * the metadata map + * @return this builder + */ public Builder additionalMetadata(Map additionalMetadata) { scorerData.setAdditionalMetadata(additionalMetadata); return this; } + /** + * Adds a single metadata entry. + * + * @param key + * the metadata key + * @param value + * the metadata value + * @return this builder + */ public Builder metadata(String key, Object value) { if (scorerData.getAdditionalMetadata() == null) { scorerData.setAdditionalMetadata(new java.util.HashMap<>()); @@ -74,6 +153,11 @@ public Builder metadata(String key, Object value) { return this; } + /** + * Builds and returns the configured ScorerData. + * + * @return the configured ScorerData + */ public ScorerData build() { return scorerData; } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java index 60ae78e..1d00220 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java @@ -3,16 +3,22 @@ import java.util.List; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.data.ScoringResult} - * instead. + * Represents a collection of scorer evaluation results. */ -@Deprecated public class ScoringResult extends com.judgmentlabs.judgeval.internal.api.models.ScoringResult { + /** + * Creates a new builder for configuring a ScoringResult. + * + * @return a new builder instance + */ public static Builder builder() { return new Builder(); } + /** + * Builder for configuring and creating ScoringResult instances. + */ public static final class Builder { private final ScoringResult result; @@ -20,11 +26,25 @@ private Builder() { this.result = new ScoringResult(); } + /** + * Sets whether the overall evaluation succeeded. + * + * @param success + * true if evaluation succeeded + * @return this builder + */ public Builder success(Boolean success) { result.setSuccess(success); return this; } + /** + * Sets the list of scorer results. + * + * @param scorersData + * the list of scorer data + * @return this builder + */ public Builder scorersData(List scorersData) { @SuppressWarnings("unchecked") List internalList = (List) (List) scorersData; @@ -32,6 +52,13 @@ public Builder scorersData(List scorersData) { return this; } + /** + * Adds a single scorer result. + * + * @param scorerData + * the scorer data to add + * @return this builder + */ public Builder scorerData(ScorerData scorerData) { if (result.getScorersData() == null) { result.setScorersData(new java.util.ArrayList<>()); @@ -41,18 +68,25 @@ public Builder scorerData(ScorerData scorerData) { return this; } + /** + * Sets the data object for the evaluation. + * + * @param dataObject + * the example data + * @return this builder + */ public Builder dataObject(Example dataObject) { - // Store Example in additional properties since setDataObject - // expects TraceSpan - // This indicates a potential API design issue - ScoringResult may - // be - // trace-specific if (dataObject != null) { result.setAdditionalProperty("example", dataObject); } return this; } + /** + * Builds and returns the configured ScoringResult. + * + * @return the configured ScoringResult + */ public ScoringResult build() { return result; } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRunBuilder.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRunBuilder.java deleted file mode 100644 index 7c114eb..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRunBuilder.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; -import com.judgmentlabs.judgeval.scorers.BaseScorer; -import com.judgmentlabs.judgeval.scorers.api_scorers.custom_scorer.CustomScorer; - -/** - * @deprecated Replaced by - * com.judgmentlabs.judgeval.v1.data.TraceEvaluationRunBuilder - */ -@Deprecated -public class TraceEvaluationRunBuilder { - private String projectName; - private String evalName; - private String model; - private String traceId; - private String spanId; - private final List scorers = new ArrayList<>(); - - public TraceEvaluationRunBuilder projectName(String v) { - this.projectName = v; - return this; - } - - public TraceEvaluationRunBuilder evalName(String v) { - this.evalName = v; - return this; - } - - public TraceEvaluationRunBuilder model(String v) { - this.model = v; - return this; - } - - public TraceEvaluationRunBuilder trace(String traceId, String spanId) { - this.traceId = traceId; - this.spanId = spanId; - return this; - } - - public TraceEvaluationRunBuilder addScorer(BaseScorer v) { - this.scorers.add(v); - return this; - } - - public TraceEvaluationRun build() { - TraceEvaluationRun run = new TraceEvaluationRun(); - run.setProjectName(projectName); - run.setEvalName(evalName); - run.setModel(model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL); - List judgment = new ArrayList<>(); - List custom = new ArrayList<>(); - for (BaseScorer s : scorers) { - if (s instanceof CustomScorer) - custom.add((com.judgmentlabs.judgeval.internal.api.models.BaseScorer) s); - else - judgment.add(s.getScorerConfig()); - } - run.setJudgmentScorers(judgment); - run.setCustomScorers(custom); - run.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); - run.setIsOffline(false); - run.setIsBucketRun(false); - run.setId(UUID.randomUUID().toString()); - run.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - return run; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/Evaluation.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/Evaluation.java similarity index 95% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/Evaluation.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/Evaluation.java index c8efdc4..42eea0c 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/Evaluation.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/Evaluation.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.evaluation; +package com.judgmentlabs.judgeval.evaluation; import java.util.Objects; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactory.java similarity index 91% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactory.java index 372929b..1c3c3e8 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactory.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.evaluation; +package com.judgmentlabs.judgeval.evaluation; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java deleted file mode 100644 index 8126891..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.judgmentlabs.judgeval.exceptions; - -public class JudgmentAPIError extends RuntimeException { - private final int statusCode; - private final String detail; - - public JudgmentAPIError(int statusCode, String detail) { - super(String.format("%d: %s", statusCode, detail)); - this.statusCode = statusCode; - this.detail = detail; - } - - public int getStatusCode() { - return statusCode; - } - - public String getDetail() { - return detail; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentRuntimeError.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentRuntimeError.java deleted file mode 100644 index 5f78bd2..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentRuntimeError.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.judgmentlabs.judgeval.exceptions; - -public class JudgmentRuntimeError extends RuntimeException { - public JudgmentRuntimeError(String message) { - super(message); - } - - public JudgmentRuntimeError(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentTestError.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentTestError.java deleted file mode 100644 index d0194a6..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentTestError.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.judgmentlabs.judgeval.exceptions; - -public class JudgmentTestError extends RuntimeException { - public JudgmentTestError(String message) { - super(message); - } - - public JudgmentTestError(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/package-info.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/package-info.java similarity index 100% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/package-info.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/package-info.java diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index 1c6e739..0a73e43 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -7,14 +7,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.judgmentlabs.judgeval.data.APIScorerType; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; +import com.judgmentlabs.judgeval.data.APIScorerType; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.scorers.APIScorer} - * instead. + * Base class for API-based scorers that evaluate using the Judgment backend. */ -@Deprecated public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer implements BaseScorer { private APIScorerType scoreType; @@ -87,10 +85,25 @@ public ScorerConfig getScorerConfig() { return cfg; } + /** + * Creates a new builder for an APIScorer subclass. + * + * @param + * the scorer type + * @param scorerClass + * the scorer class + * @return a new builder instance + */ public static Builder builder(Class scorerClass) { return new Builder<>(scorerClass); } + /** + * Builder for configuring and creating APIScorer instances. + * + * @param + * the scorer type + */ public static final class Builder { private final T scorer; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index 0eb8c46..daf9a45 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -3,23 +3,18 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; /** - * Minimal interface for scorers used by BaseTracer. Only requires the essential - * methods needed for evaluation. - * - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.scorers.BaseScorer} - * instead. + * Base interface for all scorers in the Judgment evaluation system. */ -@Deprecated public interface BaseScorer { /** - * Gets the name of the scorer. + * Returns the name of this scorer. * * @return the scorer name */ String getName(); /** - * Gets the scorer configuration for evaluation runs. + * Returns the configuration for this scorer. * * @return the scorer configuration */ diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/ScorersFactory.java similarity index 81% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/ScorersFactory.java index edc26d5..d687088 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/ScorersFactory.java @@ -1,9 +1,9 @@ -package com.judgmentlabs.judgeval.v1.scorers; +package com.judgmentlabs.judgeval.scorers; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.v1.scorers.built_in.BuiltInScorersFactory; -import com.judgmentlabs.judgeval.v1.scorers.custom_scorer.CustomScorerFactory; -import com.judgmentlabs.judgeval.v1.scorers.prompt_scorer.PromptScorerFactory; +import com.judgmentlabs.judgeval.scorers.built_in.BuiltInScorersFactory; +import com.judgmentlabs.judgeval.scorers.custom_scorer.CustomScorerFactory; +import com.judgmentlabs.judgeval.scorers.prompt_scorer.PromptScorerFactory; /** * Factory for creating scorer builders and accessing scorer types. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java deleted file mode 100644 index 750844a..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers; - -import java.util.Arrays; - -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * AnswerCorrectnessScorer scorer = AnswerCorrectnessScorer.create();
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * AnswerCorrectnessScorer scorer = client.scorers().builtIn().answerCorrectness().build();
- * }
- */ -@Deprecated -public class AnswerCorrectnessScorer extends APIScorer { - public AnswerCorrectnessScorer() { - super(APIScorerType.ANSWER_CORRECTNESS); - setRequiredParams(Arrays.asList("input", "actual_output", "expected_output")); - } - - public static APIScorer.Builder builder() { - return APIScorer.builder(AnswerCorrectnessScorer.class); - } - - public static AnswerCorrectnessScorer create() { - return new AnswerCorrectnessScorer(); - } - - public static AnswerCorrectnessScorer create(double threshold) { - return builder().threshold(threshold) - .build(); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java deleted file mode 100644 index 652dcfe..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers; - -import java.util.Arrays; - -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * AnswerRelevancyScorer scorer = AnswerRelevancyScorer.create();
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * AnswerRelevancyScorer scorer = client.scorers().builtIn().answerRelevancy().build();
- * }
- */ -@Deprecated -public class AnswerRelevancyScorer extends APIScorer { - public AnswerRelevancyScorer() { - super(APIScorerType.ANSWER_RELEVANCY); - setRequiredParams(Arrays.asList("input", "actual_output")); - } - - public static APIScorer.Builder builder() { - return APIScorer.builder(AnswerRelevancyScorer.class); - } - - public static AnswerRelevancyScorer create() { - return new AnswerRelevancyScorer(); - } - - public static AnswerRelevancyScorer create(double threshold) { - return builder().threshold(threshold) - .build(); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java deleted file mode 100644 index ee1a6b8..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers; - -import java.util.Arrays; - -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * FaithfulnessScorer scorer = FaithfulnessScorer.create();
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * FaithfulnessScorer scorer = client.scorers().builtIn().faithfulness().build();
- * }
- */ -@Deprecated -public class FaithfulnessScorer extends APIScorer { - public FaithfulnessScorer() { - super(APIScorerType.FAITHFULNESS); - setRequiredParams(Arrays.asList("input", "actual_output", "retrieval_context")); - } - - public static APIScorer.Builder builder() { - return APIScorer.builder(FaithfulnessScorer.class); - } - - public static FaithfulnessScorer create() { - return new FaithfulnessScorer(); - } - - public static FaithfulnessScorer create(double threshold) { - return builder().threshold(threshold) - .build(); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java deleted file mode 100644 index b60e3cb..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers; - -import java.util.Arrays; - -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * InstructionAdherenceScorer scorer = InstructionAdherenceScorer.create();
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * InstructionAdherenceScorer scorer = client.scorers().builtIn().instructionAdherence().build();
- * }
- */ -@Deprecated -public class InstructionAdherenceScorer extends APIScorer { - public InstructionAdherenceScorer() { - super(APIScorerType.INSTRUCTION_ADHERENCE); - setRequiredParams(Arrays.asList("input", "actual_output")); - setName("Instruction Adherence"); - } - - public static APIScorer.Builder builder() { - return APIScorer.builder(InstructionAdherenceScorer.class); - } - - public static InstructionAdherenceScorer create() { - return new InstructionAdherenceScorer(); - } - - public static InstructionAdherenceScorer create(double threshold) { - return builder().threshold(threshold) - .build(); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/custom_scorer/CustomScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/custom_scorer/CustomScorer.java deleted file mode 100644 index 0ca1f07..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/custom_scorer/CustomScorer.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers.custom_scorer; - -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -/** - * Server-hosted custom scorer representation for enqueue payloads. - * Instances serialize into ExampleEvaluationRun.custom_scorers with score_type - * "Custom", server_hosted=true, and optional class_name for server routing. - * - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * CustomScorer scorer = CustomScorer.get("my-scorer");
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * CustomScorer scorer = client.scorers().customScorer().get("my-scorer");
- * }
- */ -@Deprecated -public class CustomScorer extends APIScorer { - public CustomScorer() { - super(APIScorerType.CUSTOM); - } - - /** - * Creates a server-hosted custom scorer with the given name. - * - * @param name - * scorer identifier and default class_name - * @return configured CustomScorer - */ - public static CustomScorer get(String name) { - CustomScorer s = new CustomScorer(); - s.setName(name); - s.setClassName(name); - s.setServerHosted(true); - return s; - } - - /** - * Creates a server-hosted custom scorer with explicit class_name for routing. - * - * @param name - * scorer identifier - * @param className - * server-side scorer class name - * @return configured CustomScorer - */ - public static CustomScorer get(String name, String className) { - CustomScorer s = get(name); - s.setClassName(className); - return s; - } - - @Override - /** - * Not used for server-hosted custom scorers. - * - * @deprecated never returns; always throws - * @return never returns; always throws - */ - public ScorerConfig getScorerConfig() { - throw new UnsupportedOperationException("CustomScorer does not use ScorerConfig"); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java deleted file mode 100644 index 01f1860..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -public abstract class BasePromptScorer extends APIScorer { - private static final Map cache = new ConcurrentHashMap<>(); - - protected String prompt; - protected Map options; - protected String judgmentApiKey; - protected String organizationId; - - protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, double threshold, - Map options, String judgmentApiKey, String organizationId) { - super(scoreType); - this.prompt = prompt; - this.options = options; - this.judgmentApiKey = judgmentApiKey; - this.organizationId = organizationId; - setName(name); - super.setThreshold(threshold); - } - - public static boolean scorerExists(String name, String judgmentApiKey, String organizationId) { - try { - JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - ScorerExistsRequest request = new ScorerExistsRequest(); - request.setName(name); - ScorerExistsResponse response = client.scorerExists(request); - return Boolean.TRUE.equals(response.getExists()); - } catch (Exception e) { - throw new JudgmentAPIError(500, "Failed to check if scorer exists: " + e.getMessage()); - } - } - - public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer(String name, - String judgmentApiKey, String organizationId) { - CacheKey key = new CacheKey(name, judgmentApiKey, organizationId); - com.judgmentlabs.judgeval.internal.api.models.PromptScorer cached = cache.get(key); - if (cached != null) { - return cached; - } - - try { - JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - FetchPromptScorersRequest request = new FetchPromptScorersRequest(); - request.setNames(java.util.Collections.singletonList(name)); - - FetchPromptScorersResponse response = client.fetchScorers(request); - - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorer = Optional.ofNullable(response) - .map(FetchPromptScorersResponse::getScorers) - .filter(scorers -> scorers != null && !scorers.isEmpty()) - .map(scorers -> scorers.get(0)) - .orElseThrow( - () -> new JudgmentAPIError(404, "Failed to fetch prompt scorer '" + name + "': not found")); - - cache.put(key, scorer); - return scorer; - } catch (JudgmentAPIError e) { - throw e; - } catch (Exception e) { - throw new JudgmentAPIError(500, "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); - } - } - - public Double getThreshold() { - return super.getThreshold(); - } - - public String getPrompt() { - return prompt; - } - - public Map getOptions() { - return Optional.ofNullable(options) - .map(HashMap::new) - .orElse(null); - } - - public String getScorerName() { - return getName(); - } - - public Map getConfig() { - Map config = new HashMap<>(); - config.put("name", getName()); - config.put("prompt", prompt); - config.put("threshold", getThreshold()); - config.put("options", options); - return config; - } - - protected abstract boolean isTrace(); - - private static final class CacheKey { - private final String name; - private final String apiKey; - private final String organizationId; - - CacheKey(String name, String apiKey, String organizationId) { - this.name = name; - this.apiKey = apiKey; - this.organizationId = organizationId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - CacheKey that = (CacheKey) obj; - return Objects.equals(name, that.name) && Objects.equals(apiKey, that.apiKey) - && Objects.equals(organizationId, that.organizationId); - } - - @Override - public int hashCode() { - return Objects.hash(name, apiKey, organizationId); - } - } - - @Override - public String toString() { - return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" + getThreshold() - + ", options=" + options + ")"; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java deleted file mode 100644 index 2d9e28c..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * PromptScorer scorer = PromptScorer.get("my-scorer");
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * PromptScorer scorer = client.scorers().promptScorer().get("my-scorer");
- * }
- */ -@Deprecated -public class PromptScorer extends BasePromptScorer { - - public PromptScorer(String name, String prompt, double threshold, Map options) { - super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, - Env.JUDGMENT_ORG_ID); - } - - public PromptScorer(String name, String prompt, double threshold, Map options, - String judgmentApiKey, String organizationId) { - super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); - } - - public static PromptScorer get(String name) { - return get(name, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static PromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, - judgmentApiKey, organizationId); - - if (Boolean.TRUE.equals(scorerConfig.getIsTrace())) { - throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a PromptScorer"); - } - - Map options = null; - if (scorerConfig.getOptions() != null) { - if (scorerConfig.getOptions() instanceof Map) { - @SuppressWarnings("unchecked") - Map rawOptions = (Map) scorerConfig.getOptions(); - options = new HashMap<>(); - for (Map.Entry entry : rawOptions.entrySet()) { - if (entry.getValue() instanceof Number) { - options.put(entry.getKey(), ((Number) entry.getValue()).doubleValue()); - } - } - } - } - - return new PromptScorer(name, scorerConfig.getPrompt(), Optional.ofNullable(scorerConfig.getThreshold()) - .orElse(0.5), options, judgmentApiKey, organizationId); - } - - @Override - public ScorerConfig getScorerConfig() { - ScorerConfig cfg = new ScorerConfig(); - cfg.setScoreType(getScoreType()); - cfg.setThreshold(getThreshold()); - cfg.setName(getName()); - cfg.setStrictMode(getStrictMode()); - cfg.setRequiredParams(getRequiredParams()); - Map kwargs = new HashMap<>(); - kwargs.put("prompt", prompt); - if (options != null) { - kwargs.put("options", options); - } - if (getAdditionalProperties() != null) { - kwargs.putAll(getAdditionalProperties()); - } - cfg.setKwargs(kwargs); - return cfg; - } - - @Override - protected boolean isTrace() { - return false; - } - -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java deleted file mode 100644 index 6ba7c9f..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -/** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * TracePromptScorer scorer = TracePromptScorer.get("my-scorer");
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * PromptScorer scorer = client.scorers().tracePromptScorer().get("my-scorer");
- * }
- */ -@Deprecated -public class TracePromptScorer extends BasePromptScorer { - - public TracePromptScorer(String name, String prompt, double threshold, Map options) { - super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, - Env.JUDGMENT_ORG_ID); - } - - public TracePromptScorer(String name, String prompt, double threshold, Map options, - String judgmentApiKey, String organizationId) { - super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); - } - - public static TracePromptScorer get(String name) { - return get(name, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static TracePromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, - judgmentApiKey, organizationId); - - if (!Boolean.TRUE.equals(scorerConfig.getIsTrace())) { - throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a TracePromptScorer"); - } - - Map options = null; - if (scorerConfig.getOptions() != null) { - if (scorerConfig.getOptions() instanceof Map) { - @SuppressWarnings("unchecked") - Map rawOptions = (Map) scorerConfig.getOptions(); - options = new HashMap<>(); - for (Map.Entry entry : rawOptions.entrySet()) { - if (entry.getValue() instanceof Number) { - options.put(entry.getKey(), ((Number) entry.getValue()).doubleValue()); - } - } - } - } - - return new TracePromptScorer(name, scorerConfig.getPrompt(), - Optional.ofNullable(scorerConfig.getThreshold()) - .orElse(0.5), - options, judgmentApiKey, organizationId); - } - - @Override - public ScorerConfig getScorerConfig() { - ScorerConfig cfg = new ScorerConfig(); - cfg.setScoreType(getScoreType()); - cfg.setThreshold(getThreshold()); - cfg.setName(getName()); - cfg.setStrictMode(getStrictMode()); - cfg.setRequiredParams(getRequiredParams()); - Map kwargs = new HashMap<>(); - kwargs.put("prompt", prompt); - if (options != null) { - kwargs.put("options", options); - } - if (getAdditionalProperties() != null) { - kwargs.putAll(getAdditionalProperties()); - } - cfg.setKwargs(kwargs); - return cfg; - } - - @Override - protected boolean isTrace() { - return true; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerCorrectnessScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerCorrectnessScorer.java similarity index 91% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerCorrectnessScorer.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerCorrectnessScorer.java index 73f7b76..ba1aa5a 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerCorrectnessScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerCorrectnessScorer.java @@ -1,9 +1,9 @@ -package com.judgmentlabs.judgeval.v1.scorers.built_in; +package com.judgmentlabs.judgeval.scorers.built_in; import java.util.Arrays; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; -import com.judgmentlabs.judgeval.v1.scorers.APIScorer; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.scorers.APIScorer; /** * Scorer that evaluates the correctness of an answer against an expected diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerRelevancyScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerRelevancyScorer.java similarity index 91% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerRelevancyScorer.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerRelevancyScorer.java index 952391b..339b3a5 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/AnswerRelevancyScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/AnswerRelevancyScorer.java @@ -1,9 +1,9 @@ -package com.judgmentlabs.judgeval.v1.scorers.built_in; +package com.judgmentlabs.judgeval.scorers.built_in; import java.util.Arrays; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; -import com.judgmentlabs.judgeval.v1.scorers.APIScorer; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.scorers.APIScorer; /** * Scorer that evaluates the relevancy of an answer to the input question. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactory.java similarity index 93% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactory.java index b3c0a50..0e47522 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactory.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.scorers.built_in; +package com.judgmentlabs.judgeval.scorers.built_in; /** * Factory for creating built-in scorer builders. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/FaithfulnessScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/FaithfulnessScorer.java similarity index 91% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/FaithfulnessScorer.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/FaithfulnessScorer.java index 4e20d36..df36d8f 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/built_in/FaithfulnessScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/built_in/FaithfulnessScorer.java @@ -1,9 +1,9 @@ -package com.judgmentlabs.judgeval.v1.scorers.built_in; +package com.judgmentlabs.judgeval.scorers.built_in; import java.util.Arrays; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; -import com.judgmentlabs.judgeval.v1.scorers.APIScorer; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.scorers.APIScorer; /** * Scorer that evaluates whether an answer is faithful to the provided context. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorer.java similarity index 89% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorer.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorer.java index da8ea85..1d25556 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorer.java @@ -1,8 +1,8 @@ -package com.judgmentlabs.judgeval.v1.scorers.custom_scorer; +package com.judgmentlabs.judgeval.scorers.custom_scorer; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; -import com.judgmentlabs.judgeval.v1.scorers.APIScorer; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.scorers.APIScorer; /** * Scorer that uses custom user-defined evaluation logic. Scorers are hosted on diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorerFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorerFactory.java similarity index 93% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorerFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorerFactory.java index c575b3c..a768fd5 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/custom_scorer/CustomScorerFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/custom_scorer/CustomScorerFactory.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.scorers.custom_scorer; +package com.judgmentlabs.judgeval.scorers.custom_scorer; /** * Factory for creating custom scorer instances. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorer.java similarity index 95% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorer.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorer.java index ea111e0..6048d73 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorer.java @@ -1,12 +1,12 @@ -package com.judgmentlabs.judgeval.v1.scorers.prompt_scorer; +package com.judgmentlabs.judgeval.scorers.prompt_scorer; import java.util.HashMap; import java.util.Map; import java.util.Objects; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; -import com.judgmentlabs.judgeval.v1.scorers.APIScorer; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.scorers.APIScorer; /** * Scorer that evaluates traces using Judgment-hosted prompt scorers. diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorerFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorerFactory.java similarity index 98% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorerFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorerFactory.java index 6f38121..fc99067 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/prompt_scorer/PromptScorerFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/scorers/prompt_scorer/PromptScorerFactory.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.scorers.prompt_scorer; +package com.judgmentlabs.judgeval.scorers.prompt_scorer; import java.util.HashMap; import java.util.Map; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 75fa866..d4f5915 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -1,72 +1,91 @@ package com.judgmentlabs.judgeval.tracer; import java.lang.reflect.Type; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.EvaluationRunBuilder; -import com.judgmentlabs.judgeval.data.Example; -import com.judgmentlabs.judgeval.data.TraceEvaluationRunBuilder; +import com.judgmentlabs.judgeval.JudgmentAttributeKeys; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; +import com.judgmentlabs.judgeval.utils.Logger; +import com.judgmentlabs.judgeval.data.Example; import com.judgmentlabs.judgeval.scorers.BaseScorer; +import com.judgmentlabs.judgeval.scorers.custom_scorer.CustomScorer; import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; -import com.judgmentlabs.judgeval.utils.Logger; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; +/** + * Base tracer providing span manipulation, attribute setting, and evaluation + * capabilities. + */ public abstract class BaseTracer { - public static final String TRACER_NAME = "judgeval"; - - protected final TracerConfiguration configuration; - protected final JudgmentSyncClient apiClient; - protected final ISerializer serializer; - protected final ObjectMapper jacksonMapper; - protected final Optional projectId; - - protected BaseTracer(TracerConfiguration configuration, ISerializer serializer, boolean initialize) { - this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); - this.apiClient = new JudgmentSyncClient(configuration.apiUrl(), configuration.apiKey(), - configuration.organizationId()); - this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); + public static final String TRACER_NAME = "judgeval"; + + protected final String projectName; + protected final boolean enableEvaluation; + protected final JudgmentSyncClient apiClient; + protected final ISerializer serializer; + protected final ObjectMapper jacksonMapper; + protected final Optional projectId; + + protected BaseTracer(String projectName, + boolean enableEvaluation, JudgmentSyncClient apiClient, ISerializer serializer) { + this.projectName = Objects.requireNonNull(projectName, "projectName required"); + this.enableEvaluation = enableEvaluation; + this.apiClient = Objects.requireNonNull(apiClient, "apiClient required"); + this.serializer = Objects.requireNonNull(serializer, "serializer required"); this.jacksonMapper = new ObjectMapper(); - this.projectId = resolveProjectId(configuration.projectName()); + this.projectId = resolveProjectId(projectName); + this.projectId.ifPresentOrElse(id -> { - }, () -> Logger.error("Failed to resolve project " + configuration.projectName() - + ", please create it first at https://app.judgmentlabs.ai/org/" + configuration.organizationId() + }, () -> Logger.error("Failed to resolve project " + projectName + + ", please create it first at https://app.judgmentlabs.ai/org/" + this.apiClient.getOrganizationId() + "/projects. Skipping Judgment export.")); - - if (initialize) { - initialize(); - } } /** - * Initializes the tracer with OpenTelemetry SDK configuration and span - * exporters. Must be implemented by subclasses. + * Initializes the tracer. */ public abstract void initialize(); /** - * Gets the span exporter for sending traces to the Judgment Labs backend. - * Returns a NoOpSpanExporter if the project ID is not resolved. + * Forces pending spans to flush. * - * @return the configured SpanExporter instance + * @param timeoutMillis + * maximum time to wait in milliseconds + * @return true if flush succeeded within timeout + */ + public abstract boolean forceFlush(int timeoutMillis); + + /** + * Shuts down the tracer. + * + * @param timeoutMillis + * maximum time to wait for shutdown in milliseconds + */ + public abstract void shutdown(int timeoutMillis); + + /** + * Returns the span exporter for this tracer. + * + * @return the span exporter */ public SpanExporter getSpanExporter() { return projectId.map(this::createJudgmentSpanExporter) @@ -77,15 +96,15 @@ public SpanExporter getSpanExporter() { } /** - * Sets the kind of the current span (e.g., "llm", "tool", "span"). + * Sets the span kind attribute on the current span. * * @param kind - * the span kind to set, ignored if null + * the span kind */ public void setSpanKind(String kind) { Optional.ofNullable(kind) .ifPresent(k -> withCurrentSpan( - span -> span.setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); + span -> span.setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); } private static void withCurrentSpan(java.util.function.Consumer action) { @@ -98,13 +117,12 @@ private static boolean isValidKey(String key) { } /** - * Sets an attribute on the current span by serializing the object value. - * Empty strings and null are valid attribute values, but not valid keys. + * Sets an attribute on the current span by serializing the value. * * @param key * the attribute key * @param value - * the attribute value (null and empty strings are valid) + * the attribute value */ public void setAttribute(String key, Object value) { if (!isValidKey(key)) { @@ -116,14 +134,13 @@ public void setAttribute(String key, Object value) { } /** - * Sets an attribute on the current span by serializing the object value with - * the specified type. Empty strings and null are valid attribute values, but - * not valid keys. + * Sets an attribute on the current span by serializing the value with the + * specified type. * * @param key * the attribute key * @param value - * the attribute value (null and empty strings are valid) + * the attribute value * @param type * the type to use for serialization */ @@ -137,13 +154,12 @@ public void setAttribute(String key, Object value, Type type) { } /** - * Sets a string attribute on the current span. Empty strings and null are valid - * attribute values, but not valid keys. + * Sets a string attribute on the current span. * * @param key * the attribute key * @param value - * the string value (null and empty strings are valid) + * the attribute value */ public void setAttribute(String key, String value) { if (!isValidKey(key)) { @@ -158,7 +174,7 @@ public void setAttribute(String key, String value) { * @param key * the attribute key * @param value - * the long value + * the attribute value */ public void setAttribute(String key, long value) { if (!isValidKey(key)) { @@ -173,7 +189,7 @@ public void setAttribute(String key, long value) { * @param key * the attribute key * @param value - * the double value + * the attribute value */ public void setAttribute(String key, double value) { if (!isValidKey(key)) { @@ -188,7 +204,7 @@ public void setAttribute(String key, double value) { * @param key * the attribute key * @param value - * the boolean value + * the attribute value */ public void setAttribute(String key, boolean value) { if (!isValidKey(key)) { @@ -211,11 +227,11 @@ private Optional getSampledSpan() { } private boolean isEvaluationEnabled() { - return configuration.enableEvaluation(); + return enableEvaluation; } private void logEvaluationInfo(String method, String traceId, String spanId, String scorerName) { - Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId + ", spanId=" + Logger.info(method + ": project=" + projectName + ", traceId=" + traceId + ", spanId=" + spanId + ", scorer=" + scorerName); } @@ -228,17 +244,17 @@ private void safeExecute(String operation, Runnable action) { } /** - * Asynchronously evaluates a scorer against an example, associating it with the - * current trace. + * Asynchronously evaluates the current span using the specified scorer and + * example. + * The evaluation is queued and processed asynchronously by the Judgment + * backend. * * @param scorer - * the scorer to evaluate + * the scorer to use for evaluation * @param example - * the example to evaluate against - * @param model - * the model name, or null to use the default + * the example data to evaluate against */ - public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + public void asyncEvaluate(BaseScorer scorer, Example example) { safeExecute("evaluate scorer", () -> { if (!isEvaluationEnabled()) { return; @@ -250,34 +266,21 @@ public void asyncEvaluate(BaseScorer scorer, Example example, String model) { logEvaluationInfo("asyncEvaluate", traceId, spanId, scorer.getName()); - ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, model, traceId, spanId); + ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, traceId, spanId); enqueueEvaluation(evaluationRun); }); }); } /** - * Asynchronously evaluates a scorer against an example using the default model. + * Asynchronously evaluates the current trace using the specified scorer. + * Attaches evaluation metadata to the current span for processing after trace + * completion. * * @param scorer - * the scorer to evaluate - * @param example - * the example to evaluate against + * the scorer to use for trace evaluation */ - public void asyncEvaluate(BaseScorer scorer, Example example) { - asyncEvaluate(scorer, example, null); - } - - /** - * Asynchronously evaluates a scorer for the current trace, attaching the - * evaluation as a span attribute. - * - * @param scorer - * the scorer to evaluate - * @param model - * the model name, or null to use the default - */ - public void asyncTraceEvaluate(BaseScorer scorer, String model) { + public void asyncTraceEvaluate(BaseScorer scorer) { safeExecute("evaluate trace scorer", () -> { if (!isEvaluationEnabled()) { return; @@ -290,10 +293,11 @@ public void asyncTraceEvaluate(BaseScorer scorer, String model) { logEvaluationInfo("asyncTraceEvaluate", traceId, spanId, scorer.getName()); - TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, model, traceId, spanId); + TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, traceId, spanId); try { String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute(JudgevalTraceKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); + currentSpan.setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_PENDING_TRACE_EVAL, + traceEvalJson); } catch (Exception e) { Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); } @@ -302,21 +306,10 @@ public void asyncTraceEvaluate(BaseScorer scorer, String model) { } /** - * Asynchronously evaluates a scorer for the current trace using the default - * model. - * - * @param scorer - * the scorer to evaluate - */ - public void asyncTraceEvaluate(BaseScorer scorer) { - asyncTraceEvaluate(scorer, null); - } - - /** - * Sets multiple attributes on the current span from a map. + * Sets multiple attributes on the current span. * * @param attributes - * the map of attribute key-value pairs, ignored if null + * map of attribute keys to values */ public void setAttributes(Map attributes) { Optional.ofNullable(attributes) @@ -324,80 +317,78 @@ public void setAttributes(Map attributes) { } /** - * Sets the current span kind to "llm". + * Marks the current span as an LLM span. */ public void setLLMSpan() { setSpanKind("llm"); } /** - * Sets the current span kind to "tool". + * Marks the current span as a tool span. */ public void setToolSpan() { setSpanKind("tool"); } /** - * Sets the current span kind to "span". + * Marks the current span as a general span. */ public void setGeneralSpan() { setSpanKind("span"); } /** - * Sets the input attribute on the current span by serializing the object. + * Sets the input attribute on the current span. * * @param input - * the input object to set, ignored if null + * the input value */ public void setInput(Object input) { - setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input); + setInput(input, input.getClass()); } /** - * Sets the output attribute on the current span by serializing the object. + * Sets the output attribute on the current span. * * @param output - * the output object to set, ignored if null + * the output value */ public void setOutput(Object output) { - setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output); + setOutput(output, output.getClass()); } /** - * Sets the input attribute on the current span with a specific type for - * serialization. + * Sets the input attribute on the current span using the specified type. * * @param input - * the input object to set, ignored if null + * the input value * @param type * the type to use for serialization */ public void setInput(Object input, Type type) { - setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input, type); + setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_INPUT, input, type); } /** - * Sets the output attribute on the current span with a specific type for - * serialization. + * Sets the output attribute on the current span using the specified type. * * @param output - * the output object to set, ignored if null + * the output value * @param type * the type to use for serialization */ public void setOutput(Object output, Type type) { - setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); + setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } /** - * Creates a new span, executes the runnable within its context, and ends the - * span. + * Executes a runnable within a new span, automatically handling span lifecycle + * and errors. * * @param spanName * the name of the span * @param runnable - * the code to execute within the span context + * the code to execute within the span */ public void span(String spanName, Runnable runnable) { Span span = getTracer().spanBuilder(spanName) @@ -413,15 +404,15 @@ public void span(String spanName, Runnable runnable) { } /** - * Creates a new span, executes the callable within its context, and ends the - * span. + * Executes a callable within a new span, automatically handling span lifecycle + * and errors. * * @param - * the return type of the callable + * the return type * @param spanName * the name of the span * @param callable - * the code to execute within the span context + * the code to execute within the span * @return the result of the callable * @throws Exception * if the callable throws an exception @@ -440,41 +431,48 @@ public T span(String spanName, java.util.concurrent.Callable callable) th } /** - * Gets the OpenTelemetry tracer instance. + * Returns the OpenTelemetry tracer instance. * - * @return the configured Tracer instance + * @return the OpenTelemetry tracer */ - public Tracer getTracer() { + public io.opentelemetry.api.trace.Tracer getTracer() { return GlobalOpenTelemetry.get() .getTracer(TRACER_NAME); } /** - * Gets the tracer configuration. + * Returns the project name. + * + * @return the project name + */ + public String getProjectName() { + return projectName; + } + + /** + * Returns whether evaluation is enabled. * - * @return the TracerConfiguration instance + * @return true if evaluation is enabled */ - public TracerConfiguration getConfiguration() { - return configuration; + public boolean isEnableEvaluation() { + return enableEvaluation; } /** - * Gets the resolved project ID, if available. + * Returns the resolved project ID if available. * - * @return an Optional containing the project ID, or empty if not resolved + * @return the project ID, or empty if not resolved */ public Optional getProjectId() { return projectId; } /** - * Creates and returns a new span with the given name. The span must be entered - * with a try-with-resources block or manually ended by calling - * {@link Span#end()}. + * Creates and returns a new span with the specified name. * * @param spanName * the name of the span - * @return the newly created span + * @return the created span */ public static Span span(String spanName) { return GlobalOpenTelemetry.get() @@ -483,7 +481,7 @@ public static Span span(String spanName) { .startSpan(); } - protected Optional resolveProjectId(String name) { + private Optional resolveProjectId(String name) { try { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); @@ -501,9 +499,9 @@ private String buildEndpoint(String baseUrl) { private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { return JudgmentSpanExporter.builder() - .endpoint(buildEndpoint(configuration.apiUrl())) - .apiKey(configuration.apiKey()) - .organizationId(configuration.organizationId()) + .endpoint(buildEndpoint(apiClient.getApiUrl())) + .apiKey(apiClient.getApiKey()) + .organizationId(apiClient.getOrganizationId()) .projectId(projectId) .build(); } @@ -513,46 +511,46 @@ private String generateRunId(String prefix, String spanId) { .orElseGet(() -> String.valueOf(System.currentTimeMillis())); } - private String getModelName(String model) { - return Optional.ofNullable(model) - .orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); - } - - private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example example, String model, String traceId, + private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example example, String traceId, String spanId) { String runId = generateRunId("async_evaluate_", spanId); - return new EvaluationRunBuilder() - .projectName(configuration.projectName()) - .evalName(runId) - .model(model) - .example(example) - .trace(traceId, spanId) - .addScorer(scorer) - .build(); + + ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(); + evaluationRun.setId(UUID.randomUUID().toString()); + evaluationRun.setProjectName(projectName); + evaluationRun.setEvalName(runId); + evaluationRun.setTraceId(traceId); + evaluationRun.setTraceSpanId(spanId); + evaluationRun.setExamples(List.of(example)); + + if (scorer instanceof CustomScorer) { + evaluationRun.setJudgmentScorers(List.of()); + evaluationRun.setCustomScorers(List.of((com.judgmentlabs.judgeval.internal.api.models.BaseScorer) scorer)); + } else { + evaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + evaluationRun.setCustomScorers(List.of()); + } + evaluationRun.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + + return evaluationRun; } - private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String model, String traceId, + private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String traceId, String spanId) { String evalName = generateRunId("async_trace_evaluate_", spanId); - return new TraceEvaluationRunBuilder() - .projectName(configuration.projectName()) - .evalName(evalName) - .model(model) - .trace(traceId, spanId) - .addScorer(scorer) - .build(); - } - private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { - if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) - throw new IllegalArgumentException("Trace and span IDs are required for trace evaluations."); - List> converted = new java.util.ArrayList<>(); - for (List pair : traceAndSpanIds) { - if (pair == null || pair.size() != 2) - throw new IllegalArgumentException("Each trace and span ID pair must contain exactly 2 elements."); - converted.add(List.of(pair.get(0), pair.get(1))); - } - return converted; + TraceEvaluationRun evaluationRun = new TraceEvaluationRun(); + evaluationRun.setId(UUID.randomUUID().toString()); + evaluationRun.setProjectName(projectName); + evaluationRun.setEvalName(evalName); + evaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); + evaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + evaluationRun.setCustomScorers(List.of()); + evaluationRun.setIsOffline(false); + evaluationRun.setIsBucketRun(false); + evaluationRun.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + + return evaluationRun; } private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { @@ -562,27 +560,4 @@ private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { Logger.error("Failed to enqueue evaluation run: " + e.getMessage()); } } - - protected static class GsonSerializer implements ISerializer { - private final Gson gson = new Gson(); - - @Override - public String serialize(Object obj) { - return Optional.ofNullable(obj) - .map(o -> serialize(o, o.getClass())) - .orElse(null); - } - - @Override - public String serialize(Object obj, Type type) { - try { - return gson.toJson(obj, type); - } catch (Exception e) { - Logger.error("Failed to serialize object: " + e.getMessage()); - return Optional.ofNullable(obj) - .map(Object::toString) - .orElse(null); - } - } - } } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/ISerializer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/ISerializer.java index 737d80e..a5fe4d7 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/ISerializer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/ISerializer.java @@ -3,13 +3,27 @@ import java.lang.reflect.Type; /** - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.tracer.ISerializer} - * instead. + * Serializes objects to string representations. */ -@Deprecated public interface ISerializer { + /** + * Serializes an object to a string. + * + * @param obj + * the object to serialize + * @return the serialized string + */ String serialize(Object obj); + /** + * Serializes an object to a string using the specified type. + * + * @param obj + * the object to serialize + * @param type + * the type to use for serialization + * @return the serialized string + */ default String serialize(Object obj, Type type) { return serialize(obj); } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java deleted file mode 100644 index 65e2fd4..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.judgmentlabs.judgeval.tracer; - -public final class JudgevalTraceKeys { - public static final class AttributeKeys { - public static final String JUDGMENT_SPAN_KIND = "judgment.span_kind"; - public static final String JUDGMENT_INPUT = "judgment.input"; - public static final String JUDGMENT_OUTPUT = "judgment.output"; - public static final String JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"; - public static final String JUDGMENT_UPDATE_ID = "judgment.update_id"; - public static final String JUDGMENT_CUSTOMER_ID = "judgment.customer_id"; - public static final String JUDGMENT_AGENT_ID = "judgment.agent_id"; - public static final String JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"; - public static final String JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"; - public static final String JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"; - public static final String JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"; - public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; - public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; - public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; - public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; - - public static final String GEN_AI_PROMPT = "gen_ai.prompt"; - public static final String GEN_AI_COMPLETION = "gen_ai.completion"; - public static final String GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; - public static final String GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; - public static final String GEN_AI_SYSTEM = "gen_ai.system"; - public static final String GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; - public static final String GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; - public static final String GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = "gen_ai.usage.cache_creation_input_tokens"; - public static final String GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"; - public static final String GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; - public static final String GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; - public static final String GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"; - - private AttributeKeys() { - } - } - - public static final class ResourceKeys { - public static final String SERVICE_NAME = "service.name"; - public static final String TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"; - public static final String TELEMETRY_SDK_NAME = "telemetry.sdk.name"; - public static final String TELEMETRY_SDK_VERSION = "telemetry.sdk.version"; - public static final String JUDGMENT_PROJECT_ID = "judgment.project_id"; - - private ResourceKeys() { - } - } - - private JudgevalTraceKeys() { - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index de82d2d..3f21853 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -1,8 +1,13 @@ package com.judgmentlabs.judgeval.tracer; +import java.lang.reflect.Type; +import java.util.Objects; import java.util.Optional; +import com.google.gson.Gson; import com.judgmentlabs.judgeval.Version; +import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; +import com.judgmentlabs.judgeval.utils.Logger; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -13,75 +18,26 @@ import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.sdk.trace.export.SpanExporter; -/** - * Main tracer for Judgment Labs distributed tracing and evaluation. - * - * @deprecated Use {@link com.judgmentlabs.judgeval.v1.Judgeval} instead. - * - *

- * Migration example: - * - * 

{@code
- * // Old way:
- * Tracer tracer = Tracer.createDefault("my-project");
- * tracer.initialize();
- * 
- * // New way:
- * Judgeval client = Judgeval.builder().build();
- * Tracer tracer = client.tracer().create()
- *     .projectName("my-project")
- *     .build();
- * tracer.initialize();
- * }
- * - * @see TracerConfiguration - * @see SpanExporter - * @see com.judgmentlabs.judgeval.scorers.BaseScorer - * @see com.judgmentlabs.judgeval.data.Example - */ -@Deprecated public final class Tracer extends BaseTracer { + private SdkTracerProvider tracerProvider; - private Tracer(TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { - super(configuration, serializer, shouldInitialize); - } - - /** - * Creates a new TracerBuilder for constructing a Tracer instance. - * - * @return a new TracerBuilder - */ - public static TracerBuilder builder() { - return new TracerBuilder(); - } - - /** - * Creates a Tracer with default configuration for the given project name. - * - * @param projectName - * the name of the project - * @return a new Tracer instance with default configuration - */ - public static Tracer createDefault(String projectName) { - return builder().configuration(TracerConfiguration.createDefault(projectName)) - .build(); - } + private Tracer(Builder builder) { + super( + Objects.requireNonNull(builder.projectName, "projectName required"), + builder.enableEvaluation, + Objects.requireNonNull(builder.client, "client required"), + builder.serializer != null ? builder.serializer : new GsonSerializer()); - /** - * Creates a Tracer with the provided configuration. - * - * @param configuration - * the tracer configuration - * @return a new Tracer instance with the given configuration - */ - public static Tracer createWithConfiguration(TracerConfiguration configuration) { - return builder().configuration(configuration) - .build(); + if (builder.initialize) { + initialize(); + } } /** - * Initializes the OpenTelemetry SDK with batch span processor and registers it - * globally. + * Initializes the tracer by setting up the OpenTelemetry SDK with a span + * exporter, + * configuring the tracer provider with batch span processing, and registering + * it globally. */ @Override public void initialize() { @@ -89,76 +45,167 @@ public void initialize() { var resource = Resource.getDefault() .merge(Resource.create(Attributes.builder() - .put("service.name", configuration.projectName()) + .put("service.name", projectName) .put("telemetry.sdk.name", TRACER_NAME) .put("telemetry.sdk.version", Version.getVersion()) .build())); - SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + this.tracerProvider = SdkTracerProvider.builder() .setResource(resource) .addSpanProcessor(BatchSpanProcessor.builder(spanExporter) .build()) .build(); OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(tracerProvider) + .setTracerProvider(this.tracerProvider) .build(); GlobalOpenTelemetry.set(openTelemetry); } - public static final class TracerBuilder { - private TracerConfiguration configuration; - private ISerializer serializer = new GsonSerializer(); - private boolean initialize = false; + /** + * Forces the tracer to flush any pending spans within the specified timeout. + * + * @param timeoutMillis + * the maximum time to wait in milliseconds + * @return true if the flush completed successfully within the timeout + */ + @Override + public boolean forceFlush(int timeoutMillis) { + if (tracerProvider == null) { + Logger.error("Cannot forceFlush: tracer not initialized"); + return false; + } + return tracerProvider.forceFlush() + .join(timeoutMillis, java.util.concurrent.TimeUnit.MILLISECONDS) + .isSuccess(); + } + + /** + * Shuts down the tracer, flushing any remaining spans and releasing resources. + * + * @param timeoutMillis + * the maximum time to wait for shutdown in milliseconds + */ + @Override + public void shutdown(int timeoutMillis) { + if (tracerProvider == null) { + Logger.error("Cannot shutdown: tracer not initialized"); + return; + } + tracerProvider.shutdown() + .join(timeoutMillis, java.util.concurrent.TimeUnit.MILLISECONDS); + } + + /** + * Creates a new builder for configuring a Tracer. + * + * @return a new builder instance + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for configuring and creating Tracer instances. + */ + public static final class Builder { + private JudgmentSyncClient client; + private String projectName; + private boolean enableEvaluation = true; + private ISerializer serializer; + private boolean initialize = true; + + /** + * Sets the Judgment API client. + * + * @param client + * the API client + * @return this builder + */ + public Builder client(JudgmentSyncClient client) { + this.client = client; + return this; + } + + /** + * Sets the project name for this tracer. + * + * @param projectName + * the project name + * @return this builder + */ + public Builder projectName(String projectName) { + this.projectName = projectName; + return this; + } /** - * Sets the tracer configuration. + * Sets whether evaluation is enabled. * - * @param configuration - * the configuration to use - * @return this builder for method chaining + * @param enableEvaluation + * true to enable evaluation + * @return this builder */ - public TracerBuilder configuration(TracerConfiguration configuration) { - this.configuration = configuration; + public Builder enableEvaluation(boolean enableEvaluation) { + this.enableEvaluation = enableEvaluation; return this; } /** - * Sets the serializer to use for attribute serialization. + * Sets the custom serializer for span attributes. * * @param serializer - * the serializer to use - * @return this builder for method chaining + * the serializer + * @return this builder */ - public TracerBuilder serializer(ISerializer serializer) { + public Builder serializer(ISerializer serializer) { this.serializer = serializer; return this; } /** - * Sets whether to automatically initialize the tracer after construction. + * Sets whether to automatically initialize the tracer on build. * * @param initialize - * true to initialize automatically, false otherwise - * @return this builder for method chaining + * true to initialize on build + * @return this builder */ - public TracerBuilder initialize(boolean initialize) { + public Builder initialize(boolean initialize) { this.initialize = initialize; return this; } /** - * Builds a new Tracer instance with the configured settings. + * Builds and returns a new Tracer instance. * - * @return a new Tracer instance - * @throws IllegalArgumentException - * if configuration is not set + * @return the configured Tracer */ public Tracer build() { - return Optional.ofNullable(configuration) - .map(config -> new Tracer(config, serializer, initialize)) - .orElseThrow(() -> new IllegalArgumentException("Configuration is required")); + return new Tracer(this); + } + } + + private static class GsonSerializer implements ISerializer { + private final Gson gson = new Gson(); + + @Override + public String serialize(Object obj) { + return Optional.ofNullable(obj) + .map(o -> serialize(o, o.getClass())) + .orElse(null); + } + + @Override + public String serialize(Object obj, Type type) { + try { + return gson.toJson(obj, type); + } catch (Exception e) { + Logger.error("Failed to serialize object: " + e.getMessage()); + return Optional.ofNullable(obj) + .map(Object::toString) + .orElse(null); + } } } } diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java deleted file mode 100644 index 93270a7..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java +++ /dev/null @@ -1,148 +0,0 @@ -package com.judgmentlabs.judgeval.tracer; - -import java.util.Optional; - -/** - * Configuration for the Judgment Tracer that controls how tracing and - * evaluation behave. - *

- * This class encapsulates all configuration parameters needed to initialize a - * {@link Tracer}. - *

- * Example usage: - * - * 

{@code
- * TracerConfiguration config = TracerConfiguration.builder()
- *         .projectName("my-project")
- *         .apiKey("your-api-key")
- *         .organizationId("your-org-id")
- *         .enableEvaluation(true)
- *         .build();
- *
- * Tracer tracer = Tracer.createWithConfiguration(config);
- * }
- * - * @see Tracer - * @deprecated Replaced by - * com.judgmentlabs.judgeval.v1.tracer.TracerConfiguration - */ -@Deprecated -public final class TracerConfiguration { - private final String projectName; - private final String apiKey; - private final String organizationId; - private final String apiUrl; - private final boolean enableEvaluation; - - private TracerConfiguration(Builder builder) { - this.projectName = Optional.ofNullable(builder.projectName) - .map(String::trim) - .filter(name -> !name.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("Project name cannot be null or empty")); - this.apiKey = Optional.ofNullable(builder.apiKey) - .orElseThrow(() -> new IllegalArgumentException("API key cannot be null")); - this.organizationId = Optional.ofNullable(builder.organizationId) - .orElseThrow(() -> new IllegalArgumentException("Organization ID cannot be null")); - this.apiUrl = Optional.ofNullable(builder.apiUrl) - .orElseThrow(() -> new IllegalArgumentException("API URL cannot be null")); - this.enableEvaluation = builder.enableEvaluation; - } - - public String projectName() { - return projectName; - } - - public String apiKey() { - return apiKey; - } - - public String organizationId() { - return organizationId; - } - - public String apiUrl() { - return apiUrl; - } - - public boolean enableEvaluation() { - return enableEvaluation; - } - - public static Builder builder() { - return new Builder(); - } - - /** - * Creates a default configuration with the given project name. - *

- * This method uses default values from environment variables: - *

    - *
  • API Key: {@code Env.JUDGMENT_API_KEY} - *
  • Organization ID: {@code Env.JUDGMENT_ORG_ID} - *
  • API URL: {@code Env.JUDGMENT_API_URL} - *
  • Evaluation: enabled - *
- * - * @param projectName - * the name of the project - * @return a new TracerConfiguration with default values - * @throws IllegalArgumentException - * if project name is null or empty - */ - public static TracerConfiguration createDefault(String projectName) { - return builder().projectName(projectName) - .build(); - } - - /** - * Builder for creating TracerConfiguration instances. - *

- * Example usage: - * - * 

{@code
-     * TracerConfiguration config = TracerConfiguration.builder()
-     *         .projectName("my-project")
-     *         .apiKey("custom-api-key")
-     *         .organizationId("custom-org-id")
-     *         .apiUrl("https://custom-api.judgmentlabs.ai")
-     *         .enableEvaluation(false)
-     *         .build();
-     * }
- */ - public static final class Builder { - private String projectName; - private String apiKey = com.judgmentlabs.judgeval.Env.JUDGMENT_API_KEY; - private String organizationId = com.judgmentlabs.judgeval.Env.JUDGMENT_ORG_ID; - private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL; - private boolean enableEvaluation = true; - - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - public Builder apiKey(String apiKey) { - this.apiKey = apiKey; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public Builder apiUrl(String apiUrl) { - this.apiUrl = apiUrl; - return this; - } - - public Builder enableEvaluation(boolean enableEvaluation) { - this.enableEvaluation = enableEvaluation; - return this; - } - - public TracerConfiguration build() { - return new TracerConfiguration(this); - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactory.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerFactory.java similarity index 92% rename from judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactory.java rename to judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerFactory.java index cffa765..4684103 100644 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactory.java +++ b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/tracer/TracerFactory.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.tracer; +package com.judgmentlabs.judgeval.tracer; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/Judgeval.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/Judgeval.java deleted file mode 100644 index d2cd402..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/Judgeval.java +++ /dev/null @@ -1,117 +0,0 @@ -package com.judgmentlabs.judgeval.v1; - -import java.util.Objects; - -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.v1.evaluation.EvaluationFactory; -import com.judgmentlabs.judgeval.v1.scorers.ScorersFactory; -import com.judgmentlabs.judgeval.v1.tracer.TracerFactory; - -/** - * Main entry point for the Judgment SDK. Provides access to tracer, scorer, and - * evaluation factories. - */ -public class Judgeval { - private final String apiKey; - private final String organizationId; - private final String apiUrl; - private final JudgmentSyncClient internalClient; - - protected Judgeval(Builder builder) { - this.apiKey = Objects.requireNonNull(builder.apiKey, "apiKey required"); - this.organizationId = Objects.requireNonNull(builder.organizationId, "organizationId required"); - this.apiUrl = builder.apiUrl != null ? builder.apiUrl : Env.JUDGMENT_API_URL; - this.internalClient = new JudgmentSyncClient(apiUrl, apiKey, organizationId); - } - - /** - * Returns a factory for creating tracers. - * - * @return the tracer factory - */ - public TracerFactory tracer() { - return new TracerFactory(internalClient); - } - - /** - * Returns a factory for creating scorers. - * - * @return the scorer factory - */ - public ScorersFactory scorers() { - return new ScorersFactory(internalClient); - } - - /** - * Returns a factory for creating evaluations. - * - * @return the evaluation factory - */ - public EvaluationFactory evaluation() { - return new EvaluationFactory(internalClient); - } - - /** - * Creates a new builder for configuring a Judgeval. - * - * @return a new builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder for configuring and creating Judgeval instances. - */ - public static class Builder { - private String apiKey = Env.JUDGMENT_API_KEY; - private String organizationId = Env.JUDGMENT_ORG_ID; - private String apiUrl = Env.JUDGMENT_API_URL; - - /** - * Sets the API key for authentication. - * - * @param apiKey - * the API key - * @return this builder - */ - public Builder apiKey(String apiKey) { - this.apiKey = apiKey; - return this; - } - - /** - * Sets the organization ID. - * - * @param organizationId - * the organization ID - * @return this builder - */ - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - /** - * Sets the API URL. - * - * @param apiUrl - * the API URL - * @return this builder - */ - public Builder apiUrl(String apiUrl) { - this.apiUrl = apiUrl; - return this; - } - - /** - * Builds and returns a new Judgeval instance. - * - * @return the configured Judgeval - */ - public Judgeval build() { - return new Judgeval(this); - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/APIScorerType.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/APIScorerType.java deleted file mode 100644 index 5af16fa..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/APIScorerType.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.judgmentlabs.judgeval.v1.data; - -/** - * Available types of API-based scorers. - */ -public enum APIScorerType { - PROMPT_SCORER("Prompt Scorer"), - TRACE_PROMPT_SCORER("Trace Prompt Scorer"), - FAITHFULNESS("Faithfulness"), - ANSWER_RELEVANCY("Answer Relevancy"), - ANSWER_CORRECTNESS("Answer Correctness"), - CUSTOM("Custom"); - - private final String value; - - APIScorerType(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return value; - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/Example.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/Example.java deleted file mode 100644 index 6a5e66a..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/Example.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.judgmentlabs.judgeval.v1.data; - -import java.time.Instant; -import java.util.UUID; - -/** - * Represents an evaluation example with arbitrary properties. - */ -public class Example extends com.judgmentlabs.judgeval.internal.api.models.Example { - - public Example() { - super(); - setExampleId(UUID.randomUUID() - .toString()); - setCreatedAt(Instant.now() - .toString()); - setName(null); - } - - /** - * Creates a new builder for configuring an Example. - * - * @return a new builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder for configuring and creating Example instances. - */ - public static final class Builder { - private final Example example; - - private Builder() { - this.example = new Example(); - } - - /** - * Sets a custom property on the example. - * - * @param key - * the property key - * @param value - * the property value - * @return this builder - */ - public Builder property(String key, Object value) { - example.setAdditionalProperty(key, value); - return this; - } - - /** - * Sets the name of the example. - * - * @param name - * the example name - * @return this builder - */ - public Builder name(String name) { - example.setName(name); - return this; - } - - /** - * Builds and returns the configured Example. - * - * @return the configured Example - */ - public Example build() { - return example; - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScorerData.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScorerData.java deleted file mode 100644 index 3347aad..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScorerData.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.judgmentlabs.judgeval.v1.data; - -import java.util.Map; - -/** - * Represents the result of a single scorer evaluation. - */ -public class ScorerData extends com.judgmentlabs.judgeval.internal.api.models.ScorerData { - - /** - * Creates a new builder for configuring ScorerData. - * - * @return a new builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder for configuring and creating ScorerData instances. - */ - public static final class Builder { - private final ScorerData scorerData; - - private Builder() { - this.scorerData = new ScorerData(); - } - - /** - * Sets the scorer name. - * - * @param name - * the scorer name - * @return this builder - */ - public Builder name(String name) { - scorerData.setName(name); - return this; - } - - /** - * Sets the evaluation score. - * - * @param score - * the score value - * @return this builder - */ - public Builder score(Double score) { - scorerData.setScore(score); - return this; - } - - /** - * Sets whether the evaluation succeeded. - * - * @param success - * true if evaluation succeeded - * @return this builder - */ - public Builder success(Boolean success) { - scorerData.setSuccess(success); - return this; - } - - /** - * Sets the reason for the evaluation result. - * - * @param reason - * the evaluation reason - * @return this builder - */ - public Builder reason(String reason) { - scorerData.setReason(reason); - return this; - } - - /** - * Sets the evaluation threshold. - * - * @param threshold - * the threshold value - * @return this builder - */ - public Builder threshold(Double threshold) { - scorerData.setThreshold(threshold); - return this; - } - - /** - * Sets strict mode for evaluation. - * - * @param strictMode - * true for strict mode - * @return this builder - */ - public Builder strictMode(Boolean strictMode) { - scorerData.setStrictMode(strictMode); - return this; - } - - /** - * Sets the model used for evaluation. - * - * @param evaluationModel - * the model name - * @return this builder - */ - public Builder evaluationModel(String evaluationModel) { - scorerData.setEvaluationModel(evaluationModel); - return this; - } - - /** - * Sets an error message if evaluation failed. - * - * @param error - * the error message - * @return this builder - */ - public Builder error(String error) { - scorerData.setError(error); - return this; - } - - /** - * Sets additional metadata for the evaluation. - * - * @param additionalMetadata - * the metadata map - * @return this builder - */ - public Builder additionalMetadata(Map additionalMetadata) { - scorerData.setAdditionalMetadata(additionalMetadata); - return this; - } - - /** - * Adds a single metadata entry. - * - * @param key - * the metadata key - * @param value - * the metadata value - * @return this builder - */ - public Builder metadata(String key, Object value) { - if (scorerData.getAdditionalMetadata() == null) { - scorerData.setAdditionalMetadata(new java.util.HashMap<>()); - } - @SuppressWarnings("unchecked") - Map metadata = (Map) scorerData.getAdditionalMetadata(); - metadata.put(key, value); - return this; - } - - /** - * Builds and returns the configured ScorerData. - * - * @return the configured ScorerData - */ - public ScorerData build() { - return scorerData; - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScoringResult.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScoringResult.java deleted file mode 100644 index 0811f62..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/data/ScoringResult.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.judgmentlabs.judgeval.v1.data; - -import java.util.List; - -/** - * Represents a collection of scorer evaluation results. - */ -public class ScoringResult extends com.judgmentlabs.judgeval.internal.api.models.ScoringResult { - - /** - * Creates a new builder for configuring a ScoringResult. - * - * @return a new builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder for configuring and creating ScoringResult instances. - */ - public static final class Builder { - private final ScoringResult result; - - private Builder() { - this.result = new ScoringResult(); - } - - /** - * Sets whether the overall evaluation succeeded. - * - * @param success - * true if evaluation succeeded - * @return this builder - */ - public Builder success(Boolean success) { - result.setSuccess(success); - return this; - } - - /** - * Sets the list of scorer results. - * - * @param scorersData - * the list of scorer data - * @return this builder - */ - public Builder scorersData(List scorersData) { - @SuppressWarnings("unchecked") - List internalList = (List) (List) scorersData; - result.setScorersData(internalList); - return this; - } - - /** - * Adds a single scorer result. - * - * @param scorerData - * the scorer data to add - * @return this builder - */ - public Builder scorerData(ScorerData scorerData) { - if (result.getScorersData() == null) { - result.setScorersData(new java.util.ArrayList<>()); - } - result.getScorersData() - .add(scorerData); - return this; - } - - /** - * Sets the data object for the evaluation. - * - * @param dataObject - * the example data - * @return this builder - */ - public Builder dataObject(Example dataObject) { - if (dataObject != null) { - result.setAdditionalProperty("example", dataObject); - } - return this; - } - - /** - * Builds and returns the configured ScoringResult. - * - * @return the configured ScoringResult - */ - public ScoringResult build() { - return result; - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/APIScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/APIScorer.java deleted file mode 100644 index e3c0d96..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/APIScorer.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.judgmentlabs.judgeval.v1.scorers; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.v1.data.APIScorerType; - -/** - * Base class for API-based scorers that evaluate using the Judgment backend. - */ -public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer implements BaseScorer { - private APIScorerType scoreType; - - @JsonIgnore - private List requiredParams; - - public APIScorer(APIScorerType scoreType) { - super(); - this.scoreType = scoreType; - setName(scoreType.toString()); - setScoreType(scoreType.toString()); - this.requiredParams = new java.util.ArrayList<>(); - if (Boolean.TRUE.equals(getStrictMode())) { - setThreshold(1.0); - } - } - - public void setThreshold(double threshold) { - if (threshold < 0 || threshold > 1) { - throw new IllegalArgumentException("Threshold must be between 0 and 1, got: " + threshold); - } - super.setThreshold(threshold); - } - - @JsonProperty("score_type") - public String getScoreType() { - return scoreType.toString(); - } - - public List getRequiredParams() { - return requiredParams; - } - - public void setRequiredParams(List requiredParams) { - this.requiredParams = requiredParams; - } - - @Override - public Double getThreshold() { - return Optional.ofNullable(super.getThreshold()) - .orElse(0.5); - } - - @Override - public String getName() { - return Optional.ofNullable(super.getName()) - .map(Object::toString) - .orElse(null); - } - - @Override - public Boolean getStrictMode() { - return Optional.ofNullable(super.getStrictMode()) - .orElse(false); - } - - @Override - @JsonIgnore - public ScorerConfig getScorerConfig() { - ScorerConfig cfg = new ScorerConfig(); - cfg.setScoreType(getScoreType()); - cfg.setThreshold(getThreshold()); - cfg.setName(getName()); - cfg.setStrictMode(getStrictMode()); - cfg.setRequiredParams(getRequiredParams()); - Map kwargs = new HashMap<>(); - if (getAdditionalProperties() != null) - kwargs.putAll(getAdditionalProperties()); - cfg.setKwargs(kwargs); - return cfg; - } - - /** - * Creates a new builder for an APIScorer subclass. - * - * @param - * the scorer type - * @param scorerClass - * the scorer class - * @return a new builder instance - */ - public static Builder builder(Class scorerClass) { - return new Builder<>(scorerClass); - } - - /** - * Builder for configuring and creating APIScorer instances. - * - * @param - * the scorer type - */ - public static final class Builder { - private final T scorer; - - private Builder(Class scorerClass) { - try { - this.scorer = scorerClass.getDeclaredConstructor() - .newInstance(); - } catch (Exception e) { - throw new RuntimeException("Failed to create scorer instance", e); - } - } - - public Builder threshold(double threshold) { - scorer.setThreshold(threshold); - return this; - } - - public Builder name(String name) { - scorer.setName(name); - return this; - } - - public Builder strictMode(boolean strictMode) { - scorer.setStrictMode(strictMode); - return this; - } - - public Builder requiredParams(List requiredParams) { - scorer.setRequiredParams(requiredParams); - return this; - } - - public Builder model(String model) { - scorer.setModel(model); - return this; - } - - public Builder additionalProperty(String key, Object value) { - scorer.setAdditionalProperty(key, value); - return this; - } - - public T build() { - return scorer; - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/BaseScorer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/BaseScorer.java deleted file mode 100644 index 37e35cd..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/scorers/BaseScorer.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.judgmentlabs.judgeval.v1.scorers; - -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -/** - * Base interface for all scorers in the Judgment evaluation system. - */ -public interface BaseScorer { - /** - * Returns the name of this scorer. - * - * @return the scorer name - */ - String getName(); - - /** - * Returns the configuration for this scorer. - * - * @return the scorer configuration - */ - ScorerConfig getScorerConfig(); -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracer.java deleted file mode 100644 index 41d68b5..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracer.java +++ /dev/null @@ -1,563 +0,0 @@ -package com.judgmentlabs.judgeval.v1.tracer; - -import java.lang.reflect.Type; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.judgmentlabs.judgeval.JudgmentAttributeKeys; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; -import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; -import com.judgmentlabs.judgeval.utils.Logger; -import com.judgmentlabs.judgeval.v1.data.Example; -import com.judgmentlabs.judgeval.v1.scorers.BaseScorer; -import com.judgmentlabs.judgeval.v1.scorers.custom_scorer.CustomScorer; -import com.judgmentlabs.judgeval.v1.tracer.exporters.JudgmentSpanExporter; -import com.judgmentlabs.judgeval.v1.tracer.exporters.NoOpSpanExporter; - -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.context.Scope; -import io.opentelemetry.sdk.trace.export.SpanExporter; - -/** - * Base tracer providing span manipulation, attribute setting, and evaluation - * capabilities. - */ -public abstract class BaseTracer { - public static final String TRACER_NAME = "judgeval"; - - protected final String projectName; - protected final boolean enableEvaluation; - protected final JudgmentSyncClient apiClient; - protected final ISerializer serializer; - protected final ObjectMapper jacksonMapper; - protected final Optional projectId; - - protected BaseTracer(String projectName, - boolean enableEvaluation, JudgmentSyncClient apiClient, ISerializer serializer) { - this.projectName = Objects.requireNonNull(projectName, "projectName required"); - this.enableEvaluation = enableEvaluation; - this.apiClient = Objects.requireNonNull(apiClient, "apiClient required"); - this.serializer = Objects.requireNonNull(serializer, "serializer required"); - this.jacksonMapper = new ObjectMapper(); - this.projectId = resolveProjectId(projectName); - - this.projectId.ifPresentOrElse(id -> { - }, () -> Logger.error("Failed to resolve project " + projectName - + ", please create it first at https://app.judgmentlabs.ai/org/" + this.apiClient.getOrganizationId() - + "/projects. Skipping Judgment export.")); - } - - /** - * Initializes the tracer. - */ - public abstract void initialize(); - - /** - * Forces pending spans to flush. - * - * @param timeoutMillis - * maximum time to wait in milliseconds - * @return true if flush succeeded within timeout - */ - public abstract boolean forceFlush(int timeoutMillis); - - /** - * Shuts down the tracer. - * - * @param timeoutMillis - * maximum time to wait for shutdown in milliseconds - */ - public abstract void shutdown(int timeoutMillis); - - /** - * Returns the span exporter for this tracer. - * - * @return the span exporter - */ - public SpanExporter getSpanExporter() { - return projectId.map(this::createJudgmentSpanExporter) - .orElseGet(() -> { - Logger.error("Project not resolved; cannot create exporter, returning NoOpSpanExporter"); - return new NoOpSpanExporter(); - }); - } - - /** - * Sets the span kind attribute on the current span. - * - * @param kind - * the span kind - */ - public void setSpanKind(String kind) { - Optional.ofNullable(kind) - .ifPresent(k -> withCurrentSpan( - span -> span.setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); - } - - private static void withCurrentSpan(java.util.function.Consumer action) { - Optional.ofNullable(Span.current()) - .ifPresent(action); - } - - private static boolean isValidKey(String key) { - return key != null && !key.isEmpty(); - } - - /** - * Sets an attribute on the current span by serializing the value. - * - * @param key - * the attribute key - * @param value - * the attribute value - */ - public void setAttribute(String key, Object value) { - if (!isValidKey(key)) { - return; - } - if (value != null) { - setAttribute(key, value, value.getClass()); - } - } - - /** - * Sets an attribute on the current span by serializing the value with the - * specified type. - * - * @param key - * the attribute key - * @param value - * the attribute value - * @param type - * the type to use for serialization - */ - public void setAttribute(String key, Object value, Type type) { - if (!isValidKey(key)) { - return; - } - if (value != null) { - withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(value, type))); - } - } - - /** - * Sets a string attribute on the current span. - * - * @param key - * the attribute key - * @param value - * the attribute value - */ - public void setAttribute(String key, String value) { - if (!isValidKey(key)) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, value)); - } - - /** - * Sets a long attribute on the current span. - * - * @param key - * the attribute key - * @param value - * the attribute value - */ - public void setAttribute(String key, long value) { - if (!isValidKey(key)) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, value)); - } - - /** - * Sets a double attribute on the current span. - * - * @param key - * the attribute key - * @param value - * the attribute value - */ - public void setAttribute(String key, double value) { - if (!isValidKey(key)) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, value)); - } - - /** - * Sets a boolean attribute on the current span. - * - * @param key - * the attribute key - * @param value - * the attribute value - */ - public void setAttribute(String key, boolean value) { - if (!isValidKey(key)) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, value)); - } - - private Optional getSampledSpanContext() { - return Optional.ofNullable(Span.current()) - .filter(span -> span.getSpanContext() - .isSampled()) - .map(Span::getSpanContext); - } - - private Optional getSampledSpan() { - return Optional.ofNullable(Span.current()) - .filter(span -> span.getSpanContext() - .isSampled()); - } - - private boolean isEvaluationEnabled() { - return enableEvaluation; - } - - private void logEvaluationInfo(String method, String traceId, String spanId, String scorerName) { - Logger.info(method + ": project=" + projectName + ", traceId=" + traceId + ", spanId=" - + spanId + ", scorer=" + scorerName); - } - - private void safeExecute(String operation, Runnable action) { - try { - action.run(); - } catch (Exception e) { - Logger.error("Failed to " + operation + ": " + e.getMessage()); - } - } - - /** - * Asynchronously evaluates the current span using the specified scorer and - * example. - * The evaluation is queued and processed asynchronously by the Judgment - * backend. - * - * @param scorer - * the scorer to use for evaluation - * @param example - * the example data to evaluate against - */ - public void asyncEvaluate(BaseScorer scorer, Example example) { - safeExecute("evaluate scorer", () -> { - if (!isEvaluationEnabled()) { - return; - } - - getSampledSpanContext().ifPresent(spanContext -> { - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - logEvaluationInfo("asyncEvaluate", traceId, spanId, scorer.getName()); - - ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, traceId, spanId); - enqueueEvaluation(evaluationRun); - }); - }); - } - - /** - * Asynchronously evaluates the current trace using the specified scorer. - * Attaches evaluation metadata to the current span for processing after trace - * completion. - * - * @param scorer - * the scorer to use for trace evaluation - */ - public void asyncTraceEvaluate(BaseScorer scorer) { - safeExecute("evaluate trace scorer", () -> { - if (!isEvaluationEnabled()) { - return; - } - - getSampledSpan().ifPresent(currentSpan -> { - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - logEvaluationInfo("asyncTraceEvaluate", traceId, spanId, scorer.getName()); - - TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, traceId, spanId); - try { - String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_PENDING_TRACE_EVAL, - traceEvalJson); - } catch (Exception e) { - Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); - } - }); - }); - } - - /** - * Sets multiple attributes on the current span. - * - * @param attributes - * map of attribute keys to values - */ - public void setAttributes(Map attributes) { - Optional.ofNullable(attributes) - .ifPresent(attrs -> attrs.forEach(this::setAttribute)); - } - - /** - * Marks the current span as an LLM span. - */ - public void setLLMSpan() { - setSpanKind("llm"); - } - - /** - * Marks the current span as a tool span. - */ - public void setToolSpan() { - setSpanKind("tool"); - } - - /** - * Marks the current span as a general span. - */ - public void setGeneralSpan() { - setSpanKind("span"); - } - - /** - * Sets the input attribute on the current span. - * - * @param input - * the input value - */ - public void setInput(Object input) { - setInput(input, input.getClass()); - } - - /** - * Sets the output attribute on the current span. - * - * @param output - * the output value - */ - public void setOutput(Object output) { - setOutput(output, output.getClass()); - } - - /** - * Sets the input attribute on the current span using the specified type. - * - * @param input - * the input value - * @param type - * the type to use for serialization - */ - public void setInput(Object input, Type type) { - setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_INPUT, input, type); - } - - /** - * Sets the output attribute on the current span using the specified type. - * - * @param output - * the output value - * @param type - * the type to use for serialization - */ - public void setOutput(Object output, Type type) { - setAttribute(JudgmentAttributeKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); - } - - /** - * Executes a runnable within a new span, automatically handling span lifecycle - * and errors. - * - * @param spanName - * the name of the span - * @param runnable - * the code to execute within the span - */ - public void span(String spanName, Runnable runnable) { - Span span = getTracer().spanBuilder(spanName) - .startSpan(); - try (Scope scope = span.makeCurrent()) { - runnable.run(); - } catch (Exception e) { - span.setStatus(StatusCode.ERROR).recordException(e); - throw e; - } finally { - span.end(); - } - } - - /** - * Executes a callable within a new span, automatically handling span lifecycle - * and errors. - * - * @param - * the return type - * @param spanName - * the name of the span - * @param callable - * the code to execute within the span - * @return the result of the callable - * @throws Exception - * if the callable throws an exception - */ - public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { - Span span = getTracer().spanBuilder(spanName) - .startSpan(); - try (Scope scope = span.makeCurrent()) { - return callable.call(); - } catch (Exception e) { - span.setStatus(StatusCode.ERROR).recordException(e); - throw e; - } finally { - span.end(); - } - } - - /** - * Returns the OpenTelemetry tracer instance. - * - * @return the OpenTelemetry tracer - */ - public io.opentelemetry.api.trace.Tracer getTracer() { - return GlobalOpenTelemetry.get() - .getTracer(TRACER_NAME); - } - - /** - * Returns the project name. - * - * @return the project name - */ - public String getProjectName() { - return projectName; - } - - /** - * Returns whether evaluation is enabled. - * - * @return true if evaluation is enabled - */ - public boolean isEnableEvaluation() { - return enableEvaluation; - } - - /** - * Returns the resolved project ID if available. - * - * @return the project ID, or empty if not resolved - */ - public Optional getProjectId() { - return projectId; - } - - /** - * Creates and returns a new span with the specified name. - * - * @param spanName - * the name of the span - * @return the created span - */ - public static Span span(String spanName) { - return GlobalOpenTelemetry.get() - .getTracer(TRACER_NAME) - .spanBuilder(spanName) - .startSpan(); - } - - private Optional resolveProjectId(String name) { - try { - ResolveProjectNameRequest request = new ResolveProjectNameRequest(); - request.setProjectName(name); - ResolveProjectNameResponse response = apiClient.projectsResolve(request); - return Optional.ofNullable(response.getProjectId()) - .map(Object::toString); - } catch (Exception e) { - return Optional.empty(); - } - } - - private String buildEndpoint(String baseUrl) { - return baseUrl.endsWith("/") ? baseUrl + "otel/v1/traces" : baseUrl + "/otel/v1/traces"; - } - - private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { - return JudgmentSpanExporter.builder() - .endpoint(buildEndpoint(apiClient.getApiUrl())) - .apiKey(apiClient.getApiKey()) - .organizationId(apiClient.getOrganizationId()) - .projectId(projectId) - .build(); - } - - private String generateRunId(String prefix, String spanId) { - return prefix + Optional.ofNullable(spanId) - .orElseGet(() -> String.valueOf(System.currentTimeMillis())); - } - - private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example example, String traceId, - String spanId) { - String runId = generateRunId("async_evaluate_", spanId); - - ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(); - evaluationRun.setId(UUID.randomUUID().toString()); - evaluationRun.setProjectName(projectName); - evaluationRun.setEvalName(runId); - evaluationRun.setTraceId(traceId); - evaluationRun.setTraceSpanId(spanId); - evaluationRun.setExamples(List.of(example)); - - if (scorer instanceof CustomScorer) { - evaluationRun.setJudgmentScorers(List.of()); - evaluationRun.setCustomScorers(List.of((com.judgmentlabs.judgeval.internal.api.models.BaseScorer) scorer)); - } else { - evaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); - evaluationRun.setCustomScorers(List.of()); - } - evaluationRun.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - - return evaluationRun; - } - - private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String traceId, - String spanId) { - String evalName = generateRunId("async_trace_evaluate_", spanId); - - TraceEvaluationRun evaluationRun = new TraceEvaluationRun(); - evaluationRun.setId(UUID.randomUUID().toString()); - evaluationRun.setProjectName(projectName); - evaluationRun.setEvalName(evalName); - evaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); - evaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); - evaluationRun.setCustomScorers(List.of()); - evaluationRun.setIsOffline(false); - evaluationRun.setIsBucketRun(false); - evaluationRun.setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - - return evaluationRun; - } - - private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { - try { - apiClient.addToRunEvalQueue(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to enqueue evaluation run: " + e.getMessage()); - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/ISerializer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/ISerializer.java deleted file mode 100644 index 8d84fe8..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/ISerializer.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.judgmentlabs.judgeval.v1.tracer; - -import java.lang.reflect.Type; - -/** - * Serializes objects to string representations. - */ -public interface ISerializer { - /** - * Serializes an object to a string. - * - * @param obj - * the object to serialize - * @return the serialized string - */ - String serialize(Object obj); - - /** - * Serializes an object to a string using the specified type. - * - * @param obj - * the object to serialize - * @param type - * the type to use for serialization - * @return the serialized string - */ - default String serialize(Object obj, Type type) { - return serialize(obj); - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/Tracer.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/Tracer.java deleted file mode 100644 index 301be20..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/Tracer.java +++ /dev/null @@ -1,211 +0,0 @@ -package com.judgmentlabs.judgeval.v1.tracer; - -import java.lang.reflect.Type; -import java.util.Objects; -import java.util.Optional; - -import com.google.gson.Gson; -import com.judgmentlabs.judgeval.Version; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.utils.Logger; - -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; -import io.opentelemetry.sdk.trace.export.SpanExporter; - -public final class Tracer extends BaseTracer { - private SdkTracerProvider tracerProvider; - - private Tracer(Builder builder) { - super( - Objects.requireNonNull(builder.projectName, "projectName required"), - builder.enableEvaluation, - Objects.requireNonNull(builder.client, "client required"), - builder.serializer != null ? builder.serializer : new GsonSerializer()); - - if (builder.initialize) { - initialize(); - } - } - - /** - * Initializes the tracer by setting up the OpenTelemetry SDK with a span - * exporter, - * configuring the tracer provider with batch span processing, and registering - * it globally. - */ - @Override - public void initialize() { - SpanExporter spanExporter = getSpanExporter(); - - var resource = Resource.getDefault() - .merge(Resource.create(Attributes.builder() - .put("service.name", projectName) - .put("telemetry.sdk.name", TRACER_NAME) - .put("telemetry.sdk.version", Version.getVersion()) - .build())); - - this.tracerProvider = SdkTracerProvider.builder() - .setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder(spanExporter) - .build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(this.tracerProvider) - .build(); - - GlobalOpenTelemetry.set(openTelemetry); - } - - /** - * Forces the tracer to flush any pending spans within the specified timeout. - * - * @param timeoutMillis - * the maximum time to wait in milliseconds - * @return true if the flush completed successfully within the timeout - */ - @Override - public boolean forceFlush(int timeoutMillis) { - if (tracerProvider == null) { - Logger.error("Cannot forceFlush: tracer not initialized"); - return false; - } - return tracerProvider.forceFlush() - .join(timeoutMillis, java.util.concurrent.TimeUnit.MILLISECONDS) - .isSuccess(); - } - - /** - * Shuts down the tracer, flushing any remaining spans and releasing resources. - * - * @param timeoutMillis - * the maximum time to wait for shutdown in milliseconds - */ - @Override - public void shutdown(int timeoutMillis) { - if (tracerProvider == null) { - Logger.error("Cannot shutdown: tracer not initialized"); - return; - } - tracerProvider.shutdown() - .join(timeoutMillis, java.util.concurrent.TimeUnit.MILLISECONDS); - } - - /** - * Creates a new builder for configuring a Tracer. - * - * @return a new builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Builder for configuring and creating Tracer instances. - */ - public static final class Builder { - private JudgmentSyncClient client; - private String projectName; - private boolean enableEvaluation = true; - private ISerializer serializer; - private boolean initialize = true; - - /** - * Sets the Judgment API client. - * - * @param client - * the API client - * @return this builder - */ - public Builder client(JudgmentSyncClient client) { - this.client = client; - return this; - } - - /** - * Sets the project name for this tracer. - * - * @param projectName - * the project name - * @return this builder - */ - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - /** - * Sets whether evaluation is enabled. - * - * @param enableEvaluation - * true to enable evaluation - * @return this builder - */ - public Builder enableEvaluation(boolean enableEvaluation) { - this.enableEvaluation = enableEvaluation; - return this; - } - - /** - * Sets the custom serializer for span attributes. - * - * @param serializer - * the serializer - * @return this builder - */ - public Builder serializer(ISerializer serializer) { - this.serializer = serializer; - return this; - } - - /** - * Sets whether to automatically initialize the tracer on build. - * - * @param initialize - * true to initialize on build - * @return this builder - */ - public Builder initialize(boolean initialize) { - this.initialize = initialize; - return this; - } - - /** - * Builds and returns a new Tracer instance. - * - * @return the configured Tracer - */ - public Tracer build() { - return new Tracer(this); - } - } - - private static class GsonSerializer implements ISerializer { - private final Gson gson = new Gson(); - - @Override - public String serialize(Object obj) { - return Optional.ofNullable(obj) - .map(o -> serialize(o, o.getClass())) - .orElse(null); - } - - @Override - public String serialize(Object obj, Type type) { - try { - return gson.toJson(obj, type); - } catch (Exception e) { - Logger.error("Failed to serialize object: " + e.getMessage()); - return Optional.ofNullable(obj) - .map(Object::toString) - .orElse(null); - } - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/JudgmentSpanExporter.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/JudgmentSpanExporter.java deleted file mode 100644 index d31674d..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/JudgmentSpanExporter.java +++ /dev/null @@ -1,179 +0,0 @@ -package com.judgmentlabs.judgeval.v1.tracer.exporters; - -import java.util.Collection; -import java.util.Optional; - -import com.judgmentlabs.judgeval.utils.Logger; - -import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; - -/** - * SpanExporter implementation that sends spans to Judgment Labs with project - * identification. - *

- * This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific - * headers and project identification to all exported spans. - */ -public class JudgmentSpanExporter implements SpanExporter { - private final SpanExporter delegate; - - /** - * Creates a new JudgmentSpanExporter with the specified configuration. - * - * @param endpoint - * the OTLP endpoint URL - * @param apiKey - * the API key for authentication - * @param organizationId - * the organization ID - * @param projectId - * the project ID (must not be null or empty) - * @throws IllegalArgumentException - * if projectId is null or empty - */ - protected JudgmentSpanExporter(String endpoint, String apiKey, String organizationId, String projectId) { - if (projectId.isEmpty()) { - throw new IllegalArgumentException("projectId is required for JudgmentSpanExporter"); - } - this.delegate = OtlpHttpSpanExporter.builder() - .setEndpoint(endpoint) - .addHeader("Authorization", "Bearer " + apiKey) - .addHeader("X-Organization-Id", organizationId) - .addHeader("X-Project-Id", projectId) - .build(); - } - - /** - * Creates a new builder for constructing JudgmentSpanExporter instances. - * - * @return a new Builder instance - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Exports the collection of spans to the Judgment Labs backend. - * - * @param spans - * the collection of spans to export - * @return a CompletableResultCode representing the export operation status - */ - @Override - public CompletableResultCode export(Collection spans) { - Logger.info("Exported " + spans.size() + " spans"); - return delegate.export(spans); - } - - /** - * Flushes any pending span exports. - * - * @return a CompletableResultCode representing the flush operation status - */ - @Override - public CompletableResultCode flush() { - return delegate.flush(); - } - - /** - * Shuts down this exporter and releases any resources. - * - * @return a CompletableResultCode representing the shutdown operation status - */ - @Override - public CompletableResultCode shutdown() { - return delegate.shutdown(); - } - - /** - * Builder for creating JudgmentSpanExporter instances. - */ - public static final class Builder { - private String endpoint; - private String apiKey; - private String organizationId; - private String projectId; - - private Builder() { - } - - /** - * Sets the OTLP endpoint URL. - * - * @param endpoint - * the endpoint URL - * @return this builder for method chaining - */ - public Builder endpoint(String endpoint) { - this.endpoint = endpoint; - return this; - } - - /** - * Sets the API key for authentication. - * - * @param apiKey - * the API key - * @return this builder for method chaining - */ - public Builder apiKey(String apiKey) { - this.apiKey = apiKey; - return this; - } - - /** - * Sets the organization ID. - * - * @param organizationId - * the organization ID - * @return this builder for method chaining - */ - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - /** - * Sets the project ID. - * - * @param projectId - * the project ID - * @return this builder for method chaining - */ - public Builder projectId(String projectId) { - this.projectId = projectId; - return this; - } - - /** - * Builds a new JudgmentSpanExporter instance with the configured settings. - * - * @return a new JudgmentSpanExporter instance - * @throws IllegalArgumentException - * if any required field is null or empty - */ - public JudgmentSpanExporter build() { - String validEndpoint = Optional.ofNullable(endpoint) - .map(String::trim) - .filter(e -> !e.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("Endpoint is required")); - String validApiKey = Optional.ofNullable(apiKey) - .map(String::trim) - .filter(key -> !key.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("API key is required")); - String validOrganizationId = Optional.ofNullable(organizationId) - .map(String::trim) - .filter(id -> !id.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("Organization ID is required")); - String validProjectId = Optional.ofNullable(projectId) - .map(String::trim) - .filter(id -> !id.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("Project ID is required")); - - return new JudgmentSpanExporter(validEndpoint, validApiKey, validOrganizationId, validProjectId); - } - } -} diff --git a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/NoOpSpanExporter.java b/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/NoOpSpanExporter.java deleted file mode 100644 index 6bb5244..0000000 --- a/judgeval-java/src/main/java/com/judgmentlabs/judgeval/v1/tracer/exporters/NoOpSpanExporter.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.judgmentlabs.judgeval.v1.tracer.exporters; - -import java.util.Collection; - -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; - -/** - * A no-op implementation of SpanExporter that discards all spans. Used as a - * fallback when project resolution fails or when spans should not be exported. - */ -public class NoOpSpanExporter implements SpanExporter { - /** - * Discards the collection of spans without exporting. - * - * @param spans - * the collection of spans (ignored) - * @return a successful CompletableResultCode - */ - @Override - public CompletableResultCode export(Collection spans) { - return CompletableResultCode.ofSuccess(); - } - - /** - * Performs a no-op flush operation. - * - * @return a successful CompletableResultCode - */ - @Override - public CompletableResultCode flush() { - return CompletableResultCode.ofSuccess(); - } - - /** - * Performs a no-op shutdown operation. - * - * @return a successful CompletableResultCode - */ - @Override - public CompletableResultCode shutdown() { - return CompletableResultCode.ofSuccess(); - } -} diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/JudgevalTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/JudgevalTest.java similarity index 91% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/JudgevalTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/JudgevalTest.java index 5277d27..c72e115 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/JudgevalTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/JudgevalTest.java @@ -4,9 +4,9 @@ import org.junit.jupiter.api.Test; -import com.judgmentlabs.judgeval.v1.evaluation.EvaluationFactory; -import com.judgmentlabs.judgeval.v1.scorers.ScorersFactory; -import com.judgmentlabs.judgeval.v1.tracer.TracerFactory; +import com.judgmentlabs.judgeval.evaluation.EvaluationFactory; +import com.judgmentlabs.judgeval.scorers.ScorersFactory; +import com.judgmentlabs.judgeval.tracer.TracerFactory; class JudgevalTest { private static final String TEST_API_URL = "https://api.test.com"; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/data/ExampleTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/data/ExampleTest.java similarity index 96% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/data/ExampleTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/data/ExampleTest.java index 204fe53..67acef9 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/data/ExampleTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/data/ExampleTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.data; +package com.judgmentlabs.judgeval.data; import static org.junit.jupiter.api.Assertions.*; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactoryTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactoryTest.java similarity index 95% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactoryTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactoryTest.java index bde3d5a..0cb131e 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/evaluation/EvaluationFactoryTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/evaluation/EvaluationFactoryTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.evaluation; +package com.judgmentlabs.judgeval.evaluation; import static org.junit.jupiter.api.Assertions.*; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactoryTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/ScorersFactoryTest.java similarity index 82% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactoryTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/ScorersFactoryTest.java index 625ce85..fe58522 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/ScorersFactoryTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/ScorersFactoryTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.scorers; +package com.judgmentlabs.judgeval.scorers; import static org.junit.jupiter.api.Assertions.*; @@ -6,9 +6,9 @@ import org.junit.jupiter.api.Test; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.v1.scorers.built_in.BuiltInScorersFactory; -import com.judgmentlabs.judgeval.v1.scorers.custom_scorer.CustomScorerFactory; -import com.judgmentlabs.judgeval.v1.scorers.prompt_scorer.PromptScorerFactory; +import com.judgmentlabs.judgeval.scorers.built_in.BuiltInScorersFactory; +import com.judgmentlabs.judgeval.scorers.custom_scorer.CustomScorerFactory; +import com.judgmentlabs.judgeval.scorers.prompt_scorer.PromptScorerFactory; class ScorersFactoryTest { private static final String TEST_API_URL = "https://api.test.com"; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactoryTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactoryTest.java similarity index 93% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactoryTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactoryTest.java index fe8d71d..ae5c524 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/scorers/built_in/BuiltInScorersFactoryTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/scorers/built_in/BuiltInScorersFactoryTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.scorers.built_in; +package com.judgmentlabs.judgeval.scorers.built_in; import static org.junit.jupiter.api.Assertions.*; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracerTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/BaseTracerTest.java similarity index 96% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracerTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/BaseTracerTest.java index 86e01fd..af29ca9 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/BaseTracerTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/BaseTracerTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.tracer; +package com.judgmentlabs.judgeval.tracer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -18,8 +18,8 @@ import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; -import com.judgmentlabs.judgeval.v1.tracer.exporters.JudgmentSpanExporter; -import com.judgmentlabs.judgeval.v1.tracer.exporters.NoOpSpanExporter; +import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; +import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; import io.opentelemetry.sdk.trace.export.SpanExporter; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactoryTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerFactoryTest.java similarity index 94% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactoryTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerFactoryTest.java index f3637fe..2c6ec4a 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerFactoryTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerFactoryTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.tracer; +package com.judgmentlabs.judgeval.tracer; import static org.junit.jupiter.api.Assertions.*; diff --git a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerTest.java b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerTest.java similarity index 97% rename from judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerTest.java rename to judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerTest.java index 3a57598..5a707cf 100644 --- a/judgeval-java/src/test/java/com/judgmentlabs/judgeval/v1/tracer/TracerTest.java +++ b/judgeval-java/src/test/java/com/judgmentlabs/judgeval/tracer/TracerTest.java @@ -1,4 +1,4 @@ -package com.judgmentlabs.judgeval.v1.tracer; +package com.judgmentlabs.judgeval.tracer; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/package.json b/package.json index fc25fe5..cd327c2 100644 --- a/package.json +++ b/package.json @@ -3,28 +3,18 @@ "version": "0.0.0", "private": true, "scripts": { - "format": "npm run format:core && npm run format:openai", - "format:core": "mvn -B -f judgeval-java/pom.xml spotless:apply", - "format:openai": "mvn -B -f instrumentation/judgeval-instrumentation-openai/pom.xml spotless:apply", - "format:check": "npm run format:check:core && npm run format:check:openai", - "format:check:core": "cd judgeval-java && mvn -B spotless:check", - "format:check:openai": "cd instrumentation/judgeval-instrumentation-openai && mvn -B spotless:check", + "format": "cd judgeval-java && mvn -B spotless:apply", + "format:check": "cd judgeval-java && mvn -B spotless:check", "check": "cd judgeval-java && mvn -B compile checkstyle:check spotless:check", "lint": "cd judgeval-java && mvn -B checkstyle:check", "test": "cd judgeval-java && mvn test", - "clean": "npm run clean:core && npm run clean:openai", - "clean:core": "cd judgeval-java && mvn clean", - "clean:openai": "cd instrumentation/judgeval-instrumentation-openai && mvn clean", + "clean": "cd judgeval-java && mvn clean", "build": "cd judgeval-java && mvn -B clean compile", - "install:all": "npm run install:core && npm run install:openai", - "install:core": "cd judgeval-java && mvn -B -Dgpg.skip=true clean install", - "install:openai": "cd instrumentation/judgeval-instrumentation-openai && mvn -B -Dgpg.skip=true clean install", + "install": "cd judgeval-java && mvn -B -Dgpg.skip=true clean install", "generate:client": "npm run generate:client:raw && npm run format", "generate:client:raw": "./scripts/generate-client.sh", "ci": "cd judgeval-java && mvn -B clean compile test checkstyle:check spotless:check", - "status": "npm run status:core && npm run status:openai", - "status:core": "cd judgeval-java && echo \"[status] judgeval-java\" && G=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.groupId) && A=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.artifactId) && V=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version) && echo \"GAV: $G:$A:$V\" && (ls -1 target/*.jar 2>/dev/null || echo \"No jar built\")", - "status:openai": "cd instrumentation/judgeval-instrumentation-openai && echo \"[status] instrumentation/judgeval-instrumentation-openai\" && G=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.groupId) && A=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.artifactId) && V=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version) && echo \"GAV: $G:$A:$V\" && (ls -1 target/*.jar 2>/dev/null || echo \"No jar built\")", + "status": "cd judgeval-java && echo \"[status] judgeval-java\" && G=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.groupId) && A=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.artifactId) && V=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version) && echo \"GAV: $G:$A:$V\" && (ls -1 target/*.jar 2>/dev/null || echo \"No jar built\")", "example": "dotenv -e .env -- ./scripts/run-example.sh" }, "devDependencies": { From 2ae36f28d08b930b6cdc07e81e70f3484d788e3e Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 11 Nov 2025 17:21:18 -0800 Subject: [PATCH 2/2] deprecate: instrumentation ci --- .../publish-instrumentation-openai.yml | 111 ------------------ .../workflows/test-instrumentation-openai.yml | 43 ------- 2 files changed, 154 deletions(-) delete mode 100644 .github/workflows/publish-instrumentation-openai.yml delete mode 100644 .github/workflows/test-instrumentation-openai.yml diff --git a/.github/workflows/publish-instrumentation-openai.yml b/.github/workflows/publish-instrumentation-openai.yml deleted file mode 100644 index 36218ed..0000000 --- a/.github/workflows/publish-instrumentation-openai.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Publish judgeval-instrumentation-openai - -on: - workflow_dispatch: - release: - types: [created] - push: - branches: [main] - paths: - - "instrumentation/judgeval-instrumentation-openai/**" - -jobs: - test-and-build: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - - - name: Set up Java - uses: actions/setup-java@v4 - with: - java-version: "21" - distribution: "temurin" - cache: "maven" - - - name: Test and Build - run: | - cd instrumentation/judgeval-instrumentation-openai - mvn clean install -Dgpg.skip=true - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: judgeval-instrumentation-openai-build - path: | - instrumentation/judgeval-instrumentation-openai/target/*.jar - instrumentation/judgeval-instrumentation-openai/target/*.pom - retention-days: 1 - - publish: - needs: test-and-build - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - - name: Set up Java and Maven Central Repository - uses: actions/setup-java@v4 - with: - java-version: "21" - distribution: "temurin" - cache: "maven" - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: judgeval-instrumentation-openai-build - path: instrumentation/judgeval-instrumentation-openai/target - - - name: Verify POM groupId - run: | - echo "Checking pom.xml for correct groupId..." - grep -E ".*" instrumentation/judgeval-instrumentation-openai/pom.xml | head -1 - echo "Expected: com.judgmentlabs" - - - name: Show Maven coordinates - run: | - cd instrumentation/judgeval-instrumentation-openai - G=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.groupId) - A=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.artifactId) - V=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version) - echo "Publishing: ${G}:${A}:${V}" - - - name: Import GPG key - run: | - echo "${{ secrets.GPG_PRIVATE_KEY }}" | base64 --decode | gpg --import --batch --yes - echo "use-agent" >> ~/.gnupg/gpg.conf - echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf - echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf - gpg-connect-agent reloadagent /bye - gpg --list-secret-keys --keyid-format LONG - - - name: Deploy to Central Portal - run: | - cd instrumentation/judgeval-instrumentation-openai - mvn -e --batch-mode deploy \ - -DskipTests \ - -Dmaven.install.skip=true \ - -Dgpg.passphrase="${{ secrets.GPG_PASSPHRASE }}" \ - -Dgpg.pinentry-mode=loopback \ - -Dhttp.keepAlive=true \ - -Dmaven.wagon.http.pool=false \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \ - -Dmaven.wagon.http.retryHandler.count=3 - env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_PORTAL_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_PORTAL_TOKEN }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - - - name: Deployment Status - run: | - echo "Deployment completed successfully!" - echo "Artifacts uploaded to Central Portal" - echo "Check status at: https://central.sonatype.com/publishing/deployments" - echo "" - echo "Note: It may take 15-30 minutes for artifacts to sync to Maven Central" - echo "Once published, artifacts will be available at:" - echo " https://central.sonatype.com/artifact/com.judgmentlabs/judgeval-instrumentation-openai" diff --git a/.github/workflows/test-instrumentation-openai.yml b/.github/workflows/test-instrumentation-openai.yml deleted file mode 100644 index cffa159..0000000 --- a/.github/workflows/test-instrumentation-openai.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Test judgeval-instrumentation-openai - -on: - pull_request: - branches: [main] - paths: - - "instrumentation/judgeval-instrumentation-openai/**" - - "pom.xml" - - ".github/workflows/test-instrumentation-openai.yml" - -jobs: - test-install: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Java - uses: actions/setup-java@v4 - with: - java-version: "21" - distribution: "temurin" - - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Run Tests - run: | - cd instrumentation/judgeval-instrumentation-openai && mvn -B test - - - name: Test Install - run: | - cd instrumentation/judgeval-instrumentation-openai && mvn -B -Dgpg.skip=true clean install - - - name: Verify Build Artifacts - run: | - echo "Checking build artifacts..." - ls -la instrumentation/judgeval-instrumentation-openai/target/ || true - echo "JAR files:" - find instrumentation/judgeval-instrumentation-openai/target -name "*.jar" -type f