Merge remote-tracking branch 'origin/main' into OPIK-615

.github/workflows/lib-guardrails-tests.yml

@@ -0,0 +1,57 @@
+# Workflow to run Guar tests
+#
+# Please read inputs to provide correct values.
+#
+name: SDK Lib Guardrails Tests
+run-name: "SDK Lib Guardrails Tests ${{ github.ref_name }} by @${{ github.actor }}"
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  OPENAI_ORG_ID:  ${{ secrets.OPENAI_ORG_ID }}
+  GUARDRAILS_API_KEY: ${{ secrets.GUARDRAILS_API_KEY }}
+on:
+  workflow_call:
+
+jobs:
+  tests:
+    name: Guardrails Python ${{matrix.python_version}}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: sdks/python
+
+    strategy:
+      fail-fast: true
+      matrix:
+        python_version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{matrix.python_version}}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python_version}}
+
+      - name: Install opik
+        run: pip install .
+
+      - name: Install test tools
+        run: |
+          cd ./tests
+          pip install --no-cache-dir --disable-pip-version-check -r test_requirements.txt
+
+      - name: Install lib 
+        run: |
+          cd ./tests
+          pip install --no-cache-dir --disable-pip-version-check -r library_integration/guardrails/requirements.txt
+
+      - name: Install checks from guardrails hub
+        run: |
+          guardrails configure --token $GUARDRAILS_API_KEY --disable-metrics --enable-remote-inferencing;
+          guardrails hub install hub://guardrails/politeness_check
+
+      - name: Run tests
+        run: |
+          cd ./tests/library_integration/guardrails/
+          python -m pytest  -vv .

.github/workflows/lib-integration-tests-runner.yml

@@ -17,6 +17,7 @@ on:
           - anthropic
           - aisuite
           - haystack
+          - guardrails
   schedule:
     - cron: "0 0 */1 * *"
   pull_request:
@@ -80,3 +81,9 @@ jobs:
     if: contains(fromJSON('["haystack", "all"]'), needs.init_environment.outputs.LIBS)
     uses: ./.github/workflows/lib-haystack-tests.yml
     secrets: inherit
+
+  guardrails_tests:
+    needs: [init_environment]
+    if: contains(fromJSON('["guardrails", "all"]'), needs.init_environment.outputs.LIBS)
+    uses: ./.github/workflows/lib-guardrails-tests.yml
+    secrets: inherit

README.md

@@ -8,11 +8,11 @@
         <br>
         Opik
     </div>
-    Open-source end-to-end LLM Development Platform<br>
+    Open source LLM evaluation framework<br>
 </h1>
 
 <p align="center">
-Confidently evaluate, test and monitor LLM applications. 
+From RAG chatbots to code assistants to complex agentic pipelines and beyond, build LLM systems that run better, faster, and cheaper with tracing, evaluations, and dashboards.
 </p>
 
 <div align="center">
@@ -49,6 +49,8 @@ You can use Opik for:
 
   * **Annotations:** Annotate your LLM calls by logging feedback scores using the [Python SDK](https://www.comet.com/docs/opik/tracing/annotate_traces/#annotating-traces-and-spans-using-the-sdk?from=llm&utm_source=opik&utm_medium=github&utm_content=sdk_link&utm_campaign=opik) or the [UI](https://www.comet.com/docs/opik/tracing/annotate_traces/#annotating-traces-through-the-ui?from=llm&utm_source=opik&utm_medium=github&utm_content=ui_link&utm_campaign=opik).
 
+  * **Playground:**: Try out different prompts and models in the [prompt playground](https://www.comet.com/docs/opik/evaluation/playground/?from=llm&utm_source=opik&utm_medium=github&utm_content=playground_link&utm_campaign=opik)
+
 * **Evaluation**: Automate the evaluation process of your LLM application:
 
     * **Datasets and Experiments**: Store test cases and run experiments ([Datasets](https://www.comet.com/docs/opik/evaluation/manage_datasets/?from=llm&utm_source=opik&utm_medium=github&utm_content=datasets_link&utm_campaign=opik), [Evaluate your LLM Application](https://www.comet.com/docs/opik/evaluation/evaluate_your_llm/?from=llm&utm_source=opik&utm_medium=github&utm_content=eval_link&utm_campaign=opik))
@@ -61,7 +63,7 @@ You can use Opik for:
 
     * **Log all your production traces**: Opik has been designed to support high volumes of traces, making it easy to monitor your production applications.
 
-    * **Monitoring dashboards**: Review your feedback scores, trace count and tokens over time in the [Opik Dashboard](https://www.comet.com/docs/opik/self-host/opik_dashboard/?from=llm&utm_source=opik&utm_medium=github&utm_content=dashboard_link&utm_campaign=opik)
+    * **Monitoring dashboards**: Review your feedback scores, trace count and tokens over time in the [Opik Dashboard](https://www.comet.com/docs/opik/self-host/opik_dashboard/?from=llm&utm_source=opik&utm_medium=github&utm_content=dashboard_link&utm_campaign=opik).
 
 > [!TIP]  
 > If you are looking for features that Opik doesn't have today, please raise a new [Feature request](https://github.com/comet-ml/opik/issues/new/choose) 🚀

apps/opik-backend/src/main/java/com/comet/opik/api/Project.java

@@ -11,8 +11,11 @@
 
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
+import static com.comet.opik.api.ProjectStats.PercentageValues;
+
 @Builder(toBuilder = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 // This annotation is used to specify the strategy to be used for naming of properties for the annotated type. Required so that OpenAPI schema generation uses snake_case
@@ -29,7 +32,15 @@ public record Project(
         @JsonView({Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant lastUpdatedAt,
         @JsonView({Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) String lastUpdatedBy,
         @JsonView({
-                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable Instant lastUpdatedTraceAt){
+                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable Instant lastUpdatedTraceAt,
+        @JsonView({
+                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable List<FeedbackScoreAverage> feedbackScores,
+        @JsonView({
+                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable PercentageValues duration,
+        @JsonView({
+                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable Double totalEstimatedCost,
+        @JsonView({
+                Project.View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) @Nullable Map<String, Double> usage){
 
     public static class View {
         public static class Write {

apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKey.java

@@ -9,6 +9,7 @@
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Size;
 import lombok.Builder;
 
 import java.time.Instant;
@@ -22,8 +23,9 @@ public record ProviderApiKey(
         @JsonView( {
                 View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) UUID id,
         @JsonView({View.Public.class, View.Write.class}) @NotNull LlmProvider provider,
-        @JsonView({
+        @JsonView({View.Public.class,
                 View.Write.class}) @NotBlank @JsonDeserialize(using = ProviderApiKeyDeserializer.class) String apiKey,
+        @JsonView({View.Public.class, View.Write.class}) @Size(max = 150) String name,
         @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant createdAt,
         @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) String createdBy,
         @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant lastUpdatedAt,
@@ -49,6 +51,7 @@ public static class Public {
         }
     }
 
+    @Builder(toBuilder = true)
     public record ProviderApiKeyPage(
             @JsonView( {
                     Project.View.Public.class}) int page,

apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKeyUpdate.java

@@ -6,13 +6,15 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.Size;
 import lombok.Builder;
 
 @Builder(toBuilder = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
 public record ProviderApiKeyUpdate(
-        @NotBlank @JsonDeserialize(using = ProviderApiKeyDeserializer.class) String apiKey) {
+        @NotBlank @JsonDeserialize(using = ProviderApiKeyDeserializer.class) String apiKey,
+        @Size(max = 150) String name) {
 
     @Override
     public String toString() {

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/LlmProviderApiKeyResource.java

@@ -38,6 +38,9 @@
 
 import java.util.UUID;
 
+import static com.comet.opik.infrastructure.EncryptionUtils.decrypt;
+import static com.comet.opik.infrastructure.EncryptionUtils.maskApiKey;
+
 @Path("/v1/private/llm-provider-key")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
@@ -60,10 +63,21 @@ public Response find() {
         String workspaceId = requestContext.get().getWorkspaceId();
 
         log.info("Find LLM Provider's ApiKeys for workspaceId '{}'", workspaceId);
-        Page<ProviderApiKey> providerApiKeyPage = llmProviderApiKeyService.find(workspaceId);
+        ProviderApiKey.ProviderApiKeyPage providerApiKeyPage = llmProviderApiKeyService.find(workspaceId);
         log.info("Found LLM Provider's ApiKeys for workspaceId '{}'", workspaceId);
 
-        return Response.ok().entity(providerApiKeyPage).build();
+        return Response.ok().entity(
+                        providerApiKeyPage.toBuilder()
+                                .content(
+                                        providerApiKeyPage.content().stream()
+                                                .map(providerApiKey -> providerApiKey.toBuilder()
+                                                        .apiKey(maskApiKey(decrypt(providerApiKey.apiKey())))
+                                                        .build())
+                                                .toList()
+                                )
+                                .build()
+                )
+                .build();
     }
 
     @GET
@@ -82,7 +96,9 @@ public Response getById(@PathParam("id") UUID id) {
 
         log.info("Got LLM Provider's ApiKey by id '{}' on workspace_id '{}'", id, workspaceId);
 
-        return Response.ok().entity(providerApiKey).build();
+        return Response.ok().entity(providerApiKey.toBuilder()
+                .apiKey(maskApiKey(decrypt(providerApiKey.apiKey())))
+                .build()).build();
     }
 
     @POST

apps/opik-backend/src/main/java/com/comet/opik/domain/LlmProviderApiKeyDAO.java

@@ -1,6 +1,7 @@
 package com.comet.opik.domain;
 
 import com.comet.opik.api.ProviderApiKey;
+import com.comet.opik.api.ProviderApiKeyUpdate;
 import com.comet.opik.infrastructure.db.UUIDArgumentFactory;
 import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory;
 import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper;
@@ -19,18 +20,18 @@
 @RegisterArgumentFactory(UUIDArgumentFactory.class)
 public interface LlmProviderApiKeyDAO {
 
-    @SqlUpdate("INSERT INTO llm_provider_api_key (id, provider, workspace_id, api_key, created_by, last_updated_by) VALUES (:bean.id, :bean.provider, :workspaceId, :bean.apiKey, :bean.createdBy, :bean.lastUpdatedBy)")
+    @SqlUpdate("INSERT INTO llm_provider_api_key (id, provider, workspace_id, api_key, name, created_by, last_updated_by) " +
+            "VALUES (:bean.id, :bean.provider, :workspaceId, :bean.apiKey, :bean.name, :bean.createdBy, :bean.lastUpdatedBy)")
     void save(@Bind("workspaceId") String workspaceId,
             @BindMethods("bean") ProviderApiKey providerApiKey);
 
     @SqlUpdate("UPDATE llm_provider_api_key SET " +
-            "api_key = :apiKey, " +
-            "last_updated_by = :lastUpdatedBy " +
+            "api_key = :bean.apiKey, name = :bean.name, last_updated_by = :lastUpdatedBy " +
             "WHERE id = :id AND workspace_id = :workspaceId")
     void update(@Bind("id") UUID id,
             @Bind("workspaceId") String workspaceId,
-            @Bind("apiKey") String encryptedApiKey,
-            @Bind("lastUpdatedBy") String lastUpdatedBy);
+            @Bind("lastUpdatedBy") String lastUpdatedBy,
+            @BindMethods("bean") ProviderApiKeyUpdate providerApiKeyUpdate);
 
     @SqlQuery("SELECT * FROM llm_provider_api_key WHERE id = :id AND workspace_id = :workspaceId")
     ProviderApiKey findById(@Bind("id") UUID id, @Bind("workspaceId") String workspaceId);

apps/opik-backend/src/main/java/com/comet/opik/domain/LlmProviderApiKeyService.java

@@ -30,7 +30,7 @@ public interface LlmProviderApiKeyService {
 
     ProviderApiKey find(UUID id, String workspaceId);
 
-    Page<ProviderApiKey> find(String workspaceId);
+    ProviderApiKey.ProviderApiKeyPage find(String workspaceId);
 
     ProviderApiKey saveApiKey(ProviderApiKey providerApiKey, String userName, String workspaceId);
 
@@ -58,12 +58,11 @@ public ProviderApiKey find(@NonNull UUID id, @NonNull String workspaceId) {
             return repository.fetch(id, workspaceId).orElseThrow(this::createNotFoundError);
         });
 
-        return providerApiKey.toBuilder()
-                .build();
+        return providerApiKey;
     }
 
     @Override
-    public Page<ProviderApiKey> find(@NonNull String workspaceId) {
+    public ProviderApiKey.ProviderApiKeyPage find(@NonNull String workspaceId) {
         List<ProviderApiKey> providerApiKeys = template.inTransaction(READ_ONLY, handle -> {
             var repository = handle.attach(LlmProviderApiKeyDAO.class);
             return repository.find(workspaceId);
@@ -122,8 +121,8 @@ public void updateApiKey(@NonNull UUID id, @NonNull ProviderApiKeyUpdate provide
 
             repository.update(providerApiKey.id(),
                     workspaceId,
-                    providerApiKeyUpdate.apiKey(),
-                    userName);
+                    userName,
+                    providerApiKeyUpdate);
 
             return null;
         });

apps/opik-backend/src/main/java/com/comet/opik/domain/ProjectMetricsService.java

@@ -20,7 +20,6 @@
 @ImplementedBy(ProjectMetricsServiceImpl.class)
 public interface ProjectMetricsService {
     String ERR_START_BEFORE_END = "'start_time' must be before 'end_time'";
-    String ERR_PROJECT_METRIC_NOT_SUPPORTED = "metric '%s' is not supported";
 
     Mono<ProjectMetricResponse<Number>> getProjectMetrics(UUID projectId, ProjectMetricRequest request);
 }