From b598ac7f7efe7bc624778cfe176cdaf0905c569a Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 22 Nov 2024 16:10:50 +0100 Subject: [PATCH] NO-JIRA: Updated Python SDK fern autogenerated code (#700) * NO-JIRA: Updated Python SDK fern autogenerated code * Rev2: update README.md --- .../java/com/comet/opik/api/DataPoint.java | 3 +- .../api/metrics/ProjectMetricRequest.java | 3 +- .../api/metrics/ProjectMetricResponse.java | 4 +- .../opik/domain/ProjectMetricsService.java | 3 +- .../java/com/comet/opik/domain/SpanDAO.java | 3 +- .../comet/opik/domain/cost/ModelPrice.java | 81 +++-- .../opik/domain/cost/SpanCostCalculator.java | 3 +- .../v1/priv/ProjectMetricsResourceTest.java | 5 +- sdks/python/code_generation/fern/README.md | 33 +++ .../code_generation/fern/openapi/openapi.yaml | 225 +++++++++++++- sdks/python/code_generation/fern/readme.md | 12 - sdks/python/src/opik/rest_api/__init__.py | 18 ++ sdks/python/src/opik/rest_api/client.py | 5 + .../src/opik/rest_api/datasets/client.py | 26 +- .../src/opik/rest_api/experiments/client.py | 30 +- .../opik/rest_api/feedback_scores/__init__.py | 1 + .../opik/rest_api/feedback_scores/client.py | 123 ++++++++ .../src/opik/rest_api/projects/__init__.py | 7 + .../src/opik/rest_api/projects/client.py | 277 ++++++++++++++++++ .../opik/rest_api/projects/types/__init__.py | 8 + .../project_metric_request_public_interval.py | 7 + ...oject_metric_request_public_metric_type.py | 7 + sdks/python/src/opik/rest_api/spans/client.py | 32 ++ .../src/opik/rest_api/types/__init__.py | 12 + .../opik/rest_api/types/data_point_public.py | 43 +++ .../rest_api/types/error_message_public.py | 4 +- .../types/project_metric_response_public.py | 50 ++++ ...project_metric_response_public_interval.py | 7 + ...ject_metric_response_public_metric_type.py | 7 + .../src/opik/rest_api/types/results_public.py | 44 +++ sdks/python/src/opik/rest_api/types/span.py | 3 + .../src/opik/rest_api/types/span_public.py | 3 + .../src/opik/rest_api/types/span_write.py | 2 + 33 files changed, 1028 insertions(+), 63 deletions(-) create mode 100644 sdks/python/code_generation/fern/README.md delete mode 100644 sdks/python/code_generation/fern/readme.md create mode 100644 sdks/python/src/opik/rest_api/feedback_scores/__init__.py create mode 100644 sdks/python/src/opik/rest_api/feedback_scores/client.py create mode 100644 sdks/python/src/opik/rest_api/projects/types/__init__.py create mode 100644 sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_interval.py create mode 100644 sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_metric_type.py create mode 100644 sdks/python/src/opik/rest_api/types/data_point_public.py create mode 100644 sdks/python/src/opik/rest_api/types/project_metric_response_public.py create mode 100644 sdks/python/src/opik/rest_api/types/project_metric_response_public_interval.py create mode 100644 sdks/python/src/opik/rest_api/types/project_metric_response_public_metric_type.py create mode 100644 sdks/python/src/opik/rest_api/types/results_public.py diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/DataPoint.java b/apps/opik-backend/src/main/java/com/comet/opik/api/DataPoint.java index ab1bfee407..a87988e694 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/DataPoint.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/DataPoint.java @@ -5,4 +5,5 @@ import java.time.Instant; @Builder(toBuilder = true) -public record DataPoint(Instant time, Number value) {} +public record DataPoint(Instant time, Number value) { +} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricRequest.java b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricRequest.java index 4c0ed5da17..b3a3ff4870 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricRequest.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricRequest.java @@ -16,4 +16,5 @@ public record ProjectMetricRequest( @NonNull MetricType metricType, @NonNull TimeInterval interval, Instant intervalStart, - Instant intervalEnd) {} + Instant intervalEnd) { +} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricResponse.java b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricResponse.java index 32ac97cb11..aeb4744ca3 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricResponse.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/ProjectMetricResponse.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; -import java.time.Instant; import java.util.List; import java.util.UUID; @@ -27,5 +26,6 @@ public record ProjectMetricResponse( @Builder(toBuilder = true) @JsonIgnoreProperties(ignoreUnknown = true) @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) - public record Results(String name, List data) {} + public record Results(String name, List data) { + } } diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/ProjectMetricsService.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/ProjectMetricsService.java index a613d407ff..164c2d3842 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/ProjectMetricsService.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/ProjectMetricsService.java @@ -1,6 +1,5 @@ package com.comet.opik.domain; -import com.comet.opik.api.DataPoint; import com.comet.opik.api.metrics.ProjectMetricRequest; import com.comet.opik.api.metrics.ProjectMetricResponse; import com.comet.opik.infrastructure.db.TransactionTemplateAsync; @@ -37,7 +36,7 @@ public Mono getProjectMetrics(UUID projectId, ProjectMetr validate(request); return template.nonTransaction(connection -> projectMetricsDAO.getTraceCount(projectId, request, - connection) + connection) .map(dataPoints -> ProjectMetricResponse.builder() .projectId(projectId) .metricType(request.metricType()) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/SpanDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/SpanDAO.java index c94012e224..d42b905291 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/SpanDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/SpanDAO.java @@ -633,7 +633,8 @@ private Publisher insert(List spans, Connection connecti .bind("model" + i, span.model() != null ? span.model() : "") .bind("provider" + i, span.provider() != null ? span.provider() : "") .bind("total_estimated_cost" + i, estimatedCost.toString()) - .bind("total_estimated_cost_version" + i, estimatedCost.compareTo(ZERO_COST) > 0 ? ESTIMATED_COST_VERSION : "") + .bind("total_estimated_cost_version" + i, + estimatedCost.compareTo(ZERO_COST) > 0 ? ESTIMATED_COST_VERSION : "") .bind("tags" + i, span.tags() != null ? span.tags().toArray(String[]::new) : new String[]{}) .bind("created_by" + i, userName) .bind("last_updated_by" + i, userName); diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/ModelPrice.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/ModelPrice.java index 012586474c..78aebb93f3 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/ModelPrice.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/ModelPrice.java @@ -12,36 +12,63 @@ @Getter public enum ModelPrice { gpt_4o("gpt-4o", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost), - gpt_4o_2024_08_06("gpt-4o-2024-08-06", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost), - gpt_4o_audio_preview("gpt-4o-audio-preview", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost), - gpt_4o_audio_preview_2024_10_01("gpt-4o-audio-preview-2024-10-01", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost), - gpt_4o_2024_05_13("gpt-4o-2024-05-13", new BigDecimal("0.000005"), new BigDecimal("0.000015"), SpanCostCalculator::textGenerationCost), - gpt_4o_mini("gpt-4o-mini", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), SpanCostCalculator::textGenerationCost), - gpt_4o_mini_2024_07_18("gpt-4o-mini-2024-07-18", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), SpanCostCalculator::textGenerationCost), - o1_preview("o1-preview", new BigDecimal("0.000015"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost), - o1_preview_2024_09_12("o1-preview-2024-09-12", new BigDecimal("0.000015"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost), + gpt_4o_2024_08_06("gpt-4o-2024-08-06", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), + SpanCostCalculator::textGenerationCost), + gpt_4o_audio_preview("gpt-4o-audio-preview", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), + SpanCostCalculator::textGenerationCost), + gpt_4o_audio_preview_2024_10_01("gpt-4o-audio-preview-2024-10-01", new BigDecimal("0.0000025"), + new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost), + gpt_4o_2024_05_13("gpt-4o-2024-05-13", new BigDecimal("0.000005"), new BigDecimal("0.000015"), + SpanCostCalculator::textGenerationCost), + gpt_4o_mini("gpt-4o-mini", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), + SpanCostCalculator::textGenerationCost), + gpt_4o_mini_2024_07_18("gpt-4o-mini-2024-07-18", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), + SpanCostCalculator::textGenerationCost), + o1_preview("o1-preview", new BigDecimal("0.000015"), new BigDecimal("0.000060"), + SpanCostCalculator::textGenerationCost), + o1_preview_2024_09_12("o1-preview-2024-09-12", new BigDecimal("0.000015"), new BigDecimal("0.000060"), + SpanCostCalculator::textGenerationCost), o1_mini("o1-mini", new BigDecimal("0.000003"), new BigDecimal("0.000012"), SpanCostCalculator::textGenerationCost), - o1_mini_2024_09_12("o1-mini-2024-09-12", new BigDecimal("0.000003"), new BigDecimal("0.000012"), SpanCostCalculator::textGenerationCost), - gpt_4o_realtime_preview("gpt-4o-realtime-preview", new BigDecimal("0.000005"), new BigDecimal("0.000020"), SpanCostCalculator::textGenerationCost), - gpt_4o_realtime_preview_2024_10_01("gpt-4o-realtime-preview-2024-10-01", new BigDecimal("0.000005"), new BigDecimal("0.000020"), + o1_mini_2024_09_12("o1-mini-2024-09-12", new BigDecimal("0.000003"), new BigDecimal("0.000012"), + SpanCostCalculator::textGenerationCost), + gpt_4o_realtime_preview("gpt-4o-realtime-preview", new BigDecimal("0.000005"), new BigDecimal("0.000020"), + SpanCostCalculator::textGenerationCost), + gpt_4o_realtime_preview_2024_10_01("gpt-4o-realtime-preview-2024-10-01", new BigDecimal("0.000005"), + new BigDecimal("0.000020"), + SpanCostCalculator::textGenerationCost), + chatgpt_4o_latest("chatgpt-4o-latest", new BigDecimal("0.000005"), new BigDecimal("0.000015"), + SpanCostCalculator::textGenerationCost), + gpt_4_turbo("gpt-4-turbo", new BigDecimal("0.000010"), new BigDecimal("0.000030"), + SpanCostCalculator::textGenerationCost), + gpt_4_turbo_2024_04_09("gpt-4-turbo-2024-04-09", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), - chatgpt_4o_latest("chatgpt-4o-latest", new BigDecimal("0.000005"), new BigDecimal("0.000015"), SpanCostCalculator::textGenerationCost), - gpt_4_turbo("gpt-4-turbo", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), - gpt_4_turbo_2024_04_09("gpt-4-turbo-2024-04-09", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), gpt_4("gpt-4", new BigDecimal("0.000030"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost), - gpt_4_32k("gpt-4-32k", new BigDecimal("0.000060"), new BigDecimal("0.000120"), SpanCostCalculator::textGenerationCost), - gpt_4_0125_preview("gpt-4-0125-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), - gpt_4_1106_preview("gpt-4-1106-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), - gpt_4_vision_preview("gpt-4-vision-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo("gpt-3.5-turbo", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_0125("gpt-3.5-turbo-0125", new BigDecimal("0.00000050"), new BigDecimal("0.0000015"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_instruct("gpt-3.5-turbo-instruct", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_1106("gpt-3.5-turbo-1106", new BigDecimal("0.000001"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_0613("gpt-3.5-turbo-0613", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_16k_0613("gpt-3.5-turbo-16k-0613", new BigDecimal("0.000003"), new BigDecimal("0.000004"), SpanCostCalculator::textGenerationCost), - gpt_3_5_turbo_0301("gpt-3.5-turbo-0301", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - davinci_002("davinci-002", new BigDecimal("0.000005"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost), - babbage_002("babbage-002", new BigDecimal("0.0000004"), new BigDecimal("0.0000004"), SpanCostCalculator::textGenerationCost), + gpt_4_32k("gpt-4-32k", new BigDecimal("0.000060"), new BigDecimal("0.000120"), + SpanCostCalculator::textGenerationCost), + gpt_4_0125_preview("gpt-4-0125-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), + SpanCostCalculator::textGenerationCost), + gpt_4_1106_preview("gpt-4-1106-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), + SpanCostCalculator::textGenerationCost), + gpt_4_vision_preview("gpt-4-vision-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo("gpt-3.5-turbo", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_0125("gpt-3.5-turbo-0125", new BigDecimal("0.00000050"), new BigDecimal("0.0000015"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_instruct("gpt-3.5-turbo-instruct", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_1106("gpt-3.5-turbo-1106", new BigDecimal("0.000001"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_0613("gpt-3.5-turbo-0613", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_16k_0613("gpt-3.5-turbo-16k-0613", new BigDecimal("0.000003"), new BigDecimal("0.000004"), + SpanCostCalculator::textGenerationCost), + gpt_3_5_turbo_0301("gpt-3.5-turbo-0301", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + davinci_002("davinci-002", new BigDecimal("0.000005"), new BigDecimal("0.000002"), + SpanCostCalculator::textGenerationCost), + babbage_002("babbage-002", new BigDecimal("0.0000004"), new BigDecimal("0.0000004"), + SpanCostCalculator::textGenerationCost), DEFAULT("", new BigDecimal("0"), new BigDecimal("0"), SpanCostCalculator::defaultCost); private final String name; diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/SpanCostCalculator.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/SpanCostCalculator.java index d1fd69c3c8..3ac04c5644 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/SpanCostCalculator.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/cost/SpanCostCalculator.java @@ -9,7 +9,8 @@ class SpanCostCalculator { public static BigDecimal textGenerationCost(ModelPrice modelPrice, Map usage) { return modelPrice.getInputPrice().multiply(BigDecimal.valueOf(usage.getOrDefault("prompt_tokens", 0))) - .add(modelPrice.getOutputPrice().multiply(BigDecimal.valueOf(usage.getOrDefault("completion_tokens", 0)))); + .add(modelPrice.getOutputPrice() + .multiply(BigDecimal.valueOf(usage.getOrDefault("completion_tokens", 0)))); } public static BigDecimal defaultCost(ModelPrice modelPrice, Map usage) { diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ProjectMetricsResourceTest.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ProjectMetricsResourceTest.java index a8d30bee30..152d3a5867 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ProjectMetricsResourceTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ProjectMetricsResourceTest.java @@ -2,10 +2,10 @@ import com.comet.opik.api.DataPoint; import com.comet.opik.api.TimeInterval; +import com.comet.opik.api.Trace; import com.comet.opik.api.metrics.MetricType; import com.comet.opik.api.metrics.ProjectMetricRequest; import com.comet.opik.api.metrics.ProjectMetricResponse; -import com.comet.opik.api.Trace; import com.comet.opik.api.resources.utils.AuthTestUtils; import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; import com.comet.opik.api.resources.utils.ClientSupportUtils; @@ -350,7 +350,8 @@ private void createTraces(String projectName, Instant marker, int count) { .mapToObj(i -> factory.manufacturePojo(Trace.class).toBuilder() .projectName(projectName) .startTime(marker.plus(i, ChronoUnit.SECONDS)) - .build()).toList(); + .build()) + .toList(); traceResourceClient.batchCreateTraces(traces, API_KEY, WORKSPACE_NAME); } diff --git a/sdks/python/code_generation/fern/README.md b/sdks/python/code_generation/fern/README.md new file mode 100644 index 0000000000..77fc4cca9a --- /dev/null +++ b/sdks/python/code_generation/fern/README.md @@ -0,0 +1,33 @@ +# How to generate new client code for communication with Opik backend + +## Pre-requirements +For either the automatic script or the manual steps: +- You'll need to install the fern SDK: https://github.com/fern-api/fern + +This is a one time process in the local machine generating the code. + +## Automatic process +> [!IMPORTANT] +> You should generally run this script instead of manual steps below. +> The only pre-requirement is installing the fern SDK. + +The following script automates this whole process: +```bash +./scripts/generate_openapi.sh +``` + +You simply need to run it from the repository base directory. + +## Manual process + +These are the manual steps (not recommended), as alternative to the automatic script. +In this case, it's also a pre-requirement to install the fern SDK. +Steps: +1. Execute the `./build_and_run.sh script` from the root of repository +2. Go to http://localhost:3003/ (URL for backend API specification) +3. Download openapi specification file - `openapi.yaml` +4. Put this file into `code_generation/fern/openapi/openapi.yaml` (overwrite it).
+ Note that this path contains the version of the schema from which the code in `src/opik/rest_api` for the SDK was generated. Therefore, it might not be the latest version of the schema. +5. Run `fern generate` from inside `code_generation/fern` folder. This will generate a python code inside the directory called `sdks` near the `fern` one. +7. Replace content of `src/opik/rest_api` with the python package inside `sdks` (there will be few nested directories, navigate until you find python files) +8. Run `pre-commit run --all-files` to format code diff --git a/sdks/python/code_generation/fern/openapi/openapi.yaml b/sdks/python/code_generation/fern/openapi/openapi.yaml index de0d26956f..9dad4a237a 100644 --- a/sdks/python/code_generation/fern/openapi/openapi.yaml +++ b/sdks/python/code_generation/fern/openapi/openapi.yaml @@ -43,6 +43,8 @@ tags: description: Experiment resources - name: Feedback-definitions description: Feedback definitions related resources +- name: Feedback-scores + description: Feedback scores related resources - name: Projects description: Project related resources - name: Prompts @@ -134,6 +136,11 @@ paths: in: query schema: type: boolean + - name: prompt_id + in: query + schema: + type: string + format: uuid - name: name in: query schema: @@ -306,6 +313,10 @@ paths: in: query schema: type: string + - name: truncate + in: query + schema: + type: boolean responses: "200": description: Dataset item resource @@ -381,6 +392,10 @@ paths: type: integer format: int32 default: 10 + - name: truncate + in: query + schema: + type: boolean responses: "200": description: Dataset items resource @@ -444,6 +459,15 @@ paths: in: query schema: type: string + - name: dataset_deleted + in: query + schema: + type: boolean + - name: prompt_id + in: query + schema: + type: string + format: uuid responses: "200": description: Experiments resource @@ -715,6 +739,32 @@ paths: responses: "204": description: No Content + /v1/private/feedback-scores/names: + get: + tags: + - Feedback-scores + summary: Find Feedback Score names + description: Find Feedback Score names + operationId: findFeedbackScoreNames + parameters: + - name: project_id + in: query + schema: + type: string + format: uuid + - name: with_experiments_only + in: query + schema: + type: boolean + responses: + "200": + description: Feedback Scores resource + content: + application/json: + schema: + type: array + items: + type: string /v1/private/projects: get: tags: @@ -861,6 +911,81 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorMessage' + /v1/private/projects/{id}/metrics: + post: + tags: + - Projects + summary: Get Project Metrics + description: Gets specified metrics for a project + operationId: getProjectMetrics + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectMetricRequest_Public' + responses: + "200": + description: Project Metrics + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectMetricResponse_Public' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + /v1/private/projects/retrieve: + post: + tags: + - Projects + summary: Retrieve project + description: Retrieve project + operationId: retrieveProject + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectRetrieve_Public' + responses: + "200": + description: Project resource + content: + application/json: + schema: + $ref: '#/components/schemas/Project_Public' + "422": + description: Unprocessable Content + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' /v1/private/prompts: get: tags: @@ -2539,13 +2664,10 @@ components: ErrorMessage_Public: type: object properties: - code: - type: integer - format: int32 - message: - type: string - details: - type: string + errors: + type: array + items: + type: string ExperimentItemStreamRequest: required: - experiment_name @@ -3068,6 +3190,73 @@ components: type: string format: date-time readOnly: true + DataPoint_Public: + type: object + properties: + time: + type: string + format: date-time + value: + type: number + ProjectMetricResponse_Public: + type: object + properties: + project_id: + type: string + format: uuid + metric_type: + type: string + enum: + - FEEDBACK_SCORES + - TRACE_COUNT + - TOKEN_USAGE + interval: + type: string + enum: + - HOURLY + - DAILY + - WEEKLY + results: + type: array + items: + $ref: '#/components/schemas/Results_Public' + Results_Public: + type: object + properties: + name: + type: string + data: + type: array + items: + $ref: '#/components/schemas/DataPoint_Public' + ProjectMetricRequest_Public: + type: object + properties: + metric_type: + type: string + enum: + - FEEDBACK_SCORES + - TRACE_COUNT + - TOKEN_USAGE + interval: + type: string + enum: + - HOURLY + - DAILY + - WEEKLY + interval_start: + type: string + format: date-time + interval_end: + type: string + format: date-time + ProjectRetrieve_Public: + required: + - name + type: object + properties: + name: + type: string ProjectUpdate: type: object properties: @@ -3405,6 +3594,10 @@ components: $ref: '#/components/schemas/JsonNode' metadata: $ref: '#/components/schemas/JsonNode' + model: + type: string + provider: + type: string tags: uniqueItems: true type: array @@ -3434,6 +3627,9 @@ components: readOnly: true items: $ref: '#/components/schemas/FeedbackScore' + total_estimated_cost: + type: number + readOnly: true Span_Write: required: - name @@ -3475,6 +3671,10 @@ components: $ref: '#/components/schemas/JsonNode_Write' metadata: $ref: '#/components/schemas/JsonNode_Write' + model: + type: string + provider: + type: string tags: uniqueItems: true type: array @@ -3594,6 +3794,10 @@ components: $ref: '#/components/schemas/JsonNode_Public' metadata: $ref: '#/components/schemas/JsonNode_Public' + model: + type: string + provider: + type: string tags: uniqueItems: true type: array @@ -3623,6 +3827,9 @@ components: readOnly: true items: $ref: '#/components/schemas/FeedbackScore_Public' + total_estimated_cost: + type: number + readOnly: true SpanPage_Public: type: object properties: @@ -3711,6 +3918,10 @@ components: $ref: '#/components/schemas/JsonNode' metadata: $ref: '#/components/schemas/JsonNode' + model: + type: string + provider: + type: string tags: uniqueItems: true type: array diff --git a/sdks/python/code_generation/fern/readme.md b/sdks/python/code_generation/fern/readme.md deleted file mode 100644 index 1ec9addc06..0000000000 --- a/sdks/python/code_generation/fern/readme.md +++ /dev/null @@ -1,12 +0,0 @@ -How to generate new client code for communication with Opik backend - -0. You'll need to install fern SDK https://github.com/fern-api/fern - -1. Execute the ./build_and_run.sh script from the root of repository -2. Go to http://localhost:3003/ (URL for backend API specification) -3. Download openapi specification file - `openapi.yaml` -4. Put this file into `code_generation/fern/openapi/openapi.yaml` (overwrite it).
- Note that this path contains the version of the schema from which the code in `src/opik/rest_api` for the SDK was generated. Therefore, it might not be the latest version of the schema. -5. Run `fern generate` from inside `code_generation/fern` folder. This will generate a python code inside the directory called `sdks` near the `fern` one. -7. Replace content of `src/opik/rest_api` with the python package inside `sdks` (there will be few nested directories, navigate until you find python files) -8. Run `pre-commit run --all-files` to format code diff --git a/sdks/python/src/opik/rest_api/__init__.py b/sdks/python/src/opik/rest_api/__init__.py index 2cf7341576..f80f0b5f6c 100644 --- a/sdks/python/src/opik/rest_api/__init__.py +++ b/sdks/python/src/opik/rest_api/__init__.py @@ -17,6 +17,7 @@ ColumnCompareTypesItem, ColumnPublic, ColumnPublicTypesItem, + DataPointPublic, Dataset, DatasetItem, DatasetItemBatch, @@ -81,6 +82,9 @@ NumericalFeedbackDetailPublic, NumericalFeedbackDetailUpdate, Project, + ProjectMetricResponsePublic, + ProjectMetricResponsePublicInterval, + ProjectMetricResponsePublicMetricType, ProjectPagePublic, ProjectPublic, Prompt, @@ -94,6 +98,7 @@ PromptVersionLinkWrite, PromptVersionPagePublic, PromptVersionPublic, + ResultsPublic, Span, SpanBatch, SpanPagePublic, @@ -121,6 +126,7 @@ datasets, experiments, feedback_definitions, + feedback_scores, projects, prompts, spans, @@ -129,6 +135,10 @@ ) from .environment import OpikApiEnvironment from .feedback_definitions import FindFeedbackDefinitionsRequestType +from .projects import ( + ProjectMetricRequestPublicInterval, + ProjectMetricRequestPublicMetricType, +) from .spans import GetSpansByProjectRequestType __all__ = [ @@ -150,6 +160,7 @@ "ColumnPublic", "ColumnPublicTypesItem", "ConflictError", + "DataPointPublic", "Dataset", "DatasetItem", "DatasetItemBatch", @@ -219,6 +230,11 @@ "NumericalFeedbackDetailUpdate", "OpikApiEnvironment", "Project", + "ProjectMetricRequestPublicInterval", + "ProjectMetricRequestPublicMetricType", + "ProjectMetricResponsePublic", + "ProjectMetricResponsePublicInterval", + "ProjectMetricResponsePublicMetricType", "ProjectPagePublic", "ProjectPublic", "Prompt", @@ -232,6 +248,7 @@ "PromptVersionLinkWrite", "PromptVersionPagePublic", "PromptVersionPublic", + "ResultsPublic", "Span", "SpanBatch", "SpanPagePublic", @@ -251,6 +268,7 @@ "datasets", "experiments", "feedback_definitions", + "feedback_scores", "projects", "prompts", "spans", diff --git a/sdks/python/src/opik/rest_api/client.py b/sdks/python/src/opik/rest_api/client.py index 8933b254da..9599bc37ef 100644 --- a/sdks/python/src/opik/rest_api/client.py +++ b/sdks/python/src/opik/rest_api/client.py @@ -16,6 +16,7 @@ AsyncFeedbackDefinitionsClient, FeedbackDefinitionsClient, ) +from .feedback_scores.client import AsyncFeedbackScoresClient, FeedbackScoresClient from .projects.client import AsyncProjectsClient, ProjectsClient from .prompts.client import AsyncPromptsClient, PromptsClient from .spans.client import AsyncSpansClient, SpansClient @@ -86,6 +87,7 @@ def __init__( self.feedback_definitions = FeedbackDefinitionsClient( client_wrapper=self._client_wrapper ) + self.feedback_scores = FeedbackScoresClient(client_wrapper=self._client_wrapper) self.projects = ProjectsClient(client_wrapper=self._client_wrapper) self.prompts = PromptsClient(client_wrapper=self._client_wrapper) self.spans = SpansClient(client_wrapper=self._client_wrapper) @@ -219,6 +221,9 @@ def __init__( self.feedback_definitions = AsyncFeedbackDefinitionsClient( client_wrapper=self._client_wrapper ) + self.feedback_scores = AsyncFeedbackScoresClient( + client_wrapper=self._client_wrapper + ) self.projects = AsyncProjectsClient(client_wrapper=self._client_wrapper) self.prompts = AsyncPromptsClient(client_wrapper=self._client_wrapper) self.spans = AsyncSpansClient(client_wrapper=self._client_wrapper) diff --git a/sdks/python/src/opik/rest_api/datasets/client.py b/sdks/python/src/opik/rest_api/datasets/client.py index c810e8cd65..ef8be8d954 100644 --- a/sdks/python/src/opik/rest_api/datasets/client.py +++ b/sdks/python/src/opik/rest_api/datasets/client.py @@ -29,6 +29,7 @@ def find_datasets( page: typing.Optional[int] = None, size: typing.Optional[int] = None, with_experiments_only: typing.Optional[bool] = None, + prompt_id: typing.Optional[str] = None, name: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetPagePublic: @@ -43,6 +44,8 @@ def find_datasets( with_experiments_only : typing.Optional[bool] + prompt_id : typing.Optional[str] + name : typing.Optional[str] request_options : typing.Optional[RequestOptions] @@ -67,6 +70,7 @@ def find_datasets( "page": page, "size": size, "with_experiments_only": with_experiments_only, + "prompt_id": prompt_id, "name": name, }, request_options=request_options, @@ -416,6 +420,7 @@ def find_dataset_items_with_experiment_items( page: typing.Optional[int] = None, size: typing.Optional[int] = None, filters: typing.Optional[str] = None, + truncate: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetItemPageCompare: """ @@ -433,6 +438,8 @@ def find_dataset_items_with_experiment_items( filters : typing.Optional[str] + truncate : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -459,6 +466,7 @@ def find_dataset_items_with_experiment_items( "size": size, "experiment_ids": experiment_ids, "filters": filters, + "truncate": truncate, }, request_options=request_options, ) @@ -563,6 +571,7 @@ def get_dataset_items( *, page: typing.Optional[int] = None, size: typing.Optional[int] = None, + truncate: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetItemPagePublic: """ @@ -576,6 +585,8 @@ def get_dataset_items( size : typing.Optional[int] + truncate : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -596,7 +607,7 @@ def get_dataset_items( _response = self._client_wrapper.httpx_client.request( f"v1/private/datasets/{jsonable_encoder(id)}/items", method="GET", - params={"page": page, "size": size}, + params={"page": page, "size": size, "truncate": truncate}, request_options=request_options, ) try: @@ -678,6 +689,7 @@ async def find_datasets( page: typing.Optional[int] = None, size: typing.Optional[int] = None, with_experiments_only: typing.Optional[bool] = None, + prompt_id: typing.Optional[str] = None, name: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetPagePublic: @@ -692,6 +704,8 @@ async def find_datasets( with_experiments_only : typing.Optional[bool] + prompt_id : typing.Optional[str] + name : typing.Optional[str] request_options : typing.Optional[RequestOptions] @@ -724,6 +738,7 @@ async def main() -> None: "page": page, "size": size, "with_experiments_only": with_experiments_only, + "prompt_id": prompt_id, "name": name, }, request_options=request_options, @@ -1129,6 +1144,7 @@ async def find_dataset_items_with_experiment_items( page: typing.Optional[int] = None, size: typing.Optional[int] = None, filters: typing.Optional[str] = None, + truncate: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetItemPageCompare: """ @@ -1146,6 +1162,8 @@ async def find_dataset_items_with_experiment_items( filters : typing.Optional[str] + truncate : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1180,6 +1198,7 @@ async def main() -> None: "size": size, "experiment_ids": experiment_ids, "filters": filters, + "truncate": truncate, }, request_options=request_options, ) @@ -1300,6 +1319,7 @@ async def get_dataset_items( *, page: typing.Optional[int] = None, size: typing.Optional[int] = None, + truncate: typing.Optional[bool] = None, request_options: typing.Optional[RequestOptions] = None, ) -> DatasetItemPagePublic: """ @@ -1313,6 +1333,8 @@ async def get_dataset_items( size : typing.Optional[int] + truncate : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1341,7 +1363,7 @@ async def main() -> None: _response = await self._client_wrapper.httpx_client.request( f"v1/private/datasets/{jsonable_encoder(id)}/items", method="GET", - params={"page": page, "size": size}, + params={"page": page, "size": size, "truncate": truncate}, request_options=request_options, ) try: diff --git a/sdks/python/src/opik/rest_api/experiments/client.py b/sdks/python/src/opik/rest_api/experiments/client.py index 8a493ea07f..b2b2c30b7e 100644 --- a/sdks/python/src/opik/rest_api/experiments/client.py +++ b/sdks/python/src/opik/rest_api/experiments/client.py @@ -31,6 +31,8 @@ def find_experiments( size: typing.Optional[int] = None, dataset_id: typing.Optional[str] = None, name: typing.Optional[str] = None, + dataset_deleted: typing.Optional[bool] = None, + prompt_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None, ) -> ExperimentPagePublic: """ @@ -46,6 +48,10 @@ def find_experiments( name : typing.Optional[str] + dataset_deleted : typing.Optional[bool] + + prompt_id : typing.Optional[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -64,7 +70,14 @@ def find_experiments( _response = self._client_wrapper.httpx_client.request( "v1/private/experiments", method="GET", - params={"page": page, "size": size, "datasetId": dataset_id, "name": name}, + params={ + "page": page, + "size": size, + "datasetId": dataset_id, + "name": name, + "dataset_deleted": dataset_deleted, + "prompt_id": prompt_id, + }, request_options=request_options, ) try: @@ -436,6 +449,8 @@ async def find_experiments( size: typing.Optional[int] = None, dataset_id: typing.Optional[str] = None, name: typing.Optional[str] = None, + dataset_deleted: typing.Optional[bool] = None, + prompt_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None, ) -> ExperimentPagePublic: """ @@ -451,6 +466,10 @@ async def find_experiments( name : typing.Optional[str] + dataset_deleted : typing.Optional[bool] + + prompt_id : typing.Optional[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -477,7 +496,14 @@ async def main() -> None: _response = await self._client_wrapper.httpx_client.request( "v1/private/experiments", method="GET", - params={"page": page, "size": size, "datasetId": dataset_id, "name": name}, + params={ + "page": page, + "size": size, + "datasetId": dataset_id, + "name": name, + "dataset_deleted": dataset_deleted, + "prompt_id": prompt_id, + }, request_options=request_options, ) try: diff --git a/sdks/python/src/opik/rest_api/feedback_scores/__init__.py b/sdks/python/src/opik/rest_api/feedback_scores/__init__.py new file mode 100644 index 0000000000..67a41b2742 --- /dev/null +++ b/sdks/python/src/opik/rest_api/feedback_scores/__init__.py @@ -0,0 +1 @@ +# This file was auto-generated by Fern from our API Definition. diff --git a/sdks/python/src/opik/rest_api/feedback_scores/client.py b/sdks/python/src/opik/rest_api/feedback_scores/client.py new file mode 100644 index 0000000000..fa40e3e493 --- /dev/null +++ b/sdks/python/src/opik/rest_api/feedback_scores/client.py @@ -0,0 +1,123 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.pydantic_utilities import pydantic_v1 +from ..core.request_options import RequestOptions + + +class FeedbackScoresClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def find_feedback_score_names( + self, + *, + project_id: typing.Optional[str] = None, + with_experiments_only: typing.Optional[bool] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> typing.List[str]: + """ + Find Feedback Score names + + Parameters + ---------- + project_id : typing.Optional[str] + + with_experiments_only : typing.Optional[bool] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + typing.List[str] + Feedback Scores resource + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.feedback_scores.find_feedback_score_names() + """ + _response = self._client_wrapper.httpx_client.request( + "v1/private/feedback-scores/names", + method="GET", + params={ + "project_id": project_id, + "with_experiments_only": with_experiments_only, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(typing.List[str], _response.json()) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + +class AsyncFeedbackScoresClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def find_feedback_score_names( + self, + *, + project_id: typing.Optional[str] = None, + with_experiments_only: typing.Optional[bool] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> typing.List[str]: + """ + Find Feedback Score names + + Parameters + ---------- + project_id : typing.Optional[str] + + with_experiments_only : typing.Optional[bool] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + typing.List[str] + Feedback Scores resource + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.feedback_scores.find_feedback_score_names() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "v1/private/feedback-scores/names", + method="GET", + params={ + "project_id": project_id, + "with_experiments_only": with_experiments_only, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(typing.List[str], _response.json()) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) diff --git a/sdks/python/src/opik/rest_api/projects/__init__.py b/sdks/python/src/opik/rest_api/projects/__init__.py index 67a41b2742..18c41b8b74 100644 --- a/sdks/python/src/opik/rest_api/projects/__init__.py +++ b/sdks/python/src/opik/rest_api/projects/__init__.py @@ -1 +1,8 @@ # This file was auto-generated by Fern from our API Definition. + +from .types import ( + ProjectMetricRequestPublicInterval, + ProjectMetricRequestPublicMetricType, +) + +__all__ = ["ProjectMetricRequestPublicInterval", "ProjectMetricRequestPublicMetricType"] diff --git a/sdks/python/src/opik/rest_api/projects/client.py b/sdks/python/src/opik/rest_api/projects/client.py index 9a2b964ec1..19649a7b2a 100644 --- a/sdks/python/src/opik/rest_api/projects/client.py +++ b/sdks/python/src/opik/rest_api/projects/client.py @@ -1,5 +1,6 @@ # This file was auto-generated by Fern from our API Definition. +import datetime as dt import typing from json.decoder import JSONDecodeError @@ -10,9 +11,17 @@ from ..core.request_options import RequestOptions from ..errors.bad_request_error import BadRequestError from ..errors.conflict_error import ConflictError +from ..errors.not_found_error import NotFoundError from ..errors.unprocessable_entity_error import UnprocessableEntityError +from ..types.project_metric_response_public import ProjectMetricResponsePublic from ..types.project_page_public import ProjectPagePublic from ..types.project_public import ProjectPublic +from .types.project_metric_request_public_interval import ( + ProjectMetricRequestPublicInterval, +) +from .types.project_metric_request_public_metric_type import ( + ProjectMetricRequestPublicMetricType, +) # this is used as the default value for optional parameters OMIT = typing.cast(typing.Any, ...) @@ -269,6 +278,132 @@ def update_project( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + def get_project_metrics( + self, + id: str, + *, + metric_type: typing.Optional[ProjectMetricRequestPublicMetricType] = OMIT, + interval: typing.Optional[ProjectMetricRequestPublicInterval] = OMIT, + interval_start: typing.Optional[dt.datetime] = OMIT, + interval_end: typing.Optional[dt.datetime] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> ProjectMetricResponsePublic: + """ + Gets specified metrics for a project + + Parameters + ---------- + id : str + + metric_type : typing.Optional[ProjectMetricRequestPublicMetricType] + + interval : typing.Optional[ProjectMetricRequestPublicInterval] + + interval_start : typing.Optional[dt.datetime] + + interval_end : typing.Optional[dt.datetime] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ProjectMetricResponsePublic + Project Metrics + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.projects.get_project_metrics( + id="id", + ) + """ + _response = self._client_wrapper.httpx_client.request( + f"v1/private/projects/{jsonable_encoder(id)}/metrics", + method="POST", + json={ + "metric_type": metric_type, + "interval": interval, + "interval_start": interval_start, + "interval_end": interval_end, + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as( + ProjectMetricResponsePublic, _response.json() + ) # type: ignore + if _response.status_code == 400: + raise BadRequestError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 404: + raise NotFoundError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + def retrieve_project( + self, *, name: str, request_options: typing.Optional[RequestOptions] = None + ) -> ProjectPublic: + """ + Retrieve project + + Parameters + ---------- + name : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ProjectPublic + Project resource + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.projects.retrieve_project( + name="name", + ) + """ + _response = self._client_wrapper.httpx_client.request( + "v1/private/projects/retrieve", + method="POST", + json={"name": name}, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(ProjectPublic, _response.json()) # type: ignore + if _response.status_code == 400: + raise BadRequestError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 404: + raise NotFoundError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 422: + raise UnprocessableEntityError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + class AsyncProjectsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -560,3 +695,145 @@ async def main() -> None: except JSONDecodeError: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + + async def get_project_metrics( + self, + id: str, + *, + metric_type: typing.Optional[ProjectMetricRequestPublicMetricType] = OMIT, + interval: typing.Optional[ProjectMetricRequestPublicInterval] = OMIT, + interval_start: typing.Optional[dt.datetime] = OMIT, + interval_end: typing.Optional[dt.datetime] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> ProjectMetricResponsePublic: + """ + Gets specified metrics for a project + + Parameters + ---------- + id : str + + metric_type : typing.Optional[ProjectMetricRequestPublicMetricType] + + interval : typing.Optional[ProjectMetricRequestPublicInterval] + + interval_start : typing.Optional[dt.datetime] + + interval_end : typing.Optional[dt.datetime] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ProjectMetricResponsePublic + Project Metrics + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.projects.get_project_metrics( + id="id", + ) + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + f"v1/private/projects/{jsonable_encoder(id)}/metrics", + method="POST", + json={ + "metric_type": metric_type, + "interval": interval, + "interval_start": interval_start, + "interval_end": interval_end, + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as( + ProjectMetricResponsePublic, _response.json() + ) # type: ignore + if _response.status_code == 400: + raise BadRequestError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 404: + raise NotFoundError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + async def retrieve_project( + self, *, name: str, request_options: typing.Optional[RequestOptions] = None + ) -> ProjectPublic: + """ + Retrieve project + + Parameters + ---------- + name : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + ProjectPublic + Project resource + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.projects.retrieve_project( + name="name", + ) + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "v1/private/projects/retrieve", + method="POST", + json={"name": name}, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(ProjectPublic, _response.json()) # type: ignore + if _response.status_code == 400: + raise BadRequestError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 404: + raise NotFoundError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 422: + raise UnprocessableEntityError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) diff --git a/sdks/python/src/opik/rest_api/projects/types/__init__.py b/sdks/python/src/opik/rest_api/projects/types/__init__.py new file mode 100644 index 0000000000..76388230c1 --- /dev/null +++ b/sdks/python/src/opik/rest_api/projects/types/__init__.py @@ -0,0 +1,8 @@ +# This file was auto-generated by Fern from our API Definition. + +from .project_metric_request_public_interval import ProjectMetricRequestPublicInterval +from .project_metric_request_public_metric_type import ( + ProjectMetricRequestPublicMetricType, +) + +__all__ = ["ProjectMetricRequestPublicInterval", "ProjectMetricRequestPublicMetricType"] diff --git a/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_interval.py b/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_interval.py new file mode 100644 index 0000000000..74ab69d254 --- /dev/null +++ b/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_interval.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +ProjectMetricRequestPublicInterval = typing.Union[ + typing.Literal["HOURLY", "DAILY", "WEEKLY"], typing.Any +] diff --git a/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_metric_type.py b/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_metric_type.py new file mode 100644 index 0000000000..532c00e5e0 --- /dev/null +++ b/sdks/python/src/opik/rest_api/projects/types/project_metric_request_public_metric_type.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +ProjectMetricRequestPublicMetricType = typing.Union[ + typing.Literal["FEEDBACK_SCORES", "TRACE_COUNT", "TOKEN_USAGE"], typing.Any +] diff --git a/sdks/python/src/opik/rest_api/spans/client.py b/sdks/python/src/opik/rest_api/spans/client.py index bdb2930e17..88b11ae216 100644 --- a/sdks/python/src/opik/rest_api/spans/client.py +++ b/sdks/python/src/opik/rest_api/spans/client.py @@ -199,6 +199,8 @@ def create_span( input: typing.Optional[JsonNodeWrite] = OMIT, output: typing.Optional[JsonNodeWrite] = OMIT, metadata: typing.Optional[JsonNodeWrite] = OMIT, + model: typing.Optional[str] = OMIT, + provider: typing.Optional[str] = OMIT, tags: typing.Optional[typing.Sequence[str]] = OMIT, usage: typing.Optional[typing.Dict[str, int]] = OMIT, request_options: typing.Optional[RequestOptions] = None, @@ -231,6 +233,10 @@ def create_span( metadata : typing.Optional[JsonNodeWrite] + model : typing.Optional[str] + + provider : typing.Optional[str] + tags : typing.Optional[typing.Sequence[str]] usage : typing.Optional[typing.Dict[str, int]] @@ -273,6 +279,8 @@ def create_span( "input": input, "output": output, "metadata": metadata, + "model": model, + "provider": provider, "tags": tags, "usage": usage, }, @@ -442,6 +450,8 @@ def update_span( input: typing.Optional[JsonNode] = OMIT, output: typing.Optional[JsonNode] = OMIT, metadata: typing.Optional[JsonNode] = OMIT, + model: typing.Optional[str] = OMIT, + provider: typing.Optional[str] = OMIT, tags: typing.Optional[typing.Sequence[str]] = OMIT, usage: typing.Optional[typing.Dict[str, int]] = OMIT, request_options: typing.Optional[RequestOptions] = None, @@ -471,6 +481,10 @@ def update_span( metadata : typing.Optional[JsonNode] + model : typing.Optional[str] + + provider : typing.Optional[str] + tags : typing.Optional[typing.Sequence[str]] usage : typing.Optional[typing.Dict[str, int]] @@ -504,6 +518,8 @@ def update_span( "input": input, "output": output, "metadata": metadata, + "model": model, + "provider": provider, "tags": tags, "usage": usage, }, @@ -813,6 +829,8 @@ async def create_span( input: typing.Optional[JsonNodeWrite] = OMIT, output: typing.Optional[JsonNodeWrite] = OMIT, metadata: typing.Optional[JsonNodeWrite] = OMIT, + model: typing.Optional[str] = OMIT, + provider: typing.Optional[str] = OMIT, tags: typing.Optional[typing.Sequence[str]] = OMIT, usage: typing.Optional[typing.Dict[str, int]] = OMIT, request_options: typing.Optional[RequestOptions] = None, @@ -845,6 +863,10 @@ async def create_span( metadata : typing.Optional[JsonNodeWrite] + model : typing.Optional[str] + + provider : typing.Optional[str] + tags : typing.Optional[typing.Sequence[str]] usage : typing.Optional[typing.Dict[str, int]] @@ -894,6 +916,8 @@ async def main() -> None: "input": input, "output": output, "metadata": metadata, + "model": model, + "provider": provider, "tags": tags, "usage": usage, }, @@ -1086,6 +1110,8 @@ async def update_span( input: typing.Optional[JsonNode] = OMIT, output: typing.Optional[JsonNode] = OMIT, metadata: typing.Optional[JsonNode] = OMIT, + model: typing.Optional[str] = OMIT, + provider: typing.Optional[str] = OMIT, tags: typing.Optional[typing.Sequence[str]] = OMIT, usage: typing.Optional[typing.Dict[str, int]] = OMIT, request_options: typing.Optional[RequestOptions] = None, @@ -1115,6 +1141,10 @@ async def update_span( metadata : typing.Optional[JsonNode] + model : typing.Optional[str] + + provider : typing.Optional[str] + tags : typing.Optional[typing.Sequence[str]] usage : typing.Optional[typing.Dict[str, int]] @@ -1156,6 +1186,8 @@ async def main() -> None: "input": input, "output": output, "metadata": metadata, + "model": model, + "provider": provider, "tags": tags, "usage": usage, }, diff --git a/sdks/python/src/opik/rest_api/types/__init__.py b/sdks/python/src/opik/rest_api/types/__init__.py index 04dc13de0b..6c4611dd59 100644 --- a/sdks/python/src/opik/rest_api/types/__init__.py +++ b/sdks/python/src/opik/rest_api/types/__init__.py @@ -16,6 +16,7 @@ from .column_compare_types_item import ColumnCompareTypesItem from .column_public import ColumnPublic from .column_public_types_item import ColumnPublicTypesItem +from .data_point_public import DataPointPublic from .dataset import Dataset from .dataset_item import DatasetItem from .dataset_item_batch import DatasetItemBatch @@ -86,6 +87,11 @@ from .numerical_feedback_detail_public import NumericalFeedbackDetailPublic from .numerical_feedback_detail_update import NumericalFeedbackDetailUpdate from .project import Project +from .project_metric_response_public import ProjectMetricResponsePublic +from .project_metric_response_public_interval import ProjectMetricResponsePublicInterval +from .project_metric_response_public_metric_type import ( + ProjectMetricResponsePublicMetricType, +) from .project_page_public import ProjectPagePublic from .project_public import ProjectPublic from .prompt import Prompt @@ -99,6 +105,7 @@ from .prompt_version_link_write import PromptVersionLinkWrite from .prompt_version_page_public import PromptVersionPagePublic from .prompt_version_public import PromptVersionPublic +from .results_public import ResultsPublic from .span import Span from .span_batch import SpanBatch from .span_page_public import SpanPagePublic @@ -132,6 +139,7 @@ "ColumnCompareTypesItem", "ColumnPublic", "ColumnPublicTypesItem", + "DataPointPublic", "Dataset", "DatasetItem", "DatasetItemBatch", @@ -196,6 +204,9 @@ "NumericalFeedbackDetailPublic", "NumericalFeedbackDetailUpdate", "Project", + "ProjectMetricResponsePublic", + "ProjectMetricResponsePublicInterval", + "ProjectMetricResponsePublicMetricType", "ProjectPagePublic", "ProjectPublic", "Prompt", @@ -209,6 +220,7 @@ "PromptVersionLinkWrite", "PromptVersionPagePublic", "PromptVersionPublic", + "ResultsPublic", "Span", "SpanBatch", "SpanPagePublic", diff --git a/sdks/python/src/opik/rest_api/types/data_point_public.py b/sdks/python/src/opik/rest_api/types/data_point_public.py new file mode 100644 index 0000000000..cf50defe09 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/data_point_public.py @@ -0,0 +1,43 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +from ..core.datetime_utils import serialize_datetime +from ..core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1 + + +class DataPointPublic(pydantic_v1.BaseModel): + time: typing.Optional[dt.datetime] = None + value: typing.Optional[float] = None + + def json(self, **kwargs: typing.Any) -> str: + kwargs_with_defaults: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + return super().json(**kwargs_with_defaults) + + def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: + kwargs_with_defaults_exclude_unset: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + kwargs_with_defaults_exclude_none: typing.Any = { + "by_alias": True, + "exclude_none": True, + **kwargs, + } + + return deep_union_pydantic_dicts( + super().dict(**kwargs_with_defaults_exclude_unset), + super().dict(**kwargs_with_defaults_exclude_none), + ) + + class Config: + frozen = True + smart_union = True + extra = pydantic_v1.Extra.allow + json_encoders = {dt.datetime: serialize_datetime} diff --git a/sdks/python/src/opik/rest_api/types/error_message_public.py b/sdks/python/src/opik/rest_api/types/error_message_public.py index df4a75a943..b723102c64 100644 --- a/sdks/python/src/opik/rest_api/types/error_message_public.py +++ b/sdks/python/src/opik/rest_api/types/error_message_public.py @@ -8,9 +8,7 @@ class ErrorMessagePublic(pydantic_v1.BaseModel): - code: typing.Optional[int] = None - message: typing.Optional[str] = None - details: typing.Optional[str] = None + errors: typing.Optional[typing.List[str]] = None def json(self, **kwargs: typing.Any) -> str: kwargs_with_defaults: typing.Any = { diff --git a/sdks/python/src/opik/rest_api/types/project_metric_response_public.py b/sdks/python/src/opik/rest_api/types/project_metric_response_public.py new file mode 100644 index 0000000000..a5c0676e67 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/project_metric_response_public.py @@ -0,0 +1,50 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +from ..core.datetime_utils import serialize_datetime +from ..core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1 +from .project_metric_response_public_interval import ProjectMetricResponsePublicInterval +from .project_metric_response_public_metric_type import ( + ProjectMetricResponsePublicMetricType, +) +from .results_public import ResultsPublic + + +class ProjectMetricResponsePublic(pydantic_v1.BaseModel): + project_id: typing.Optional[str] = None + metric_type: typing.Optional[ProjectMetricResponsePublicMetricType] = None + interval: typing.Optional[ProjectMetricResponsePublicInterval] = None + results: typing.Optional[typing.List[ResultsPublic]] = None + + def json(self, **kwargs: typing.Any) -> str: + kwargs_with_defaults: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + return super().json(**kwargs_with_defaults) + + def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: + kwargs_with_defaults_exclude_unset: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + kwargs_with_defaults_exclude_none: typing.Any = { + "by_alias": True, + "exclude_none": True, + **kwargs, + } + + return deep_union_pydantic_dicts( + super().dict(**kwargs_with_defaults_exclude_unset), + super().dict(**kwargs_with_defaults_exclude_none), + ) + + class Config: + frozen = True + smart_union = True + extra = pydantic_v1.Extra.allow + json_encoders = {dt.datetime: serialize_datetime} diff --git a/sdks/python/src/opik/rest_api/types/project_metric_response_public_interval.py b/sdks/python/src/opik/rest_api/types/project_metric_response_public_interval.py new file mode 100644 index 0000000000..679f99ae46 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/project_metric_response_public_interval.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +ProjectMetricResponsePublicInterval = typing.Union[ + typing.Literal["HOURLY", "DAILY", "WEEKLY"], typing.Any +] diff --git a/sdks/python/src/opik/rest_api/types/project_metric_response_public_metric_type.py b/sdks/python/src/opik/rest_api/types/project_metric_response_public_metric_type.py new file mode 100644 index 0000000000..2fcde79ca5 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/project_metric_response_public_metric_type.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +ProjectMetricResponsePublicMetricType = typing.Union[ + typing.Literal["FEEDBACK_SCORES", "TRACE_COUNT", "TOKEN_USAGE"], typing.Any +] diff --git a/sdks/python/src/opik/rest_api/types/results_public.py b/sdks/python/src/opik/rest_api/types/results_public.py new file mode 100644 index 0000000000..79c865abe1 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/results_public.py @@ -0,0 +1,44 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +from ..core.datetime_utils import serialize_datetime +from ..core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1 +from .data_point_public import DataPointPublic + + +class ResultsPublic(pydantic_v1.BaseModel): + name: typing.Optional[str] = None + data: typing.Optional[typing.List[DataPointPublic]] = None + + def json(self, **kwargs: typing.Any) -> str: + kwargs_with_defaults: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + return super().json(**kwargs_with_defaults) + + def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: + kwargs_with_defaults_exclude_unset: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + kwargs_with_defaults_exclude_none: typing.Any = { + "by_alias": True, + "exclude_none": True, + **kwargs, + } + + return deep_union_pydantic_dicts( + super().dict(**kwargs_with_defaults_exclude_unset), + super().dict(**kwargs_with_defaults_exclude_none), + ) + + class Config: + frozen = True + smart_union = True + extra = pydantic_v1.Extra.allow + json_encoders = {dt.datetime: serialize_datetime} diff --git a/sdks/python/src/opik/rest_api/types/span.py b/sdks/python/src/opik/rest_api/types/span.py index 008b51ea8b..09f3aa13ad 100644 --- a/sdks/python/src/opik/rest_api/types/span.py +++ b/sdks/python/src/opik/rest_api/types/span.py @@ -27,6 +27,8 @@ class Span(pydantic_v1.BaseModel): input: typing.Optional[JsonNode] = None output: typing.Optional[JsonNode] = None metadata: typing.Optional[JsonNode] = None + model: typing.Optional[str] = None + provider: typing.Optional[str] = None tags: typing.Optional[typing.List[str]] = None usage: typing.Optional[typing.Dict[str, int]] = None created_at: typing.Optional[dt.datetime] = None @@ -34,6 +36,7 @@ class Span(pydantic_v1.BaseModel): created_by: typing.Optional[str] = None last_updated_by: typing.Optional[str] = None feedback_scores: typing.Optional[typing.List[FeedbackScore]] = None + total_estimated_cost: typing.Optional[float] = None def json(self, **kwargs: typing.Any) -> str: kwargs_with_defaults: typing.Any = { diff --git a/sdks/python/src/opik/rest_api/types/span_public.py b/sdks/python/src/opik/rest_api/types/span_public.py index a05cfc24c8..098eb648a0 100644 --- a/sdks/python/src/opik/rest_api/types/span_public.py +++ b/sdks/python/src/opik/rest_api/types/span_public.py @@ -22,6 +22,8 @@ class SpanPublic(pydantic_v1.BaseModel): input: typing.Optional[JsonNodePublic] = None output: typing.Optional[JsonNodePublic] = None metadata: typing.Optional[JsonNodePublic] = None + model: typing.Optional[str] = None + provider: typing.Optional[str] = None tags: typing.Optional[typing.List[str]] = None usage: typing.Optional[typing.Dict[str, int]] = None created_at: typing.Optional[dt.datetime] = None @@ -29,6 +31,7 @@ class SpanPublic(pydantic_v1.BaseModel): created_by: typing.Optional[str] = None last_updated_by: typing.Optional[str] = None feedback_scores: typing.Optional[typing.List[FeedbackScorePublic]] = None + total_estimated_cost: typing.Optional[float] = None def json(self, **kwargs: typing.Any) -> str: kwargs_with_defaults: typing.Any = { diff --git a/sdks/python/src/opik/rest_api/types/span_write.py b/sdks/python/src/opik/rest_api/types/span_write.py index 52746d463f..819d431c89 100644 --- a/sdks/python/src/opik/rest_api/types/span_write.py +++ b/sdks/python/src/opik/rest_api/types/span_write.py @@ -25,6 +25,8 @@ class SpanWrite(pydantic_v1.BaseModel): input: typing.Optional[JsonNodeWrite] = None output: typing.Optional[JsonNodeWrite] = None metadata: typing.Optional[JsonNodeWrite] = None + model: typing.Optional[str] = None + provider: typing.Optional[str] = None tags: typing.Optional[typing.List[str]] = None usage: typing.Optional[typing.Dict[str, int]] = None