Skip to content

Commit

Permalink
Fix comments
Browse files Browse the repository at this point in the history
  • Loading branch information
Borys Tkachenko committed Nov 22, 2024
1 parent b33fa7b commit ecda221
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 58 deletions.
34 changes: 17 additions & 17 deletions apps/opik-backend/src/main/java/com/comet/opik/domain/SpanDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ INSERT INTO spans(
:metadata<item.index>,
:model<item.index>,
:provider<item.index>,
toDecimal32(:total_estimated_cost<item.index>, 7),
toDecimal32(:total_estimated_cost<item.index>, 8),
:total_estimated_cost_version<item.index>,
:tags<item.index>,
mapFromArrays(:usage_keys<item.index>, :usage_values<item.index>),
Expand Down Expand Up @@ -195,7 +195,7 @@ INSERT INTO spans(
new_span.provider
) as provider,
multiIf(
old_span.total_estimated_cost > toDecimal32(0, 7), old_span.total_estimated_cost,
old_span.total_estimated_cost > 0, old_span.total_estimated_cost,
new_span.total_estimated_cost
) as total_estimated_cost,
multiIf(
Expand Down Expand Up @@ -235,7 +235,7 @@ old_span.total_estimated_cost > toDecimal32(0, 7), old_span.total_estimated_cost
:metadata as metadata,
:model as model,
:provider as provider,
toDecimal32(:total_estimated_cost, 7) as total_estimated_cost,
toDecimal32(:total_estimated_cost, 8) as total_estimated_cost,
:total_estimated_cost_version as total_estimated_cost_version,
:tags as tags,
mapFromArrays(:usage_keys, :usage_values) as usage,
Expand Down Expand Up @@ -297,7 +297,7 @@ INSERT INTO spans (
<if(metadata)> :metadata <else> metadata <endif> as metadata,
<if(model)> :model <else> model <endif> as model,
<if(provider)> :provider <else> provider <endif> as provider,
<if(total_estimated_cost)> toDecimal32(:total_estimated_cost, 7) <else> total_estimated_cost <endif> as total_estimated_cost,
<if(total_estimated_cost)> toDecimal32(:total_estimated_cost, 8) <else> total_estimated_cost <endif> as total_estimated_cost,
<if(total_estimated_cost_version)> :total_estimated_cost_version <else> total_estimated_cost_version <endif> as total_estimated_cost_version,
<if(tags)> :tags <else> tags <endif> as tags,
<if(usage)> CAST((:usageKeys, :usageValues), 'Map(String, Int64)') <else> usage <endif> as usage,
Expand Down Expand Up @@ -393,8 +393,8 @@ INSERT INTO spans(
new_span.provider
) as provider,
multiIf(
new_span.total_estimated_cost > toDecimal32(0, 7), new_span.total_estimated_cost,
old_span.total_estimated_cost > toDecimal32(0, 7), old_span.total_estimated_cost,
new_span.total_estimated_cost > 0, new_span.total_estimated_cost,
old_span.total_estimated_cost > 0, old_span.total_estimated_cost,
new_span.total_estimated_cost
) as total_estimated_cost,
multiIf(
Expand Down Expand Up @@ -437,7 +437,7 @@ old_span.total_estimated_cost > toDecimal32(0, 7), old_span.total_estimated_cost
<if(metadata)> :metadata <else> '' <endif> as metadata,
<if(model)> :model <else> '' <endif> as model,
<if(provider)> :provider <else> '' <endif> as provider,
<if(total_estimated_cost)> toDecimal32(:total_estimated_cost, 7) <else> toDecimal32(0, 7) <endif> as total_estimated_cost,
<if(total_estimated_cost)> toDecimal32(:total_estimated_cost, 8) <else> toDecimal32(0, 8) <endif> as total_estimated_cost,
<if(total_estimated_cost_version)> :total_estimated_cost_version <else> '' <endif> as total_estimated_cost_version,
<if(tags)> :tags <else> [] <endif> as tags,
<if(usage)> CAST((:usageKeys, :usageValues), 'Map(String, Int64)') <else> mapFromArrays([], []) <endif> as usage,
Expand Down Expand Up @@ -581,7 +581,7 @@ AND id in (
""";

private static final String ESTIMATED_COST_VERSION = "1.0";
private static final BigDecimal COST_NOT_AVAILABLE = new BigDecimal("0.0000000");
private static final BigDecimal ZERO_COST = new BigDecimal("0.00000000");

private final @NonNull ConnectionFactory connectionFactory;
private final @NonNull FeedbackScoreDAO feedbackScoreDAO;
Expand Down Expand Up @@ -618,7 +618,7 @@ private Publisher<? extends Result> insert(List<Span> spans, Connection connecti
int i = 0;
for (Span span : spans) {

double estimatedCost = calculateCost(span);
BigDecimal estimatedCost = calculateCost(span);

statement.bind("id" + i, span.id())
.bind("project_id" + i, span.projectId())
Expand All @@ -632,8 +632,8 @@ private Publisher<? extends Result> insert(List<Span> spans, Connection connecti
.bind("metadata" + i, span.metadata() != null ? span.metadata().toString() : "")
.bind("model" + i, span.model() != null ? span.model() : "")
.bind("provider" + i, span.provider() != null ? span.provider() : "")
.bind("total_estimated_cost" + i, estimatedCost)
.bind("total_estimated_cost_version" + i, estimatedCost > 0 ? ESTIMATED_COST_VERSION : "")
.bind("total_estimated_cost" + i, estimatedCost.toString())
.bind("total_estimated_cost_version" + i, estimatedCost.compareTo(ZERO_COST) > 0 ? ESTIMATED_COST_VERSION : "")
.bind("tags" + i, span.tags() != null ? span.tags().toArray(String[]::new) : new String[]{})
.bind("created_by" + i, userName)
.bind("last_updated_by" + i, userName);
Expand Down Expand Up @@ -717,9 +717,9 @@ private Publisher<? extends Result> insert(Span span, Connection connection) {
statement.bind("provider", "");
}

double estimatedCost = calculateCost(span);
statement.bind("total_estimated_cost", estimatedCost);
if (estimatedCost > 0) {
BigDecimal estimatedCost = calculateCost(span);
statement.bind("total_estimated_cost", estimatedCost.toString());
if (estimatedCost.compareTo(ZERO_COST) > 0) {
statement.bind("total_estimated_cost_version", ESTIMATED_COST_VERSION);
} else {
statement.bind("total_estimated_cost_version", "");
Expand Down Expand Up @@ -841,7 +841,7 @@ private void bindUpdateParams(SpanUpdate spanUpdate, Statement statement) {

if (StringUtils.isNotBlank(spanUpdate.model()) && Objects.nonNull(spanUpdate.usage())) {
statement.bind("total_estimated_cost",
ModelPrice.fromString(spanUpdate.model()).calculateCost(spanUpdate.usage()));
ModelPrice.fromString(spanUpdate.model()).calculateCost(spanUpdate.usage()).toString());
statement.bind("total_estimated_cost_version", ESTIMATED_COST_VERSION);
}
}
Expand Down Expand Up @@ -938,7 +938,7 @@ private Publisher<Span> mapToDto(Result result) {
.orElse(null))
.model(row.get("model", String.class))
.provider(row.get("provider", String.class))
.totalEstimatedCost(row.get("total_estimated_cost", BigDecimal.class).equals(COST_NOT_AVAILABLE)
.totalEstimatedCost(row.get("total_estimated_cost", BigDecimal.class).equals(ZERO_COST)
? null
: row.get("total_estimated_cost", BigDecimal.class))
.tags(Optional.of(Arrays.stream(row.get("tags", String[].class)).collect(Collectors.toSet()))
Expand Down Expand Up @@ -980,7 +980,7 @@ private Mono<List<Span>> enhanceWithFeedbackScores(List<Span> spans) {
.doFinally(signalType -> endSegment(segment));
}

private double calculateCost(Span span) {
private BigDecimal calculateCost(Span span) {
// Later we could just use span.model(), but now it's still located inside metadata
String model = StringUtils.isNotBlank(span.model())
? span.model()
Expand Down
71 changes: 36 additions & 35 deletions apps/opik-backend/src/main/java/com/comet/opik/domain/cost/ModelPrice.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,50 +3,51 @@
import lombok.Getter;
import lombok.RequiredArgsConstructor;

import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Map;
import java.util.function.BiFunction;

@RequiredArgsConstructor
@Getter
public enum ModelPrice {
gpt_4o("gpt-4o", 2.5, 10, SpanCostCalculator::textGenerationCost),
gpt_4o_2024_08_06("gpt-4o-2024-08-06", 2.5, 10, SpanCostCalculator::textGenerationCost),
gpt_4o_audio_preview("gpt-4o-audio-preview", 2.5, 10, SpanCostCalculator::textGenerationCost),
gpt_4o_audio_preview_2024_10_01("gpt-4o-audio-preview-2024-10-01", 2.5, 10, SpanCostCalculator::textGenerationCost),
gpt_4o_2024_05_13("gpt-4o-2024-05-13", 5, 15, SpanCostCalculator::textGenerationCost),
gpt_4o_mini("gpt-4o-mini", 0.15, 0.6, SpanCostCalculator::textGenerationCost),
gpt_4o_mini_2024_07_18("gpt-4o-mini-2024-07-18", 0.15, 0.6, SpanCostCalculator::textGenerationCost),
o1_preview("o1-preview", 15, 60, SpanCostCalculator::textGenerationCost),
o1_preview_2024_09_12("o1-preview-2024-09-12", 15, 60, SpanCostCalculator::textGenerationCost),
o1_mini("o1-mini", 3, 12, SpanCostCalculator::textGenerationCost),
o1_mini_2024_09_12("o1-mini-2024-09-12", 3, 12, SpanCostCalculator::textGenerationCost),
gpt_4o_realtime_preview("gpt-4o-realtime-preview", 5, 20, SpanCostCalculator::textGenerationCost),
gpt_4o_realtime_preview_2024_10_01("gpt-4o-realtime-preview-2024-10-01", 5, 20,
gpt_4o("gpt-4o", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost),
gpt_4o_2024_08_06("gpt-4o-2024-08-06", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost),
gpt_4o_audio_preview("gpt-4o-audio-preview", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost),
gpt_4o_audio_preview_2024_10_01("gpt-4o-audio-preview-2024-10-01", new BigDecimal("0.0000025"), new BigDecimal("0.000010"), SpanCostCalculator::textGenerationCost),
gpt_4o_2024_05_13("gpt-4o-2024-05-13", new BigDecimal("0.000005"), new BigDecimal("0.000015"), SpanCostCalculator::textGenerationCost),
gpt_4o_mini("gpt-4o-mini", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), SpanCostCalculator::textGenerationCost),
gpt_4o_mini_2024_07_18("gpt-4o-mini-2024-07-18", new BigDecimal("0.00000015"), new BigDecimal("0.00000060"), SpanCostCalculator::textGenerationCost),
o1_preview("o1-preview", new BigDecimal("0.000015"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost),
o1_preview_2024_09_12("o1-preview-2024-09-12", new BigDecimal("0.000015"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost),
o1_mini("o1-mini", new BigDecimal("0.000003"), new BigDecimal("0.000012"), SpanCostCalculator::textGenerationCost),
o1_mini_2024_09_12("o1-mini-2024-09-12", new BigDecimal("0.000003"), new BigDecimal("0.000012"), SpanCostCalculator::textGenerationCost),
gpt_4o_realtime_preview("gpt-4o-realtime-preview", new BigDecimal("0.000005"), new BigDecimal("0.000020"), SpanCostCalculator::textGenerationCost),
gpt_4o_realtime_preview_2024_10_01("gpt-4o-realtime-preview-2024-10-01", new BigDecimal("0.000005"), new BigDecimal("0.000020"),
SpanCostCalculator::textGenerationCost),
chatgpt_4o_latest("chatgpt-4o-latest", 5, 15, SpanCostCalculator::textGenerationCost),
gpt_4_turbo("gpt-4-turbo", 10, 30, SpanCostCalculator::textGenerationCost),
gpt_4_turbo_2024_04_09("gpt-4-turbo-2024-04-09", 10, 30, SpanCostCalculator::textGenerationCost),
gpt_4("gpt-4", 30, 60, SpanCostCalculator::textGenerationCost),
gpt_4_32k("gpt-4-32k", 60, 120, SpanCostCalculator::textGenerationCost),
gpt_4_0125_preview("gpt-4-0125-preview", 10, 30, SpanCostCalculator::textGenerationCost),
gpt_4_1106_preview("gpt-4-1106-preview", 10, 30, SpanCostCalculator::textGenerationCost),
gpt_4_vision_preview("gpt-4-vision-preview", 10, 30, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo("gpt-3.5-turbo", 1.5, 2, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0125("gpt-3.5-turbo-0125", 0.5, 1.5, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_instruct("gpt-3.5-turbo-instruct", 1.5, 2, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_1106("gpt-3.5-turbo-1106", 1, 2, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0613("gpt-3.5-turbo-0613", 1.5, 2, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_16k_0613("gpt-3.5-turbo-16k-0613", 3, 4, SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0301("gpt-3.5-turbo-0301", 1.5, 2, SpanCostCalculator::textGenerationCost),
davinci_002("davinci-002", 2, 2, SpanCostCalculator::textGenerationCost),
babbage_002("babbage-002", .4, 0.4, SpanCostCalculator::textGenerationCost),
DEFAULT("", 0, 0, SpanCostCalculator::defaultCost);
chatgpt_4o_latest("chatgpt-4o-latest", new BigDecimal("0.000005"), new BigDecimal("0.000015"), SpanCostCalculator::textGenerationCost),
gpt_4_turbo("gpt-4-turbo", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost),
gpt_4_turbo_2024_04_09("gpt-4-turbo-2024-04-09", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost),
gpt_4("gpt-4", new BigDecimal("0.000030"), new BigDecimal("0.000060"), SpanCostCalculator::textGenerationCost),
gpt_4_32k("gpt-4-32k", new BigDecimal("0.000060"), new BigDecimal("0.000120"), SpanCostCalculator::textGenerationCost),
gpt_4_0125_preview("gpt-4-0125-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost),
gpt_4_1106_preview("gpt-4-1106-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost),
gpt_4_vision_preview("gpt-4-vision-preview", new BigDecimal("0.000010"), new BigDecimal("0.000030"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo("gpt-3.5-turbo", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0125("gpt-3.5-turbo-0125", new BigDecimal("0.00000050"), new BigDecimal("0.0000015"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_instruct("gpt-3.5-turbo-instruct", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_1106("gpt-3.5-turbo-1106", new BigDecimal("0.000001"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0613("gpt-3.5-turbo-0613", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_16k_0613("gpt-3.5-turbo-16k-0613", new BigDecimal("0.000003"), new BigDecimal("0.000004"), SpanCostCalculator::textGenerationCost),
gpt_3_5_turbo_0301("gpt-3.5-turbo-0301", new BigDecimal("0.0000015"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
davinci_002("davinci-002", new BigDecimal("0.000005"), new BigDecimal("0.000002"), SpanCostCalculator::textGenerationCost),
babbage_002("babbage-002", new BigDecimal("0.0000004"), new BigDecimal("0.0000004"), SpanCostCalculator::textGenerationCost),
DEFAULT("", new BigDecimal("0"), new BigDecimal("0"), SpanCostCalculator::defaultCost);

private final String name;
private final double inputPricePer1M;
private final double outputPricePer1M;
private final BiFunction<ModelPrice, Map<String, Integer>, Double> calculator;
private final BigDecimal inputPrice;
private final BigDecimal outputPrice;
private final BiFunction<ModelPrice, Map<String, Integer>, BigDecimal> calculator;

public static ModelPrice fromString(String modelName) {
return Arrays.stream(values())
Expand All @@ -55,7 +56,7 @@ public static ModelPrice fromString(String modelName) {
.orElse(DEFAULT);
}

public double calculateCost(Map<String, Integer> usage) {
public BigDecimal calculateCost(Map<String, Integer> usage) {
return calculator.apply(this, usage);
}
}
11 changes: 6 additions & 5 deletions apps/opik-backend/src/main/java/com/comet/opik/domain/cost/SpanCostCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@

import lombok.experimental.UtilityClass;

import java.math.BigDecimal;
import java.util.Map;

@UtilityClass
class SpanCostCalculator {
public static double textGenerationCost(ModelPrice modelPrice, Map<String, Integer> usage) {
return (modelPrice.getInputPricePer1M() * usage.getOrDefault("prompt_tokens", 0)
+ modelPrice.getOutputPricePer1M() * usage.getOrDefault("completion_tokens", 0)) / 1000000;
public static BigDecimal textGenerationCost(ModelPrice modelPrice, Map<String, Integer> usage) {
return modelPrice.getInputPrice().multiply(BigDecimal.valueOf(usage.getOrDefault("prompt_tokens", 0)))
.add(modelPrice.getOutputPrice().multiply(BigDecimal.valueOf(usage.getOrDefault("completion_tokens", 0))));
}

public static double defaultCost(ModelPrice modelPrice, Map<String, Integer> usage) {
return 0;
public static BigDecimal defaultCost(ModelPrice modelPrice, Map<String, Integer> usage) {
return new BigDecimal("0");
}
}
2 changes: 1 addition & 1 deletion ...nd/src/main/resources/liquibase/db-app-analytics/migrations/000007_add_model_to_spans.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
ALTER TABLE ${ANALYTICS_DB_DATABASE_NAME}.spans
ADD COLUMN IF NOT EXISTS model String DEFAULT '',
ADD COLUMN IF NOT EXISTS provider String DEFAULT '',
ADD COLUMN IF NOT EXISTS total_estimated_cost Decimal32(7) DEFAULT toDecimal32(0, 7),
ADD COLUMN IF NOT EXISTS total_estimated_cost Decimal32(8),
ADD COLUMN IF NOT EXISTS total_estimated_cost_version String DEFAULT '';

--rollback ALTER TABLE ${ANALYTICS_DB_DATABASE_NAME}.spans DROP COLUMN IF EXISTS model, DROP COLUMN IF EXISTS provider, DROP COLUMN IF EXISTS total_estimated_cost, DROP COLUMN IF EXISTS total_estimated_cost_version;

0 comments on commit ecda221

Please sign in to comment.