diff --git a/apps/opik-backend/config.yml b/apps/opik-backend/config.yml index 0bbd465603..0e19446fcd 100644 --- a/apps/opik-backend/config.yml +++ b/apps/opik-backend/config.yml @@ -91,3 +91,15 @@ cors: encryption: key: ${OPIK_ENCRYPTION_KEY:-'GiTHubiLoVeYouAA'} + +llmProviderClient: + maxAttempts: ${LLM_PROVIDER_CLIENT_MAX_ATTEMPTS:-3} + delayMillis: ${LLM_PROVIDER_CLIENT_DELAY_MILLIS:-500} + jitterScale: ${LLM_PROVIDER_CLIENT_JITTER_SCALE:-0.2} + backoffExp: ${LLM_PROVIDER_CLIENT_BACKOFF_EXP:-1.5} + callTimeout: ${LLM_PROVIDER_CLIENT_CALL_TIMEOUT:-60s} + connectTimeout: ${LLM_PROVIDER_CLIENT_CONNECT_TIMEOUT:-60s} + readTimeout: ${LLM_PROVIDER_CLIENT_READ_TIMEOUT:-60s} + writeTimeout: ${LLM_PROVIDER_CLIENT_WRITE_TIMEOUT:-60s} + openApiClient: + url: ${LLM_PROVIDER_CLIENT_WRITE_TIMEOUT:-} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java b/apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java index 9d2da4d1fd..c85c0dbdfd 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java @@ -17,10 +17,12 @@ import com.comet.opik.infrastructure.ratelimit.RateLimitModule; import com.comet.opik.infrastructure.redis.RedisModule; import com.comet.opik.utils.JsonBigDecimalDeserializer; +import com.comet.opik.utils.OpenAiMessageJsonDeserializer; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.module.SimpleModule; +import dev.ai4j.openai4j.chat.Message; import io.dropwizard.configuration.EnvironmentVariableSubstitutor; import io.dropwizard.configuration.SubstitutingSourceProvider; import io.dropwizard.core.Application; @@ -89,7 +91,9 @@ public void run(OpikConfiguration configuration, Environment environment) { environment.getObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategies.SnakeCaseStrategy.INSTANCE); environment.getObjectMapper().configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); environment.getObjectMapper() - .registerModule(new SimpleModule().addDeserializer(BigDecimal.class, new JsonBigDecimalDeserializer())); + .registerModule(new SimpleModule() + .addDeserializer(BigDecimal.class, JsonBigDecimalDeserializer.INSTANCE) + .addDeserializer(Message.class, OpenAiMessageJsonDeserializer.INSTANCE)); jersey.property(ServerProperties.RESPONSE_SET_STATUS_OVER_SEND_ERROR, true); diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/LlmProvider.java b/apps/opik-backend/src/main/java/com/comet/opik/api/LlmProvider.java index 234433e39b..12d7cd45ed 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/LlmProvider.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/LlmProvider.java @@ -1,13 +1,26 @@ package com.comet.opik.api; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.util.Arrays; + @Getter @RequiredArgsConstructor public enum LlmProvider { - @JsonProperty("openai") - OPEN_AI; + OPEN_AI("openai"); + + @JsonValue + private final String value; + + @JsonCreator + public static LlmProvider fromString(String value) { + return Arrays.stream(values()) + .filter(llmProvider -> llmProvider.value.equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown llm provider '%s'".formatted(value))); + } } diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKey.java b/apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKey.java index 0cac56cd6e..6514eca34e 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKey.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/ProviderApiKey.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.Builder; -import lombok.NonNull; import java.time.Instant; import java.util.List; @@ -21,7 +21,7 @@ public record ProviderApiKey( @JsonView( { View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) UUID id, - @JsonView({View.Public.class, View.Write.class}) @NonNull LlmProvider provider, + @JsonView({View.Public.class, View.Write.class}) @NotNull LlmProvider provider, @JsonView({ View.Write.class}) @NotBlank @JsonDeserialize(using = ProviderApiKeyDeserializer.class) String apiKey, @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant createdAt, diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResource.java b/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResource.java index fd2b66d11d..55aecc46dd 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResource.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResource.java @@ -1,12 +1,10 @@ package com.comet.opik.api.resources.v1.priv; import com.codahale.metrics.annotation.Timed; -import com.comet.opik.domain.TextStreamer; +import com.comet.opik.domain.ChatCompletionService; import com.comet.opik.infrastructure.auth.RequestContext; import dev.ai4j.openai4j.chat.ChatCompletionRequest; import dev.ai4j.openai4j.chat.ChatCompletionResponse; -import dev.ai4j.openai4j.shared.CompletionTokensDetails; -import dev.ai4j.openai4j.shared.Usage; import io.dropwizard.jersey.errors.ErrorMessage; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -28,10 +26,6 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; - -import java.security.SecureRandom; -import java.util.UUID; @Path("/v1/private/chat/completions") @Produces(MediaType.APPLICATION_JSON) @@ -43,52 +37,34 @@ public class ChatCompletionsResource { private final @NonNull Provider requestContextProvider; - private final @NonNull TextStreamer textStreamer; - private final @NonNull SecureRandom secureRandom; + private final @NonNull ChatCompletionService chatCompletionService; @POST @Produces({MediaType.SERVER_SENT_EVENTS, MediaType.APPLICATION_JSON}) - @Operation(operationId = "getChatCompletions", summary = "Get chat completions", description = "Get chat completions", responses = { - @ApiResponse(responseCode = "501", description = "Chat completions response", content = { + @Operation(operationId = "createChatCompletions", summary = "Create chat completions", description = "Create chat completions", responses = { + @ApiResponse(responseCode = "200", description = "Chat completions response", content = { @Content(mediaType = "text/event-stream", array = @ArraySchema(schema = @Schema(type = "object", anyOf = { ChatCompletionResponse.class, ErrorMessage.class}))), @Content(mediaType = "application/json", schema = @Schema(implementation = ChatCompletionResponse.class))}), }) - public Response get( + public Response create( @RequestBody(content = @Content(schema = @Schema(implementation = ChatCompletionRequest.class))) @NotNull @Valid ChatCompletionRequest request) { var workspaceId = requestContextProvider.get().getWorkspaceId(); String type; Object entity; if (Boolean.TRUE.equals(request.stream())) { - log.info("Streaming chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + log.info("Creating and streaming chat completions, workspaceId '{}', model '{}'", + workspaceId, request.model()); type = MediaType.SERVER_SENT_EVENTS; - var flux = Flux.range(0, 10).map(i -> newResponse(request.model())); - entity = textStreamer.getOutputStream(flux); + entity = chatCompletionService.createAndStreamResponse(request, workspaceId); } else { - log.info("Getting chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + log.info("Creating chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); type = MediaType.APPLICATION_JSON; - entity = newResponse(request.model()); + entity = chatCompletionService.create(request, workspaceId); } - var response = Response.status(Response.Status.NOT_IMPLEMENTED).type(type).entity(entity).build(); - log.info("Returned chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + var response = Response.ok().type(type).entity(entity).build(); + log.info("Created chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); return response; } - - private ChatCompletionResponse newResponse(String model) { - return ChatCompletionResponse.builder() - .id(UUID.randomUUID().toString()) - .created((int) (System.currentTimeMillis() / 1000)) - .model(model) - .usage(Usage.builder() - .totalTokens(secureRandom.nextInt()) - .promptTokens(secureRandom.nextInt()) - .completionTokens(secureRandom.nextInt()) - .completionTokensDetails(CompletionTokensDetails.builder() - .reasoningTokens(secureRandom.nextInt()) - .build()) - .build()) - .systemFingerprint(UUID.randomUUID().toString()) - .build(); - } } diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/ChatCompletionService.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/ChatCompletionService.java new file mode 100644 index 0000000000..dd6aa06aa8 --- /dev/null +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/ChatCompletionService.java @@ -0,0 +1,185 @@ +package com.comet.opik.domain; + +import com.comet.opik.api.LlmProvider; +import com.comet.opik.infrastructure.EncryptionUtils; +import com.comet.opik.infrastructure.LlmProviderClientConfig; +import com.comet.opik.utils.JsonUtils; +import dev.ai4j.openai4j.OpenAiClient; +import dev.ai4j.openai4j.OpenAiHttpException; +import dev.ai4j.openai4j.chat.ChatCompletionRequest; +import dev.ai4j.openai4j.chat.ChatCompletionResponse; +import dev.langchain4j.internal.RetryUtils; +import io.dropwizard.jersey.errors.ErrorMessage; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.ClientErrorException; +import jakarta.ws.rs.InternalServerErrorException; +import jakarta.ws.rs.ServerErrorException; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.glassfish.jersey.server.ChunkedOutput; +import ru.vyarus.dropwizard.guice.module.yaml.bind.Config; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +@Singleton +@Slf4j +public class ChatCompletionService { + + private static final String UNEXPECTED_ERROR_CALLING_LLM_PROVIDER = "Unexpected error calling LLM provider"; + + private final LlmProviderClientConfig llmProviderClientConfig; + private final LlmProviderApiKeyService llmProviderApiKeyService; + private final RetryUtils.RetryPolicy retryPolicy; + + @Inject + public ChatCompletionService( + @NonNull @Config LlmProviderClientConfig llmProviderClientConfig, + @NonNull LlmProviderApiKeyService llmProviderApiKeyService) { + this.llmProviderApiKeyService = llmProviderApiKeyService; + this.llmProviderClientConfig = llmProviderClientConfig; + this.retryPolicy = newRetryPolicy(); + } + + private RetryUtils.RetryPolicy newRetryPolicy() { + var retryPolicyBuilder = RetryUtils.retryPolicyBuilder(); + Optional.ofNullable(llmProviderClientConfig.getMaxAttempts()).ifPresent(retryPolicyBuilder::maxAttempts); + Optional.ofNullable(llmProviderClientConfig.getJitterScale()).ifPresent(retryPolicyBuilder::jitterScale); + Optional.ofNullable(llmProviderClientConfig.getBackoffExp()).ifPresent(retryPolicyBuilder::backoffExp); + return retryPolicyBuilder + .delayMillis(llmProviderClientConfig.getDelayMillis()) + .build(); + } + + public ChatCompletionResponse create(@NonNull ChatCompletionRequest request, @NonNull String workspaceId) { + log.info("Creating chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + var openAiClient = getAndConfigureOpenAiClient(request, workspaceId); + ChatCompletionResponse chatCompletionResponse; + try { + chatCompletionResponse = retryPolicy.withRetry(() -> openAiClient.chatCompletion(request).execute()); + } catch (RuntimeException runtimeException) { + log.error(UNEXPECTED_ERROR_CALLING_LLM_PROVIDER, runtimeException); + if (runtimeException.getCause() instanceof OpenAiHttpException openAiHttpException) { + if (openAiHttpException.code() >= 400 && openAiHttpException.code() <= 499) { + throw new ClientErrorException(openAiHttpException.getMessage(), openAiHttpException.code()); + } + throw new ServerErrorException(openAiHttpException.getMessage(), openAiHttpException.code()); + } + throw new InternalServerErrorException(UNEXPECTED_ERROR_CALLING_LLM_PROVIDER); + } + log.info("Created chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + return chatCompletionResponse; + } + + public ChunkedOutput createAndStreamResponse( + @NonNull ChatCompletionRequest request, @NonNull String workspaceId) { + log.info("Creating and streaming chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + var openAiClient = getAndConfigureOpenAiClient(request, workspaceId); + var chunkedOutput = new ChunkedOutput(String.class, "\r\n"); + openAiClient.chatCompletion(request) + .onPartialResponse(chatCompletionResponse -> send(chatCompletionResponse, chunkedOutput)) + .onComplete(() -> close(chunkedOutput)) + .onError(throwable -> handle(throwable, chunkedOutput)) + .execute(); + log.info("Created and streaming chat completions, workspaceId '{}', model '{}'", workspaceId, request.model()); + return chunkedOutput; + } + + private OpenAiClient getAndConfigureOpenAiClient(ChatCompletionRequest request, String workspaceId) { + var llmProvider = getLlmProvider(request.model()); + var encryptedApiKey = getEncryptedApiKey(workspaceId, llmProvider); + return newOpenAiClient(encryptedApiKey); + } + + /** + * The agreed requirement is to resolve the LLM provider and its API key based on the model. + * Currently, only OPEN AI is supported, so model param is ignored. + * No further validation is needed on the model, as it's just forwarded in the OPEN AI request and will be rejected + * if not valid. + */ + private LlmProvider getLlmProvider(String model) { + return LlmProvider.OPEN_AI; + } + + /** + * Finding API keys isn't paginated at the moment, since only OPEN AI is supported. + * Even in the future, the number of supported LLM providers per workspace is going to be very low. + */ + private String getEncryptedApiKey(String workspaceId, LlmProvider llmProvider) { + return llmProviderApiKeyService.find(workspaceId).content().stream() + .filter(providerApiKey -> llmProvider.equals(providerApiKey.provider())) + .findFirst() + .orElseThrow(() -> new BadRequestException("API key not configured for LLM provider '%s'".formatted( + llmProvider.getValue()))) + .apiKey(); + } + + /** + * Initially, only OPEN AI is supported, so no need for a more sophisticated client resolution to start with. + * At the moment, openai4j client and also langchain4j wrappers, don't support dynamic API keys. That can imply + * an important performance penalty for next phases. The following options should be evaluated: + * - Cache clients, but can be unsafe. + * - Find and evaluate other clients. + * - Implement our own client. + * TODO as part of : OPIK-522 + */ + private OpenAiClient newOpenAiClient(String encryptedApiKey) { + var openAiClientBuilder = OpenAiClient.builder(); + Optional.ofNullable(llmProviderClientConfig.getOpenApiClient()) + .map(LlmProviderClientConfig.OpenApiClientConfig::url) + .ifPresent(baseUrl -> { + if (StringUtils.isNotBlank(baseUrl)) { + openAiClientBuilder.baseUrl(baseUrl); + } + }); + Optional.ofNullable(llmProviderClientConfig.getCallTimeout()) + .ifPresent(callTimeout -> openAiClientBuilder.callTimeout(callTimeout.toJavaDuration())); + Optional.ofNullable(llmProviderClientConfig.getConnectTimeout()) + .ifPresent(connectTimeout -> openAiClientBuilder.connectTimeout(connectTimeout.toJavaDuration())); + Optional.ofNullable(llmProviderClientConfig.getReadTimeout()) + .ifPresent(readTimeout -> openAiClientBuilder.readTimeout(readTimeout.toJavaDuration())); + Optional.ofNullable(llmProviderClientConfig.getWriteTimeout()) + .ifPresent(writeTimeout -> openAiClientBuilder.writeTimeout(writeTimeout.toJavaDuration())); + return openAiClientBuilder + .openAiApiKey(EncryptionUtils.decrypt(encryptedApiKey)) + .build(); + } + + private void send(Object item, ChunkedOutput chunkedOutput) { + if (chunkedOutput.isClosed()) { + log.warn("Output stream is already closed"); + return; + } + try { + chunkedOutput.write(JsonUtils.writeValueAsString(item)); + } catch (IOException ioException) { + throw new UncheckedIOException(ioException); + } + } + + private void handle(Throwable throwable, ChunkedOutput chunkedOutput) { + log.error(UNEXPECTED_ERROR_CALLING_LLM_PROVIDER, throwable); + var errorMessage = new ErrorMessage(UNEXPECTED_ERROR_CALLING_LLM_PROVIDER); + if (throwable instanceof OpenAiHttpException openAiHttpException) { + errorMessage = new ErrorMessage(openAiHttpException.code(), openAiHttpException.getMessage()); + } + try { + send(errorMessage, chunkedOutput); + } catch (UncheckedIOException uncheckedIOException) { + log.error("Failed to stream error message to client", uncheckedIOException); + } + close(chunkedOutput); + } + + private void close(ChunkedOutput chunkedOutput) { + try { + chunkedOutput.close(); + } catch (IOException ioException) { + log.error("Failed to close output stream", ioException); + } + } +} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/TextStreamer.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/TextStreamer.java deleted file mode 100644 index a2151ef8dd..0000000000 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/TextStreamer.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.comet.opik.domain; - -import com.comet.opik.utils.JsonUtils; -import io.dropwizard.jersey.errors.ErrorMessage; -import jakarta.inject.Singleton; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.glassfish.jersey.server.ChunkedOutput; -import reactor.core.publisher.Flux; -import reactor.core.scheduler.Schedulers; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.concurrent.TimeoutException; - -@Singleton -@Slf4j -public class TextStreamer { - - public ChunkedOutput getOutputStream(@NonNull Flux flux) { - var outputStream = new ChunkedOutput(String.class, "\n"); - Schedulers.boundedElastic() - .schedule(() -> flux.doOnNext(item -> send(item, outputStream)) - .onErrorResume(throwable -> handleError(throwable, outputStream)) - .doFinally(signalType -> close(outputStream)) - .subscribe()); - return outputStream; - } - - private void send(Object item, ChunkedOutput outputStream) { - try { - outputStream.write(JsonUtils.writeValueAsString(item)); - } catch (IOException exception) { - throw new UncheckedIOException(exception); - } - } - - private Flux handleError(Throwable throwable, ChunkedOutput outputStream) { - if (throwable instanceof TimeoutException) { - try { - send(new ErrorMessage(500, "Streaming operation timed out"), outputStream); - } catch (UncheckedIOException uncheckedIOException) { - log.error("Failed to stream error message to client", uncheckedIOException); - } - } - return Flux.error(throwable); - } - - private void close(ChunkedOutput outputStream) { - try { - outputStream.close(); - } catch (IOException ioException) { - log.error("Error while closing output stream", ioException); - } - } -} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/LlmProviderClientConfig.java b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/LlmProviderClientConfig.java new file mode 100644 index 0000000000..bb32740560 --- /dev/null +++ b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/LlmProviderClientConfig.java @@ -0,0 +1,42 @@ +package com.comet.opik.infrastructure; + +import io.dropwizard.util.Duration; +import io.dropwizard.validation.MinDuration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Positive; +import lombok.Data; + +import java.util.concurrent.TimeUnit; + +@Data +public class LlmProviderClientConfig { + + public record OpenApiClientConfig(String url) { + } + + @Min(1) + private Integer maxAttempts; + + @Min(1) + private int delayMillis = 500; + + @Positive private Double jitterScale; + + @Positive private Double backoffExp; + + @MinDuration(value = 1, unit = TimeUnit.MILLISECONDS) + private Duration callTimeout; + + @MinDuration(value = 1, unit = TimeUnit.MILLISECONDS) + private Duration connectTimeout; + + @MinDuration(value = 1, unit = TimeUnit.MILLISECONDS) + private Duration readTimeout; + + @MinDuration(value = 1, unit = TimeUnit.MILLISECONDS) + private Duration writeTimeout; + + @Valid + private OpenApiClientConfig openApiClient; +} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/OpikConfiguration.java b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/OpikConfiguration.java index b15b3fddb2..9ff1f39091 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/OpikConfiguration.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/OpikConfiguration.java @@ -59,4 +59,8 @@ public class OpikConfiguration extends JobConfiguration { @NotNull @JsonProperty @ToString.Exclude private EncryptionConfig encryption = new EncryptionConfig(); + + @Valid + @NotNull @JsonProperty + private LlmProviderClientConfig llmProviderClient = new LlmProviderClientConfig(); } diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonBigDecimalDeserializer.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonBigDecimalDeserializer.java index bdfd6cb719..a4cf82e7c2 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonBigDecimalDeserializer.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonBigDecimalDeserializer.java @@ -13,6 +13,8 @@ public class JsonBigDecimalDeserializer extends NumberDeserializers.BigDecimalDeserializer { + public static final JsonBigDecimalDeserializer INSTANCE = new JsonBigDecimalDeserializer(); + @Override public BigDecimal deserialize(JsonParser p, DeserializationContext context) throws IOException { return Optional.ofNullable(super.deserialize(p, context)) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonUtils.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonUtils.java index 9959d88811..61e04f22ae 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonUtils.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/JsonUtils.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import dev.ai4j.openai4j.chat.Message; import lombok.NonNull; import lombok.experimental.UtilityClass; @@ -26,7 +27,9 @@ public class JsonUtils { .setSerializationInclusion(JsonInclude.Include.NON_NULL) .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) .configure(SerializationFeature.INDENT_OUTPUT, false) - .registerModule(new JavaTimeModule().addDeserializer(BigDecimal.class, new JsonBigDecimalDeserializer())); + .registerModule(new JavaTimeModule() + .addDeserializer(BigDecimal.class, JsonBigDecimalDeserializer.INSTANCE) + .addDeserializer(Message.class, OpenAiMessageJsonDeserializer.INSTANCE)); public static JsonNode getJsonNodeFromString(@NonNull String value) { try { diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/OpenAiMessageJsonDeserializer.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/OpenAiMessageJsonDeserializer.java new file mode 100644 index 0000000000..5606e1ad15 --- /dev/null +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/OpenAiMessageJsonDeserializer.java @@ -0,0 +1,39 @@ +package com.comet.opik.utils; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import dev.ai4j.openai4j.chat.AssistantMessage; +import dev.ai4j.openai4j.chat.FunctionMessage; +import dev.ai4j.openai4j.chat.Message; +import dev.ai4j.openai4j.chat.Role; +import dev.ai4j.openai4j.chat.SystemMessage; +import dev.ai4j.openai4j.chat.ToolMessage; +import dev.ai4j.openai4j.chat.UserMessage; + +import java.io.IOException; + +/** + * The Message interface of openai4j has not appropriate deserialization support for all its polymorphic implementors + * such as UserMessage, AssistantMessage etc. so deserialization fails. + * As we can't annotate them Message interface with JsonTypeInfo and JsonSubTypes, solving this issue by creating + * a custom deserializer. + */ +public class OpenAiMessageJsonDeserializer extends JsonDeserializer { + + public static final OpenAiMessageJsonDeserializer INSTANCE = new OpenAiMessageJsonDeserializer(); + + @Override + public Message deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException { + JsonNode jsonNode = jsonParser.readValueAsTree(); + var role = context.readTreeAsValue(jsonNode.get("role"), Role.class); + return switch (role) { + case SYSTEM -> context.readTreeAsValue(jsonNode, SystemMessage.class); + case USER -> context.readTreeAsValue(jsonNode, UserMessage.class); + case ASSISTANT -> context.readTreeAsValue(jsonNode, AssistantMessage.class); + case TOOL -> context.readTreeAsValue(jsonNode, ToolMessage.class); + case FUNCTION -> context.readTreeAsValue(jsonNode, FunctionMessage.class); + }; + } +} diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/ChatCompletionsClient.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/ChatCompletionsClient.java index 3ebf24667c..c1f435ea64 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/ChatCompletionsClient.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/ChatCompletionsClient.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import dev.ai4j.openai4j.chat.ChatCompletionRequest; import dev.ai4j.openai4j.chat.ChatCompletionResponse; +import io.dropwizard.jersey.errors.ErrorMessage; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.GenericType; import jakarta.ws.rs.core.HttpHeaders; @@ -29,6 +30,9 @@ public class ChatCompletionsClient { private static final TypeReference CHAT_COMPLETION_RESPONSE_TYPE_REFERENCE = new TypeReference<>() { }; + private static final TypeReference ERROR_MESSAGE_TYPE_REFERENCE = new TypeReference<>() { + }; + private final ClientSupport clientSupport; private final String baseURI; @@ -37,47 +41,96 @@ public ChatCompletionsClient(ClientSupport clientSupport) { this.baseURI = "http://localhost:%d".formatted(clientSupport.getPort()); } - public ChatCompletionResponse get(String apiKey, String workspaceName, ChatCompletionRequest request) { + public ChatCompletionResponse create(String apiKey, String workspaceName, ChatCompletionRequest request) { assertThat(request.stream()).isFalse(); - try (var response = clientSupport.target(RESOURCE_PATH.formatted(baseURI)) + try (var response = clientSupport.target(getCreateUrl()) .request() .accept(MediaType.APPLICATION_JSON_TYPE) .header(HttpHeaders.AUTHORIZATION, apiKey) .header(RequestContext.WORKSPACE_HEADER, workspaceName) .post(Entity.json(request))) { - assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_NOT_IMPLEMENTED); + assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_OK); return response.readEntity(ChatCompletionResponse.class); } } - public List getStream(String apiKey, String workspaceName, ChatCompletionRequest request) { + public ErrorMessage create(String apiKey, String workspaceName, ChatCompletionRequest request, + int expectedStatusCode) { + assertThat(request.stream()).isFalse(); + + try (var response = clientSupport.target(getCreateUrl()) + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .header(HttpHeaders.AUTHORIZATION, apiKey) + .header(RequestContext.WORKSPACE_HEADER, workspaceName) + .post(Entity.json(request))) { + + assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(expectedStatusCode); + + return response.readEntity(ErrorMessage.class); + } + } + + public List createAndStream( + String apiKey, String workspaceName, ChatCompletionRequest request) { assertThat(request.stream()).isTrue(); - try (var response = clientSupport.target(RESOURCE_PATH.formatted(baseURI)) + try (var response = clientSupport.target(getCreateUrl()) .request() .accept(MediaType.SERVER_SENT_EVENTS) .header(HttpHeaders.AUTHORIZATION, apiKey) .header(RequestContext.WORKSPACE_HEADER, workspaceName) .post(Entity.json(request))) { - assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_NOT_IMPLEMENTED); + assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_OK); + + return getStreamedEntities(response); + } + } - return getStreamedItems(response); + public List createAndStreamError( + String apiKey, String workspaceName, ChatCompletionRequest request) { + assertThat(request.stream()).isTrue(); + + try (var response = clientSupport.target(getCreateUrl()) + .request() + .accept(MediaType.SERVER_SENT_EVENTS) + .header(HttpHeaders.AUTHORIZATION, apiKey) + .header(RequestContext.WORKSPACE_HEADER, workspaceName) + .post(Entity.json(request))) { + + assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(HttpStatus.SC_OK); + + return getStreamedError(response); + } + } + + private String getCreateUrl() { + return RESOURCE_PATH.formatted(baseURI); + } + + private List getStreamedEntities(Response response) { + var entities = new ArrayList(); + try (var inputStream = response.readEntity(CHUNKED_INPUT_STRING_GENERIC_TYPE)) { + String chunk; + while ((chunk = inputStream.read()) != null) { + entities.add(JsonUtils.readValue(chunk, CHAT_COMPLETION_RESPONSE_TYPE_REFERENCE)); + } } + return entities; } - private List getStreamedItems(Response response) { - var items = new ArrayList(); + private List getStreamedError(Response response) { + var errorMessages = new ArrayList(); try (var inputStream = response.readEntity(CHUNKED_INPUT_STRING_GENERIC_TYPE)) { - inputStream.setParser(ChunkedInput.createParser("\n")); - String stringItem; - while ((stringItem = inputStream.read()) != null) { - items.add(JsonUtils.readValue(stringItem, CHAT_COMPLETION_RESPONSE_TYPE_REFERENCE)); + String chunk; + while ((chunk = inputStream.read()) != null) { + errorMessages.add(JsonUtils.readValue(chunk, ERROR_MESSAGE_TYPE_REFERENCE)); } } - return items; + return errorMessages; } } diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/LlmProviderApiKeyResourceClient.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/LlmProviderApiKeyResourceClient.java index ad3514057d..58653644df 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/LlmProviderApiKeyResourceClient.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/LlmProviderApiKeyResourceClient.java @@ -9,7 +9,6 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; -import lombok.RequiredArgsConstructor; import ru.vyarus.dropwizard.guice.test.ClientSupport; import uk.co.jemos.podam.api.PodamUtils; @@ -18,16 +17,25 @@ import static com.comet.opik.infrastructure.auth.RequestContext.WORKSPACE_HEADER; import static org.assertj.core.api.Assertions.assertThat; -@RequiredArgsConstructor public class LlmProviderApiKeyResourceClient { private static final String RESOURCE_PATH = "%s/v1/private/llm-provider-key"; private final ClientSupport client; private final String baseURI; - public ProviderApiKey createProviderApiKey(String providerApiKey, String apiKey, String workspaceName, - int expectedStatus) { - ProviderApiKey body = ProviderApiKey.builder().provider(randomLlmProvider()).apiKey(providerApiKey).build(); + public LlmProviderApiKeyResourceClient(ClientSupport client) { + this.client = client; + this.baseURI = "http://localhost:%d".formatted(client.getPort()); + } + + public ProviderApiKey createProviderApiKey( + String providerApiKey, String apiKey, String workspaceName, int expectedStatus) { + return createProviderApiKey(providerApiKey, randomLlmProvider(), apiKey, workspaceName, expectedStatus); + } + + public ProviderApiKey createProviderApiKey( + String providerApiKey, LlmProvider llmProvider, String apiKey, String workspaceName, int expectedStatus) { + ProviderApiKey body = ProviderApiKey.builder().provider(llmProvider).apiKey(providerApiKey).build(); try (var actualResponse = client.target(RESOURCE_PATH.formatted(baseURI)) .request() .accept(MediaType.APPLICATION_JSON_TYPE) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResourceTest.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResourceTest.java index c7e41ba620..9b03b2579c 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResourceTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResourceTest.java @@ -1,5 +1,6 @@ package com.comet.opik.api.resources.v1.priv; +import com.comet.opik.api.LlmProvider; import com.comet.opik.api.resources.utils.AuthTestUtils; import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; import com.comet.opik.api.resources.utils.ClientSupportUtils; @@ -9,10 +10,13 @@ import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils; import com.comet.opik.api.resources.utils.WireMockUtils; import com.comet.opik.api.resources.utils.resources.ChatCompletionsClient; +import com.comet.opik.api.resources.utils.resources.LlmProviderApiKeyResourceClient; import com.comet.opik.podam.PodamFactoryUtils; import com.redis.testcontainers.RedisContainer; +import dev.ai4j.openai4j.chat.ChatCompletionModel; import dev.ai4j.openai4j.chat.ChatCompletionRequest; -import lombok.extern.slf4j.Slf4j; +import dev.ai4j.openai4j.chat.Role; +import org.apache.http.HttpStatus; import org.jdbi.v3.core.Jdbi; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; @@ -33,7 +37,6 @@ import static org.assertj.core.api.Assertions.assertThat; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@Slf4j public class ChatCompletionsResourceTest { private static final String API_KEY = RandomStringUtils.randomAlphanumeric(25); @@ -41,28 +44,32 @@ public class ChatCompletionsResourceTest { private static final String WORKSPACE_NAME = RandomStringUtils.randomAlphanumeric(20); private static final String USER = RandomStringUtils.randomAlphanumeric(20); - private static final RedisContainer REDIS = RedisContainerUtils.newRedisContainer(); + private static final RedisContainer REDIS_CONTAINER = RedisContainerUtils.newRedisContainer(); private static final MySQLContainer MY_SQL_CONTAINER = MySQLContainerUtils.newMySQLContainer(); private static final ClickHouseContainer CLICK_HOUSE_CONTAINER = ClickHouseContainerUtils.newClickHouseContainer(); - private static final WireMockUtils.WireMockRuntime wireMock = WireMockUtils.startWireMock(); + private static final WireMockUtils.WireMockRuntime WIRE_MOCK = WireMockUtils.startWireMock(); @RegisterExtension - private static final TestDropwizardAppExtension app; + private static final TestDropwizardAppExtension APP; static { - Startables.deepStart(REDIS, MY_SQL_CONTAINER, CLICK_HOUSE_CONTAINER).join(); + Startables.deepStart(REDIS_CONTAINER, MY_SQL_CONTAINER, CLICK_HOUSE_CONTAINER).join(); var databaseAnalyticsFactory = ClickHouseContainerUtils.newDatabaseAnalyticsFactory( CLICK_HOUSE_CONTAINER, ClickHouseContainerUtils.DATABASE_NAME); - app = TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( - MY_SQL_CONTAINER.getJdbcUrl(), databaseAnalyticsFactory, wireMock.runtimeInfo(), REDIS.getRedisURI()); + APP = TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( + MY_SQL_CONTAINER.getJdbcUrl(), + databaseAnalyticsFactory, + WIRE_MOCK.runtimeInfo(), + REDIS_CONTAINER.getRedisURI()); } private final PodamFactory podamFactory = PodamFactoryUtils.newPodamFactory(); private ChatCompletionsClient chatCompletionsClient; + private LlmProviderApiKeyResourceClient llmProviderApiKeyResourceClient; @BeforeAll void setUpAll(ClientSupport clientSupport, Jdbi jdbi) throws SQLException { @@ -77,35 +84,138 @@ void setUpAll(ClientSupport clientSupport, Jdbi jdbi) throws SQLException { ClientSupportUtils.config(clientSupport); - mockTargetWorkspace(API_KEY, WORKSPACE_NAME, WORKSPACE_ID); + mockTargetWorkspace(WORKSPACE_NAME, WORKSPACE_ID); this.chatCompletionsClient = new ChatCompletionsClient(clientSupport); + this.llmProviderApiKeyResourceClient = new LlmProviderApiKeyResourceClient(clientSupport); } - private static void mockTargetWorkspace(String apiKey, String workspaceName, String workspaceId) { - AuthTestUtils.mockTargetWorkspace(wireMock.server(), apiKey, workspaceName, workspaceId, USER); + private static void mockTargetWorkspace(String workspaceName, String workspaceId) { + AuthTestUtils.mockTargetWorkspace(WIRE_MOCK.server(), API_KEY, workspaceName, workspaceId, USER); } @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class Get { + class Create { @Test - void get() { - var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class).stream(false).build(); + void create() { + var workspaceName = RandomStringUtils.randomAlphanumeric(20); + var workspaceId = UUID.randomUUID().toString(); + mockTargetWorkspace(workspaceName, workspaceId); + createLlmProviderApiKey(workspaceName); + var expectedModel = ChatCompletionModel.GPT_4O_MINI.toString(); + + var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class) + .stream(false) + .model(expectedModel) + .addUserMessage("Say 'Hello World'") + .build(); + + var response = chatCompletionsClient.create(API_KEY, workspaceName, request); + + assertThat(response.model()).containsIgnoringCase(expectedModel); + assertThat(response.choices()).anySatisfy(choice -> { + assertThat(choice.message().content()).containsIgnoringCase("Hello World"); + assertThat(choice.message().role()).isEqualTo(Role.ASSISTANT); + }); + } - var response = chatCompletionsClient.get(API_KEY, WORKSPACE_NAME, request); + @Test + void createReturnsBadRequestWhenNoLlmProviderApiKey() { + var workspaceName = RandomStringUtils.randomAlphanumeric(20); + var workspaceId = UUID.randomUUID().toString(); + mockTargetWorkspace(workspaceName, workspaceId); + var expectedModel = ChatCompletionModel.GPT_4O_MINI.toString(); + + var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class) + .stream(false) + .model(expectedModel) + .addUserMessage("Say 'Hello World'") + .build(); + + var errorMessage = chatCompletionsClient.create(API_KEY, workspaceName, request, HttpStatus.SC_BAD_REQUEST); + + assertThat(errorMessage.getCode()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + assertThat(errorMessage.getMessage()) + .containsIgnoringCase("API key not configured for LLM provider '%s'" + .formatted(LlmProvider.OPEN_AI.getValue())); + } - assertThat(response).isNotNull(); + @Test + void createReturnsBadRequestWhenNoModel() { + var workspaceName = RandomStringUtils.randomAlphanumeric(20); + var workspaceId = UUID.randomUUID().toString(); + mockTargetWorkspace(workspaceName, workspaceId); + createLlmProviderApiKey(workspaceName); + + var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class) + .stream(false) + .addUserMessage("Say 'Hello World'") + .build(); + + var errorMessage = chatCompletionsClient.create(API_KEY, workspaceName, request, HttpStatus.SC_BAD_REQUEST); + + assertThat(errorMessage.getCode()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + assertThat(errorMessage.getMessage()) + .containsIgnoringCase("Only %s model is available".formatted(ChatCompletionModel.GPT_4O_MINI)); } @Test - void getStream() { - var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class).stream(true).build(); + void createAndStreamResponse() { + var workspaceName = RandomStringUtils.randomAlphanumeric(20); + var workspaceId = UUID.randomUUID().toString(); + mockTargetWorkspace(workspaceName, workspaceId); + createLlmProviderApiKey(workspaceName); + var expectedModel = ChatCompletionModel.GPT_4O_MINI.toString(); + + var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class) + .stream(true) + .model(expectedModel) + .addUserMessage("Say 'Hello World'") + .build(); + + var response = chatCompletionsClient.createAndStream(API_KEY, workspaceName, request); + + assertThat(response) + .allSatisfy(entity -> assertThat(entity.model()) + .containsIgnoringCase(expectedModel)); + + var choices = response.stream().flatMap(entity -> entity.choices().stream()).toList(); + assertThat(choices) + .anySatisfy(choice -> assertThat(choice.delta().content()) + .containsIgnoringCase("Hello")); + assertThat(choices) + .anySatisfy(choice -> assertThat(choice.delta().content()) + .containsIgnoringCase("World")); + assertThat(choices).anySatisfy(choice -> assertThat(choice.delta().role()) + .isEqualTo(Role.ASSISTANT)); + } + } - var response = chatCompletionsClient.getStream(API_KEY, WORKSPACE_NAME, request); + @Test + void createAndStreamResponseReturnsBadRequestWhenNoModel() { + var workspaceName = RandomStringUtils.randomAlphanumeric(20); + var workspaceId = UUID.randomUUID().toString(); + mockTargetWorkspace(workspaceName, workspaceId); + createLlmProviderApiKey(workspaceName); - assertThat(response).hasSize(10); - } + var request = podamFactory.manufacturePojo(ChatCompletionRequest.Builder.class) + .stream(true) + .addUserMessage("Say 'Hello World'") + .build(); + + var errorMessages = chatCompletionsClient.createAndStreamError(API_KEY, workspaceName, request); + + assertThat(errorMessages).hasSize(1); + assertThat(errorMessages.getFirst().getCode()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + assertThat(errorMessages.getFirst().getMessage()) + .containsIgnoringCase("Only %s model is available".formatted(ChatCompletionModel.GPT_4O_MINI)); + } + + private void createLlmProviderApiKey(String workspaceName) { + var llmProviderApiKey = UUID.randomUUID().toString(); + llmProviderApiKeyResourceClient.createProviderApiKey( + llmProviderApiKey, LlmProvider.OPEN_AI, API_KEY, workspaceName, 201); } } diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/LlmProviderApiKeyResourceTest.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/LlmProviderApiKeyResourceTest.java index 5f3b617971..a4d486dbe9 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/LlmProviderApiKeyResourceTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/LlmProviderApiKeyResourceTest.java @@ -72,8 +72,6 @@ class LlmProviderApiKeyResourceTest { private final PodamFactory factory = PodamFactoryUtils.newPodamFactory(); - private String baseURI; - private ClientSupport client; private TransactionTemplate mySqlTemplate; private LlmProviderApiKeyResourceClient llmProviderApiKeyResourceClient; @@ -88,10 +86,8 @@ void setUpAll(ClientSupport client, Jdbi jdbi, ClickHouseContainerUtils.migrationParameters()); } - this.baseURI = "http://localhost:%d".formatted(client.getPort()); - this.client = client; this.mySqlTemplate = mySqlTemplate; - this.llmProviderApiKeyResourceClient = new LlmProviderApiKeyResourceClient(this.client, this.baseURI); + this.llmProviderApiKeyResourceClient = new LlmProviderApiKeyResourceClient(client); ClientSupportUtils.config(client); } diff --git a/apps/opik-backend/src/test/resources/config-test.yml b/apps/opik-backend/src/test/resources/config-test.yml index 5f284ebe48..ecabf312c7 100644 --- a/apps/opik-backend/src/test/resources/config-test.yml +++ b/apps/opik-backend/src/test/resources/config-test.yml @@ -85,3 +85,9 @@ cors: encryption: key: ${OPIK_ENCRYPTION_KEY:-'GiTHubiLoVeYouAA'} + +llmProviderClient: + openApiClient: + # See demo endpoint Langchain4j documentation: https://docs.langchain4j.dev/get-started + # Not https but only used for testing purposes. It's fine as long as not sensitive data is sent. + url: http://langchain4j.dev/demo/openai/v1