From 017a52b8730fc8f8ed71415049189f79ad56aa8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 23 Aug 2024 12:47:49 +0200 Subject: [PATCH] fix: upgrade netty and jackson mapper (#130) --- .github/workflows/ci.yml | 20 ++- .../langstream/agents/camel/CamelSource.java | 6 +- .../agents/grpc/AbstractGrpcAgent.java | 9 +- .../agents/grpc/PythonGrpcServer.java | 10 +- .../langstream-agent-http-request/pom.xml | 2 +- .../agents/http/HttpRequestAgent.java | 5 +- .../agents/http/LangServeClient.java | 11 +- .../agents/http/LangServeInvokeAgent.java | 2 - .../langstream-agent-webcrawler/pom.xml | 2 +- .../agents/webcrawler/WebCrawlerSource.java | 7 +- .../commons/state/LocalDiskStateStorage.java | 8 +- .../commons/state/MemoryStateStorage.java | 3 - .../agents/commons/state/S3StateStorage.java | 8 +- .../provider/StorageProviderSource.java | 7 +- .../ai/agents/commons/MutableRecord.java | 42 +++-- .../ai/agents/commons/jstl/JstlEvaluator.java | 4 +- .../ai/agents/commons/jstl/JstlFunctions.java | 8 +- .../jstl/JstlTransformContextAdapter.java | 6 +- .../agents/text/DocumentToJsonAgent.java | 7 +- .../langstream-ai-agents/pom.xml | 7 +- .../ai/agents/GenAIToolKitAgent.java | 7 +- .../services/impl/BedrockServiceProvider.java | 13 +- .../services/impl/HuggingFaceProvider.java | 12 +- .../agents/services/impl/OllamaProvider.java | 24 +-- .../services/impl/VertexAIProvider.java | 12 +- .../impl/bedrock/BaseInvokeModelRequest.java | 6 +- .../services/impl/bedrock/BedrockClient.java | 10 +- .../oss/streaming/ai/MergeKeyValueStep.java | 6 +- .../ai/embeddings/EmbeddingsService.java | 10 +- .../ai/util/TransformFunctionUtil.java | 7 +- .../ai/agents/GenAIToolKitAgentTest.java | 37 +++-- .../functions/transforms/GenAITest.java | 15 +- .../functions/transforms/JsonNodeSchema.java | 12 +- .../transforms/TransformFunction.java | 14 +- .../streaming/ai/ChatCompletionsStepTest.java | 7 +- .../streaming/ai/ComputeAIEmbeddingsTest.java | 6 +- .../oss/streaming/ai/ComputeStepTest.java | 14 +- .../streaming/ai/MergeKeyValueStepTest.java | 10 +- .../oss/streaming/ai/QueryStepTest.java | 16 +- .../agents/vector/InterpolationUtils.java | 5 +- .../agents/vector/QueryVectorDBAgent.java | 10 +- .../vector/couchbase/CouchbaseDataSource.java | 10 +- .../ElasticSearchDataSource.java | 24 +-- .../elasticsearch/ElasticSearchWriter.java | 18 +- .../vector/milvus/MilvusDataSource.java | 10 +- .../agents/vector/milvus/MilvusModel.java | 3 +- .../opensearch/OpenSearchDataSource.java | 23 +-- .../vector/opensearch/OpenSearchWriter.java | 18 +- .../PineconeAssetsManagerProvider.java | 9 +- .../vector/pinecone/PineconeDataSource.java | 10 +- .../agents/vector/solr/SolrDataSource.java | 10 +- .../datasource/impl/CassandraWriterTest.java | 10 +- .../datasource/impl/CouchbaseWriterTest.java | 5 +- .../impl/PineconeDataSourceTest.java | 6 +- .../github/GitHubAuthenticationProvider.java | 11 +- .../google/GoogleAuthenticationProvider.java | 7 +- .../jwt/admin/HttpAuthenticationProvider.java | 6 +- .../langstream-jwt-api-gateway-auth/pom.xml | 6 +- .../jwt/admin/JwtAuthenticationProvider.java | 6 +- ...thenticationProviderConfigurationTest.java | 5 +- langstream-api-gateway/pom.xml | 2 +- .../apigateway/gateways/ConsumeGateway.java | 2 - .../apigateway/gateways/ProduceGateway.java | 5 +- .../apigateway/http/GatewayResource.java | 6 +- .../metrics/ApiGatewayMetricsProvider.java | 1 - .../apigateway/util/StreamingClusterUtil.java | 9 +- .../websocket/handlers/AbstractHandler.java | 25 ++- .../apigateway/http/GatewayResourceTest.java | 11 +- .../http/ServiceAgentGatewayResourceTest.java | 5 +- .../handlers/ProduceConsumeHandlerTest.java | 24 ++- langstream-api/pom.xml | 7 +- ...GatewayAuthenticationProviderRegistry.java | 9 +- .../api/util/ObjectMapperFactory.java | 62 +++++++ .../src/test/resources/logback-test.xml | 34 ++++ langstream-auth-jwt/pom.xml | 2 +- langstream-cli/pom.xml | 3 +- .../k8s/codestorage/AzureBlobCodeStorage.java | 6 +- .../k8s/codestorage/S3CodeStorage.java | 6 +- .../codestorage/LocalDiskCodeStorage.java | 9 +- .../ApplicationPlaceholderResolver.java | 20 ++- .../impl/deploy/ApplicationDeployer.java | 16 +- .../langstream/impl/parser/ModelBuilder.java | 8 +- .../AIProvidersResourceProvider.java | 5 +- .../BaseDataSourceResourceProvider.java | 9 +- .../storage/GlobalMetadataStoreManager.java | 22 +-- .../impl/uti/ClassConfigValidator.java | 50 +++--- .../impl/parser/ModelBuilderTest.java | 2 +- .../impl/uti/ClassConfigValidatorTest.java | 115 +++++++------ .../tests/util/BaseEndToEndTest.java | 6 +- .../tests/util/ConsumeGatewayMessage.java | 3 +- .../impl/k8s/tests/KubeTestServer.java | 35 ++-- .../k8s/api/crds/agents/AgentSpec.java | 12 +- .../k8s/api/crds/apps/ApplicationSpec.java | 17 +- .../k8s/agents/AgentResourcesFactory.java | 8 +- .../k8s/util/JSONAssertComparator.java | 3 +- .../deployer/k8s/util/SerializationUtil.java | 39 +---- .../deployer/k8s/util/SpecDiffer.java | 41 ++--- .../k8s/agents/AgentResourcesFactoryTest.java | 2 +- .../k8s/apps/AppResourcesFactoryTest.java | 4 +- .../deployer/k8s/util/SpecDifferTest.java | 39 +++++ .../k8s/ResolvedDeployerConfiguration.java | 16 +- .../k8s/TenantsConfigurationReader.java | 7 +- .../k8s/controllers/apps/AppController.java | 8 +- .../k8s/controllers/AppControllerIT.java | 4 +- .../impl/k8s/KubernetesClusterRuntime.java | 8 +- .../agents/QueryVectorDBAgentProvider.java | 9 +- .../KubernetesClusterRuntimeDockerTest.java | 4 +- .../k8s/AbstractKubernetesGenericStore.java | 2 - .../k8s/apps/KubernetesApplicationStore.java | 15 +- .../storage/k8s/apps/TenantResources.java | 8 +- .../global/KubernetesGlobalMetadataStore.java | 4 +- .../apps/KubernetesApplicationStoreTest.java | 4 +- .../kafka/runner/KafkaProducerWrapper.java | 6 +- .../kafka/runner/KafkaReaderWrapper.java | 9 +- .../runtime/KafkaStreamingClusterRuntime.java | 7 +- .../pravega/PravegaClientUtils.java | 7 +- ...ravegaTopicConnectionsRuntimeProvider.java | 11 +- .../PravegaStreamingClusterRuntime.java | 7 +- .../langstream/pulsar/PulsarClientUtils.java | 7 +- ...PulsarTopicConnectionsRuntimeProvider.java | 27 +-- .../pulsar/PulsarStreamingClusterRuntime.java | 6 +- .../langstream-runtime-impl/pom.xml | 2 +- .../agent/AgentCodeDownloaderStarter.java | 14 +- .../runtime/agent/AgentRunnerStarter.java | 7 +- .../runtime/agent/api/AgentInfoServlet.java | 10 +- .../application/ApplicationSetupRunner.java | 14 +- .../ApplicationSetupRunnerStarter.java | 23 ++- .../runtime/deployer/RuntimeDeployer.java | 21 +-- .../deployer/RuntimeDeployerStarter.java | 22 ++- .../agents/AzureBlobStorageSourceIT.java | 4 +- .../agents/BedrockCompletionsIT.java | 4 +- .../agents/CassandraAssetQueryWriteIT.java | 6 +- .../CassandraVectorAssetQueryWriteIT.java | 6 +- .../langstream/agents/ChatCompletionsIT.java | 2 +- .../agents/ComputeEmbeddingsIT.java | 16 +- .../agents/CouchbaseAssetQueryWriteIT.java | 5 +- .../agents/ElasticSearchVectorIT.java | 6 +- .../agents/FlowControlRunnerIT.java | 8 +- .../agents/HttpRequestAgentRunnerIT.java | 6 +- .../ai/langstream/agents/JdbcDatabaseIT.java | 3 +- .../langstream/agents/OpenSearchVectorIT.java | 4 +- .../java/ai/langstream/agents/PineconeIT.java | 3 +- .../agents/RerankAgentRunnerIT.java | 5 +- .../java/ai/langstream/agents/S3AssetIT.java | 3 + .../java/ai/langstream/agents/S3SourceIT.java | 11 +- .../agents/SolrAssetQueryWriteIT.java | 5 +- .../langstream/agents/TextCompletionsIT.java | 2 +- .../agents/TextProcessingAgentsRunnerIT.java | 6 +- .../langstream/agents/WebCrawlerSourceIT.java | 12 +- .../ai/langstream/assets/DeployAssetsIT.java | 25 +-- .../langstream/pravega/SimplePravegaIT.java | 4 + .../agent/AgentCodeDownloaderStarterTest.java | 3 +- .../runtime/agent/AgentRunnerStarterTest.java | 3 +- .../deployer/RuntimeDeployerStarterTest.java | 3 +- .../AbstractApplicationRunner.java | 8 +- .../runtime/tester/KubeTestServer.java | 35 ++-- .../tester/LocalApplicationRunner.java | 4 +- .../ai/langstream/runtime/tester/Main.java | 5 +- langstream-webservice/pom.xml | 2 +- .../application/ApplicationResource.java | 5 +- .../webservice/archetype/ArchetypeStore.java | 4 +- .../doc/DocumentationGeneratorStarter.java | 11 +- .../src/main/resources/application.properties | 1 + .../ApplicationResourceLogsTest.java | 10 +- .../application/ApplicationResourceTest.java | 88 +++++----- .../archetype/ArchetypeResourceTest.java | 155 ++++++------------ pom.xml | 16 +- 167 files changed, 1027 insertions(+), 1069 deletions(-) create mode 100644 langstream-api/src/main/java/ai/langstream/api/util/ObjectMapperFactory.java create mode 100644 langstream-api/src/test/resources/logback-test.xml create mode 100644 langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/SpecDifferTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5522f9357..56b1309e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: name: Unit tests (${{ matrix.name }}) runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + checks: write # junit reporter strategy: fail-fast: false matrix: @@ -48,20 +50,29 @@ jobs: - name: Kafka - IT - Group 0 test_cmd: | export TESTS_RUNTIME_TYPE=kafka - ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.excludedGroups=group-1 $MAVEN_COMMON_SKIP_FLAGS + ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.excludedGroups=group-1,group-2 $MAVEN_COMMON_SKIP_FLAGS - name: Kafka - IT - Group 1 test_cmd: | export TESTS_RUNTIME_TYPE=kafka ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.groups=group-1 $MAVEN_COMMON_SKIP_FLAGS + - name: Kafka - IT - Group 2 + test_cmd: | + export TESTS_RUNTIME_TYPE=kafka + ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.groups=group-2 $MAVEN_COMMON_SKIP_FLAGS - name: Pulsar - IT - Group 0 test_cmd: | export TESTS_RUNTIME_TYPE=pulsar - ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.excludedGroups=group-1 $MAVEN_COMMON_SKIP_FLAGS + ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.excludedGroups=group-1,group-2 $MAVEN_COMMON_SKIP_FLAGS - name: Pulsar - IT - Group 1 test_cmd: | export TESTS_RUNTIME_TYPE=pulsar ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.groups=group-1 $MAVEN_COMMON_SKIP_FLAGS + + - name: Pulsar - IT - Group 2 + test_cmd: | + export TESTS_RUNTIME_TYPE=pulsar + ./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -DintegrationTests.groups=group-2 $MAVEN_COMMON_SKIP_FLAGS - name: Deployer test_cmd: ./mvnw verify -f langstream-k8s-deployer -Pdocker $MAVEN_COMMON_SKIP_FLAGS - name: Api Gateway @@ -123,6 +134,11 @@ jobs: git diff exit 1 fi + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() + with: + report_paths: '**/target/surefire-reports/TEST-*.xml' # e2e-tests: # name: End to End tests diff --git a/langstream-agents/langstream-agent-camel/src/main/java/ai/langstream/agents/camel/CamelSource.java b/langstream-agents/langstream-agent-camel/src/main/java/ai/langstream/agents/camel/CamelSource.java index 1e9b13ac3..d79fd654f 100644 --- a/langstream-agents/langstream-agent-camel/src/main/java/ai/langstream/agents/camel/CamelSource.java +++ b/langstream-agents/langstream-agent-camel/src/main/java/ai/langstream/agents/camel/CamelSource.java @@ -18,8 +18,8 @@ import ai.langstream.api.runner.code.*; import ai.langstream.api.runner.code.Record; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; @@ -38,8 +38,6 @@ @Slf4j public class CamelSource extends AbstractAgentCode implements AgentSource { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private String componentUri; private DefaultCamelContext camelContext; @@ -157,7 +155,7 @@ private static Object safeObject(Object v) throws JsonProcessingException { } else if (v instanceof CharSequence || v instanceof Number || v instanceof Boolean) { converted = v; } else { - converted = MAPPER.writeValueAsString(v); + converted = ObjectMapperFactory.getDefaultMapper().writeValueAsString(v); } return converted; } diff --git a/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/AbstractGrpcAgent.java b/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/AbstractGrpcAgent.java index 6fd4315a0..691c23102 100644 --- a/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/AbstractGrpcAgent.java +++ b/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/AbstractGrpcAgent.java @@ -19,9 +19,9 @@ import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.topics.TopicProducer; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.ByteString; import com.google.protobuf.Empty; import io.grpc.ManagedChannel; @@ -52,7 +52,6 @@ @Slf4j abstract class AbstractGrpcAgent extends AbstractAgentCode { - protected static final ObjectMapper MAPPER = new ObjectMapper(); protected ManagedChannel channel; // For each schema sent, we increment the schemaId @@ -217,8 +216,10 @@ private synchronized void sendTopicProducerWriteResult( @Override protected Map buildAdditionalInfo() { try { - return MAPPER.readValue( - blockingStub.agentInfo(Empty.getDefaultInstance()).getJsonInfo(), Map.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue( + blockingStub.agentInfo(Empty.getDefaultInstance()).getJsonInfo(), + Map.class); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/PythonGrpcServer.java b/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/PythonGrpcServer.java index e55c76eea..b1e06efed 100644 --- a/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/PythonGrpcServer.java +++ b/langstream-agents/langstream-agent-grpc/src/main/java/ai/langstream/agents/grpc/PythonGrpcServer.java @@ -16,7 +16,7 @@ package ai.langstream.agents.grpc; import ai.langstream.api.runner.code.AgentContext; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import com.google.protobuf.Empty; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -32,8 +32,6 @@ public class PythonGrpcServer { private static final int MAX_TRIALS = 15; - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final Path codeDirectory; private final Map configuration; private final String agentId; @@ -116,8 +114,10 @@ private Process startPythonProcess(boolean runMode, int targetPort) throws IOExc "langstream_grpc", runMode ? "run" : "cleanup", "[::]:%s".formatted(targetPort), - MAPPER.writeValueAsString(configuration), - MAPPER.writeValueAsString(agentContextConfiguration)) + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(configuration), + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(agentContextConfiguration)) .inheritIO() .redirectOutput(ProcessBuilder.Redirect.INHERIT) .redirectError(ProcessBuilder.Redirect.INHERIT); diff --git a/langstream-agents/langstream-agent-http-request/pom.xml b/langstream-agents/langstream-agent-http-request/pom.xml index aa50a89bd..1976efd50 100644 --- a/langstream-agents/langstream-agent-http-request/pom.xml +++ b/langstream-agents/langstream-agent-http-request/pom.xml @@ -70,7 +70,7 @@ test - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/HttpRequestAgent.java b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/HttpRequestAgent.java index 239cf7aff..55c128dff 100644 --- a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/HttpRequestAgent.java +++ b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/HttpRequestAgent.java @@ -23,8 +23,8 @@ import ai.langstream.api.runner.code.RecordSink; import ai.langstream.api.runtime.ComponentType; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.samskivert.mustache.Mustache; import com.samskivert.mustache.Template; import java.net.CookieManager; @@ -49,7 +49,6 @@ @Slf4j public class HttpRequestAgent extends AbstractAgentCode implements AgentProcessor { - static final ObjectMapper mapper = new ObjectMapper(); private final Map avroValueSchemaCache = new ConcurrentHashMap<>(); private final Map avroKeySchemaCache = new ConcurrentHashMap<>(); @@ -199,7 +198,7 @@ public void processRecord(Record record, RecordSink recordSink) { private Object parseResponseBody(HttpResponse response) { try { - return mapper.readValue(response.body(), Map.class); + return ObjectMapperFactory.getDefaultMapper().readValue(response.body(), Map.class); } catch (JsonProcessingException ex) { log.debug("Not able to parse response to json: {}, {}", response.body(), ex); } diff --git a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeClient.java b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeClient.java index 256847ec4..da1444927 100644 --- a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeClient.java +++ b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeClient.java @@ -15,9 +15,9 @@ */ package ai.langstream.agents.http; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.StringWriter; import java.net.CookieManager; @@ -37,8 +37,6 @@ @Slf4j public class LangServeClient { - static final ObjectMapper mapper = new ObjectMapper(); - final HttpClient httpClient; final Options options; @@ -144,7 +142,8 @@ private Object parseResponseBody(String body, boolean streaming) { try { if (body.startsWith("{")) { Map map = - mapper.readValue(body, new TypeReference>() {}); + ObjectMapperFactory.getDefaultMapper() + .readValue(body, new TypeReference>() {}); if (!streaming) { Object output = map.get("output"); if (output == null || output instanceof String) { @@ -159,7 +158,7 @@ private Object parseResponseBody(String body, boolean streaming) { return map.get(options.contentField); } } else if (body.startsWith("\"")) { - body = mapper.readValue(body, String.class); + body = ObjectMapperFactory.getDefaultMapper().readValue(body, String.class); } } catch (JsonProcessingException ex) { log.info("Not able to parse response to json: {}, {}", body, ex); @@ -268,7 +267,7 @@ interface StreamingChunksConsumer { private String buildBody(Map values) throws IOException { Map request = Map.of("input", values); - return mapper.writeValueAsString(request); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(request); } public CompletableFuture execute( diff --git a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeInvokeAgent.java b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeInvokeAgent.java index d090f5b36..db71dabd6 100644 --- a/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeInvokeAgent.java +++ b/langstream-agents/langstream-agent-http-request/src/main/java/ai/langstream/agents/http/LangServeInvokeAgent.java @@ -25,7 +25,6 @@ import ai.langstream.api.runner.topics.TopicProducer; import ai.langstream.api.runtime.ComponentType; import ai.langstream.api.util.ConfigurationUtils; -import com.fasterxml.jackson.databind.ObjectMapper; import com.samskivert.mustache.Mustache; import com.samskivert.mustache.Template; import java.io.IOException; @@ -55,7 +54,6 @@ record FieldDefinition(String name, JstlEvaluator expressionEvaluator) { private int minChunksPerMessage; private String contentField; private boolean debug; - static final ObjectMapper mapper = new ObjectMapper(); private final Map avroValueSchemaCache = new ConcurrentHashMap<>(); private final Map avroKeySchemaCache = new ConcurrentHashMap<>(); diff --git a/langstream-agents/langstream-agent-webcrawler/pom.xml b/langstream-agents/langstream-agent-webcrawler/pom.xml index d3cd1b69a..f3d7a0003 100644 --- a/langstream-agents/langstream-agent-webcrawler/pom.xml +++ b/langstream-agents/langstream-agent-webcrawler/pom.xml @@ -95,7 +95,7 @@ test - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/WebCrawlerSource.java b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/WebCrawlerSource.java index 200b24e5d..4b91df211 100644 --- a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/WebCrawlerSource.java +++ b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/WebCrawlerSource.java @@ -29,8 +29,8 @@ import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; import io.minio.*; import java.nio.file.Path; import java.time.Instant; @@ -46,7 +46,6 @@ @Slf4j public class WebCrawlerSource extends AbstractAgentCode implements AgentSource { - public static final ObjectMapper MAPPER = new ObjectMapper(); private int maxUnflushedPages; private String bucketName; @@ -536,7 +535,9 @@ private void sendSourceActivitySummaryIfNeeded() throws Exception { currentSourceActivitySummary.deletedUrls().size()); // Convert the new object to JSON - String value = MAPPER.writeValueAsString(summaryWithCounts); + String value = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(summaryWithCounts); List
allHeaders = new ArrayList<>(sourceRecordHeaders); allHeaders.add( new SimpleRecord.SimpleHeader("recordType", "sourceActivitySummary")); diff --git a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/LocalDiskStateStorage.java b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/LocalDiskStateStorage.java index af4c590b5..bde91332c 100644 --- a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/LocalDiskStateStorage.java +++ b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/LocalDiskStateStorage.java @@ -15,7 +15,7 @@ */ package ai.langstream.ai.agents.commons.state; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.minio.*; import java.io.IOException; import java.nio.file.Files; @@ -55,14 +55,12 @@ public static Path computePath( pathPrefix + "." + agentId + ".status.json"); } - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final Path path; @Override public void store(T status) throws Exception { log.info("Storing state to the disk at path {}", path); - MAPPER.writeValue(path.toFile(), status); + ObjectMapperFactory.getDefaultMapper().writeValue(path.toFile(), status); } @Override @@ -70,7 +68,7 @@ public T get(Class clazz) throws Exception { if (Files.exists(path)) { log.info("Restoring state from {}", path); try { - return MAPPER.readValue(path.toFile(), clazz); + return ObjectMapperFactory.getDefaultMapper().readValue(path.toFile(), clazz); } catch (IOException e) { log.error("Error parsing state file", e); return null; diff --git a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/MemoryStateStorage.java b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/MemoryStateStorage.java index 405a615b4..b3d003af3 100644 --- a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/MemoryStateStorage.java +++ b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/MemoryStateStorage.java @@ -15,13 +15,10 @@ */ package ai.langstream.ai.agents.commons.state; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @Slf4j public class MemoryStateStorage implements StateStorage { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private T value; @Override diff --git a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/S3StateStorage.java b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/S3StateStorage.java index 769516266..cab654df4 100644 --- a/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/S3StateStorage.java +++ b/langstream-agents/langstream-agents-commons-state-storage/src/main/java/ai/langstream/ai/agents/commons/state/S3StateStorage.java @@ -15,7 +15,7 @@ */ package ai.langstream.ai.agents.commons.state; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.minio.*; import io.minio.errors.ErrorResponseException; import io.minio.errors.MinioException; @@ -49,8 +49,6 @@ public static String computeObjectName( return pathPrefix + "." + suffix + ".status.json"; } - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final MinioClient minioClient; private final String bucketName; private final String objectName; @@ -74,7 +72,7 @@ private void makeBucketIfNotExists() { @Override public void store(T status) throws Exception { - byte[] content = MAPPER.writeValueAsBytes(status); + byte[] content = ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(status); log.info("Storing state in {}, {} bytes", objectName, content.length); putWithRetries( () -> @@ -135,7 +133,7 @@ public T get(Class clazz) throws Exception { byte[] content = result.readAllBytes(); log.info("Restoring state from {}, {} bytes", objectName, content.length); try { - return MAPPER.readValue(content, clazz); + return ObjectMapperFactory.getDefaultMapper().readValue(content, clazz); } catch (IOException e) { log.error("Error parsing state file", e); return null; diff --git a/langstream-agents/langstream-agents-commons-storage-provider/src/main/java/ai/langstream/ai/agents/commons/storage/provider/StorageProviderSource.java b/langstream-agents/langstream-agents-commons-storage-provider/src/main/java/ai/langstream/ai/agents/commons/storage/provider/StorageProviderSource.java index 674b19a43..3430ab2b3 100644 --- a/langstream-agents/langstream-agents-commons-storage-provider/src/main/java/ai/langstream/ai/agents/commons/storage/provider/StorageProviderSource.java +++ b/langstream-agents/langstream-agents-commons-storage-provider/src/main/java/ai/langstream/ai/agents/commons/storage/provider/StorageProviderSource.java @@ -24,8 +24,8 @@ import ai.langstream.api.runner.code.*; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -37,7 +37,6 @@ public abstract class StorageProviderSource extends AbstractAgentCode implements AgentSource { - public static final ObjectMapper MAPPER = new ObjectMapper(); private Map agentConfiguration; private final Set objectsToCommit = ConcurrentHashMap.newKeySet(); @Getter private StateStorage stateStorage; @@ -301,7 +300,9 @@ private void sendSourceActivitySummaryIfNeeded() throws Exception { currentSourceActivitySummary.deletedObjects().size()); // Convert the new object to JSON - String value = MAPPER.writeValueAsString(summaryWithCounts); + String value = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(summaryWithCounts); SimpleRecord simpleRecord = buildSimpleRecord(value, "sourceActivitySummary"); sourceActivitySummaryProducer.write(simpleRecord).get(); } else { diff --git a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/MutableRecord.java b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/MutableRecord.java index 93bd42850..9acd5606b 100644 --- a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/MutableRecord.java +++ b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/MutableRecord.java @@ -18,10 +18,10 @@ import ai.langstream.api.runner.code.Header; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -55,7 +55,6 @@ @Slf4j @Data public class MutableRecord { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private TransformSchemaType keySchemaType; private Object keyNativeSchema; private Object keyObject; @@ -105,16 +104,17 @@ public MutableRecord copy() { public void convertMapToStringOrBytes() throws JsonProcessingException { if (valueObject instanceof Map) { if (valueSchemaType == TransformSchemaType.STRING) { - valueObject = OBJECT_MAPPER.writeValueAsString(valueObject); + valueObject = + ObjectMapperFactory.getDefaultMapper().writeValueAsString(valueObject); } else if (valueSchemaType == TransformSchemaType.BYTES) { - valueObject = OBJECT_MAPPER.writeValueAsBytes(valueObject); + valueObject = ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(valueObject); } } if (keyObject instanceof Map) { if (keySchemaType == TransformSchemaType.STRING) { - keyObject = OBJECT_MAPPER.writeValueAsString(keyObject); + keyObject = ObjectMapperFactory.getDefaultMapper().writeValueAsString(keyObject); } else if (keySchemaType == TransformSchemaType.BYTES) { - keyObject = OBJECT_MAPPER.writeValueAsBytes(keyObject); + keyObject = ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(keyObject); } } } @@ -228,7 +228,10 @@ private void addOrReplaceJsonValueFields( } ObjectNode json = (ObjectNode) valueObject; newFields.forEach( - (field, value) -> json.set(field.name(), OBJECT_MAPPER.valueToTree(value))); + (field, value) -> + json.set( + field.name(), + ObjectMapperFactory.getDefaultMapper().valueToTree(value))); valueObject = json; } @@ -263,7 +266,10 @@ private void addOrReplaceJsonKeyFields( } ObjectNode json = (ObjectNode) keyObject; newFields.forEach( - (field, value) -> json.set(field.name(), OBJECT_MAPPER.valueToTree(value))); + (field, value) -> + json.set( + field.name(), + ObjectMapperFactory.getDefaultMapper().valueToTree(value))); keyObject = json; } @@ -290,11 +296,13 @@ private static Object toJsonSerializable(TransformSchemaType schemaType, Object switch (schemaType) { case AVRO: // TODO: do better than the double conversion AVRO -> JsonNode -> Map - return OBJECT_MAPPER.convertValue( - JsonConverter.toJson((GenericRecord) val), - new TypeReference>() {}); + return ObjectMapperFactory.getDefaultMapper() + .convertValue( + JsonConverter.toJson((GenericRecord) val), + new TypeReference>() {}); case JSON: - return OBJECT_MAPPER.convertValue(val, new TypeReference>() {}); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(val, new TypeReference>() {}); default: throw new UnsupportedOperationException("Unsupported schemaType " + schemaType); } @@ -302,7 +310,7 @@ private static Object toJsonSerializable(TransformSchemaType schemaType, Object @SneakyThrows public static String toJson(Object object) { - return OBJECT_MAPPER.writeValueAsString(object); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(object); } public void setResultField( @@ -533,11 +541,11 @@ private static TransformSchemaType getSchemaType(Class javaType) { public static Object attemptJsonConversion(Object value) { try { if (value instanceof String) { - return OBJECT_MAPPER.readValue( - (String) value, new TypeReference>() {}); + return ObjectMapperFactory.getDefaultMapper() + .readValue((String) value, new TypeReference>() {}); } else if (value instanceof byte[]) { - return OBJECT_MAPPER.readValue( - (byte[]) value, new TypeReference>() {}); + return ObjectMapperFactory.getDefaultMapper() + .readValue((byte[]) value, new TypeReference>() {}); } } catch (IOException e) { if (log.isDebugEnabled()) { diff --git a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlEvaluator.java b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlEvaluator.java index e42d11c62..d65dcce1a 100644 --- a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlEvaluator.java +++ b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlEvaluator.java @@ -16,7 +16,7 @@ package ai.langstream.ai.agents.commons.jstl; import ai.langstream.ai.agents.commons.MutableRecord; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import jakarta.el.ELContext; import jakarta.el.ExpressionFactory; import jakarta.el.PropertyNotFoundException; @@ -275,7 +275,7 @@ public synchronized T evaluate(MutableRecord mutableRecord) { Object valueObject = mutableRecord.getValueObject(); if (valueObject instanceof String s) { try { - new ObjectMapper().readValue(s, Object.class); + ObjectMapperFactory.getDefaultMapper().readValue(s, Object.class); } catch (IOException error) { throw new IllegalArgumentException( "The property referred by " diff --git a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlFunctions.java b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlFunctions.java index ba6e815c5..7a28e4f84 100644 --- a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlFunctions.java +++ b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlFunctions.java @@ -17,7 +17,7 @@ import ai.langstream.ai.agents.commons.MutableRecord; import ai.langstream.ai.agents.commons.jstl.predicate.JstlPredicate; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import jakarta.el.ELException; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -43,8 +43,6 @@ /** Provides convenience methods to use in jstl expression. All functions should be static. */ @Slf4j public class JstlFunctions { - private static final ObjectMapper MAPPER = new ObjectMapper(); - @Setter private static Clock clock = Clock.systemUTC(); public static String uppercase(Object input) { @@ -56,7 +54,7 @@ public static String lowercase(Object input) { } public static String toJson(Object input) throws Exception { - return MAPPER.writeValueAsString(input); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(input); } public static Object fromJson(Object input) throws Exception { @@ -67,7 +65,7 @@ public static Object fromJson(Object input) throws Exception { if (s.isEmpty()) { return null; } - return MAPPER.readValue(s, Object.class); + return ObjectMapperFactory.getDefaultMapper().readValue(s, Object.class); } public static Object toDouble(Object input) { diff --git a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlTransformContextAdapter.java b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlTransformContextAdapter.java index 6b5cbeaa2..52fdf0fcb 100644 --- a/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlTransformContextAdapter.java +++ b/langstream-agents/langstream-agents-commons/src/main/java/ai/langstream/ai/agents/commons/jstl/JstlTransformContextAdapter.java @@ -17,9 +17,9 @@ import ai.langstream.ai.agents.commons.AvroUtil; import ai.langstream.ai.agents.commons.MutableRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.time.Instant; import java.time.LocalDate; @@ -176,7 +176,6 @@ public Object transform(String key) { static class JsonNodeTransformer implements Transformer { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final JsonNode jsonNode; private final org.apache.avro.Schema schema; @@ -199,7 +198,8 @@ public Object transform(String key) { node, schema != null ? schema.getField(key).schema() : null)); } try { - Object value = OBJECT_MAPPER.treeToValue(node, Object.class); + Object value = + ObjectMapperFactory.getDefaultMapper().treeToValue(node, Object.class); return adaptValue(schema, key, value); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/langstream-agents/langstream-agents-text-processing/src/main/java/ai/langstream/agents/text/DocumentToJsonAgent.java b/langstream-agents/langstream-agents-text-processing/src/main/java/ai/langstream/agents/text/DocumentToJsonAgent.java index 206e8f5f4..35bfecd91 100644 --- a/langstream-agents/langstream-agents-text-processing/src/main/java/ai/langstream/agents/text/DocumentToJsonAgent.java +++ b/langstream-agents/langstream-agents-text-processing/src/main/java/ai/langstream/agents/text/DocumentToJsonAgent.java @@ -18,7 +18,7 @@ import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.code.SingleRecordAgentProcessor; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -28,7 +28,6 @@ @Slf4j public class DocumentToJsonAgent extends SingleRecordAgentProcessor { - private static final ObjectMapper MAPPER = new ObjectMapper(); private String textField; private boolean copyProperties; @@ -66,6 +65,8 @@ public List processRecord(Record record) throws Exception { } return List.of( - SimpleRecord.copyFrom(record).value(MAPPER.writeValueAsString(asJson)).build()); + SimpleRecord.copyFrom(record) + .value(ObjectMapperFactory.getDefaultMapper().writeValueAsString(asJson)) + .build()); } } diff --git a/langstream-agents/langstream-ai-agents/pom.xml b/langstream-agents/langstream-ai-agents/pom.xml index 1236f7354..56f053248 100644 --- a/langstream-agents/langstream-ai-agents/pom.xml +++ b/langstream-agents/langstream-ai-agents/pom.xml @@ -182,11 +182,6 @@ netty-transport-native-unix-common ${netty.version} - - io.netty - netty-tcnative-boringssl-static - ${netty.boringssl.version} - org.apache.pulsar @@ -318,7 +313,7 @@ test - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/GenAIToolKitAgent.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/GenAIToolKitAgent.java index 5d1a3e87a..5d15a9813 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/GenAIToolKitAgent.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/GenAIToolKitAgent.java @@ -29,6 +29,7 @@ import ai.langstream.api.runner.code.RecordSink; import ai.langstream.api.runner.topics.TopicProducer; import ai.langstream.api.runtime.ComponentType; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.StepPredicatePair; import com.datastax.oss.streaming.ai.TransformStep; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; @@ -38,7 +39,6 @@ import com.datastax.oss.streaming.ai.streaming.StreamingAnswersConsumer; import com.datastax.oss.streaming.ai.streaming.StreamingAnswersConsumerFactory; import com.datastax.oss.streaming.ai.util.TransformFunctionUtil; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,7 +52,6 @@ @Slf4j public class GenAIToolKitAgent extends AbstractAgentCode implements AgentProcessor { - private static final ObjectMapper MAPPER = new ObjectMapper(); private StepPredicatePair step; private TransformStepConfig config; private QueryStepDataSource dataSource; @@ -140,7 +139,9 @@ public void start() throws Exception { serviceProvider = ServiceProviderRegistry.getServiceProvider(configuration, reporter); configuration.remove("vertex"); - config = MAPPER.convertValue(configuration, TransformStepConfig.class); + config = + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, TransformStepConfig.class); dataSource = DataSourceProviderRegistry.getQueryStepDataSource(datasourceConfiguration); if (dataSource != null) { dataSource.initialize(datasourceConfiguration); diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/BedrockServiceProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/BedrockServiceProvider.java index 9bb398088..3d418efe8 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/BedrockServiceProvider.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/BedrockServiceProvider.java @@ -21,6 +21,7 @@ import ai.langstream.ai.agents.services.impl.bedrock.BedrockClient; import ai.langstream.ai.agents.services.impl.bedrock.TitanEmbeddingsModel; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.completions.ChatChoice; import com.datastax.oss.streaming.ai.completions.ChatCompletions; import com.datastax.oss.streaming.ai.completions.ChatMessage; @@ -29,8 +30,6 @@ import com.datastax.oss.streaming.ai.embeddings.EmbeddingsService; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -121,9 +120,6 @@ public void close() { @AllArgsConstructor private static class BedrockCompletionsService implements CompletionsService { - static final ObjectMapper MAPPER = - new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private final BedrockClient client; @Override @@ -155,9 +151,10 @@ public CompletableFuture getTextCompletions( final String model = (String) agentConfiguration.get("model"); final BedrockOptions bedrockOptions = - MAPPER.convertValue( - agentConfiguration.getOrDefault("options", Map.of()), - BedrockOptions.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + agentConfiguration.getOrDefault("options", Map.of()), + BedrockOptions.class); final Map parameters = bedrockOptions.getRequestParameters(); final String textCompletionExpression = diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java index 705b21fe4..fa011ea23 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java @@ -17,6 +17,7 @@ import ai.langstream.ai.agents.services.ServiceProviderProvider; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.completions.ChatChoice; import com.datastax.oss.streaming.ai.completions.ChatCompletions; import com.datastax.oss.streaming.ai.completions.ChatMessage; @@ -30,8 +31,6 @@ import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -46,9 +45,6 @@ public class HuggingFaceProvider implements ServiceProviderProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map agentConfiguration) { return agentConfiguration.containsKey("huggingface"); @@ -184,7 +180,7 @@ private CompletableFuture> callHFService( // https://huggingface.co/docs/api-inference/quicktour String url = this.url + "/models/%s"; String finalUrl = url.formatted(model); - String request = MAPPER.writeValueAsString(content); + String request = ObjectMapperFactory.getDefaultMapper().writeValueAsString(content); log.info("URL: {}", finalUrl); log.info("Request: {}", request); CompletableFuture> responseHandle = @@ -205,7 +201,9 @@ private static List convertResponse(HttpResponse response) if (log.isDebugEnabled()) { log.debug("Response: {}", body); } - List responseBeans = MAPPER.readValue(body, new TypeReference<>() {}); + List responseBeans = + ObjectMapperFactory.getDefaultMapper() + .readValue(body, new TypeReference<>() {}); return responseBeans; } diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/OllamaProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/OllamaProvider.java index 588f01c38..39b0166a7 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/OllamaProvider.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/OllamaProvider.java @@ -18,6 +18,7 @@ import ai.langstream.ai.agents.services.ServiceProviderProvider; import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.completions.ChatChoice; import com.datastax.oss.streaming.ai.completions.ChatCompletions; import com.datastax.oss.streaming.ai.completions.ChatMessage; @@ -26,8 +27,6 @@ import com.datastax.oss.streaming.ai.embeddings.EmbeddingsService; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.StringWriter; import java.net.URI; import java.net.http.HttpClient; @@ -47,12 +46,6 @@ @Slf4j public class OllamaProvider implements ServiceProviderProvider { - static ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map agentConfiguration) { return agentConfiguration.containsKey("ollama"); @@ -124,7 +117,9 @@ record EmbeddingResponse(List embedding, String error) {} private CompletableFuture> computeEmbeddings(String prompt) { String request; try { - request = MAPPER.writeValueAsString(new Request(model, prompt)); + request = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(new Request(model, prompt)); final HttpRequest.BodyPublisher bodyPublisher = HttpRequest.BodyPublishers.ofString(request); @@ -146,7 +141,9 @@ private CompletableFuture> computeEmbeddings(String prompt) { try { String body = response.body(); EmbeddingResponse embeddings = - mapper.readValue(body, EmbeddingResponse.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + body, EmbeddingResponse.class); if (embeddings.error != null) { throw new RuntimeException(embeddings.error); } @@ -199,7 +196,8 @@ record ResponseLine( @Override @SneakyThrows public synchronized void onNext(String body) { - ResponseLine responseLine = mapper.readValue(body, ResponseLine.class); + ResponseLine responseLine = + ObjectMapperFactory.getDefaultMapper().readValue(body, ResponseLine.class); String content = responseLine.response(); boolean last = responseLine.done(); @@ -280,7 +278,9 @@ public CompletableFuture getChatCompletions( String request; try { - request = MAPPER.writeValueAsString(new Request(model, prompt)); + request = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(new Request(model, prompt)); final HttpRequest.BodyPublisher bodyPublisher = HttpRequest.BodyPublishers.ofString(request); diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/VertexAIProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/VertexAIProvider.java index 714eee336..ea893960d 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/VertexAIProvider.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/VertexAIProvider.java @@ -18,6 +18,7 @@ import ai.langstream.ai.agents.services.ServiceProviderProvider; import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.completions.ChatChoice; import com.datastax.oss.streaming.ai.completions.ChatCompletions; import com.datastax.oss.streaming.ai.completions.ChatMessage; @@ -26,8 +27,6 @@ import com.datastax.oss.streaming.ai.embeddings.EmbeddingsService; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.auth.oauth2.CredentialRefreshListener; import com.google.api.client.auth.oauth2.TokenErrorResponse; @@ -60,9 +59,6 @@ public class VertexAIProvider implements ServiceProviderProvider { private static final String VERTEX_URL_TEMPLATE = "%s/v1/projects/%s/locations/%s/publishers/google/models/%s:predict"; - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map agentConfiguration) { return agentConfiguration.containsKey("vertex"); @@ -218,7 +214,9 @@ private CompletableFuture executeVertexCall( String finalUrl = VERTEX_URL_TEMPLATE.formatted(url, project, region, model); String request; try { - request = MAPPER.writeValueAsString(requestEmbeddings); + request = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(requestEmbeddings); } catch (JsonProcessingException e) { return CompletableFuture.failedFuture(e); } @@ -498,6 +496,6 @@ private static T handleResponse(Class responseType, HttpResponse } String body = response.body(); log.info("Response: {}", body); - return MAPPER.readValue(body, responseType); + return ObjectMapperFactory.getDefaultMapper().readValue(body, responseType); } } diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BaseInvokeModelRequest.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BaseInvokeModelRequest.java index 0b6cdfbfd..b7abb558f 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BaseInvokeModelRequest.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BaseInvokeModelRequest.java @@ -15,16 +15,14 @@ */ package ai.langstream.ai.agents.services.impl.bedrock; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import lombok.SneakyThrows; public abstract class BaseInvokeModelRequest { - protected static final ObjectMapper MAPPER = new ObjectMapper(); - @SneakyThrows public String generateJsonBody() { - return MAPPER.writeValueAsString(getBodyObject()); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(getBodyObject()); } public abstract S getBodyObject(); diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BedrockClient.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BedrockClient.java index 1c420584d..11f990630 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BedrockClient.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/bedrock/BedrockClient.java @@ -15,9 +15,7 @@ */ package ai.langstream.ai.agents.services.impl.bedrock; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; +import ai.langstream.api.util.ObjectMapperFactory; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -37,10 +35,6 @@ @Slf4j public class BedrockClient implements AutoCloseable { - protected static final ObjectMapper MAPPER = - new ObjectMapper() - .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private final BedrockRuntimeClient client; private final ExecutorService executorService; @@ -64,7 +58,7 @@ public CompletableFuture invokeModel( private static T parseResponse(InputStream in, Class toClass) { try { - return MAPPER.readValue(in, toClass); + return ObjectMapperFactory.getDefaultMapper().readValue(in, toClass); } catch (Exception e) { try { log.error( diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/MergeKeyValueStep.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/MergeKeyValueStep.java index e36cd36bc..347539e60 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/MergeKeyValueStep.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/MergeKeyValueStep.java @@ -17,8 +17,8 @@ import ai.langstream.ai.agents.commons.MutableRecord; import ai.langstream.ai.agents.commons.TransformSchemaType; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.List; import java.util.Map; @@ -28,7 +28,6 @@ import org.apache.avro.generic.GenericRecord; public class MergeKeyValueStep implements TransformStep { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final Map> schemaCache = new ConcurrentHashMap<>(); @@ -43,7 +42,8 @@ public void process(MutableRecord mutableRecord) { if (keyObject instanceof Map && valueObject instanceof Map) { Map value = (Map) valueObject; Map keyCopy = - OBJECT_MAPPER.convertValue(keyObject, new TypeReference<>() {}); + ObjectMapperFactory.getDefaultMapper() + .convertValue(keyObject, new TypeReference<>() {}); keyCopy.forEach(value::putIfAbsent); } else if (keySchemaType == TransformSchemaType.AVRO && mutableRecord.getValueSchemaType() == TransformSchemaType.AVRO) { diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/embeddings/EmbeddingsService.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/embeddings/EmbeddingsService.java index 0ac9c4cc3..48b7bd667 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/embeddings/EmbeddingsService.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/embeddings/EmbeddingsService.java @@ -15,7 +15,7 @@ */ package com.datastax.oss.streaming.ai.embeddings; -import com.fasterxml.jackson.annotation.JsonInclude; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; @@ -24,11 +24,9 @@ public interface EmbeddingsService extends AutoCloseable { static ObjectMapper createObjectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - return mapper; + return ObjectMapperFactory.getDefaultMapper() + .copy() + .configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true); } default void close() throws Exception {} diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/util/TransformFunctionUtil.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/util/TransformFunctionUtil.java index 3f8f4cbb7..f4558adad 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/util/TransformFunctionUtil.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/util/TransformFunctionUtil.java @@ -18,6 +18,7 @@ import ai.langstream.ai.agents.commons.MutableRecord; import ai.langstream.ai.agents.commons.TransformSchemaType; import ai.langstream.ai.agents.commons.jstl.predicate.JstlPredicate; +import ai.langstream.api.util.ObjectMapperFactory; import com.azure.ai.openai.OpenAIAsyncClient; import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; @@ -63,7 +64,6 @@ import com.datastax.oss.streaming.ai.model.config.UnwrapKeyValueConfig; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.datastax.oss.streaming.ai.streaming.StreamingAnswersConsumerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -90,7 +90,6 @@ @Slf4j public class TransformFunctionUtil { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final List FIELD_NAMES = Arrays.asList( "value", "key", "destinationTopic", "messageKey", "topicName", "eventTime"); @@ -320,11 +319,11 @@ public static UnwrapKeyValueStep newUnwrapKeyValueFunction(UnwrapKeyValueConfig } public static Map convertToMap(Object object) { - return OBJECT_MAPPER.convertValue(object, Map.class); + return ObjectMapperFactory.getDefaultMapper().convertValue(object, Map.class); } public static T convertFromMap(Map map, Class type) { - return OBJECT_MAPPER.convertValue(map, type); + return ObjectMapperFactory.getDefaultMapper().convertValue(map, type); } public static ChatCompletionsStep newChatCompletionsFunction( diff --git a/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/GenAIToolKitAgentTest.java b/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/GenAIToolKitAgentTest.java index d2699cd7c..32184fcd0 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/GenAIToolKitAgentTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/GenAIToolKitAgentTest.java @@ -23,27 +23,26 @@ import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; class GenAIToolKitAgentTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); - @Test void testCompute() throws Exception { String value = - MAPPER.writeValueAsString( - Map.of( - "fieldInt", - 1, - "fieldText", - "text", - "fieldCsv", - "a,b,c", - "fieldJson", - "{\"this\":\"that\"}")); + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString( + Map.of( + "fieldInt", + 1, + "fieldText", + "text", + "fieldCsv", + "a,b,c", + "fieldJson", + "{\"this\":\"that\"}")); assertEquals(1, compute("value.fieldInt", value)); assertEquals("text", compute("value.fieldText", value)); @@ -54,10 +53,11 @@ void testCompute() throws Exception { // compute cannot return a Map, so we return a JSON String assertEquals( Map.of("f1", "a", "f2", "b", "f3", "c"), - MAPPER.readValue( - compute("fn:toJson(fn:unpack(value.fieldCsv, 'f1,f2,f3'))", value) - .toString(), - Map.class)); + ObjectMapperFactory.getDefaultMapper() + .readValue( + compute("fn:toJson(fn:unpack(value.fieldCsv, 'f1,f2,f3'))", value) + .toString(), + Map.class)); } Object compute(String expression, Object value) throws Exception { @@ -83,7 +83,8 @@ Object compute(String expression, Object value) throws Exception { SimpleRecord record = SimpleRecord.builder().value(value).build(); Record result = agent.processRecord(record).get().get(0); Map resultValueParsed = - MAPPER.readValue(result.value().toString(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(result.value().toString(), Map.class); agent.close(); return resultValueParsed.get("computedField"); } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/GenAITest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/GenAITest.java index ef5fcdca5..6991c382e 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/GenAITest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/GenAITest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.when; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import com.azure.ai.openai.OpenAIAsyncClient; import com.azure.ai.openai.models.ChatCompletions; import com.azure.ai.openai.models.ChatCompletionsOptions; @@ -33,7 +34,6 @@ import com.azure.ai.openai.models.ChatRequestUserMessage; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.datastax.oss.streaming.ai.services.OpenAIServiceProvider; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.util.List; @@ -209,7 +209,7 @@ void testChatCompletions() throws Exception { .thenAnswer( a -> Flux.just( - new ObjectMapper() + ObjectMapperFactory.getDefaultMapper() .readValue(completion, ChatCompletions.class))); when(transformFunction.buildServiceProvider(any())) .thenReturn(new OpenAIServiceProvider(client, MetricsReporter.DISABLED)); @@ -282,7 +282,7 @@ void testChatCompletionsWithLogField() throws Exception { .thenAnswer( a -> Flux.just( - new ObjectMapper() + ObjectMapperFactory.getDefaultMapper() .readValue(completion, ChatCompletions.class))); when(transformFunction.buildServiceProvider(any())) .thenReturn(new OpenAIServiceProvider(client, MetricsReporter.DISABLED)); @@ -298,13 +298,8 @@ void testChatCompletionsWithLogField() throws Exception { Utils.getRecord(messageSchema.getValueSchema(), (byte[]) messageValue.getValue()); assertEquals("result", valueAvroRecord.get("completion").toString()); assertEquals( - "{\"options\":{\"type\":\"ai-chat-completions\",\"when\":null,\"model\":\"test-model\"," - + "\"messages\":[{\"role\":\"user\",\"content\":\"{{ value.valueField1 }} {{ key.keyField2 }}\"}]," - + "\"stream-to-topic\":null,\"stream-response-completion-field\":null,\"min-chunks-per-message\":20," - + "\"completion-field\":\"value.completion\",\"stream\":true,\"log-field\":\"value.log\"," - + "\"max-tokens\":null,\"temperature\":null,\"top-p\":null,\"logit-bias\":null,\"user\":null," - + "\"stop\":null,\"presence-penalty\":null,\"frequency-penalty\":null,\"options\":null}," - + "\"messages\":[{\"role\":\"user\",\"content\":\"value1 key2\"}],\"model\":\"test-model\"}", + """ + {"messages":[{"role":"user","content":"value1 key2"}],"model":"test-model","options":{"completion-field":"value.completion","log-field":"value.log","messages":[{"content":"{{ value.valueField1 }} {{ key.keyField2 }}","role":"user"}],"min-chunks-per-message":20,"model":"test-model","stream":true,"type":"ai-chat-completions"}}""", valueAvroRecord.get("log").toString()); } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/JsonNodeSchema.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/JsonNodeSchema.java index 591c8bb01..3ebd36f08 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/JsonNodeSchema.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/JsonNodeSchema.java @@ -15,10 +15,9 @@ */ package com.datastax.oss.pulsar.functions.transforms; -import com.fasterxml.jackson.annotation.JsonInclude; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.charset.StandardCharsets; import java.util.Optional; import org.apache.pulsar.client.api.Schema; @@ -27,13 +26,6 @@ import org.apache.pulsar.common.schema.SchemaType; public final class JsonNodeSchema implements Schema { - private static final ThreadLocal JSON_MAPPER = - ThreadLocal.withInitial( - () -> { - ObjectMapper mapper = new ObjectMapper(); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - return mapper; - }); private final org.apache.avro.Schema nativeSchema; private final SchemaInfo schemaInfo; @@ -53,7 +45,7 @@ public JsonNodeSchema(org.apache.avro.Schema schema) { @Override public byte[] encode(JsonNode message) { try { - return JSON_MAPPER.get().writeValueAsBytes(message); + return ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(message); } catch (JsonProcessingException e) { throw new SchemaSerializationException(e); } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java index 604f9f6f6..600d1f3a6 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java @@ -22,6 +22,7 @@ import ai.langstream.ai.agents.commons.TransformSchemaType; import ai.langstream.ai.agents.services.impl.HuggingFaceProvider; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.StepPredicatePair; import com.datastax.oss.streaming.ai.TransformStep; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; @@ -31,8 +32,6 @@ import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.datastax.oss.streaming.ai.util.TransformFunctionUtil; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SchemaValidatorsConfig; @@ -142,8 +141,9 @@ public class TransformFunction @SneakyThrows public void initialize(Context context) { Map userConfigMap = context.getUserConfigMap(); - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - JsonNode jsonNode = mapper.convertValue(userConfigMap, JsonNode.class); + JsonNode jsonNode = + ObjectMapperFactory.getDefaultYamlMapper() + .convertValue(userConfigMap, JsonNode.class); URNFactory urnFactory = urn -> { @@ -157,7 +157,7 @@ public void initialize(Context context) { }; JsonSchemaFactory factory = JsonSchemaFactory.builder(JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4)) - .objectMapper(mapper) + .objectMapper(ObjectMapperFactory.getDefaultYamlMapper().copy()) .addUrnFactory(urnFactory) .build(); SchemaValidatorsConfig jsonSchemaConfig = new SchemaValidatorsConfig(); @@ -218,7 +218,9 @@ public void initialize(Context context) { }); } - transformConfig = mapper.convertValue(userConfigMap, TransformStepConfig.class); + transformConfig = + ObjectMapperFactory.getDefaultYamlMapper() + .convertValue(userConfigMap, TransformStepConfig.class); serviceProvider = buildServiceProvider(transformConfig); dataSource = buildDataSource(transformConfig.getDatasource()); diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ChatCompletionsStepTest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ChatCompletionsStepTest.java index bbcb3dcc8..bc7200492 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ChatCompletionsStepTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ChatCompletionsStepTest.java @@ -25,6 +25,7 @@ import ai.langstream.ai.agents.services.impl.OpenAICompletionService; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import com.azure.ai.openai.OpenAIAsyncClient; import com.azure.ai.openai.models.ChatCompletions; import com.azure.ai.openai.models.ChatCompletionsOptions; @@ -80,7 +81,7 @@ public class ChatCompletionsStepTest { + " ]" + "}") .replace("'", "\""); - private static final ObjectMapper mapper = new ObjectMapper(); + private static final ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); private OpenAICompletionService completionService; private OpenAIAsyncClient openAIClient; @@ -484,11 +485,11 @@ void testJsonValueFieldOutput() throws Exception { public static Object[][] jsonStringFieldOutput() { return new Object[][] { - {Schema.STRING, "{\"name\":\"Jane\"}", "{\"name\":\"Jane\",\"chat\":\"result\"}"}, + {Schema.STRING, "{\"name\":\"Jane\"}", "{\"chat\":\"result\",\"name\":\"Jane\"}"}, { Schema.BYTES, "{\"name\":\"Jane\"}".getBytes(StandardCharsets.UTF_8), - "{\"name\":\"Jane\",\"chat\":\"result\"}".getBytes(StandardCharsets.UTF_8) + "{\"chat\":\"result\",\"name\":\"Jane\"}".getBytes(StandardCharsets.UTF_8) } }; } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeAIEmbeddingsTest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeAIEmbeddingsTest.java index 5bf8bb059..ec43e877f 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeAIEmbeddingsTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeAIEmbeddingsTest.java @@ -20,8 +20,8 @@ import ai.langstream.ai.agents.commons.MutableRecord; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.embeddings.MockEmbeddingsService; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.Arrays; import java.util.List; @@ -135,7 +135,9 @@ void testJson() throws Exception { final ObjectNode jsonNode = (ObjectNode) outputRecord.getValue(); assertNotNull(jsonNode.get("newField")); - final List asList = new ObjectMapper().convertValue(jsonNode.get("newField"), List.class); + final List asList = + ObjectMapperFactory.getDefaultMapper() + .convertValue(jsonNode.get("newField"), List.class); assertEquals(asList, expectedEmbeddings); assertEquals(outputRecord.getSchema().getSchemaInfo().getType(), SchemaType.JSON); } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeStepTest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeStepTest.java index be2c66df7..b8cc28025 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeStepTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/ComputeStepTest.java @@ -430,13 +430,12 @@ void testStringJson(Schema schema) throws Exception { assertEquals(outputRecord.getSchema(), schema); Object expected = - "{\"name\":\"Jane\",\"age\":\"43\",\"date\":18999,\"timestamp\":1672525445006,\"time\":83085006," - + "\"integerStr\":\"13360\"}"; + "{\"age\":\"43\",\"date\":18999,\"integerStr\":\"13360\",\"name\":\"Jane\",\"time\":83085006,\"timestamp\":1672525445006}"; if (schemaType == SchemaType.BYTES) { expected = ((String) expected).getBytes(StandardCharsets.UTF_8); - assertArrayEquals((byte[]) outputRecord.getValue(), (byte[]) expected); + assertArrayEquals((byte[]) expected, (byte[]) outputRecord.getValue()); } else { - assertEquals(outputRecord.getValue(), expected); + assertEquals(expected, outputRecord.getValue()); } } @@ -485,13 +484,12 @@ void testKVStringJson(Schema schema) throws Exception { assertEquals(messageValue.getValue(), 42); Object expected = - "{\"name\":\"Jane\",\"age\":\"44\",\"date\":18999,\"timestamp\":1672525445006,\"time\":83085006," - + "\"integerStr\":\"13360\"}"; + "{\"age\":\"44\",\"date\":18999,\"integerStr\":\"13360\",\"name\":\"Jane\",\"time\":83085006,\"timestamp\":1672525445006}"; if (schemaType == SchemaType.BYTES) { expected = ((String) expected).getBytes(StandardCharsets.UTF_8); - assertArrayEquals((byte[]) messageValue.getKey(), (byte[]) expected); + assertArrayEquals((byte[]) expected, (byte[]) messageValue.getKey()); } else { - assertEquals(messageValue.getKey(), expected); + assertEquals(expected, messageValue.getKey()); } } diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/MergeKeyValueStepTest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/MergeKeyValueStepTest.java index eb058b6e0..3e4a3a20d 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/MergeKeyValueStepTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/MergeKeyValueStepTest.java @@ -112,9 +112,8 @@ void testKeyValueStringJson() throws Exception { KeyValue messageValue = (KeyValue) outputRecord.getValue(); assertEquals( - messageValue.getValue(), - "{\"valueField1\":\"value1\",\"valueField2\":\"value2\"," - + "\"valueField3\":\"value3\",\"keyField1\":\"key1\",\"keyField2\":\"key2\",\"keyField3\":\"key3\"}"); + "{\"keyField1\":\"key1\",\"keyField2\":\"key2\",\"keyField3\":\"key3\",\"valueField1\":\"value1\",\"valueField2\":\"value2\",\"valueField3\":\"value3\"}", + messageValue.getValue()); } @Test @@ -142,9 +141,8 @@ void testKeyValueBytesJson() throws Exception { KeyValue messageValue = (KeyValue) outputRecord.getValue(); assertEquals( - new String((byte[]) messageValue.getValue(), StandardCharsets.UTF_8), - "{\"valueField1\":\"value1\",\"valueField2\":\"value2\",\"valueField3\":\"value3\"," - + "\"keyField1\":\"key1\",\"keyField2\":\"key2\",\"keyField3\":\"key3\"}"); + "{\"keyField1\":\"key1\",\"keyField2\":\"key2\",\"keyField3\":\"key3\",\"valueField1\":\"value1\",\"valueField2\":\"value2\",\"valueField3\":\"value3\"}", + new String((byte[]) messageValue.getValue(), StandardCharsets.UTF_8)); } @Test diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/QueryStepTest.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/QueryStepTest.java index 890bdfd38..f3acad3e4 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/QueryStepTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/streaming/ai/QueryStepTest.java @@ -19,8 +19,8 @@ import ai.langstream.ai.agents.commons.MutableRecord; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; -import com.fasterxml.jackson.databind.ObjectMapper; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; @@ -50,8 +50,6 @@ @Slf4j public class QueryStepTest { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - @Test void testPrimitive() throws Exception { Record record = @@ -175,8 +173,10 @@ public SchemaType getSchemaType() { @Override public Object getNativeObject() { - return OBJECT_MAPPER.valueToTree( - new PoJo("a", 42, true, List.of("b", "c"), List.of(43, 44))); + return ObjectMapperFactory.getDefaultMapper() + .valueToTree( + new PoJo( + "a", 42, true, List.of("b", "c"), List.of(43, 44))); } }; Record record = @@ -333,7 +333,8 @@ public List> fetchData(String query, List params) { Record result = Utils.process(record, queryStepFindSomeResults); KeyValue keyValueResult = (KeyValue) result.getValue(); Map parsed = - new ObjectMapper().readValue(keyValueResult.getValue(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(keyValueResult.getValue(), Map.class); assertEquals( parsed, Map.of( @@ -354,7 +355,8 @@ public List> fetchData(String query, List params) { Utils.process(record, queryStepFindNoResults); KeyValue keyValueResultNoResults = resultNoResults.getValue(); Map parsedNoResults = - new ObjectMapper().readValue(keyValueResultNoResults.getValue(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(keyValueResultNoResults.getValue(), Map.class); assertEquals( parsedNoResults, Map.of( diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/InterpolationUtils.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/InterpolationUtils.java index 6d5d504af..210d9c05f 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/InterpolationUtils.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/InterpolationUtils.java @@ -15,6 +15,7 @@ */ package ai.langstream.agents.vector; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; @@ -25,7 +26,9 @@ public class InterpolationUtils { private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapperFactory.getDefaultMapper() + .copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); public static String interpolate(String query, List array) { if (query == null || !query.contains("?")) { diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/QueryVectorDBAgent.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/QueryVectorDBAgent.java index 12bc288dc..f293ac620 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/QueryVectorDBAgent.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/QueryVectorDBAgent.java @@ -23,13 +23,12 @@ import ai.langstream.ai.agents.datasource.DataSourceProviderRegistry; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SingleRecordAgentProcessor; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.QueryStep; import com.datastax.oss.streaming.ai.StepPredicatePair; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.datastax.oss.streaming.ai.model.config.QueryConfig; import com.datastax.oss.streaming.ai.util.TransformFunctionUtil; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collection; import java.util.List; import java.util.Map; @@ -39,9 +38,6 @@ @Slf4j public class QueryVectorDBAgent extends SingleRecordAgentProcessor { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private QueryStepDataSource dataSource; private QueryStep queryExecutor; @@ -56,7 +52,9 @@ public void init(Map configuration) throws Exception { dataSource.initialize(datasourceConfiguration); configuration.put("type", "query"); - QueryConfig queryConfig = MAPPER.convertValue(configuration, QueryConfig.class); + QueryConfig queryConfig = + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, QueryConfig.class); queryExecutor = (QueryStep) TransformFunctionUtil.newQuery(queryConfig, dataSource); JstlPredicate when = queryConfig.getWhen() == null ? null : new JstlPredicate(queryConfig.getWhen()); diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/couchbase/CouchbaseDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/couchbase/CouchbaseDataSource.java index dd614169c..046ca3ea6 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/couchbase/CouchbaseDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/couchbase/CouchbaseDataSource.java @@ -18,6 +18,7 @@ import ai.langstream.agents.vector.InterpolationUtils; import ai.langstream.ai.agents.commons.jstl.JstlFunctions; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import com.couchbase.client.core.error.DocumentNotFoundException; import com.couchbase.client.java.Cluster; import com.couchbase.client.java.json.JsonArray; @@ -30,8 +31,6 @@ import com.couchbase.client.java.search.vector.VectorSearch; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -45,9 +44,6 @@ @Slf4j public class CouchbaseDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "couchbase".equals(dataSourceConfig.get("service")); @@ -68,7 +64,9 @@ public static final class CouchbaseConfig { @Override public QueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { - CouchbaseConfig config = MAPPER.convertValue(dataSourceConfig, CouchbaseConfig.class); + CouchbaseConfig config = + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, CouchbaseConfig.class); return new CouchbaseQueryStepDataSource(config); } diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchDataSource.java index 04bfdd254..123e9c7cc 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchDataSource.java @@ -19,6 +19,7 @@ import static ai.langstream.agents.vector.elasticsearch.ElasticSearchDataSource.ElasticSearchQueryStepDataSource.convertSearchRequest; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch.core.SearchRequest; @@ -28,11 +29,7 @@ import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.*; @@ -48,9 +45,6 @@ @Slf4j public class ElasticSearchDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "elasticsearch".equals(dataSourceConfig.get("service")); @@ -72,7 +66,8 @@ public ElasticSearchQueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { ElasticSearchConfig clientConfig = - MAPPER.convertValue(dataSourceConfig, ElasticSearchConfig.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, ElasticSearchConfig.class); return new ElasticSearchQueryStepDataSource(clientConfig); } @@ -168,7 +163,9 @@ public List> fetchData(String query, List params) { @SneakyThrows public static SearchRequest convertSearchRequest(String query, List params) { - final Map asMap = buildObjectFromJson(query, Map.class, params, OBJECT_MAPPER); + final Map asMap = + buildObjectFromJson( + query, Map.class, params, ObjectMapperFactory.getDefaultMapper()); SearchRequest.Builder builder = new SearchRequest.Builder(); Object index = asMap.remove("index"); if (index == null) { @@ -181,7 +178,9 @@ public static SearchRequest convertSearchRequest(String query, List para builder.index(String.valueOf(index)); } return builder.withJson( - new ByteArrayInputStream(OBJECT_MAPPER.writeValueAsBytes(asMap))) + new ByteArrayInputStream( + ObjectMapperFactory.getDefaultMapper() + .writeValueAsBytes(asMap))) .build(); } @@ -196,9 +195,4 @@ public void close() { } } } - - protected static final ObjectMapper OBJECT_MAPPER = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); } diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchWriter.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchWriter.java index fbab75884..72a0777ab 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchWriter.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/elasticsearch/ElasticSearchWriter.java @@ -24,6 +24,7 @@ import ai.langstream.api.database.VectorDatabaseWriterProvider; import ai.langstream.api.runner.code.Record; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.util.OrderedAsyncBatchExecutor; import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.Time; @@ -35,10 +36,7 @@ import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.bulk.DeleteOperation; import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.samskivert.mustache.Mustache; import com.samskivert.mustache.Template; import java.io.IOException; @@ -71,11 +69,6 @@ public ElasticSearchVectorDatabaseWriter createImplementation( public static class ElasticSearchVectorDatabaseWriter implements VectorDatabaseWriter, AutoCloseable { - protected static final ObjectMapper OBJECT_MAPPER = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - @Getter private final ElasticSearchDataSource.ElasticSearchQueryStepDataSource dataSource; private OrderedAsyncBatchExecutor batchExecutor; @@ -129,10 +122,11 @@ public void initialise(Map agentConfiguration) throws Exception }); id = buildEvaluator(agentConfiguration, "id", String.class); bulkParameters = - OBJECT_MAPPER.convertValue( - ConfigurationUtils.getMap( - "bulk-parameters", Map.of(), agentConfiguration), - BulkParameters.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + ConfigurationUtils.getMap( + "bulk-parameters", Map.of(), agentConfiguration), + BulkParameters.class); final int flushInterval = ConfigurationUtils.getInt("flush-interval", 1000, agentConfiguration); diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java index 0d627cbe2..557a517b5 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java @@ -18,10 +18,9 @@ import static ai.langstream.agents.vector.InterpolationUtils.buildObjectFromJson; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import io.milvus.client.MilvusServiceClient; import io.milvus.param.ConnectParam; import io.milvus.param.R; @@ -37,9 +36,6 @@ @Slf4j public class MilvusDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "milvus".equals(dataSourceConfig.get("service")); @@ -75,7 +71,9 @@ public static final class MilvusConfig { public MilvusQueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { - MilvusConfig clientConfig = MAPPER.convertValue(dataSourceConfig, MilvusConfig.class); + MilvusConfig clientConfig = + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, MilvusConfig.class); return new MilvusQueryStepDataSource(clientConfig); } diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusModel.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusModel.java index a06cdc243..39f207937 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusModel.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusModel.java @@ -15,6 +15,7 @@ */ package ai.langstream.agents.vector.milvus; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; @@ -50,7 +51,7 @@ public class MilvusModel { private static final ObjectMapper MAPPER = builderMapper(); private static ObjectMapper builderMapper() { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper().copy(); SimpleModule module = new SimpleModule(); module.addDeserializer( DataType.class, diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchDataSource.java index 8c6b75b40..e76ed1588 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchDataSource.java @@ -18,12 +18,9 @@ import static ai.langstream.agents.vector.InterpolationUtils.buildObjectFromJson; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -60,9 +57,6 @@ @Slf4j public class OpenSearchDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "opensearch".equals(dataSourceConfig.get("service")); @@ -88,7 +82,8 @@ public OpenSearchQueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { OpenSearchConfig clientConfig = - MAPPER.convertValue(dataSourceConfig, OpenSearchConfig.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, OpenSearchConfig.class); return new OpenSearchQueryStepDataSource(clientConfig); } @@ -206,7 +201,9 @@ public List> fetchData(String query, List params) { @NotNull static SearchRequest convertSearchRequest( String query, List params, String indexName) throws IllegalAccessException { - final Map asMap = buildObjectFromJson(query, Map.class, params, OBJECT_MAPPER); + final Map asMap = + buildObjectFromJson( + query, Map.class, params, ObjectMapperFactory.getDefaultMapper()); final SearchRequest searchRequest = OpenSearchDataSource.parseOpenSearchRequestBodyJson( asMap, SearchRequest._DESERIALIZER); @@ -226,17 +223,13 @@ public void close() { } } - protected static final ObjectMapper OBJECT_MAPPER = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); protected static final JacksonJsonpMapper JACKSON_JSONP_MAPPER = - new JacksonJsonpMapper(OBJECT_MAPPER); + new JacksonJsonpMapper(ObjectMapperFactory.getDefaultMapper().copy()); public static T parseOpenSearchRequestBodyJson( String json, JsonpDeserializer deserializer) throws IOException { return parseOpenSearchRequestBodyJson( - OBJECT_MAPPER.readValue(json, Map.class), deserializer); + ObjectMapperFactory.getDefaultMapper().readValue(json, Map.class), deserializer); } public static T parseOpenSearchRequestBodyJson( diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchWriter.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchWriter.java index ea0b432f8..9a39784ed 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchWriter.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/opensearch/OpenSearchWriter.java @@ -23,11 +23,9 @@ import ai.langstream.api.database.VectorDatabaseWriterProvider; import ai.langstream.api.runner.code.Record; import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.util.OrderedAsyncBatchExecutor; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -67,11 +65,6 @@ public OpenSearchVectorDatabaseWriter createImplementation( public static class OpenSearchVectorDatabaseWriter implements VectorDatabaseWriter, AutoCloseable { - protected static final ObjectMapper OBJECT_MAPPER = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - @Getter private final OpenSearchDataSource.OpenSearchQueryStepDataSource dataSource; private OrderedAsyncBatchExecutor batchExecutor; @@ -123,10 +116,11 @@ public void initialise(Map agentConfiguration) throws Exception }); id = buildEvaluator(agentConfiguration, "id", String.class); bulkParameters = - OBJECT_MAPPER.convertValue( - ConfigurationUtils.getMap( - "bulk-parameters", Map.of(), agentConfiguration), - BulkParameters.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + ConfigurationUtils.getMap( + "bulk-parameters", Map.of(), agentConfiguration), + BulkParameters.class); final int flushInterval = ConfigurationUtils.getInt("flush-interval", 1000, agentConfiguration); diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeAssetsManagerProvider.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeAssetsManagerProvider.java index f5f38d1df..765e07ec5 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeAssetsManagerProvider.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeAssetsManagerProvider.java @@ -19,8 +19,7 @@ import ai.langstream.api.runner.assets.AssetManager; import ai.langstream.api.runner.assets.AssetManagerProvider; import ai.langstream.api.util.ConfigurationUtils; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.pinecone.exceptions.PineconeNotFoundException; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -29,9 +28,6 @@ @Slf4j public class PineconeAssetsManagerProvider implements AssetManagerProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(String assetType) { return "pinecone-index".equals(assetType); @@ -61,7 +57,8 @@ private static class PineconeIndexAssetManager implements AssetManager { public void initialize(AssetDefinition assetDefinition) throws Exception { this.datasource = buildDataSource(assetDefinition); this.indexConfig = - MAPPER.convertValue(assetDefinition.getConfig(), PineconeIndexConfig.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(assetDefinition.getConfig(), PineconeIndexConfig.class); } @Override diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java index 5901abe2c..55c5e4222 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java @@ -18,10 +18,9 @@ import static ai.langstream.agents.vector.InterpolationUtils.buildObjectFromJson; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import io.grpc.StatusRuntimeException; import io.pinecone.clients.Index; import io.pinecone.clients.Pinecone; @@ -48,9 +47,6 @@ @Slf4j public class PineconeDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "pinecone".equals(dataSourceConfig.get("service")); @@ -86,7 +82,9 @@ public static final class PineconeConfig { public PineconeQueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { - PineconeConfig clientConfig = MAPPER.convertValue(dataSourceConfig, PineconeConfig.class); + PineconeConfig clientConfig = + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, PineconeConfig.class); return new PineconeQueryStepDataSource(clientConfig); } diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/solr/SolrDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/solr/SolrDataSource.java index cf406f8b7..7b4507449 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/solr/SolrDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/solr/SolrDataSource.java @@ -17,10 +17,9 @@ import ai.langstream.agents.vector.InterpolationUtils; import ai.langstream.ai.agents.datasource.DataSourceProvider; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -38,9 +37,6 @@ @Slf4j public class SolrDataSource implements DataSourceProvider { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Override public boolean supports(Map dataSourceConfig) { return "solr".equals(dataSourceConfig.get("service")); @@ -72,7 +68,9 @@ public static final class SolrConfig { public SolrQueryStepDataSource createDataSourceImplementation( Map dataSourceConfig) { - SolrConfig clientConfig = MAPPER.convertValue(dataSourceConfig, SolrConfig.class); + SolrConfig clientConfig = + ObjectMapperFactory.getDefaultMapper() + .convertValue(dataSourceConfig, SolrConfig.class); return new SolrQueryStepDataSource(clientConfig); } diff --git a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CassandraWriterTest.java b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CassandraWriterTest.java index 2f8d19ca2..6aba63683 100644 --- a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CassandraWriterTest.java +++ b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CassandraWriterTest.java @@ -25,9 +25,9 @@ import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.ResultSet; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,7 +87,9 @@ void testWrite() throws Exception { Map value = Map.of("id", "1", "description", "test-description", "name", "test-name"); - SimpleRecord record = SimpleRecord.of(null, new ObjectMapper().writeValueAsString(value)); + SimpleRecord record = + SimpleRecord.of( + null, ObjectMapperFactory.getDefaultMapper().writeValueAsString(value)); agent.write(record).thenRun(() -> committed.add(record)).get(); assertEquals(committed.get(0), record); @@ -130,7 +132,9 @@ void testWriteAstra() throws Exception { Map value = Map.of("id", "1", "description", "test-description", "name", "test-name"); - SimpleRecord record = SimpleRecord.of(null, new ObjectMapper().writeValueAsString(value)); + SimpleRecord record = + SimpleRecord.of( + null, ObjectMapperFactory.getDefaultMapper().writeValueAsString(value)); agent.write(record).thenRun(() -> committed.add(record)).get(); assertEquals(committed.get(0), record); diff --git a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CouchbaseWriterTest.java b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CouchbaseWriterTest.java index 85fa9c251..c7889c766 100644 --- a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CouchbaseWriterTest.java +++ b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/CouchbaseWriterTest.java @@ -30,8 +30,8 @@ import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -214,7 +214,8 @@ void testCouchbaseWrite() throws Exception { "funtostayatymca.pdf"); SimpleRecord record = - SimpleRecord.of(null, new ObjectMapper().writeValueAsString(value)); + SimpleRecord.of( + null, ObjectMapperFactory.getDefaultMapper().writeValueAsString(value)); agent.write(record).thenRun(() -> committed.add(record)).get(); } diff --git a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/PineconeDataSourceTest.java b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/PineconeDataSourceTest.java index d6526ab62..2c75a40b6 100644 --- a/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/PineconeDataSourceTest.java +++ b/langstream-agents/langstream-vector-agents/src/test/java/ai/langstream/agents/vector/datasource/impl/PineconeDataSourceTest.java @@ -31,8 +31,8 @@ import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.code.SimpleRecord; +import ai.langstream.api.util.ObjectMapperFactory; import com.datastax.oss.streaming.ai.datasource.QueryStepDataSource; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import lombok.extern.slf4j.Slf4j; @@ -87,7 +87,9 @@ void testPineconeWrite() throws Exception { Map value = Map.of("id", i, "vector", vectors.get(i), "genre", genre, "title", title); SimpleRecord record = - SimpleRecord.of(null, new ObjectMapper().writeValueAsString(value)); + SimpleRecord.of( + null, + ObjectMapperFactory.getDefaultMapper().writeValueAsString(value)); List committed = new CopyOnWriteArrayList<>(); agent.write(record).thenRun(() -> committed.add(record)).get(); assertEquals(committed.get(0), record); diff --git a/langstream-api-gateway-auth/langstream-github-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/github/GitHubAuthenticationProvider.java b/langstream-api-gateway-auth/langstream-github-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/github/GitHubAuthenticationProvider.java index 18c4a09d0..cacd1f5b1 100644 --- a/langstream-api-gateway-auth/langstream-github-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/github/GitHubAuthenticationProvider.java +++ b/langstream-api-gateway-auth/langstream-github-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/github/GitHubAuthenticationProvider.java @@ -18,7 +18,7 @@ import ai.langstream.api.gateway.GatewayAuthenticationProvider; import ai.langstream.api.gateway.GatewayAuthenticationResult; import ai.langstream.api.gateway.GatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -30,8 +30,6 @@ @Slf4j public class GitHubAuthenticationProvider implements GatewayAuthenticationProvider { - private static final ObjectMapper mapper = new ObjectMapper(); - private String clientId; private final HttpClient client = HttpClient.newHttpClient(); @@ -43,7 +41,9 @@ public String type() { @Override public void initialize(Map configuration) { final GitHubAuthenticationProviderConfiguration config = - mapper.convertValue(configuration, GitHubAuthenticationProviderConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + configuration, GitHubAuthenticationProviderConfiguration.class); clientId = config.getClientId(); log.info("Initialized GitHub authentication with configuration: {}", config); } @@ -80,7 +80,8 @@ public GatewayAuthenticationResult authenticate(GatewayRequestContext context) { log.info("X-OAuth-Client-Id: {}", responseClientId); log.info("Required: X-OAuth-Client-Id: {}", clientId); - Map result = mapper.readValue(body, Map.class); + Map result = + ObjectMapperFactory.getDefaultMapper().readValue(body, Map.class); if (log.isDebugEnabled()) { response.headers().map().forEach((k, v) -> log.debug("Header {}: {}", k, v)); } diff --git a/langstream-api-gateway-auth/langstream-google-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/google/GoogleAuthenticationProvider.java b/langstream-api-gateway-auth/langstream-google-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/google/GoogleAuthenticationProvider.java index 92048af34..75fd52da3 100644 --- a/langstream-api-gateway-auth/langstream-google-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/google/GoogleAuthenticationProvider.java +++ b/langstream-api-gateway-auth/langstream-google-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/google/GoogleAuthenticationProvider.java @@ -18,7 +18,7 @@ import ai.langstream.api.gateway.GatewayAuthenticationProvider; import ai.langstream.api.gateway.GatewayAuthenticationResult; import ai.langstream.api.gateway.GatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken; import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier; import com.google.api.client.http.javanet.NetHttpTransport; @@ -33,7 +33,6 @@ public class GoogleAuthenticationProvider implements GatewayAuthenticationProvid protected static final String FIELD_EMAIL = "email"; protected static final String FIELD_NAME = "name"; protected static final String FIELD_LOCALE = "locale"; - private static final ObjectMapper mapper = new ObjectMapper(); private GoogleIdTokenVerifier verifier; @@ -45,7 +44,9 @@ public String type() { @Override public void initialize(Map configuration) { final GoogleAuthenticationProviderConfiguration config = - mapper.convertValue(configuration, GoogleAuthenticationProviderConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + configuration, GoogleAuthenticationProviderConfiguration.class); final String clientId = config.getClientId(); if (clientId == null || clientId.isBlank()) { throw new IllegalArgumentException("clientId is required for Google Authentication."); diff --git a/langstream-api-gateway-auth/langstream-http-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/HttpAuthenticationProvider.java b/langstream-api-gateway-auth/langstream-http-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/HttpAuthenticationProvider.java index 5a215638d..7713147a3 100644 --- a/langstream-api-gateway-auth/langstream-http-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/HttpAuthenticationProvider.java +++ b/langstream-api-gateway-auth/langstream-http-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/HttpAuthenticationProvider.java @@ -18,7 +18,7 @@ import ai.langstream.api.gateway.GatewayAuthenticationProvider; import ai.langstream.api.gateway.GatewayAuthenticationResult; import ai.langstream.api.gateway.GatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -31,7 +31,6 @@ @Slf4j public class HttpAuthenticationProvider implements GatewayAuthenticationProvider { - private static final ObjectMapper mapper = new ObjectMapper(); private HttpAuthenticationProviderConfiguration httpConfiguration; private HttpClient httpClient; @@ -44,7 +43,8 @@ public String type() { @SneakyThrows public void initialize(Map configuration) { httpConfiguration = - mapper.convertValue(configuration, HttpAuthenticationProviderConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, HttpAuthenticationProviderConfiguration.class); httpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(30)) diff --git a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/pom.xml b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/pom.xml index 05c6da9d3..3fae96b2e 100644 --- a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/pom.xml +++ b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/pom.xml @@ -63,14 +63,10 @@ test - com.github.tomakehurst + org.wiremock wiremock test - - com.github.tomakehurst - wiremock - com.fasterxml.jackson.dataformat jackson-dataformat-yaml diff --git a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProvider.java b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProvider.java index b21670c95..e0603025c 100644 --- a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProvider.java +++ b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/main/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProvider.java @@ -18,10 +18,10 @@ import ai.langstream.api.gateway.GatewayAuthenticationProvider; import ai.langstream.api.gateway.GatewayAuthenticationResult; import ai.langstream.api.gateway.GatewayRequestContext; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.auth.common.store.RevokedTokensAwareAuthenticationProvider; import ai.langstream.auth.jwt.AuthenticationProviderToken; import ai.langstream.auth.jwt.JwtProperties; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import lombok.SneakyThrows; @@ -31,7 +31,6 @@ public class JwtAuthenticationProvider extends RevokedTokensAwareAuthenticationProvider implements GatewayAuthenticationProvider { - private static final ObjectMapper mapper = new ObjectMapper(); private AuthenticationProviderToken authenticationProviderToken; private List adminRoles; @@ -44,7 +43,8 @@ public String type() { @SneakyThrows public void initialize(Map configuration) { final JwtAuthenticationProviderConfiguration tokenProperties = - mapper.convertValue(configuration, JwtAuthenticationProviderConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, JwtAuthenticationProviderConfiguration.class); if (tokenProperties.adminRoles() != null) { this.adminRoles = tokenProperties.adminRoles(); diff --git a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/test/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProviderConfigurationTest.java b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/test/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProviderConfigurationTest.java index b8d0593f6..69e067fc9 100644 --- a/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/test/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProviderConfigurationTest.java +++ b/langstream-api-gateway-auth/langstream-jwt-api-gateway-auth/src/test/java/ai/langstream/apigateway/auth/impl/jwt/admin/JwtAuthenticationProviderConfigurationTest.java @@ -17,8 +17,8 @@ import static org.junit.jupiter.api.Assertions.*; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.util.List; import java.util.Map; import lombok.SneakyThrows; @@ -26,7 +26,8 @@ class JwtAuthenticationProviderConfigurationTest { - protected static final ObjectMapper yamlConfigReader = new ObjectMapper(new YAMLFactory()); + protected static final ObjectMapper yamlConfigReader = + ObjectMapperFactory.getDefaultYamlMapper(); @Test void parseCamelCase() { diff --git a/langstream-api-gateway/pom.xml b/langstream-api-gateway/pom.xml index 63749f6d5..fc18078fa 100644 --- a/langstream-api-gateway/pom.xml +++ b/langstream-api-gateway/pom.xml @@ -140,7 +140,7 @@ test - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ConsumeGateway.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ConsumeGateway.java index 45a10e9d9..6f9da4a06 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ConsumeGateway.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ConsumeGateway.java @@ -31,7 +31,6 @@ import ai.langstream.apigateway.api.ConsumePushMessage; import ai.langstream.apigateway.util.StreamingClusterUtil; import ai.langstream.apigateway.websocket.AuthenticatedGatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; @@ -51,7 +50,6 @@ @Slf4j public class ConsumeGateway implements AutoCloseable { - protected static final ObjectMapper mapper = new ObjectMapper(); private final TopicConnectionsRuntimeRegistry topicConnectionsRuntimeRegistry; private final ClusterRuntimeRegistry clusterRuntimeRegistry; private final TopicConnectionsRuntimeCache topicConnectionsRuntimeCache; diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ProduceGateway.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ProduceGateway.java index e713cea94..4cc0e1673 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ProduceGateway.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/gateways/ProduceGateway.java @@ -27,6 +27,7 @@ import ai.langstream.api.runtime.ClusterRuntimeRegistry; import ai.langstream.api.runtime.StreamingClusterRuntime; import ai.langstream.api.runtime.Topic; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.api.ProducePayload; import ai.langstream.apigateway.api.ProduceRequest; import ai.langstream.apigateway.api.ProduceResponse; @@ -34,7 +35,6 @@ import ai.langstream.apigateway.websocket.AuthenticatedGatewayRequestContext; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -45,8 +45,7 @@ @Slf4j public class ProduceGateway implements AutoCloseable { - protected static final ObjectMapper mapper = - new ObjectMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + protected static final ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); @Getter public static class ProduceException extends Exception { diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/http/GatewayResource.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/http/GatewayResource.java index 2065f8c47..80cef78a3 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/http/GatewayResource.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/http/GatewayResource.java @@ -23,6 +23,7 @@ import ai.langstream.api.runner.code.SystemHeaders; import ai.langstream.api.runtime.ClusterRuntimeRegistry; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.api.ConsumePushMessage; import ai.langstream.apigateway.api.ProducePayload; import ai.langstream.apigateway.api.ProduceRequest; @@ -31,7 +32,6 @@ import ai.langstream.apigateway.metrics.ApiGatewayMetrics; import ai.langstream.apigateway.runner.TopicConnectionsRuntimeProviderBean; import ai.langstream.apigateway.websocket.AuthenticatedGatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.PreDestroy; import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.constraints.NotBlank; @@ -82,7 +82,6 @@ public class GatewayResource { "/service/{tenant}/{application}/{gateway}/**"; protected static final String SERVICE_REQUEST_ID_HEADER = SystemHeaders.SERVICE_REQUEST_ID_HEADER.getKey(); - protected static final ObjectMapper mapper = new ObjectMapper(); private final TopicConnectionsRuntimeProviderBean topicConnectionsRuntimeRegistryProvider; private final ClusterRuntimeRegistry clusterRuntimeRegistry; private final TopicProducerCache topicProducerCache; @@ -405,7 +404,8 @@ private static ResponseEntity buildResponseFromReceivedRecord( } } try { - String asString = mapper.writeValueAsString(consumePushMessage); + String asString = + ObjectMapperFactory.getDefaultMapper().writeValueAsString(consumePushMessage); return ResponseEntity.ok(asString); } catch (Exception e) { throw new RuntimeException(e); diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/metrics/ApiGatewayMetricsProvider.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/metrics/ApiGatewayMetricsProvider.java index 9aead723b..d1ce4dabf 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/metrics/ApiGatewayMetricsProvider.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/metrics/ApiGatewayMetricsProvider.java @@ -24,7 +24,6 @@ public class ApiGatewayMetricsProvider { @Bean(destroyMethod = "close") public ApiGatewayMetrics apiGatewayMetrics() { - System.out.println("CALL ApiGatewayMetricsProvider"); return new ApiGatewayMetrics(Metrics.globalRegistry); } } diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/util/StreamingClusterUtil.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/util/StreamingClusterUtil.java index 1daaf3829..51de2f901 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/util/StreamingClusterUtil.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/util/StreamingClusterUtil.java @@ -16,17 +16,16 @@ package ai.langstream.apigateway.util; import ai.langstream.api.model.StreamingCluster; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Pair; public class StreamingClusterUtil { - private static final ObjectMapper mapper = new ObjectMapper(); - @SneakyThrows public static String asKey(StreamingCluster streamingCluster) { - return mapper.writeValueAsString( - Pair.of(streamingCluster.type(), streamingCluster.configuration())); + return ObjectMapperFactory.getDefaultMapper() + .writeValueAsString( + Pair.of(streamingCluster.type(), streamingCluster.configuration())); } } diff --git a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/websocket/handlers/AbstractHandler.java b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/websocket/handlers/AbstractHandler.java index 5c60bd3dc..3f3cc6460 100644 --- a/langstream-api-gateway/src/main/java/ai/langstream/apigateway/websocket/handlers/AbstractHandler.java +++ b/langstream-api-gateway/src/main/java/ai/langstream/apigateway/websocket/handlers/AbstractHandler.java @@ -31,11 +31,11 @@ import ai.langstream.api.runtime.StreamingClusterRuntime; import ai.langstream.api.runtime.Topic; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.api.ProduceResponse; import ai.langstream.apigateway.gateways.*; import ai.langstream.apigateway.util.StreamingClusterUtil; import ai.langstream.apigateway.websocket.AuthenticatedGatewayRequestContext; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.List; import java.util.Map; @@ -51,7 +51,6 @@ @Slf4j public abstract class AbstractHandler extends TextWebSocketHandler { - protected static final ObjectMapper mapper = new ObjectMapper(); protected static final String ATTRIBUTE_PRODUCE_GATEWAY = "__produce_gateway"; protected static final String ATTRIBUTE_CONSUME_GATEWAY = "__consume_gateway"; protected final TopicConnectionsRuntimeRegistry topicConnectionsRuntimeRegistry; @@ -259,11 +258,16 @@ public TopicProducer get() { .category(EventRecord.Categories.Gateway) .type(type.toString()) .timestamp(System.currentTimeMillis()) - .source(mapper.convertValue(source, Map.class)) - .data(mapper.convertValue(data, Map.class)) + .source( + ObjectMapperFactory.getDefaultMapper() + .convertValue(source, Map.class)) + .data( + ObjectMapperFactory.getDefaultMapper() + .convertValue(data, Map.class)) .build(); - final String recordValue = mapper.writeValueAsString(event); + final String recordValue = + ObjectMapperFactory.getDefaultMapper().writeValueAsString(event); final SimpleRecord record = SimpleRecord.builder().value(recordValue).build(); eventsProducer.write(record).get(); @@ -281,7 +285,8 @@ protected void startReadingMessages(WebSocketSession webSocketSession, Executor () -> !webSocketSession.isOpen(), message -> { try { - String jsonStringMessage = mapper.writeValueAsString(message); + String jsonStringMessage = + ObjectMapperFactory.getDefaultMapper().writeValueAsString(message); webSocketSession.sendMessage(new TextMessage(jsonStringMessage)); } catch (IOException ex) { throw new RuntimeException(ex); @@ -344,7 +349,9 @@ protected void produceMessage( (ProduceGateway) context.attributes().get(ATTRIBUTE_PRODUCE_GATEWAY); produceGateway.produceMessage(message.getPayload(), payloadSchema); webSocketSession.sendMessage( - new TextMessage(mapper.writeValueAsString(ProduceResponse.OK))); + new TextMessage( + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(ProduceResponse.OK))); } catch (ProduceGateway.ProduceException exception) { sendResponse(webSocketSession, exception.getStatus(), exception.getMessage()); } @@ -380,6 +387,8 @@ private static void sendResponse( WebSocketSession webSocketSession, ProduceResponse.Status status, String reason) throws IOException { webSocketSession.sendMessage( - new TextMessage(mapper.writeValueAsString(new ProduceResponse(status, reason)))); + new TextMessage( + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(new ProduceResponse(status, reason)))); } } diff --git a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/GatewayResourceTest.java b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/GatewayResourceTest.java index e38561490..e90731034 100644 --- a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/GatewayResourceTest.java +++ b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/GatewayResourceTest.java @@ -35,6 +35,7 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.ApiGatewayTestUtil; import ai.langstream.apigateway.api.ConsumePushMessage; import ai.langstream.apigateway.config.GatewayTestAuthenticationProperties; @@ -42,7 +43,6 @@ import ai.langstream.impl.deploy.ApplicationDeployer; import ai.langstream.impl.parser.ModelBuilder; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -92,7 +92,7 @@ abstract class GatewayResourceTest { log.info("Agents directory is {}", agentsDirectory); } - protected static final ObjectMapper MAPPER = new ObjectMapper(); + protected static final ObjectMapper MAPPER = ObjectMapperFactory.getDefaultMapper(); static List topics; ExecutorService futuresExecutor; @@ -172,7 +172,7 @@ private static Application buildApp(String instanceYaml) throws Exception { ModelBuilder.buildApplicationInstance( Map.of( "module.yaml", - new ObjectMapper(new YAMLFactory()) + ObjectMapperFactory.getDefaultYamlMapper() .writeValueAsString(module)), instanceYaml, null) @@ -274,7 +274,8 @@ void produceJsonAndExpectBadRequest( HttpResponse response = sendRequest(url, content, headers); assertEquals(400, response.statusCode()); log.info("Response body: {}", response.body()); - final Map map = new ObjectMapper().readValue(response.body(), Map.class); + final Map map = + ObjectMapperFactory.getDefaultMapper().readValue(response.body(), Map.class); String detail = (String) map.get("detail"); assertTrue(detail.contains(errorMessage)); } @@ -1021,8 +1022,6 @@ private void assertMessageContent(MsgRecord expected, String actual) { final MsgRecord actualMsgRecord = new MsgRecord(consume.record().key(), consume.record().value(), headers); - System.out.println("type: " + actualMsgRecord.value().getClass()); - assertEquals(expected.value(), actualMsgRecord.value()); assertEquals(expected, actualMsgRecord); } diff --git a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/ServiceAgentGatewayResourceTest.java b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/ServiceAgentGatewayResourceTest.java index aaed5ea3f..73205ae89 100644 --- a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/ServiceAgentGatewayResourceTest.java +++ b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/http/ServiceAgentGatewayResourceTest.java @@ -27,9 +27,8 @@ import ai.langstream.api.model.Gateways; import ai.langstream.api.model.StoredApplication; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.parser.ModelBuilder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -135,7 +134,7 @@ private static Application buildApp(String instanceYaml) throws Exception { ModelBuilder.buildApplicationInstance( Map.of( "module.yaml", - new ObjectMapper(new YAMLFactory()) + ObjectMapperFactory.getDefaultYamlMapper() .writeValueAsString(module)), instanceYaml, null) diff --git a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/websocket/handlers/ProduceConsumeHandlerTest.java b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/websocket/handlers/ProduceConsumeHandlerTest.java index 96f94b745..fbc9fbbc9 100644 --- a/langstream-api-gateway/src/test/java/ai/langstream/apigateway/websocket/handlers/ProduceConsumeHandlerTest.java +++ b/langstream-api-gateway/src/test/java/ai/langstream/apigateway/websocket/handlers/ProduceConsumeHandlerTest.java @@ -39,6 +39,7 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.apigateway.api.ConsumePushMessage; import ai.langstream.apigateway.api.ProduceRequest; import ai.langstream.apigateway.api.ProduceResponse; @@ -48,7 +49,6 @@ import ai.langstream.impl.parser.ModelBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -102,7 +102,7 @@ abstract class ProduceConsumeHandlerTest { log.info("Agents directory is {}", agentsDirectory); } - protected static final ObjectMapper MAPPER = new ObjectMapper(); + protected static final ObjectMapper MAPPER = ObjectMapperFactory.getDefaultMapper(); static List topics; static Gateways testGateways; @@ -168,7 +168,7 @@ private static Application buildApp(String instanceYaml) throws Exception { ModelBuilder.buildApplicationInstance( Map.of( "module.yaml", - new ObjectMapper(new YAMLFactory()) + ObjectMapperFactory.getDefaultYamlMapper() .writeValueAsString(module)), instanceYaml, null) @@ -640,14 +640,22 @@ private void assertMessagesContent(List expected, List actual actual.stream() .map( string -> { + Map asMap; try { + asMap = MAPPER.readValue(string, Map.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + if (asMap.containsKey("record")) { ConsumePushMessage consume = - MAPPER.readValue(string, ConsumePushMessage.class); + MAPPER.convertValue( + asMap, ConsumePushMessage.class); return new MsgRecord( consume.record().key(), consume.record().value(), consume.record().headers()); - } catch (JsonProcessingException e) { + } else { + log.info("Skipping message: {}", string); return null; } }) @@ -1187,7 +1195,9 @@ public void onClose(CloseReason cr) { @SneakyThrows private ProduceResponse connectAndProduce(URI connectTo, ProduceRequest produceRequest) { - return connectAndProduce(connectTo, new ObjectMapper().writeValueAsString(produceRequest)); + return connectAndProduce( + connectTo, + ObjectMapperFactory.getDefaultMapper().writeValueAsString(produceRequest)); } @SneakyThrows @@ -1208,7 +1218,7 @@ public void onOpen(Session session) { @SneakyThrows public void onMessage(String msg) { response.set( - new ObjectMapper() + ObjectMapperFactory.getDefaultMapper() .readValue(msg, ProduceResponse.class)); countDownLatch.countDown(); } diff --git a/langstream-api/pom.xml b/langstream-api/pom.xml index 319f8203a..7d188aa23 100644 --- a/langstream-api/pom.xml +++ b/langstream-api/pom.xml @@ -47,17 +47,18 @@ com.fasterxml.jackson.core jackson-annotations - provided com.fasterxml.jackson.core jackson-databind - provided com.fasterxml.jackson.core jackson-core - provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml diff --git a/langstream-api/src/main/java/ai/langstream/api/gateway/GatewayAuthenticationProviderRegistry.java b/langstream-api/src/main/java/ai/langstream/api/gateway/GatewayAuthenticationProviderRegistry.java index ac41c1d69..4f91ced05 100644 --- a/langstream-api/src/main/java/ai/langstream/api/gateway/GatewayAuthenticationProviderRegistry.java +++ b/langstream-api/src/main/java/ai/langstream/api/gateway/GatewayAuthenticationProviderRegistry.java @@ -15,7 +15,7 @@ */ package ai.langstream.api.gateway; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.Map; import java.util.Objects; import java.util.ServiceLoader; @@ -23,8 +23,6 @@ import lombok.SneakyThrows; public class GatewayAuthenticationProviderRegistry { - private static final ObjectMapper mapper = new ObjectMapper(); - record ProviderCacheKey(String type, String configString) {} private static final Map cachedProviders = @@ -37,7 +35,10 @@ public static GatewayAuthenticationProvider loadProvider( final Map finalConfiguration = configuration == null ? Map.of() : configuration; ProviderCacheKey key = - new ProviderCacheKey(type, mapper.writeValueAsString(finalConfiguration)); + new ProviderCacheKey( + type, + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(finalConfiguration)); return cachedProviders.computeIfAbsent( key, k -> { diff --git a/langstream-api/src/main/java/ai/langstream/api/util/ObjectMapperFactory.java b/langstream-api/src/main/java/ai/langstream/api/util/ObjectMapperFactory.java new file mode 100644 index 000000000..a3dd8a232 --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/util/ObjectMapperFactory.java @@ -0,0 +1,62 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ObjectMapperFactory { + + private static final ObjectMapper MAPPER = configureObjectMapper(new ObjectMapper()); + private static final ObjectMapper PRETTY_PRINT_MAPPER = + MAPPER.copy().configure(SerializationFeature.INDENT_OUTPUT, true); + + private static final ObjectMapper YAML_MAPPER = + configureObjectMapper( + new ObjectMapper( + YAMLFactory.builder() + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.SPLIT_LINES) + .build())); + + public static ObjectMapper getDefaultMapper() { + return MAPPER; + } + + public static ObjectMapper getPrettyPrintMapper() { + return PRETTY_PRINT_MAPPER; + } + + public static ObjectMapper getDefaultYamlMapper() { + return YAML_MAPPER; + } + + private static ObjectMapper configureObjectMapper(ObjectMapper mapper) { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.configure(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION, true); + mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + return mapper; + } +} diff --git a/langstream-api/src/test/resources/logback-test.xml b/langstream-api/src/test/resources/logback-test.xml new file mode 100644 index 000000000..fdfe741f8 --- /dev/null +++ b/langstream-api/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + + diff --git a/langstream-auth-jwt/pom.xml b/langstream-auth-jwt/pom.xml index 7f85cbabe..99c5d6fc0 100644 --- a/langstream-auth-jwt/pom.xml +++ b/langstream-auth-jwt/pom.xml @@ -66,7 +66,7 @@ jjwt-jackson - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-cli/pom.xml b/langstream-cli/pom.xml index 57e813d1e..f1961a71c 100644 --- a/langstream-cli/pom.xml +++ b/langstream-cli/pom.xml @@ -28,7 +28,6 @@ LangStream - CLI ${project.build.directory} - 3.0.0-beta-10 linux/amd64 1.10 1.4 @@ -47,7 +46,7 @@ ${project.version} - com.github.tomakehurst + org.wiremock wiremock ${wiremock.version} test diff --git a/langstream-codestorage-providers/langstream-codestorage-azure-blob-storage/src/main/java/ai/langstream/impl/storage/k8s/codestorage/AzureBlobCodeStorage.java b/langstream-codestorage-providers/langstream-codestorage-azure-blob-storage/src/main/java/ai/langstream/impl/storage/k8s/codestorage/AzureBlobCodeStorage.java index ce7725a97..f19b79bcb 100644 --- a/langstream-codestorage-providers/langstream-codestorage-azure-blob-storage/src/main/java/ai/langstream/impl/storage/k8s/codestorage/AzureBlobCodeStorage.java +++ b/langstream-codestorage-providers/langstream-codestorage-azure-blob-storage/src/main/java/ai/langstream/impl/storage/k8s/codestorage/AzureBlobCodeStorage.java @@ -20,6 +20,7 @@ import ai.langstream.api.codestorage.CodeStorageException; import ai.langstream.api.codestorage.LocalZipFileArchiveFile; import ai.langstream.api.codestorage.UploadableCodeArchive; +import ai.langstream.api.util.ObjectMapperFactory; import com.azure.core.http.rest.Response; import com.azure.core.util.BinaryData; import com.azure.core.util.Context; @@ -30,7 +31,6 @@ import com.azure.storage.blob.models.BlockBlobItem; import com.azure.storage.blob.options.BlobParallelUploadOptions; import com.azure.storage.common.StorageSharedKeyCredential; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -46,7 +46,6 @@ @Slf4j public class AzureBlobCodeStorage implements CodeStorage { - private static final ObjectMapper mapper = new ObjectMapper(); // metadata keys must not contain dashes since it's not supported by azure protected static final String OBJECT_METADATA_KEY_TENANT = "langstreamtenant"; protected static final String OBJECT_METADATA_KEY_APPLICATION = "langstreamapplication"; @@ -60,7 +59,8 @@ public class AzureBlobCodeStorage implements CodeStorage { @SneakyThrows public AzureBlobCodeStorage(Map configuration) { final AzureBlobCodeStorageConfiguration azureConfig = - mapper.convertValue(configuration, AzureBlobCodeStorageConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, AzureBlobCodeStorageConfiguration.class); if (azureConfig.getEndpoint() == null) { throw new IllegalArgumentException("Azure 'endpoint' must be provided"); diff --git a/langstream-codestorage-providers/langstream-codestorage-s3/src/main/java/ai/langstream/impl/storage/k8s/codestorage/S3CodeStorage.java b/langstream-codestorage-providers/langstream-codestorage-s3/src/main/java/ai/langstream/impl/storage/k8s/codestorage/S3CodeStorage.java index 450130584..249c018bb 100644 --- a/langstream-codestorage-providers/langstream-codestorage-s3/src/main/java/ai/langstream/impl/storage/k8s/codestorage/S3CodeStorage.java +++ b/langstream-codestorage-providers/langstream-codestorage-s3/src/main/java/ai/langstream/impl/storage/k8s/codestorage/S3CodeStorage.java @@ -20,7 +20,7 @@ import ai.langstream.api.codestorage.CodeStorageException; import ai.langstream.api.codestorage.LocalZipFileArchiveFile; import ai.langstream.api.codestorage.UploadableCodeArchive; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.minio.BucketExistsArgs; import io.minio.DownloadObjectArgs; import io.minio.MakeBucketArgs; @@ -48,7 +48,6 @@ @Slf4j public class S3CodeStorage implements CodeStorage { - private static final ObjectMapper mapper = new ObjectMapper(); protected static final String OBJECT_METADATA_KEY_TENANT = "langstream-tenant"; protected static final String OBJECT_METADATA_KEY_APPLICATION = "langstream-application"; protected static final String OBJECT_METADATA_KEY_VERSION = "langstream-version"; @@ -66,7 +65,8 @@ public class S3CodeStorage implements CodeStorage { @SneakyThrows public S3CodeStorage(Map configuration) { final S3CodeStorageConfiguration s3CodeStorageConfiguration = - mapper.convertValue(configuration, S3CodeStorageConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, S3CodeStorageConfiguration.class); bucketName = s3CodeStorageConfiguration.getBucketName(); final String endpoint = s3CodeStorageConfiguration.getEndpoint(); diff --git a/langstream-core/src/main/java/ai/langstream/impl/codestorage/LocalDiskCodeStorage.java b/langstream-core/src/main/java/ai/langstream/impl/codestorage/LocalDiskCodeStorage.java index 7857288dd..9e18a26b7 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/codestorage/LocalDiskCodeStorage.java +++ b/langstream-core/src/main/java/ai/langstream/impl/codestorage/LocalDiskCodeStorage.java @@ -20,8 +20,8 @@ import ai.langstream.api.codestorage.CodeStorageException; import ai.langstream.api.codestorage.LocalZipFileArchiveFile; import ai.langstream.api.codestorage.UploadableCodeArchive; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -36,8 +36,6 @@ @Slf4j public class LocalDiskCodeStorage implements CodeStorage { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final Path rootPath; private final Path metadataFilePath; @@ -58,7 +56,8 @@ private void initMetadata() { if (file.exists()) { // load the metadata List codeArchives = - MAPPER.readValue(file, new TypeReference<>() {}); + ObjectMapperFactory.getDefaultMapper() + .readValue(file, new TypeReference<>() {}); archives.addAll(codeArchives); } else { log.info("No metadata file found at {}, creating an empty file", file); @@ -73,7 +72,7 @@ private void persistMetadata() { } File file = metadataFilePath.toFile(); log.info("Loading {} archives metadata from {}", archives.size(), file); - MAPPER.writeValue(file, new ArrayList<>(archives)); + ObjectMapperFactory.getDefaultMapper().writeValue(file, new ArrayList<>(archives)); } @Override diff --git a/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java b/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java index f36e89377..3d929dbfb 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java +++ b/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java @@ -28,6 +28,7 @@ import ai.langstream.api.model.ResourcesSpec; import ai.langstream.api.model.StreamingCluster; import ai.langstream.api.model.TopicDefinition; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import java.io.IOException; @@ -45,11 +46,9 @@ @Slf4j public class ApplicationPlaceholderResolver { - private static final ObjectMapper mapper = - new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); - private static final ObjectMapper mapperForTemplates = - new ObjectMapper() + ObjectMapperFactory.getDefaultMapper() + .copy() .configure( SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); // help with tests (also of applications using LS) @@ -63,7 +62,8 @@ public static Application resolvePlaceholders(Application instance) { final Map context = createContext(instance); if (log.isDebugEnabled()) { log.debug( - "Resolving placeholders with context:\n{}", mapper.writeValueAsString(context)); + "Resolving placeholders with context:\n{}", + ObjectMapperFactory.getPrettyPrintMapper().writeValueAsString(context)); } if (log.isDebugEnabled()) { log.debug("Resolve context: {}", context); @@ -418,10 +418,16 @@ private static Object resolveProperty(Object context, String property) { } private static Application deepCopy(Application instance) throws IOException { - return mapper.readValue(mapper.writeValueAsBytes(instance), Application.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(instance), + Application.class); } private static Map deepCopy(Map context) throws IOException { - return mapper.readValue(mapper.writeValueAsBytes(context), Map.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(context), + Map.class); } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/deploy/ApplicationDeployer.java b/langstream-core/src/main/java/ai/langstream/impl/deploy/ApplicationDeployer.java index 165fdfe39..d66c118e4 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/deploy/ApplicationDeployer.java +++ b/langstream-core/src/main/java/ai/langstream/impl/deploy/ApplicationDeployer.java @@ -27,9 +27,9 @@ import ai.langstream.api.runner.code.*; import ai.langstream.api.runner.topics.*; import ai.langstream.api.runtime.*; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.common.ApplicationPlaceholderResolver; import ai.langstream.impl.common.DefaultAgentNode; -import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -46,8 +46,6 @@ @Slf4j public final class ApplicationDeployer implements AutoCloseable { - static final ObjectMapper MAPPER = new ObjectMapper(); - private ClusterRuntimeRegistry registry; private PluginsRegistry pluginsRegistry; @Builder.Default private DeployContext deployContext = DeployContext.NO_DEPLOY_CONTEXT; @@ -109,7 +107,8 @@ private void setupAssets( try { for (AssetNode assetNode : executionPlan.getAssets()) { AssetDefinition asset = - MAPPER.convertValue(assetNode.config(), AssetDefinition.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(assetNode.config(), AssetDefinition.class); try { boolean created = setupAsset(asset, assetManagerRegistry); if (created) { @@ -240,11 +239,13 @@ private static SimpleRecord createAssetEventRecord( .category(EventRecord.Categories.Asset) .type(eventType) .timestamp(System.currentTimeMillis()) - .source(MAPPER.convertValue(source, Map.class)) + .source( + ObjectMapperFactory.getDefaultMapper() + .convertValue(source, Map.class)) .data(data) .build(); - final String recordValue = MAPPER.writeValueAsString(event); + final String recordValue = ObjectMapperFactory.getDefaultMapper().writeValueAsString(event); return SimpleRecord.builder().value(recordValue).build(); } @@ -408,7 +409,8 @@ private void cleanupAssets( try { for (AssetNode assetNode : executionPlan.getAssets()) { AssetDefinition asset = - MAPPER.convertValue(assetNode.config(), AssetDefinition.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(assetNode.config(), AssetDefinition.class); try { boolean deleted = cleanupAsset(asset); if (deleted) { diff --git a/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java b/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java index 0cc2c1607..c6a75098e 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java +++ b/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java @@ -23,11 +23,11 @@ import ai.langstream.api.model.*; import ai.langstream.api.model.Module; import ai.langstream.api.runtime.AgentNode; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.uti.FileUtils; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -59,7 +59,7 @@ @Slf4j public class ModelBuilder { - static final ObjectMapper yamlParser = new ObjectMapper(new YAMLFactory()); + static final ObjectMapper yamlParser = ObjectMapperFactory.getDefaultYamlMapper(); public static ApplicationWithPackageInfo buildApplicationInstanceFromArchetype( Path archetypePath, Map applicationParameters) throws Exception { @@ -380,9 +380,7 @@ public static ApplicationWithPackageInfo buildApplicationInstance( if (defaultsHolder.globals != null && !defaultsHolder.globals.isEmpty()) { applicationWithPackageInfo.hasInstanceDefinition = true; parseInstance( - "instance:", - applicationWithPackageInfo.getApplication(), - defaultsHolder.globals); + "{}", applicationWithPackageInfo.getApplication(), defaultsHolder.globals); } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java index 7388bfc03..20b9344b8 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java @@ -23,8 +23,8 @@ import ai.langstream.api.doc.ResourceConfig; import ai.langstream.api.model.Resource; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -47,7 +47,6 @@ public class AIProvidersResourceProvider extends AbstractResourceProvider { VERTEX_CONFIGURATION, BEDROCK_CONFIGURATION, OLLAMA_CONFIGURATION); - protected static final ObjectMapper MAPPER = new ObjectMapper(); public AIProvidersResourceProvider() { super(SUPPORTED_TYPES); @@ -84,7 +83,7 @@ private void validateVertexConfigurationResource(Resource resource) { } if (!serviceAccountJson.isEmpty()) { try { - MAPPER.readValue(serviceAccountJson, Map.class); + ObjectMapperFactory.getDefaultMapper().readValue(serviceAccountJson, Map.class); } catch (Exception e) { throw new IllegalArgumentException( "Invalid JSON for field serviceAccountJson in " + describe(resource).get(), diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java index 0d4482666..871e8a766 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java @@ -22,8 +22,8 @@ import ai.langstream.api.runtime.ComputeClusterRuntime; import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.api.runtime.ResourceNodeProvider; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.uti.ClassConfigValidator; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.LinkedHashMap; import java.util.Map; import lombok.AllArgsConstructor; @@ -32,7 +32,6 @@ @AllArgsConstructor public class BaseDataSourceResourceProvider implements ResourceNodeProvider { - protected static final ObjectMapper MAPPER = new ObjectMapper(); private final String resourceType; private final Map supportedServices; @@ -93,7 +92,9 @@ public Map generateSupportedTypesDocumentati @SneakyThrows private static ResourceConfigurationModel deepCopy(ResourceConfigurationModel instance) { - return MAPPER.readValue( - MAPPER.writeValueAsBytes(instance), ResourceConfigurationModel.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(instance), + ResourceConfigurationModel.class); } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/storage/GlobalMetadataStoreManager.java b/langstream-core/src/main/java/ai/langstream/impl/storage/GlobalMetadataStoreManager.java index d1c2fb90f..a0f296f22 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/storage/GlobalMetadataStoreManager.java +++ b/langstream-core/src/main/java/ai/langstream/impl/storage/GlobalMetadataStoreManager.java @@ -17,12 +17,11 @@ import ai.langstream.api.storage.ApplicationStore; import ai.langstream.api.storage.GlobalMetadataStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.tenant.CreateTenantRequest; import ai.langstream.api.webservice.tenant.TenantConfiguration; import ai.langstream.api.webservice.tenant.UpdateTenantRequest; import ai.langstream.impl.storage.tenants.TenantException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; import java.util.stream.Collectors; import lombok.AllArgsConstructor; @@ -33,9 +32,6 @@ @Slf4j public class GlobalMetadataStoreManager { - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - public static class TenantNotFoundException extends Exception { public TenantNotFoundException(String message) { @@ -43,7 +39,6 @@ public TenantNotFoundException(String message) { } } - protected static final String TENANT_KEY_PREFIX = "t-"; private final GlobalMetadataStore globalMetadataStore; private final ApplicationStore applicationStore; @@ -74,7 +69,8 @@ public void createTenant(String tenant, CreateTenantRequest request) throws Tena final TenantConfiguration configuration = createConfiguration(tenant, request); log.info("Creating tenant {} with configuration {}", tenant, configuration); - globalMetadataStore.put(key, mapper.writeValueAsString(configuration)); + globalMetadataStore.put( + key, ObjectMapperFactory.getDefaultMapper().writeValueAsString(configuration)); applicationStore.onTenantCreated(tenant); } @@ -100,12 +96,14 @@ public void updateTenant(String tenant, UpdateTenantRequest request) throws Tena throw new TenantException( "Tenant " + tenant + " not found", TenantException.Type.NotFound); } - final TenantConfiguration mergeConfig = mapper.readValue(before, TenantConfiguration.class); + final TenantConfiguration mergeConfig = + ObjectMapperFactory.getDefaultMapper().readValue(before, TenantConfiguration.class); mergeConfiguration(request, mergeConfig); log.info("Updating tenant {} with configuration {}", tenant, mergeConfig); - globalMetadataStore.put(key, mapper.writeValueAsString(mergeConfig)); + globalMetadataStore.put( + key, ObjectMapperFactory.getDefaultMapper().writeValueAsString(mergeConfig)); applicationStore.onTenantUpdated(tenant); } @@ -122,7 +120,9 @@ private void mergeConfiguration(UpdateTenantRequest request, TenantConfiguration public void putTenant(String tenant, TenantConfiguration tenantConfiguration) { final String key = keyedTenantName(tenant); final String before = globalMetadataStore.get(key); - globalMetadataStore.put(key, mapper.writeValueAsString(tenantConfiguration)); + globalMetadataStore.put( + key, + ObjectMapperFactory.getDefaultMapper().writeValueAsString(tenantConfiguration)); if (before != null) { applicationStore.onTenantUpdated(tenant); } else { @@ -141,7 +141,7 @@ public TenantConfiguration getTenant(String tenant) { @SneakyThrows private TenantConfiguration parseTenantConfiguration(String res) { - return mapper.readValue(res, TenantConfiguration.class); + return ObjectMapperFactory.getDefaultMapper().readValue(res, TenantConfiguration.class); } @SneakyThrows diff --git a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java index d3fb2052a..ed50d9b85 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java +++ b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java @@ -22,19 +22,15 @@ import ai.langstream.api.doc.AssetConfigurationModel; import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.doc.ConfigPropertyIgnore; -import ai.langstream.api.doc.ConfigPropertyModel; import ai.langstream.api.doc.ExtendedValidationType; import ai.langstream.api.doc.ResourceConfig; import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.model.AgentConfiguration; import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.model.Resource; -import com.fasterxml.jackson.annotation.JsonInclude; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.github.victools.jsonschema.generator.ConfigFunction; import com.github.victools.jsonschema.generator.FieldScope; @@ -59,21 +55,13 @@ @Slf4j public class ClassConfigValidator { - static final ObjectMapper jsonWriter = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, true) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - - static final ObjectMapper validatorMapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - static final Map agentModels = new ConcurrentHashMap<>(); static final Map resourceModels = new ConcurrentHashMap<>(); static final Map assetModels = new ConcurrentHashMap<>(); public static T convertValidatedConfiguration( Map agentConfiguration, Class clazz) { - return validatorMapper.convertValue(agentConfiguration, clazz); + return ObjectMapperFactory.getDefaultMapper().convertValue(agentConfiguration, clazz); } public static AgentConfigurationModel generateAgentModelFromClass(Class clazz) { @@ -230,7 +218,11 @@ private static void validateModelFromClass( Class modelClazz, Map asMap, boolean allowUnknownProperties) { - asMap = validatorMapper.readValue(validatorMapper.writeValueAsBytes(asMap), Map.class); + asMap = + ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(asMap), + Map.class); final AgentConfigurationModel agentConfigurationModel = generateAgentModelFromClass(modelClazz); @@ -358,17 +350,16 @@ public static class Prop { public static Map readPropertiesFromClass( Class clazz) { JsonNode jsonSchema = getJsonSchema(clazz); - final ObjectMapper mapper = - new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - final ParsedJsonSchema parsed = mapper.convertValue(jsonSchema, ParsedJsonSchema.class); + final ParsedJsonSchema parsed = + ObjectMapperFactory.getDefaultMapper() + .convertValue(jsonSchema, ParsedJsonSchema.class); Map props = new LinkedHashMap<>(); if (parsed.getProperties() != null) { for (Map.Entry schema : parsed.getProperties().entrySet()) { final ai.langstream.api.doc.ConfigPropertyModel parsedProp = - parseProp(mapper, schema.getValue()); + parseProp(schema.getValue()); if (parsedProp != null) { props.put(schema.getKey(), parsedProp); } @@ -379,7 +370,7 @@ public static Map readPropert @SneakyThrows private static ai.langstream.api.doc.ConfigPropertyModel parseProp( - ObjectMapper mapper, ParsedJsonSchema.Prop value) { + ParsedJsonSchema.Prop value) { ai.langstream.api.doc.ConfigPropertyModel newProp = new ai.langstream.api.doc.ConfigPropertyModel(); @@ -387,7 +378,8 @@ private static ai.langstream.api.doc.ConfigPropertyModel parseProp( final String jsonDesc = value.getDescription(); if (jsonDesc != null) { final ConfigPropertyModel property = - mapper.readValue(jsonDesc, ConfigPropertyModel.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(jsonDesc, ConfigPropertyModel.class); if (property.isIgnore()) { return null; } @@ -404,13 +396,12 @@ private static ai.langstream.api.doc.ConfigPropertyModel parseProp( if (value.getProperties() != null) { newProp.setProperties( value.getProperties().entrySet().stream() - .map(prop -> Pair.of(prop.getKey(), parseProp(mapper, prop.getValue()))) + .map(prop -> Pair.of(prop.getKey(), parseProp(prop.getValue()))) .filter(prop -> prop.getRight() != null) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight))); } if (value.getItems() != null) { - final ai.langstream.api.doc.ConfigPropertyModel items = - parseProp(mapper, value.getItems()); + final ai.langstream.api.doc.ConfigPropertyModel items = parseProp(value.getItems()); if (items != null) { newProp.setItems(items); @@ -459,7 +450,8 @@ public String apply(FieldScope fieldScope) { } } - return jsonWriter.writeValueAsString(model); + return ObjectMapperFactory.getPrettyPrintMapper() + .writeValueAsString(model); } }); SchemaGeneratorConfig config = configBuilder.build(); @@ -491,7 +483,11 @@ public static Map validateGenericClassAndApplyDefaults( boolean allowUnknownProperties) { final Map asMap = - validatorMapper.readValue(validatorMapper.writeValueAsBytes(inputValue), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper() + .writeValueAsBytes(inputValue), + Map.class); final Map properties = readPropertiesFromClass(modelClazz); diff --git a/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java b/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java index cd8e10a7d..ac2e8f2eb 100644 --- a/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java +++ b/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java @@ -289,7 +289,7 @@ void testParseRealDirectoryWithDefaults() throws Exception { String instanceContentWithEmptyGlobals = """ instance: - globals: + globals: {} """; applicationWithPackageInfo = ModelBuilder.buildApplicationInstance( diff --git a/langstream-core/src/test/java/ai/langstream/impl/uti/ClassConfigValidatorTest.java b/langstream-core/src/test/java/ai/langstream/impl/uti/ClassConfigValidatorTest.java index ebb43a4d1..29783b630 100644 --- a/langstream-core/src/test/java/ai/langstream/impl/uti/ClassConfigValidatorTest.java +++ b/langstream-core/src/test/java/ai/langstream/impl/uti/ClassConfigValidatorTest.java @@ -20,11 +20,10 @@ import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.doc.ConfigPropertyIgnore; import ai.langstream.api.doc.ExtendedValidationType; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import java.util.List; import java.util.Map; import lombok.Data; @@ -78,70 +77,66 @@ public void testParseClass() throws Exception { Assertions.assertEquals( """ - { - "name" : "my super agent", - "description" : "this agent is AWESOME!", + { + "name" : "my super agent", + "description" : "this agent is AWESOME!", + "properties" : { + "doNotIgnoreMe" : { + "required" : false + }, + "expression" : { + "required" : false, + "type" : "string", + "extendedValidationType" : "EL_EXPRESSION" + }, + "my-int" : { + "description" : "my description", + "required" : true, + "type" : "integer", + "defaultValue" : "11" + }, + "myclass2" : { + "required" : false, + "type" : "object", + "properties" : { + "inner" : { + "required" : false, + "type" : "string" + }, + "innerDoc" : { + "description" : "my description", + "required" : true, + "type" : "string", + "defaultValue" : "110.1a" + } + } + }, + "theList" : { + "required" : true, + "type" : "array", + "items" : { + "required" : true, + "type" : "object", "properties" : { - "doNotIgnoreMe" : { - "required" : false - }, - "expression" : { + "inner" : { "required" : false, - "type" : "string", - "extendedValidationType" : "EL_EXPRESSION" + "type" : "string" }, - "my-int" : { + "innerDoc" : { "description" : "my description", "required" : true, - "type" : "integer", - "defaultValue" : "11" - }, - "myclass2" : { - "required" : false, - "type" : "object", - "properties" : { - "innerDoc" : { - "description" : "my description", - "required" : true, - "type" : "string", - "defaultValue" : "110.1a" - }, - "inner" : { - "required" : false, - "type" : "string" - } - } - }, - "theList" : { - "required" : true, - "type" : "array", - "items" : { - "required" : true, - "type" : "object", - "properties" : { - "innerDoc" : { - "description" : "my description", - "required" : true, - "type" : "string", - "defaultValue" : "110.1a" - }, - "inner" : { - "required" : false, - "type" : "string" - } - } - } - }, - "theMap" : { - "required" : true, - "type" : "object" + "type" : "string", + "defaultValue" : "110.1a" } } - }""", - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, true) - .setSerializationInclusion( - com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL) - .writeValueAsString(model)); + } + }, + "theMap" : { + "required" : true, + "type" : "object" + } + } + }""", + ObjectMapperFactory.getPrettyPrintMapper().writeValueAsString(model)); } } diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java index 6032f541a..d678bf852 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.fail; import ai.langstream.api.model.StreamingCluster; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource; import ai.langstream.impl.parser.ModelBuilder; @@ -30,7 +31,6 @@ import ai.langstream.tests.util.kafka.RemoteKafkaProvider; import ai.langstream.tests.util.pulsar.LocalPulsarStandaloneProvider; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.api.model.HasMetadata; @@ -130,8 +130,8 @@ public class BaseEndToEndTest implements TestWatcher { public static final File TEST_LOGS_DIR = new File("target", "e2e-test-logs"); protected static final String TENANT_NAMESPACE_PREFIX = "ls-tenant-"; - protected static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - protected static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()); + protected static final ObjectMapper JSON_MAPPER = ObjectMapperFactory.getDefaultMapper(); + protected static final ObjectMapper YAML_MAPPER = ObjectMapperFactory.getDefaultYamlMapper(); protected static KubeCluster kubeCluster; protected static StreamingClusterProvider streamingClusterProvider; protected static StreamingCluster streamingCluster; diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/ConsumeGatewayMessage.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/ConsumeGatewayMessage.java index 60db9d8d4..0ca65517e 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/ConsumeGatewayMessage.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/ConsumeGatewayMessage.java @@ -15,6 +15,7 @@ */ package ai.langstream.tests.util; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; import lombok.AllArgsConstructor; @@ -26,7 +27,7 @@ @NoArgsConstructor @AllArgsConstructor public class ConsumeGatewayMessage { - protected static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + protected static final ObjectMapper JSON_MAPPER = ObjectMapperFactory.getDefaultMapper(); @SneakyThrows public static ConsumeGatewayMessage readValue(String line) { diff --git a/langstream-k8s-common/src/test/java/ai/langstream/impl/k8s/tests/KubeTestServer.java b/langstream-k8s-common/src/test/java/ai/langstream/impl/k8s/tests/KubeTestServer.java index 8656e2d4c..8c731b357 100644 --- a/langstream-k8s-common/src/test/java/ai/langstream/impl/k8s/tests/KubeTestServer.java +++ b/langstream-k8s-common/src/test/java/ai/langstream/impl/k8s/tests/KubeTestServer.java @@ -17,11 +17,10 @@ import static org.mockito.ArgumentMatchers.isNull; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource; import ai.langstream.impl.k8s.KubernetesClientFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.NamespacedKubernetesClient; @@ -131,15 +130,11 @@ public Map spyAgentCustomResources( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = - new ObjectMapper() - .enable( - SerializationFeature - .ORDER_MAP_ENTRIES_BY_KEYS); final AgentCustomResource agent = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - AgentCustomResource.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + AgentCustomResource.class); log.info("received patch request for agent {}", agentId); currentAgents.put(agentId, agent); return agent; @@ -190,11 +185,11 @@ public Map spyAgentCustomResourcesSecrets( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = new ObjectMapper(); final Secret secret = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - Secret.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + Secret.class); log.info( "received patch request secret for agent {}: {}", agentId, @@ -300,15 +295,11 @@ public Map spyApplicationCustomResources( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = - new ObjectMapper() - .enable( - SerializationFeature - .ORDER_MAP_ENTRIES_BY_KEYS); final ApplicationCustomResource app = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - ApplicationCustomResource.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + ApplicationCustomResource.class); log.info("received patch request for app {}", appId); currentApplications.put(appId, app); return app; diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/agents/AgentSpec.java b/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/agents/AgentSpec.java index 205b57c03..729f2f94b 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/agents/AgentSpec.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/agents/AgentSpec.java @@ -15,10 +15,9 @@ */ package ai.langstream.deployer.k8s.api.crds.agents; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.NamespacedSpec; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import lombok.Getter; import lombok.NoArgsConstructor; @@ -44,9 +43,6 @@ public record Options( boolean autoUpgradeAgentPodTemplate, long applicationSeed) {} - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private String agentId; private String applicationId; @Deprecated private String image; @@ -62,7 +58,9 @@ public record Options( private synchronized Options parseOptions() { if (parsedOptions == null) { if (options != null) { - parsedOptions = MAPPER.readValue(options, Options.class); + parsedOptions = + ObjectMapperFactory.getDefaultYamlMapper() + .readValue(options, Options.class); } } return parsedOptions; @@ -70,7 +68,7 @@ private synchronized Options parseOptions() { @SneakyThrows public void serializeAndSetOptions(Options options) { - this.options = MAPPER.writeValueAsString(options); + this.options = ObjectMapperFactory.getDefaultYamlMapper().writeValueAsString(options); } @JsonIgnore diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/apps/ApplicationSpec.java b/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/apps/ApplicationSpec.java index 6440a97ee..a9a31ef4f 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/apps/ApplicationSpec.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-api/src/main/java/ai/langstream/deployer/k8s/api/crds/apps/ApplicationSpec.java @@ -15,9 +15,8 @@ */ package ai.langstream.deployer.k8s.api.crds.apps; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.NamespacedSpec; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import lombok.NoArgsConstructor; import lombok.SneakyThrows; @@ -26,19 +25,18 @@ @NoArgsConstructor public class ApplicationSpec extends NamespacedSpec { - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @SneakyThrows public static String serializeApplication( SerializedApplicationInstance serializedApplicationInstance) { - return mapper.writeValueAsString(serializedApplicationInstance); + return ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(serializedApplicationInstance); } @SneakyThrows public static SerializedApplicationInstance deserializeApplication( String serializedApplicationInstance) { - return mapper.readValue(serializedApplicationInstance, SerializedApplicationInstance.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue(serializedApplicationInstance, SerializedApplicationInstance.class); } @SneakyThrows @@ -46,12 +44,13 @@ public static ApplicationSpecOptions deserializeOptions(String options) { if (options == null) { return new ApplicationSpecOptions(); } - return mapper.readValue(options, ApplicationSpecOptions.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue(options, ApplicationSpecOptions.class); } @SneakyThrows public static String serializeOptions(ApplicationSpecOptions options) { - return mapper.writeValueAsString(options); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(options); } @Deprecated private String image; diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java index 1cb27408f..3f083225a 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java @@ -19,6 +19,7 @@ import ai.langstream.api.model.ApplicationStatus; import ai.langstream.api.runner.code.AgentStatusResponse; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.CRDConstants; import ai.langstream.deployer.k8s.PodTemplate; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; @@ -30,8 +31,6 @@ import ai.langstream.runtime.api.agent.DownloadAgentCodeConfiguration; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.api.model.apps.StatefulSet; import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder; @@ -67,9 +66,6 @@ @Slf4j public class AgentResourcesFactory { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - protected static final String AGENT_SECRET_DATA_APP = "app-config"; public static Service generateHeadlessService(AgentCustomResource agentCustomResource) { @@ -798,7 +794,7 @@ private static List queryAgentStatus(String url, HttpClient .build(), HttpResponse.BodyHandlers.ofString()) .body(); - return MAPPER.readValue(body, new TypeReference<>() {}); + return ObjectMapperFactory.getDefaultMapper().readValue(body, new TypeReference<>() {}); } catch (IOException | InterruptedException e) { log.warn("Failed to query agent info from {}", url, e); return List.of(); diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/JSONAssertComparator.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/JSONAssertComparator.java index a75f12d63..0439b5653 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/JSONAssertComparator.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/JSONAssertComparator.java @@ -194,6 +194,7 @@ private static String getValueByDotNotation(final Map map, Strin } } - return currentMap.get(words.get(words.size() - 1)).toString(); + Object value = currentMap.get(words.get(words.size() - 1)); + return value == null ? "" : value.toString(); } } diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SerializationUtil.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SerializationUtil.java index aad23730c..f980d02b2 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SerializationUtil.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SerializationUtil.java @@ -15,45 +15,19 @@ */ package ai.langstream.deployer.k8s.util; -import com.fasterxml.jackson.annotation.JsonInclude; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import lombok.SneakyThrows; public class SerializationUtil { - private static final ObjectMapper mapper = - new ObjectMapper() - .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + private static final ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); - private static final ObjectMapper jsonPrettyPrint = - new ObjectMapper() - .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) - .configure(SerializationFeature.INDENT_OUTPUT, true) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - private static final ObjectMapper yamlMapper = - new ObjectMapper( - YAMLFactory.builder() - .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) - .disable(YAMLGenerator.Feature.SPLIT_LINES) - .build()) - .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); + private static final ObjectMapper jsonPrettyPrint = ObjectMapperFactory.getPrettyPrintMapper(); + private static final ObjectMapper yamlMapper = ObjectMapperFactory.getDefaultYamlMapper(); private SerializationUtil() {} - @SneakyThrows - public static T deepCloneObject(T object) { - if (object == null) { - return null; - } - return (T) mapper.readValue(mapper.writeValueAsString(object), object.getClass()); - } - @SneakyThrows public static String writeAsJson(Object object) { return mapper.writeValueAsString(object); @@ -69,11 +43,6 @@ public static T readJson(String string, Class objectClass) { return mapper.readValue(string, objectClass); } - @SneakyThrows - public static T convertValue(Object from, Class objectClass) { - return mapper.convertValue(from, objectClass); - } - @SneakyThrows public static byte[] writeAsJsonBytes(Object object) { return mapper.writeValueAsBytes(object); diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SpecDiffer.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SpecDiffer.java index 006174867..1d9ef8448 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SpecDiffer.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/SpecDiffer.java @@ -16,6 +16,7 @@ package ai.langstream.deployer.k8s.util; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -53,7 +54,8 @@ public List diffs() { private SpecDiffer() {} - public static JSONComparator.Result generateDiff(String expectedJson, String actualJson) { + private static JSONComparator.Result generateDiffFromStrings( + String expectedJson, String actualJson) { if (expectedJson == null && actualJson == null) { return JSONComparator.RESULT_EQUALS; } @@ -66,36 +68,11 @@ public static JSONComparator.Result generateDiff(String expectedJson, String act return new JSONAssertComparator().compare(expectedJson, actualJson); } - public static JSONComparator.Result generateDiff(Object expectedSpec, Object actualSpec) { - if (expectedSpec == null && actualSpec == null) { - return JSONComparator.RESULT_EQUALS; - } - if (expectedSpec == null) { - return EXPECTED_WAS_NULL_RESULT; - } - if (actualSpec == null) { - return ACTUAL_WAS_NULL_RESULT; - } - final String expectedStr = SerializationUtil.writeAsJson(expectedSpec); - final String actualStr = SerializationUtil.writeAsJson(actualSpec); - return generateDiff(expectedStr, actualStr); - } - - public static JSONComparator.Result generateDiff(Object expectedSpec, String actualJson) { - if (expectedSpec == null && actualJson == null) { - return JSONComparator.RESULT_EQUALS; - } - if (expectedSpec == null) { - return EXPECTED_WAS_NULL_RESULT; - } - if (actualJson == null) { - return ACTUAL_WAS_NULL_RESULT; - } - final String expectedStr = SerializationUtil.writeAsJson(expectedSpec); - return generateDiff(expectedStr, actualJson); - } - public static JSONComparator.Result generateDiff(String expectedJson, Object actualSpec) { + if (actualSpec instanceof String) { + throw new IllegalArgumentException( + "actualSpec should be a parsed object, not a string"); + } if (expectedJson == null && actualSpec == null) { return JSONComparator.RESULT_EQUALS; } @@ -106,7 +83,9 @@ public static JSONComparator.Result generateDiff(String expectedJson, Object act return EXPECTED_WAS_NULL_RESULT; } final String actualStr = SerializationUtil.writeAsJson(actualSpec); - return generateDiff(expectedJson, actualStr); + final String expectedJsonStrSameSerialization = + SerializationUtil.writeAsJson(SerializationUtil.readJson(expectedJson, Map.class)); + return generateDiffFromStrings(expectedJsonStrSameSerialization, actualStr); } public static void logDetailedSpecDiff(JSONComparator.Result diff) { diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactoryTest.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactoryTest.java index e2c04e4a7..44804087d 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactoryTest.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactoryTest.java @@ -136,7 +136,7 @@ void testStatefulsetAndService() { name: code-download initContainers: - args: - - "echo '{\\"codeDownloadPath\\":\\"/app-code-download\\",\\"tenant\\":\\"my-tenant\\",\\"applicationId\\":\\"the-'\\"'\\"'app\\",\\"codeArchiveId\\":null}' > /download-config/config" + - "echo '{\\"codeDownloadPath\\":\\"/app-code-download\\",\\"tenant\\":\\"my-tenant\\",\\"applicationId\\":\\"the-'\\"'\\"'app\\"}' > /download-config/config" command: - bash - -c diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/apps/AppResourcesFactoryTest.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/apps/AppResourcesFactoryTest.java index dd6d6b23e..32c2984b0 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/apps/AppResourcesFactoryTest.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/apps/AppResourcesFactoryTest.java @@ -245,7 +245,7 @@ void testDeployerJob() { controller: true name: test-'app data: - app-config: "{\\"applicationId\\":\\"test-'app\\",\\"tenant\\":\\"my-tenant\\",\\"application\\":\\"{app: true}\\",\\"codeStorageArchiveId\\":\\"iiii\\",\\"deployFlags\\":{\\"runtimeVersion\\":null,\\"autoUpgradeRuntimeImagePullPolicy\\":false,\\"autoUpgradeAgentResources\\":false,\\"autoUpgradeAgentPodTemplate\\":false,\\"seed\\":0}}" + app-config: "{\\"applicationId\\":\\"test-'app\\",\\"tenant\\":\\"my-tenant\\",\\"application\\":\\"{app: true}\\",\\"codeStorageArchiveId\\":\\"iiii\\",\\"deployFlags\\":{\\"autoUpgradeRuntimeImagePullPolicy\\":false,\\"autoUpgradeAgentResources\\":false,\\"autoUpgradeAgentPodTemplate\\":false,\\"seed\\":0}}" cluster-runtime-config: "{\\"config1\\":\\"value\\"}" """, SerializationUtil.writeAsYaml( @@ -588,7 +588,7 @@ void testNoUpdateFlags() { .getData() .get("app-config") .contains( - "\"deployFlags\":{\"runtimeVersion\":null,\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false,\"seed\":0}")); + "\"deployFlags\":{\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false,\"seed\":0}")); } @Test diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/SpecDifferTest.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/SpecDifferTest.java new file mode 100644 index 000000000..18c3d423d --- /dev/null +++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/SpecDifferTest.java @@ -0,0 +1,39 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.deployer.k8s.util; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +class SpecDifferTest { + + @Test + void testJsonOrderAndNulls() throws Exception { + String json1 = + """ + { + "c": 3, + "a": 1, + "b": null + }"""; + + JSONComparator.Result result = SpecDiffer.generateDiff(json1, Map.of("a", 1, "c", 3)); + SpecDiffer.logDetailedSpecDiff(result); + assertTrue(result.areEquals()); + } +} diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/ResolvedDeployerConfiguration.java b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/ResolvedDeployerConfiguration.java index 9390327dd..fd2bfe2ea 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/ResolvedDeployerConfiguration.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/ResolvedDeployerConfiguration.java @@ -15,14 +15,10 @@ */ package ai.langstream.deployer.k8s; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.agents.AgentResourceUnitConfiguration; import ai.langstream.deployer.k8s.util.SerializationUtil; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import jakarta.inject.Singleton; import java.util.Map; import lombok.Getter; @@ -33,15 +29,7 @@ @JBossLog public class ResolvedDeployerConfiguration { - private static final ObjectMapper yamlMapper = - new ObjectMapper( - YAMLFactory.builder() - .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) - .disable(YAMLGenerator.Feature.SPLIT_LINES) - .build()) - .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); + private static final ObjectMapper yamlMapper = ObjectMapperFactory.getDefaultYamlMapper(); @SneakyThrows public ResolvedDeployerConfiguration(DeployerConfiguration configuration) { diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/TenantsConfigurationReader.java b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/TenantsConfigurationReader.java index e9a87070a..0e410e943 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/TenantsConfigurationReader.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/TenantsConfigurationReader.java @@ -17,9 +17,8 @@ import ai.langstream.api.storage.GlobalMetadataStore; import ai.langstream.api.storage.GlobalMetadataStoreRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.tenant.TenantConfiguration; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.enterprise.context.ApplicationScoped; import lombok.SneakyThrows; import lombok.extern.jbosslog.JBossLog; @@ -27,8 +26,6 @@ @ApplicationScoped @JBossLog public class TenantsConfigurationReader { - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private final GlobalMetadataStore globalMetadataStore; public TenantsConfigurationReader(ResolvedDeployerConfiguration resolvedDeployerConfiguration) { @@ -58,6 +55,6 @@ public TenantConfiguration getTenantConfiguration(String tenant) { @SneakyThrows private TenantConfiguration parseTenantConfiguration(String res) { - return mapper.readValue(res, TenantConfiguration.class); + return ObjectMapperFactory.getDefaultMapper().readValue(res, TenantConfiguration.class); } } diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java index d60ea6c7b..fe8524604 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java @@ -16,6 +16,7 @@ package ai.langstream.deployer.k8s.controllers.apps; import ai.langstream.api.model.ApplicationLifecycleStatus; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationSpec; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationSpecOptions; @@ -27,9 +28,7 @@ import ai.langstream.deployer.k8s.util.KubeUtil; import ai.langstream.deployer.k8s.util.SerializationUtil; import ai.langstream.deployer.k8s.util.SpecDiffer; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.batch.v1.Job; @@ -56,10 +55,7 @@ public class AppController extends BaseController implements ErrorStatusHandler { private static final ObjectMapper lastAppliedJsonMapper = - new ObjectMapper() - .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + ObjectMapperFactory.getDefaultMapper(); protected static final Duration DEFAULT_RESCHEDULE_DURATION = Duration.ofSeconds(5); diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/test/java/ai/langstream/deployer/k8s/controllers/AppControllerIT.java b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/test/java/ai/langstream/deployer/k8s/controllers/AppControllerIT.java index c4f7a7c2c..eacad0153 100644 --- a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/test/java/ai/langstream/deployer/k8s/controllers/AppControllerIT.java +++ b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/test/java/ai/langstream/deployer/k8s/controllers/AppControllerIT.java @@ -400,7 +400,7 @@ private void checkSetupJob(Job job) { assertEquals(2, configMap.getData().size()); assertEquals( - "{\"applicationId\":\"my-app\",\"tenant\":\"my-tenant\",\"application\":\"{\\\"modules\\\": {}}\",\"codeArchiveId\":null}", + "{\"applicationId\":\"my-app\",\"tenant\":\"my-tenant\",\"application\":\"{\\\"modules\\\": {}}\"}", configMap.getData().get("app-config")); assertEquals("{}", configMap.getData().get("cluster-runtime-config")); @@ -483,7 +483,7 @@ private void checkDeployerJob(Job job, boolean cleanup) { assertEquals(2, configMap.getData().size()); assertEquals( - "{\"applicationId\":\"my-app\",\"tenant\":\"my-tenant\",\"application\":\"{\\\"modules\\\": {}}\",\"codeStorageArchiveId\":null,\"deployFlags\":{\"runtimeVersion\":null,\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false,\"seed\":0}}", + "{\"applicationId\":\"my-app\",\"tenant\":\"my-tenant\",\"application\":\"{\\\"modules\\\": {}}\",\"deployFlags\":{\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false,\"seed\":0}}", configMap.getData().get("app-config")); assertEquals("{}", configMap.getData().get("cluster-runtime-config")); diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntime.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntime.java index bf6a5017c..8ea4fc8ed 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntime.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntime.java @@ -20,6 +20,7 @@ import ai.langstream.api.model.ResourcesSpec; import ai.langstream.api.model.StreamingCluster; import ai.langstream.api.runtime.*; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.application.ApplicationCodeInfo; import ai.langstream.deployer.k8s.agents.AgentResourcesFactory; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; @@ -31,9 +32,7 @@ import ai.langstream.impl.common.DefaultAgentNode; import ai.langstream.impl.k8s.KubernetesClientFactory; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import java.security.MessageDigest; @@ -44,10 +43,7 @@ @Slf4j public class KubernetesClusterRuntime extends BasicClusterRuntime { - static final ObjectMapper mapper = - new ObjectMapper() - .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + static final ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); public static final String CLUSTER_TYPE = "kubernetes"; static final List OPTIMISERS = diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProvider.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProvider.java index c06cc974a..295cae424 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProvider.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProvider.java @@ -27,12 +27,12 @@ import ai.langstream.api.runtime.ComputeClusterRuntime; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.agents.AbstractComposableAgentProvider; import ai.langstream.impl.agents.ai.steps.QueryConfiguration; import ai.langstream.impl.uti.ClassConfigValidator; import ai.langstream.runtime.impl.k8s.KubernetesClusterRuntime; import ai.langstream.runtime.impl.k8s.agents.vectors.*; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -46,8 +46,6 @@ @Slf4j public class QueryVectorDBAgentProvider extends AbstractComposableAgentProvider { - protected static final ObjectMapper MAPPER = new ObjectMapper(); - @Getter @Setter public abstract static class VectorDatabaseWriterConfig { @@ -250,6 +248,9 @@ public Map generateSupportedTypesDocumentation( @SneakyThrows private static AgentConfigurationModel deepCopy(AgentConfigurationModel instance) { - return MAPPER.readValue(MAPPER.writeValueAsBytes(instance), AgentConfigurationModel.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue( + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(instance), + AgentConfigurationModel.class); } } diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntimeDockerTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntimeDockerTest.java index 23c27b025..0099dc27a 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntimeDockerTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/KubernetesClusterRuntimeDockerTest.java @@ -33,6 +33,7 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.application.ApplicationCodeInfo; import ai.langstream.deployer.k8s.agents.AgentResourcesFactory; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; @@ -45,7 +46,6 @@ import ai.langstream.kafka.runtime.KafkaTopic; import ai.langstream.runtime.api.agent.AgentSpec; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -77,7 +77,7 @@ private ApplicationDeployer getDeployer(DeployContext deployContext) { new ClusterRuntimeRegistry( Map.of( "kubernetes", - new ObjectMapper() + ObjectMapperFactory.getDefaultMapper() .convertValue(config, Map.class)))) .pluginsRegistry(new PluginsRegistry()) .topicConnectionsRuntimeRegistry(new TopicConnectionsRuntimeRegistry()) diff --git a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/AbstractKubernetesGenericStore.java b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/AbstractKubernetesGenericStore.java index 97b88a96b..4b07c8b90 100644 --- a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/AbstractKubernetesGenericStore.java +++ b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/AbstractKubernetesGenericStore.java @@ -16,7 +16,6 @@ package ai.langstream.impl.storage.k8s; import ai.langstream.api.storage.GenericStore; -import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.api.model.ObjectMeta; @@ -34,7 +33,6 @@ public abstract class AbstractKubernetesGenericStore implements GenericStore { - protected static final ObjectMapper mapper = new ObjectMapper(); protected static final String PREFIX = "langstream-"; private static String resourceName(String key) { diff --git a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStore.java b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStore.java index 4eba6cb3e..00f7df830 100644 --- a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStore.java +++ b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStore.java @@ -18,6 +18,7 @@ import ai.langstream.api.model.*; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.agents.AgentResourcesFactory; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationSpec; @@ -27,8 +28,6 @@ import ai.langstream.deployer.k8s.limits.ApplicationResourceLimitsChecker; import ai.langstream.deployer.k8s.util.KubeUtil; import ai.langstream.impl.k8s.KubernetesClientFactory; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Pod; @@ -63,8 +62,6 @@ public class KubernetesApplicationStore implements ApplicationStore { protected static final String SECRET_KEY = "secrets"; - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); protected static final SimpleDateFormat UTC_RFC3339; protected static final SimpleDateFormat UTC_K8S_LOGS; @@ -88,7 +85,8 @@ public String storeType() { @Override public void initialize(Map configuration) { this.properties = - mapper.convertValue(configuration, KubernetesApplicationStoreProperties.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, KubernetesApplicationStoreProperties.class); this.client = KubernetesClientFactory.get(null); } @@ -232,8 +230,9 @@ public void put( Map.of( SECRET_KEY, encodeSecret( - mapper.writeValueAsString( - applicationInstance.getSecrets())))) + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString( + applicationInstance.getSecrets())))) .build(); client.resource(secret).inNamespace(namespace).serverSideApply(); } @@ -300,7 +299,7 @@ public Secrets getSecrets(String tenant, String applicationId) { final String s = secret.getData().get(SECRET_KEY); final String decoded = new String(Base64.getDecoder().decode(s), StandardCharsets.UTF_8); - return mapper.readValue(decoded, Secrets.class); + return ObjectMapperFactory.getDefaultMapper().readValue(decoded, Secrets.class); } return null; } diff --git a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/TenantResources.java b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/TenantResources.java index e0e38ee10..bf8cf5a9a 100644 --- a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/TenantResources.java +++ b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/apps/TenantResources.java @@ -17,9 +17,9 @@ import static ai.langstream.impl.storage.k8s.apps.KubernetesApplicationStore.encodeSecret; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.CRDConstants; import ai.langstream.runtime.api.ClusterConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.SecretBuilder; @@ -38,7 +38,6 @@ @Slf4j public class TenantResources { - private static final ObjectMapper mapper = new ObjectMapper(); private final KubernetesApplicationStoreProperties properties; private final KubernetesClient client; private final String tenant; @@ -60,7 +59,10 @@ void ensureTenantResources() { private void ensureClusterConfiguration() { final ClusterConfiguration clusterConfiguration = new ClusterConfiguration(properties.getControlPlaneUrl()); - final String encoded = encodeSecret(mapper.writeValueAsString(clusterConfiguration)); + final String encoded = + encodeSecret( + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(clusterConfiguration)); final Secret secret = new SecretBuilder() .withNewMetadata() diff --git a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/global/KubernetesGlobalMetadataStore.java b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/global/KubernetesGlobalMetadataStore.java index 3cdc13d23..a1ba805b0 100644 --- a/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/global/KubernetesGlobalMetadataStore.java +++ b/langstream-k8s-storage/src/main/java/ai/langstream/impl/storage/k8s/global/KubernetesGlobalMetadataStore.java @@ -16,6 +16,7 @@ package ai.langstream.impl.storage.k8s.global; import ai.langstream.api.storage.GlobalMetadataStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.k8s.KubernetesClientFactory; import ai.langstream.impl.storage.k8s.AbstractKubernetesGenericStore; import io.fabric8.kubernetes.api.model.ConfigMap; @@ -38,7 +39,8 @@ public class KubernetesGlobalMetadataStore extends AbstractKubernetesGenericStor @Override public void initialize(Map configuration) { final KubernetesGlobalMetadataStoreProperties properties = - mapper.convertValue(configuration, KubernetesGlobalMetadataStoreProperties.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, KubernetesGlobalMetadataStoreProperties.class); if (properties.getNamespace() == null) { final String fromEnv = System.getenv("KUBERNETES_NAMESPACE"); diff --git a/langstream-k8s-storage/src/test/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStoreTest.java b/langstream-k8s-storage/src/test/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStoreTest.java index 966ec71eb..7130c976f 100644 --- a/langstream-k8s-storage/src/test/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStoreTest.java +++ b/langstream-k8s-storage/src/test/java/ai/langstream/impl/storage/k8s/apps/KubernetesApplicationStoreTest.java @@ -66,11 +66,11 @@ void testApp() { assertNull(createdCr.getSpec().getImage()); assertNull(createdCr.getSpec().getImagePullPolicy()); assertEquals( - "{\"resources\":{},\"modules\":{},\"instance\":null,\"gateways\":null,\"agentRunners\":{}}", + "{\"resources\":{},\"modules\":{},\"agentRunners\":{}}", createdCr.getSpec().getApplication()); assertEquals(tenant, createdCr.getSpec().getTenant()); assertEquals( - "{\"deleteMode\":\"CLEANUP_REQUIRED\",\"markedForDeletion\":false,\"seed\":0,\"runtimeVersion\":null,\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false}", + "{\"deleteMode\":\"CLEANUP_REQUIRED\",\"markedForDeletion\":false,\"seed\":0,\"autoUpgradeRuntimeImagePullPolicy\":false,\"autoUpgradeAgentResources\":false,\"autoUpgradeAgentPodTemplate\":false}", createdCr.getSpec().getOptions()); final Secret createdSecret = diff --git a/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaProducerWrapper.java b/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaProducerWrapper.java index dc6164ef3..f02e80399 100644 --- a/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaProducerWrapper.java +++ b/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaProducerWrapper.java @@ -22,7 +22,7 @@ import ai.langstream.api.runner.code.Header; import ai.langstream.api.runner.code.Record; import ai.langstream.api.runner.topics.TopicProducer; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.confluent.kafka.serializers.KafkaAvroSerializer; import java.util.ArrayList; import java.util.Collection; @@ -260,15 +260,13 @@ public String toString() { } private static class ObjectToJsonSerializer implements Serializer { - private static final ObjectMapper MAPPER = new ObjectMapper(); - @Override public void configure(Map configs, boolean isKey) {} @Override @SneakyThrows public byte[] serialize(String topic, Object data) { - return MAPPER.writeValueAsBytes(data); + return ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(data); } } } diff --git a/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaReaderWrapper.java b/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaReaderWrapper.java index 07313eb52..7ee3f7b2b 100644 --- a/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaReaderWrapper.java +++ b/langstream-kafka-runtime/src/main/java/ai/langstream/kafka/runner/KafkaReaderWrapper.java @@ -21,8 +21,8 @@ import ai.langstream.api.runner.topics.TopicReadResult; import ai.langstream.api.runner.topics.TopicReader; import ai.langstream.api.util.ClassloaderUtils; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -40,7 +40,6 @@ @Slf4j class KafkaReaderWrapper implements TopicReader { - static final ObjectMapper mapper = new ObjectMapper(); private final Map configuration; private final String topicName; private final TopicOffsetPosition initialPosition; @@ -101,7 +100,8 @@ public void start() throws IOException { } private OffsetPerPartition parseOffset() throws IOException { - return mapper.readValue(initialPosition.offset(), OffsetPerPartition.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue(initialPosition.offset(), OffsetPerPartition.class); } @Override @@ -134,7 +134,8 @@ public TopicReadResult read() throws JsonProcessingException { partitions.put(key.partition() + "", topicPartitionLongEntry.getValue() + ""); } final OffsetPerPartition offsetPerPartition = new OffsetPerPartition(partitions); - byte[] offset = mapper.writeValueAsBytes(offsetPerPartition); + byte[] offset = + ObjectMapperFactory.getDefaultMapper().writeValueAsBytes(offsetPerPartition); return new TopicReadResult() { @Override public List records() { diff --git a/langstream-kafka/src/main/java/ai/langstream/kafka/runtime/KafkaStreamingClusterRuntime.java b/langstream-kafka/src/main/java/ai/langstream/kafka/runtime/KafkaStreamingClusterRuntime.java index 4d2511be4..218439e9f 100644 --- a/langstream-kafka/src/main/java/ai/langstream/kafka/runtime/KafkaStreamingClusterRuntime.java +++ b/langstream-kafka/src/main/java/ai/langstream/kafka/runtime/KafkaStreamingClusterRuntime.java @@ -20,7 +20,7 @@ import ai.langstream.api.runtime.AgentNode; import ai.langstream.api.runtime.StreamingClusterRuntime; import ai.langstream.api.runtime.Topic; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -28,12 +28,11 @@ @Slf4j public class KafkaStreamingClusterRuntime implements StreamingClusterRuntime { - static final ObjectMapper mapper = new ObjectMapper(); - public static KafkaClusterRuntimeConfiguration getKafkaClusterRuntimeConfiguration( StreamingCluster streamingCluster) { final Map configuration = streamingCluster.configuration(); - return mapper.convertValue(configuration, KafkaClusterRuntimeConfiguration.class); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, KafkaClusterRuntimeConfiguration.class); } @Override diff --git a/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaClientUtils.java b/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaClientUtils.java index 525b4b890..86ed63038 100644 --- a/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaClientUtils.java +++ b/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaClientUtils.java @@ -19,7 +19,7 @@ import ai.langstream.api.model.StreamingCluster; import ai.langstream.api.util.ConfigurationUtils; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import io.pravega.client.ClientConfig; import io.pravega.client.EventStreamClientFactory; import io.pravega.client.admin.ReaderGroupManager; @@ -28,8 +28,6 @@ import java.util.Map; public class PravegaClientUtils { - private static final ObjectMapper MAPPER = new ObjectMapper(); - public static StreamManager buildStreamManager( PravegaClusterRuntimeConfiguration pravegaClusterRuntimeConfiguration) throws Exception { @@ -85,6 +83,7 @@ public static String getScope(PravegaClusterRuntimeConfiguration configuration) public static PravegaClusterRuntimeConfiguration getPravegarClusterRuntimeConfiguration( StreamingCluster streamingCluster) { final Map configuration = streamingCluster.configuration(); - return MAPPER.convertValue(configuration, PravegaClusterRuntimeConfiguration.class); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, PravegaClusterRuntimeConfiguration.class); } } diff --git a/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaTopicConnectionsRuntimeProvider.java b/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaTopicConnectionsRuntimeProvider.java index 9f2cdf36b..11ea6974a 100644 --- a/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaTopicConnectionsRuntimeProvider.java +++ b/langstream-pravega-runtime/src/main/java/ai/langstream/pravega/PravegaTopicConnectionsRuntimeProvider.java @@ -33,8 +33,8 @@ import ai.langstream.api.runner.topics.TopicReader; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.Topic; +import ai.langstream.api.util.ObjectMapperFactory; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.pravega.client.EventStreamClientFactory; import io.pravega.client.admin.ReaderGroupManager; import io.pravega.client.admin.StreamManager; @@ -63,7 +63,6 @@ @Slf4j public class PravegaTopicConnectionsRuntimeProvider implements TopicConnectionsRuntimeProvider { - private static final ObjectMapper mapper = new ObjectMapper(); @Override public boolean supports(String streamingClusterType) { @@ -470,7 +469,9 @@ public static SimpleRecord convertToRecord(EventRead stringEventRead, St throws JsonProcessingException { Collection
headers = new ArrayList<>(); log.info("decoding event {}", stringEventRead.getEvent()); - RecordWrapper wrapper = mapper.readValue(stringEventRead.getEvent(), RecordWrapper.class); + RecordWrapper wrapper = + ObjectMapperFactory.getDefaultMapper() + .readValue(stringEventRead.getEvent(), RecordWrapper.class); if (wrapper.headers != null) { wrapper.headers.forEach( (key, value) -> headers.add(new SimpleRecord.SimpleHeader(key, value))); @@ -495,7 +496,7 @@ private static String serialiseKey(Object o) throws IOException { return o.toString(); } - return mapper.writeValueAsString(o); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(o); } private static String serialiseValue(Record record) throws IOException { @@ -505,7 +506,7 @@ private static String serialiseValue(Record record) throws IOException { } RecordWrapper wrapper = new RecordWrapper(record.key(), record.value(), headers, record.timestamp()); - return mapper.writeValueAsString(wrapper); + return ObjectMapperFactory.getDefaultMapper().writeValueAsString(wrapper); } public record RecordWrapper( diff --git a/langstream-pravega/src/main/java/ai/langstream/pravega/PravegaStreamingClusterRuntime.java b/langstream-pravega/src/main/java/ai/langstream/pravega/PravegaStreamingClusterRuntime.java index 99e1f0700..619807dd2 100644 --- a/langstream-pravega/src/main/java/ai/langstream/pravega/PravegaStreamingClusterRuntime.java +++ b/langstream-pravega/src/main/java/ai/langstream/pravega/PravegaStreamingClusterRuntime.java @@ -20,14 +20,12 @@ import ai.langstream.api.runtime.AgentNode; import ai.langstream.api.runtime.StreamingClusterRuntime; import ai.langstream.api.runtime.Topic; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j public class PravegaStreamingClusterRuntime implements StreamingClusterRuntime { - private static final ObjectMapper MAPPER = new ObjectMapper(); - @Override public Topic createTopicImplementation( TopicDefinition topicDefinition, StreamingCluster streamingCluster) { @@ -65,6 +63,7 @@ public Map createProducerConfiguration( public static PravegaClusterRuntimeConfiguration getPravegaClusterRuntimeConfiguration( StreamingCluster streamingCluster) { final Map configuration = streamingCluster.configuration(); - return MAPPER.convertValue(configuration, PravegaClusterRuntimeConfiguration.class); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, PravegaClusterRuntimeConfiguration.class); } } diff --git a/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/PulsarClientUtils.java b/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/PulsarClientUtils.java index 34b3f9583..cd63ac0c6 100644 --- a/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/PulsarClientUtils.java +++ b/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/PulsarClientUtils.java @@ -16,15 +16,13 @@ package ai.langstream.pulsar; import ai.langstream.api.model.StreamingCluster; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.HashMap; import java.util.Map; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; public class PulsarClientUtils { - private static final ObjectMapper MAPPER = new ObjectMapper(); - public static PulsarAdmin buildPulsarAdmin( PulsarClusterRuntimeConfiguration pulsarClusterRuntimeConfiguration) throws Exception { Map adminConfig = pulsarClusterRuntimeConfiguration.admin(); @@ -64,6 +62,7 @@ public static PulsarClient buildPulsarClient(StreamingCluster streamingCluster) public static PulsarClusterRuntimeConfiguration getPulsarClusterRuntimeConfiguration( StreamingCluster streamingCluster) { final Map configuration = streamingCluster.configuration(); - return MAPPER.convertValue(configuration, PulsarClusterRuntimeConfiguration.class); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, PulsarClusterRuntimeConfiguration.class); } } diff --git a/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/runner/PulsarTopicConnectionsRuntimeProvider.java b/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/runner/PulsarTopicConnectionsRuntimeProvider.java index 39a8a8dba..19d850e06 100644 --- a/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/runner/PulsarTopicConnectionsRuntimeProvider.java +++ b/langstream-pulsar-runtime/src/main/java/ai/langstream/pulsar/runner/PulsarTopicConnectionsRuntimeProvider.java @@ -35,11 +35,11 @@ import ai.langstream.api.runner.topics.TopicReader; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.Topic; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.pulsar.PulsarClientUtils; import ai.langstream.pulsar.PulsarClusterRuntimeConfiguration; import ai.langstream.pulsar.PulsarName; import ai.langstream.pulsar.PulsarTopic; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -73,7 +73,6 @@ @Slf4j public class PulsarTopicConnectionsRuntimeProvider implements TopicConnectionsRuntimeProvider { - private static final ObjectMapper mapper = new ObjectMapper(); @Override public boolean supports(String streamingClusterType) { @@ -447,7 +446,8 @@ private PulsarTopicReader( if (initialPosition.position() == TopicOffsetPosition.Position.Absolute) { try { this.topicMessageIds = - mapper.readerForMapOf(byte[].class) + ObjectMapperFactory.getDefaultMapper() + .readerForMapOf(byte[].class) .readValue(initialPosition.offset()); } catch (IOException e) { throw new RuntimeException(e); @@ -505,7 +505,9 @@ public TopicReadResult read() throws Exception { records = List.of(new PulsarConsumerRecord(finalKey, finalValue, receive)); topicMessageIds.put( receive.getTopicName(), receive.getMessageId().toByteArray()); - offset = mapper.writeValueAsBytes(topicMessageIds); + offset = + ObjectMapperFactory.getDefaultMapper() + .writeValueAsBytes(topicMessageIds); } else { records = List.of(); offset = null; @@ -660,16 +662,18 @@ public void start() { if (configuration.containsKey("valueSchema")) { log.info("Using schema from topic definition {}", localTopic); SchemaDefinition valueSchemaDefinition = - mapper.convertValue( - configuration.remove("valueSchema"), - SchemaDefinition.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + configuration.remove("valueSchema"), + SchemaDefinition.class); Schema valueSchema = Schema.getSchema(getSchemaInfo(valueSchemaDefinition)); if (configuration.containsKey("keySchema")) { SchemaDefinition keySchemaDefinition = - mapper.convertValue( - configuration.remove("keySchema"), - SchemaDefinition.class); + ObjectMapperFactory.getDefaultMapper() + .convertValue( + configuration.remove("keySchema"), + SchemaDefinition.class); Schema keySchema = Schema.getSchema(getSchemaInfo(keySchemaDefinition)); schema = @@ -801,7 +805,8 @@ private static Object convertValue(Object value, Schema schema) { if (Map.class.isAssignableFrom(value.getClass()) || Collection.class.isAssignableFrom(value.getClass())) { try { - return mapper.writeValueAsString(value); + return ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(value); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/langstream-pulsar/src/main/java/ai/langstream/pulsar/PulsarStreamingClusterRuntime.java b/langstream-pulsar/src/main/java/ai/langstream/pulsar/PulsarStreamingClusterRuntime.java index 735e8e266..a184a0694 100644 --- a/langstream-pulsar/src/main/java/ai/langstream/pulsar/PulsarStreamingClusterRuntime.java +++ b/langstream-pulsar/src/main/java/ai/langstream/pulsar/PulsarStreamingClusterRuntime.java @@ -21,13 +21,12 @@ import ai.langstream.api.runtime.AgentNode; import ai.langstream.api.runtime.StreamingClusterRuntime; import ai.langstream.api.runtime.Topic; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j public class PulsarStreamingClusterRuntime implements StreamingClusterRuntime { - private static final ObjectMapper MAPPER = new ObjectMapper(); @Override public Topic createTopicImplementation( @@ -73,6 +72,7 @@ public Map createProducerConfiguration( public static PulsarClusterRuntimeConfiguration getPulsarClusterRuntimeConfiguration( StreamingCluster streamingCluster) { final Map configuration = streamingCluster.configuration(); - return MAPPER.convertValue(configuration, PulsarClusterRuntimeConfiguration.class); + return ObjectMapperFactory.getDefaultMapper() + .convertValue(configuration, PulsarClusterRuntimeConfiguration.class); } } diff --git a/langstream-runtime/langstream-runtime-impl/pom.xml b/langstream-runtime/langstream-runtime-impl/pom.xml index 400d76348..08b2fc5c8 100644 --- a/langstream-runtime/langstream-runtime-impl/pom.xml +++ b/langstream-runtime/langstream-runtime-impl/pom.xml @@ -257,7 +257,7 @@ tests - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarter.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarter.java index d51a78d6d..97eb7f786 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarter.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarter.java @@ -15,12 +15,11 @@ */ package ai.langstream.runtime.agent; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.RuntimeStarter; import ai.langstream.runtime.api.ClusterConfiguration; import ai.langstream.runtime.api.agent.AgentCodeDownloaderConstants; import ai.langstream.runtime.api.agent.DownloadAgentCodeConfiguration; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Files; import java.nio.file.Path; import lombok.SneakyThrows; @@ -29,8 +28,6 @@ /** This is the main entry point for the pods that run the LangStream runtime code downloader. */ @Slf4j public class AgentCodeDownloaderStarter extends RuntimeStarter { - private static final ObjectMapper MAPPER = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static MainErrorHandler mainErrorHandler = error -> { @@ -77,11 +74,14 @@ public void start(String... args) throws Exception { AgentCodeDownloaderConstants.TOKEN_ENV_DEFAULT); DownloadAgentCodeConfiguration configuration = - MAPPER.readValue( - downloadCodeConfigPath.toFile(), DownloadAgentCodeConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + downloadCodeConfigPath.toFile(), + DownloadAgentCodeConfiguration.class); ClusterConfiguration clusterConfiguration = - MAPPER.readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); final String token; if (tokenPath != null) { token = Files.readString(tokenPath); diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentRunnerStarter.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentRunnerStarter.java index 0a9e45992..ff504f8b2 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentRunnerStarter.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/AgentRunnerStarter.java @@ -25,12 +25,11 @@ import static ai.langstream.runtime.api.agent.AgentRunnerConstants.POD_CONFIG_ENV_DEFAULT; import ai.langstream.api.runner.code.MetricsReporter; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.RuntimeStarter; import ai.langstream.runtime.agent.api.AgentAPIController; import ai.langstream.runtime.agent.metrics.PrometheusMetricsReporter; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicBoolean; import lombok.SneakyThrows; @@ -39,7 +38,6 @@ /** This is the main entry point for the pods that run the LangStream runtime and Java code. */ @Slf4j public class AgentRunnerStarter extends RuntimeStarter { - private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory()); private static final MetricsReporter metricsReporter = new PrometheusMetricsReporter(); private static final MainErrorHandler mainErrorHandler = @@ -109,7 +107,8 @@ public void start(String... args) throws Exception { } RuntimePodConfiguration configuration = - MAPPER.readValue(podRuntimeConfiguration.toFile(), RuntimePodConfiguration.class); + ObjectMapperFactory.getDefaultYamlMapper() + .readValue(podRuntimeConfiguration.toFile(), RuntimePodConfiguration.class); AtomicBoolean continueLoop = new AtomicBoolean(true); Runtime.getRuntime() diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/api/AgentInfoServlet.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/api/AgentInfoServlet.java index 4f10ef496..18ef1e8ee 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/api/AgentInfoServlet.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/agent/api/AgentInfoServlet.java @@ -15,7 +15,7 @@ */ package ai.langstream.runtime.agent.api; -import com.fasterxml.jackson.databind.ObjectMapper; +import ai.langstream.api.util.ObjectMapperFactory; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -24,8 +24,6 @@ @Slf4j public class AgentInfoServlet extends HttpServlet { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final AgentAPIController agentAPIController; public AgentInfoServlet(AgentAPIController agentAPIController) { @@ -35,7 +33,8 @@ public AgentInfoServlet(AgentAPIController agentAPIController) { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("application/json"); - MAPPER.writeValue(resp.getOutputStream(), agentAPIController.serveWorkerStatus()); + ObjectMapperFactory.getDefaultMapper() + .writeValue(resp.getOutputStream(), agentAPIController.serveWorkerStatus()); } @Override @@ -44,7 +43,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I String uri = req.getRequestURI(); if (uri.endsWith("/restart")) { try { - MAPPER.writeValue(resp.getOutputStream(), agentAPIController.restart()); + ObjectMapperFactory.getDefaultMapper() + .writeValue(resp.getOutputStream(), agentAPIController.restart()); } catch (Throwable error) { log.error("Error while restarting the agents"); resp.getOutputStream().write((error + "").getBytes()); diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunner.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunner.java index 9bd39a7af..6eb0f12af 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunner.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunner.java @@ -15,8 +15,6 @@ */ package ai.langstream.runtime.application; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; - import ai.langstream.api.model.Application; import ai.langstream.api.model.Secrets; import ai.langstream.api.runner.assets.AssetManagerRegistry; @@ -26,12 +24,12 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.deploy.ApplicationDeployer; import ai.langstream.impl.nar.NarFileHandler; import ai.langstream.runtime.agent.AgentRunner; import ai.langstream.runtime.api.application.ApplicationSetupConfiguration; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URL; import java.nio.file.Path; import java.util.List; @@ -41,12 +39,6 @@ @Slf4j public class ApplicationSetupRunner { - private static final ObjectMapper MAPPER = - new ObjectMapper() - .configure( - FAIL_ON_UNKNOWN_PROPERTIES, - false); // this helps with forward compatibility - public void runSetup( Map> clusterRuntimeConfiguration, ApplicationSetupConfiguration configuration, @@ -108,7 +100,9 @@ private Application parseApplicationInstance( throws JsonProcessingException { final String applicationConfig = configuration.getApplication(); - final Application appInstance = MAPPER.readValue(applicationConfig, Application.class); + final Application appInstance = + ObjectMapperFactory.getDefaultMapper() + .readValue(applicationConfig, Application.class); appInstance.setSecrets(secrets); return appInstance; } diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunnerStarter.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunnerStarter.java index 95a3be476..ffe223695 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunnerStarter.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/application/ApplicationSetupRunnerStarter.java @@ -15,9 +15,8 @@ */ package ai.langstream.runtime.application; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; - import ai.langstream.api.model.Secrets; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.RuntimeStarter; import ai.langstream.runtime.agent.AgentCodeDownloader; import ai.langstream.runtime.api.ClusterConfiguration; @@ -25,7 +24,6 @@ import ai.langstream.runtime.api.agent.DownloadAgentCodeConfiguration; import ai.langstream.runtime.api.application.ApplicationSetupConfiguration; import ai.langstream.runtime.api.application.ApplicationSetupConstants; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -36,12 +34,6 @@ @Slf4j public class ApplicationSetupRunnerStarter extends RuntimeStarter { - private static final ObjectMapper MAPPER = - new ObjectMapper() - .configure( - FAIL_ON_UNKNOWN_PROPERTIES, - false); // this helps with forward compatibility - private static final ErrorHandler errorHandler = error -> { log.error("Unexpected error", error); @@ -144,7 +136,8 @@ private Path downloadApplicationCode( final Path downloadDirectory = Files.createTempDirectory("langstream-code"); ClusterConfiguration clusterConfiguration = - MAPPER.readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); AgentCodeDownloader downloader = new AgentCodeDownloader(); @@ -168,7 +161,9 @@ private Secrets loadSecrets(Path secretsPath) throws IOException { Secrets secrets = null; if (secretsPath != null) { log.info("Loading secrets from {}", secretsPath); - secrets = MAPPER.readValue(secretsPath.toFile(), Secrets.class); + secrets = + ObjectMapperFactory.getDefaultMapper() + .readValue(secretsPath.toFile(), Secrets.class); } else { log.info("No secrets file provided"); } @@ -179,7 +174,8 @@ private ApplicationSetupConfiguration loadApplicationSetupConfiguration(Path app throws IOException { log.info("Loading app configuration from {}", appConfigPath); final ApplicationSetupConfiguration configuration = - MAPPER.readValue(appConfigPath.toFile(), ApplicationSetupConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(appConfigPath.toFile(), ApplicationSetupConfiguration.class); log.info("applicationSetupConfiguration {}", configuration); return configuration; } @@ -189,7 +185,8 @@ private Map> loadClusterRuntimeConfiguration( log.info("Loading cluster runtime config from {}", clusterRuntimeConfigPath); final Map> clusterRuntimeConfiguration = (Map>) - MAPPER.readValue(clusterRuntimeConfigPath.toFile(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(clusterRuntimeConfigPath.toFile(), Map.class); log.info("clusterRuntimeConfiguration {}", clusterRuntimeConfiguration); return clusterRuntimeConfiguration; } diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployer.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployer.java index f2e2e3212..dad2583bf 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployer.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployer.java @@ -15,8 +15,6 @@ */ package ai.langstream.runtime.deployer; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; - import ai.langstream.admin.client.AdminClient; import ai.langstream.admin.client.AdminClientLogger; import ai.langstream.api.model.Application; @@ -25,11 +23,11 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.application.ApplicationCodeInfo; import ai.langstream.impl.deploy.ApplicationDeployer; import ai.langstream.runtime.api.ClusterConfiguration; import ai.langstream.runtime.api.deployer.RuntimeDeployerConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.Map; import lombok.SneakyThrows; @@ -39,12 +37,6 @@ @Slf4j public class RuntimeDeployer { - private static final ObjectMapper MAPPER = - new ObjectMapper() - .configure( - FAIL_ON_UNKNOWN_PROPERTIES, - false); // this helps with forward compatibility - public void deploy( Map> clusterRuntimeConfiguration, RuntimeDeployerConfiguration configuration, @@ -60,7 +52,9 @@ public void deploy( configuration.getCodeStorageArchiveId()); final String applicationConfig = configuration.getApplication(); - final Application appInstance = MAPPER.readValue(applicationConfig, Application.class); + final Application appInstance = + ObjectMapperFactory.getDefaultMapper() + .readValue(applicationConfig, Application.class); appInstance.setSecrets(secrets); final String tenant = configuration.getTenant(); @@ -103,7 +97,9 @@ public void delete( final String applicationId = configuration.getApplicationId(); final String applicationConfig = configuration.getApplication(); - final Application appInstance = MAPPER.readValue(applicationConfig, Application.class); + final Application appInstance = + ObjectMapperFactory.getDefaultMapper() + .readValue(applicationConfig, Application.class); appInstance.setSecrets(secrets); try (ApplicationDeployer deployer = @@ -189,7 +185,8 @@ public ApplicationCodeInfo getApplicationCodeInfo( } final String codeInfo = adminClient.applications().getCodeInfo(applicationId, codeArchiveId); - return MAPPER.readValue(codeInfo, ApplicationCodeInfo.class); + return ObjectMapperFactory.getDefaultMapper() + .readValue(codeInfo, ApplicationCodeInfo.class); } @Override diff --git a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployerStarter.java b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployerStarter.java index aad039775..a9f8ae847 100644 --- a/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployerStarter.java +++ b/langstream-runtime/langstream-runtime-impl/src/main/java/ai/langstream/runtime/deployer/RuntimeDeployerStarter.java @@ -15,15 +15,13 @@ */ package ai.langstream.runtime.deployer; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; - import ai.langstream.api.model.Secrets; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.RuntimeStarter; import ai.langstream.runtime.api.ClusterConfiguration; import ai.langstream.runtime.api.agent.AgentCodeDownloaderConstants; import ai.langstream.runtime.api.deployer.RuntimeDeployerConfiguration; import ai.langstream.runtime.api.deployer.RuntimeDeployerConstants; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -34,11 +32,6 @@ @Slf4j public class RuntimeDeployerStarter extends RuntimeStarter { - private static final ObjectMapper MAPPER = - new ObjectMapper() - .configure( - FAIL_ON_UNKNOWN_PROPERTIES, - false); // this helps with forward compatibility private static final ErrorHandler errorHandler = error -> { log.error("Unexpected error", error); @@ -106,18 +99,22 @@ public void start(String... args) throws Exception { log.info("Loading cluster runtime config from {}", clusterRuntimeConfigPath); final Map> clusterRuntimeConfiguration = (Map>) - MAPPER.readValue(clusterRuntimeConfigPath.toFile(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(clusterRuntimeConfigPath.toFile(), Map.class); log.info("clusterRuntimeConfiguration {}", clusterRuntimeConfiguration); log.info("Loading app configuration from {}", appConfigPath); final RuntimeDeployerConfiguration configuration = - MAPPER.readValue(appConfigPath.toFile(), RuntimeDeployerConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(appConfigPath.toFile(), RuntimeDeployerConfiguration.class); log.info("RuntimeDeployerConfiguration {}", configuration); Secrets secrets = null; if (secretsPath != null) { log.info("Loading secrets from {}", secretsPath); - secrets = MAPPER.readValue(secretsPath.toFile(), Secrets.class); + secrets = + ObjectMapperFactory.getDefaultMapper() + .readValue(secretsPath.toFile(), Secrets.class); } else { log.info("No secrets file provided"); } @@ -147,7 +144,8 @@ private ClusterConfiguration getClusterConfiguration() throws IOException { clusterConfiguration = null; } else { clusterConfiguration = - MAPPER.readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(clusterConfigPath.toFile(), ClusterConfiguration.class); } return clusterConfiguration; } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/AzureBlobStorageSourceIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/AzureBlobStorageSourceIT.java index 62e4f4744..4a1c38a44 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/AzureBlobStorageSourceIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/AzureBlobStorageSourceIT.java @@ -15,12 +15,12 @@ */ package ai.langstream.agents; -import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; import ai.langstream.agents.azureblobstorage.AzureBlobStorageSource; import ai.langstream.api.runner.topics.TopicConsumer; +import ai.langstream.testrunners.AbstractApplicationRunner; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; import com.azure.core.util.BinaryData; import com.azure.storage.blob.BlobContainerClient; @@ -38,7 +38,7 @@ @Slf4j @Testcontainers -@Tag(INTEGRATION_TESTS_GROUP1) +@Tag(AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2) class AzureBlobStorageSourceIT extends AbstractGenericStreamingApplicationRunner { @Container private static final LocalStackContainer localstack = diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/BedrockCompletionsIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/BedrockCompletionsIT.java index d545bc042..6e6e91a57 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/BedrockCompletionsIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/BedrockCompletionsIT.java @@ -151,8 +151,8 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { String expected = """ - {"question":"the car","session-id":"2139847128764192","answer":"A car is a vehicle","prompt":"{\\"options\\":{\\"type\\":\\"ai-chat-completions\\",\\"when\\":null,\\"model\\":\\"ai21.j2-mid-v1\\",\\"messages\\":[{\\"role\\":null,\\"content\\":\\"What can you tell me about {{{ value.question }}} ?\\"}],\\"stream-to-topic\\":null,\\"stream-response-completion-field\\":null,\\"min-chunks-per-message\\":3,\\"completion-field\\":\\"value.answer\\",\\"stream\\":true,\\"log-field\\":\\"value.prompt\\",\\"max-tokens\\":null,\\"temperature\\":null,\\"top-p\\":null,\\"logit-bias\\":null,\\"user\\":null,\\"stop\\":null,\\"presence-penalty\\":null,\\"frequency-penalty\\":null,\\"options\\":{\\"request-parameters\\":{\\"maxTokens\\":5,\\"temperature\\":0.2},\\"response-completions-expression\\":\\"completions[0].data.text\\"}},\\"messages\\":[{\\"role\\":null,\\"content\\":\\"What can you tell me about the car ?\\"}],\\"model\\":\\"ai21.j2-mid-v1\\"}"}""" - .formatted(prompt, outputTopic); + {"answer":"A car is a vehicle","prompt":"{\\"messages\\":[{\\"content\\":\\"What can you tell me about the car ?\\"}],\\"model\\":\\"ai21.j2-mid-v1\\",\\"options\\":{\\"completion-field\\":\\"value.answer\\",\\"log-field\\":\\"value.prompt\\",\\"messages\\":[{\\"content\\":\\"%s\\"}],\\"min-chunks-per-message\\":3,\\"model\\":\\"ai21.j2-mid-v1\\",\\"options\\":{\\"request-parameters\\":{\\"maxTokens\\":5,\\"temperature\\":0.2},\\"response-completions-expression\\":\\"completions[0].data.text\\"},\\"stream\\":true,\\"type\\":\\"ai-chat-completions\\"}}","question":"the car","session-id":"2139847128764192"}""" + .formatted(prompt); waitForMessages(consumer, List.of(expected)); } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraAssetQueryWriteIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraAssetQueryWriteIT.java index bd70cef60..1fce92708 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraAssetQueryWriteIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraAssetQueryWriteIT.java @@ -15,7 +15,7 @@ */ package ai.langstream.agents; -import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -41,7 +41,7 @@ @Slf4j @Testcontainers -@Tag(INTEGRATION_TESTS_GROUP1) +@Tag(INTEGRATION_TESTS_GROUP2) class CassandraAssetQueryWriteIT extends AbstractGenericStreamingApplicationRunner { @Container @@ -147,7 +147,7 @@ public void testCassandra() throws Exception { waitForMessages( consumer, List.of( - "{\"documentId\":2,\"queryresult\":{\"name\":\"A\",\"description\":\"A description\",\"id\":1},\"name\":\"A\",\"description\":\"A description\"}")); + "{\"description\":\"A description\",\"documentId\":2,\"name\":\"A\",\"queryresult\":{\"description\":\"A description\",\"id\":1,\"name\":\"A\"}}")); CqlSessionBuilder builder = new CqlSessionBuilder(); builder.addContactPoint(cassandra.getContactPoint()); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraVectorAssetQueryWriteIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraVectorAssetQueryWriteIT.java index 8b62b7c65..1ce2c6dc8 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraVectorAssetQueryWriteIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CassandraVectorAssetQueryWriteIT.java @@ -15,7 +15,7 @@ */ package ai.langstream.agents; -import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; @@ -45,7 +45,7 @@ @Slf4j @Testcontainers -@Tag(INTEGRATION_TESTS_GROUP1) +@Tag(INTEGRATION_TESTS_GROUP2) class CassandraVectorAssetQueryWriteIT extends AbstractGenericStreamingApplicationRunner { @Container @@ -154,7 +154,7 @@ public void testCassandra() throws Exception { waitForMessages( consumer, List.of( - "{\"documentId\":2,\"embeddings\":[0.1,0.2,0.3,0.4,0.5],\"queryresult\":{\"embeddings\":null,\"name\":\"A\",\"description\":\"A description\",\"id\":1},\"name\":\"A\",\"description\":\"A description\"}")); + "{\"description\":\"A description\",\"documentId\":2,\"embeddings\":[0.1,0.2,0.3,0.4,0.5],\"name\":\"A\",\"queryresult\":{\"description\":\"A description\",\"id\":1,\"name\":\"A\"}}")); CqlSessionBuilder builder = new CqlSessionBuilder(); builder.addContactPoint(cassandra.getContactPoint()); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ChatCompletionsIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ChatCompletionsIT.java index b990c9db6..1bdd324d0 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ChatCompletionsIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ChatCompletionsIT.java @@ -185,7 +185,7 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { String expected = """ - {"question":"the car","session-id":"2139847128764192","answer":"A car is a vehicle","prompt":"{\\"options\\":{\\"type\\":\\"ai-chat-completions\\",\\"when\\":null,\\"model\\":\\"gpt-35-turbo\\",\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"%s\\"}],\\"stream-to-topic\\":\\"%s\\",\\"stream-response-completion-field\\":\\"value\\",\\"min-chunks-per-message\\":3,\\"completion-field\\":\\"value.answer\\",\\"stream\\":true,\\"log-field\\":\\"value.prompt\\",\\"max-tokens\\":null,\\"temperature\\":null,\\"top-p\\":null,\\"logit-bias\\":null,\\"user\\":null,\\"stop\\":null,\\"presence-penalty\\":null,\\"frequency-penalty\\":null,\\"options\\":null},\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"What can you tell me about the car ?\\"}],\\"model\\":\\"gpt-35-turbo\\"}"}""" + {"answer":"A car is a vehicle","prompt":"{\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"What can you tell me about the car ?\\"}],\\"model\\":\\"gpt-35-turbo\\",\\"options\\":{\\"completion-field\\":\\"value.answer\\",\\"log-field\\":\\"value.prompt\\",\\"messages\\":[{\\"content\\":\\"%s\\",\\"role\\":\\"user\\"}],\\"min-chunks-per-message\\":3,\\"model\\":\\"gpt-35-turbo\\",\\"stream\\":true,\\"stream-response-completion-field\\":\\"value\\",\\"stream-to-topic\\":\\"%s\\",\\"type\\":\\"ai-chat-completions\\"}}","question":"the car","session-id":"2139847128764192"}""" .formatted(prompt, streamToTopic); List mainOutputRecords = waitForMessages(consumer, List.of(expected)); Record record = mainOutputRecords.get(0); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ComputeEmbeddingsIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ComputeEmbeddingsIT.java index acfd5c6e7..097475cc5 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ComputeEmbeddingsIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ComputeEmbeddingsIT.java @@ -15,7 +15,7 @@ */ package ai.langstream.agents; -import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static com.github.tomakehurst.wiremock.client.WireMock.containing; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; @@ -63,7 +63,7 @@ @Slf4j @WireMockTest -@Tag(INTEGRATION_TESTS_GROUP1) +@Tag(INTEGRATION_TESTS_GROUP2) class ComputeEmbeddingsIT extends AbstractGenericStreamingApplicationRunner { @AllArgsConstructor @@ -350,7 +350,7 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { Set expectedMessages = new HashSet<>(); for (String embedding : config.expectedEmbeddings) { String expected = - "{\"name\":\"some name\",\"description\":\"some description\",\"embeddings\":%s}" + "{\"description\":\"some description\",\"embeddings\":%s,\"name\":\"some name\"}" .formatted(embedding); expectedMessages.add(expected); } @@ -528,11 +528,11 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { }; } String expectedContent = - "{\"name\":\" " - + name - + "\",\"description\":\"some description\",\"embeddings\":" + "{\"description\":\"some description\",\"embeddings\":" + embeddings - + "}"; + + ",\"name\":\" " + + name + + "\"}"; expected.add(expectedContent); } @@ -688,7 +688,7 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { waitForMessages( consumer, List.of( - "{\"name\":\"foo\",\"embeddings\":[1.0,5.400000095367432,8.699999809265137]}")); + "{\"embeddings\":[1.0,5.400000095367432,8.699999809265137],\"name\":\"foo\"}")); } } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CouchbaseAssetQueryWriteIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CouchbaseAssetQueryWriteIT.java index e9be58d28..429cc656e 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CouchbaseAssetQueryWriteIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/CouchbaseAssetQueryWriteIT.java @@ -20,8 +20,8 @@ import ai.langstream.ai.agents.commons.jstl.JstlFunctions; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -184,7 +184,8 @@ public void testCouchbase() throws Exception { @SneakyThrows public void accept(String m) { Map map = - new ObjectMapper().readValue(m, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(m, Map.class); Map queryresult = (Map) map.get("queryresult"); assertEquals("1", queryresult.get("id")); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ElasticSearchVectorIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ElasticSearchVectorIT.java index d5042286c..e5f0e53f6 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ElasticSearchVectorIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/ElasticSearchVectorIT.java @@ -15,6 +15,8 @@ */ package ai.langstream.agents; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; + import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; @@ -23,6 +25,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.junit.jupiter.Container; @@ -31,6 +34,7 @@ @Slf4j @Testcontainers +@Tag(INTEGRATION_TESTS_GROUP1) class ElasticSearchVectorIT extends AbstractGenericStreamingApplicationRunner { @Container static ElasticsearchContainer ELASTICSEARCH = @@ -179,7 +183,7 @@ public void test(boolean https, String host, int port, String apiKey) throws Exc waitForMessages( consumer, List.of( - "{\"embeddings\":[1,1,2],\"query-result\":[{\"embeddings\":[1,1,1],\"similarity\":0.9394879,\"index\":\"my-index-000\",\"id\":\"key9\",\"content\":\"hello9\"}]}")); + "{\"embeddings\":[1,1,2],\"query-result\":[{\"content\":\"hello9\",\"embeddings\":[1,1,1],\"id\":\"key9\",\"index\":\"my-index-000\",\"similarity\":0.9394879}]}")); } } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/FlowControlRunnerIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/FlowControlRunnerIT.java index 10c4edf05..0bc8ce7ec 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/FlowControlRunnerIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/FlowControlRunnerIT.java @@ -15,6 +15,7 @@ */ package ai.langstream.agents; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -22,17 +23,19 @@ import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.function.BiConsumer; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @Slf4j +@Tag(INTEGRATION_TESTS_GROUP2) class FlowControlRunnerIT extends AbstractGenericStreamingApplicationRunner { @Test @@ -282,7 +285,8 @@ public void accept(List consumerRecords, List objects) { Object key = consumerRecord.key(); log.info("Received key {}", key); Map jsonKey = - new ObjectMapper().readValue(key.toString(), Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(key.toString(), Map.class); // try to parse the UUID UUID.fromString(jsonKey.get("id").toString()); assertEquals( diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/HttpRequestAgentRunnerIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/HttpRequestAgentRunnerIT.java index c01550b1d..6fa4b3e20 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/HttpRequestAgentRunnerIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/HttpRequestAgentRunnerIT.java @@ -87,7 +87,7 @@ void testGetJson() throws Exception { final String e1 = """ - {"id":"my-model","classification":"good","api":{"id":"my-model","created":"2021-08-31T12:00:00Z","model":"gpt-35-turbo","object":"text-generation","choices":[{"text":"It is a car."}]}}"""; + {"api":{"choices":[{"text":"It is a car."}],"created":"2021-08-31T12:00:00Z","id":"my-model","model":"gpt-35-turbo","object":"text-generation"},"classification":"good","id":"my-model"}"""; runAndAssertMessage(application, e1); } @@ -128,7 +128,7 @@ void testGetRawText() throws Exception { final String e1 = """ - {"id":"my-model","classification":"good","api":"some-string"}"""; + {"api":"some-string","classification":"good","id":"my-model"}"""; runAndAssertMessage(application, e1); } @@ -182,7 +182,7 @@ void testPostWithBody() throws Exception { final String e1 = """ - {"id":"my-model","classification":"good","api":{"id":"my-model","created":"2021-08-31T12:00:00Z","model":"gpt-35-turbo","object":"text-generation","choices":[{"text":"It is a car."}]}}"""; + {"api":{"choices":[{"text":"It is a car."}],"created":"2021-08-31T12:00:00Z","id":"my-model","model":"gpt-35-turbo","object":"text-generation"},"classification":"good","id":"my-model"}"""; runAndAssertMessage(application, e1); } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/JdbcDatabaseIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/JdbcDatabaseIT.java index a096c1e5f..49ea95dc9 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/JdbcDatabaseIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/JdbcDatabaseIT.java @@ -19,6 +19,7 @@ import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; import ai.langstream.utils.HerdDBExtension; import com.fasterxml.jackson.databind.ObjectMapper; @@ -36,7 +37,7 @@ @Testcontainers class JdbcDatabaseIT extends AbstractGenericStreamingApplicationRunner { - static final ObjectMapper MAPPER = new ObjectMapper(); + static final ObjectMapper MAPPER = ObjectMapperFactory.getDefaultMapper(); @RegisterExtension static HerdDBExtension herdDB = new HerdDBExtension(); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/OpenSearchVectorIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/OpenSearchVectorIT.java index 92f15a077..33f4cf250 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/OpenSearchVectorIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/OpenSearchVectorIT.java @@ -159,9 +159,7 @@ public void test() throws Exception { waitForMessages( consumer, List.of( - "{\"embeddings\":[999,999,5],\"query-result\":[{\"score\":1.0," - + "\"document\":{\"embeddings\":[999,999,5],\"content\":\"hello5\"}," - + "\"index\":\"my-index-1\",\"id\":\"key5\"}]}")); + "{\"embeddings\":[999,999,5],\"query-result\":[{\"document\":{\"content\":\"hello5\",\"embeddings\":[999,999,5]},\"id\":\"key5\",\"index\":\"my-index-1\",\"score\":1.0}]}")); } } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/PineconeIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/PineconeIT.java index 91d2a9af6..04c2eda5f 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/PineconeIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/PineconeIT.java @@ -17,6 +17,7 @@ import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; @@ -230,7 +231,7 @@ public void testWithDynamicIndex() throws Exception { try (TopicProducer producer = createProducer("insert-topic"); TopicConsumer consumer = createConsumer("result-topic")) { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); for (int i = 0; i < 2; i++) { String content = mapper.writeValueAsString( diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/RerankAgentRunnerIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/RerankAgentRunnerIT.java index fc37b4ca9..22de54de1 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/RerankAgentRunnerIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/RerankAgentRunnerIT.java @@ -19,10 +19,10 @@ import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; import ai.langstream.utils.HerdDBExtension; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -42,7 +42,8 @@ class RerankAgentRunnerIT extends AbstractGenericStreamingApplicationRunner { private static void validateResults(String message) { log.info("Validating message: {}", message); Map parsed = - new ObjectMapper().readValue(message, new TypeReference>() {}); + ObjectMapperFactory.getDefaultMapper() + .readValue(message, new TypeReference>() {}); List> relatedDocuments = (List>) parsed.get("related_documents"); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3AssetIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3AssetIT.java index 691269ea5..2bb8bd8f8 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3AssetIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3AssetIT.java @@ -15,6 +15,7 @@ */ package ai.langstream.agents; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; @@ -25,6 +26,7 @@ import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; @@ -33,6 +35,7 @@ @Slf4j @Testcontainers +@Tag(INTEGRATION_TESTS_GROUP2) class S3AssetIT extends AbstractGenericStreamingApplicationRunner { @Container diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3SourceIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3SourceIT.java index 2ab6a2631..fdfb18814 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3SourceIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/S3SourceIT.java @@ -15,14 +15,15 @@ */ package ai.langstream.agents; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static org.junit.jupiter.api.Assertions.*; import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; import ai.langstream.ai.agents.commons.state.S3StateStorage; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; -import com.fasterxml.jackson.databind.ObjectMapper; import io.minio.*; import io.minio.errors.ErrorResponseException; import java.io.ByteArrayInputStream; @@ -32,6 +33,7 @@ import java.util.UUID; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; @@ -40,6 +42,7 @@ @Slf4j @Testcontainers +@Tag(INTEGRATION_TESTS_GROUP2) class S3SourceIT extends AbstractGenericStreamingApplicationRunner { @Container @@ -310,7 +313,8 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { @SneakyThrows public void accept(Object o) { Map map = - new ObjectMapper().readValue((String) o, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); List> newObjects = (List>) map.get("newObjects"); @@ -338,7 +342,8 @@ public void accept(Object o) { @SneakyThrows public void accept(Object o) { Map map = - new ObjectMapper().readValue((String) o, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); List> newObjects = (List>) map.get("newObjects"); List> updatedObjects = diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/SolrAssetQueryWriteIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/SolrAssetQueryWriteIT.java index 85c465bfd..cee975fc9 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/SolrAssetQueryWriteIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/SolrAssetQueryWriteIT.java @@ -20,8 +20,8 @@ import ai.langstream.ai.agents.commons.jstl.JstlFunctions; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -211,7 +211,8 @@ public void testSolr() throws Exception { @SneakyThrows public void accept(String m) { Map map = - new ObjectMapper().readValue(m, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue(m, Map.class); Map queryresult = (Map) map.get("queryresult"); assertEquals("1", queryresult.get("id")); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextCompletionsIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextCompletionsIT.java index 3882a91ef..707b3f28a 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextCompletionsIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextCompletionsIT.java @@ -190,7 +190,7 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { executeAgentRunners(applicationRuntime); String expected = - "{\"question\":\"the car\",\"answer\":\"I am an AI language model and I do not have personal experiences or the\",\"prompt\":\"{\\\"options\\\":{\\\"type\\\":\\\"ai-text-completions\\\",\\\"when\\\":null,\\\"model\\\":\\\"gpt-3.5-turbo-instruct\\\",\\\"prompt\\\":[\\\"What can you tell me about {{{ value.question }}} ?\\\"],\\\"stream-to-topic\\\":null,\\\"stream-response-completion-field\\\":null,\\\"min-chunks-per-message\\\":3,\\\"completion-field\\\":\\\"value.answer\\\",\\\"stream\\\":true,\\\"log-field\\\":\\\"value.prompt\\\",\\\"logprobs-field\\\":\\\"value.logprobs\\\",\\\"logprobs\\\":5.0,\\\"max-tokens\\\":null,\\\"temperature\\\":null,\\\"top-p\\\":null,\\\"logit-bias\\\":null,\\\"user\\\":null,\\\"stop\\\":null,\\\"presence-penalty\\\":null,\\\"frequency-penalty\\\":null,\\\"options\\\":null},\\\"messages\\\":[\\\"What can you tell me about the car ?\\\"],\\\"model\\\":\\\"gpt-3.5-turbo-instruct\\\"}\",\"logprobs\":{\"tokens\":[\"I\",\" am\",\" an\",\" AI\",\" language\",\" model\",\" and\",\" I\",\" do\",\" not\",\" have\",\" personal\",\" experiences\",\" or\",\" the\"],\"logprobs\":[-0.50947005,-0.81064594,-0.0639758,-0.007819127,-4.2176867,-9.771052E-5,-0.38906613,-1.1028589,-0.18535662,-9.115311E-5,-0.0122308275,-0.9290634,-0.2772571,-0.06607247,-2.1178281]}}"; + "{\"answer\":\"I am an AI language model and I do not have personal experiences or the\",\"logprobs\":{\"logprobs\":[-0.50947005,-0.81064594,-0.0639758,-0.007819127,-4.2176867,-9.771052E-5,-0.38906613,-1.1028589,-0.18535662,-9.115311E-5,-0.0122308275,-0.9290634,-0.2772571,-0.06607247,-2.1178281],\"tokens\":[\"I\",\" am\",\" an\",\" AI\",\" language\",\" model\",\" and\",\" I\",\" do\",\" not\",\" have\",\" personal\",\" experiences\",\" or\",\" the\"]},\"prompt\":\"{\\\"messages\\\":[\\\"What can you tell me about the car ?\\\"],\\\"model\\\":\\\"gpt-3.5-turbo-instruct\\\",\\\"options\\\":{\\\"completion-field\\\":\\\"value.answer\\\",\\\"log-field\\\":\\\"value.prompt\\\",\\\"logprobs\\\":5.0,\\\"logprobs-field\\\":\\\"value.logprobs\\\",\\\"min-chunks-per-message\\\":3,\\\"model\\\":\\\"gpt-3.5-turbo-instruct\\\",\\\"prompt\\\":[\\\"What can you tell me about {{{ value.question }}} ?\\\"],\\\"stream\\\":true,\\\"type\\\":\\\"ai-text-completions\\\"}}\",\"question\":\"the car\"}"; waitForMessages(consumer, List.of(expected)); } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextProcessingAgentsRunnerIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextProcessingAgentsRunnerIT.java index 939cf8af1..3b2749c2b 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextProcessingAgentsRunnerIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/TextProcessingAgentsRunnerIT.java @@ -186,9 +186,9 @@ public void testSplitThenJson(boolean useIntermediateTopic) throws Exception { waitForMessages( consumer, List.of( - "{\"chunk_text_length\":\"47\",\"text\":\"This text is written in English, but it is very\",\"text_num_chunks\":\"3\",\"chunk_id\":\"0\",\"chunk_num_tokens\":\"47\"}", - "{\"chunk_text_length\":\"5\",\"text\":\"long,\",\"text_num_chunks\":\"3\",\"chunk_id\":\"1\",\"chunk_num_tokens\":\"5\"}", - "{\"chunk_text_length\":\"40\",\"text\":\"so you may want to split it into chunks.\",\"text_num_chunks\":\"3\",\"chunk_id\":\"2\",\"chunk_num_tokens\":\"40\"}")); + "{\"chunk_id\":\"0\",\"chunk_num_tokens\":\"47\",\"chunk_text_length\":\"47\",\"text\":\"This text is written in English, but it is very\",\"text_num_chunks\":\"3\"}", + "{\"chunk_id\":\"1\",\"chunk_num_tokens\":\"5\",\"chunk_text_length\":\"5\",\"text\":\"long,\",\"text_num_chunks\":\"3\"}", + "{\"chunk_id\":\"2\",\"chunk_num_tokens\":\"40\",\"chunk_text_length\":\"40\",\"text\":\"so you may want to split it into chunks.\",\"text_num_chunks\":\"3\"}")); } } } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/WebCrawlerSourceIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/WebCrawlerSourceIT.java index d8eedc803..45279e8bd 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/WebCrawlerSourceIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/agents/WebCrawlerSourceIT.java @@ -15,14 +15,14 @@ */ package ai.langstream.agents; -import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP1; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; import static com.github.tomakehurst.wiremock.client.WireMock.*; import static org.junit.jupiter.api.Assertions.*; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runner.topics.TopicProducer; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import java.nio.file.Files; @@ -38,7 +38,7 @@ @Slf4j @WireMockTest -@Tag(INTEGRATION_TESTS_GROUP1) +@Tag(INTEGRATION_TESTS_GROUP2) class WebCrawlerSourceIT extends AbstractGenericStreamingApplicationRunner { static WireMockRuntimeInfo wireMockRuntimeInfo; @@ -174,7 +174,8 @@ tenant, appId, application, buildInstanceYaml(), expectedAgents)) { @SneakyThrows public void accept(Object o) { Map map = - new ObjectMapper().readValue((String) o, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); List> newUrls = (List>) map.get("newObjects"); @@ -212,7 +213,8 @@ public void accept(Object o) { @SneakyThrows public void accept(Object o) { Map map = - new ObjectMapper().readValue((String) o, Map.class); + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); List> newUrls = (List>) map.get("newObjects"); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsIT.java index 89236801a..c4bdb16fd 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsIT.java @@ -20,6 +20,7 @@ import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.runner.topics.TopicConsumer; import ai.langstream.api.runtime.ExecutionPlan; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.mockagents.MockAssetManagerCodeProvider; import ai.langstream.testrunners.AbstractGenericStreamingApplicationRunner; import com.fasterxml.jackson.databind.ObjectMapper; @@ -122,12 +123,12 @@ public void testDeployAsset() throws Exception { @SneakyThrows public void accept(Object o) { log.info("Received: {}", o); - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); Map read = mapper.readValue((String) o, Map.class); assertEquals("AssetCreated", read.get("type")); assertEquals("Asset", read.get("category")); assertEquals( - "{\"tenant\":\"tenant\",\"applicationId\":\"app\",\"asset\":{\"id\":\"my-table\",\"name\":\"my-table\",\"config\":{\"datasource\":{\"configuration\":{\"service\":\"jdbc\",\"driverClass\":\"org.postgresql.Driver\",\"url\":\"bar\"}},\"table\":\"my-table\"},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"asset-type\":\"mock-database-resource\",\"events-topic\":\"%s\"}}" + "{\"applicationId\":\"app\",\"asset\":{\"asset-type\":\"mock-database-resource\",\"config\":{\"datasource\":{\"configuration\":{\"driverClass\":\"org.postgresql.Driver\",\"service\":\"jdbc\",\"url\":\"bar\"}},\"table\":\"my-table\"},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"events-topic\":\"%s\",\"id\":\"my-table\",\"name\":\"my-table\"},\"tenant\":\"tenant\"}" .formatted(eventsTopic), mapper.writeValueAsString(read.get("source"))); assertNotNull(read.get("timestamp")); @@ -146,14 +147,16 @@ public void accept(Object o) { @SneakyThrows public void accept(Object o) { log.info("Received: {}", o); - ObjectMapper mapper = new ObjectMapper(); - Map read = mapper.readValue((String) o, Map.class); + Map read = + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); assertEquals("AssetDeleted", read.get("type")); assertEquals("Asset", read.get("category")); assertEquals( - "{\"tenant\":\"tenant\",\"applicationId\":\"app\",\"asset\":{\"id\":\"my-table\",\"name\":\"my-table\",\"config\":{\"datasource\":{\"configuration\":{\"service\":\"jdbc\",\"driverClass\":\"org.postgresql.Driver\",\"url\":\"bar\"}},\"table\":\"my-table\"},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"asset-type\":\"mock-database-resource\",\"events-topic\":\"%s\"}}" + "{\"applicationId\":\"app\",\"asset\":{\"asset-type\":\"mock-database-resource\",\"config\":{\"datasource\":{\"configuration\":{\"driverClass\":\"org.postgresql.Driver\",\"service\":\"jdbc\",\"url\":\"bar\"}},\"table\":\"my-table\"},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"events-topic\":\"%s\",\"id\":\"my-table\",\"name\":\"my-table\"},\"tenant\":\"tenant\"}" .formatted(eventsTopic), - mapper.writeValueAsString(read.get("source"))); + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(read.get("source"))); assertNotNull(read.get("timestamp")); } })); @@ -231,14 +234,16 @@ public void testDeployAssetFailed() throws Exception { @SneakyThrows public void accept(Object o) { log.info("Received: {}", o); - ObjectMapper mapper = new ObjectMapper(); - Map read = mapper.readValue((String) o, Map.class); + Map read = + ObjectMapperFactory.getDefaultMapper() + .readValue((String) o, Map.class); assertEquals("AssetCreationFailed", read.get("type")); assertEquals("Asset", read.get("category")); assertEquals( - "{\"tenant\":\"tenant\",\"applicationId\":\"app\",\"asset\":{\"id\":\"my-table\",\"name\":\"my-table\",\"config\":{\"fail\":true,\"datasource\":{\"configuration\":{\"service\":\"jdbc\",\"driverClass\":\"org.postgresql.Driver\",\"url\":\"bar\"}}},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"asset-type\":\"mock-database-resource\",\"events-topic\":\"%s\"}}" + "{\"applicationId\":\"app\",\"asset\":{\"asset-type\":\"mock-database-resource\",\"config\":{\"datasource\":{\"configuration\":{\"driverClass\":\"org.postgresql.Driver\",\"service\":\"jdbc\",\"url\":\"bar\"}},\"fail\":true},\"creation-mode\":\"create-if-not-exists\",\"deletion-mode\":\"delete\",\"events-topic\":\"%s\",\"id\":\"my-table\",\"name\":\"my-table\"},\"tenant\":\"tenant\"}" .formatted(eventsTopic), - mapper.writeValueAsString(read.get("source"))); + ObjectMapperFactory.getDefaultMapper() + .writeValueAsString(read.get("source"))); assertEquals( "Mock failure to deploy asset", ((Map) read.get("data")) diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/pravega/SimplePravegaIT.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/pravega/SimplePravegaIT.java index 33dfa94a2..c6c16229e 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/pravega/SimplePravegaIT.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/pravega/SimplePravegaIT.java @@ -15,6 +15,8 @@ */ package ai.langstream.pravega; +import static ai.langstream.testrunners.AbstractApplicationRunner.INTEGRATION_TESTS_GROUP2; + import ai.langstream.api.model.StreamingCluster; import ai.langstream.api.runner.code.SimpleRecord; import ai.langstream.api.runner.topics.TopicConnectionsRuntime; @@ -28,9 +30,11 @@ import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @Slf4j +@Tag(INTEGRATION_TESTS_GROUP2) class SimplePravegaIT extends AbstractGenericStreamingApplicationRunner { public SimplePravegaIT() { super("pravega"); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarterTest.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarterTest.java index 4137e2d1f..a80d2bb6e 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarterTest.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentCodeDownloaderStarterTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.*; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.api.ClusterConfiguration; import ai.langstream.runtime.api.agent.AgentCodeDownloaderConstants; import ai.langstream.runtime.api.agent.DownloadAgentCodeConfiguration; @@ -31,7 +32,7 @@ class AgentCodeDownloaderStarterTest { - static ObjectMapper mapper = new ObjectMapper(); + static ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); @Test public void test() throws Exception { diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentRunnerStarterTest.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentRunnerStarterTest.java index f253d322b..6e713aa53 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentRunnerStarterTest.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/agent/AgentRunnerStarterTest.java @@ -17,6 +17,7 @@ import static org.mockito.ArgumentMatchers.eq; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.api.agent.AgentRunnerConstants; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,7 +31,7 @@ class AgentRunnerStarterTest { - static ObjectMapper mapper = new ObjectMapper(); + static ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); @Test public void test() throws Exception { diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/deployer/RuntimeDeployerStarterTest.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/deployer/RuntimeDeployerStarterTest.java index b416a6fd4..c78c8229e 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/deployer/RuntimeDeployerStarterTest.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/deployer/RuntimeDeployerStarterTest.java @@ -16,6 +16,7 @@ package ai.langstream.runtime.deployer; import ai.langstream.api.model.Secrets; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.runtime.api.agent.AgentRunnerConstants; import ai.langstream.runtime.api.deployer.RuntimeDeployerConfiguration; import ai.langstream.runtime.api.deployer.RuntimeDeployerConstants; @@ -40,7 +41,7 @@ class RuntimeDeployerStarterTest { log.info("Agents directory is {}", agentsDirectory); } - static ObjectMapper mapper = new ObjectMapper(); + static ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); @Test public void test() throws Exception { diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/testrunners/AbstractApplicationRunner.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/testrunners/AbstractApplicationRunner.java index 7aa868eec..e61b7088f 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/testrunners/AbstractApplicationRunner.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/testrunners/AbstractApplicationRunner.java @@ -28,6 +28,7 @@ import ai.langstream.api.runtime.DeployContext; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.agents.AgentResourcesFactory; import ai.langstream.impl.deploy.ApplicationDeployer; import ai.langstream.impl.k8s.tests.KubeTestServer; @@ -37,7 +38,6 @@ import ai.langstream.runtime.agent.api.AgentAPIController; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.github.dockerjava.api.model.Image; import io.fabric8.kubernetes.api.model.Secret; import java.io.IOException; @@ -70,12 +70,10 @@ @Slf4j public abstract class AbstractApplicationRunner { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper OBJECT_MAPPER = ObjectMapperFactory.getDefaultMapper(); public static final String INTEGRATION_TESTS_GROUP1 = "group-1"; - public static final ObjectMapper JSON_MAPPER = - new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true); - + public static final String INTEGRATION_TESTS_GROUP2 = "group-2"; private static final int DEFAULT_NUM_LOOPS = 10; public static final Path agentsDirectory; diff --git a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/KubeTestServer.java b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/KubeTestServer.java index 61888ebbb..d41d649ac 100644 --- a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/KubeTestServer.java +++ b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/KubeTestServer.java @@ -17,11 +17,10 @@ import static org.mockito.ArgumentMatchers.isNull; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource; import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource; import ai.langstream.impl.k8s.KubernetesClientFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.NamespacedKubernetesClient; @@ -113,15 +112,11 @@ public Map spyAgentCustomResources( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = - new ObjectMapper() - .enable( - SerializationFeature - .ORDER_MAP_ENTRIES_BY_KEYS); final AgentCustomResource agent = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - AgentCustomResource.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + AgentCustomResource.class); log.debug("received patch request for agent {}", agentId); currentAgents.put(agentId, agent); return agent; @@ -172,11 +167,11 @@ public Map spyAgentCustomResourcesSecrets( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = new ObjectMapper(); final Secret secret = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - Secret.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + Secret.class); log.debug( "received patch request secret for agent {}: {}", agentId, @@ -274,15 +269,11 @@ public Map spyApplicationCustomResources( final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); recordedRequest.getBody().copyTo(byteArrayOutputStream); - final ObjectMapper mapper = - new ObjectMapper() - .enable( - SerializationFeature - .ORDER_MAP_ENTRIES_BY_KEYS); final ApplicationCustomResource app = - mapper.readValue( - byteArrayOutputStream.toByteArray(), - ApplicationCustomResource.class); + ObjectMapperFactory.getDefaultMapper() + .readValue( + byteArrayOutputStream.toByteArray(), + ApplicationCustomResource.class); log.debug("received patch request for app {}", appId); currentApplications.put(appId, app); return app; diff --git a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/LocalApplicationRunner.java b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/LocalApplicationRunner.java index d61ade6d6..ab29b80a3 100644 --- a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/LocalApplicationRunner.java +++ b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/LocalApplicationRunner.java @@ -21,6 +21,7 @@ import ai.langstream.api.runner.code.MetricsReporter; import ai.langstream.api.runner.topics.TopicConnectionsRuntimeRegistry; import ai.langstream.api.runtime.*; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.deployer.k8s.agents.AgentResourcesFactory; import ai.langstream.impl.deploy.ApplicationDeployer; import ai.langstream.impl.nar.NarFileHandler; @@ -30,7 +31,6 @@ import ai.langstream.runtime.agent.metrics.PrometheusMetricsReporter; import ai.langstream.runtime.api.agent.RuntimePodConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.fabric8.kubernetes.api.model.Secret; import java.io.IOException; import java.io.OutputStream; @@ -56,7 +56,7 @@ public class LocalApplicationRunner implements AutoCloseable, InMemoryApplicationStore.AgentInfoCollector { - private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper MAPPER = ObjectMapperFactory.getDefaultYamlMapper(); final KubeTestServer kubeServer = new KubeTestServer(); final InMemoryApplicationStore applicationStore = new InMemoryApplicationStore(); diff --git a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/Main.java b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/Main.java index 0d0fa8463..4f4069a18 100644 --- a/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/Main.java +++ b/langstream-runtime/langstream-runtime-tester/src/main/java/ai/langstream/runtime/tester/Main.java @@ -18,6 +18,7 @@ import ai.langstream.api.model.Application; import ai.langstream.api.storage.GlobalMetadataStore; import ai.langstream.api.storage.GlobalMetadataStoreRegistry; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.application.ApplicationDescription; import ai.langstream.api.webservice.tenant.TenantConfiguration; import ai.langstream.apigateway.LangStreamApiGateway; @@ -226,7 +227,6 @@ private static Server bootstrapHttpServer(LocalApplicationRunner runner) throws } private static class AgentControllerServlet extends HttpServlet { - private static final ObjectMapper MAPPER = new ObjectMapper(); private final LocalApplicationRunner agentAPIController; @@ -247,7 +247,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I for (AgentAPIController controller : agents) { result.add(controller.restart()); } - MAPPER.writeValue(resp.getOutputStream(), result); + ObjectMapperFactory.getDefaultMapper() + .writeValue(resp.getOutputStream(), result); } catch (Throwable error) { log.error("Error while restarting the agents"); resp.getOutputStream().write((error + "").getBytes()); diff --git a/langstream-webservice/pom.xml b/langstream-webservice/pom.xml index e044f581b..e4954b440 100644 --- a/langstream-webservice/pom.xml +++ b/langstream-webservice/pom.xml @@ -204,7 +204,7 @@ test - com.github.tomakehurst + org.wiremock wiremock test diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java b/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java index 907b0b7bb..a7f3b8a1d 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java @@ -21,12 +21,12 @@ import ai.langstream.api.model.ApplicationSpecs; import ai.langstream.api.model.StoredApplication; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.api.webservice.application.ApplicationCodeInfo; import ai.langstream.api.webservice.application.ApplicationDescription; import ai.langstream.impl.common.ApplicationPlaceholderResolver; import ai.langstream.impl.parser.ModelBuilder; import ai.langstream.webservice.security.infrastructure.primary.TokenAuthFilter; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -414,7 +414,8 @@ static class LogLineConsumer implements ApplicationStore.LogLineConsumer { private static final String[] LOG_COLORS = new String[] {"32", "33", "34", "35", "36", "37", "38"}; - private static final ObjectWriter newlineDelimitedJSONWriter = new ObjectMapper().writer(); + private static final ObjectWriter newlineDelimitedJSONWriter = + ObjectMapperFactory.getDefaultMapper().writer(); private final ApplicationStore.PodLogHandler podLog; private final Consumer lastSent; diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java index a932e22bb..cdb6b0d63 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java @@ -16,9 +16,9 @@ package ai.langstream.webservice.archetype; import ai.langstream.api.archetype.ArchetypeDefinition; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.cli.utils.ApplicationPackager; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -31,7 +31,7 @@ @Slf4j public class ArchetypeStore { - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper mapper = ObjectMapperFactory.getDefaultYamlMapper(); private final Map archetypeDefinitions = new HashMap<>(); private final Map archetypePaths = new HashMap<>(); diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java index 2b8bf5788..aabeb31b7 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java @@ -16,9 +16,7 @@ package ai.langstream.webservice.doc; import ai.langstream.api.doc.ApiConfigurationModel; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; +import ai.langstream.api.util.ObjectMapperFactory; import java.nio.file.Path; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -34,16 +32,11 @@ public static void main(String[] args) { System.exit(-1); } - final ObjectMapper jsonWriter = - new ObjectMapper() - .configure(SerializationFeature.INDENT_OUTPUT, true) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - final String outputDir = args[0]; final String version = args[1]; final Path agentsFile = Path.of(outputDir).resolve("api.json"); final ApiConfigurationModel model = DocumentationGenerator.generateDocs(version); - jsonWriter.writeValue(agentsFile.toFile(), model); + ObjectMapperFactory.getPrettyPrintMapper().writeValue(agentsFile.toFile(), model); System.out.println( "Generated documentation with %d agents, %d resources, %d assets" .formatted( diff --git a/langstream-webservice/src/main/resources/application.properties b/langstream-webservice/src/main/resources/application.properties index 72b37f0df..0b6ef5ed1 100644 --- a/langstream-webservice/src/main/resources/application.properties +++ b/langstream-webservice/src/main/resources/application.properties @@ -28,6 +28,7 @@ application.security.token.kubernetes-namespace-prefix=langstream- spring.jackson.serialization.indent-output=true spring.jackson.serialization.order-map-entries-by-keys=true +spring.jackson.default-property-inclusion=non_null application.storage.global.type=local application.storage.apps.type=kubernetes diff --git a/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceLogsTest.java b/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceLogsTest.java index 63c2c7208..907bc1d62 100644 --- a/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceLogsTest.java +++ b/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceLogsTest.java @@ -68,7 +68,7 @@ public void close() {} assertEquals("\u001B[32m[mypod-0] line logs\u001B[0m\n", output.get()); } else { assertEquals( - "{\"replica\":\"mypod-0\",\"message\":\"line logs\",\"timestamp\":1696926662058}\n", + "{\"message\":\"line logs\",\"replica\":\"mypod-0\",\"timestamp\":1696926662058}\n", output.get()); } @@ -82,8 +82,8 @@ public void close() {} output.get()); } else { assertEquals( - "{\"replica\":\"mypod-0\",\"message\":\"Replica mypod-0 is not running, will retry in 10 seconds" - + ". State: Error, Reason: Some long exception\"}\n", + "{\"message\":\"Replica mypod-0 is not running, will retry in 10 seconds" + + ". State: Error, Reason: Some long exception\",\"replica\":\"mypod-0\"}\n", output.get()); } @@ -96,8 +96,8 @@ public void close() {} output.get()); } else { assertEquals( - "{\"replica\":\"mypod-0\",\"message\":\"Replica mypod-0 logs not available, will retry in 10 " - + "seconds\"}\n", + "{\"message\":\"Replica mypod-0 logs not available, will retry in 10 " + + "seconds\",\"replica\":\"mypod-0\"}\n", output.get()); } } diff --git a/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceTest.java b/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceTest.java index 3d0ab0742..2b9c2c03f 100644 --- a/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceTest.java +++ b/langstream-webservice/src/test/java/ai/langstream/webservice/application/ApplicationResourceTest.java @@ -102,48 +102,46 @@ void testAppCrud() throws Exception { result -> assertEquals( """ - { - "application-id" : "test", - "application" : { - "resources" : { }, - "modules" : [ { - "id" : "default", - "pipelines" : [ { - "id" : "app1", - "module" : "default", - "name" : "test", - "resources" : { - "parallelism" : 1, - "size" : 1 - }, - "errors" : { - "retries" : 0, - "on-failure" : "fail" - }, - "agents" : [ ] - } ], - "topics" : [ ] - } ], - "instance" : { - "streamingCluster" : { - "type" : "pulsar", - "configuration" : { } - }, - "computeCluster" : { - "type" : "none", - "configuration" : { } - }, - "globals" : null - } - }, - "status" : { - "status" : { - "status" : "CREATED", - "reason" : null - }, - "executors" : [ ] - } - }""", + { + "application-id" : "test", + "application" : { + "resources" : { }, + "modules" : [ { + "id" : "default", + "pipelines" : [ { + "id" : "app1", + "module" : "default", + "name" : "test", + "resources" : { + "parallelism" : 1, + "size" : 1 + }, + "errors" : { + "retries" : 0, + "on-failure" : "fail" + }, + "agents" : [ ] + } ], + "topics" : [ ] + } ], + "instance" : { + "streamingCluster" : { + "type" : "pulsar", + "configuration" : { } + }, + "computeCluster" : { + "type" : "none", + "configuration" : { } + } + } + }, + "status" : { + "status" : { + "status" : "CREATED" + }, + "executors" : [ ] + } + }""", result.getResponse().getContentAsString())); mockMvc.perform(get("/api/applications/my-tenant")) @@ -354,7 +352,6 @@ void testDeployDryRun() throws Exception { "id" : "app1-ai-chat-completions-1", "name" : "ai-chat-completions", "type" : "ai-chat-completions", - "input" : null, "output" : { "connectionType" : "TOPIC", "definition" : "history-topic", @@ -371,16 +368,11 @@ void testDeployDryRun() throws Exception { "retries" : 0, "on-failure" : "fail" }, - "signalsFrom" : null, "deletionMode" : "cleanup" } ] } ], "topics" : [ { "name" : "history-topic", - "config" : null, - "options" : null, - "keySchema" : null, - "valueSchema" : null, "partitions" : 0, "implicit" : false, "creation-mode" : "none", diff --git a/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java b/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java index e835ecd85..339d09032 100644 --- a/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java +++ b/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java @@ -24,6 +24,7 @@ import ai.langstream.api.model.Secrets; import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.api.util.ObjectMapperFactory; import ai.langstream.impl.k8s.tests.KubeK3sServer; import ai.langstream.webservice.WebAppTestConfig; import ai.langstream.webservice.application.CodeStorageService; @@ -48,7 +49,7 @@ @DirtiesContext class ArchetypeResourceTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = ObjectMapperFactory.getDefaultMapper(); @Autowired MockMvc mockMvc; @@ -82,105 +83,57 @@ void testArchetypesMetadata() throws Exception { log.info("Result {}", result.getResponse().getContentAsString()); assertEquals( """ - { - "archetype" : { - "id" : "simple", - "title" : "Simple", - "labels" : null, - "description" : null, - "icon" : null, - "sections" : [ { - "title" : "Section 1", - "description" : "Xxxxx", - "parameters" : [ { - "default" : null, - "name" : "s1", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.string-value", - "required" : false - }, { - "default" : null, - "name" : "i1", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.input-value", - "required" : false - }, { - "default" : null, - "name" : "r1", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.int-value", - "required" : false - }, { - "default" : null, - "name" : "m1", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.map-value", - "required" : false - }, { - "default" : null, - "name" : "l1", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.list-value", - "required" : false - }, { - "default" : null, - "name" : "m2", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "globals.nested-map.key2.key2-1", - "required" : false - } ] - }, { - "title" : "Section 2", - "description" : "Xxxxx", - "parameters" : [ { - "default" : null, - "name" : "s2", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "secrets.open-ai.foo", - "required" : false - }, { - "default" : null, - "name" : "i2", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "secrets.open-ai.foo-int", - "required" : false - }, { - "default" : null, - "name" : "k2", - "label" : null, - "description" : null, - "type" : null, - "subtype" : null, - "binding" : "secrets.kafka.bootstrap-servers", - "required" : false - } ] - } ] - } - }""", + { + "archetype" : { + "id" : "simple", + "title" : "Simple", + "sections" : [ { + "title" : "Section 1", + "description" : "Xxxxx", + "parameters" : [ { + "name" : "s1", + "binding" : "globals.string-value", + "required" : false + }, { + "name" : "i1", + "binding" : "globals.input-value", + "required" : false + }, { + "name" : "r1", + "binding" : "globals.int-value", + "required" : false + }, { + "name" : "m1", + "binding" : "globals.map-value", + "required" : false + }, { + "name" : "l1", + "binding" : "globals.list-value", + "required" : false + }, { + "name" : "m2", + "binding" : "globals.nested-map.key2.key2-1", + "required" : false + } ] + }, { + "title" : "Section 2", + "description" : "Xxxxx", + "parameters" : [ { + "name" : "s2", + "binding" : "secrets.open-ai.foo", + "required" : false + }, { + "name" : "i2", + "binding" : "secrets.open-ai.foo-int", + "required" : false + }, { + "name" : "k2", + "binding" : "secrets.kafka.bootstrap-servers", + "required" : false + } ] + } ] + } + }""", result.getResponse().getContentAsString()); }); } @@ -216,7 +169,7 @@ void testDeployFromArchetype() throws Exception { .andExpect(status().isOk()) .andExpect( result -> { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = ObjectMapperFactory.getDefaultMapper(); String body = result.getResponse().getContentAsString(); log.info("Deploy result {}", body); Map parse = diff --git a/pom.xml b/pom.xml index bdfb1ae50..dfd2a162b 100644 --- a/pom.xml +++ b/pom.xml @@ -37,14 +37,12 @@ 1.10.2 1.0.0-beta.8 1.2.24 - - 4.1.97.Final - 2.0.61.Final - 0.0.21.Final + 4.1.111.Final + 0.0.24.Final 1.19.8 - 5.9.3 - 1.18.30 - 2.14.2 + 5.11.0 + 1.18.34 + 2.17.2 32.1.3-jre 1.33.0 1.4.14 @@ -69,7 +67,7 @@ 8.5.10 1.70 1.4.1 - 3.0.1 + 3.9.1 0.25.5 0.11.5 1.16.0 @@ -447,7 +445,7 @@ ${kindcontainer.version} - com.github.tomakehurst + org.wiremock wiremock ${wiremock.version} test