From 3223c0262cc0e8876f4095ce32d3ee1cbe950dd0 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Thu, 26 Oct 2023 09:34:28 +0200 Subject: [PATCH] Archetypes - part 1 (#642) --- .../webcrawler-source/crawler.yaml | 2 +- .../langstream/admin/client/AdminClient.java | 21 ++ .../admin/client/model/Archetypes.java | 22 ++ langstream-api-gateway/pom.xml | 2 - .../api/archetype/ArchetypeDefinition.java | 42 +++ .../cli/commands/RootArchetypeCmd.java | 29 ++ .../ai/langstream/cli/commands/RootCmd.java | 1 + .../AbstractDeployApplicationCmd.java | 58 +--- .../commands/archetypes/BaseArchetypeCmd.java | 31 ++ .../archetypes/ListArchetypesCmd.java | 53 ++++ .../cli/utils/ApplicationPackager.java | 79 ++++++ .../commands/applications/AppsCmdTest.java | 22 +- .../langstream/impl/parser/ModelBuilder.java | 188 ++++++++++-- .../impl/parser/ModelBuilderTest.java | 18 +- langstream-webservice/pom.xml | 4 +- .../archetypes/website-qa-chatbot/.gitignore | 1 + .../website-qa-chatbot/archetype.yaml | 166 +++++++++++ .../website-qa-chatbot/chatbot.yaml | 87 ++++++ .../website-qa-chatbot/configuration.yaml | 35 +++ .../website-qa-chatbot/crawler.yaml | 97 +++++++ .../website-qa-chatbot/gateways.yaml | 43 +++ .../website-qa-chatbot/instance.yaml | 30 ++ .../website-qa-chatbot/secrets.yaml | 50 ++++ .../website-qa-chatbot/write-to-astra.yaml | 62 ++++ .../LangStreamControlPlaneWebApplication.java | 2 + .../application/ApplicationResource.java | 2 +- .../archetype/ArchetypeBasicInfo.java | 32 +++ .../archetype/ArchetypeResource.java | 194 +++++++++++++ .../archetype/ArchetypeService.java | 50 ++++ .../webservice/archetype/ArchetypeStore.java | 92 ++++++ .../archetype/ArchetypeStoreFactory.java | 34 +++ .../config/ArchetypesProperties.java | 30 ++ .../src/main/resources/application.properties | 4 +- .../test/archetypes/simple/.langstreamignore | 149 ++++++++++ .../src/test/archetypes/simple/archetype.yaml | 44 +++ .../test/archetypes/simple/configuration.yaml | 24 ++ .../src/test/archetypes/simple/gateways.yaml | 28 ++ .../src/test/archetypes/simple/instance.yaml | 38 +++ .../src/test/archetypes/simple/pipeline.yaml | 29 ++ .../test/archetypes/simple/python/example.py | 24 ++ .../src/test/archetypes/simple/secrets.yaml | 27 ++ .../webservice/application/AppTestHelper.java | 4 +- .../archetype/ArchetypeResourceTest.java | 268 ++++++++++++++++++ pom.xml | 2 +- 44 files changed, 2117 insertions(+), 103 deletions(-) create mode 100644 langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Archetypes.java create mode 100644 langstream-api/src/main/java/ai/langstream/api/archetype/ArchetypeDefinition.java create mode 100644 langstream-cli/src/main/java/ai/langstream/cli/commands/RootArchetypeCmd.java create mode 100644 langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/BaseArchetypeCmd.java create mode 100644 langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/ListArchetypesCmd.java create mode 100644 langstream-cli/src/main/java/ai/langstream/cli/utils/ApplicationPackager.java create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/.gitignore create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/archetype.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/chatbot.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/configuration.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/crawler.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/gateways.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/instance.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/secrets.yaml create mode 100644 langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/write-to-astra.yaml create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeBasicInfo.java create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeResource.java create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeService.java create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStoreFactory.java create mode 100644 langstream-webservice/src/main/java/ai/langstream/webservice/config/ArchetypesProperties.java create mode 100644 langstream-webservice/src/test/archetypes/simple/.langstreamignore create mode 100644 langstream-webservice/src/test/archetypes/simple/archetype.yaml create mode 100644 langstream-webservice/src/test/archetypes/simple/configuration.yaml create mode 100644 langstream-webservice/src/test/archetypes/simple/gateways.yaml create mode 100644 langstream-webservice/src/test/archetypes/simple/instance.yaml create mode 100644 langstream-webservice/src/test/archetypes/simple/pipeline.yaml create mode 100644 langstream-webservice/src/test/archetypes/simple/python/example.py create mode 100644 langstream-webservice/src/test/archetypes/simple/secrets.yaml create mode 100644 langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java diff --git a/examples/applications/webcrawler-source/crawler.yaml b/examples/applications/webcrawler-source/crawler.yaml index 5477a26bf..020a7ba10 100644 --- a/examples/applications/webcrawler-source/crawler.yaml +++ b/examples/applications/webcrawler-source/crawler.yaml @@ -87,7 +87,7 @@ pipeline: type: "compute-ai-embeddings" output: "chunks-topic" configuration: - model: "text-embedding-ada-002" # This needs to match the name of the model deployment, not the base model + model: "${secrets.open-ai.embeddings-model}" embeddings-field: "value.embeddings_vector" text: "{{ value.text }}" batch-size: 10 diff --git a/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java b/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java index 75d51ef91..300848ffd 100644 --- a/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java +++ b/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java @@ -19,6 +19,7 @@ import ai.langstream.admin.client.http.HttpClientProperties; import ai.langstream.admin.client.http.Retry; import ai.langstream.admin.client.model.Applications; +import ai.langstream.admin.client.model.Archetypes; import ai.langstream.admin.client.util.MultiPartBodyPublisher; import ai.langstream.admin.client.util.Slf4jLAdminClientLogger; import java.io.InputStream; @@ -204,6 +205,26 @@ public Applications applications() { return new ApplicationsImpl(); } + public Archetypes archetypes() { + return new ArchetypesImpl(); + } + + private class ArchetypesImpl implements Archetypes { + @Override + @SneakyThrows + public String list() { + final String tenant = configuration.getTenant(); + return http(newGet(String.format("/archetypes/%s", tenant))).body(); + } + + @Override + @SneakyThrows + public String get(String archetype) { + final String tenant = configuration.getTenant(); + return http(newGet(String.format("/archetypes/%s/%s", tenant, archetype))).body(); + } + } + private class ApplicationsImpl implements Applications { @Override public String deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher) { diff --git a/langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Archetypes.java b/langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Archetypes.java new file mode 100644 index 000000000..11918c1fc --- /dev/null +++ b/langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Archetypes.java @@ -0,0 +1,22 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.admin.client.model; + +public interface Archetypes { + String list(); + + String get(String archetypeId); +} diff --git a/langstream-api-gateway/pom.xml b/langstream-api-gateway/pom.xml index b4b1e1650..b465cb315 100644 --- a/langstream-api-gateway/pom.xml +++ b/langstream-api-gateway/pom.xml @@ -32,8 +32,6 @@ UTF-8 UTF-8 2.1.0 - 3.3.1 - diff --git a/langstream-api/src/main/java/ai/langstream/api/archetype/ArchetypeDefinition.java b/langstream-api/src/main/java/ai/langstream/api/archetype/ArchetypeDefinition.java new file mode 100644 index 000000000..c6674e9d1 --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/archetype/ArchetypeDefinition.java @@ -0,0 +1,42 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.archetype; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public record ArchetypeDefinition(Archetype archetype) { + + public record Archetype( + String id, + String title, + List labels, + String description, + String icon, + List
sections) {} + + public record Section(String title, String description, List parameters) {} + + public record Parameter( + String name, + String label, + String description, + String type, + String subtype, + String binding, + boolean required, + @JsonProperty("default") Object defaultVal) {} +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/RootArchetypeCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootArchetypeCmd.java new file mode 100644 index 000000000..1bf57a26d --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootArchetypeCmd.java @@ -0,0 +1,29 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.commands; + +import ai.langstream.cli.commands.archetypes.ListArchetypesCmd; +import lombok.Getter; +import picocli.CommandLine; + +@CommandLine.Command( + name = "archetypes", + header = "Use LangStream Archetypes", + subcommands = {ListArchetypesCmd.class}) +@Getter +public class RootArchetypeCmd { + @CommandLine.ParentCommand private RootCmd rootCmd; +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java index e2725afb9..f826e47e5 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java @@ -27,6 +27,7 @@ scope = CommandLine.ScopeType.INHERIT, header = "LangStream CLI", subcommands = { + RootArchetypeCmd.class, RootAppCmd.class, ConfigureCmd.class, RootTenantCmd.class, diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java index 0976bf52e..d6f5b4338 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java @@ -15,19 +15,16 @@ */ package ai.langstream.cli.commands.applications; +import static ai.langstream.cli.utils.ApplicationPackager.buildZip; + import ai.langstream.admin.client.util.MultiPartBodyPublisher; -import ai.langstream.cli.commands.GitIgnoreParser; import ai.langstream.cli.util.LocalFileReferenceResolver; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.function.Consumer; import lombok.SneakyThrows; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.model.ZipParameters; import picocli.CommandLine; public abstract class AbstractDeployApplicationCmd extends BaseApplicationCmd { @@ -255,57 +252,6 @@ public void run() { } } - public static Path buildZip(File appDirectory, Consumer logger) throws IOException { - final Path tempZip = Files.createTempFile("app", ".zip"); - try (final ZipFile zip = new ZipFile(tempZip.toFile())) { - addApp(appDirectory, zip, logger); - } - return tempZip; - } - - private static void addApp(File appDirectory, ZipFile zip, Consumer logger) - throws IOException { - if (appDirectory == null) { - return; - } - logger.accept(String.format("packaging app: %s", appDirectory.getAbsolutePath())); - if (appDirectory.isDirectory()) { - File ignoreFile = appDirectory.toPath().resolve(".langstreamignore").toFile(); - if (ignoreFile.exists()) { - GitIgnoreParser parser = new GitIgnoreParser(ignoreFile.toPath()); - addDirectoryFilesWithLangstreamIgnore(appDirectory, appDirectory, parser, zip); - } else { - for (File file : appDirectory.listFiles()) { - if (file.isDirectory()) { - zip.addFolder(file); - } else { - zip.addFile(file); - } - } - } - } else { - zip.addFile(appDirectory); - } - logger.accept("app packaged"); - } - - private static void addDirectoryFilesWithLangstreamIgnore( - File appDirectory, File directory, GitIgnoreParser parser, ZipFile zip) - throws IOException { - for (File file : directory.listFiles()) { - if (!parser.matches(file)) { - if (file.isDirectory()) { - addDirectoryFilesWithLangstreamIgnore(appDirectory, file, parser, zip); - } else { - ZipParameters zipParameters = new ZipParameters(); - String filename = appDirectory.toURI().relativize(file.toURI()).getPath(); - zipParameters.setFileNameInZip(filename); - zip.addFile(file, zipParameters); - } - } - } - } - public static MultiPartBodyPublisher buildMultipartContentForAppZip( Map formData) { final MultiPartBodyPublisher multiPartBodyPublisher = new MultiPartBodyPublisher(); diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/BaseArchetypeCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/BaseArchetypeCmd.java new file mode 100644 index 000000000..6cf8b6025 --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/BaseArchetypeCmd.java @@ -0,0 +1,31 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.commands.archetypes; + +import ai.langstream.cli.commands.BaseCmd; +import ai.langstream.cli.commands.RootArchetypeCmd; +import ai.langstream.cli.commands.RootCmd; +import picocli.CommandLine; + +public abstract class BaseArchetypeCmd extends BaseCmd { + + @CommandLine.ParentCommand private RootArchetypeCmd rootAppCmd; + + @Override + protected RootCmd getRootCmd() { + return rootAppCmd.getRootCmd(); + } +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/ListArchetypesCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/ListArchetypesCmd.java new file mode 100644 index 000000000..7494d4eba --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/ListArchetypesCmd.java @@ -0,0 +1,53 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.commands.archetypes; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.function.BiFunction; +import lombok.SneakyThrows; +import picocli.CommandLine; + +@CommandLine.Command(name = "list", header = "List all LangStream archetypes") +public class ListArchetypesCmd extends BaseArchetypeCmd { + + protected static final String[] COLUMNS_FOR_RAW = {"id", "labels"}; + + @CommandLine.Option( + names = {"-o"}, + description = "Output format. Formats are: yaml, json, raw. Default value is raw.") + private Formats format = Formats.raw; + + @Override + @SneakyThrows + public void run() { + ensureFormatIn(format, Formats.raw, Formats.json, Formats.yaml); + final String body = getClient().archetypes().list(); + print(format, body, COLUMNS_FOR_RAW, getRawFormatValuesSupplier()); + } + + public static BiFunction getRawFormatValuesSupplier() { + return (jsonNode, s) -> { + switch (s) { + case "id": + return searchValueInJson(jsonNode, "archetype.id"); + case "labels": + return searchValueInJson(jsonNode, "archetype.labels"); + default: + return jsonNode.get(s); + } + }; + } +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/utils/ApplicationPackager.java b/langstream-cli/src/main/java/ai/langstream/cli/utils/ApplicationPackager.java new file mode 100644 index 000000000..1b536adbe --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/utils/ApplicationPackager.java @@ -0,0 +1,79 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.utils; + +import ai.langstream.cli.commands.GitIgnoreParser; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Consumer; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.model.ZipParameters; + +public class ApplicationPackager { + + public static Path buildZip(File appDirectory, Consumer logger) throws IOException { + final Path tempZip = Files.createTempFile("app", ".zip"); + try (final ZipFile zip = new ZipFile(tempZip.toFile())) { + addApp(appDirectory, zip, logger); + } + return tempZip; + } + + private static void addApp(File appDirectory, ZipFile zip, Consumer logger) + throws IOException { + if (appDirectory == null) { + return; + } + logger.accept(String.format("packaging app: %s", appDirectory.getAbsolutePath())); + if (appDirectory.isDirectory()) { + File ignoreFile = appDirectory.toPath().resolve(".langstreamignore").toFile(); + if (ignoreFile.exists()) { + GitIgnoreParser parser = new GitIgnoreParser(ignoreFile.toPath()); + addDirectoryFilesWithLangstreamIgnore(appDirectory, appDirectory, parser, zip); + } else { + for (File file : appDirectory.listFiles()) { + if (file.isDirectory()) { + zip.addFolder(file); + } else { + zip.addFile(file); + } + } + } + } else { + zip.addFile(appDirectory); + } + logger.accept("app packaged"); + } + + private static void addDirectoryFilesWithLangstreamIgnore( + File appDirectory, File directory, GitIgnoreParser parser, ZipFile zip) + throws IOException { + for (File file : directory.listFiles()) { + if (!parser.matches(file)) { + if (file.isDirectory()) { + addDirectoryFilesWithLangstreamIgnore(appDirectory, file, parser, zip); + } else { + ZipParameters zipParameters = new ZipParameters(); + String filename = appDirectory.toURI().relativize(file.toURI()).getPath(); + zipParameters.setFileNameInZip(filename); + zip.addFile(file, zipParameters); + } + } + } + } +} diff --git a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java index 94a297115..29c80e3c8 100644 --- a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java +++ b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java @@ -15,6 +15,7 @@ */ package ai.langstream.cli.commands.applications; +import static ai.langstream.cli.utils.ApplicationPackager.buildZip; import static com.github.tomakehurst.wiremock.client.WireMock.aMultipart; import static com.github.tomakehurst.wiremock.client.WireMock.binaryEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; @@ -44,8 +45,7 @@ public void testDeploy() throws Exception { final String instance = createTempFile("instance: {}"); final String secrets = createTempFile("secrets: []"); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT)) @@ -105,8 +105,7 @@ public void testDeployWithDependencies() throws Exception { final String instance = createTempFile("instance: {}"); final String secrets = createTempFile("secrets: []"); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT)) .withMultipartRequestBody( @@ -140,8 +139,7 @@ public void testUpdateAll() throws Exception { final String instance = createTempFile("instance: {}"); final String secrets = createTempFile("secrets: []"); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.patch(urlEqualTo(String.format("/api/applications/%s/my-app", TENANT))) .withMultipartRequestBody( @@ -175,8 +173,7 @@ public void testDeployDryRun() throws Exception { final String instance = createTempFile("instance: {}"); final String secrets = createTempFile("secrets: []"); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.post(String.format("/api/applications/%s/my-app?dry-run=true", TENANT)) @@ -245,8 +242,7 @@ public void testUpdateAppAndInstance() throws Exception { final String app = createTempFile("module: module-1", langstream); final String instance = createTempFile("instance: {}"); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.patch(urlEqualTo(String.format("/api/applications/%s/my-app", TENANT))) .withMultipartRequestBody( @@ -274,8 +270,7 @@ public void testUpdateApp() throws Exception { Path langstream = Files.createTempDirectory("langstream"); final String app = createTempFile("module: module-1", langstream); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.patch(urlEqualTo(String.format("/api/applications/%s/my-app", TENANT))) .withMultipartRequestBody( @@ -473,8 +468,7 @@ public void testDeployWithFilePlaceholders() throws Exception { + " project: myproject\n"), jsonFileRelative)); - final Path zipFile = - AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println); + final Path zipFile = buildZip(langstream.toFile(), System.out::println); wireMock.register( WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT)) diff --git a/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java b/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java index 37ff1a664..805cb96f2 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java +++ b/langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java @@ -19,6 +19,7 @@ import static ai.langstream.api.model.ErrorsSpec.FAIL; import static ai.langstream.api.model.ErrorsSpec.SKIP; +import ai.langstream.api.archetype.ArchetypeDefinition; import ai.langstream.api.model.AgentConfiguration; import ai.langstream.api.model.Application; import ai.langstream.api.model.AssetDefinition; @@ -72,7 +73,124 @@ @Slf4j public class ModelBuilder { - static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + static final ObjectMapper yamlParser = new ObjectMapper(new YAMLFactory()); + + public static ApplicationWithPackageInfo buildApplicationInstanceFromArchetype( + Path archetypePath, Map applicationParameters) throws Exception { + Path archetypeFile = archetypePath.resolve("archetype.yaml"); + ArchetypeDefinition archetypeDefinition = + yamlParser.readValue(archetypeFile.toFile(), ArchetypeDefinition.class); + + Path instance = archetypePath.resolve("instance.yaml"); + Map globalsMap = new HashMap<>(); + InstanceFileModel instanceFileModel; + if (Files.isRegularFile(instance)) { + instanceFileModel = yamlParser.readValue(instance.toFile(), InstanceFileModel.class); + if (instanceFileModel.instance.globals() != null) { + globalsMap = new HashMap<>(instanceFileModel.instance.globals()); + } + } else { + throw new IllegalArgumentException( + "An archetype must always contain an instance.yaml file"); + } + + Path secrets = archetypePath.resolve("secrets.yaml"); + Map secretsMap = new HashMap<>(); + SecretsFileModel secretsModel; + if (Files.isRegularFile(secrets)) { + secretsModel = yamlParser.readValue(secrets.toFile(), SecretsFileModel.class); + secretsModel + .secrets() + .forEach( + s -> { + secretsMap.put(s.id(), s.data()); + }); + } else { + throw new IllegalArgumentException( + "An archetype must always contain an secrets.yaml file"); + } + + applyArchetypeParameters( + archetypeDefinition, applicationParameters, globalsMap, secretsMap); + + InstanceFileModel newInstanceFileModel = + new InstanceFileModel( + new Instance( + instanceFileModel.instance.streamingCluster(), + instanceFileModel.instance().computeCluster(), + globalsMap)); + SecretsFileModel newSecretsFileModel = + new SecretsFileModel( + secretsModel.secrets().stream() + .map( + s -> + new Secret( + s.id(), + s.name(), + (Map) + secretsMap.get(s.id()))) + .collect(Collectors.toList())); + + String instanceContent = yamlParser.writeValueAsString(newInstanceFileModel); + String secretsContent = yamlParser.writeValueAsString(newSecretsFileModel); + log.info("Generated instance.yaml file:\n{}", instanceContent); + log.info("Generated secrets.yaml file:\n{}", secretsContent); + + return buildApplicationInstance( + List.of(archetypePath), instanceContent, secretsContent, true); + } + + static void applyArchetypeParameters( + ArchetypeDefinition archetypeDefinition, + Map applicationParameters, + Map globals, + Map secretsMap) { + + archetypeDefinition + .archetype() + .sections() + .forEach( + section -> { + if (section.parameters() != null) { + section.parameters() + .forEach( + parameter -> { + String binding = parameter.binding(); + Object value = + applicationParameters.get( + parameter.name()); + log.info( + "Processing parameter, name: {}, binding: {}, value {}", + parameter.name(), + binding, + value); + String[] path = binding.split("\\."); + String first = path[0]; + Map context = + switch (first) { + case "secrets" -> secretsMap; + case "globals" -> globals; + default -> throw new IllegalArgumentException( + "Invalid binding " + + binding); + }; + applyValue(context, path, 1, value); + }); + } + }); + } + + private static void applyValue( + Map context, String[] path, int index, Object value) { + if (index == path.length - 1) { + context.put(path[index], value); + } else { + context = + (Map) + context.computeIfAbsent(path[index], k -> new HashMap<>()); + applyValue(context, path, index + 1, value); + } + } @Getter public static class ApplicationWithPackageInfo { @@ -100,14 +218,25 @@ public ApplicationWithPackageInfo(Application application) { * @throws Exception if an error occurs */ public static ApplicationWithPackageInfo buildApplicationInstance( - List applicationDirectories, String instanceContent, String secretsContent) + List applicationDirectories, + String instanceContent, + String secretsContent, + boolean fromArchetype) throws Exception { return buildApplicationInstance( applicationDirectories, instanceContent, secretsContent, new MessageDigestFunction(DigestUtils::getSha256Digest), - new MessageDigestFunction(DigestUtils::getSha256Digest)); + new MessageDigestFunction(DigestUtils::getSha256Digest), + fromArchetype); + } + + public static ApplicationWithPackageInfo buildApplicationInstance( + List applicationDirectories, String instanceContent, String secretsContent) + throws Exception { + return buildApplicationInstance( + applicationDirectories, instanceContent, secretsContent, false); } static class MessageDigestFunction implements ChecksumFunction { @@ -148,7 +277,8 @@ static ApplicationWithPackageInfo buildApplicationInstance( String instanceContent, String secretsContent, ChecksumFunction pyChecksumFunction, - ChecksumFunction javaChecksumFunction) + ChecksumFunction javaChecksumFunction, + boolean fromArchetype) throws Exception { Map applicationContents = new HashMap<>(); @@ -201,7 +331,8 @@ static ApplicationWithPackageInfo buildApplicationInstance( } } final ApplicationWithPackageInfo applicationWithPackageInfo = - buildApplicationInstance(applicationContents, instanceContent, secretsContent); + buildApplicationInstance( + applicationContents, instanceContent, secretsContent, fromArchetype); applicationWithPackageInfo.javaBinariesDigest = javaChecksumFunction.digest(); applicationWithPackageInfo.pyBinariesDigest = pyChecksumFunction.digest(); @@ -233,12 +364,25 @@ private static void recursiveAppendFiles( public static ApplicationWithPackageInfo buildApplicationInstance( Map files, String instanceContent, String secretsContent) throws Exception { + return buildApplicationInstance(files, instanceContent, secretsContent, false); + } + + public static ApplicationWithPackageInfo buildApplicationInstance( + Map files, + String instanceContent, + String secretsContent, + boolean fromArchetype) + throws Exception { final ApplicationWithPackageInfo applicationWithPackageInfo = new ApplicationWithPackageInfo(new Application()); DefaultsHolder defaultsHolder = new DefaultsHolder(); for (Map.Entry entry : files.entrySet()) { parseApplicationFile( - entry.getKey(), entry.getValue(), applicationWithPackageInfo, defaultsHolder); + entry.getKey(), + entry.getValue(), + applicationWithPackageInfo, + defaultsHolder, + fromArchetype); } if (instanceContent != null) { applicationWithPackageInfo.hasInstanceDefinition = true; @@ -267,7 +411,8 @@ private static void parseApplicationFile( String fileName, String content, ApplicationWithPackageInfo applicationWithPackageInfo, - DefaultsHolder defaultsHolder) + DefaultsHolder defaultsHolder, + boolean fromArchetype) throws IOException { if (!isPipelineFile(fileName)) { // skip @@ -277,11 +422,17 @@ private static void parseApplicationFile( switch (fileName) { case "instance.yaml": - throw new IllegalArgumentException( - "instance.yaml must not be included in the application zip"); + if (!fromArchetype) { + throw new IllegalArgumentException( + "instance.yaml must not be included in the application zip"); + } + break; case "secrets.yaml": - throw new IllegalArgumentException( - "secrets.yaml must not be included in the application zip"); + if (!fromArchetype) { + throw new IllegalArgumentException( + "secrets.yaml must not be included in the application zip"); + } + break; case "configuration.yaml": applicationWithPackageInfo.hasAppDefinition = true; parseConfiguration( @@ -291,6 +442,11 @@ private static void parseApplicationFile( applicationWithPackageInfo.hasAppDefinition = true; parseGateways(content, applicationWithPackageInfo.getApplication()); break; + case "archetype.yaml": + // ignore + // only validate that the file is valid + yamlParser.readValue(content, ArchetypeDefinition.class); + break; default: applicationWithPackageInfo.hasAppDefinition = true; parsePipelineFile(fileName, content, applicationWithPackageInfo.getApplication()); @@ -312,7 +468,7 @@ private static void parseConfiguration( String content, Application application, DefaultsHolder defaultsHolder) throws IOException { ConfigurationFileModel configurationFileModel = - mapper.readValue(content, ConfigurationFileModel.class); + yamlParser.readValue(content, ConfigurationFileModel.class); if (configurationFileModel.configuration == null) { throw new IllegalArgumentException( "configuration entry is not present in configuration.yaml"); @@ -345,7 +501,7 @@ private static void parseConfiguration( } private static void parseGateways(String content, Application application) throws IOException { - Gateways gatewaysFileModel = mapper.readValue(content, Gateways.class); + Gateways gatewaysFileModel = yamlParser.readValue(content, Gateways.class); if (gatewaysFileModel.gateways() != null) { gatewaysFileModel.gateways().forEach(ModelBuilder::validateGateway); } @@ -456,7 +612,7 @@ private static void parsePipelineFile(String filename, String content, Applicati throws IOException { try { PipelineFileModel pipelineConfiguration = - mapper.readValue(content, PipelineFileModel.class); + yamlParser.readValue(content, PipelineFileModel.class); if (pipelineConfiguration.getModule() == null) { pipelineConfiguration.setModule(Module.DEFAULT_MODULE); } @@ -606,7 +762,7 @@ private static void parsePipelineFile(String filename, String content, Applicati } private static void parseSecrets(String content, Application application) throws IOException { - SecretsFileModel secretsFileModel = mapper.readValue(content, SecretsFileModel.class); + SecretsFileModel secretsFileModel = yamlParser.readValue(content, SecretsFileModel.class); if (log.isDebugEnabled()) { // don't write secrets in logs log.debug("Secrets: {}", secretsFileModel); } @@ -633,7 +789,7 @@ private static void validateSecret(Secret s, Set ids) { private static void parseInstance( String content, Application application, Map defaultValues) throws IOException { - InstanceFileModel instanceModel = mapper.readValue(content, InstanceFileModel.class); + InstanceFileModel instanceModel = yamlParser.readValue(content, InstanceFileModel.class); log.info("Instance Configuration: {}", instanceModel); Instance instance = instanceModel.instance; if (instance == null) { diff --git a/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java b/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java index 0fe2a9b02..b13c657c4 100644 --- a/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java +++ b/langstream-core/src/test/java/ai/langstream/impl/parser/ModelBuilderTest.java @@ -64,7 +64,8 @@ void testPyChecksum() throws Exception { null, null, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); Assertions.assertEquals( "asubdir/script2.py:print('hello world3'),script.py:print('hello world'),script2.py:print('hello world2'),", applicationWithPackageInfo.getPyBinariesDigest()); @@ -89,7 +90,8 @@ void testJavaLibChecksum() throws Exception { null, null, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); Assertions.assertEquals( "my-jar-1.jar:some bin content,my-jar-2.jar:some bin content2,", applicationWithPackageInfo.getJavaBinariesDigest()); @@ -169,7 +171,8 @@ void testParseRealDirectoryWithDefaults() throws Exception { instanceContent, secretsContent, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); Application applicationInstance = applicationWithPackageInfo.getApplication(); Map globals = applicationInstance.getInstance().globals(); assertEquals("value1", globals.get("var1")); @@ -201,7 +204,8 @@ void testParseRealDirectoryWithDefaults() throws Exception { instanceContentWithoutGlobals, secretsContent, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); applicationInstance = applicationWithPackageInfo.getApplication(); globals = applicationInstance.getInstance().globals(); assertEquals("default-value1", globals.get("var1")); @@ -220,7 +224,8 @@ void testParseRealDirectoryWithDefaults() throws Exception { instanceContentWithEmptyGlobals, secretsContent, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); applicationInstance = applicationWithPackageInfo.getApplication(); globals = applicationInstance.getInstance().globals(); assertEquals("default-value1", globals.get("var1")); @@ -234,7 +239,8 @@ void testParseRealDirectoryWithDefaults() throws Exception { null, secretsContent, new StrChecksumFunction(), - new StrChecksumFunction()); + new StrChecksumFunction(), + false); applicationInstance = applicationWithPackageInfo.getApplication(); globals = applicationInstance.getInstance().globals(); assertEquals("default-value1", globals.get("var1")); diff --git a/langstream-webservice/pom.xml b/langstream-webservice/pom.xml index 04b590d23..982a7de07 100644 --- a/langstream-webservice/pom.xml +++ b/langstream-webservice/pom.xml @@ -33,8 +33,6 @@ UTF-8 UTF-8 2.1.0 - 3.3.1 - @@ -179,8 +177,8 @@ ${project.groupId} langstream-cli ${project.version} - test + ${project.groupId} langstream-k8s-deployer-api diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/.gitignore b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/.gitignore new file mode 100644 index 000000000..55dea2dd3 --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/.gitignore @@ -0,0 +1 @@ +java/lib/* \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/archetype.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/archetype.yaml new file mode 100644 index 000000000..facf876a0 --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/archetype.yaml @@ -0,0 +1,166 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +archetype: + id: "website-qa-chatbot" + title: "Website QA Chatbot" + labels: + - qa + - chat-bot + - website + - astra + description: "This archetype is a chatbot that answers questions from a website. It uses a pre-trained model from OpenAI to compute embeddings for the questions and then uses a Cassandra database to store the answers." + icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAYAAABccqhmAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAUGVYSWZNTQAqAAAACAACARIAAwAAAAEAAQAAh2kABAAAAAEAAAAmAAAAAAADoAEAAwAAAAEAAQAAoAIABAAAAAEAAAEAoAMABAAAAAEAAAEAAAAAAHU3NFcAAAIyaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj4yNTY8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MjU2PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6Q29sb3JTcGFjZT4xPC9leGlmOkNvbG9yU3BhY2U+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgqgFRNPAAAb40lEQVR4Ae2da5Ac1XXH7+3Z1Rtpd0fCsg2WnUAgYAwmaKUVYDAGEjAuQoyQwYA2X/AXLZKCYptUpbJxpcqYEEsgvuAPqUollVQgYECYV8DIFiAkUQ7YJuRBgoixeWhnVxLSSrsz0zfntGa1s6uZ2e6eftzT/VcVbM/te8/jd06fudOP20rhHwiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAAAiAABPQwJBdAuaB2zoPvPnzc8pVd7lS5gLy9JPaqC6K+iJlTJfhv0p1aKOHjFL7tDL7qG2I2t4rKOdVXXB2dm95ZS99xr+MEkAByFhghwb6eo2q3KiN6aOD+Txl1Oy2XNTqPSoQL2tHv+w4ziMoCG3RtG4wCoB1IQlu0Mgdq5ZVy5Wb6Vv9FmPMGcEl+BtByWKU1j82jv7bxd3zHtGD24/6G4lethJAAbA1Mj7sGl638kJXV79DB/4XaQqfaCxJ2QEy8Z86lHN31/273/ZhLrpYSCDRpLHQf5EmHfyT3t8ZHzd3GWWuS90BrcYco7fq+Yv+qud7z3FRwD9BBFAABAXro02rTh4bL/8FTcRvo6l+h1Wm08lDrdVgsWfBA/TToGKVbTCmKQEUgKZo7NpRWr/iCtet/iMd/IvtsmyqNVrrf3e0WdNz36u/nLoHn2wk4NhoFGyaJEDf9Lq0bvmfG7f6tO0HP1tN9p5VNeqV4YHeGye9wJatBDADsDUyZNeBjX0945XKP9BRdZXFZjY1jZJra/Gc8+/Q3/hBuWkn7EiVAApAqvibK6eD/7Ryufwcnd1f1ryX/XvoJ8HLs2d3XnfSPS9/aL+1+bMQBcDCmJcGVpxiTPVF6Qf/cbRa/bxz/pxLu+56ceR4GzasIIBzAFaEYdIIPtNvlCv+m3/SI9oy6nOVw2NPmk1Xzp/Sjg+pE0ABSD0EkwaMbLi06+hY+dk47+ab1JbsFvm0sjQ28pgZuKq9W5OTNTvz2lAALAmxGVw9q1o5/CSd8DvXEpMiN4OKwJdKat8/mwdXFyIXDoGhCKAAhMIW/aDS8N6/pDv7+qKXbJdEY9S1pR1777TLqvxag5OAFsS+tKG3z1TMDioAufhmpKQrFzp0b/eWPa9ZgD/XJmAGkHL4zeBX5rlV83d5OfgZN13d6KxW1d/zz56U8edePQpAyilQKr33Pfrdf3rKZiSuns4HfLZU2vudxBVD4RQC+AkwBUeyH0YGll9SNeYF+kbMZRzo4SGXVh26qLhl985kyUPbBAHMACZIpPCXHpmjE3/5PPgZN50QdOjnz3dTQA+VNQK5/OaxIfql25evcl3zkg22pG2D0+GswiwgnShgBpAOd+W6CpfCauzdivmzlMKQe7WYAaSQAsMb+85xy+XX8zz9r8dOSWiczs5zezbv/EV9O7bjJ4AZQPyMT9BQrZTvxME/iYVZMJPJFmwlRQAzgKRI1/SMbFj56Wql+laervv7QaypBhQ6Cqdh2XE/tKLrgxlAdCx9SXIrlbU4+E9ExUyYzYl70BInARSAOOk2kO1qdUuDZjQRAbBJPg1QABJkPrJ++RfoPtjfTlClLFXExmMky2rR1qIAJBg+uv+9P0F1IlWBUbJhw0nAhHibjX1zS5XK+3QP/MKEVIpUQ2sIHix2dCzVm3ceEemAMKMxA0goYKVq+Xoc/DPDZkbMauae6BEFARSAKCj6kWE0znD74cR9wMovqbb74SdA2whnFuCt8quq7/DDLzP3Rg/vKUFVWFbcuutd0IiXABIyXr6edKPNWhz8/kEzK2bmfwR6hiWAAhCWXJBxxuDafxBe3BfMghIL1R8FIBQ2/4P4sV86sXWG/xHoyQSYGbMDjXgJoADEy1e5Btf+wyIGu7Dk/I9DAfDPKnBPM3jpHK3M6sADMcAjwOyYIXDERwAFID62aqQ0eh2d0OqKUUWmRTM7ZphpJ1N2DgUgxgC4Bmey28ULhu0SbD0e9wG05hN67+ENF3/8SOXor/Dob2iE3kBeJ2Bux5xT52/Z8V57kjC6EQHMABpRiaDtSHXsVhz87YNkhsyyfUmQ0IgACkAjKpG0GSRtJBxZCFhGhnKaIBSAaUCi+Dg00NdL17HPikIWZHj3BJzFTMEiegIoANEzJYmV/ljE5loomMYRfhSAiKnWXni5JmKxEKfUGrxMNPo0QAGImOnw0DvX0vS/J2Kx8YjT+m3F/wn4x0yZrQBTRZmIAhBxuFztinmKzTHqYf4vYgSxiZPENjYIEQvGfQARAj20fsXHjrruu/Rt1RGh2NhEObrQx8JdUxXxdl5aLqwyx3FOWXDvrg9ig5IzwZgBRBjwI657s5SDn26w+XXPfa/s4v/owBKx8AazZcYRhiz3olAAIk0BQdertXqYDnzD/xGCRyLFEKswQYxj5RCNcBSAaDiq/RtWnk9r/n8uInGxi3GU/pcJJfXbE23W/iXGHmtrDZRlGApARPGqVARdp9b6g+7i1S9NuH5sW78/8dn2v6JYWw4TBSCCAJkHbus0Wn0tAlGJiKAzv4/owUF3Qhlv00KcP5z4bPtfZs3MbbdTgn0oABFEqfTGa9fQ9H9JBKKSEWEmp//HFTZqO77Tsg1i7TG3zCyJ5qAARBE1I+faP539LxW/sOwn09322rQamt5u7WdBzK1lSIahALQZnYN3XLqYvv2vblNMYsNp+vxDfcND1ekKj7XpR6e3W/uZmHvsrTVQhmEoAG3GqVw+/HW6jibm96hDl/+audxqX7MxabUzc2aflv6s6EUBaDOSritn1V860be/5+zPP9/MZd7HfZrtt61dEnvb2E3YgwIwQSLE3+GNfefQ0+rnhRia1pDH9Td+UG6mvLbv8Wb77Ws35x2LgX2WSbEIBaCNSFWr5T9uY3jiQ03dzT/NlPvp02xsGu3SYpAGo1Y6UQBa0Wmxj9ar76C32N7UootVu+iW348Wq8XPzmQU9+G+M/WzZj/FwIuFNQbJMgQFIGS8hkqHr6L3V30s5PDEh9Himk/orU+NzaSY+3DfmfpZs59i4MXCGoNkGYICEDJe9AjN2pBDUxlG1/+P3/s/kwFB+s4kK4n90mKRBBO/OlAA/JKq67f/2xd10+urr6lrsn1ztFhc+rRfI2t9R/32T7sfx4JjkrYdEvWjAISIWmX06E1088/sEENTGUKX9p7Ug9t8H9Dcl8ekYmwYpRQLLyZhxuZ8DApAmARwdX+YYWmNCXODT5gxafnn6RUWk1RZ1SlHAaiD4WdzaMPK36WTZBf46WtFH63GqnNm/yioLd09C+hEoD4adFxa/TkmHJu09EvViwIQMHK66oq69k9r/jyz5O6XAl/W04PbD9FjwzNeNgyIL9bu0mITKwyfwlEAfILibubB1QVjXFn3nzvG99n/E1C0M/YEYfE3cGw4RvFryo4GFIAAsRz56dtX0kMonwgwJNWu9A1eLjgLtoU1gseyjLDjkx7HseEYJa1Xsj4UgADRq2ot6to/rfj5XPeW7aEf7uGxLCMAotS7SotR2sBQAHxGYPhbly+iS3+i3kzjqDam/zUuUcjwiTiabhQjL1bRSMu8FBQAnyGujh6kNf/MHJ/dU+9G9/NXOhc4j7VrCMtgWe3KSW68mXMsVslplKwJBcBn9LQy/T672tJt+8Lv7i61a0xNxvZ25SQ5XmCsksQzRRcKwBQcjT8cuH3F6fRWmpWN99rZSr/dm678E9TiKGUF1R2mP8eKYxZmbN7GoAD4iHjZFXbtXyt3rtaRLfPNsujW4OPLiPtAlnoXaTFLCxgKwAzkzeAgMxL1Pjpj9ItRvkCTZbHMGVDZtvvmWuxss8sqe1AAZgjH8MhTX6LbTE+doZtVu7Xj/9Ffv4bHIdOv7jD9OGYcuzBj8zQGBWCGaBtXzpr/7ArduGNUoRD9yz5Jpid7Bl427ZYWuzTYUUzxrxmBfd+88CQ1OsbvzJvXrI9t7bSYx87F9+9ZFYddQ+uWv0zfrH1xyI5J5qiaN3tpmGchYrLHOrGYAbQIiTNavoF2izn42ZU4p+pxym4RhnZ2zavFsB0ZmR6LAtAivK52+1vstnKX4zjRT/9rnsYpOy6YEmMYF4tGclEAGlGhtv0bl/8WrTV3YZPdVjbT9P/V7i2v7I3LOJbNOuKSH4dcjiHHMg7ZWZCJAtAkiuWq6qeny6SdI4ns5p8mWLg5CR0t1AfbxTHkWAYblZ/eKAANYk13kmn65rilwS6rmzodJ/aDs7OzI/z6AinR41hyTFNSb7VaFIAG4Rlav/JSSphPN9hlb5PWry+6b9d/x23gos0731KkK249UcrnWHJMo5SZFVkoAA0iqYVd+2cXHJPc1DxJXQ3CE6pJYkxDORpwEArANGBm05Xz6Vr3V6c1W//RdBQSm5onqSsq8BxTjm1U8rIiBwVgWiRL4/uvp6YF05qt/kg/bt9cvOWVN5MyknWxzqT0RaRnQS22EYnLhhgUgOlxdOVd+zcx3Ps/Hcv0z2nonG5D4M8CYxvYx4ADUADqgI3csWoZfbykrknEZoeJ7tl/vw6nodOvbS36XVKLcYsu+dqFAlAXb3e8vFbgtf+3urfuTvysfE3nW3X4rN/k2HKMrTc0QQNRAOpgu8aIu/ZP6/XFfu2/DtGUzTR1TzEkwAeJMQ7gXuCuKAA1ZMMDvRfT5mmBCaY8wHRE/+y/X5fS1O3Xxgb9TqvFusGu/DWhANRiXtWynvtns+lM/DtLNu9O7d581s02SDtsJMY6LsYoAETWbOybS+/QWx0X5Ljk0s2tqU3/J3yywYYJW/z+5VhzzP32z3I/FACK7lCl8kd0u+hCaYF2Ck5iN/80Y2ODDc1sa9bOseaYN9ufp3YUAIo2TWP7pQWdbP5Nz+Zdr6RtN9tAjwj/Om07guqXGPOgPvrpn/sCUBpYcQr9CLjMDyzL+jxMZ+Hpyla6/47ZYGJbhCQ+78xlx2IfnwYJknNfAIyq3mqMEsehkOLlv+mJbZMt021r9pljzrFvtj8v7eISP/rAaIFJoD/sKn55R/Qswkn0bNH6g3Cj0xwlMfbR8sp1ASht6O2jE0JnRIs0fmn07rtH9OCgNW/qYVu0MZG9iSh+gsc0cOw5B5LSZ6OeXBcAVZV37d9LIoum/8eT2kabjhvXYkNqDrRwKciu3BYAM3DVbDqDtiYILBv60hn3UrE4f7sNttTbwDaxbfVtErY5BzgXJNgah425LQDDZugP6URQVxxQY5Wp9WN6cHslVh0hhHs2kW0hhqY6hHOAcyFVI1JUntsC4Gp51/45Txzjpn7zT7N8tdm2ZjZzu9RcaOWT3325LACHN1z8cbqCfoVfSLb0o5tXDnQv/szzttgz3Q62jW2c3m77Z84Fzgnb7YzDvlwWgCOVsVtojbhCHEBjlamdx/XgQ+Ox6mhDuGcb2diGiFSGci5wTqSiPGWluSwAdOefwGv/lCkSzrRLsLHhQSc0Jxr64r8xdwVgaP3y5VTxz/aPyJqeh4o9c5+xxpomhtRsPNRkt7XNnBOcG9YaGJNhuSsAylVCl4TST9CZ9qMx5UFkYo/ZqJ+ITGCSgsTmRnhIuSoAZnD1LGXU18LjSm+ko+N/7VdU3kmydYrPlBtejkxpzPaHXBWA4ZG3v0JTvaK4kGp9pGf2oqek2N1TPPlJsnVUir0TdnJucI5MfM7D31wVALeq+yUGlS6tPaXvefawFNv14LZRekz4aSn21tspNUfqfQiynZsC8NGmVSdrrf4gCBxb+joqvYU/wzKQaDP7yjnCuRLWb2njclMAxscrN9PTXx3SAkTLFY1V53aKO6nm2Uy2S+PNOcK5Is3usPbmpgDQevAir/3TAzbPLrn7pY/CBjitcWwz256W/nb0Ss2VMD7nogCMDFzweWXMuWEApT9G3vR/kplQ2ylXvJyZdCSzW7koAFWjRV77p5N/5Y75s7ZJzb5CYd7j7INE+6XmTFDWmS8A5oHbOo02NwUFY0N/o/XzXXe9OGKDLWFs6N6yfT/7EGZs2mM4Zzh30rYjbv2ZLwClN16/mm7+WRI3yDjkO2Lvq5+kIdYHyhkvdyZdyeRW5guAMtV+iZGjE2jVzs55j0q0vd5m9oF9qW8Tsy00d4LwzXQBOHhnb5G+/b8cBIg1fbX6ycK/2T5kjT0hDfF8IF9CDk93GOWOl0PpWhGr9kwXgPFD5uu05pvU33HWrvwTIiNF+sK5wzkUwl8xQzJdAOimDpln/7Vy58xR4pbZbpb17AvdYWfNMubN7GzULjWHGvnSqC2zBWD49gs+Sw6f38hp29uM0i8t+Os979tup1/72Bf2yW9/y/qdX8sly8yKxpzMFgBjdH80iJKXQtfORU6ZW5GS7JPkXGoVE96XyQJgHlxdcI0S+duNDhSjlSPwZZutU419Yt9a97JzL+cS55Sd1rVnVSYLwMiOd+ipP7O0PTQpjdZ6V3HrrndT0h6bWs8n8i02BbEKNkuP5VSsSlIRnskC4Cq3PxWaESilr/+HIxBjpQjJvknOqVbJkLkCsP/bF3XTPFPsqi6drs5sAZDsG+cU51arg0nivswVgMqhsRvpl6bUd739rOv+3W9LTCQ/Ntd8+5mfvtb1oZzycss6w9ozKHMFQGmZ1/45jNoR+vhsgBwU7aPg3GoWokwVgKF1fWfSyx57mzlre/usjuxO/yfYS/aRc4tzbMKXLPzNVAFQutIvNSh0iewXC7+/+7+k2u/XbvaRffXb37p+gnOsEcvMFAAzOOjQb3+xa7nRKrqZu/mnUcJxm2hfKce8XGvmnLD2zBSA4ZEnr6B13T8pjP+kuR3Zvfw36WRtS7CvnGOcayf4JLQhMwXArZp+oTHgpaj/o7h5zxtS7Q9qN/vKPgcdZ0t/ybk2nWEmCsDwty5fRMtnXzvdOTGfTY6+/SeCItlnyjUv5yZ8Efw3EwWgeuTAGlr1d67UOBQ68vP7fyJGon2mXPNybsIZwX8zUQC0kfrGX8ocrf6ne8ue1wTnUCjTPZ/J91CDLRgkOufq+IkvAAduX3E6Ldqwqs4nUZv0Wzizt/7OFAjJvnPOce7N5KPt+8UXgLJbXWs75Jb25ejy3wkchPsuPvcoIKILAFVhWnBW3XJCYglpoG/A/1t87549QsyN3Ez2nRlELjgpgZR7Xg4mpS8GPaILQGl972V0e+anYuCSiEjJj8dGBUgyA849zsGoWKQhR3QBIGD9aUCLSmfB7cjt7/8JhhlgIDoH6bZsmf/2ffPCk9ToGC+cOU+iBwT+N8Wte06h22LpUfP8/uMpdGlg+bsE4RNCKYyqebOXSnyDM/MWOwNwjoytJvtFHvxeomtNS2Xn++BnDh4DYuExkfm/ebVcFGm92ALgCn3j70SWmEIhNw//TPjc7K90FpJzUWQB2L+u9zNamYubJZT97frDxatO3WG/nclYeIyF/jAZbdFr4VzknIxecvwSRRaAsqPW0m9GsecvKKyP6hseqsYfXhkaaizEvgiVc5FzUgbtqVaKKwDedVdjbp3qhqxPTsHB9H9ayMQzoZyUeE+AuAKw//beL9CDPyKnW5zzdNJruKd77gvT8j/3H5kJsxELgnLSy01hDogrADRv7hfGeKq5Rj2mB7dXpjbik8eE2EgmITE3RRUAs+nK+TTNul5ykjg6+wt/ho2PdDacm5yjYf1PY5yoAlAa2/9VgrQgDVBR6KQp7sHu4rJ/jUJWFmUwGzqze0CwbwtqOSrGBVEFgN73J/JMa102bNODD43XfcZmHQGPjdZP1DUJ3JSVo2IKwPD6iz5FJ/++KDAjJk02+Vv5Z9J5n1vSGVGOernq0920u4kpAK47ditfb00bWGj9Wh0udhaeCT0+JwOLi+c9Ta4ekuou5yjnqhT7xRQAOsEifPqvf6Q37zwiJTHSspOuBhyla6VPpqU/Cr2SclVEARhet/JCCsxpUQQnLRmO9KltguAywOq0Ws4mSC2cKhEFoKqr/eHcs2SU1kd65nSJ/lZLkqTHipglqTNqXVJy1voCYDb2zaUf/jdEHaAk5dFDv0/re549nKROybqYFTMT7QPlLOeu7T5YXwCG3Op19Jtqoe0gW9knedmrVn7FuU86M85Zzt04GUUh2/oCQKdURZ/8o0Uvxx3lbIsiWHmS4THTakyyzxJy1+oCMLSxj1/2ebnkJKAVr54tbt11ULIPadjuMTNa+l2Tl9dyOA2EvnRaXQBUuULLLstdtowjQDOA3C/86SsTG3SSzs7LXcrhBq5Z02R3AdCyr/3Tvf+VjvmzRT/hlmamMjs6AVxO04a2dVuew9YWgKH1K1dQBT2z7QCkKsD8uOuuF0dSNUGwco+dVi8IdoHuXldnci7b6oO1BUBVK/22QvNrF722CCv/+IXVpF8mGFqcy1YWADNw1Wz69bymSU6IaKbErc5Sc8Suc2cLZGbILG2xJ5wdes2xnA43Os5RVhaAYT10rVGmO07HY5etzU9P2rpjX+x6Mq7AY0gsJbvJucw5baMPVhYA15V98s8LNM7+R5fvGWBpa05bVwAO/enypTTl+/3osid5SXTm2sx15j6SvOZsamSWdEnQlewd5zTntm0+WFcAjoypm2nKVLANVBB76DbWl+Zv2fFekDHo25wAszRKvdy8h/17OKc5t22z1LoCoF3hq/5ShAkqbv6JONOzwNTG3LaqAOy7fcXvUaU8O+LcSVQcT/9VJ1b+jRw6MfXYRi44OYGc25zjyWmcWZNVBUAb4c/9M2+t9hS/v+dXM6NHjyAEPKbENsgYG/valuPWFAAzuHqWMvpGG4MWxCb6rYqbf4IAC9A3E2wpx71cD+B3nF2tKQCl0t5raIpUjNPZJGR3dmD6HxfnLLDlHOdcj4tRULnWFAD65bw2qPHW9df637o27/lf6+zKiEEeW2Is3h2Lct2KAvDRplUn02/nq6UHlt4Tj+l/zEHMBGPKdS/nY2blR7wVBWB8bPwmWkKpw4/BNvcxDi7/xR2fLDDmXOecj5uVH/lWFABX6X4/xtrch579/+WSe1/9T5ttzIJtzJhZS/clCzkvPQawHwRAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAARAAAQaEPh/dr6OzlM6coUAAAAASUVORK5CYII=" + sections: + - title: Website + description: "Define the website to crawl" + parameters: + - name: "website-url" + label: "Website URL" + description: "The URL of the website to crawl" + type: "strings-array" + subtype: "url" + required: true + binding: "globals.website.urls" + default: + - "https://docs.langstream.ai/" + - "https://langstream.ai/" + - name: "website-crawler-depth" + label: "Crawler depth" + description: "Maximum depth of the crawler" + type: "integer" + binding: "globals.website.max-depth" + default: 10 + required: false + - title: OpenAI parameters + description: "This archetype uses a pre-trained model from OpenAI to compute embeddings for the questions. You need to provide the name of the model deployment." + parameters: + - name: "embeddings-model" + label: "Embeddings model" + description: "The name of the model deployment from OpenAI." + type: "string" + default: "" + required: true + binding: "secrets.open-ai.embeddings-model" + - name: "access-key" + label: "Access key" + description: "The access key for the OpenAI API." + type: "string" + required: true + binding: "secrets.open-ai.access-key" + - name: "chat-completion-model" + label: "Chat completion model" + description: "The name of the model deployment from OpenAI." + type: "string" + required: true + binding: "secrets.open-ai.chat-completions-model" + - title: Prompt + description: "Here you can define the prompt to send to the LLM" + parameters: + - name: "llm-prompt" + label: "Prompt" + description: "This is the template to send to the LLM" + type: "prompt" + subtype: "openai-prompt" # each LLM has its own way of defining the prompt + binding: "globals.llm.prompt" + default: | + - role: system + content: | + An user is going to perform a questions, The documents below may help you in answering to their questions. + Please try to leverage them in your answer as much as possible. + Take into consideration that the user is always asking questions about the LangStream project. + If you provide code or YAML snippets, please explicitly state that they are examples. + Do not provide information that is not related to the LangStream project. + + Documents: + {{# value.related_documents }} + {{ text}} + {{/ value.related_documents }} + - role: user + content: "{{ value.question }}" + required: true + - title: Astra DB parameters + description: "This application uses a AstraDB database to store the answers. You need to provide the connection information for the database." + parameters: + - name: "astra-database" + label: "Database" + description: "The name of the Astra DB database" + type: "string" + subtype: "astra-database" + binding: "secrets.astra.database" + required: true + - name: "astra-client-id" + label: "Client ID" + description: "The client ID for the Astra DB database" + type: "string" + subtype: "astra-client-id" + required: true + binding: "secrets.astra.clientId" + - name: "astra-client-secret" + label: "Client secret" + description: "The client secret for the Astra DB database" + type: "string" + subtype: "astra-client-secret" + required: true + binding: "secrets.astra.secret" + - name: "astra-token" + label: "Token" + description: "The token for the Astra DB database" + type: "string" + subtype: "astra-token" + binding: "secrets.astra.token" + required: true + - title: S3 Bucket parameters + description: "The crawler uses an S3 bucket to store the state of the crawler. You need to provide the connection information for the S3 bucket." + parameters: + - name: "s3-bucket" + label: "Bucket name" + description: "The name of the S3 bucket" + type: "string" + subtype: "s3-bucket" + binding: "secrets.s3.bucket-name" + required: true + - name: "s3-access-key" + label: "S3 Access key" + description: "The access key for the S3 bucket" + type: "string" + subtype: "s3-access-key" + binding: "secrets.s3.access-key" + required: true + - title: Kafka parameters + description: "LangStream uses Kafka to connect the agents" + parameters: + - name: "kafka-bootstrap-servers" + label: "Kafka bootstrap servers" + description: "The connection string to the broker" + type: "string" + subtype: "kafka-bootstrap-servers" + binding: "secrets.kafka.bootstrap-servers" + required: true + - name: "kafka-username" + label: "Username" + description: "The username for the Kafka broker" + type: "string" + subtype: "kafka-username" + binding: "secrets.kafka.username" + required: true + - name: "kafka-password" + label: "Password" + description: "The password for the Kafka broker" + type: "password" + subtype: "kafka-password" + binding: "secrets.kafka.password" + required: true diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/chatbot.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/chatbot.yaml new file mode 100644 index 000000000..12ec6254b --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/chatbot.yaml @@ -0,0 +1,87 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +topics: + - name: "questions-topic" + creation-mode: create-if-not-exists + - name: "answers-topic" + creation-mode: create-if-not-exists + - name: "log-topic" + creation-mode: create-if-not-exists +errors: + on-failure: "skip" +pipeline: + - name: "convert-to-structure" + type: "document-to-json" + input: "questions-topic" + configuration: + text-field: "question" + - name: "compute-embeddings" + type: "compute-ai-embeddings" + configuration: + model: "${secrets.open-ai.embeddings-model}" # This needs to match the name of the model deployment, not the base model + embeddings-field: "value.question_embeddings" + text: "{{ value.question }}" + flush-interval: 0 + - name: "lookup-related-documents" + type: "query" + configuration: + datasource: "AstraDatasource" + query: "SELECT text,embeddings_vector FROM documents.documents ORDER BY embeddings_vector ANN OF ? LIMIT 20" + fields: + - "value.question_embeddings" + output-field: "value.related_documents" + - name: "re-rank documents with MMR" + type: "re-rank" + configuration: + max: 5 # keep only the top 5 documents, because we have an hard limit on the prompt size + field: "value.related_documents" + query-text: "value.question" + query-embeddings: "value.question_embeddings" + output-field: "value.related_documents" + text-field: "record.text" + embeddings-field: "record.embeddings_vector.values" + algorithm: "MMR" + lambda: 0.5 + k1: 1.2 + b: 0.75 + - name: "ai-chat-completions" + type: "ai-chat-completions" + configuration: + model: "${secrets.open-ai.chat-completions-model}" # This needs to be set to the model deployment name, not the base name + # on the log-topic we add a field with the answer + completion-field: "value.answer" + # we are also logging the prompt we sent to the LLM + log-field: "value.prompt" + # here we configure the streaming behavior + # as soon as the LLM answers with a chunk we send it to the answers-topic + stream-to-topic: "answers-topic" + # on the streaming answer we send the answer as whole message + # the 'value' syntax is used to refer to the whole value of the message + stream-response-completion-field: "value" + # we want to stream the answer as soon as we have 20 chunks + # in order to reduce latency for the first message the agent sends the first message + # with 1 chunk, then with 2 chunks....up to the min-chunks-per-message value + # eventually we want to send bigger messages to reduce the overhead of each message on the topic + min-chunks-per-message: 20 + messages: "${globals.llm.prompt}" + - name: "cleanup-response" + type: "drop-fields" + output: "log-topic" + configuration: + fields: + - "question_embeddings" + - "related_documents" \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/configuration.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/configuration.yaml new file mode 100644 index 000000000..db28f9906 --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/configuration.yaml @@ -0,0 +1,35 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +configuration: + resources: + - type: "open-ai-configuration" + name: "OpenAI Azure configuration" + configuration: + url: "${secrets.open-ai.url}" + access-key: "${secrets.open-ai.access-key}" + provider: "${secrets.open-ai.provider}" + - type: "datasource" + name: "AstraDatasource" + configuration: + service: "astra" + clientId: "${secrets.astra.clientId}" + secret: "${secrets.astra.secret}" + secureBundle: "${secrets.astra.secureBundle}" + database: "${secrets.astra.database}" + token: "${secrets.astra.token}" + environment: "${secrets.astra.environment}" \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/crawler.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/crawler.yaml new file mode 100644 index 000000000..aa90ba84f --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/crawler.yaml @@ -0,0 +1,97 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "Crawl a website" +topics: + - name: "chunks-topic" + creation-mode: create-if-not-exists +resources: + size: 2 +pipeline: + - name: "Crawl the WebSite" + type: "webcrawler-source" + configuration: + seed-urls: "${globals.website.urls}" + allowed-domains: "${globals.website.urls}" + forbidden-paths: [] + min-time-between-requests: 500 + reindex-interval-seconds: 3600 + max-error-count: 5 + max-urls: 1000 + max-depth: "${globals.website.max-depth}" + handle-robots-file: true + user-agent: "" # this is computed automatically, but you can override it + scan-html-documents: true + http-timeout: 10000 + handle-cookies: true + max-unflushed-pages: 100 + bucketName: "${secrets.s3.bucket-name}" + endpoint: "${secrets.s3.endpoint}" + access-key: "${secrets.s3.access-key}" + secret-key: "${secrets.s3.secret}" + region: "${secrets.s3.region}" + - name: "Extract text" + type: "text-extractor" + - name: "Normalise text" + type: "text-normaliser" + configuration: + make-lowercase: true + trim-spaces: true + - name: "Detect language" + type: "language-detector" + configuration: + allowedLanguages: ["en", "fr"] + property: "language" + - name: "Split into chunks" + type: "text-splitter" + configuration: + splitter_type: "RecursiveCharacterTextSplitter" + chunk_size: 400 + separators: ["\n\n", "\n", " ", ""] + keep_separator: false + chunk_overlap: 100 + length_function: "cl100k_base" + - name: "Convert to structured data" + type: "document-to-json" + configuration: + text-field: text + copy-properties: true + - name: "prepare-structure" + type: "compute" + configuration: + fields: + - name: "value.filename" + expression: "properties.url" + type: STRING + - name: "value.chunk_id" + expression: "properties.chunk_id" + type: STRING + - name: "value.language" + expression: "properties.language" + type: STRING + - name: "value.chunk_num_tokens" + expression: "properties.chunk_num_tokens" + type: STRING + - name: "compute-embeddings" + id: "step1" + type: "compute-ai-embeddings" + output: "chunks-topic" + configuration: + model: "${secrets.open-ai.embeddings-model}" + embeddings-field: "value.embeddings_vector" + text: "{{ value.text }}" + batch-size: 10 + flush-interval: 500 \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/gateways.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/gateways.yaml new file mode 100644 index 000000000..132788270 --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/gateways.yaml @@ -0,0 +1,43 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +gateways: + - id: "user-input" + type: produce + topic: "questions-topic" + parameters: + - sessionId + produceOptions: + headers: + - key: langstream-client-session-id + valueFromParameters: sessionId + + - id: "bot-output" + type: consume + topic: "answers-topic" + parameters: + - sessionId + consumeOptions: + filters: + headers: + - key: langstream-client-session-id + valueFromParameters: sessionId + + + - id: "llm-debug" + type: consume + topic: "log-topic" \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/instance.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/instance.yaml new file mode 100644 index 000000000..383543faf --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/instance.yaml @@ -0,0 +1,30 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is a sample file to connect to DataStax Astra Streaming service +# But it works with any Kafka Cluster that enables SASL/PLAIN authentication and SSL encryption + +instance: + streamingCluster: + type: "kafka" + configuration: + admin: + bootstrap.servers: "${ secrets.kafka.bootstrap-servers }" + security.protocol: SASL_SSL + sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username='${ secrets.kafka.username }' password='${ secrets.kafka.password }';" + sasl.mechanism: PLAIN + session.timeout.ms: "45000" diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/secrets.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/secrets.yaml new file mode 100644 index 000000000..ca0dcc60b --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/secrets.yaml @@ -0,0 +1,50 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +secrets: + - id: kafka + data: + username: "${KAFKA_USERNAME:-}" + password: "${KAFKA_PASSWORD:-}" + tenant: "${KAFKA_TENANT:-}" + bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS:-}" + - id: open-ai + data: + access-key: "${OPEN_AI_ACCESS_KEY:-}" + url: "${OPEN_AI_URL:-}" + provider: "${OPEN_AI_PROVIDER:-openai}" + embeddings-model: "${OPEN_AI_EMBEDDINGS_MODEL:-text-embedding-ada-002}" + chat-completions-model: "${OPEN_AI_CHAT_COMPLETIONS_MODEL:-gpt-3.5-turbo}" + text-completions-model: "${OPEN_AI_TEXT_COMPLETIONS_MODEL:-gpt-3.5-turbo-instruct}" + - id: astra + data: + clientId: ${ASTRA_CLIENT_ID:-} + secret: ${ASTRA_SECRET:-} + token: ${ASTRA_TOKEN:-} + database: ${ASTRA_DATABASE:-} + - id: s3 + data: + bucket-name: "${S3_BUCKET_NAME:-documents}" + access-key: "${S3_ACCESS_KEY:-minioadmin}" + secret: "${S3_SECRET:-minioadmin}" + region: "${S3_REGION:-}" + - id: google + data: + client-id: "${GOOGLE_CLIENT_ID:-}" + - id: github + data: + client-id: "${GITHUB_CLIENT_ID:-}" \ No newline at end of file diff --git a/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/write-to-astra.yaml b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/write-to-astra.yaml new file mode 100644 index 000000000..dcba707db --- /dev/null +++ b/langstream-webservice/src/main/docker/jib/app/archetypes/website-qa-chatbot/write-to-astra.yaml @@ -0,0 +1,62 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "Write to AstraDB" +topics: + - name: "chunks-topic" + creation-mode: create-if-not-exists +assets: + - name: "documents-table" + asset-type: "cassandra-table" + creation-mode: create-if-not-exists + config: + table-name: "documents" + keyspace: "documents" + datasource: "AstraDatasource" + create-statements: + - | + CREATE TABLE IF NOT EXISTS documents.documents ( + filename TEXT, + chunk_id int, + num_tokens int, + language TEXT, + text TEXT, + embeddings_vector VECTOR, + PRIMARY KEY (filename, chunk_id)); + - | + CREATE CUSTOM INDEX IF NOT EXISTS documents_ann_index ON documents.documents(embeddings_vector) USING 'StorageAttachedIndex'; +pipeline: + - name: "Delete stale chunks" + input: "chunks-topic" + type: "query" + configuration: + datasource: "AstraDatasource" + when: "fn:toInt(properties.text_num_chunks) == (fn:toInt(properties.chunk_id) + 1)" + mode: "execute" + query: "DELETE FROM documents.documents WHERE filename = ? AND chunk_id > ?" + output-field: "value.delete-results" + fields: + - "value.filename" + - "fn:toInt(value.chunk_id)" + - name: "Write to Astra" + type: "vector-db-sink" + resources: + size: 2 + configuration: + datasource: "AstraDatasource" + table-name: "documents" + keyspace: "documents" + mapping: "filename=value.filename, chunk_id=value.chunk_id, language=value.language, text=value.text, embeddings_vector=value.embeddings_vector, num_tokens=value.chunk_num_tokens" \ No newline at end of file diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/LangStreamControlPlaneWebApplication.java b/langstream-webservice/src/main/java/ai/langstream/webservice/LangStreamControlPlaneWebApplication.java index 3f5343844..b458876d3 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/LangStreamControlPlaneWebApplication.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/LangStreamControlPlaneWebApplication.java @@ -16,6 +16,7 @@ package ai.langstream.webservice; import ai.langstream.webservice.config.ApplicationDeployProperties; +import ai.langstream.webservice.config.ArchetypesProperties; import ai.langstream.webservice.config.AuthTokenProperties; import ai.langstream.webservice.config.StorageProperties; import ai.langstream.webservice.config.TenantProperties; @@ -30,6 +31,7 @@ @EnableConfigurationProperties({ LangStreamProperties.class, StorageProperties.class, + ArchetypesProperties.class, TenantProperties.class, AuthTokenProperties.class, ApplicationDeployProperties.class diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java b/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java index ddc33857f..d4eac3623 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/application/ApplicationResource.java @@ -242,7 +242,7 @@ private ParsedApplication parseApplicationInstance( return parsedApplication; } - private void withApplicationZip( + public static void withApplicationZip( Optional file, BiConsumer> appDirectoriesConsumer) throws Exception { if (file.isPresent()) { diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeBasicInfo.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeBasicInfo.java new file mode 100644 index 000000000..10d0f8db2 --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeBasicInfo.java @@ -0,0 +1,32 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import ai.langstream.api.archetype.ArchetypeDefinition; +import java.util.List; + +public record ArchetypeBasicInfo( + String id, String title, List labels, String description, String icon) { + + public static ArchetypeBasicInfo fromArchetypeDefinition(ArchetypeDefinition definition) { + return new ArchetypeBasicInfo( + definition.archetype().id(), + definition.archetype().title(), + definition.archetype().labels(), + definition.archetype().description(), + definition.archetype().icon()); + } +} diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeResource.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeResource.java new file mode 100644 index 000000000..20f72a5cc --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeResource.java @@ -0,0 +1,194 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import ai.langstream.api.archetype.ArchetypeDefinition; +import ai.langstream.api.model.Application; +import ai.langstream.api.webservice.application.ApplicationDescription; +import ai.langstream.impl.common.ApplicationPlaceholderResolver; +import ai.langstream.impl.parser.ModelBuilder; +import ai.langstream.webservice.application.ApplicationService; +import ai.langstream.webservice.application.CodeStorageService; +import ai.langstream.webservice.security.infrastructure.primary.TokenAuthFilter; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.constraints.NotBlank; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +@RestController +@Tag(name = "archetypes") +@RequestMapping("/api/archetypes") +@Slf4j +@AllArgsConstructor +public class ArchetypeResource { + + ArchetypeService archetypeService; + ApplicationService applicationService; + CodeStorageService codeStorageService; + + private void performAuthorization(Authentication authentication, final String tenant) { + if (authentication == null) { + return; + } + if (!authentication.isAuthenticated()) { + throw new IllegalStateException(); + } + if (authentication.getAuthorities() != null) { + final GrantedAuthority grantedAuthority = + authentication.getAuthorities().stream() + .filter( + authority -> + authority + .getAuthority() + .equals(TokenAuthFilter.ROLE_ADMIN)) + .findFirst() + .orElse(null); + if (grantedAuthority != null) { + return; + } + } + if (authentication.getPrincipal() == null) { + throw new IllegalStateException(); + } + final String principal = authentication.getPrincipal().toString(); + if (tenant.equals(principal)) { + return; + } + throw new ResponseStatusException(HttpStatus.FORBIDDEN); + } + + @GetMapping("/{tenant}") + @Operation(summary = "Get all archetypes") + List getArchetypes( + Authentication authentication, @NotBlank @PathVariable("tenant") String tenant) { + performAuthorization(authentication, tenant); + return archetypeService.getAllArchetypesBasicInfo(tenant); + } + + @GetMapping("/{tenant}/{id}") + @Operation(summary = "Get metadata about an archetype") + ArchetypeDefinition getArchetype( + Authentication authentication, + @NotBlank @PathVariable("tenant") String tenant, + @NotBlank @PathVariable("id") String id) { + performAuthorization(authentication, tenant); + ArchetypeDefinition result = archetypeService.getArchetype(tenant, id); + if (result == null) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + return result; + } + + @Data + static class ParsedApplication { + private ModelBuilder.ApplicationWithPackageInfo application; + private String codeArchiveReference; + } + + @PostMapping( + value = "/{tenant}/{idarchetype}/applications/{idapplication}", + consumes = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Create and deploy an application from an archetype") + ApplicationDescription.ApplicationDefinition deployApplication( + Authentication authentication, + @NotBlank @PathVariable("tenant") String tenant, + @NotBlank @PathVariable("idapplication") String applicationId, + @NotBlank @PathVariable("idarchetype") String idarchetype, + @RequestBody Map applicationParameters, + @RequestParam(value = "dry-run", required = false) boolean dryRun) + throws Exception { + performAuthorization(authentication, tenant); + + ArchetypeDefinition archetype = archetypeService.getArchetype(tenant, idarchetype); + if (archetype == null) { + log.info("Archetype {} not found", idarchetype); + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + + final ParsedApplication parsedApplication = + buildApplicationFromArchetype( + applicationId, idarchetype, applicationParameters, tenant, dryRun); + final Application application; + if (dryRun) { + application = + ApplicationPlaceholderResolver.resolvePlaceholders( + parsedApplication.getApplication().getApplication()); + + } else { + applicationService.deployApplication( + tenant, + applicationId, + parsedApplication.getApplication(), + parsedApplication.getCodeArchiveReference()); + application = parsedApplication.getApplication().getApplication(); + } + return new ApplicationDescription.ApplicationDefinition(application); + } + + private ParsedApplication buildApplicationFromArchetype( + String name, + String archetypeId, + Map applicationParameters, + String tenant, + boolean dryRun) + throws Exception { + final ParsedApplication parsedApplication = new ParsedApplication(); + Path archetypePath = archetypeService.getArchetypePath(archetypeId); + final ModelBuilder.ApplicationWithPackageInfo app = + ModelBuilder.buildApplicationInstanceFromArchetype( + archetypePath, applicationParameters); + final Path zip = archetypeService.buildArchetypeZip(archetypeId); + + final String codeArchiveReference; + if (dryRun) { + codeArchiveReference = null; + } else { + codeArchiveReference = + codeStorageService.deployApplicationCodeStorage( + tenant, + name, + zip, + app.getPyBinariesDigest(), + app.getJavaBinariesDigest()); + } + log.info( + "Parsed application {} {} with code archive {}", + name, + app.getApplication(), + codeArchiveReference); + parsedApplication.setApplication(app); + parsedApplication.setCodeArchiveReference(codeArchiveReference); + + return parsedApplication; + } +} diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeService.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeService.java new file mode 100644 index 000000000..b144c6cb5 --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeService.java @@ -0,0 +1,50 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import ai.langstream.api.archetype.ArchetypeDefinition; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +@AllArgsConstructor +public class ArchetypeService { + + private final ArchetypeStore archetypeStore; + + public ArchetypeDefinition getArchetype(String tenant, String id) { + return archetypeStore.get(id); + } + + public List getAllArchetypesBasicInfo(String tenant) { + return archetypeStore.list().stream() + .map(id -> ArchetypeBasicInfo.fromArchetypeDefinition(archetypeStore.get(id))) + .collect(Collectors.toList()); + } + + public Path getArchetypePath(String archetypeId) { + return archetypeStore.getArchetypePath(archetypeId); + } + + public Path buildArchetypeZip(String archetypeId) throws Exception { + return archetypeStore.buildArchetypeZip(archetypeId); + } +} diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java new file mode 100644 index 000000000..a932e22bb --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStore.java @@ -0,0 +1,92 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import ai.langstream.api.archetype.ArchetypeDefinition; +import ai.langstream.cli.utils.ApplicationPackager; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ArchetypeStore { + + private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + private final Map archetypeDefinitions = new HashMap<>(); + private final Map archetypePaths = new HashMap<>(); + + public void load(String path) throws Exception { + Path directory = Paths.get(path); + log.info("Loading archetypes from {}", directory); + if (Files.isDirectory(directory)) { + directory = directory.toAbsolutePath(); + try (var stream = Files.newDirectoryStream(directory)) { + stream.forEach( + file -> { + log.info("Loading archetype from {}", file); + try { + if (Files.isDirectory(file)) { + String id = loadArchetype(file); + archetypePaths.put(id, file); + } + } catch (Exception e) { + log.error("Failed to load archetype from {}", file, e); + throw new RuntimeException(e); + } + }); + } + } + } + + private String loadArchetype(Path file) throws Exception { + Path archetypeFile = file.resolve("archetype.yaml"); + if (Files.isRegularFile(archetypeFile)) { + ArchetypeDefinition archetypeDefinition = + mapper.readValue(archetypeFile.toFile(), ArchetypeDefinition.class); + archetypeDefinitions.put(archetypeDefinition.archetype().id(), archetypeDefinition); + return archetypeDefinition.archetype().id(); + } else { + throw new IllegalArgumentException("Archetype file not found: " + archetypeFile); + } + } + + public List list() { + return new ArrayList<>(archetypeDefinitions.keySet()); + } + + public ArchetypeDefinition get(String archetypeId) { + return archetypeDefinitions.get(archetypeId); + } + + public Path getArchetypePath(String archetypeId) { + return archetypePaths.get(archetypeId); + } + + public Path buildArchetypeZip(String archetypeId) throws Exception { + Path path = getArchetypePath(archetypeId); + if (path == null) { + throw new IllegalArgumentException("Archetype not found: " + archetypeId); + } + return ApplicationPackager.buildZip(path.toFile(), msg -> log.info("{}", msg)); + } +} diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStoreFactory.java b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStoreFactory.java new file mode 100644 index 000000000..d8c562f9b --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/archetype/ArchetypeStoreFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import ai.langstream.webservice.config.ArchetypesProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ArchetypeStoreFactory { + + @Autowired ArchetypesProperties archetypesProperties; + + @Bean + public ArchetypeStore getArchetypeStore() throws Exception { + ArchetypeStore store = new ArchetypeStore(); + store.load(archetypesProperties.getPath()); + return store; + } +} diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/config/ArchetypesProperties.java b/langstream-webservice/src/main/java/ai/langstream/webservice/config/ArchetypesProperties.java new file mode 100644 index 000000000..b7b5d177c --- /dev/null +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/config/ArchetypesProperties.java @@ -0,0 +1,30 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "application.archetypes") +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ArchetypesProperties { + + private String path = "/app/archetypes"; +} diff --git a/langstream-webservice/src/main/resources/application.properties b/langstream-webservice/src/main/resources/application.properties index 028a8ad8c..72b37f0df 100644 --- a/langstream-webservice/src/main/resources/application.properties +++ b/langstream-webservice/src/main/resources/application.properties @@ -39,4 +39,6 @@ application.storage.code.type=none application.tenants.default-tenant.create=true application.tenants.default-tenant.name=default -application.apps.gateway.require-authentication=false \ No newline at end of file +application.apps.gateway.require-authentication=false + +application.archetypes.path=/app/archetypes \ No newline at end of file diff --git a/langstream-webservice/src/test/archetypes/simple/.langstreamignore b/langstream-webservice/src/test/archetypes/simple/.langstreamignore new file mode 100644 index 000000000..1363f07bc --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/.langstreamignore @@ -0,0 +1,149 @@ +# .langstreamignore file inspired by https://github.com/github/gitignore/blob/main/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# These folders hold the libs built for the target +# and we need them in the package +!python/lib/ +!java/lib/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +pdm.lock +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ diff --git a/langstream-webservice/src/test/archetypes/simple/archetype.yaml b/langstream-webservice/src/test/archetypes/simple/archetype.yaml new file mode 100644 index 000000000..373958380 --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/archetype.yaml @@ -0,0 +1,44 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +archetype: + title: Simple + id: simple + sections: + - title: Section 1 + description: "Xxxxx" + parameters: + - name: "s1" + binding: "globals.string-value" + - name: "i1" + binding: "globals.input-value" + - name: "r1" + binding: "globals.int-value" + - name: "m1" + binding: "globals.map-value" + - name: "l1" + binding: "globals.list-value" + - name: "m2" + binding: "globals.nested-map.key2.key2-1" + - title: Section 2 + description: "Xxxxx" + parameters: + - name: "s2" + binding: "secrets.open-ai.foo" + - name: "i2" + binding: "secrets.open-ai.foo-int" + - name: "k2" + binding: "secrets.kafka.bootstrap-servers" diff --git a/langstream-webservice/src/test/archetypes/simple/configuration.yaml b/langstream-webservice/src/test/archetypes/simple/configuration.yaml new file mode 100644 index 000000000..cd4bed54e --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/configuration.yaml @@ -0,0 +1,24 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +configuration: + resources: + - type: "open-ai-configuration" + name: "OpenAI Azure configuration" + configuration: + url: "${secrets.open-ai.url}" + access-key: "${secrets.open-ai.access-key}" + provider: "${secrets.open-ai.provider}" \ No newline at end of file diff --git a/langstream-webservice/src/test/archetypes/simple/gateways.yaml b/langstream-webservice/src/test/archetypes/simple/gateways.yaml new file mode 100644 index 000000000..8b02c5323 --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/gateways.yaml @@ -0,0 +1,28 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +gateways: + - id: chat + type: chat + chat-options: + answers-topic: output-topic + questions-topic: input-topic + headers: + - key: langstream-client-session-id + value-from-parameters: sessionId + + diff --git a/langstream-webservice/src/test/archetypes/simple/instance.yaml b/langstream-webservice/src/test/archetypes/simple/instance.yaml new file mode 100644 index 000000000..025230ad2 --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/instance.yaml @@ -0,0 +1,38 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +instance: + globals: + string-value: "string-value" + int-value: 42 + list-value: + - "list-value-1" + - "list-value-2" + map-value: + key1: "map-value-1" + key2: "map-value-2" + nested-map: + key1: + key1-1: "nested-map-value-1-1" + key1-2: "nested-map-value-1-2" + key2: + key2-1: "nested-map-value-2-1" + key2-2: "nested-map-value-2-2" + streamingCluster: + type: "kafka" + configuration: + admin: + bootstrap.servers: ${secrets.kafka.bootstrap-servers} diff --git a/langstream-webservice/src/test/archetypes/simple/pipeline.yaml b/langstream-webservice/src/test/archetypes/simple/pipeline.yaml new file mode 100644 index 000000000..b99888136 --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/pipeline.yaml @@ -0,0 +1,29 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "Exclamation processor" +topics: + - name: "input-topic" + creation-mode: create-if-not-exists + - name: "output-topic" + creation-mode: create-if-not-exists +pipeline: + - name: "Process using Python" + type: "python-processor" + input: "input-topic" + output: "output-topic" + configuration: + className: example.Exclamation \ No newline at end of file diff --git a/langstream-webservice/src/test/archetypes/simple/python/example.py b/langstream-webservice/src/test/archetypes/simple/python/example.py new file mode 100644 index 000000000..3fd401623 --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/python/example.py @@ -0,0 +1,24 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from langstream import SimpleRecord, Processor + + +# Example Python processor that adds an exclamation mark to the end of the record value +class Exclamation(Processor): + def process(self, record): + return [SimpleRecord(record.value() + "!!", headers=record.headers())] diff --git a/langstream-webservice/src/test/archetypes/simple/secrets.yaml b/langstream-webservice/src/test/archetypes/simple/secrets.yaml new file mode 100644 index 000000000..63af20a4e --- /dev/null +++ b/langstream-webservice/src/test/archetypes/simple/secrets.yaml @@ -0,0 +1,27 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +secrets: + - id: kafka + data: + username: "" + password: "" + tenant: "" + bootstrap_servers: "" + - id: open-ai + data: + foo: "bar" + foo-int: 75 diff --git a/langstream-webservice/src/test/java/ai/langstream/webservice/application/AppTestHelper.java b/langstream-webservice/src/test/java/ai/langstream/webservice/application/AppTestHelper.java index 55018b693..42fafb0a8 100644 --- a/langstream-webservice/src/test/java/ai/langstream/webservice/application/AppTestHelper.java +++ b/langstream-webservice/src/test/java/ai/langstream/webservice/application/AppTestHelper.java @@ -15,10 +15,10 @@ */ package ai.langstream.webservice.application; +import static ai.langstream.cli.utils.ApplicationPackager.buildZip; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import ai.langstream.cli.commands.applications.AbstractDeployApplicationCmd; import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -48,7 +48,7 @@ private static File createTempFile(String content) { private static MockMultipartFile getMultipartFile(String application) throws Exception { final Path zip = - AbstractDeployApplicationCmd.buildZip( + buildZip( application == null ? null : createTempFile(application), System.out::println); return new MockMultipartFile( diff --git a/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java b/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java new file mode 100644 index 000000000..e835ecd85 --- /dev/null +++ b/langstream-webservice/src/test/java/ai/langstream/webservice/archetype/ArchetypeResourceTest.java @@ -0,0 +1,268 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.webservice.archetype; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import ai.langstream.api.model.Secrets; +import ai.langstream.api.storage.ApplicationStore; +import ai.langstream.impl.k8s.tests.KubeK3sServer; +import ai.langstream.webservice.WebAppTestConfig; +import ai.langstream.webservice.application.CodeStorageService; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.servlet.MockMvc; + +@SpringBootTest(properties = {"application.archetypes.path=src/test/archetypes"}) +@AutoConfigureMockMvc +@Slf4j +@Import(WebAppTestConfig.class) +@DirtiesContext +class ArchetypeResourceTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Autowired MockMvc mockMvc; + + @RegisterExtension static final KubeK3sServer k3s = new KubeK3sServer(true); + + @Autowired ApplicationStore applicationStore; + + @Autowired CodeStorageService codeStorageService; + + @Test + void testArchetypesMetadata() throws Exception { + mockMvc.perform(put("/api/tenants/my-tenant")).andExpect(status().isOk()); + mockMvc.perform(get("/api/archetypes/my-tenant")) + .andExpect(status().isOk()) + .andExpect( + result -> { + List> list = + MAPPER.readValue( + result.getResponse().getContentAsString(), + new TypeReference>>() {}); + assertEquals(1, list.size()); + assertEquals("simple", list.get(0).get("id")); + }); + mockMvc.perform(get("/api/archetypes/my-tenant/not-exists")) + .andExpect(status().isNotFound()); + + mockMvc.perform(get("/api/archetypes/my-tenant/simple")) + .andExpect(status().isOk()) + .andExpect( + result -> { + log.info("Result {}", result.getResponse().getContentAsString()); + assertEquals( + """ + { + "archetype" : { + "id" : "simple", + "title" : "Simple", + "labels" : null, + "description" : null, + "icon" : null, + "sections" : [ { + "title" : "Section 1", + "description" : "Xxxxx", + "parameters" : [ { + "default" : null, + "name" : "s1", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.string-value", + "required" : false + }, { + "default" : null, + "name" : "i1", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.input-value", + "required" : false + }, { + "default" : null, + "name" : "r1", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.int-value", + "required" : false + }, { + "default" : null, + "name" : "m1", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.map-value", + "required" : false + }, { + "default" : null, + "name" : "l1", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.list-value", + "required" : false + }, { + "default" : null, + "name" : "m2", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "globals.nested-map.key2.key2-1", + "required" : false + } ] + }, { + "title" : "Section 2", + "description" : "Xxxxx", + "parameters" : [ { + "default" : null, + "name" : "s2", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "secrets.open-ai.foo", + "required" : false + }, { + "default" : null, + "name" : "i2", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "secrets.open-ai.foo-int", + "required" : false + }, { + "default" : null, + "name" : "k2", + "label" : null, + "description" : null, + "type" : null, + "subtype" : null, + "binding" : "secrets.kafka.bootstrap-servers", + "required" : false + } ] + } ] + } + }""", + result.getResponse().getContentAsString()); + }); + } + + @Test + void testDeployFromArchetype() throws Exception { + mockMvc.perform(put("/api/tenants/my-tenant")).andExpect(status().isOk()); + mockMvc.perform( + post("/api/archetypes/my-tenant/simple/applications/app-id") + .contentType("application/json") + .content( + """ + { + "s1": "value 1", + "i1": 50, + "r1": 89, + "m1": { + "key1": "value 1", + "key2": { + "key2-1": "value 2-1" + } + }, + "m2": "a", + "l1": [ + "value 1", + "value 2" + ], + "s2": "value secret 2", + "i2": 100, + "k2": "value 3" + } + """)) + .andExpect(status().isOk()) + .andExpect( + result -> { + ObjectMapper mapper = new ObjectMapper(); + String body = result.getResponse().getContentAsString(); + log.info("Deploy result {}", body); + Map parse = + mapper.readValue( + body, new TypeReference>() {}); + Map instance = + (Map) parse.get("instance"); + Map globals = + (Map) instance.get("globals"); + assertEquals(50, globals.get("input-value")); + assertEquals(89, globals.get("int-value")); + assertEquals(List.of("value 1", "value 2"), globals.get("list-value")); + assertEquals( + Map.of( + "key1", + "value 1", + "key2", + Map.of("key2-1", "value 2-1")), + globals.get("map-value")); + assertEquals( + Map.of( + "key1", + Map.of( + "key1-1", + "nested-map-value-1-1", + "key1-2", + "nested-map-value-1-2"), + "key2", + Map.of( + "key2-1", + "a", + "key2-2", + "nested-map-value-2-2")), + globals.get("nested-map")); + }); + + String codeArchiveReference = + applicationStore.getSpecs("my-tenant", "app-id").getCodeArchiveReference(); + Secrets secrets = applicationStore.getSecrets("my-tenant", "app-id"); + assertEquals("value secret 2", secrets.secrets().get("open-ai").data().get("foo")); + assertEquals(100, secrets.secrets().get("open-ai").data().get("foo-int")); + assertEquals("value 3", secrets.secrets().get("kafka").data().get("bootstrap-servers")); + + byte[] bytes = + codeStorageService.downloadApplicationCode( + "my-tenant", "app-id", codeArchiveReference); + assertNotNull(bytes); + } +} diff --git a/pom.xml b/pom.xml index a681878ba..ac4d81a8f 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,7 @@ 3.11.0 3.0.0 3.1.2 - 3.3.1 + 3.4.0 3.0.7 2.11.5 4.7.4