Archetypes - part 1 (#642)

examples/applications/webcrawler-source/crawler.yaml

@@ -87,7 +87,7 @@ pipeline:
     type: "compute-ai-embeddings"
     output: "chunks-topic"
     configuration:
-      model: "text-embedding-ada-002" # This needs to match the name of the model deployment, not the base model
+      model: "${secrets.open-ai.embeddings-model}"
       embeddings-field: "value.embeddings_vector"
       text: "{{ value.text }}"
       batch-size: 10

langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java

@@ -19,6 +19,7 @@
 import ai.langstream.admin.client.http.HttpClientProperties;
 import ai.langstream.admin.client.http.Retry;
 import ai.langstream.admin.client.model.Applications;
+import ai.langstream.admin.client.model.Archetypes;
 import ai.langstream.admin.client.util.MultiPartBodyPublisher;
 import ai.langstream.admin.client.util.Slf4jLAdminClientLogger;
 import java.io.InputStream;
@@ -204,6 +205,26 @@ public Applications applications() {
         return new ApplicationsImpl();
     }
 
+    public Archetypes archetypes() {
+        return new ArchetypesImpl();
+    }
+
+    private class ArchetypesImpl implements Archetypes {
+        @Override
+        @SneakyThrows
+        public String list() {
+            final String tenant = configuration.getTenant();
+            return http(newGet(String.format("/archetypes/%s", tenant))).body();
+        }
+
+        @Override
+        @SneakyThrows
+        public String get(String archetype) {
+            final String tenant = configuration.getTenant();
+            return http(newGet(String.format("/archetypes/%s/%s", tenant, archetype))).body();
+        }
+    }
+
     private class ApplicationsImpl implements Applications {
         @Override
         public String deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher) {

langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Archetypes.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.admin.client.model;
+
+public interface Archetypes {
+    String list();
+
+    String get(String archetypeId);
+}

langstream-api-gateway/pom.xml

@@ -32,8 +32,6 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<springdoc-openapi-starter-webmvc.version>2.1.0</springdoc-openapi-starter-webmvc.version>
-		<jib-maven-plugin.version>3.3.1</jib-maven-plugin.version>
-
   </properties>
 
 	<dependencies>

langstream-api/src/main/java/ai/langstream/api/archetype/ArchetypeDefinition.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.archetype;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+public record ArchetypeDefinition(Archetype archetype) {
+
+    public record Archetype(
+            String id,
+            String title,
+            List<String> labels,
+            String description,
+            String icon,
+            List<Section> sections) {}
+
+    public record Section(String title, String description, List<Parameter> parameters) {}
+
+    public record Parameter(
+            String name,
+            String label,
+            String description,
+            String type,
+            String subtype,
+            String binding,
+            boolean required,
+            @JsonProperty("default") Object defaultVal) {}
+}

langstream-cli/src/main/java/ai/langstream/cli/commands/RootArchetypeCmd.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.cli.commands;
+
+import ai.langstream.cli.commands.archetypes.ListArchetypesCmd;
+import lombok.Getter;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+        name = "archetypes",
+        header = "Use LangStream Archetypes",
+        subcommands = {ListArchetypesCmd.class})
+@Getter
+public class RootArchetypeCmd {
+    @CommandLine.ParentCommand private RootCmd rootCmd;
+}

langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java

@@ -27,6 +27,7 @@
         scope = CommandLine.ScopeType.INHERIT,
         header = "LangStream CLI",
         subcommands = {
+            RootArchetypeCmd.class,
             RootAppCmd.class,
             ConfigureCmd.class,
             RootTenantCmd.class,

langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java

@@ -15,19 +15,16 @@
  */
 package ai.langstream.cli.commands.applications;
 
+import static ai.langstream.cli.utils.ApplicationPackager.buildZip;
+
 import ai.langstream.admin.client.util.MultiPartBodyPublisher;
-import ai.langstream.cli.commands.GitIgnoreParser;
 import ai.langstream.cli.util.LocalFileReferenceResolver;
 import java.io.File;
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.function.Consumer;
 import lombok.SneakyThrows;
-import net.lingala.zip4j.ZipFile;
-import net.lingala.zip4j.model.ZipParameters;
 import picocli.CommandLine;
 
 public abstract class AbstractDeployApplicationCmd extends BaseApplicationCmd {
@@ -255,57 +252,6 @@ public void run() {
         }
     }
 
-    public static Path buildZip(File appDirectory, Consumer<String> logger) throws IOException {
-        final Path tempZip = Files.createTempFile("app", ".zip");
-        try (final ZipFile zip = new ZipFile(tempZip.toFile())) {
-            addApp(appDirectory, zip, logger);
-        }
-        return tempZip;
-    }
-
-    private static void addApp(File appDirectory, ZipFile zip, Consumer<String> logger)
-            throws IOException {
-        if (appDirectory == null) {
-            return;
-        }
-        logger.accept(String.format("packaging app: %s", appDirectory.getAbsolutePath()));
-        if (appDirectory.isDirectory()) {
-            File ignoreFile = appDirectory.toPath().resolve(".langstreamignore").toFile();
-            if (ignoreFile.exists()) {
-                GitIgnoreParser parser = new GitIgnoreParser(ignoreFile.toPath());
-                addDirectoryFilesWithLangstreamIgnore(appDirectory, appDirectory, parser, zip);
-            } else {
-                for (File file : appDirectory.listFiles()) {
-                    if (file.isDirectory()) {
-                        zip.addFolder(file);
-                    } else {
-                        zip.addFile(file);
-                    }
-                }
-            }
-        } else {
-            zip.addFile(appDirectory);
-        }
-        logger.accept("app packaged");
-    }
-
-    private static void addDirectoryFilesWithLangstreamIgnore(
-            File appDirectory, File directory, GitIgnoreParser parser, ZipFile zip)
-            throws IOException {
-        for (File file : directory.listFiles()) {
-            if (!parser.matches(file)) {
-                if (file.isDirectory()) {
-                    addDirectoryFilesWithLangstreamIgnore(appDirectory, file, parser, zip);
-                } else {
-                    ZipParameters zipParameters = new ZipParameters();
-                    String filename = appDirectory.toURI().relativize(file.toURI()).getPath();
-                    zipParameters.setFileNameInZip(filename);
-                    zip.addFile(file, zipParameters);
-                }
-            }
-        }
-    }
-
     public static MultiPartBodyPublisher buildMultipartContentForAppZip(
             Map<String, Object> formData) {
         final MultiPartBodyPublisher multiPartBodyPublisher = new MultiPartBodyPublisher();

langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/BaseArchetypeCmd.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.cli.commands.archetypes;
+
+import ai.langstream.cli.commands.BaseCmd;
+import ai.langstream.cli.commands.RootArchetypeCmd;
+import ai.langstream.cli.commands.RootCmd;
+import picocli.CommandLine;
+
+public abstract class BaseArchetypeCmd extends BaseCmd {
+
+    @CommandLine.ParentCommand private RootArchetypeCmd rootAppCmd;
+
+    @Override
+    protected RootCmd getRootCmd() {
+        return rootAppCmd.getRootCmd();
+    }
+}

langstream-cli/src/main/java/ai/langstream/cli/commands/archetypes/ListArchetypesCmd.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.cli.commands.archetypes;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.function.BiFunction;
+import lombok.SneakyThrows;
+import picocli.CommandLine;
+
+@CommandLine.Command(name = "list", header = "List all LangStream archetypes")
+public class ListArchetypesCmd extends BaseArchetypeCmd {
+
+    protected static final String[] COLUMNS_FOR_RAW = {"id", "labels"};
+
+    @CommandLine.Option(
+            names = {"-o"},
+            description = "Output format. Formats are: yaml, json, raw. Default value is raw.")
+    private Formats format = Formats.raw;
+
+    @Override
+    @SneakyThrows
+    public void run() {
+        ensureFormatIn(format, Formats.raw, Formats.json, Formats.yaml);
+        final String body = getClient().archetypes().list();
+        print(format, body, COLUMNS_FOR_RAW, getRawFormatValuesSupplier());
+    }
+
+    public static BiFunction<JsonNode, String, Object> getRawFormatValuesSupplier() {
+        return (jsonNode, s) -> {
+            switch (s) {
+                case "id":
+                    return searchValueInJson(jsonNode, "archetype.id");
+                case "labels":
+                    return searchValueInJson(jsonNode, "archetype.labels");
+                default:
+                    return jsonNode.get(s);
+            }
+        };
+    }
+}