LangStream · nicoloboschi · Oct 19, 2023 · Oct 17, 2023 · Oct 17, 2023 · Oct 17, 2023
diff --git a/bin/get-cli.sh b/bin/get-cli.sh
@@ -45,18 +45,7 @@ echo "                   |___/                                    ";
 
 
 get_latest_release_tarball_url() {
-  if ! command -v jq > /dev/null; then
-  	echo "Not found."
-  	echo "======================================================================================================"
-  	echo " Please install jq on your system using your favourite package manager."
-  	echo ""
-  	echo " Restart after installing jq."
-  	echo "======================================================================================================"
-  	echo " In alternative you can set a fixed LangStream CLI version by setting LANGSTREAM_CLI_URL."
-    echo "======================================================================================================"
-  	echo ""
-  	exit 1
-  fi
+
    curl -Ss https://api.github.com/repos/LangStream/langstream/releases/latest | jq -r '.assets[] | select((.name | contains("langstream-cli")) and (.name | contains(".zip"))) | .browser_download_url'
 }
 
@@ -69,6 +58,17 @@ echo ""
 echo "$(tput setaf 6)Checking archive:$(tput setaf 7)"
 if [ -z "$LANGSTREAM_CLI_URL" ]; then
   echo "$(tput setaf 2)LANGSTREAM_CLI_URL$(tput setaf 7) environment not set, checking for the latest release"
+  if ! command -v jq > /dev/null; then
+    	echo "======================================================================================================"
+    	echo " Please install jq on your system using your favourite package manager."
+    	echo ""
+    	echo " Restart after installing jq."
+    	echo "======================================================================================================"
+    	echo " In alternative you can set a fixed LangStream CLI version by setting LANGSTREAM_CLI_URL."
+      echo "======================================================================================================"
+    	echo ""
+    	exit 1
+  fi
   LANGSTREAM_CLI_URL=$(get_latest_release_tarball_url)
   echo "$(tput setaf 2)[OK]$(tput setaf 7) - Using $LANGSTREAM_CLI_URL"
 else
@@ -101,7 +101,6 @@ esac
 echo "$(tput setaf 2)[OK]$(tput setaf 7) - Ready to install $(basename $downloaded_extracted_dir)."
 
 if ! command -v unzip > /dev/null; then
-	echo "Not found."
 	echo "======================================================================================================"
 	echo " Please install unzip on your system using your favourite package manager."
 	echo ""
@@ -113,7 +112,6 @@ fi
 echo "$(tput setaf 2)[OK]$(tput setaf 7) - unzip command is available"
 
 if ! command -v curl > /dev/null; then
-	echo "Not found."
 	echo ""
 	echo "======================================================================================================"
 	echo " Please install curl on your system using your favourite package manager."

diff --git a/conf/cli.yaml b/conf/cli.yaml
@@ -23,7 +23,7 @@ profiles:
   local-docker-run:
     webServiceUrl: "http://localhost:8090"
     apiGatewayUrl: "ws://localhost:8091"
-    tenant: "local-docker-run"
+    tenant: "default"
     token: null
     name: "local-docker-run"
 currentProfile: "default"
diff --git a/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java b/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java
@@ -19,10 +19,24 @@
 import ai.langstream.cli.commands.RootCmd;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import lombok.SneakyThrows;
 import picocli.CommandLine;
 
 public class LangStreamCLI {
 
+    @SneakyThrows
+    public static Path getLangstreamCLIHomeDirectory() {
+        final String userHome = System.getProperty("user.home");
+        if (!userHome.isBlank() && !"?".equals(userHome)) {
+            final Path langstreamDir = Path.of(userHome, ".langstream");
+            Files.createDirectories(langstreamDir);
+            return langstreamDir;
+        }
+        return null;
+    }
+
     public static void main(String... args) {
         int exitCode = execute(args);
         System.exit(exitCode);

diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmd.java
@@ -20,22 +20,28 @@
 import ai.langstream.admin.client.http.GenericRetryExecution;
 import ai.langstream.admin.client.http.HttpClientProperties;
 import ai.langstream.admin.client.http.NoRetryPolicy;
+import ai.langstream.cli.LangStreamCLI;
 import ai.langstream.cli.NamedProfile;
 import ai.langstream.cli.api.model.Gateways;
 import ai.langstream.cli.commands.VersionProvider;
 import ai.langstream.cli.commands.applications.MermaidAppDiagramGenerator;
 import ai.langstream.cli.commands.applications.UIAppCmd;
 import ai.langstream.cli.util.LocalFileReferenceResolver;
 import java.io.File;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.input.Tailer;
 import org.apache.commons.io.input.TailerListener;
 import picocli.CommandLine;
@@ -45,6 +51,8 @@ public class LocalRunApplicationCmd extends BaseDockerCmd {
 
     protected static final String LOCAL_DOCKER_RUN_PROFILE = "local-docker-run";
 
+    private static final Set<Path> temporaryFiles = new HashSet<>();
+
     @CommandLine.Parameters(description = "ID of the application")
     private String applicationId;
 
@@ -168,8 +176,8 @@ public void run() {
         }
         final File secretsFile = checkFileExistsOrDownload(secretFilePath);
 
-        log("Tenant " + tenant);
-        log("Application " + applicationId);
+        log("Tenant: " + tenant);
+        log("Application: " + applicationId);
         log("Application directory: " + appDirectory.getAbsolutePath());
         if (singleAgentId != null && !singleAgentId.isEmpty()) {
             log("Filter agent: " + singleAgentId);
@@ -241,6 +249,17 @@ public void run() {
 
         updateLocalDockerRunProfile(tenant);
 
+        Runtime.getRuntime()
+                .addShutdownHook(
+                        new Thread(
+                                () -> {
+                                    log("Cleaning environment");
+                                    for (Path temporaryFile : temporaryFiles) {
+                                        debug("Deleting temporary file: " + temporaryFile);
+                                        FileUtils.deleteQuietly(temporaryFile.toFile());
+                                    }
+                                }));
+
         executeOnDocker(
                 tenant,
                 applicationId,
@@ -281,12 +300,13 @@ private void executeOnDocker(
             boolean startDatabase,
             boolean dryRun)
             throws Exception {
-        File tmpInstanceFile = Files.createTempFile("instance", ".yaml").toFile();
-        Files.write(tmpInstanceFile.toPath(), instanceContents.getBytes(StandardCharsets.UTF_8));
+        final File appTmp = generateTempFile("app");
+        FileUtils.copyDirectory(appDirectory, appTmp);
+        makeDirOrFileReadable(appTmp);
+        File tmpInstanceFile = createReadableTempFile("instance", instanceContents);
         File tmpSecretsFile = null;
         if (secretsContents != null) {
-            tmpSecretsFile = Files.createTempFile("secrets", ".yaml").toFile();
-            Files.write(tmpSecretsFile.toPath(), secretsContents.getBytes(StandardCharsets.UTF_8));
+            tmpSecretsFile = createReadableTempFile("secrets", secretsContents);
         }
         String imageName = dockerImageName + ":" + dockerImageVersion;
         List<String> commandLine = new ArrayList<>();
@@ -315,7 +335,7 @@ private void executeOnDocker(
         }
 
         commandLine.add("-v");
-        commandLine.add(appDirectory.getAbsolutePath() + ":/code/application");
+        commandLine.add(appTmp.getAbsolutePath() + ":/code/application");
         commandLine.add("-v");
         commandLine.add(tmpInstanceFile.getAbsolutePath() + ":/code/instance.yaml");
         if (tmpSecretsFile != null) {
@@ -381,6 +401,38 @@ private void executeOnDocker(
         }
     }
 
+    private static File createReadableTempFile(String prefix, String instanceContents)
+            throws IOException {
+        File tempFile = generateTempFile(prefix);
+        Files.write(tempFile.toPath(), instanceContents.getBytes(StandardCharsets.UTF_8));
+        makeDirOrFileReadable(tempFile);
+        return tempFile;
+    }
+
+    private static void makeDirOrFileReadable(File file) throws IOException {
+        if (file.isDirectory()) {
+            for (File child : file.listFiles()) {
+                makeDirOrFileReadable(child);
+            }
+        }
+        Files.setPosixFilePermissions(file.toPath(), PosixFilePermissions.fromString("rw-r--r--"));
+    }
+
+    @SneakyThrows
+    private static File generateTempFile(String prefix) {
+        Path home = LangStreamCLI.getLangstreamCLIHomeDirectory();
+        final String generatedName = ".langstream_" + prefix + "_" + System.nanoTime();
+        if (home == null) {
+            home = new File(".").toPath();
+        } else {
+            home = home.resolve("tmp");
+            Files.createDirectories(home);
+        }
+        File tempFile = home.resolve(generatedName).toFile();
+        temporaryFiles.add(tempFile.toPath());
+        return tempFile;
+    }
+
     private void startUI(String tenant, String applicationId, Path outputLog, Process process) {
         String body;
         try (final AdminClient localAdminClient =

diff --git a/...tream-cli/src/test/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmdTest.java b/...tream-cli/src/test/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmdTest.java
@@ -19,9 +19,17 @@
 
 import ai.langstream.cli.NamedProfile;
 import ai.langstream.cli.commands.applications.CommandTestBase;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 
@@ -30,11 +38,21 @@ class LocalRunApplicationCmdTest extends CommandTestBase {
     @Test
     void testArgs() throws Exception {
         final Path tempDir = Files.createTempDirectory(this.tempDir, "langstream");
+        final Path secrets = Files.createTempFile("langstream", ".yaml");
+        Files.write(secrets, "secrets: []".getBytes(StandardCharsets.UTF_8));
 
         final String appDir = tempDir.toFile().getAbsolutePath();
         CommandResult result =
                 executeCommand(
-                        "docker", "run", "my-app", "-app", appDir, "--docker-command", "echo");
+                        "docker",
+                        "run",
+                        "my-app",
+                        "-app",
+                        appDir,
+                        "-s",
+                        secrets.toFile().getAbsolutePath(),
+                        "--docker-command",
+                        "echo");
         assertEquals("", result.err());
         assertEquals(0, result.exitCode());
 
@@ -44,10 +62,7 @@ void testArgs() throws Exception {
                 lastLine.contains(
                         "run --rm -i -e START_BROKER=true -e START_MINIO=true -e START_HERDDB=true "
                                 + "-e LANSGSTREAM_TESTER_TENANT=default -e LANSGSTREAM_TESTER_APPLICATIONID=my-app "
-                                + "-e LANSGSTREAM_TESTER_STARTWEBSERVICES=true -e LANSGSTREAM_TESTER_DRYRUN=false "
-                                + "-v "
-                                + appDir
-                                + ":/code/application "));
+                                + "-e LANSGSTREAM_TESTER_STARTWEBSERVICES=true -e LANSGSTREAM_TESTER_DRYRUN=false "));
         assertTrue(
                 lastLine.contains(
                         "--add-host minio.minio-dev.svc.cluster.local:127.0.0.1 "
@@ -57,10 +72,43 @@ void testArgs() throws Exception {
                                 + "-p 8090:8090 "
                                 + "ghcr.io/langstream/langstream-runtime-tester:unknown"));
 
+        final List<String> volumes = extractVolumes(lastLine);
+        assertEquals(3, volumes.size());
+        volumes.forEach(
+                volume -> {
+                    final String hostPath = volume.split(":")[0];
+                    final File file = new File(hostPath);
+                    assertTrue(file.exists());
+                    final Path langstreamTmp =
+                            Path.of(System.getProperty("user.home"), ".langstream", "tmp");
+                    assertEquals(langstreamTmp, file.toPath().getParent());
+                    final Set<PosixFilePermission> posixFilePermissions;
+                    try {
+                        posixFilePermissions = Files.getPosixFilePermissions(file.toPath());
+                        assertTrue(posixFilePermissions.contains(PosixFilePermission.OTHERS_READ));
+                        assertTrue(posixFilePermissions.contains(PosixFilePermission.OWNER_READ));
+                        assertTrue(posixFilePermissions.contains(PosixFilePermission.GROUP_READ));
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+
         final NamedProfile namedProfile = getConfig().getProfiles().get("local-docker-run");
         assertNotNull(namedProfile);
         assertEquals("default", namedProfile.getTenant());
         assertEquals("http://localhost:8090", namedProfile.getWebServiceUrl());
         assertEquals("ws://localhost:8091", namedProfile.getApiGatewayUrl());
     }
+
+    private static List<String> extractVolumes(String input) {
+        List<String> volumes = new ArrayList<>();
+        Pattern pattern = Pattern.compile("-v\\s+([^\\s]+)");
+        Matcher matcher = pattern.matcher(input);
+
+        while (matcher.find()) {
+            volumes.add(matcher.group(1));
+        }
+
+        return volumes;
+    }
 }
diff --git a/langstream-runtime/langstream-runtime-tester/src/main/docker/Dockerfile b/langstream-runtime/langstream-runtime-tester/src/main/docker/Dockerfile
@@ -31,13 +31,15 @@ RUN chmod -R g+w /kafka \
        && bin/kafka-storage.sh format -t $UUID -c config/kraft/server.properties \
        && mkdir -p /tmp/kraft-combined-logs \
        && chmod a+rwx /tmp/kraft-combined-logs \
+       && rm -rf /kafka/kafka.tgz \
        && chmod a+x /minio/minio \
        && chmod a+x /herddb \
        && cd /herddb \
        && unzip herddb-services-0.28.0.zip \
        && mv herddb-services-0.28.0 herddb \
        && chmod a+x /herddb/herddb \
        && chmod a+x /herddb/herddb/bin/* \
+       && rm -rf herddb-services-0.28.0.zip \
        && chown 10000:0 -R /herddb
 
 # Add the runtime code at the end. This optimizes docker layers to not depend on artifacts-specific changes.