Add method to delete old files when deploying a file tree (#31)

* Add method to delete old files when deploying a file tree * Use print0 * Fix empty deploy folder
wpilibsuite · Aug 20, 2024 · 2e7c8ca · 2e7c8ca
1 parent 8c9714b
commit 2e7c8ca
Show file tree

Hide file tree

Showing 7 changed files with 82 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -82,6 +82,7 @@ deploy {
                 // FileTreeArtifact is like a FileCollectionArtifact, but the directory structure is preserved
                 myFileTreeArtifact(getArtifactTypeClass('FileTreeArtifact)) {
                     files = fileTree(dir: 'mydir')      // Required. Set the fileTree (e.g. filetree, ziptree) to deploy
+                    deleteOldFiles = false              // Set to true to delete old files in the remote deploy directory
                 }
 
                 myCommandArtifact(getArtifactTypeClass('CommandArtifact)) {

diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileTreeArtifact.java b/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileTreeArtifact.java
@@ -1,10 +1,12 @@
 package edu.wpi.first.deployutils.deploy.artifact;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
@@ -25,6 +27,7 @@ public FileTreeArtifact(String name, RemoteTarget target) {
         super(name, target);
         files = target.getProject().getObjects().property(FileTree.class);
         cacheMethod = target.getProject().getObjects().property(CacheMethod.class);
+        deleteOldFiles = target.getProject().getObjects().property(Boolean.class);
     }
 
     private final Property<FileTree> files;
@@ -33,6 +36,12 @@ public Property<FileTree> getFiles() {
         return files;
     }
 
+    private final Property<Boolean> deleteOldFiles;
+
+    public Property<Boolean> getDeleteOldFiles() {
+        return deleteOldFiles;
+    }
+
     @Override
     public Property<CacheMethod> getCacheMethod() {
         return cacheMethod;
@@ -53,6 +62,27 @@ public void deploy(DeployContext context) {
             });
 
             context.execute("mkdir -p " + String.join(" ", mkdirs));
+
+            if (deleteOldFiles.getOrElse(false)) {
+                ETLogger logger = context.getLogger();
+                if (logger != null) {
+                    logger.silent(true);
+                }
+
+                String existingFilesString = context.execute("find . -type f -print0").getResult();
+
+                Stream<String> toRemoveFiles = Arrays.stream(existingFilesString.split("\0"))
+                    .filter(x -> x.startsWith("./"))
+                    .map(x -> x.substring(2))
+                    .filter(x -> !f.containsKey(x));
+
+                if (logger != null) {
+                    logger.silent(false);
+                }
+
+                context.delete(toRemoveFiles);
+            }
+
             context.put(f, cacheMethod.getOrElse(null));
         } else {
             ETLogger logger = context.getLogger();

diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/context/DefaultDeployContext.java b/src/main/java/edu/wpi/first/deployutils/deploy/context/DefaultDeployContext.java
@@ -3,9 +3,11 @@
 import java.io.File;
 import java.io.InputStream;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
@@ -125,4 +127,16 @@ public DeployContext subContext(String workingDir) {
     public void put(InputStream source, String dest) {
         session.put(source, dest);
     }
+
+    @Override
+    public void delete(Stream<String> files) {
+        List<String> entries = files.map(x -> {
+            logger.log("  -D-> " + x + " @ " + workingDir);
+            return PathUtils.combine(workingDir, x);
+        }).collect(Collectors.toList());
+        session.delete(entries);
+        if (!entries.isEmpty()) {
+            logger.log("  " + entries.size() + " file(s) were deleted");
+        }
+    }
 }
diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/context/DeployContext.java b/src/main/java/edu/wpi/first/deployutils/deploy/context/DeployContext.java
@@ -4,6 +4,7 @@
 import java.io.InputStream;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import edu.wpi.first.deployutils.deploy.CommandDeployResult;
 import edu.wpi.first.deployutils.deploy.cache.CacheMethod;
@@ -34,6 +35,8 @@ public interface DeployContext {
     // Put an input stream, with no caching
     void put(InputStream source, String dest);
 
+    void delete(Stream<String> files);
+
     String friendlyString();
 
     DeployContext subContext(String workingDir);

diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/sessions/DrySessionController.java b/src/main/java/edu/wpi/first/deployutils/deploy/sessions/DrySessionController.java
@@ -3,6 +3,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.List;
 import java.util.Map;
 
 import javax.inject.Inject;
@@ -49,4 +50,8 @@ public int getPort() {
     @Override
     public void put(InputStream source, String dest) {
     }
+
+    @Override
+    public void delete(List<String> files) {
+    }
 }
diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SessionController.java b/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SessionController.java
@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.io.InputStream;
+import java.util.List;
 import java.util.Map;
 
 import edu.wpi.first.deployutils.deploy.CommandDeployResult;
@@ -13,5 +14,7 @@ public interface SessionController extends AutoCloseable {
 
     void put(InputStream source, String dest);
 
+    void delete(List<String> files);
+
     String friendlyString();
 }
diff --git a/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SshSessionController.java b/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SshSessionController.java
@@ -9,6 +9,7 @@
 import java.nio.file.Files;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -91,7 +92,6 @@ private CommandDeployResult executeInternal(String command) throws IOException {
         }
     }
 
-
     private void putInternal(Map<String, File> files) throws IOException {
         int sem = acquire();
 
@@ -106,6 +106,18 @@ private void putInternal(Map<String, File> files) throws IOException {
         }
     }
 
+    private void deleteInternal(List<String> files) throws IOException {
+        int sem = acquire();
+
+        try (SftpClient sftp = SftpClientFactory.instance().createSftpClient(session)) {
+            for (String file : files) {
+                sftp.remove(file);
+            }
+        } finally {
+            release(sem);
+        }
+    }
+
     @Override
     public void close() throws IOException {
         session.close();
@@ -135,6 +147,7 @@ private void putInternal(InputStream source, String dest) throws IOException {
         int sem = acquire();
 
         try (SftpClient sftp = SftpClientFactory.instance().createSftpClient(session)) {
+            sftp.remove(dest);
             try (var remoteFile = sftp.write(dest)) {
                 source.transferTo(remoteFile);
             }
@@ -169,4 +182,16 @@ public void put(InputStream source, String dest) {
             throw new RuntimeException(e);
         }
     }
+
+    @Override
+    public void delete(List<String> files) {
+        if (files.isEmpty()) {
+            return;
+        }
+        try {
+            deleteInternal(files);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }