Hadoop: fix guid refresh by avoid use closed fs (#4180)

juicedata · Nov 22, 2023 · e7f9eb2 · e7f9eb2
1 parent 9ed1432
commit e7f9eb2
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 7 deletions.
diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
@@ -455,7 +455,7 @@ private boolean isEmpty(String str) {
     return str == null || str.trim().isEmpty();
   }
 
-  private String readFile(String file) {
+  private String readFile(String file) throws IOException {
     Path path = new Path(file);
     URI uri = path.toUri();
     FileSystem fs;
@@ -488,11 +488,11 @@ private String readFile(String file) {
       return res;
     } catch (IOException e) {
       LOG.warn(String.format("read %s failed", file), e);
-      return null;
+      throw e;
     }
   }
 
-  private void updateUidAndGrouping(String uidFile, String groupFile) {
+  private void updateUidAndGrouping(String uidFile, String groupFile) throws IOException {
     String uidstr = null;
     if (uidFile != null && !"".equals(uidFile.trim())) {
       uidstr = readFile(uidFile);

diff --git a/sdk/java/src/main/java/io/juicefs/utils/BgTaskUtil.java b/sdk/java/src/main/java/io/juicefs/utils/BgTaskUtil.java
@@ -41,18 +41,27 @@ private BgTaskUtil() {
   // use timer to run trash emptier because it will occupy a thread
   private static final List<Timer> timers = new ArrayList<>();
   private static final List<FileSystem> fileSystems = new ArrayList<>();
-  private static Set<String> runningBgTask = new HashSet<>();
+  private static final Set<String> runningBgTask = new HashSet<>();
 
-  public static void startScheduleTask(String name, String type, Runnable task, long initialDelay, long period, TimeUnit unit) {
+  public interface Task {
+    void run() throws Exception;
+  }
+
+  public static void startScheduleTask(String name, String type, Task task, long initialDelay, long period, TimeUnit unit) {
     synchronized (runningBgTask) {
       if (isRunning(name, type)) {
         return;
       }
       threadPool.scheduleAtFixedRate(() -> {
         try {
+          LOG.debug("Background task started for {} {}", name, type);
           task.run();
         } catch (Exception e) {
-          LOG.error("Background task failed", e);
+          LOG.warn("Background task failed for {} {}", name, type, e);
+          synchronized (runningBgTask) {
+            runningBgTask.remove(genKey(name, type));
+          }
+          throw new RuntimeException(e);
         }
       }, initialDelay, period, unit);
       runningBgTask.add(genKey(name, type));
@@ -65,7 +74,7 @@ public static void startTrashEmptier(String name, String type, FileSystem fs, Ru
       if (isRunning(name, type)) {
         return;
       }
-      Timer timer = new Timer(true);
+      Timer timer = new Timer("trash emptier", true);
       timer.schedule(new TimerTask() {
         @Override
         public void run() {