Merge pull request #43 from tencentyun/check_local_record

config to check local record before upload
tencentyun · Sep 26, 2023 · 9dba35c · 9dba35c
2 parents 045d3c0 + c351299
commit 9dba35c
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 36 deletions.
diff --git a/conf/config.ini b/conf/config.ini
@@ -113,6 +113,12 @@ fileListMode=off
 # 当fileListMode=on的时候，fileListPath参数为localPath下待迁移文件的相对路径列表
 fileListPath=/data/config/myFileList.txt
 
+# 是否检查本地db里有记录. 如果本地db里有记录, 则不再上传此对象. 建议保持默认为 true, 避免迁移时重复上传.
+# 如果置为 false, 则不再检查本地db里是否有记录, 即使任务已经执行, 也会尝试重复上传, 甚至可能覆盖服务端数据, 一般不建议使用, 只用于极特殊场景.
+# 上传前是否检查服务端有数据，由开关 skipSamePath 控制. 执行顺序是先判断 checkLocalRecord, 再判断 skipSamePath, 其中任何一个判断结果跳过，再跳过不执行上传
+# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
+checkLocalRecord=true
+
 ## 从阿里迁移到COS的配置分节
 [migrateAli]
 bucket=mybucket-test

diff --git a/...ate_tool-1.4.12-jar-with-dependencies.jar → ...ate_tool-1.4.13-jar-with-dependencies.jar b/...ate_tool-1.4.12-jar-with-dependencies.jar → ...ate_tool-1.4.13-jar-with-dependencies.jar
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.qcloud</groupId>
 	<artifactId>cos_migrate_tool</artifactId>
-	<version>1.4.12</version>
+	<version>1.4.13</version>
 	<packaging>jar</packaging>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java b/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java
@@ -140,7 +140,7 @@ public void setResume(String isResume) {
         if (isResume.compareToIgnoreCase("true") == 0) {
             this.isResume = true;
         } else if (isResume.compareToIgnoreCase("false") != 0) {
-            throw new IllegalArgumentException("resume invalid.should be true or false");
+            throw new IllegalArgumentException(isResume + " is invalid, isResume should be true or false");
         }
     }
 
@@ -154,7 +154,7 @@ public void setRealTimeCompare(String realTimeCompare)  throws IllegalArgumentEx
         } else if (realTimeCompare.equalsIgnoreCase("off")) {
             this.realTimeCompare = false;
         } else {
-            throw new IllegalArgumentException("invalid realTimeCompare config. only support on/off");
+            throw new IllegalArgumentException(realTimeCompare + " is invalid, realTimeCompare only support on/off");
         }
     }
 
@@ -272,14 +272,14 @@ public String getCosPath() {
 
     public void setCosPath(String cosPath) throws IllegalArgumentException {
         if (!cosPath.startsWith("/")) {
-            throw new IllegalArgumentException("cospath must start with /");
+            throw new IllegalArgumentException(cosPath + " is invalid, cospath must start with /");
         }
         this.cosPath = PathUtils.formatCosFolderPath(cosPath);
     }
 
     public void setDbCosPath(String dbCosPath) throws IllegalArgumentException {
         if (!dbCosPath.startsWith("/")) {
-            throw new IllegalArgumentException("dbcospath must start with /");
+            throw new IllegalArgumentException(dbCosPath + " is invalid, dbcospath must start with /");
         }
         this.dbCosPath = PathUtils.formatCosFolderPath(dbCosPath);
     }
@@ -302,7 +302,7 @@ public void setEnableHttps(String enableHttpsStr) throws IllegalArgumentExceptio
         } else if (enableHttpsStr.equalsIgnoreCase("off")) {
             this.enableHttps = false;
         } else {
-            throw new IllegalArgumentException("invalid https config. only support on/off");
+            throw new IllegalArgumentException(enableHttpsStr + " is invalid, https config only support on/off");
         }
     }
 
@@ -312,7 +312,7 @@ public void setTaskExecutorNumberStr(String taskExecutorNumberStr)
         try {
             int number = Integer.valueOf(taskExecutorNumberStr);
             if (number <= 0) {
-                throw new IllegalArgumentException("taskExecutorNumber must be greater than 0");
+                throw new IllegalArgumentException(taskExecutorNumberStr + " is invalid, taskExecutorNumber must be greater than 0");
             }
             this.taskExecutorNumber = number;
         } catch (NumberFormatException e) {
@@ -347,7 +347,7 @@ public void setSmallFileUploadExecutorNum(String smallFileExecutorNumStr)
         try {
             int number = Integer.valueOf(smallFileExecutorNumStr);
             if (number <= 0 || number > 1024) {
-                throw new IllegalArgumentException("legal smallFileExecutorNum  is [1, 1024] ");
+                throw new IllegalArgumentException(smallFileExecutorNumStr + " is invalid, legal smallFileExecutorNum  is [1, 1024] ");
             }
             this.smallFileExecutorNumber = number;
             this.taskExecutorNumber = this.smallFileExecutorNumber + this.bigFileExecutorNum;
@@ -365,7 +365,7 @@ public void setBigFileUploadExecutorNum(String bigFileExecutorNumStr) {
         try {
             int number = Integer.valueOf(bigFileExecutorNumStr);
             if (number <= 0 || number > 64) {
-                throw new IllegalArgumentException("legal bigFileExecutorNum is [1, 64] ");
+                throw new IllegalArgumentException(bigFileExecutorNumStr + " is invalid, legal bigFileExecutorNum is [1, 64] ");
             }
             this.bigFileExecutorNum = number;
             this.taskExecutorNumber = this.smallFileExecutorNumber + this.bigFileExecutorNum;
@@ -382,7 +382,8 @@ public void setSmallFileThreshold(String smallFileThresholdStr) {
             final long maxSmallFile = 5L * 1024 * 1024 * 1024; // 最大5GB
             if (number < minSmallFile || number > maxSmallFile) {
                 throw new IllegalArgumentException(String.format(
-                        "legal smallFileThreshold is [%d, %d], 5MB ~ 5GB", minSmallFile, maxSmallFile));
+                        "invalid smallFileThresholdStr: %s, legal smallFileThreshold is [%d, %d], 5MB ~ 5GB"
+                        , smallFileThresholdStr, minSmallFile, maxSmallFile));
             }
             this.smallFileThreshold = number;
         } catch (NumberFormatException e) {
@@ -404,8 +405,7 @@ public void setEntireFileMd5Attached(String entireFileMd5AttachedStr) {
         } else if (entireFileMd5AttachedStr.equalsIgnoreCase("off")) {
             this.entireFileMd5Attached = false;
         } else {
-            throw new IllegalArgumentException(
-                    "invalid entireFileMd5Attached config. only support on/off");
+            throw new IllegalArgumentException(entireFileMd5AttachedStr + " is invalid, entireFileMd5Attached only support on/off");
         }
     }
 
@@ -415,17 +415,16 @@ public void setDaemonMode(String daemonModeStr) {
         } else if (daemonModeStr.equalsIgnoreCase("off")) {
             this.damonMode = false;
         } else {
-            throw new IllegalArgumentException(
-                    "invalid daemonMode config. only support on/off");
+            throw new IllegalArgumentException(daemonModeStr + " is invalid, daemonMode only support on/off");
         }
     }
 
-    public void setDaemonModeInterVal(String daemonModeStr) {
-        daemonModeStr = daemonModeStr.trim();
+    public void setDaemonModeInterVal(String daemonModeInvatervalStr) {
+        daemonModeInvatervalStr = daemonModeInvatervalStr.trim();
         try {
-            int number = Integer.valueOf(daemonModeStr);
+            int number = Integer.valueOf(daemonModeInvatervalStr);
             if (number <= 0) {
-                throw new IllegalArgumentException("damonInterVal must be greater than 0");
+                throw new IllegalArgumentException(daemonModeInvatervalStr + " is invalid, damonInterVal must be greater than 0");
             }
             this.damonInterVal = number;
         } catch (NumberFormatException e) {
@@ -445,34 +444,34 @@ public void setTimeWindowsStr(String timeWindowStr) {
         timeWindowStr = timeWindowStr.trim();
         String[] timeWindowArray = timeWindowStr.split(",");
         if (timeWindowArray.length != 2) {
-            throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+            throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
         }
         try {
             String[] timeBeginMemberArray = timeWindowArray[0].split(":");
             if (timeBeginMemberArray.length != 2) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             int hour = Integer.valueOf(timeBeginMemberArray[0]);
             if (hour < 0 || hour >= 24) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             int minute = Integer.valueOf(timeBeginMemberArray[1]);
             if (minute < 0 || minute >= 60) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             this.timeWindowBegin = hour * 60 + minute;
 
             String[] timeEndMemberArray = timeWindowArray[1].split(":");
             if (timeEndMemberArray.length != 2) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             hour = Integer.valueOf(timeEndMemberArray[0]);
             if (hour < 0 || hour > 24) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             minute = Integer.valueOf(timeEndMemberArray[1]);
             if (minute < 0 || minute >= 60) {
-                throw new IllegalArgumentException("executeTimeWindow is invalid, the legal example 03:30,21:00");
+                throw new IllegalArgumentException(timeWindowStr + " is invalid, the legal executeTimeWindow example 03:30,21:00");
             }
             this.timeWindowEnd = hour * 60 + minute;
 
@@ -543,7 +542,7 @@ public void setShortConnection(String shortConnection) throws IllegalArgumentExc
         } else if (shortConnection.equalsIgnoreCase("false")) {
             isShortConnection = false;
         } else {
-            throw new IllegalArgumentException("invalid short connection config. only support true/false");
+            throw new IllegalArgumentException(shortConnection + " is invalid, short connection config. only support true/false");
         }
     }
 
@@ -557,7 +556,7 @@ public void setRequestTimeoutEnable(String strRequestTimeoutEnable) throws Illeg
         } else if (strRequestTimeoutEnable.equalsIgnoreCase("off")) {
             requestTimeoutEnable = false;
         } else {
-            throw new IllegalArgumentException("invalid request timeout enable config. only support on/off");
+            throw new IllegalArgumentException(strRequestTimeoutEnable + " is invalid, request timeout enable config. only support on/off");
         }
     }
 
@@ -570,7 +569,7 @@ public void setRequestTimeoutMS(String strRequestTimeoutMS) throws IllegalArgume
         try {
             int number = Integer.valueOf(strRequestTimeoutMS);
             if (number <= 0) {
-                throw new IllegalArgumentException("legal requestTimeoutMS  must be gt 0");
+                throw new IllegalArgumentException(strRequestTimeoutMS + " is invalid, legal requestTimeoutMS  must be gt 0");
             }
             requestTimeoutMS = number;
         } catch (NumberFormatException e) {
@@ -587,11 +586,31 @@ public void setRequestTryCount(String strRequestTryCount) throws IllegalArgument
         try {
             int number = Integer.valueOf(strRequestTryCount);
             if (number <= 0) {
-                throw new IllegalArgumentException("legal requestTryCount must be gt 0");
+                throw new IllegalArgumentException(strRequestTryCount + " is invalid, legal requestTryCount must be gt 0");
             }
             requestTryCount = number;
         } catch (NumberFormatException e) {
             throw new IllegalArgumentException("invalid requestTryCount");
         }
     }
+
+    /**
+     * 打印关键参数
+     */
+    public String toString() {
+        return "CommonConfig:"
+                + "cosPath=" + cosPath
+                + ",https=" + isEnableHttps()
+                + ",shortConnection=" + isShortConnection()
+                + ",smallFileThreshold=" + getSmallFileThreshold()
+                + ",smallFileExecutorNum=" + getSmallFileExecutorNumber()
+                + ",bigFileExecutorNum=" + getBigFileExecutorNum()
+                + ",bigFileUploadPartSize=" + getBigFileUploadPartSize()
+                + ",threadTrafficLimit=" + getThreadTrafficLimit()
+                + ",executeTimeWindow=" + getTimeWindowBegin() + "," + getTimeWindowEnd()
+                + ",resume=" + isResume()
+                + ",skipSamePath=" + skipSamePath()
+                + ",requestTryCount=" + getRequestTryCount()
+                ;
+    }
 }
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java b/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java
@@ -74,6 +74,7 @@ public class ConfigParser {
     private static final String IGNORE_EMPTY_FILE = "ignoreEmptyFile";
     private static final String FILE_LIST_MODE = "fileListMode";
     private static final String FILE_LIST_PATH = "fileListPath";
+    private static final String CHECK_LOCAL_RECORD = "checkLocalRecord";
 
     private static final String ALI_SECTION_NAME = "migrateAli";
     private static final String AWS_SECTION_NAME = "migrateAws";
@@ -694,13 +695,18 @@ private boolean initCopyFromLocalConfig(Preferences prefs,
                 copyLocalConfig.setIgnoreEmptyFile(true);
             }
             String fileListMode = getConfigValue(prefs, LOCAL_SECTION_NAME, FILE_LIST_MODE);
-            if(fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
+            if (fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
                 copyLocalConfig.setFileListMode(true);
             }
             String fileListPath = getConfigValue(prefs, LOCAL_SECTION_NAME, FILE_LIST_PATH);
-            if(fileListPath != null) {
+            if (fileListPath != null) {
                 copyLocalConfig.setFileListPath(fileListPath);
             }
+
+            String strCheckLocal = getConfigValue(prefs, LOCAL_SECTION_NAME, CHECK_LOCAL_RECORD);
+            if (strCheckLocal != null) {
+                copyLocalConfig.setCheckLocalRecord(strCheckLocal);
+            }
         } catch (Exception e) {
             System.err.println(e.getMessage());
             log.error(e.getMessage());

diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java b/src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java
@@ -16,6 +16,7 @@ public class CopyFromLocalConfig extends CommonConfig {
     boolean ignoreEmptyFile = false;
     boolean fileListMode = false;
     private String fileListPath;
+    private boolean isCheckLocalRecord = true;
 
     public void setIgnoreEmptyFile(boolean ignoreEmptyFile) {
         this.ignoreEmptyFile = ignoreEmptyFile;
@@ -100,7 +101,7 @@ public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr)
         try {
             long number = Long.valueOf(ignoreModifiedTimeLessThanStr);
             if (number <= 0) {
-                throw new IllegalArgumentException("legal ignoreModifiedTimeLessThan must be positive");
+                throw new IllegalArgumentException(ignoreModifiedTimeLessThanStr + " is invalid, ignoreModifiedTimeLessThan must be positive");
             }
             this.ignoreModifiedTimeLessThan = number;
         } catch (NumberFormatException e) {
@@ -126,4 +127,62 @@ public String getFileListPath() {
     public void setFileListPath(String fileListPath) {
         this.fileListPath = fileListPath;
     }
+
+    public void setCheckLocalRecord(String checkLocalRecord) {
+        if (checkLocalRecord.compareToIgnoreCase("true") == 0) {
+            isCheckLocalRecord = true;
+        } else if (checkLocalRecord.compareToIgnoreCase("false") == 0) {
+            isCheckLocalRecord = false;
+        } else {
+            throw new IllegalArgumentException(checkLocalRecord + " is invalid, checkLocalRecord should be true or false");
+        }
+    }
+
+    public boolean checkLocalRecord() {
+        return isCheckLocalRecord;
+    }
+
+
+    public String toString() {
+        String strExclude = "";
+        if (!excludes.isEmpty()) {
+            StringBuilder sb = new StringBuilder();
+            for (String element : excludes) {
+                sb.append(element).append(",");
+            }
+            strExclude = sb.toString();
+        }
+
+
+        String strIgnoreSuffix = "";
+        if (!ignoreSuffixs.isEmpty()) {
+            StringBuilder sb = new StringBuilder();
+            for (String element : ignoreSuffixs) {
+                sb.append(element).append(",");
+            }
+            strIgnoreSuffix = sb.toString();
+        }
+
+        String strIncludeSuffix = "";
+        if (!includeSuffixs.isEmpty()) {
+            StringBuilder sb = new StringBuilder();
+            for (String element : includeSuffixs) {
+                sb.append(element).append(",");
+            }
+            strIncludeSuffix = sb.toString();
+        }
+
+        return super.toString()
+                + ". CopyFromLocalConfig:"
+                + "localPath=" + getLocalPath()
+                + ",excludes=" + strExclude
+                + ",ignoreModifiedTimeLessThanSeconds=" + getIgnoreModifiedTimeLessThan()
+                + ",ignoreSuffix=" + strIgnoreSuffix
+                + ",ignoreEmptyFile=" + ignoreEmptyFile
+                + ",includeSuffix=" + strIncludeSuffix
+                + ",fileListMode=" + isFileListMode()
+                + ",fileListPath=" + getFileListPath()
+                + ",checkLocalRecord=" + checkLocalRecord()
+                ;
+    }
 }
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateLocalTask.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateLocalTask.java
@@ -29,6 +29,7 @@ public class MigrateLocalTask extends Task {
     private File localFile;
     private StorageClass storageClass;
     private boolean entireMd5Attached;
+    private boolean isCheckLocalRecord;
 
 
     public MigrateLocalTask(Semaphore semaphore, CopyFromLocalConfig copyFromLocalConfig,
@@ -41,6 +42,7 @@ public MigrateLocalTask(Semaphore semaphore, CopyFromLocalConfig copyFromLocalCo
         this.localFile = localFile;
         this.storageClass = copyFromLocalConfig.getStorageClass();
         this.entireMd5Attached = copyFromLocalConfig.isEntireFileMd5Attached();
+        isCheckLocalRecord = copyFromLocalConfig.checkLocalRecord();
     }
 
     private String buildCOSPath(String localPath) {
@@ -72,10 +74,13 @@ public void doTask() {
 
         MigrateLocalRecordElement migrateLocalRecordElement =
                 new MigrateLocalRecordElement(bucketName, localPath, cosPath, mtime, fileSize);
-        // 如果记录存在
-        if (isExist(migrateLocalRecordElement, true)) {
-            TaskStatics.instance.addSkipCnt();
-            return;
+
+        if (isCheckLocalRecord) {
+            // 如果记录存在
+            if (isExist(migrateLocalRecordElement, true)) {
+                TaskStatics.instance.addSkipCnt();
+                return;
+            }
         }
 
         if (config.skipSamePath()) {