support upload file by cosn fs

conf/config.ini

@@ -119,6 +119,37 @@ fileListPath=/data/config/myFileList.txt
 # 这个开关不会控制上传后是否记录数据库, 一定会记录的.
 checkLocalRecord=true
 
+# 从本地迁移到COS配置并且使用cosn文件系统分节
+[migrateLocalToCosnFs]
+# 本地目录, 表示将该路径下的数据都迁移到COS, 对于linux绝对路径, 如/a/b/c, 对于windows绝对路径，注意分隔符为两个反斜杠，如E:\\a\\b\\c
+localPath=E:\\code\\java\\workspace\\cos_migrate_tool\\test_data
+# 要排除的目录或者文件的绝对路径, 表示将localPath下面某些目录或者文件不进行迁移，多个绝对路径之前用分号分割，不填表示localpath下面的全部迁移
+excludes=
+# 排除更新时间与当前时间相差不足一定时间段的文件，单位为秒
+# 默认不设置, 表示不根据lastmodified时间进行筛选
+# 适用于客户在更新文件的同时又在运行迁移工具, 不准备把正在更新的文件迁移上传到COS, 比如设置为300, 表示只上传更新了5分钟以上的文件
+ignoreModifiedTimeLessThanSeconds=
+# 多个后缀用;隔开，例如:.txt;.tmp;
+ignoreSuffix=
+# on：忽略空文件，off：不忽略。默认不忽略
+ignoreEmptyFile=
+
+# 设定要迁移的文件后缀，不符合后缀的文件会被排除
+# 多个后缀用;隔开，例如:.txt;.tmp;
+includeSuffix=
+# on: 使用列表文件指定所有待迁移文件的相对路径; off: migration递归遍历localPath, 添加待迁移文件; 默认off
+# 可使用excludes, ignoreModifiedTimeLessThanSeconds, ignoreSuffix, ignoreEmptyFile忽略列表中的文件
+fileListMode=off
+# 当fileListMode=on的时候，fileListPath参数为localPath下待迁移文件的相对路径列表
+fileListPath=/data/config/myFileList.txt
+
+# 是否检查本地db里有记录. 如果本地db里有记录, 则不再上传此对象. 建议保持默认为 true, 避免迁移时重复上传.
+# 如果置为 false, 则不再检查本地db里是否有记录, 即使任务已经执行, 也会尝试重复上传, 甚至可能覆盖服务端数据, 一般不建议使用, 只用于极特殊场景.
+# 上传前是否检查服务端有数据，由开关 skipSamePath 控制. 执行顺序是先判断 checkLocalRecord, 再判断 skipSamePath, 其中任何一个判断结果跳过，再跳过不执行上传
+# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
+checkLocalRecord=true
+
+
 ## 从阿里迁移到COS的配置分节
 [migrateAli]
 bucket=mybucket-test

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.qcloud</groupId>
 	<artifactId>cos_migrate_tool</artifactId>
-	<version>1.4.13</version>
+	<version>1.4.13.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -96,6 +96,31 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+
+		<dependency>
+			<groupId>com.qcloud.cos</groupId>
+			<artifactId>hadoop-cos</artifactId>
+			<version>8.2.7</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.qcloud</groupId>
+			<artifactId>chdfs_hadoop_plugin_network</artifactId>
+			<version>2.8</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.qcloud</groupId>
+			<artifactId>cos_api-bundle</artifactId>
+			<version>5.6.112</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+			<version>2.8.5</version>
+		</dependency>
+
 	</dependencies>
 
 	<build>

src/main/java/com/qcloud/cos_migrate_tool/app/App.java

@@ -7,18 +7,7 @@
 import java.io.InputStreamReader;
 
 import com.qcloud.cos.internal.SkipMd5CheckStrategy;
-import com.qcloud.cos_migrate_tool.config.CommonConfig;
-import com.qcloud.cos_migrate_tool.config.ConfigParser;
-import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromUpyunConfig;
-import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
-import com.qcloud.cos_migrate_tool.config.MigrateType;
+import com.qcloud.cos_migrate_tool.config.*;
 import com.qcloud.cos_migrate_tool.meta.TaskStatics;
 import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
@@ -31,6 +20,7 @@
 import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.TaskExecutor;
 
+import com.qcloud.cos_migrate_tool.task_by_hadoop_fs.MigrateLocalToCosnTaskExecutor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,6 +47,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
             return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
         } else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_UPYUN)) {
             return new MigrateUpyunTaskExecutor((CopyFromUpyunConfig) config);
+        } else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)) {
+            return new MigrateLocalToCosnTaskExecutor((CopyFromLocalToCosnConfig) config);
         } else {
             System.out.println("unknown migrate type");
         }

src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java

@@ -76,6 +76,8 @@ public class ConfigParser {
     private static final String FILE_LIST_PATH = "fileListPath";
     private static final String CHECK_LOCAL_RECORD = "checkLocalRecord";
 
+    private static final String LOCAL_TO_COSN_FS_SECTION_NAME = "migrateLocalToCosnFs";
+
     private static final String ALI_SECTION_NAME = "migrateAli";
     private static final String AWS_SECTION_NAME = "migrateAws";
     private static final String QINIU_SECTION_NAME = "migrateQiniu";
@@ -303,6 +305,14 @@ public boolean parse() {
             if (!initCopyFromUpyunConfig(prefs, (CopyFromUpyunConfig) config)) {
                 return false;
             }
+        } else if (migrateType.equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)){
+            if (!checkMigrateLocalToCosnFsConfig(prefs)) {
+                return false;
+            }
+            config = new CopyFromLocalToCosnConfig();
+            if (!initCopyFromLocalToCosnConfig(prefs, (CopyFromLocalToCosnConfig) config)) {
+                return false;
+            }
         }
 
 
@@ -402,6 +412,13 @@ private boolean checkMigrateLocalConfig(Preferences prefs) {
         return true;
     }
 
+    private boolean checkMigrateLocalToCosnFsConfig(Preferences prefs) {
+        if (!isKeyExist(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH)) {
+            return false;
+        }
+        return true;
+    }
+
     private boolean checkMigrateCopyBucketConfig(Preferences prefs) {
         if (!isKeyExist(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_REGION)) {
             return false;
@@ -715,6 +732,69 @@ private boolean initCopyFromLocalConfig(Preferences prefs,
         return true;
     }
 
+    private boolean initCopyFromLocalToCosnConfig(Preferences prefs,
+                                            CopyFromLocalToCosnConfig copyLocalConfig) {
+        if (!initCommonConfig(prefs, copyLocalConfig)) {
+            return false;
+        }
+        try {
+
+            String localPathConfig = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH);
+            assert (localPathConfig != null);
+            copyLocalConfig.setLocalPath(localPathConfig);
+
+            String excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_EXECLUDE);
+            if (excludes != null && !excludes.trim().isEmpty()) {
+                copyLocalConfig.setExcludes(excludes);
+            } else {
+                excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, "excludes");
+                if (excludes != null && !excludes.trim().isEmpty()) {
+                    copyLocalConfig.setExcludes(excludes);
+                }
+            }
+
+            String ignoreModifiedTimeLessThanStr =
+                    getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_MODIFIED_TIME_LESS_THAN);
+            if (ignoreModifiedTimeLessThanStr != null
+                    && !ignoreModifiedTimeLessThanStr.trim().isEmpty()) {
+                copyLocalConfig.setIgnoreModifiedTimeLessThan(ignoreModifiedTimeLessThanStr);
+            }
+
+            String ignoreSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_SUFFIX);
+            if (ignoreSuffix != null && !ignoreSuffix.trim().isEmpty()) {
+                copyLocalConfig.setIgnoreSuffix(ignoreSuffix);
+            }
+
+            String includeSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, INCLUDE_SUFFIX);
+            if (includeSuffix != null && !includeSuffix.trim().isEmpty()) {
+                copyLocalConfig.setIncludeSuffix(includeSuffix);
+            }
+
+            String ignoreEmptyFile =  getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_EMPTY_FILE);
+            if (ignoreEmptyFile != null && (ignoreEmptyFile.compareToIgnoreCase("on") == 0)) {
+                copyLocalConfig.setIgnoreEmptyFile(true);
+            }
+            String fileListMode = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_MODE);
+            if (fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
+                copyLocalConfig.setFileListMode(true);
+            }
+            String fileListPath = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_PATH);
+            if (fileListPath != null) {
+                copyLocalConfig.setFileListPath(fileListPath);
+            }
+
+            String strCheckLocal = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, CHECK_LOCAL_RECORD);
+            if (strCheckLocal != null) {
+                copyLocalConfig.setCheckLocalRecord(strCheckLocal);
+            }
+        } catch (Exception e) {
+            System.err.println(e.getMessage());
+            log.error(e.getMessage());
+            return false;
+        }
+        return true;
+    }
+
     private boolean initCopyFromUrllistConfig(Preferences prefs,
             CopyFromUrllistConfig copyUrllistConfig) {
         if (!initCommonConfig(prefs, copyUrllistConfig)) {

src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java

@@ -112,6 +112,7 @@ public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr)
     public long getIgnoreModifiedTimeLessThan() {
         return ignoreModifiedTimeLessThan;
     }
+
     public boolean isFileListMode() {
         return fileListMode;
     }