Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support upload file by cosn fs #45

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

support upload file by cosn fs #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions conf/config.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,37 @@ fileListPath=/data/config/myFileList.txt
# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
checkLocalRecord=true

# 从本地迁移到COS配置并且使用cosn文件系统分节
[migrateLocalToCosnFs]
# 本地目录, 表示将该路径下的数据都迁移到COS, 对于linux绝对路径, 如/a/b/c, 对于windows绝对路径，注意分隔符为两个反斜杠，如E:\\a\\b\\c
localPath=E:\\code\\java\\workspace\\cos_migrate_tool\\test_data
# 要排除的目录或者文件的绝对路径, 表示将localPath下面某些目录或者文件不进行迁移，多个绝对路径之前用分号分割，不填表示localpath下面的全部迁移
excludes=
# 排除更新时间与当前时间相差不足一定时间段的文件，单位为秒
# 默认不设置, 表示不根据lastmodified时间进行筛选
# 适用于客户在更新文件的同时又在运行迁移工具, 不准备把正在更新的文件迁移上传到COS, 比如设置为300, 表示只上传更新了5分钟以上的文件
ignoreModifiedTimeLessThanSeconds=
# 多个后缀用;隔开，例如:.txt;.tmp;
ignoreSuffix=
# on：忽略空文件，off：不忽略。默认不忽略
ignoreEmptyFile=

# 设定要迁移的文件后缀，不符合后缀的文件会被排除
# 多个后缀用;隔开，例如:.txt;.tmp;
includeSuffix=
# on: 使用列表文件指定所有待迁移文件的相对路径; off: migration递归遍历localPath, 添加待迁移文件; 默认off
# 可使用excludes, ignoreModifiedTimeLessThanSeconds, ignoreSuffix, ignoreEmptyFile忽略列表中的文件
fileListMode=off
# 当fileListMode=on的时候，fileListPath参数为localPath下待迁移文件的相对路径列表
fileListPath=/data/config/myFileList.txt

# 是否检查本地db里有记录. 如果本地db里有记录, 则不再上传此对象. 建议保持默认为 true, 避免迁移时重复上传.
# 如果置为 false, 则不再检查本地db里是否有记录, 即使任务已经执行, 也会尝试重复上传, 甚至可能覆盖服务端数据, 一般不建议使用, 只用于极特殊场景.
# 上传前是否检查服务端有数据，由开关 skipSamePath 控制. 执行顺序是先判断 checkLocalRecord, 再判断 skipSamePath, 其中任何一个判断结果跳过，再跳过不执行上传
# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
checkLocalRecord=true


## 从阿里迁移到COS的配置分节
[migrateAli]
bucket=mybucket-test
Expand Down
Binary file renamed ...ate_tool-1.4.13-jar-with-dependencies.jar → ...e_tool-1.4.13.1-jar-with-dependencies.jar
View file
Open in desktop
Binary file not shown.
27 changes: 26 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.qcloud</groupId>
<artifactId>cos_migrate_tool</artifactId>
<version>1.4.13</version>
<version>1.4.13.1</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand Down Expand Up @@ -96,6 +96,31 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.qcloud.cos</groupId>
<artifactId>hadoop-cos</artifactId>
<version>8.2.7</version>
</dependency>

<dependency>
<groupId>com.qcloud</groupId>
<artifactId>chdfs_hadoop_plugin_network</artifactId>
<version>2.8</version>
</dependency>

<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.112</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>

</dependencies>

<build>
Expand Down
16 changes: 4 additions & 12 deletions src/main/java/com/qcloud/cos_migrate_tool/app/App.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,7 @@
import java.io.InputStreamReader;

import com.qcloud.cos.internal.SkipMd5CheckStrategy;
import com.qcloud.cos_migrate_tool.config.CommonConfig;
import com.qcloud.cos_migrate_tool.config.ConfigParser;
import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromUpyunConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
import com.qcloud.cos_migrate_tool.config.MigrateType;
import com.qcloud.cos_migrate_tool.config.*;
import com.qcloud.cos_migrate_tool.meta.TaskStatics;
import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
Expand All @@ -31,6 +20,7 @@
import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
import com.qcloud.cos_migrate_tool.task.TaskExecutor;

import com.qcloud.cos_migrate_tool.hadoop_fs_task.MigrateLocalToCosnTaskExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -57,6 +47,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_UPYUN)) {
return new MigrateUpyunTaskExecutor((CopyFromUpyunConfig) config);
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)) {
return new MigrateLocalToCosnTaskExecutor((CopyFromLocalToCosnConfig) config);
} else {
System.out.println("unknown migrate type");
}
Expand Down
80 changes: 80 additions & 0 deletions src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ public class ConfigParser {
private static final String FILE_LIST_PATH = "fileListPath";
private static final String CHECK_LOCAL_RECORD = "checkLocalRecord";

private static final String LOCAL_TO_COSN_FS_SECTION_NAME = "migrateLocalToCosnFs";

private static final String ALI_SECTION_NAME = "migrateAli";
private static final String AWS_SECTION_NAME = "migrateAws";
private static final String QINIU_SECTION_NAME = "migrateQiniu";
Expand Down Expand Up @@ -303,6 +305,14 @@ public boolean parse() {
if (!initCopyFromUpyunConfig(prefs, (CopyFromUpyunConfig) config)) {
return false;
}
} else if (migrateType.equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)){
if (!checkMigrateLocalToCosnFsConfig(prefs)) {
return false;
}
config = new CopyFromLocalToCosnConfig();
if (!initCopyFromLocalToCosnConfig(prefs, (CopyFromLocalToCosnConfig) config)) {
return false;
}
}


Expand Down Expand Up @@ -402,6 +412,13 @@ private boolean checkMigrateLocalConfig(Preferences prefs) {
return true;
}

private boolean checkMigrateLocalToCosnFsConfig(Preferences prefs) {
if (!isKeyExist(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH)) {
return false;
}
return true;
}

private boolean checkMigrateCopyBucketConfig(Preferences prefs) {
if (!isKeyExist(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_REGION)) {
return false;
Expand Down Expand Up @@ -715,6 +732,69 @@ private boolean initCopyFromLocalConfig(Preferences prefs,
return true;
}

private boolean initCopyFromLocalToCosnConfig(Preferences prefs,
CopyFromLocalToCosnConfig copyLocalConfig) {
if (!initCommonConfig(prefs, copyLocalConfig)) {
return false;
}
try {

String localPathConfig = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH);
assert (localPathConfig != null);
copyLocalConfig.setLocalPath(localPathConfig);

String excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_EXECLUDE);
if (excludes != null && !excludes.trim().isEmpty()) {
copyLocalConfig.setExcludes(excludes);
} else {
excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, "excludes");
if (excludes != null && !excludes.trim().isEmpty()) {
copyLocalConfig.setExcludes(excludes);
}
}

String ignoreModifiedTimeLessThanStr =
getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_MODIFIED_TIME_LESS_THAN);
if (ignoreModifiedTimeLessThanStr != null
&& !ignoreModifiedTimeLessThanStr.trim().isEmpty()) {
copyLocalConfig.setIgnoreModifiedTimeLessThan(ignoreModifiedTimeLessThanStr);
}

String ignoreSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_SUFFIX);
if (ignoreSuffix != null && !ignoreSuffix.trim().isEmpty()) {
copyLocalConfig.setIgnoreSuffix(ignoreSuffix);
}

String includeSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, INCLUDE_SUFFIX);
if (includeSuffix != null && !includeSuffix.trim().isEmpty()) {
copyLocalConfig.setIncludeSuffix(includeSuffix);
}

String ignoreEmptyFile = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_EMPTY_FILE);
if (ignoreEmptyFile != null && (ignoreEmptyFile.compareToIgnoreCase("on") == 0)) {
copyLocalConfig.setIgnoreEmptyFile(true);
}
String fileListMode = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_MODE);
if (fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
copyLocalConfig.setFileListMode(true);
}
String fileListPath = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_PATH);
if (fileListPath != null) {
copyLocalConfig.setFileListPath(fileListPath);
}

String strCheckLocal = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, CHECK_LOCAL_RECORD);
if (strCheckLocal != null) {
copyLocalConfig.setCheckLocalRecord(strCheckLocal);
}
} catch (Exception e) {
System.err.println(e.getMessage());
log.error(e.getMessage());
return false;
}
return true;
}

private boolean initCopyFromUrllistConfig(Preferences prefs,
CopyFromUrllistConfig copyUrllistConfig) {
if (!initCommonConfig(prefs, copyUrllistConfig)) {
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr)
public long getIgnoreModifiedTimeLessThan() {
return ignoreModifiedTimeLessThan;
}

public boolean isFileListMode() {
return fileListMode;
}
Expand Down
Loading