diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..15107a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.idea/ +.project +db/* +log/* +*.iml +target/ \ No newline at end of file diff --git a/conf/config.ini b/conf/config.ini index 455dee5..5602b1f 100644 --- a/conf/config.ini +++ b/conf/config.ini @@ -13,6 +13,8 @@ type=migrateLocal secretId=AKIDXXXXXXXXXXXXXXXXXXXXXXX # 用户的秘钥 secret_key (可在 https://console.qcloud.com/capi 查看) secretKey=yyyyyyyyyyyyyyyyyyyyyyyyyyy +# 如果使用临时密钥访问存储桶，此处配置临时密钥的Token,该密钥需要有目的桶的PutObject权限(如果迁移类型是migrateBucketCopy，则该密钥需要有源桶的GetObject权限以及目的桶的PutObject权限) +# token=zzzzzzzzzzzzzzzzz # 目的Bucket的名称, 命名规则为{name}-{appid}，即bucket名必须包含appid, 例如movie-1251000000 bucketName=mybucket-1251668577 # 目的bucket的region信息. COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224 @@ -99,8 +101,6 @@ srcHttps=off # 访问ali时, 读取ali响应的超时时间，单位ms, 默认10000 #srcSocketTimeout=10000 - - # 从七牛迁移到COS [migrateQiniu] # 从七牛迁移到COS的配置分节 @@ -165,8 +165,15 @@ srcEndPointSuffix= srcSecretId=xxxxxxxxxxx # 源bucket隶属的用户的秘钥 secret_key(可在 https://console.qcloud.com/capi 查看) srcSecretKey=yyyyyyyyyyyyyyyy -# 要迁移的cos路径的 +# 如果使用临时密钥访问存储桶,此处配置临时密钥的Token,该密钥需要有源桶的GetBucket（列出存储桶对象）和HeadObject权限 +# srcToken=zzzzzzzzzzz +# 要迁移的cos路径的 srcCosPath=/ +# 如果只需要迁移部分文件，可以通过srcFileList指定迁移文件列表，例如，只需要迁移aaa/1.jpg和bb/2.jpg，则： +# $cat /data/fileList.txt +# aaa/1.jpg +# bbb/2.jpg +#srcFileList=/data/fileList.txt [migrateCsp] bucket=xxx diff --git a/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar b/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar index 5a6374b..6118258 100644 Binary files a/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar and b/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar differ diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java b/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java index 8fe8e5a..b488601 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java @@ -17,6 +17,7 @@ public class CommonConfig { private String region; private String ak; private String sk; + private String token; private String cosPath; private boolean enableHttps; private boolean entireFileMd5Attached; @@ -157,6 +158,17 @@ public void setSk(String sk) { this.sk = sk; } + public String getToken() { + return token; + } + + public void setToken(String token) { + if (token.trim().isEmpty()) { + throw new IllegalArgumentException("token is empty"); + } + this.token = token.trim(); + } + public String getCosPath() { return cosPath; } diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java b/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java index 7441a64..e43bca9 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java @@ -34,6 +34,7 @@ public class ConfigParser { private static final String COMMON_BUCKETNAME = "bucketName"; private static final String COMMON_AK = "secretId"; private static final String COMMON_SK = "secretKey"; + private static final String COMMON_TOKEN = "token"; private static final String COMMON_COSPATH = "cosPath"; private static final String COMMON_DB_COSPATH = "dbCosPath"; private static final String COMMON_HTTPS = "https"; @@ -94,6 +95,7 @@ public class ConfigParser { private static final String COPY_SRC_BUCKETNAME = "srcBucketName"; private static final String COPY_SRC_SECRETID = "srcSecretId"; private static final String COPY_SRC_SECRETKEY = "srcSecretKey"; + private static final String COPY_SRC_TOKEN = "srcToken"; private static final String COPY_SRC_COSPATH = "srcCosPath"; private static final String COPY_SRC_ENDPOINT_SUFFIX = "srcEndPointSuffix"; private static final String COPY_SRC_FILE_LIST = "srcFileList"; @@ -457,7 +459,10 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) { assert (sk != null); commonConfig.setSk(sk); - + String token = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_TOKEN); + if (token != null) { + commonConfig.setToken(token); + } } String cosPathConfig = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_COSPATH); @@ -893,6 +898,11 @@ private boolean initCopyBucketConfig(Preferences prefs, CopyBucketConfig copyBuc assert (srcSecretKey != null); copyBucketConfig.setSrcSk(srcSecretKey); + String token = getConfigValue(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_TOKEN); + if (token != null && !token.trim().isEmpty()) { + copyBucketConfig.setSrcToken(token); + } + String srcCosPath = getConfigValue(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_COSPATH); assert (srcCosPath != null); copyBucketConfig.setSrcCosPath(srcCosPath); diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java b/src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java index 069d28b..11359a5 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java @@ -11,6 +11,7 @@ public class CopyBucketConfig extends CommonConfig { private String srcBucket; private String srcAk; private String srcSk; + private String srcToken; private String srcCosPath; private String srcEndpointSuffix; private String srcFileList = ""; @@ -74,6 +75,17 @@ public void setSrcSk(String srcSk) { this.srcSk = srcSk.trim(); } + public String getSrcToken() { + return srcToken; + } + + public void setSrcToken(String srcToken) { + if (srcToken.trim().isEmpty()) { + throw new IllegalArgumentException("srcToken is empty"); + } + this.srcToken = srcToken.trim(); + } + public String getSrcCosPath() { return srcCosPath; } diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java index adfd556..4a3ffc7 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java @@ -15,6 +15,7 @@ import com.qcloud.cos.COSClient; import com.qcloud.cos.ClientConfig; import com.qcloud.cos.auth.BasicCOSCredentials; +import com.qcloud.cos.auth.BasicSessionCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.http.HttpProtocol; import com.qcloud.cos.model.COSObjectSummary; @@ -38,8 +39,15 @@ public class MigrateCopyBucketTaskExecutor extends TaskExecutor { public MigrateCopyBucketTaskExecutor(CopyBucketConfig config) { super(MigrateType.MIGRATE_FROM_COS_BUCKET_COPY, config); - - COSCredentials srcCred = new BasicCOSCredentials(config.getSrcAk(), config.getSrcSk()); + String src_token = ((CopyBucketConfig) config).getSrcToken(); + COSCredentials srcCred = null; + if (src_token != null && !src_token.isEmpty()) { + log.info("Use temporary token to list object"); + System.out.println("Use temporary token to list object"); + srcCred = new BasicSessionCredentials(config.getSrcAk(), config.getSrcSk(), src_token); + } else { + srcCred = new BasicCOSCredentials(config.getSrcAk(), config.getSrcSk()); + } ClientConfig clientConfig = new ClientConfig(new Region(config.getSrcRegion())); if (config.isEnableHttps()) { clientConfig.setHttpProtocol(HttpProtocol.https); diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java index faa686b..97f2c5f 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java @@ -10,6 +10,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import com.qcloud.cos.auth.BasicSessionCredentials; +import com.qcloud.cos_migrate_tool.config.CopyBucketConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +59,18 @@ public TaskExecutor(MigrateType migrateType, CommonConfig config) { this.smallFileUploadExecutorNum = config.getSmallFileExecutorNumber(); this.bigFileUploadExecutorNum = config.getBigFileExecutorNum(); - COSCredentials cred = new BasicCOSCredentials(config.getAk(), config.getSk()); + COSCredentials cred = null; + String token = config.getToken(); + // supporting temporary token + if (token != null && !token.isEmpty()) { + log.info("Use temporary token to put object"); + System.out.println("Use temporary token to put object"); + cred = new BasicSessionCredentials(config.getAk(), config.getSk(), token); + } else { + cred = new BasicCOSCredentials(config.getAk(), config.getSk()); + } + + //COSCredentials cred = new BasicCOSCredentials(config.getAk(), config.getSk()); ClientConfig clientConfig = new ClientConfig(new Region(config.getRegion())); if (config.isEnableHttps()) { clientConfig.setHttpProtocol(HttpProtocol.https);