From d9f71e2a8b97ca68df819c719fedfbbdcd7428e3 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Tue, 21 Aug 2018 17:29:52 +0800 Subject: [PATCH 01/18] support user-assigned cmk id header --- .../com/aliyun/oss/internal/OSSHeaders.java | 1 + .../oss/internal/OSSObjectOperation.java | 1 + .../aliyun/oss/model/CopyObjectRequest.java | 22 ++ .../com/aliyun/oss/model/ObjectMetadata.java | 21 ++ .../aliyun/oss/model/PolicyConditions.java | 2 + .../oss/integrationtests/CMKIDTest.java | 296 ++++++++++++++++++ 6 files changed, 343 insertions(+) create mode 100644 src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java diff --git a/src/main/java/com/aliyun/oss/internal/OSSHeaders.java b/src/main/java/com/aliyun/oss/internal/OSSHeaders.java index 6700d09d..9470f3e6 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSHeaders.java +++ b/src/main/java/com/aliyun/oss/internal/OSSHeaders.java @@ -31,6 +31,7 @@ public interface OSSHeaders extends HttpHeaders { static final String OSS_VERSION_ID = "x-oss-version-id"; static final String OSS_SERVER_SIDE_ENCRYPTION = "x-oss-server-side-encryption"; + static final String OSS_SERVER_SIDE_ENCRYPTION_KEY_ID = "x-oss-server-side-encryption-key-id"; static final String GET_OBJECT_IF_MODIFIED_SINCE = "If-Modified-Since"; static final String GET_OBJECT_IF_UNMODIFIED_SINCE = "If-Unmodified-Since"; diff --git a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java index 20c18a1d..8967bf48 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java @@ -930,6 +930,7 @@ private static void populateCopyObjectHeaders(CopyObjectRequest copyObjectReques copyObjectRequest.getNonmatchingEtagConstraints()); addHeader(headers, OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION, copyObjectRequest.getServerSideEncryption()); + addHeader(headers, OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID, copyObjectRequest.getServerSideEncryptionKeyId()); ObjectMetadata newObjectMetadata = copyObjectRequest.getNewObjectMetadata(); if (newObjectMetadata != null) { diff --git a/src/main/java/com/aliyun/oss/model/CopyObjectRequest.java b/src/main/java/com/aliyun/oss/model/CopyObjectRequest.java index f51689b7..f8b2dbc3 100644 --- a/src/main/java/com/aliyun/oss/model/CopyObjectRequest.java +++ b/src/main/java/com/aliyun/oss/model/CopyObjectRequest.java @@ -44,6 +44,9 @@ public class CopyObjectRequest extends WebServiceRequest { // Target server's encryption algorithm. private String serverSideEncryption; + // Target server's encryption key ID. + private String serverSideEncryptionKeyID; + // Target object's metadata information. private ObjectMetadata newObjectMetadata; @@ -297,4 +300,23 @@ public String getServerSideEncryption() { public void setServerSideEncryption(String serverSideEncryption) { this.serverSideEncryption = serverSideEncryption; } + + /** + * Gets the target object's server side encryption key ID. + * + * @return Server side encryption key ID，null if no encryption key ID. + */ + public String getServerSideEncryptionKeyId() { + return this.serverSideEncryptionKeyID; + } + + /** + * Sets the target object's server side encryption key ID. + * + * @param serverSideEncryptionKeyId + * Server side encryption key ID，null if no encryption key ID. + */ + public void setServerSideEncryptionKeyId(String serverSideEncryptionKeyId) { + this.serverSideEncryptionKeyID = serverSideEncryptionKeyId; + } } diff --git a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java index f00e0069..75d8d0a1 100644 --- a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java +++ b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java @@ -42,6 +42,8 @@ public class ObjectMetadata { public static final String AES_256_SERVER_SIDE_ENCRYPTION = "AES256"; + public static final String KMS_SERVER_SIDE_ENCRYPTION = "KMS"; + /** *

* Gets the user's custom metadata. @@ -282,6 +284,25 @@ public void setServerSideEncryption(String serverSideEncryption) { metadata.put(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION, serverSideEncryption); } + /** + * Gets the object's server side encryption key ID. + * + * @return The server side encryption key ID. Null means no encryption key ID. + */ + public String getServerSideEncryptionKeyId() { + return (String) metadata.get(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID); + } + + /** + * Sets the object's server side encryption key ID. + * + * @param serverSideEncryptionKeyId + * The server side encryption key ID. + */ + public void setServerSideEncryptionKeyId(String serverSideEncryptionKeyId) { + metadata.put(OSSHeaders.OSS_SERVER_SIDE_ENCRYPTION_KEY_ID, serverSideEncryptionKeyId); + } + /** * Gets the object's storage type, which only supports "normal" and * "appendable" for now. diff --git a/src/main/java/com/aliyun/oss/model/PolicyConditions.java b/src/main/java/com/aliyun/oss/model/PolicyConditions.java index c59491eb..d0640243 100644 --- a/src/main/java/com/aliyun/oss/model/PolicyConditions.java +++ b/src/main/java/com/aliyun/oss/model/PolicyConditions.java @@ -153,6 +153,7 @@ public class PolicyConditions { public final static String COND_SUCCESS_ACTION_REDIRECT = "success_action_redirect"; public final static String COND_SUCCESS_ACTION_STATUS = "success_action_status"; public final static String COND_X_OSS_META_PREFIX = "x-oss-meta-"; + public final static String COND_X_OSS_SERVER_SIDE_PREFIX = "x-oss-server-side-"; private static Map> _supportedMatchRules = new HashMap>(); private List _conds = new ArrayList(); @@ -176,6 +177,7 @@ public class PolicyConditions { _supportedMatchRules.put(COND_SUCCESS_ACTION_REDIRECT, ordinaryMatchModes); _supportedMatchRules.put(COND_SUCCESS_ACTION_STATUS, ordinaryMatchModes); _supportedMatchRules.put(COND_X_OSS_META_PREFIX, ordinaryMatchModes); + _supportedMatchRules.put(COND_X_OSS_SERVER_SIDE_PREFIX, ordinaryMatchModes); } /** diff --git a/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java new file mode 100644 index 00000000..bf774cae --- /dev/null +++ b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java @@ -0,0 +1,296 @@ +package com.aliyun.oss.integrationtests; + +import com.aliyun.oss.model.*; +import junit.framework.Assert; +import org.apache.commons.codec.binary.Base64; +import org.junit.Test; + +import javax.activation.MimetypesFileTypeMap; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +public class CMKIDTest extends TestBase { + + private String CMS_ID = "7fcfd1f0-339c-4f5e-9ba7-58bb04b9930f"; + + private String key = "CMKIDTest"; + + private String uploadLocalFilePath = "upload"; + + private String downloadLocalFilePath = "download"; + + @Test + public void testPutObjectWithCMKID() { + try { + final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + + final ObjectMetadata metadata = new ObjectMetadata(); + metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + metadata.setServerSideEncryptionKeyId(CMS_ID); + ossClient.putObject(bucketName, key, sampleFile, metadata); + ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + + System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testPostObjectWithCMKID() { + HttpURLConnection conn = null; + + try { + final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + + String urlStr = TestConfig.OSS_TEST_ENDPOINT.replace("http://", "http://" + bucketName + "."); + + Map formFields = new LinkedHashMap(); + + formFields.put("key", this.key); + formFields.put("Content-Disposition", "attachment;filename=" + + sampleFile.getAbsolutePath()); + formFields.put("OSSAccessKeyId", TestConfig.OSS_TEST_ACCESS_KEY_ID); + formFields.put("x-oss-server-side-encryption", ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + formFields.put("x-oss-server-side-encryption-key-id", CMS_ID); + String policy + = "{\"expiration\": \"2120-01-01T12:00:00.000Z\",\"conditions\": [[\"content-length-range\", 0, 104857600]]}"; + String encodePolicy = new String(Base64.encodeBase64(policy.getBytes())); + formFields.put("policy", encodePolicy); + String signaturecom = computeSignature(TestConfig.OSS_TEST_ACCESS_KEY_SECRET, encodePolicy); + formFields.put("Signature", signaturecom); + + String boundary = "9431149156168"; + + URL url = new URL(urlStr); + conn = (HttpURLConnection)url.openConnection(); + conn.setConnectTimeout(5000); + conn.setReadTimeout(30000); + conn.setDoOutput(true); + conn.setDoInput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("User-Agent", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.6)"); + conn.setRequestProperty("Content-Type", + "multipart/form-data; boundary=" + boundary); + OutputStream out = new DataOutputStream(conn.getOutputStream()); + + if (formFields != null) { + StringBuffer strBuf = new StringBuffer(); + Iterator> iter = formFields.entrySet().iterator(); + int i = 0; + + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String inputName = entry.getKey(); + String inputValue = entry.getValue(); + + if (inputValue == null) { + continue; + } + + if (i == 0) { + strBuf.append("--").append(boundary).append("\r\n"); + strBuf.append("Content-Disposition: form-data; name=\"" + + inputName + "\"\r\n\r\n"); + strBuf.append(inputValue); + } else { + strBuf.append("\r\n").append("--").append(boundary).append("\r\n"); + strBuf.append("Content-Disposition: form-data; name=\"" + + inputName + "\"\r\n\r\n"); + strBuf.append(inputValue); + } + + i++; + } + out.write(strBuf.toString().getBytes()); + } + + String contentType = new MimetypesFileTypeMap().getContentType(sampleFile.getName()); + if (contentType == null || contentType.equals("")) { + contentType = "application/octet-stream"; + } + + StringBuffer strBuf = new StringBuffer(); + strBuf.append("\r\n").append("--").append(boundary) + .append("\r\n"); + strBuf.append("Content-Disposition: form-data; name=\"file\"; " + + "filename=\"" + sampleFile.getName() + "\"\r\n"); + strBuf.append("Content-Type: " + contentType + "\r\n\r\n"); + + out.write(strBuf.toString().getBytes()); + + DataInputStream in = new DataInputStream(new FileInputStream(sampleFile)); + int bytes = 0; + byte[] bufferOut = new byte[1024]; + while ((bytes = in.read(bufferOut)) != -1) { + out.write(bufferOut, 0, bytes); + } + in.close(); + + byte[] endData = ("\r\n--" + boundary + "--\r\n").getBytes(); + out.write(endData); + out.flush(); + out.close(); + + strBuf = new StringBuffer(); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + strBuf.append(line).append("\n"); + } + System.out.println(strBuf.toString()); + reader.close(); + + ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + + System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + private static String computeSignature(String accessKeySecret, String encodePolicy) + throws UnsupportedEncodingException, NoSuchAlgorithmException, InvalidKeyException { + // convert to UTF-8 + byte[] key = accessKeySecret.getBytes("UTF-8"); + byte[] data = encodePolicy.getBytes("UTF-8"); + + // hmac-sha1 + Mac mac = Mac.getInstance("HmacSHA1"); + mac.init(new SecretKeySpec(key, "HmacSHA1")); + byte[] sha = mac.doFinal(data); + + // base64 + return new String(Base64.encodeBase64(sha)); + } + + @Test + public void testInitMultipartUploadWithCMKID() { + try { + final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + metadata.setServerSideEncryptionKeyId(CMS_ID); + + InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key, metadata); + InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request); + String uploadId = result.getUploadId(); + List partETags = new ArrayList(); + + final long partSize = 1 * 1024 * 1024L; + long fileLength = sampleFile.length(); + int partCount = (int) (fileLength / partSize); + if (fileLength % partSize != 0) { + partCount++; + } + + for (int i = 0; i < partCount; i++) { + long startPos = i * partSize; + long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize; + InputStream instream = new FileInputStream(sampleFile); + instream.skip(startPos); + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(key); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(instream); + uploadPartRequest.setPartSize(curPartSize); + uploadPartRequest.setPartNumber(i + 1); + UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest); + partETags.add(uploadPartResult.getPartETag()); + } + + Collections.sort(partETags, new Comparator() { + public int compare(PartETag p1, PartETag p2) { + return p1.getPartNumber() - p2.getPartNumber(); + } + }); + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags); + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + + ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + + System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testCopyObjectWithCMKID() { + try { + final String sourceBucketName = "src_bucket"; + final String sourceKey = "src_key"; + + final CopyObjectRequest request = + new CopyObjectRequest(sourceBucketName, sourceKey, bucketName, key); + + final ObjectMetadata metadata = new ObjectMetadata(); + metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + metadata.setServerSideEncryptionKeyId(CMS_ID); + request.setNewObjectMetadata(metadata); + ossClient.copyObject(request); + + String sourceLocalFilePath = "source"; + String copyLocalFilePath = "copy"; + + ossClient.getObject(new GetObjectRequest(sourceBucketName, sourceKey), new File(sourceLocalFilePath)); + ossClient.getObject(new GetObjectRequest(bucketName, key), new File(copyLocalFilePath)); + + System.out.println(compareFile(sourceLocalFilePath, copyLocalFilePath)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testAppendObjectWithCMKID() { + try { + String content1 = "Hello OSS A "; + String content2 = "Hello OSS B "; + + ObjectMetadata meta = new ObjectMetadata(); + + meta.setContentType("text/plain"); + meta.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + meta.setServerSideEncryptionKeyId(CMS_ID); + + AppendObjectRequest appendObjectRequest = new AppendObjectRequest(bucketName, key, new ByteArrayInputStream(content1.getBytes()),meta); + + appendObjectRequest.setPosition(0L); + + AppendObjectResult appendObjectResult = ossClient.appendObject(appendObjectRequest); + + appendObjectRequest.setPosition(appendObjectResult.getNextPosition()); + appendObjectRequest.setInputStream(new ByteArrayInputStream(content2.getBytes())); + + ossClient.appendObject(appendObjectRequest); + + OSSObject o = ossClient.getObject(bucketName, key); + BufferedReader reader = new BufferedReader(new InputStreamReader(o.getObjectContent())); + StringBuilder sb = new StringBuilder(); + + while (true) { + String line = reader.readLine(); + if (line == null) break; + sb.append(line); + } + + System.out.println(sb.toString().equals(content1 + content2)); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } +} \ No newline at end of file From af62f6b1bf5c69f2323af5a95005ef60300440f3 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Mon, 8 Oct 2018 11:40:52 +0800 Subject: [PATCH 02/18] fix cmkid ut --- .../oss/integrationtests/CMKIDTest.java | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java index bf774cae..4a56e4d5 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java @@ -17,26 +17,23 @@ public class CMKIDTest extends TestBase { - private String CMS_ID = "7fcfd1f0-339c-4f5e-9ba7-58bb04b9930f"; - private String key = "CMKIDTest"; - private String uploadLocalFilePath = "upload"; + private String uploadLocalFilePath = "uploadFile"; - private String downloadLocalFilePath = "download"; + private String downloadLocalFilePath = "downloadFile"; @Test public void testPutObjectWithCMKID() { try { - final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + final File sampleFile = createSampleFile(uploadLocalFilePath, 1 * 1024 * 1024); final ObjectMetadata metadata = new ObjectMetadata(); metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); - metadata.setServerSideEncryptionKeyId(CMS_ID); + metadata.setServerSideEncryptionKeyId(TestConfig.CMK_ID); ossClient.putObject(bucketName, key, sampleFile, metadata); - ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); - - System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + ObjectMetadata objectMetadata = ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + Assert.assertEquals(TestConfig.CMK_ID, objectMetadata.getServerSideEncryptionKeyId()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -47,7 +44,7 @@ public void testPostObjectWithCMKID() { HttpURLConnection conn = null; try { - final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + final File sampleFile = createSampleFile(uploadLocalFilePath, 1 * 1024 * 1024); String urlStr = TestConfig.OSS_TEST_ENDPOINT.replace("http://", "http://" + bucketName + "."); @@ -58,7 +55,7 @@ public void testPostObjectWithCMKID() { + sampleFile.getAbsolutePath()); formFields.put("OSSAccessKeyId", TestConfig.OSS_TEST_ACCESS_KEY_ID); formFields.put("x-oss-server-side-encryption", ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); - formFields.put("x-oss-server-side-encryption-key-id", CMS_ID); + formFields.put("x-oss-server-side-encryption-key-id", TestConfig.CMK_ID); String policy = "{\"expiration\": \"2120-01-01T12:00:00.000Z\",\"conditions\": [[\"content-length-range\", 0, 104857600]]}"; String encodePolicy = new String(Base64.encodeBase64(policy.getBytes())); @@ -148,9 +145,8 @@ public void testPostObjectWithCMKID() { System.out.println(strBuf.toString()); reader.close(); - ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); - - System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + ObjectMetadata objectMetadata = ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + Assert.assertEquals(TestConfig.CMK_ID, objectMetadata.getServerSideEncryptionKeyId()); } catch (Exception e) { Assert.fail(e.getMessage()); } finally { @@ -178,10 +174,10 @@ private static String computeSignature(String accessKeySecret, String encodePoli @Test public void testInitMultipartUploadWithCMKID() { try { - final File sampleFile = createSampleFile(uploadLocalFilePath, 2 * 1024 * 1024); + final File sampleFile = createSampleFile(uploadLocalFilePath, 1 * 1024 * 1024); ObjectMetadata metadata = new ObjectMetadata(); metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); - metadata.setServerSideEncryptionKeyId(CMS_ID); + metadata.setServerSideEncryptionKeyId(TestConfig.CMK_ID); InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key, metadata); InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request); @@ -220,9 +216,9 @@ public int compare(PartETag p1, PartETag p2) { new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags); ossClient.completeMultipartUpload(completeMultipartUploadRequest); - ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); + ObjectMetadata objectMetadata = ossClient.getObject(new GetObjectRequest(bucketName, key), new File(downloadLocalFilePath)); - System.out.println(compareFile(sampleFile.getAbsolutePath(), downloadLocalFilePath)); + Assert.assertEquals(TestConfig.CMK_ID, objectMetadata.getServerSideEncryptionKeyId()); } catch (IOException e) { Assert.fail(e.getMessage()); } @@ -231,25 +227,23 @@ public int compare(PartETag p1, PartETag p2) { @Test public void testCopyObjectWithCMKID() { try { - final String sourceBucketName = "src_bucket"; + final String sourceBucketName = createBucket(); final String sourceKey = "src_key"; - + final File sampleFile = createSampleFile(uploadLocalFilePath, 1 * 1024 * 1024); final CopyObjectRequest request = new CopyObjectRequest(sourceBucketName, sourceKey, bucketName, key); - final ObjectMetadata metadata = new ObjectMetadata(); - metadata.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); - metadata.setServerSideEncryptionKeyId(CMS_ID); - request.setNewObjectMetadata(metadata); + ossClient.putObject(sourceBucketName, sourceKey, sampleFile); + + request.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); + request.setServerSideEncryptionKeyId(TestConfig.CMK_ID); ossClient.copyObject(request); - String sourceLocalFilePath = "source"; String copyLocalFilePath = "copy"; - ossClient.getObject(new GetObjectRequest(sourceBucketName, sourceKey), new File(sourceLocalFilePath)); - ossClient.getObject(new GetObjectRequest(bucketName, key), new File(copyLocalFilePath)); + ObjectMetadata metadataCopy = ossClient.getObject(new GetObjectRequest(bucketName, key), new File(copyLocalFilePath)); - System.out.println(compareFile(sourceLocalFilePath, copyLocalFilePath)); + Assert.assertEquals(TestConfig.CMK_ID, metadataCopy.getServerSideEncryptionKeyId()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -265,7 +259,7 @@ public void testAppendObjectWithCMKID() { meta.setContentType("text/plain"); meta.setServerSideEncryption(ObjectMetadata.KMS_SERVER_SIDE_ENCRYPTION); - meta.setServerSideEncryptionKeyId(CMS_ID); + meta.setServerSideEncryptionKeyId(TestConfig.CMK_ID); AppendObjectRequest appendObjectRequest = new AppendObjectRequest(bucketName, key, new ByteArrayInputStream(content1.getBytes()),meta); @@ -288,7 +282,7 @@ public void testAppendObjectWithCMKID() { sb.append(line); } - System.out.println(sb.toString().equals(content1 + content2)); + Assert.assertTrue(sb.toString().equals(content1 + content2)); } catch (IOException e) { Assert.fail(e.getMessage()); } From 1fa897424bb129c29ff51d3a3fca20376f96fd71 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Mon, 8 Oct 2018 11:52:21 +0800 Subject: [PATCH 03/18] add cmkid test config param --- src/test/java/com/aliyun/oss/integrationtests/TestConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/aliyun/oss/integrationtests/TestConfig.java b/src/test/java/com/aliyun/oss/integrationtests/TestConfig.java index 99b78700..047f17eb 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/TestConfig.java +++ b/src/test/java/com/aliyun/oss/integrationtests/TestConfig.java @@ -45,4 +45,6 @@ public final class TestConfig { public static String PROXY_USER = null; public static String PROXY_PASSWORD = null; + // OSS cmk id configuration + public static String CMK_ID = null; } From 292287d5aac1d4d1a2e79d2da3d9580d24764637 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Fri, 24 Aug 2018 14:50:59 +0800 Subject: [PATCH 04/18] support get vod playlist in live channel --- src/main/java/com/aliyun/oss/OSS.java | 39 +++++++++++++++++++ src/main/java/com/aliyun/oss/OSSClient.java | 13 +++++++ .../oss/internal/LiveChannelOperation.java | 36 ++++++++++++++++- .../oss/model/GenerateVodPlaylistRequest.java | 14 +++---- .../oss/model/GetVodPlaylistRequest.java | 30 ++++++++++++++ .../aliyun/oss/integrationtests/RtmpTest.java | 31 ++++++++++++++- 6 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 src/main/java/com/aliyun/oss/model/GetVodPlaylistRequest.java diff --git a/src/main/java/com/aliyun/oss/OSS.java b/src/main/java/com/aliyun/oss/OSS.java index 8dd43568..5f8f3ce1 100644 --- a/src/main/java/com/aliyun/oss/OSS.java +++ b/src/main/java/com/aliyun/oss/OSS.java @@ -2142,6 +2142,45 @@ public void generateVodPlaylist(String bucketName, String liveChannelName, Strin public void generateVodPlaylist(GenerateVodPlaylistRequest generateVodPlaylistRequest) throws OSSException, ClientException; + /** + * Generates and returns a VOD playlist (m3u8 format) for the *.ts files with specified + * time range under the Live Channel, but this VOD playlist would not be stored in OSS Server. + * + * @param bucketName + * Bucket name. + * @param liveChannelName + * Live Channel name. + * @param startTime + * The start time of the playlist in epoch time (means *.ts files + * time is same or later than it) + * @param endTime + * The end time of the playlist in epoch time(means *.ts files + * time is no later than it). + * @return A {@link OSSObject} instance. + * @throws OSSException + * OSS Server side exception. + * @throws ClientException + * OSS Client side exception. + */ + public OSSObject getVodPlaylist(String bucketName, String liveChannelName, long startTime, + long endTime) throws OSSException, ClientException; + + /** + * Generates and returns a VOD playlist (m3u8 format) for the *.ts files with specified + * time range under the Live Channel, but this VOD playlist would not be stored in OSS Server. + * + * @param getVodPlaylistRequest + * A {@link GetVodPlaylistRequest} instance the specifies + * the bucket name and the Live Channel name. + * @return A {@link OSSObject} instance. + * @throws OSSException + * OSS Server side exception. + * @throws ClientException + * OSS Client side exception. + */ + public OSSObject getVodPlaylist(GetVodPlaylistRequest getVodPlaylistRequest) + throws OSSException, ClientException; + /** * Generates a RTMP pushing streaming address in the Live Channel. * diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index bee1e83b..cfd5bbe6 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -1278,6 +1278,19 @@ public void generateVodPlaylist(GenerateVodPlaylistRequest generateVodPlaylistRe liveChannelOperation.generateVodPlaylist(generateVodPlaylistRequest); } + @Override + public OSSObject getVodPlaylist(String bucketName, String liveChannelName, long startTime, + long endTime) throws OSSException, ClientException { + return this.getVodPlaylist( + new GetVodPlaylistRequest(bucketName, liveChannelName, startTime, endTime)); + } + + @Override + public OSSObject getVodPlaylist(GetVodPlaylistRequest getVodPlaylistRequest) + throws OSSException, ClientException { + return liveChannelOperation.getVodPlaylist(getVodPlaylistRequest); + } + @Override public String generateRtmpUri(String bucketName, String liveChannelName, String PlaylistName, long expires) throws OSSException, ClientException { diff --git a/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java b/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java index ca2805ed..9860d29a 100644 --- a/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java +++ b/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java @@ -32,6 +32,7 @@ import static com.aliyun.oss.internal.ResponseParsers.getLiveChannelInfoResponseParser; import static com.aliyun.oss.internal.ResponseParsers.getLiveChannelStatResponseParser; import static com.aliyun.oss.internal.ResponseParsers.listLiveChannelsReponseParser; +import static com.aliyun.oss.internal.ResponseParsers.GetObjectResponseParser; import java.io.ByteArrayInputStream; import java.util.ArrayList; @@ -65,6 +66,8 @@ import com.aliyun.oss.model.LiveChannelStat; import com.aliyun.oss.model.LiveRecord; import com.aliyun.oss.model.SetLiveChannelRequest; +import com.aliyun.oss.model.GetVodPlaylistRequest; +import com.aliyun.oss.model.OSSObject; /** * Live channel operation. @@ -268,7 +271,7 @@ public void generateVodPlaylist(GenerateVodPlaylistRequest generateVodPlaylistRe String bucketName = generateVodPlaylistRequest.getBucketName(); String liveChannelName = generateVodPlaylistRequest.getLiveChannelName(); String playlistName = generateVodPlaylistRequest.getPlaylistName(); - Long startTime = generateVodPlaylistRequest.getStratTime(); + Long startTime = generateVodPlaylistRequest.getStartTime(); Long endTime = generateVodPlaylistRequest.getEndTime(); assertParameterNotNull(bucketName, "bucketName"); @@ -276,7 +279,7 @@ public void generateVodPlaylist(GenerateVodPlaylistRequest generateVodPlaylistRe assertParameterNotNull(liveChannelName, "liveChannelName"); ensureLiveChannelNameValid(liveChannelName); assertParameterNotNull(playlistName, "playlistName"); - assertParameterNotNull(startTime, "stratTime"); + assertParameterNotNull(startTime, "startTime"); assertParameterNotNull(endTime, "endTime"); Map parameters = new HashMap(); @@ -293,6 +296,35 @@ public void generateVodPlaylist(GenerateVodPlaylistRequest generateVodPlaylistRe doOperation(request, emptyResponseParser, bucketName, key); } + public OSSObject getVodPlaylist(GetVodPlaylistRequest getVodPlaylistRequest) throws OSSException, ClientException { + + assertParameterNotNull(getVodPlaylistRequest, "getVodPlaylistRequest"); + + String bucketName = getVodPlaylistRequest.getBucketName(); + String liveChannelName = getVodPlaylistRequest.getLiveChannelName(); + Long startTime = getVodPlaylistRequest.getStartTime(); + Long endTime = getVodPlaylistRequest.getEndTime(); + + assertParameterNotNull(bucketName, "bucketName"); + ensureBucketNameValid(bucketName); + assertParameterNotNull(liveChannelName, "liveChannelName"); + ensureLiveChannelNameValid(liveChannelName); + assertParameterNotNull(startTime, "startTime"); + assertParameterNotNull(endTime, "endTime"); + + Map parameters = new HashMap(); + parameters.put(RequestParameters.SUBRESOURCE_VOD, null); + parameters.put(RequestParameters.SUBRESOURCE_START_TIME, startTime.toString()); + parameters.put(RequestParameters.SUBRESOURCE_END_TIME, endTime.toString()); + + RequestMessage request = new OSSRequestMessageBuilder(getInnerClient()).setEndpoint(getEndpoint()) + .setMethod(HttpMethod.GET).setBucket(bucketName).setKey(liveChannelName).setParameters(parameters) + .setInputStream(new ByteArrayInputStream(new byte[0])).setInputSize(0) + .setOriginalRequest(getVodPlaylistRequest).build(); + + return doOperation(request, new GetObjectResponseParser(bucketName, liveChannelName), bucketName, liveChannelName, true); + } + public String generateRtmpUri(GenerateRtmpUriRequest request) throws OSSException, ClientException { assertParameterNotNull(request, "request"); diff --git a/src/main/java/com/aliyun/oss/model/GenerateVodPlaylistRequest.java b/src/main/java/com/aliyun/oss/model/GenerateVodPlaylistRequest.java index 3ea9f248..83c195c0 100644 --- a/src/main/java/com/aliyun/oss/model/GenerateVodPlaylistRequest.java +++ b/src/main/java/com/aliyun/oss/model/GenerateVodPlaylistRequest.java @@ -26,11 +26,11 @@ public GenerateVodPlaylistRequest(String bucketName, String liveChannelName, Str this.playlistName = playlistName; } - public GenerateVodPlaylistRequest(String bucketName, String liveChannelName, String playlistName, long stratTime, + public GenerateVodPlaylistRequest(String bucketName, String liveChannelName, String playlistName, long startTime, long endTime) { super(bucketName, liveChannelName); this.playlistName = playlistName; - this.stratTime = stratTime; + this.startTime = startTime; this.endTime = endTime; } @@ -42,12 +42,12 @@ public void setPlaylistName(String playlistName) { this.playlistName = playlistName; } - public Long getStratTime() { - return stratTime; + public Long getStartTime() { + return startTime; } - public void setStratTime(long stratTime) { - this.stratTime = stratTime; + public void setStartTime(long startTime) { + this.startTime = startTime; } public Long getEndTime() { @@ -59,6 +59,6 @@ public void setEndTime(long endTime) { } private String playlistName; - private Long stratTime; + private Long startTime; private Long endTime; } diff --git a/src/main/java/com/aliyun/oss/model/GetVodPlaylistRequest.java b/src/main/java/com/aliyun/oss/model/GetVodPlaylistRequest.java new file mode 100644 index 00000000..26919b92 --- /dev/null +++ b/src/main/java/com/aliyun/oss/model/GetVodPlaylistRequest.java @@ -0,0 +1,30 @@ +package com.aliyun.oss.model; + +public class GetVodPlaylistRequest extends LiveChannelGenericRequest { + + public GetVodPlaylistRequest(String bucketName, String liveChannelName, long startTime, + long endTime) { + super(bucketName, liveChannelName); + this.startTime = startTime; + this.endTime = endTime; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + private Long startTime; + private Long endTime; +} diff --git a/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java b/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java index 18afdd00..8ac7c5d0 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java @@ -42,6 +42,7 @@ import com.aliyun.oss.model.LiveChannelTarget; import com.aliyun.oss.model.LiveRecord; import com.aliyun.oss.model.PushflowStatus; +import com.aliyun.oss.model.OSSObject; /** * Test rtmp @@ -56,6 +57,7 @@ public void testCreateLiveChannelDefault() { CreateLiveChannelRequest createLiveChannelRequest = new CreateLiveChannelRequest( bucketName, liveChannel); CreateLiveChannelResult createLiveChannelResult = ossClient.createLiveChannel(createLiveChannelRequest); + ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite); Assert.assertEquals(createLiveChannelResult.getPublishUrls().size(), 1); Assert.assertTrue(createLiveChannelResult.getPublishUrls().get(0).startsWith("rtmp://")); Assert.assertTrue(createLiveChannelResult.getPublishUrls().get(0).endsWith("live/" + liveChannel)); @@ -553,7 +555,34 @@ public void testGenerateVodPlaylist() { Assert.fail(e.getMessage()); } } - + + @Test + public void testGetVodPlaylist() { + final String liveChannel = "normal-get-vod-playlist"; + + try { + CreateLiveChannelRequest createLiveChannelRequest = new CreateLiveChannelRequest( + bucketName, liveChannel); + ossClient.createLiveChannel(createLiveChannelRequest); + ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite); + + long startTime = System.currentTimeMillis() / 1000 - 8*3600; + long endTime = System.currentTimeMillis() / 1000 + 3600; + try { + OSSObject o = ossClient.getVodPlaylist(bucketName, liveChannel, startTime, endTime); + Assert.assertEquals(bucketName, o.getBucketName()); + Assert.assertEquals(liveChannel, o.getKey()); + } catch (OSSException e) { + Assert.assertEquals(e.getErrorCode(), OSSErrorCode.INVALID_ARGUMENT); + Assert.assertTrue(e.getMessage().indexOf("No ts file found in specified time span.") > -1); + } + + ossClient.deleteLiveChannel(bucketName, liveChannel); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + @Test public void testGeneratePushflowUri() { final String liveChannel = "normal-generate-pushflow-uri"; From 869f579c94241ed9bae45d8087ee08075c76d48e Mon Sep 17 00:00:00 2001 From: shallwewu Date: Fri, 24 Aug 2018 14:54:16 +0800 Subject: [PATCH 05/18] refine --- src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java b/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java index 8ac7c5d0..559ffdcf 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/RtmpTest.java @@ -566,7 +566,7 @@ public void testGetVodPlaylist() { ossClient.createLiveChannel(createLiveChannelRequest); ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite); - long startTime = System.currentTimeMillis() / 1000 - 8*3600; + long startTime = System.currentTimeMillis() / 1000 - 3600; long endTime = System.currentTimeMillis() / 1000 + 3600; try { OSSObject o = ossClient.getVodPlaylist(bucketName, liveChannel, startTime, endTime); From ba2647b34c741a45006fed154e000ca6bc9461b4 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Tue, 4 Sep 2018 14:19:19 +0800 Subject: [PATCH 06/18] support sign-v2 --- .../com/aliyun/oss/ClientConfiguration.java | 112 ++++--- src/main/java/com/aliyun/oss/OSSClient.java | 94 +----- ...cSHA1Signature.java => HmacSignature.java} | 39 ++- .../oss/common/auth/ServiceSignature.java | 5 +- .../aliyun/oss/common/utils/StringUtils.java | 20 ++ .../com/aliyun/oss/internal/OSSConstants.java | 3 - .../com/aliyun/oss/internal/OSSOperation.java | 6 +- .../aliyun/oss/internal/OSSRequestSigner.java | 19 +- .../com/aliyun/oss/internal/OSSUtils.java | 6 - .../oss/internal/RequestParameters.java | 14 +- .../aliyun/oss/internal/SignParameters.java | 47 +++ .../com/aliyun/oss/internal/SignUtils.java | 130 +++++++-- .../com/aliyun/oss/internal/SignV2Utils.java | 275 ++++++++++++++++++ .../model/GeneratePresignedUrlRequest.java | 78 +++-- .../aliyun/oss/model/WebServiceRequest.java | 16 + .../aliyun/oss/integrationtests/SignTest.java | 105 +++++++ 16 files changed, 744 insertions(+), 225 deletions(-) rename src/main/java/com/aliyun/oss/common/auth/{HmacSHA1Signature.java => HmacSignature.java} (74%) create mode 100644 src/main/java/com/aliyun/oss/internal/SignParameters.java create mode 100644 src/main/java/com/aliyun/oss/internal/SignV2Utils.java create mode 100644 src/test/java/com/aliyun/oss/integrationtests/SignTest.java diff --git a/src/main/java/com/aliyun/oss/ClientConfiguration.java b/src/main/java/com/aliyun/oss/ClientConfiguration.java index ed5ea4e9..513b6541 100644 --- a/src/main/java/com/aliyun/oss/ClientConfiguration.java +++ b/src/main/java/com/aliyun/oss/ClientConfiguration.java @@ -34,6 +34,7 @@ import com.aliyun.oss.common.utils.ResourceManager; import com.aliyun.oss.common.utils.VersionInfoUtils; import com.aliyun.oss.internal.OSSConstants; +import com.aliyun.oss.internal.SignParameters; /** * Client configurations for accessing to OSS services. @@ -59,6 +60,8 @@ public class ClientConfiguration { public static final String DEFAULT_CNAME_EXCLUDE_LIST = "aliyuncs.com,aliyun-inc.com,aliyun.com"; + public static final String DEFAULT_SIGNATURE_VERSION = SignParameters.AUTH_V1; + protected String userAgent = DEFAULT_USER_AGENT; protected int maxErrorRetry = DEFAULT_MAX_RETRIES; protected int connectionRequestTimeout = DEFAULT_CONNECTION_REQUEST_TIMEOUT; @@ -94,9 +97,11 @@ public class ClientConfiguration { protected List signerHandlers = new LinkedList(); + protected String signatureVersion = DEFAULT_SIGNATURE_VERSION; + /** * Gets the user agent string. - * + * * @return The user agent string. */ public String getUserAgent() { @@ -105,7 +110,7 @@ public String getUserAgent() { /** * Sets the user agent string. - * + * * @param userAgent * The user agent string. */ @@ -115,7 +120,7 @@ public void setUserAgent(String userAgent) { /** * Gets proxy host. - * + * * @return The proxy host in string. */ public String getProxyHost() { @@ -124,7 +129,7 @@ public String getProxyHost() { /** * Sets the proxy host. - * + * * @param proxyHost * The proxy host in string. */ @@ -134,7 +139,7 @@ public void setProxyHost(String proxyHost) { /** * Gets the proxy host's port. - * + * * @return The proxy host. */ public int getProxyPort() { @@ -143,7 +148,7 @@ public int getProxyPort() { /** * Sets proxy port. - * + * * @param proxyPort * The proxy port. * @throws ClientException @@ -159,7 +164,7 @@ public void setProxyPort(int proxyPort) throws ClientException { /** * Gets the proxy user name. - * + * * @return The user name. */ public String getProxyUsername() { @@ -168,7 +173,7 @@ public String getProxyUsername() { /** * Sets the proxy user name. - * + * * @param proxyUsername * The user name. */ @@ -178,7 +183,7 @@ public void setProxyUsername(String proxyUsername) { /** * Gets the proxy user password. - * + * * @return The proxy user password. */ public String getProxyPassword() { @@ -187,7 +192,7 @@ public String getProxyPassword() { /** * Sets the proxy user password. - * + * * @param proxyPassword * The proxy user password. */ @@ -198,7 +203,7 @@ public void setProxyPassword(String proxyPassword) { /** * Gets the proxy server's domain, which could do the NTLM authentiation * (optional). - * + * * @return The proxy domain name. */ public String getProxyDomain() { @@ -208,7 +213,7 @@ public String getProxyDomain() { /** * Sets the proxy server's domain, which could do the NTLM authentication * (optional). - * + * * @param proxyDomain * The proxy domain name. */ @@ -218,7 +223,7 @@ public void setProxyDomain(String proxyDomain) { /** * Gets the proxy host's NTLM authentication server. - * + * * @return The NTLM authentication server name. */ public String getProxyWorkstation() { @@ -228,7 +233,7 @@ public String getProxyWorkstation() { /** * Sets the proxy host's NTLM authentication server(optional, if the proxy * server does not require NTLM authentication, then it's not needed). - * + * * @param proxyWorkstation * The proxy host's NTLM authentication server name. */ @@ -238,7 +243,7 @@ public void setProxyWorkstation(String proxyWorkstation) { /** * Gets the max connection count. - * + * * @return The max connection count. By default it's 1024. */ public int getMaxConnections() { @@ -247,7 +252,7 @@ public int getMaxConnections() { /** * Sets the max connection count. - * + * * @param maxConnections * The max connection count. */ @@ -258,7 +263,7 @@ public void setMaxConnections(int maxConnections) { /** * Gets the socket timeout in millisecond. 0 means infinite timeout, not * recommended. - * + * * @return The socket timeout in millisecond. */ public int getSocketTimeout() { @@ -268,7 +273,7 @@ public int getSocketTimeout() { /** * Sets the socket timeout in millisecond. 0 means infinite timeout, not * recommended. - * + * * @param socketTimeout * The socket timeout in millisecond. */ @@ -278,7 +283,7 @@ public void setSocketTimeout(int socketTimeout) { /** * Gets the socket connection timeout in millisecond. - * + * * @return The socket connection timeout in millisecond. */ public int getConnectionTimeout() { @@ -287,7 +292,7 @@ public int getConnectionTimeout() { /** * Sets the socket connection timeout in millisecond. - * + * * @param connectionTimeout * The socket connection timeout in millisecond. */ @@ -299,7 +304,7 @@ public void setConnectionTimeout(int connectionTimeout) { * Gets the timeout in millisecond for retrieving an available connection * from the connection manager. 0 means infinite and -1 means not defined. * By default it's -1. - * + * * @return The timeout in millisecond. */ public int getConnectionRequestTimeout() { @@ -309,7 +314,7 @@ public int getConnectionRequestTimeout() { /** * Sets the timeout in millisecond for retrieving an available connection * from the connection manager. - * + * * @param connectionRequestTimeout * The timeout in millisecond. */ @@ -319,7 +324,7 @@ public void setConnectionRequestTimeout(int connectionRequestTimeout) { /** * Gets the max retry count upon a retryable error. By default it's 3. - * + * * @return The max retry count. */ public int getMaxErrorRetry() { @@ -328,7 +333,7 @@ public int getMaxErrorRetry() { /** * Sets the max retry count upon a retryable error. By default it's 3. - * + * * @param maxErrorRetry * The max retry count. */ @@ -339,7 +344,7 @@ public void setMaxErrorRetry(int maxErrorRetry) { /** * Gets the connection TTL (time to live). Http connection is cached by the * connection manager with a TTL. - * + * * @return The connection TTL in millisecond. */ public long getConnectionTTL() { @@ -349,7 +354,7 @@ public long getConnectionTTL() { /** * Sets the connection TTL (time to live). Http connection is cached by the * connection manager with a TTL. - * + * * @param connectionTTL * The connection TTL in millisecond. */ @@ -376,7 +381,7 @@ public void setUseReaper(boolean useReaper) { /** * Gets the connection's max idle time. If a connection has been idle for * more than this number, it would be closed. - * + * * @return The connection's max idle time in millisecond. */ public long getIdleConnectionTime() { @@ -386,7 +391,7 @@ public long getIdleConnectionTime() { /** * Sets the connection's max idle time. If a connection has been idle for * more than this number, it would be closed. - * + * * @param idleConnectionTime * The connection's max idle time in millisecond. */ @@ -411,7 +416,7 @@ public void setProtocol(Protocol protocol) { /** * Gets the immutable excluded CName list----any domain ends with an item in * this list will not do Cname resolution. - * + * * @return The excluded CName list, immutable. */ public List getCnameExcludeList() { @@ -428,7 +433,7 @@ public List getCnameExcludeList() { /** * Sets the immutable excluded CName list----any domain ends with an item in * this list will not do Cname resolution. - * + * * @param cnameExcludeList * The excluded CName list, immutable. */ @@ -449,7 +454,7 @@ public void setCnameExcludeList(List cnameExcludeList) { /** * Append default excluded CName list. - * + * * @param excludeList * The excluded CName list. */ @@ -464,7 +469,7 @@ private static void AppendDefaultExcludeList(List excludeList) { /** * Gets the flag if supporting Cname in the endpoint. By default it's true. - * + * * @return True if supporting Cname; False if not. */ public boolean isSupportCname() { @@ -473,7 +478,7 @@ public boolean isSupportCname() { /** * Sets the flag if supporting Cname in the endpoint. By default it's true. - * + * *

* If this value is set true, when building a canonical url, the host would * be checked against the Cname excluded list. If that host is found in the @@ -481,7 +486,7 @@ public boolean isSupportCname() { * domain). If the host is found, then it's thought as CName. If this value * is set false, then always uses TLD to access the endpoint. *

- * + * * @param supportCname * The flag if supporting CName. */ @@ -494,7 +499,7 @@ public ClientConfiguration setSupportCname(boolean supportCname) { * Gets the flag of using SLD (Second Level Domain) style to access the * endpoint. By default it's false. When using SLD, then the bucket endpoint * would be: http://host/bucket. Otherwise, it will be http://bucket.host - * + * * @return True if it's enabled; False if it's disabled. */ public boolean isSLDEnabled() { @@ -504,7 +509,7 @@ public boolean isSLDEnabled() { /** * Sets the flag of using SLD (Second Level Domain) style to access the * endpoint. By default it's false. - * + * * @param enabled * True if it's enabled; False if it's disabled. */ @@ -516,7 +521,7 @@ public ClientConfiguration setSLDEnabled(boolean enabled) { /** * The connection idle time threshold in millisecond that triggers the * validation. By default it's 2000. - * + * * @return The connection idle time threshold. */ public int getValidateAfterInactivity() { @@ -525,7 +530,7 @@ public int getValidateAfterInactivity() { /** * Gets the flag of enabling request timeout. By default it's disabled. - * + * * @return true enabled; false disabled. */ public boolean isRequestTimeoutEnabled() { @@ -534,7 +539,7 @@ public boolean isRequestTimeoutEnabled() { /** * Gets the flag of enabling request timeout. By default it's disabled. - * + * * @param requestTimeoutEnabled * true to enable; false to disable. */ @@ -585,7 +590,7 @@ public Map getDefaultHeaders() { * Sets the default http headers. All these headers would be automatically * added in every request. And if a header is also specified in the request, * the default one will be overwritten. - * + * * @param defaultHeaders * Default http headers. */ @@ -595,7 +600,7 @@ public void setDefaultHeaders(Map defaultHeaders) { /** * Add a default header into the default header list. - * + * * @param key * The default header name. * @param value @@ -608,7 +613,7 @@ public void addDefaultHeader(String key, String value) { /** * Gets the flag of enabling CRC checksum on upload and download. By default * it's true. - * + * * @return true enable CRC;false disable CRC. */ public boolean isCrcCheckEnabled() { @@ -618,7 +623,7 @@ public boolean isCrcCheckEnabled() { /** * Sets the flag of enabling CRC checksum on upload and download. By default * it's true. - * + * * @param crcCheckEnabled * True to enable CRC; False to disable CRC. */ @@ -628,7 +633,7 @@ public void setCrcCheckEnabled(boolean crcCheckEnabled) { /** * Gets signer handlers - * + * * @return signer handlers */ public List getSignerHandlers() { @@ -637,7 +642,7 @@ public List getSignerHandlers() { /** * Sets signer handlers using for authentication of the proxy server. - * + * * @param signerHandlers */ public void setSignerHandlers(List signerHandlers) { @@ -652,4 +657,21 @@ public void setSignerHandlers(List signerHandlers) { } } + /** + * Gets signature version + * + * @return signature version + */ + public String getSignatureVersion() { + return signatureVersion; + } + + /** + * Sets signature version for all request. + * + * @param signatureVersion + */ + public void setSignatureVersion(String signatureVersion) { + this.signatureVersion = signatureVersion; + } } diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index cfd5bbe6..83c6ba71 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -26,10 +26,6 @@ import static com.aliyun.oss.internal.OSSConstants.DEFAULT_OSS_ENDPOINT; import static com.aliyun.oss.internal.OSSUtils.OSS_RESOURCE_MANAGER; import static com.aliyun.oss.internal.OSSUtils.ensureBucketNameValid; -import static com.aliyun.oss.internal.OSSUtils.populateResponseHeaderParameters; -import static com.aliyun.oss.internal.RequestParameters.OSS_ACCESS_KEY_ID; -import static com.aliyun.oss.internal.RequestParameters.SECURITY_TOKEN; -import static com.aliyun.oss.internal.RequestParameters.SIGNATURE; import java.io.File; import java.io.FileInputStream; @@ -44,8 +40,6 @@ import java.net.URL; import java.net.UnknownHostException; import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -54,26 +48,12 @@ import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.common.auth.ServiceSignature; import com.aliyun.oss.common.comm.DefaultServiceClient; -import com.aliyun.oss.common.comm.RequestMessage; import com.aliyun.oss.common.comm.ResponseMessage; import com.aliyun.oss.common.comm.ServiceClient; import com.aliyun.oss.common.comm.TimeoutServiceClient; import com.aliyun.oss.common.utils.BinaryUtil; import com.aliyun.oss.common.utils.DateUtil; -import com.aliyun.oss.common.utils.HttpHeaders; -import com.aliyun.oss.common.utils.HttpUtil; -import com.aliyun.oss.internal.CORSOperation; -import com.aliyun.oss.internal.LiveChannelOperation; -import com.aliyun.oss.internal.OSSBucketOperation; -import com.aliyun.oss.internal.OSSDownloadOperation; -import com.aliyun.oss.internal.OSSHeaders; -import com.aliyun.oss.internal.OSSMultipartOperation; -import com.aliyun.oss.internal.OSSObjectOperation; -import com.aliyun.oss.internal.OSSUdfOperation; -import com.aliyun.oss.internal.OSSUploadOperation; -import com.aliyun.oss.internal.OSSUtils; -import com.aliyun.oss.internal.RequestParameters; -import com.aliyun.oss.internal.SignUtils; +import com.aliyun.oss.internal.*; import com.aliyun.oss.model.*; import com.aliyun.oss.model.SetBucketCORSRequest.CORSRule; @@ -691,7 +671,6 @@ public URL generatePresignedUrl(GeneratePresignedUrlRequest request) throws Clie assertParameterNotNull(request, "request"); - String bucketName = request.getBucketName(); if (request.getBucketName() == null) { throw new IllegalArgumentException(OSS_RESOURCE_MANAGER.getString("MustSetBucketName")); } @@ -700,74 +679,13 @@ public URL generatePresignedUrl(GeneratePresignedUrlRequest request) throws Clie if (request.getExpiration() == null) { throw new IllegalArgumentException(OSS_RESOURCE_MANAGER.getString("MustSetExpiration")); } + String url; - Credentials currentCreds = credsProvider.getCredentials(); - String accessId = currentCreds.getAccessKeyId(); - String accessKey = currentCreds.getSecretAccessKey(); - boolean useSecurityToken = currentCreds.useSecurityToken(); - HttpMethod method = request.getMethod() != null ? request.getMethod() : HttpMethod.GET; - - String expires = String.valueOf(request.getExpiration().getTime() / 1000L); - String key = request.getKey(); - ClientConfiguration config = serviceClient.getClientConfiguration(); - String resourcePath = OSSUtils.determineResourcePath(bucketName, key, config.isSLDEnabled()); - - RequestMessage requestMessage = new RequestMessage(bucketName, key); - requestMessage.setEndpoint(OSSUtils.determineFinalEndpoint(endpoint, bucketName, config)); - requestMessage.setMethod(method); - requestMessage.setResourcePath(resourcePath); - requestMessage.setHeaders(request.getHeaders()); - - requestMessage.addHeader(HttpHeaders.DATE, expires); - if (request.getContentType() != null && !request.getContentType().trim().equals("")) { - requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType()); - } - if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) { - requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5()); - } - for (Map.Entry h : request.getUserMetadata().entrySet()) { - requestMessage.addHeader(OSSHeaders.OSS_USER_METADATA_PREFIX + h.getKey(), h.getValue()); - } - - Map responseHeaderParams = new HashMap(); - populateResponseHeaderParameters(responseHeaderParams, request.getResponseHeaders()); - if (responseHeaderParams.size() > 0) { - requestMessage.setParameters(responseHeaderParams); - } - - if (request.getQueryParameter() != null && request.getQueryParameter().size() > 0) { - for (Map.Entry entry : request.getQueryParameter().entrySet()) { - requestMessage.addParameter(entry.getKey(), entry.getValue()); - } - } - - if (request.getProcess() != null && !request.getProcess().trim().equals("")) { - requestMessage.addParameter(RequestParameters.SUBRESOURCE_PROCESS, request.getProcess()); - } - - if (useSecurityToken) { - requestMessage.addParameter(SECURITY_TOKEN, currentCreds.getSecurityToken()); - } - - String canonicalResource = "/" + ((bucketName != null) ? bucketName : "") + ((key != null ? "/" + key : "")); - String canonicalString = SignUtils.buildCanonicalString(method.toString(), canonicalResource, requestMessage, - expires); - String signature = ServiceSignature.create().computeSignature(accessKey, canonicalString); - - Map params = new LinkedHashMap(); - params.put(HttpHeaders.EXPIRES, expires); - params.put(OSS_ACCESS_KEY_ID, accessId); - params.put(SIGNATURE, signature); - params.putAll(requestMessage.getParameters()); - - String queryString = HttpUtil.paramToQueryString(params, DEFAULT_CHARSET_NAME); - - /* Compse HTTP request uri. */ - String url = requestMessage.getEndpoint().toString(); - if (!url.endsWith("/")) { - url += "/"; + if (serviceClient.getClientConfiguration().getSignatureVersion() != null && serviceClient.getClientConfiguration().getSignatureVersion().equals(SignParameters.AUTH_V2)) { + url = SignV2Utils.buildSignedURL(request, credsProvider.getCredentials(), serviceClient.getClientConfiguration(), endpoint); + } else { + url = SignUtils.buildSignedURL(request, credsProvider.getCredentials(), serviceClient.getClientConfiguration(), endpoint); } - url += resourcePath + "?" + queryString; try { return new URL(url); diff --git a/src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java b/src/main/java/com/aliyun/oss/common/auth/HmacSignature.java similarity index 74% rename from src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java rename to src/main/java/com/aliyun/oss/common/auth/HmacSignature.java index 2579eac7..2da8be25 100644 --- a/src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java +++ b/src/main/java/com/aliyun/oss/common/auth/HmacSignature.java @@ -29,15 +29,21 @@ import com.aliyun.oss.common.utils.BinaryUtil; /** - * Used for computing Hmac-SHA1 signature. + * Used for computing Hmac signature. */ -public class HmacSHA1Signature extends ServiceSignature { +public class HmacSignature extends ServiceSignature { + + /* Hmac SHA1 Signature method */ + public static final String SHA1 = "HmacSHA1"; + + /* Hmac SHA256 Signature method */ + public static final String SHA256 = "HmacSHA256"; /* The default encoding. */ - private static final String DEFAULT_ENCODING = "UTF-8"; + private String encoding = "UTF-8"; /* Signature method. */ - private static final String ALGORITHM = "HmacSHA1"; + private String algorithm = SHA1; /* Signature version. */ private static final String VERSION = "1"; @@ -47,8 +53,19 @@ public class HmacSHA1Signature extends ServiceSignature { /* Prototype of the Mac instance. */ private static Mac macInstance; + public HmacSignature() {} + + public HmacSignature(String algorithm) { + this.algorithm = algorithm; + } + + public HmacSignature(String algorithm, String encoding) { + this.algorithm = algorithm; + this.encoding = encoding; + } + public String getAlgorithm() { - return ALGORITHM; + return algorithm; } public String getVersion() { @@ -57,10 +74,10 @@ public String getVersion() { public String computeSignature(String key, String data) { try { - byte[] signData = sign(key.getBytes(DEFAULT_ENCODING), data.getBytes(DEFAULT_ENCODING)); + byte[] signData = sign(key.getBytes(encoding), data.getBytes(encoding)); return BinaryUtil.toBase64String(signData); } catch (UnsupportedEncodingException ex) { - throw new RuntimeException("Unsupported algorithm: " + DEFAULT_ENCODING, ex); + throw new RuntimeException("Unsupported algorithm: " + encoding, ex); } } @@ -72,7 +89,7 @@ private byte[] sign(byte[] key, byte[] data) { if (macInstance == null) { synchronized (LOCK) { if (macInstance == null) { - macInstance = Mac.getInstance(ALGORITHM); + macInstance = Mac.getInstance(algorithm); } } } @@ -82,12 +99,12 @@ private byte[] sign(byte[] key, byte[] data) { mac = (Mac) macInstance.clone(); } catch (CloneNotSupportedException e) { // If it is not clonable, create a new one. - mac = Mac.getInstance(ALGORITHM); + mac = Mac.getInstance(algorithm); } - mac.init(new SecretKeySpec(key, ALGORITHM)); + mac.init(new SecretKeySpec(key, algorithm)); return mac.doFinal(data); } catch (NoSuchAlgorithmException ex) { - throw new RuntimeException("Unsupported algorithm: " + ALGORITHM, ex); + throw new RuntimeException("Unsupported algorithm: " + algorithm, ex); } catch (InvalidKeyException ex) { throw new RuntimeException("Invalid key: " + key, ex); } diff --git a/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java b/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java index a26b8f37..611601a2 100644 --- a/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java +++ b/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java @@ -52,11 +52,12 @@ public abstract class ServiceSignature { /** * * Creates the default ServiceSignature instance which is - * {@link HmacSHA1Signature}. + * {@link HmacSignature}. * * @return The default ServiceSignature instance */ public static ServiceSignature create() { - return new HmacSHA1Signature(); + return new HmacSignature(); } + } \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/common/utils/StringUtils.java b/src/main/java/com/aliyun/oss/common/utils/StringUtils.java index 9dcb38a6..b64691fb 100644 --- a/src/main/java/com/aliyun/oss/common/utils/StringUtils.java +++ b/src/main/java/com/aliyun/oss/common/utils/StringUtils.java @@ -23,6 +23,7 @@ import java.math.BigInteger; import java.nio.charset.Charset; import java.text.Collator; +import java.util.Collection; import java.util.Locale; /** @@ -148,6 +149,25 @@ public static String join(String joiner, String... parts) { return builder.toString(); } + /** + * Joins the strings in collection with joiner between each string + * @param joiner the string to insert between the strings in collection + * @param collection the collection to join + */ + public static String join(String joiner, Collection collection) { + StringBuilder builder = new StringBuilder(); + int i = 0; + + for (String part : collection) { + builder.append(part); + if (i < collection.size() - 1) { + builder.append(joiner); + } + i++; + } + return builder.toString(); + } + /** * A null-safe trim method. If the input string is null, returns null; * otherwise returns a trimmed version of the input. diff --git a/src/main/java/com/aliyun/oss/internal/OSSConstants.java b/src/main/java/com/aliyun/oss/internal/OSSConstants.java index ff4186cf..30067677 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSConstants.java +++ b/src/main/java/com/aliyun/oss/internal/OSSConstants.java @@ -42,9 +42,6 @@ public final class OSSConstants { public static final int OBJECT_NAME_MAX_LENGTH = 1024; - public static final String OSS_AUTHORIZATION_PREFIX = "OSS "; - public static final String OSS_AUTHORIZATION_SEPERATOR = ":"; - public static final String LOGGER_PACKAGE_NAME = "com.aliyun.oss"; public static final String PROTOCOL_HTTP = "http://"; diff --git a/src/main/java/com/aliyun/oss/internal/OSSOperation.java b/src/main/java/com/aliyun/oss/internal/OSSOperation.java index 57ce32e9..55c91307 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSOperation.java @@ -165,16 +165,16 @@ protected T doOperation(RequestMessage request, ResponseParser parser, St } } - private static RequestSigner createSigner(HttpMethod method, String bucketName, String key, Credentials creds) { + private static RequestSigner createSigner(HttpMethod method, String bucketName, String key, Credentials creds, String signatureVersion) { String resourcePath = "/" + ((bucketName != null) ? bucketName + "/" : "") + ((key != null ? key : "")); - return new OSSRequestSigner(method.toString(), resourcePath, creds); + return new OSSRequestSigner(method.toString(), resourcePath, creds, signatureVersion); } protected ExecutionContext createDefaultContext(HttpMethod method, String bucketName, String key) { ExecutionContext context = new ExecutionContext(); context.setCharset(DEFAULT_CHARSET_NAME); - context.setSigner(createSigner(method, bucketName, key, credsProvider.getCredentials())); + context.setSigner(createSigner(method, bucketName, key, credsProvider.getCredentials(), client.getClientConfiguration().getSignatureVersion())); context.addResponseHandler(errorResponseHandler); if (method == HttpMethod.POST) { context.setRetryStrategy(noRetryStrategy); diff --git a/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java b/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java index e2d8f54a..97411194 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java +++ b/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java @@ -22,7 +22,6 @@ import com.aliyun.oss.ClientException; import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.RequestSigner; -import com.aliyun.oss.common.auth.ServiceSignature; import com.aliyun.oss.common.comm.RequestMessage; public class OSSRequestSigner implements RequestSigner { @@ -31,12 +30,16 @@ public class OSSRequestSigner implements RequestSigner { /* Note that resource path should not have been url-encoded. */ private String resourcePath; + private Credentials creds; - public OSSRequestSigner(String httpMethod, String resourcePath, Credentials creds) { + private String signatureVersion; + + public OSSRequestSigner(String httpMethod, String resourcePath, Credentials creds, String signatureVersion) { this.httpMethod = httpMethod; this.resourcePath = resourcePath; this.creds = creds; + this.signatureVersion = signatureVersion; } @Override @@ -45,9 +48,15 @@ public void sign(RequestMessage request) throws ClientException { String secretAccessKey = creds.getSecretAccessKey(); if (accessKeyId.length() > 0 && secretAccessKey.length() > 0) { - String canonicalString = SignUtils.buildCanonicalString(httpMethod, resourcePath, request, null); - String signature = ServiceSignature.create().computeSignature(secretAccessKey, canonicalString); - request.addHeader(OSSHeaders.AUTHORIZATION, OSSUtils.composeRequestAuthorization(accessKeyId, signature)); + String signature; + + if (SignParameters.AUTH_V2.equals(signatureVersion)) { + signature = SignV2Utils.buildSignature(secretAccessKey, httpMethod, resourcePath, request); + request.addHeader(OSSHeaders.AUTHORIZATION, SignV2Utils.composeRequestAuthorization(accessKeyId,signature, request)); + } else { + signature = SignUtils.buildSignature(secretAccessKey, httpMethod, resourcePath, request); + request.addHeader(OSSHeaders.AUTHORIZATION, SignUtils.composeRequestAuthorization(accessKeyId, signature)); + } } } } diff --git a/src/main/java/com/aliyun/oss/internal/OSSUtils.java b/src/main/java/com/aliyun/oss/internal/OSSUtils.java index 450945bd..ad509961 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSUtils.java +++ b/src/main/java/com/aliyun/oss/internal/OSSUtils.java @@ -21,8 +21,6 @@ import static com.aliyun.oss.internal.OSSConstants.DEFAULT_CHARSET_NAME; import static com.aliyun.oss.internal.OSSConstants.OBJECT_NAME_MAX_LENGTH; -import static com.aliyun.oss.internal.OSSConstants.OSS_AUTHORIZATION_PREFIX; -import static com.aliyun.oss.internal.OSSConstants.OSS_AUTHORIZATION_SEPERATOR; import static com.aliyun.oss.internal.OSSConstants.RESOURCE_NAME_COMMON; import static com.aliyun.oss.internal.OSSConstants.RESOURCE_NAME_OSS; @@ -393,10 +391,6 @@ public static String joinETags(List eTags) { return sb.toString(); } - public static String composeRequestAuthorization(String accessKeyId, String signature) { - return OSS_AUTHORIZATION_PREFIX + accessKeyId + OSS_AUTHORIZATION_SEPERATOR + signature; - } - /** * Encode the callback with JSON. */ diff --git a/src/main/java/com/aliyun/oss/internal/RequestParameters.java b/src/main/java/com/aliyun/oss/internal/RequestParameters.java index 05f80830..589869e2 100644 --- a/src/main/java/com/aliyun/oss/internal/RequestParameters.java +++ b/src/main/java/com/aliyun/oss/internal/RequestParameters.java @@ -79,9 +79,6 @@ public final class RequestParameters { public static final String PART_NUMBER_MARKER = "part-number-marker"; public static final String RULE_ID = "rule-id"; - public static final String SIGNATURE = "Signature"; - public static final String OSS_ACCESS_KEY_ID = "OSSAccessKeyId"; - public static final String SECURITY_TOKEN = "security-token"; public static final String POSITION = "position"; @@ -99,4 +96,15 @@ public final class RequestParameters { public static final String SINCE = "since"; public static final String TAIL = "tail"; + /* V1 signature params */ + public static final String SIGNATURE = "Signature"; + public static final String OSS_ACCESS_KEY_ID = "OSSAccessKeyId"; + + /* V2 signature params */ + public static final String OSS_SIGNATURE_VERSION = "x-oss-signature-version"; + public static final String OSS_EXPIRES = "x-oss-expires"; + public static final String OSS_ACCESS_KEY_ID_PARAM = "x-oss-access-key-id"; + public static final String OSS_ADDITIONAL_HEADERS = "x-oss-additional-headers"; + public static final String OSS_SIGNATURE = "x-oss-signature"; + } diff --git a/src/main/java/com/aliyun/oss/internal/SignParameters.java b/src/main/java/com/aliyun/oss/internal/SignParameters.java new file mode 100644 index 00000000..b3772289 --- /dev/null +++ b/src/main/java/com/aliyun/oss/internal/SignParameters.java @@ -0,0 +1,47 @@ +package com.aliyun.oss.internal; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.aliyun.oss.common.utils.CodingUtils.assertTrue; +import static com.aliyun.oss.internal.RequestParameters.*; +import static com.aliyun.oss.internal.RequestParameters.SUBRESOURCE_RESTORE; +import static com.aliyun.oss.internal.RequestParameters.SUBRESOURCE_UDF_LOG; +import static com.aliyun.oss.model.ResponseHeaderOverrides.*; +import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_TYPE; +import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_EXPIRES; + +public class SignParameters { + + public static final String AUTH_V1 = "auth-v1"; + + public static final String AUTH_V2 = "auth-v2"; + + public static final String AUTHORIZATION_PREFIX = "OSS "; + + public static final String AUTHORIZATION_PREFIX_V2 = "OSS2 "; + + public static final String AUTHORIZATION_ACCESS_KEY_ID = "AccessKeyId"; + + public static final String AUTHORIZATION_ADDITIONAL_HEADERS = "AdditionalHeaders"; + + public static final String AUTHORIZATION_SIGNATURE = "Signature"; + + public static final String NEW_LINE = "\n"; + + public static final List SIGNED_PARAMTERS = Arrays.asList(new String[] { SUBRESOURCE_ACL, + SUBRESOURCE_UPLOADS, SUBRESOURCE_LOCATION, SUBRESOURCE_CORS, SUBRESOURCE_LOGGING, SUBRESOURCE_WEBSITE, + SUBRESOURCE_REFERER, SUBRESOURCE_LIFECYCLE, SUBRESOURCE_DELETE, SUBRESOURCE_APPEND, SUBRESOURCE_TAGGING, + SUBRESOURCE_OBJECTMETA, UPLOAD_ID, PART_NUMBER, SECURITY_TOKEN, POSITION, RESPONSE_HEADER_CACHE_CONTROL, + RESPONSE_HEADER_CONTENT_DISPOSITION, RESPONSE_HEADER_CONTENT_ENCODING, RESPONSE_HEADER_CONTENT_LANGUAGE, + RESPONSE_HEADER_CONTENT_TYPE, RESPONSE_HEADER_EXPIRES, SUBRESOURCE_IMG, SUBRESOURCE_STYLE, STYLE_NAME, + SUBRESOURCE_REPLICATION, SUBRESOURCE_REPLICATION_PROGRESS, SUBRESOURCE_REPLICATION_LOCATION, + SUBRESOURCE_CNAME, SUBRESOURCE_BUCKET_INFO, SUBRESOURCE_COMP, SUBRESOURCE_QOS, SUBRESOURCE_LIVE, + SUBRESOURCE_STATUS, SUBRESOURCE_VOD, SUBRESOURCE_START_TIME, SUBRESOURCE_END_TIME, SUBRESOURCE_PROCESS, + SUBRESOURCE_PROCESS_CONF, SUBRESOURCE_SYMLINK, SUBRESOURCE_STAT, SUBRESOURCE_UDF, SUBRESOURCE_UDF_NAME, + SUBRESOURCE_UDF_IMAGE, SUBRESOURCE_UDF_IMAGE_DESC, SUBRESOURCE_UDF_APPLICATION, SUBRESOURCE_UDF_LOG, + SUBRESOURCE_RESTORE, }); + + +} diff --git a/src/main/java/com/aliyun/oss/internal/SignUtils.java b/src/main/java/com/aliyun/oss/internal/SignUtils.java index 3202078e..43852c8c 100644 --- a/src/main/java/com/aliyun/oss/internal/SignUtils.java +++ b/src/main/java/com/aliyun/oss/internal/SignUtils.java @@ -21,44 +21,38 @@ import static com.aliyun.oss.common.utils.CodingUtils.assertTrue; import static com.aliyun.oss.internal.RequestParameters.*; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CACHE_CONTROL; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_DISPOSITION; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_ENCODING; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_LANGUAGE; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_TYPE; -import static com.aliyun.oss.model.ResponseHeaderOverrides.RESPONSE_HEADER_EXPIRES; import java.util.Arrays; -import java.util.List; import java.util.Map; +import static com.aliyun.oss.internal.OSSConstants.DEFAULT_CHARSET_NAME; +import static com.aliyun.oss.internal.OSSUtils.populateResponseHeaderParameters; +import static com.aliyun.oss.internal.RequestParameters.SIGNATURE; +import static com.aliyun.oss.internal.SignParameters.AUTHORIZATION_PREFIX; + +import java.net.URI; +import java.util.*; import java.util.Map.Entry; -import java.util.TreeMap; +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.HttpMethod; +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.ServiceSignature; import com.aliyun.oss.common.comm.RequestMessage; import com.aliyun.oss.common.utils.HttpHeaders; +import com.aliyun.oss.common.utils.HttpUtil; +import com.aliyun.oss.model.GeneratePresignedUrlRequest; public class SignUtils { - private static final String NEW_LINE = "\n"; - - private static final List SIGNED_PARAMTERS = Arrays.asList(new String[] { SUBRESOURCE_ACL, - SUBRESOURCE_UPLOADS, SUBRESOURCE_LOCATION, SUBRESOURCE_CORS, SUBRESOURCE_LOGGING, SUBRESOURCE_WEBSITE, - SUBRESOURCE_REFERER, SUBRESOURCE_LIFECYCLE, SUBRESOURCE_DELETE, SUBRESOURCE_APPEND, SUBRESOURCE_TAGGING, - SUBRESOURCE_OBJECTMETA, UPLOAD_ID, PART_NUMBER, SECURITY_TOKEN, POSITION, RESPONSE_HEADER_CACHE_CONTROL, - RESPONSE_HEADER_CONTENT_DISPOSITION, RESPONSE_HEADER_CONTENT_ENCODING, RESPONSE_HEADER_CONTENT_LANGUAGE, - RESPONSE_HEADER_CONTENT_TYPE, RESPONSE_HEADER_EXPIRES, SUBRESOURCE_IMG, SUBRESOURCE_STYLE, STYLE_NAME, - SUBRESOURCE_REPLICATION, SUBRESOURCE_REPLICATION_PROGRESS, SUBRESOURCE_REPLICATION_LOCATION, - SUBRESOURCE_CNAME, SUBRESOURCE_BUCKET_INFO, SUBRESOURCE_COMP, SUBRESOURCE_QOS, SUBRESOURCE_LIVE, - SUBRESOURCE_STATUS, SUBRESOURCE_VOD, SUBRESOURCE_START_TIME, SUBRESOURCE_END_TIME, SUBRESOURCE_PROCESS, - SUBRESOURCE_PROCESS_CONF, SUBRESOURCE_SYMLINK, SUBRESOURCE_STAT, SUBRESOURCE_UDF, SUBRESOURCE_UDF_NAME, - SUBRESOURCE_UDF_IMAGE, SUBRESOURCE_UDF_IMAGE_DESC, SUBRESOURCE_UDF_APPLICATION, SUBRESOURCE_UDF_LOG, - SUBRESOURCE_RESTORE, SUBRESOURCE_CSV_SELECT, SUBRESOURCE_CSV_META, SUBRESOURCE_SQL}); + public static String composeRequestAuthorization(String accessKeyId, String signature) { + return AUTHORIZATION_PREFIX + accessKeyId + ":" + signature; + } public static String buildCanonicalString(String method, String resourcePath, RequestMessage request, String expires) { StringBuilder canonicalString = new StringBuilder(); - canonicalString.append(method + NEW_LINE); + canonicalString.append(method + SignParameters.NEW_LINE); Map headers = request.getHeaders(); TreeMap headersToSign = new TreeMap(); @@ -97,7 +91,7 @@ public static String buildCanonicalString(String method, String resourcePath, Re canonicalString.append(value); } - canonicalString.append(NEW_LINE); + canonicalString.append(SignParameters.NEW_LINE); } // Append canonical resource to canonical string @@ -112,14 +106,14 @@ public static String buildRtmpCanonicalString(String canonicalizedResource, Requ StringBuilder canonicalString = new StringBuilder(); // Append expires - canonicalString.append(expires + NEW_LINE); + canonicalString.append(expires + SignParameters.NEW_LINE); // Append canonicalized parameters for (Map.Entry entry : request.getParameters().entrySet()) { String key = entry.getKey(); String value = entry.getValue(); canonicalString.append(key).append(':').append(value); - canonicalString.append(NEW_LINE); + canonicalString.append(SignParameters.NEW_LINE); } // Append canonicalized resource @@ -128,8 +122,77 @@ public static String buildRtmpCanonicalString(String canonicalizedResource, Requ return canonicalString.toString(); } - private static String buildCanonicalizedResource(String resourcePath, Map parameters) { + public static String buildSignedURL(GeneratePresignedUrlRequest request, Credentials currentCreds, ClientConfiguration config, URI endpoint) { + String bucketName = request.getBucketName(); + String accessId = currentCreds.getAccessKeyId(); + String accessKey = currentCreds.getSecretAccessKey(); + boolean useSecurityToken = currentCreds.useSecurityToken(); + HttpMethod method = request.getMethod() != null ? request.getMethod() : HttpMethod.GET; + + String expires = String.valueOf(request.getExpiration().getTime() / 1000L); + String key = request.getKey(); + String resourcePath = OSSUtils.determineResourcePath(bucketName, key, config.isSLDEnabled()); + + RequestMessage requestMessage = new RequestMessage(bucketName, key); + requestMessage.setEndpoint(OSSUtils.determineFinalEndpoint(endpoint, bucketName, config)); + requestMessage.setMethod(method); + requestMessage.setResourcePath(resourcePath); + requestMessage.setHeaders(request.getHeaders()); + + requestMessage.addHeader(HttpHeaders.DATE, expires); + if (request.getContentType() != null && !request.getContentType().trim().equals("")) { + requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType()); + } + if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) { + requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5()); + } + for (Map.Entry h : request.getUserMetadata().entrySet()) { + requestMessage.addHeader(OSSHeaders.OSS_USER_METADATA_PREFIX + h.getKey(), h.getValue()); + } + + Map responseHeaderParams = new HashMap(); + populateResponseHeaderParameters(responseHeaderParams, request.getResponseHeaders()); + if (responseHeaderParams.size() > 0) { + requestMessage.setParameters(responseHeaderParams); + } + + if (request.getQueryParameter() != null && request.getQueryParameter().size() > 0) { + for (Map.Entry entry : request.getQueryParameter().entrySet()) { + requestMessage.addParameter(entry.getKey(), entry.getValue()); + } + } + + if (request.getProcess() != null && !request.getProcess().trim().equals("")) { + requestMessage.addParameter(RequestParameters.SUBRESOURCE_PROCESS, request.getProcess()); + } + + if (useSecurityToken) { + requestMessage.addParameter(SECURITY_TOKEN, currentCreds.getSecurityToken()); + } + + String canonicalResource = "/" + ((bucketName != null) ? bucketName : "") + ((key != null ? "/" + key : "")); + String canonicalString = buildCanonicalString(method.toString(), canonicalResource, requestMessage, + expires); + String signature = ServiceSignature.create().computeSignature(accessKey, canonicalString); + + Map params = new LinkedHashMap(); + params.put(HttpHeaders.EXPIRES, expires); + params.put(OSS_ACCESS_KEY_ID, accessId); + params.put(SIGNATURE, signature); + params.putAll(requestMessage.getParameters()); + + String queryString = HttpUtil.paramToQueryString(params, DEFAULT_CHARSET_NAME); + /* Compse HTTP request uri. */ + String url = requestMessage.getEndpoint().toString(); + if (!url.endsWith("/")) { + url += "/"; + } + url += resourcePath + "?" + queryString; + return url; + } + + public static String buildCanonicalizedResource(String resourcePath, Map parameters) { assertTrue(resourcePath.startsWith("/"), "Resource path should start with slash character"); StringBuilder builder = new StringBuilder(); @@ -139,23 +202,28 @@ private static String buildCanonicalizedResource(String resourcePath, Map buildSortedAdditionalHeaderNames(Set headerNames, Set additionalHeaderNames) { + TreeSet ts = new TreeSet(); + + if (headerNames != null && additionalHeaderNames != null) { + for (String additionalHeaderName : additionalHeaderNames) { + if (headerNames.contains(additionalHeaderName)) { + ts.add(additionalHeaderName.toLowerCase()); + } + } + } + return ts; + } + + private static Set buildRawAdditionalHeaderNames(Set headerNames, Set additionalHeaderNames) { + Set hs = new HashSet(); + + if (headerNames != null && additionalHeaderNames != null) { + for (String additionalHeaderName : additionalHeaderNames) { + if (headerNames.contains(additionalHeaderName)) { + hs.add(additionalHeaderName); + } + } + } + return hs; + } + + public static String buildCanonicalString(String method, String resourcePath, RequestMessage request, Set additionalHeaderNames) { + StringBuilder canonicalString = new StringBuilder(); + canonicalString.append(method + SignParameters.NEW_LINE); + Map headers = request.getHeaders(); + TreeMap fixedHeadersToSign = new TreeMap(); + TreeMap canonicalizedOssHeadersToSign = new TreeMap(); + + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + if (header.getKey() != null) { + String lowerKey = header.getKey().toLowerCase(); + if (lowerKey.equals(HttpHeaders.CONTENT_TYPE.toLowerCase()) + || lowerKey.equals(HttpHeaders.CONTENT_MD5.toLowerCase()) + || lowerKey.equals(HttpHeaders.DATE.toLowerCase())) { + fixedHeadersToSign.put(lowerKey, header.getValue().trim()); + } else if (lowerKey.startsWith(OSSHeaders.OSS_PREFIX)){ + canonicalizedOssHeadersToSign.put(lowerKey, header.getValue().trim()); + } + } + } + } + + if (!fixedHeadersToSign.containsKey(HttpHeaders.CONTENT_TYPE.toLowerCase())) { + fixedHeadersToSign.put(HttpHeaders.CONTENT_TYPE.toLowerCase(), ""); + } + if (!fixedHeadersToSign.containsKey(HttpHeaders.CONTENT_MD5.toLowerCase())) { + fixedHeadersToSign.put(HttpHeaders.CONTENT_MD5.toLowerCase(), ""); + } + + for (String additionalHeaderName : additionalHeaderNames) { + if (additionalHeaderName != null && headers.get(additionalHeaderName) != null) { + canonicalizedOssHeadersToSign.put(additionalHeaderName.toLowerCase(), headers.get(additionalHeaderName).trim()); + } + } + + // Append fixed headers to sign to canonical string + for (Map.Entry entry : fixedHeadersToSign.entrySet()) { + Object value = entry.getValue(); + + canonicalString.append(value); + canonicalString.append(SignParameters.NEW_LINE); + } + + // Append canonicalized oss headers to sign to canonical string + for (Map.Entry entry : canonicalizedOssHeadersToSign.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + + canonicalString.append(key).append(':').append(value).append(SignParameters.NEW_LINE); + } + + + // Append additional header names + TreeSet ts = new TreeSet(); + for (String additionalHeaderName : additionalHeaderNames) { + ts.add(additionalHeaderName.toLowerCase()); + } + String separator = ""; + + for (String additionalHeaderName : ts) { + canonicalString.append(separator).append(additionalHeaderName); + separator = ";"; + } + canonicalString.append(SignParameters.NEW_LINE); + + // Append canonical resource to canonical string + canonicalString.append(buildCanonicalizedResource(resourcePath, request.getParameters())); + + return canonicalString.toString(); + } + + public static String buildSignedURL(GeneratePresignedUrlRequest request, Credentials currentCreds, ClientConfiguration config, URI endpoint) { + String bucketName = request.getBucketName(); + String accessId = currentCreds.getAccessKeyId(); + String accessKey = currentCreds.getSecretAccessKey(); + boolean useSecurityToken = currentCreds.useSecurityToken(); + HttpMethod method = request.getMethod() != null ? request.getMethod() : HttpMethod.GET; + + String expires = String.valueOf(request.getExpiration().getTime() / 1000L); + String key = request.getKey(); + String resourcePath = OSSUtils.determineResourcePath(bucketName, key, config.isSLDEnabled()); + + RequestMessage requestMessage = new RequestMessage(bucketName, key); + requestMessage.setEndpoint(OSSUtils.determineFinalEndpoint(endpoint, bucketName, config)); + requestMessage.setMethod(method); + requestMessage.setResourcePath(resourcePath); + requestMessage.setHeaders(request.getHeaders()); + + requestMessage.addHeader(HttpHeaders.DATE, expires); + if (request.getContentType() != null && !request.getContentType().trim().equals("")) { + requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType()); + } + if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) { + requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5()); + } + for (Map.Entry h : request.getUserMetadata().entrySet()) { + requestMessage.addHeader(OSSHeaders.OSS_USER_METADATA_PREFIX + h.getKey(), h.getValue()); + } + Map responseHeaderParams = new HashMap(); + populateResponseHeaderParameters(responseHeaderParams, request.getResponseHeaders()); + if (responseHeaderParams.size() > 0) { + requestMessage.setParameters(responseHeaderParams); + } + + if (request.getQueryParameter() != null && request.getQueryParameter().size() > 0) { + for (Map.Entry entry : request.getQueryParameter().entrySet()) { + requestMessage.addParameter(entry.getKey(), entry.getValue()); + } + } + + if (request.getProcess() != null && !request.getProcess().trim().equals("")) { + requestMessage.addParameter(SUBRESOURCE_PROCESS, request.getProcess()); + } + + if (useSecurityToken) { + requestMessage.addParameter(SECURITY_TOKEN, currentCreds.getSecurityToken()); + } + + String canonicalResource = "/" + ((bucketName != null) ? bucketName : "") + ((key != null ? "/" + key : "")); + Set rawAdditionalHeaderNames = buildRawAdditionalHeaderNames(request.getHeaders().keySet(), request.getAdditionalHeaderNames()); + Set sortedAdditionalHeaderNames = buildSortedAdditionalHeaderNames(request.getHeaders().keySet(), request.getAdditionalHeaderNames()); + String canonicalString = buildCanonicalString(method.toString(), canonicalResource, requestMessage, rawAdditionalHeaderNames); + String signature = new HmacSignature(HmacSignature.SHA256).computeSignature(accessKey, canonicalString); + + Map params = new LinkedHashMap(); + params.put(OSS_SIGNATURE_VERSION, "OSS2"); + params.put(OSS_EXPIRES, expires); + params.put(OSS_ACCESS_KEY_ID_PARAM, accessId); + params.put(OSS_ADDITIONAL_HEADERS, StringUtils.join(";", sortedAdditionalHeaderNames)); + params.put(OSS_SIGNATURE, signature); + params.putAll(requestMessage.getParameters()); + + String queryString = HttpUtil.paramToQueryString(params, DEFAULT_CHARSET_NAME); + + /* Compse HTTP request uri. */ + String url = requestMessage.getEndpoint().toString(); + if (!url.endsWith("/")) { + url += "/"; + } + url += resourcePath + "?" + queryString; + return url; + } + + private static String buildCanonicalizedResource(String resourcePath, Map parameters) { + assertTrue(resourcePath.startsWith("/"), "Resource path should start with slash character"); + + StringBuilder builder = new StringBuilder(); + builder.append(uriEncoding(resourcePath)); + + if (parameters != null) { + String[] parameterNames = parameters.keySet().toArray(new String[parameters.size()]); + Arrays.sort(parameterNames); + + char separator = '?'; + for (String paramName : parameterNames) { + builder.append(separator); + builder.append(uriEncoding(paramName)); + String paramValue = parameters.get(paramName); + if (paramValue != null) { + builder.append("=").append(uriEncoding(paramValue)); + } + + separator = '&'; + } + } + + return builder.toString(); + } + + public static String uriEncoding(String uri) { + String result = ""; + + for (char c : uri.toCharArray()) { + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') + || (c >= '0' && c <= '9') || c == '_' || c == '-' + || c == '~' || c == '.') { + result += c; + } else if (c == '/') { + result += "%2F"; + } else { + String temp = Integer.toHexString((int)c); + + if (temp.length() < 2) { + temp = "0" + temp; + } + result += "%" + temp.toUpperCase(); + } + } + + return result; + } + + public static String buildSignature(String secretAccessKey, String httpMethod, String resourcePath, RequestMessage request) { + String canonicalString = buildCanonicalString(httpMethod, resourcePath, request, + buildRawAdditionalHeaderNames(request.getOriginalRequest().getHeaders().keySet(), request.getOriginalRequest().getAdditionalHeaderNames())); + return new HmacSignature(HmacSignature.SHA256).computeSignature(secretAccessKey, canonicalString); + } + +} diff --git a/src/main/java/com/aliyun/oss/model/GeneratePresignedUrlRequest.java b/src/main/java/com/aliyun/oss/model/GeneratePresignedUrlRequest.java index fc647208..02a9129f 100644 --- a/src/main/java/com/aliyun/oss/model/GeneratePresignedUrlRequest.java +++ b/src/main/java/com/aliyun/oss/model/GeneratePresignedUrlRequest.java @@ -19,9 +19,7 @@ package com.aliyun.oss.model; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import com.aliyun.oss.HttpMethod; @@ -70,9 +68,11 @@ public class GeneratePresignedUrlRequest { private Map headers = new HashMap(); + private Set additionalHeaderNames = new HashSet(); + /** * Constructor with GET as the httpMethod - * + * * @param bucketName * Bucket name. * @param key @@ -84,7 +84,7 @@ public GeneratePresignedUrlRequest(String bucketName, String key) { /** * Constructor. - * + * * @param bucketName * Bucket name. * @param key @@ -100,7 +100,7 @@ public GeneratePresignedUrlRequest(String bucketName, String key, HttpMethod met /** * Gets Http method. - * + * * @return HTTP method. */ public HttpMethod getMethod() { @@ -109,7 +109,7 @@ public HttpMethod getMethod() { /** * Sets Http method. - * + * * @param method * HTTP method. */ @@ -122,7 +122,7 @@ public void setMethod(HttpMethod method) { /** * Gets {@link Bucket} name - * + * * @return Bucket name */ public String getBucketName() { @@ -131,7 +131,7 @@ public String getBucketName() { /** * Sets the {@link Bucket} name. - * + * * @param bucketName * {@link Bucket} name. */ @@ -141,7 +141,7 @@ public void setBucketName(String bucketName) { /** * Gets the {@link OSSObject} key. - * + * * @return Object key. */ public String getKey() { @@ -150,7 +150,7 @@ public String getKey() { /** * Sets {@link OSSObject} key. - * + * * @param key * {@link OSSObject} key. */ @@ -160,7 +160,7 @@ public void setKey(String key) { /** * Gets the expiration time of the Url - * + * * @return The expiration time of the Url. */ public Date getExpiration() { @@ -169,7 +169,7 @@ public Date getExpiration() { /** * Sets the expiration time of the Url - * + * * @param expiration * The expiration time of the Url. */ @@ -179,7 +179,7 @@ public void setExpiration(Date expiration) { /** * Sets the content-type header which indicates the file's type. - * + * * @param contentType * The file's content type. */ @@ -189,7 +189,7 @@ public void setContentType(String contentType) { /** * Gets the content type header. - * + * * @return Content-Type Header */ public String getContentType() { @@ -198,7 +198,7 @@ public String getContentType() { /** * Sets the file's MD5 value. - * + * * @param contentMD5 * The target file's MD5 value. */ @@ -208,7 +208,7 @@ public void setContentMD5(String contentMD5) { /** * Gets the file's MD5 value. - * + * * @return Content-MD5 */ public String getContentMD5() { @@ -217,7 +217,7 @@ public String getContentMD5() { /** * Sets the response headers to override. - * + * * @param responseHeaders * The response headers to override. */ @@ -227,7 +227,7 @@ public void setResponseHeaders(ResponseHeaderOverrides responseHeaders) { /** * Gets the response headers to override. - * + * * @return The response headers to override. */ public ResponseHeaderOverrides getResponseHeaders() { @@ -246,7 +246,7 @@ public ResponseHeaderOverrides getResponseHeaders() { * when it's returned from OSS. For example, if the key is MyUserMeta，the * key returned by this method will be myusermeta. *

- * + * * @return A {@link Map} instance that contains the user's customized * metadata. */ @@ -257,7 +257,7 @@ public Map getUserMetadata() { /** * Gets user's customized metadata. They will be represented in x-oss-meta* * headers. - * + * * @param userMetadata * User's metadata */ @@ -270,7 +270,7 @@ public void setUserMetadata(Map userMetadata) { /** * Add a user's customized metadata. - * + * * @param key * The metadata key. Note: this key should not have prefix of * 'x-oss-meta-'. @@ -283,7 +283,7 @@ public void addUserMetadata(String key, String value) { /** * Gets the query parameters. - * + * * @return Query parameters. */ public Map getQueryParameter() { @@ -292,7 +292,7 @@ public Map getQueryParameter() { /** * Sets the query parameters. - * + * * @param queryParam * Query parameters. */ @@ -313,7 +313,7 @@ public void addQueryParameter(String key, String value) { /** * Gets the process header. - * + * * @return The process header. */ public String getProcess() { @@ -322,7 +322,7 @@ public String getProcess() { /** * Sets the process header. - * + * * @param process * The process header. */ @@ -332,7 +332,7 @@ public void setProcess(String process) { /** * Gets HTTP Headers - * + * * @return HTTP Headers */ public Map getHeaders() { @@ -341,7 +341,7 @@ public Map getHeaders() { /** * Sets Http headers. - * + * * @param headers * HTTP Headers。 */ @@ -360,4 +360,26 @@ public void addHeader(String key, String value) { this.headers.put(key, value); } + /** + * Gets additional HTTP header names. + * + * @return Additional HTTP header names. + */ + public Set getAdditionalHeaderNames() { + return additionalHeaderNames; + } + + /** + * Sets additional HTTP header names + * + * @param additionalHeaderNames + * additional http header names. + */ + public void setAdditionalHeaderNames(Set additionalHeaderNames) { + this.additionalHeaderNames = additionalHeaderNames; + } + + public void addAdditionalHeaderName(String name) { + this.additionalHeaderNames.add(name); + } } diff --git a/src/main/java/com/aliyun/oss/model/WebServiceRequest.java b/src/main/java/com/aliyun/oss/model/WebServiceRequest.java index 538c8661..48ea5ff8 100644 --- a/src/main/java/com/aliyun/oss/model/WebServiceRequest.java +++ b/src/main/java/com/aliyun/oss/model/WebServiceRequest.java @@ -19,8 +19,10 @@ package com.aliyun.oss.model; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import com.aliyun.oss.event.ProgressListener; @@ -38,6 +40,8 @@ public abstract class WebServiceRequest { private Map parameters = new LinkedHashMap(); private Map headers = new LinkedHashMap(); + private Set additionalHeaderNames = new HashSet(); + public void setProgressListener(ProgressListener progressListener) { this.progressListener = (progressListener == null) ? ProgressListener.NOOP : progressListener; } @@ -77,6 +81,18 @@ public void addHeader(String key, String value) { this.headers.put(key, value); } + public Set getAdditionalHeaderNames() { + return additionalHeaderNames; + } + + public void setAdditionalHeaderNames(Set additionalHeaderNames) { + this.additionalHeaderNames = additionalHeaderNames; + } + + public void addAdditionalHeaderName(String name) { + this.additionalHeaderNames.add(name); + } + public boolean isLogEnabled() { return logEnabled; } diff --git a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java new file mode 100644 index 00000000..1b7dcb20 --- /dev/null +++ b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java @@ -0,0 +1,105 @@ +package com.aliyun.oss.integrationtests; + +import com.aliyun.oss.ClientBuilderConfiguration; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.common.utils.StringUtils; +import com.aliyun.oss.internal.SignParameters; +import com.aliyun.oss.internal.SignV2Utils; +import com.aliyun.oss.model.GeneratePresignedUrlRequest; +import com.aliyun.oss.model.PutObjectRequest; +import junit.framework.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.*; + +import static com.aliyun.oss.integrationtests.TestUtils.genFixedLengthFile; +import static com.aliyun.oss.integrationtests.TestUtils.removeFile; + +public class SignTest { + + private static final String endpoint = "http://oss-cn-hangzhou.aliyuncs.com"; + + private static final String accessKeyID = "LTAI2pSNlDGMkFrB"; + + private static final String accessKeySecret = "VWEukXofmBnajjymqMvYVwG2LdFN4B"; + + private static final String bucket = "test-sign"; + + @Test + public void testSignV2() { + String key = "test-sign-V2"; + ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); + conf.setSignatureVersion(SignParameters.AUTH_V2); + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyID, accessKeySecret, conf); + ossClient.createBucket(bucket); + String filePath = null; + + try { + filePath = genFixedLengthFile(1 * 1024 * 1024); //1MB + PutObjectRequest request = new PutObjectRequest(bucket, key, new File(filePath)); + request.addHeader("x-oss-head1", "31232"); + request.addHeader("abc", "4fdfsd"); + request.addHeader("ZAbc", "4fde324fsd"); + request.addHeader("XYZ", "4fde324fsd"); + request.addAdditionalHeaderName("ZAbc"); + request.addAdditionalHeaderName("x-oss-head1"); + request.addAdditionalHeaderName("abc"); + request.addParameter("param1", "value1"); + + ossClient.putObject(request); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } finally { + if (filePath != null) { + removeFile(filePath); + } + } + } + + @Test + public void testGenerateSignedV2URL() { + String key = "test-sign-v2-url"; + ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); + conf.setSignatureVersion(SignParameters.AUTH_V2); + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyID, accessKeySecret, conf); + Date expiration = new Date(new Date().getTime() + 1000 * 60 *10); + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, key); + request.setExpiration(expiration); + URL url = null; + + Set abc = new HashSet(); + List ls = new LinkedList(abc); + StringUtils.join(";", ls); + try { + URI endpointURI = new URI(endpoint); + String head1 = "test1"; + String head2 = "B\rTest!"; + + request.addUserMetadata("user1", "ddd"); + request.addHeader(head1, "aaa"); + request.addHeader("atest", "bbb"); + request.addHeader(head2, "ccc"); + request.addAdditionalHeaderName(head1); + request.addAdditionalHeaderName(head2); + request.addQueryParameter("queryParam1", "value1"); + url = ossClient.generatePresignedUrl(request); + + StringBuilder expectedUrlPrefix = new StringBuilder(); + expectedUrlPrefix.append(endpointURI.getScheme()).append("://").append(bucket).append(".").append(endpointURI.getHost()).append("/") + .append(key).append("?x-oss-signature-version=OSS2").append("&x-oss-expires=").append(Long.toString(expiration.getTime() / 1000)) + .append("&x-oss-access-key-id=").append(accessKeyID).append("&x-oss-additional-headers=").append(SignV2Utils.uriEncoding(head2.toLowerCase() + ";" + head1.toLowerCase())).append("&x-oss-signature"); + + Assert.assertTrue(url.toString().startsWith(expectedUrlPrefix.toString())); + + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + + } +} From 19766ed2eb912e768ff33bf75e95550264b28afa Mon Sep 17 00:00:00 2001 From: shallwewu Date: Tue, 9 Oct 2018 15:16:34 +0800 Subject: [PATCH 07/18] support signature version switch & fix V2 URL sign --- src/main/java/com/aliyun/oss/OSS.java | 8 ++ src/main/java/com/aliyun/oss/OSSClient.java | 9 ++ .../oss/common/auth/HmacSHA1Signature.java | 64 ++++++++++ .../oss/common/auth/HmacSHA256Signature.java | 63 ++++++++++ .../aliyun/oss/common/auth/HmacSignature.java | 112 ------------------ .../oss/common/auth/ServiceSignature.java | 38 +++++- .../aliyun/oss/internal/SignParameters.java | 2 + .../com/aliyun/oss/internal/SignV2Utils.java | 17 ++- .../aliyun/oss/integrationtests/SignTest.java | 81 +++++++------ 9 files changed, 234 insertions(+), 160 deletions(-) create mode 100644 src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java create mode 100644 src/main/java/com/aliyun/oss/common/auth/HmacSHA256Signature.java delete mode 100644 src/main/java/com/aliyun/oss/common/auth/HmacSignature.java diff --git a/src/main/java/com/aliyun/oss/OSS.java b/src/main/java/com/aliyun/oss/OSS.java index 5f8f3ce1..bf0a7a62 100644 --- a/src/main/java/com/aliyun/oss/OSS.java +++ b/src/main/java/com/aliyun/oss/OSS.java @@ -52,6 +52,14 @@ public interface OSS { */ public void switchCredentials(Credentials creds); + /** + * Switches to another signature version + * + * @param signatureVersion + * the signature version to switch to。 + */ + public void switchSignatureVersion(String signatureVersion); + /** * Shuts down the OSS instance (release all resources) The OSS instance is * not usable after its shutdown() is called. diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index 83c6ba71..3eae2957 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -300,6 +300,15 @@ public void switchCredentials(Credentials creds) { this.credsProvider.setCredentials(creds); } + @Override + public void switchSignatureVersion(String signatureVersion) { + if (signatureVersion != SignParameters.AUTH_V1 && signatureVersion != SignParameters.AUTH_V2) { + throw new IllegalArgumentException("unsupported signature version" + signatureVersion); + } + + this.getClientConfiguration().setSignatureVersion(signatureVersion); + } + public CredentialsProvider getCredentialsProvider() { return this.credsProvider; } diff --git a/src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java b/src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java new file mode 100644 index 00000000..588dd727 --- /dev/null +++ b/src/main/java/com/aliyun/oss/common/auth/HmacSHA1Signature.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.aliyun.oss.common.auth; + +import java.io.UnsupportedEncodingException; +import javax.crypto.Mac; + +import com.aliyun.oss.common.utils.BinaryUtil; + +/** + * Used for computing Hmac-SHA1 signature. + */ +public class HmacSHA1Signature extends ServiceSignature { + + /* The default encoding. */ + private static final String DEFAULT_ENCODING = "UTF-8"; + + /* Signature method. */ + private static final String ALGORITHM = "HmacSHA1"; + + /* Signature version. */ + private static final String VERSION = "1"; + + private static final Object LOCK = new Object(); + + /* Prototype of the Mac instance. */ + private static Mac macInstance; + + public String getAlgorithm() { + return ALGORITHM; + } + + public String getVersion() { + return VERSION; + } + + public String computeSignature(String key, String data) { + try { + byte[] signData = sign(key.getBytes(DEFAULT_ENCODING), data.getBytes(DEFAULT_ENCODING), macInstance, + LOCK, ALGORITHM); + return BinaryUtil.toBase64String(signData); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException("Unsupported algorithm: " + DEFAULT_ENCODING, ex); + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/common/auth/HmacSHA256Signature.java b/src/main/java/com/aliyun/oss/common/auth/HmacSHA256Signature.java new file mode 100644 index 00000000..d7d2dd10 --- /dev/null +++ b/src/main/java/com/aliyun/oss/common/auth/HmacSHA256Signature.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.aliyun.oss.common.auth; + +import java.io.UnsupportedEncodingException; +import javax.crypto.Mac; + +import com.aliyun.oss.common.utils.BinaryUtil; + +/** + * Used for computing Hmac-SHA256 signature. + */ +public class HmacSHA256Signature extends ServiceSignature { + + /* The default encoding. */ + private static final String DEFAULT_ENCODING = "UTF-8"; + + /* Signature method. */ + private static final String ALGORITHM = "HmacSHA256"; + + /* Signature version. */ + private static final String VERSION = "1"; + + private static final Object LOCK = new Object(); + + /* Prototype of the Mac instance. */ + private static Mac macInstance; + + public String getAlgorithm() { + return ALGORITHM; + } + + public String getVersion() { + return VERSION; + } + + public String computeSignature(String key, String data) { + try { + byte[] signData = sign(key.getBytes(DEFAULT_ENCODING), data.getBytes(DEFAULT_ENCODING), macInstance, + LOCK, ALGORITHM); + return BinaryUtil.toBase64String(signData); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException("Unsupported algorithm: " + DEFAULT_ENCODING, ex); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/common/auth/HmacSignature.java b/src/main/java/com/aliyun/oss/common/auth/HmacSignature.java deleted file mode 100644 index 2da8be25..00000000 --- a/src/main/java/com/aliyun/oss/common/auth/HmacSignature.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.aliyun.oss.common.auth; - -import java.io.UnsupportedEncodingException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -import com.aliyun.oss.common.utils.BinaryUtil; - -/** - * Used for computing Hmac signature. - */ -public class HmacSignature extends ServiceSignature { - - /* Hmac SHA1 Signature method */ - public static final String SHA1 = "HmacSHA1"; - - /* Hmac SHA256 Signature method */ - public static final String SHA256 = "HmacSHA256"; - - /* The default encoding. */ - private String encoding = "UTF-8"; - - /* Signature method. */ - private String algorithm = SHA1; - - /* Signature version. */ - private static final String VERSION = "1"; - - private static final Object LOCK = new Object(); - - /* Prototype of the Mac instance. */ - private static Mac macInstance; - - public HmacSignature() {} - - public HmacSignature(String algorithm) { - this.algorithm = algorithm; - } - - public HmacSignature(String algorithm, String encoding) { - this.algorithm = algorithm; - this.encoding = encoding; - } - - public String getAlgorithm() { - return algorithm; - } - - public String getVersion() { - return VERSION; - } - - public String computeSignature(String key, String data) { - try { - byte[] signData = sign(key.getBytes(encoding), data.getBytes(encoding)); - return BinaryUtil.toBase64String(signData); - } catch (UnsupportedEncodingException ex) { - throw new RuntimeException("Unsupported algorithm: " + encoding, ex); - } - } - - private byte[] sign(byte[] key, byte[] data) { - try { - // Because Mac.getInstance(String) calls a synchronized method, it - // could block on - // invoked concurrently, so use prototype pattern to improve perf. - if (macInstance == null) { - synchronized (LOCK) { - if (macInstance == null) { - macInstance = Mac.getInstance(algorithm); - } - } - } - - Mac mac = null; - try { - mac = (Mac) macInstance.clone(); - } catch (CloneNotSupportedException e) { - // If it is not clonable, create a new one. - mac = Mac.getInstance(algorithm); - } - mac.init(new SecretKeySpec(key, algorithm)); - return mac.doFinal(data); - } catch (NoSuchAlgorithmException ex) { - throw new RuntimeException("Unsupported algorithm: " + algorithm, ex); - } catch (InvalidKeyException ex) { - throw new RuntimeException("Invalid key: " + key, ex); - } - } -} \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java b/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java index 611601a2..d34da79d 100644 --- a/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java +++ b/src/main/java/com/aliyun/oss/common/auth/ServiceSignature.java @@ -19,6 +19,11 @@ package com.aliyun.oss.common.auth; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + /** * The interface to compute the signature of the data. */ @@ -52,12 +57,41 @@ public abstract class ServiceSignature { /** * * Creates the default ServiceSignature instance which is - * {@link HmacSignature}. + * {@link HmacSHA1Signature}. * * @return The default ServiceSignature instance */ public static ServiceSignature create() { - return new HmacSignature(); + return new HmacSHA1Signature(); + } + + protected byte[] sign(byte[] key, byte[] data, Mac macInstance, Object lock, String algorithm) { + try { + // Because Mac.getInstance(String) calls a synchronized method, it + // could block on + // invoked concurrently, so use prototype pattern to improve perf. + if (macInstance == null) { + synchronized (lock) { + if (macInstance == null) { + macInstance = Mac.getInstance(algorithm); + } + } + } + + Mac mac; + try { + mac = (Mac) macInstance.clone(); + } catch (CloneNotSupportedException e) { + // If it is not clonable, create a new one. + mac = Mac.getInstance(algorithm); + } + mac.init(new SecretKeySpec(key, algorithm)); + return mac.doFinal(data); + } catch (NoSuchAlgorithmException ex) { + throw new RuntimeException("Unsupported algorithm: " + algorithm, ex); + } catch (InvalidKeyException ex) { + throw new RuntimeException("Invalid key: " + key, ex); + } } } \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/internal/SignParameters.java b/src/main/java/com/aliyun/oss/internal/SignParameters.java index b3772289..705b2935 100644 --- a/src/main/java/com/aliyun/oss/internal/SignParameters.java +++ b/src/main/java/com/aliyun/oss/internal/SignParameters.java @@ -22,6 +22,8 @@ public class SignParameters { public static final String AUTHORIZATION_PREFIX_V2 = "OSS2 "; + public static final String AUTHORIZATION_V2 = "OSS2"; + public static final String AUTHORIZATION_ACCESS_KEY_ID = "AccessKeyId"; public static final String AUTHORIZATION_ADDITIONAL_HEADERS = "AdditionalHeaders"; diff --git a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java index 533627c9..fb2357dd 100644 --- a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java +++ b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java @@ -3,11 +3,10 @@ import com.aliyun.oss.ClientConfiguration; import com.aliyun.oss.HttpMethod; import com.aliyun.oss.common.auth.Credentials; -import com.aliyun.oss.common.auth.HmacSignature; +import com.aliyun.oss.common.auth.HmacSHA256Signature; import com.aliyun.oss.common.comm.RequestMessage; import com.aliyun.oss.common.utils.HttpHeaders; import com.aliyun.oss.common.utils.HttpUtil; -import com.aliyun.oss.common.utils.StringUtils; import com.aliyun.oss.model.GeneratePresignedUrlRequest; import java.net.URI; @@ -193,22 +192,20 @@ public static String buildSignedURL(GeneratePresignedUrlRequest request, Credent } String canonicalResource = "/" + ((bucketName != null) ? bucketName : "") + ((key != null ? "/" + key : "")); + requestMessage.addParameter(OSS_SIGNATURE_VERSION, SignParameters.AUTHORIZATION_V2); + requestMessage.addParameter(OSS_EXPIRES, expires); + requestMessage.addParameter(OSS_ACCESS_KEY_ID_PARAM, accessId); Set rawAdditionalHeaderNames = buildRawAdditionalHeaderNames(request.getHeaders().keySet(), request.getAdditionalHeaderNames()); - Set sortedAdditionalHeaderNames = buildSortedAdditionalHeaderNames(request.getHeaders().keySet(), request.getAdditionalHeaderNames()); String canonicalString = buildCanonicalString(method.toString(), canonicalResource, requestMessage, rawAdditionalHeaderNames); - String signature = new HmacSignature(HmacSignature.SHA256).computeSignature(accessKey, canonicalString); + String signature = new HmacSHA256Signature().computeSignature(accessKey, canonicalString); Map params = new LinkedHashMap(); - params.put(OSS_SIGNATURE_VERSION, "OSS2"); - params.put(OSS_EXPIRES, expires); - params.put(OSS_ACCESS_KEY_ID_PARAM, accessId); - params.put(OSS_ADDITIONAL_HEADERS, StringUtils.join(";", sortedAdditionalHeaderNames)); params.put(OSS_SIGNATURE, signature); params.putAll(requestMessage.getParameters()); String queryString = HttpUtil.paramToQueryString(params, DEFAULT_CHARSET_NAME); - /* Compse HTTP request uri. */ + /* Compose HTTP request uri. */ String url = requestMessage.getEndpoint().toString(); if (!url.endsWith("/")) { url += "/"; @@ -269,7 +266,7 @@ public static String uriEncoding(String uri) { public static String buildSignature(String secretAccessKey, String httpMethod, String resourcePath, RequestMessage request) { String canonicalString = buildCanonicalString(httpMethod, resourcePath, request, buildRawAdditionalHeaderNames(request.getOriginalRequest().getHeaders().keySet(), request.getOriginalRequest().getAdditionalHeaderNames())); - return new HmacSignature(HmacSignature.SHA256).computeSignature(secretAccessKey, canonicalString); + return new HmacSHA256Signature().computeSignature(secretAccessKey, canonicalString); } } diff --git a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java index 1b7dcb20..03fb7ed2 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java @@ -1,19 +1,15 @@ package com.aliyun.oss.integrationtests; import com.aliyun.oss.ClientBuilderConfiguration; -import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.common.utils.StringUtils; import com.aliyun.oss.internal.SignParameters; -import com.aliyun.oss.internal.SignV2Utils; import com.aliyun.oss.model.GeneratePresignedUrlRequest; import com.aliyun.oss.model.PutObjectRequest; import junit.framework.Assert; import org.junit.Test; import java.io.File; -import java.io.IOException; import java.net.URI; import java.net.URL; import java.util.*; @@ -23,27 +19,21 @@ public class SignTest { - private static final String endpoint = "http://oss-cn-hangzhou.aliyuncs.com"; - - private static final String accessKeyID = "LTAI2pSNlDGMkFrB"; - - private static final String accessKeySecret = "VWEukXofmBnajjymqMvYVwG2LdFN4B"; - - private static final String bucket = "test-sign"; - @Test public void testSignV2() { String key = "test-sign-V2"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); conf.setSignatureVersion(SignParameters.AUTH_V2); - OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyID, accessKeySecret, conf); + OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); + long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); + String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; ossClient.createBucket(bucket); String filePath = null; try { filePath = genFixedLengthFile(1 * 1024 * 1024); //1MB PutObjectRequest request = new PutObjectRequest(bucket, key, new File(filePath)); - request.addHeader("x-oss-head1", "31232"); + request.addHeader("x-oss-head1", "test1"); request.addHeader("abc", "4fdfsd"); request.addHeader("ZAbc", "4fde324fsd"); request.addHeader("XYZ", "4fde324fsd"); @@ -53,7 +43,7 @@ public void testSignV2() { request.addParameter("param1", "value1"); ossClient.putObject(request); - } catch (IOException e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } finally { if (filePath != null) { @@ -67,39 +57,58 @@ public void testGenerateSignedV2URL() { String key = "test-sign-v2-url"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); conf.setSignatureVersion(SignParameters.AUTH_V2); - OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyID, accessKeySecret, conf); + OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); Date expiration = new Date(new Date().getTime() + 1000 * 60 *10); - GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, key); - request.setExpiration(expiration); - URL url = null; + long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); + String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; + + ossClient.createBucket(bucket); + URL url; + String filePath; - Set abc = new HashSet(); - List ls = new LinkedList(abc); - StringUtils.join(";", ls); try { - URI endpointURI = new URI(endpoint); - String head1 = "test1"; - String head2 = "B\rTest!"; - - request.addUserMetadata("user1", "ddd"); - request.addHeader(head1, "aaa"); - request.addHeader("atest", "bbb"); - request.addHeader(head2, "ccc"); - request.addAdditionalHeaderName(head1); - request.addAdditionalHeaderName(head2); - request.addQueryParameter("queryParam1", "value1"); + filePath = genFixedLengthFile(1 * 1024 * 1024); //1MB + PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, new File(filePath)); + + ossClient.putObject(putObjectRequest); + + URI endpointURI = new URI(TestConfig.OSS_TEST_ENDPOINT); + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, key); + request.setExpiration(expiration); url = ossClient.generatePresignedUrl(request); StringBuilder expectedUrlPrefix = new StringBuilder(); + expectedUrlPrefix.append(endpointURI.getScheme()).append("://").append(bucket).append(".").append(endpointURI.getHost()).append("/") - .append(key).append("?x-oss-signature-version=OSS2").append("&x-oss-expires=").append(Long.toString(expiration.getTime() / 1000)) - .append("&x-oss-access-key-id=").append(accessKeyID).append("&x-oss-additional-headers=").append(SignV2Utils.uriEncoding(head2.toLowerCase() + ";" + head1.toLowerCase())).append("&x-oss-signature"); + .append(key).append("?x-oss-"); Assert.assertTrue(url.toString().startsWith(expectedUrlPrefix.toString())); - } catch (Exception e) { Assert.fail(e.getMessage()); } + } + @Test + public void testSwitchSignatureVersion() { + String key = "test-switch-signature-version"; + ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); + conf.setSignatureVersion(SignParameters.AUTH_V2); + OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); + long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); + String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; + ossClient.createBucket(bucket); + String filePath; + + try { + filePath = genFixedLengthFile(1 * 1024 * 1024); //1MB + + ossClient.putObject(bucket, key, new File(filePath)); + + ossClient.switchSignatureVersion(SignParameters.AUTH_V1); + + ossClient.putObject(bucket, key, new File(filePath)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } } } From b4019c15965b630b3dd98dc1367d661b1ef7ff4d Mon Sep 17 00:00:00 2001 From: shallwewu Date: Mon, 27 Aug 2018 15:03:06 +0800 Subject: [PATCH 08/18] head object api --- src/main/java/com/aliyun/oss/OSS.java | 22 ++++++++ src/main/java/com/aliyun/oss/OSSClient.java | 10 ++++ .../oss/internal/OSSObjectOperation.java | 5 +- .../aliyun/oss/internal/ResponseParsers.java | 13 +++++ .../com/aliyun/oss/model/ObjectMetadata.java | 16 ++++++ .../oss/integrationtests/HeadObjectTest.java | 56 +++++++++++++++++++ 6 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/aliyun/oss/integrationtests/HeadObjectTest.java diff --git a/src/main/java/com/aliyun/oss/OSS.java b/src/main/java/com/aliyun/oss/OSS.java index bf0a7a62..9cddb941 100644 --- a/src/main/java/com/aliyun/oss/OSS.java +++ b/src/main/java/com/aliyun/oss/OSS.java @@ -714,6 +714,28 @@ public SimplifiedObjectMeta getSimplifiedObjectMeta(GenericRequest genericReques */ public SelectObjectMetadata createSelectObjectMetadata(CreateSelectObjectMetadataRequest createSelectObjectMetadataRequest) throws OSSException, ClientException; + /** + * Gets all the head data of {@link OSSObject}. + * + * @param bucketName + * Bucket name. + * @param key + * Object key. + * + * @return The {@link ObjectMetadata} instance. + */ + public ObjectMetadata headObject(String bucketName, String key) throws OSSException, ClientException; + + /** + * Gets all the head data of {@link OSSObject}. + * + * @param headObjectRequest + * A {@link HeadObjectRequest} instance which specifies the + * bucket name and object key, and some constraint information can be set. + * @return The {@link ObjectMetadata} instance. + */ + public ObjectMetadata headObject(HeadObjectRequest headObjectRequest) throws OSSException, ClientException; + /** * Append the data to the appendable object specified in * {@link AppendObjectRequest}. It's not applicable to normal OSS object. diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index 3eae2957..a9b2be14 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -582,6 +582,16 @@ public ObjectMetadata getObjectMetadata(GenericRequest genericRequest) throws OS return objectOperation.getObjectMetadata(genericRequest); } + @Override + public ObjectMetadata headObject(String bucketName, String key) throws OSSException, ClientException { + return this.headObject(new HeadObjectRequest(bucketName, key)); + } + + @Override + public ObjectMetadata headObject(HeadObjectRequest headObjectRequest) throws OSSException, ClientException { + return objectOperation.headObject(headObjectRequest); + } + @Override public AppendObjectResult appendObject(AppendObjectRequest appendObjectRequest) throws OSSException, ClientException { diff --git a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java index 8967bf48..8f43094d 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java @@ -62,6 +62,7 @@ import static com.aliyun.oss.internal.ResponseParsers.putObjectProcessReponseParser; import static com.aliyun.oss.internal.ResponseParsers.getSimplifiedObjectMetaResponseParser; import static com.aliyun.oss.internal.ResponseParsers.getSymbolicLinkResponseParser; +import static com.aliyun.oss.internal.ResponseParsers.headObjectResponseParser; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -528,7 +529,7 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsReque /** * Check if the object key exists under the specified bucket. */ - public void headObject(HeadObjectRequest headObjectRequest) throws OSSException, ClientException { + public ObjectMetadata headObject(HeadObjectRequest headObjectRequest) throws OSSException, ClientException { assertParameterNotNull(headObjectRequest, "headObjectRequest"); @@ -554,7 +555,7 @@ public void headObject(HeadObjectRequest headObjectRequest) throws OSSException, .setMethod(HttpMethod.HEAD).setBucket(bucketName).setKey(key).setHeaders(headers) .setOriginalRequest(headObjectRequest).build(); - doOperation(request, emptyResponseParser, bucketName, key); + return doOperation(request, headObjectResponseParser, bucketName, key); } public void setObjectAcl(SetObjectAclRequest setObjectAclRequest) throws OSSException, ClientException { diff --git a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java index 5a39b6b2..3cb712ed 100644 --- a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java +++ b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java @@ -151,6 +151,7 @@ public final class ResponseParsers { public static final GetSimplifiedObjectMetaResponseParser getSimplifiedObjectMetaResponseParser = new GetSimplifiedObjectMetaResponseParser(); public static final RestoreObjectResponseParser restoreObjectResponseParser = new RestoreObjectResponseParser(); public static final ProcessObjectResponseParser processObjectResponseParser = new ProcessObjectResponseParser(); + public static final HeadObjectResponseParser headObjectResponseParser = new HeadObjectResponseParser(); public static final CompleteMultipartUploadResponseParser completeMultipartUploadResponseParser = new CompleteMultipartUploadResponseParser(); public static final CompleteMultipartUploadProcessResponseParser completeMultipartUploadProcessResponseParser = new CompleteMultipartUploadProcessResponseParser(); @@ -734,6 +735,18 @@ public ObjectMetadata parse(ResponseMessage response) throws ResponseParseExcept } + public static final class HeadObjectResponseParser implements ResponseParser { + + @Override + public ObjectMetadata parse(ResponseMessage response) throws ResponseParseException { + try { + return parseObjectMetadata(response.getHeaders()); + } finally { + safeCloseResponse(response); + } + } + } + public static final class CopyObjectResponseParser implements ResponseParser { @Override diff --git a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java index 75d8d0a1..589b29c9 100644 --- a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java +++ b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java @@ -19,6 +19,7 @@ package com.aliyun.oss.model; +import java.math.BigInteger; import java.text.ParseException; import java.util.Collections; import java.util.Date; @@ -343,6 +344,21 @@ public String getRequestId() { return (String) metadata.get(OSSHeaders.OSS_HEADER_REQUEST_ID); } + /** + * Gets the service crc. + * + * @return service crc. + */ + public Long getServiceCRC() { + String strSrvCrc = (String) metadata.get(OSSHeaders.OSS_HASH_CRC64_ECMA); + + if (strSrvCrc != null) { + BigInteger bi = new BigInteger(strSrvCrc); + return bi.longValue(); + } + return null; + } + /** * Gets the object's storage class, which is "standard", "IA" or "Archive". * diff --git a/src/test/java/com/aliyun/oss/integrationtests/HeadObjectTest.java b/src/test/java/com/aliyun/oss/integrationtests/HeadObjectTest.java new file mode 100644 index 00000000..f93fdfb7 --- /dev/null +++ b/src/test/java/com/aliyun/oss/integrationtests/HeadObjectTest.java @@ -0,0 +1,56 @@ +package com.aliyun.oss.integrationtests; + +import com.aliyun.oss.model.HeadObjectRequest; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectRequest; +import com.aliyun.oss.model.PutObjectResult; +import junit.framework.Assert; +import org.junit.Test; + +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +import static com.aliyun.oss.integrationtests.TestUtils.genFixedLengthInputStream; + +public class HeadObjectTest extends TestBase { + + @Test + public void testHeadObject() { + final String key = "head-object"; + final long inputStreanLength = 1024; + + try { + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, + genFixedLengthInputStream(inputStreanLength), null); + PutObjectResult putObjectResult = ossClient.putObject(putObjectRequest); + Assert.assertEquals(putObjectResult.getRequestId().length(), REQUEST_ID_LEN); + + HeadObjectRequest headObjectRequest = new HeadObjectRequest(bucketName, key); + List matchingETags = new LinkedList(); + matchingETags.add(putObjectResult.getETag()); + headObjectRequest.setMatchingETagConstraints(matchingETags); + ObjectMetadata o = ossClient.headObject(headObjectRequest); + Assert.assertEquals(o.getETag(), putObjectResult.getETag()); + + headObjectRequest = new HeadObjectRequest(bucketName, key); + List nonmatchingEtags = new LinkedList(); + nonmatchingEtags.add("nonmatching"); + headObjectRequest.setNonmatchingETagConstraints(nonmatchingEtags); + o = ossClient.headObject(headObjectRequest); + Assert.assertEquals(o.getETag(), putObjectResult.getETag()); + + headObjectRequest = new HeadObjectRequest(bucketName, key); + headObjectRequest.setModifiedSinceConstraint(new Date(System.currentTimeMillis() - 3600 * 1000)); + o = ossClient.headObject(headObjectRequest); + Assert.assertEquals(o.getETag(), putObjectResult.getETag()); + + headObjectRequest = new HeadObjectRequest(bucketName, key); + headObjectRequest.setUnmodifiedSinceConstraint(new Date(System.currentTimeMillis() + 3600 * 1000)); + o = ossClient.headObject(headObjectRequest); + Assert.assertEquals(o.getETag(), putObjectResult.getETag()); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } +} From 148dca0a19888a1b795e53986ea66c4de71c0220 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Tue, 28 Aug 2018 11:28:30 +0800 Subject: [PATCH 09/18] support download/upload file crc64 --- .../oss/internal/OSSDownloadOperation.java | 58 +++++++++++++++++++ .../oss/internal/OSSMultipartOperation.java | 2 +- .../oss/internal/OSSObjectOperation.java | 2 +- .../oss/internal/OSSUploadOperation.java | 40 +++++++++++++ .../com/aliyun/oss/model/ObjectMetadata.java | 2 +- .../oss/integrationtests/CRCChecksumTest.java | 23 +++++++- 6 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java index dce8c350..6fc71187 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java @@ -48,6 +48,9 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import com.aliyun.oss.InconsistentException; +import com.aliyun.oss.common.utils.CRC64; +import com.aliyun.oss.common.utils.IOUtils; import com.aliyun.oss.event.ProgressEventType; import com.aliyun.oss.event.ProgressListener; import com.aliyun.oss.event.ProgressPublisher; @@ -58,6 +61,7 @@ import com.aliyun.oss.model.OSSObject; import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.SimplifiedObjectMeta; +import com.aliyun.oss.model.HeadObjectRequest; /** * OSSDownloadOperation @@ -254,11 +258,34 @@ public void setException(Exception exception) { this.exception = exception; } + public Long getClientCRC() { return clientCRC; } + + public void setClientCRC(Long clientCRC) { this.clientCRC = clientCRC; } + + public Long getServerCRC() { + return serverCRC; + } + + public void setServerCRC(Long serverCRC) { + this.serverCRC = serverCRC; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } private int number; // part number, starting from 1. private long start; // start index in the part. private long end; // end index in the part. private boolean failed; // flag of part upload failure. private Exception exception; // Exception during part upload. + private Long clientCRC; // client crc of this part + private Long serverCRC; // server crc of this file + + private long length; } static class DownloadResult { @@ -354,6 +381,19 @@ private DownloadFileResult downloadFileWithCheckpoint(DownloadFileRequest downlo } } + // check crc64 + if(objectOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { + Long clientCRC = calcObjectCRCFromParts(downloadResult.getPartResults()); + Long serverCRC = downloadResult.getPartResults().get(0).getServerCRC(); + try { + OSSUtils.checkChecksum(clientCRC, serverCRC, downloadResult.getObjectMetadata().getRequestId()); + } catch (Exception e) { + ProgressPublisher.publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); + throw new InconsistentException(clientCRC, serverCRC, downloadResult.getObjectMetadata().getRequestId()); + } + } + + // Publish the complete status. ProgressPublisher.publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT); @@ -398,6 +438,18 @@ public static void createFixedFile(String filePath, long length) throws IOExcept } } + private static Long calcObjectCRCFromParts(List partResults) { + long crc = 0; + + for (PartResult partResult : partResults) { + if (partResult.getClientCRC() == null || partResult.getLength() <= 0) { + return null; + } + crc = CRC64.combine(crc, partResult.getClientCRC(), partResult.getLength()); + } + return new Long(crc); + } + private DownloadResult download(DownloadCheckPoint downloadCheckPoint, DownloadFileRequest downloadFileRequest) throws Throwable { DownloadResult downloadResult = new DownloadResult(); @@ -509,6 +561,12 @@ public PartResult call() throws Exception { output.write(buffer, 0, bytesRead); } + if (objectOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { + Long clientCRC = IOUtils.getCRCValue(content); + tr.setClientCRC(clientCRC); + tr.setServerCRC(objectMetadata.getServerCRC()); + tr.setLength(objectMetadata.getContentLength()); + } downloadCheckPoint.update(partIndex, true); if (downloadFileRequest.isEnableCheckpoint()) { downloadCheckPoint.dump(downloadFileRequest.getCheckpointFile()); diff --git a/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java b/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java index 5a8cfab9..0d57a81a 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java @@ -178,8 +178,8 @@ public int compare(PartETag p1, PartETag p2) { reponseHandlers); } - result.setClientCRC(calcObjectCRCFromParts(completeMultipartUploadRequest.getPartETags())); if (getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { + result.setClientCRC(calcObjectCRCFromParts(completeMultipartUploadRequest.getPartETags())); OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId()); } diff --git a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java index 8f43094d..f7b8f4e7 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java @@ -527,7 +527,7 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsReque } /** - * Check if the object key exists under the specified bucket. + * Get head information. */ public ObjectMetadata headObject(HeadObjectRequest headObjectRequest) throws OSSException, ClientException { diff --git a/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java b/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java index 4a924db0..925c038f 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java @@ -43,6 +43,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import com.aliyun.oss.InconsistentException; +import com.aliyun.oss.common.utils.CRC64; import com.aliyun.oss.event.ProgressEventType; import com.aliyun.oss.event.ProgressListener; import com.aliyun.oss.event.ProgressPublisher; @@ -260,11 +262,20 @@ public void setException(Exception exception) { this.exception = exception; } + public Long getPartCRC() { + return partCRC; + } + + public void setPartCRC(Long partCRC) { + this.partCRC = partCRC; + } + private int number; // part number private long offset; // offset in the file private long length; // part size private boolean failed; // part upload failure flag private Exception exception; // part upload exception + private Long partCRC; } public OSSUploadOperation(OSSMultipartOperation multipartOperation) { @@ -340,6 +351,18 @@ private UploadFileResult uploadFileWithCheckpoint(UploadFileRequest uploadFileRe CompleteMultipartUploadResult multipartUploadResult = complete(uploadCheckPoint, uploadFileRequest); uploadFileResult.setMultipartUploadResult(multipartUploadResult); + // check crc64 + if (multipartOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { + Long clientCRC = calcObjectCRCFromParts(partResults); + multipartUploadResult.setClientCRC(clientCRC); + try { + OSSUtils.checkChecksum(clientCRC, multipartUploadResult.getServerCRC(), multipartUploadResult.getRequestId()); + } catch (Exception e) { + ProgressPublisher.publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); + throw new InconsistentException(clientCRC, multipartUploadResult.getServerCRC(), multipartUploadResult.getRequestId()); + } + } + // The checkpoint is enabled and upload the checkpoint file. if (uploadFileRequest.isEnableCheckpoint()) { remove(uploadFileRequest.getCheckpointFile()); @@ -348,6 +371,18 @@ private UploadFileResult uploadFileWithCheckpoint(UploadFileRequest uploadFileRe return uploadFileResult; } + private static Long calcObjectCRCFromParts(List partResults) { + long crc = 0; + + for (PartResult partResult : partResults) { + if (partResult.getPartCRC() == null || partResult.getLength() <= 0) { + return null; + } + crc = CRC64.combine(crc, partResult.getPartCRC(), partResult.getLength()); + } + return new Long(crc); + } + private void prepare(UploadCheckPoint uploadCheckPoint, UploadFileRequest uploadFileRequest) { uploadCheckPoint.magic = UploadCheckPoint.UPLOAD_MAGIC; uploadCheckPoint.uploadFile = uploadFileRequest.getUploadFile(); @@ -462,6 +497,11 @@ public PartResult call() throws Exception { UploadPartResult uploadPartResult = multipartOperation.uploadPart(uploadPartRequest); + if(multipartOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { + OSSUtils.checkChecksum(uploadPartResult.getClientCRC(), uploadPartResult.getServerCRC(), uploadPartResult.getRequestId()); + tr.setPartCRC(uploadPartResult.getClientCRC()); + tr.setLength(uploadPartResult.getPartSize()); + } PartETag partETag = new PartETag(uploadPartResult.getPartNumber(), uploadPartResult.getETag()); uploadCheckPoint.update(partIndex, partETag, true); if (uploadFileRequest.isEnableCheckpoint()) { diff --git a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java index 589b29c9..d858165a 100644 --- a/src/main/java/com/aliyun/oss/model/ObjectMetadata.java +++ b/src/main/java/com/aliyun/oss/model/ObjectMetadata.java @@ -349,7 +349,7 @@ public String getRequestId() { * * @return service crc. */ - public Long getServiceCRC() { + public Long getServerCRC() { String strSrvCrc = (String) metadata.get(OSSHeaders.OSS_HASH_CRC64_ECMA); if (strSrvCrc != null) { diff --git a/src/test/java/com/aliyun/oss/integrationtests/CRCChecksumTest.java b/src/test/java/com/aliyun/oss/integrationtests/CRCChecksumTest.java index 44d13300..f36345c4 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/CRCChecksumTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/CRCChecksumTest.java @@ -48,6 +48,8 @@ import com.aliyun.oss.model.PutObjectResult; import com.aliyun.oss.model.UploadPartRequest; import com.aliyun.oss.model.UploadPartResult; +import com.aliyun.oss.model.UploadFileRequest; +import com.aliyun.oss.model.UploadFileResult; public class CRCChecksumTest extends TestBase { @@ -186,7 +188,26 @@ public void testMutilUploadCRC() { Assert.fail(e.getMessage()); } } - + + @Test + public void testUploadFileCRC() { + final String key = "upload-file-crc"; + + try { + File file = createSampleFile(key, 1024 * 500); + + UploadFileRequest uploadFileRequest = new UploadFileRequest(bucketName, key); + uploadFileRequest.setUploadFile(file.getAbsolutePath()); + uploadFileRequest.setTaskNum(10); + uploadFileRequest.setEnableCheckpoint(true); + + UploadFileResult uploadRes = ossClient.uploadFile(uploadFileRequest); + Assert.assertEquals(uploadRes.getMultipartUploadResult().getClientCRC(), uploadRes.getMultipartUploadResult().getServerCRC()); + } catch (Throwable e) { + Assert.fail(e.getMessage()); + } + } + @Test public void testGetObjectCRC() { String key = "get-object-crc"; From 67dc2b4525cc28c12269da1b10ce616c7939b763 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Mon, 8 Oct 2018 20:23:37 +0800 Subject: [PATCH 10/18] fix checkpoint for crc check --- .../oss/internal/OSSDownloadOperation.java | 23 ++++++++++++++++--- .../oss/internal/OSSMultipartOperation.java | 2 +- .../oss/internal/OSSUploadOperation.java | 12 +++++++++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java index 6fc71187..6df8eccd 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java @@ -205,13 +205,16 @@ public int hashCode() { result = prime * result + (isCompleted ? 1231 : 1237); result = prime * result + (int) (end ^ (end >>> 32)); result = prime * result + (int) (start ^ (start >>> 32)); + result = prime * result + (int) (crc ^ (crc >>> 32)); return result; } public int index; // part index (starting from 0). public long start; // start index; public long end; // end index; - public boolean isCompleted; // flag of part download finished or not. + public boolean isCompleted; // flag of part download finished or not; + public long length; // length of part + public long crc; // part crc. } static class PartResult { @@ -222,6 +225,14 @@ public PartResult(int number, long start, long end) { this.end = end; } + public PartResult(int number, long start, long end, long length, long clientCRC) { + this.number = number; + this.start = start; + this.end = end; + this.length = length; + this.clientCRC = clientCRC; + } + public long getStart() { return start; } @@ -374,7 +385,11 @@ private DownloadFileResult downloadFileWithCheckpoint(DownloadFileRequest downlo // Concurrently download parts. DownloadResult downloadResult = download(downloadCheckPoint, downloadFileRequest); + Long serverCRC = null; for (PartResult partResult : downloadResult.getPartResults()) { + if (partResult.getServerCRC() != null) { + serverCRC = partResult.getServerCRC(); + } if (partResult.isFailed()) { ProgressPublisher.publishProgress(listener, ProgressEventType.TRANSFER_PART_FAILED_EVENT); throw partResult.getException(); @@ -384,7 +399,6 @@ private DownloadFileResult downloadFileWithCheckpoint(DownloadFileRequest downlo // check crc64 if(objectOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { Long clientCRC = calcObjectCRCFromParts(downloadResult.getPartResults()); - Long serverCRC = downloadResult.getPartResults().get(0).getServerCRC(); try { OSSUtils.checkChecksum(clientCRC, serverCRC, downloadResult.getObjectMetadata().getRequestId()); } catch (Exception e) { @@ -480,7 +494,8 @@ private DownloadResult download(DownloadCheckPoint downloadCheckPoint, DownloadF tasks.add(task); } else { taskResults.add(new PartResult(i + 1, downloadCheckPoint.downloadParts.get(i).start, - downloadCheckPoint.downloadParts.get(i).end)); + downloadCheckPoint.downloadParts.get(i).end, downloadCheckPoint.downloadParts.get(i).length, + downloadCheckPoint.downloadParts.get(i).crc)); } } service.shutdown(); @@ -566,6 +581,8 @@ public PartResult call() throws Exception { tr.setClientCRC(clientCRC); tr.setServerCRC(objectMetadata.getServerCRC()); tr.setLength(objectMetadata.getContentLength()); + downloadPart.length = objectMetadata.getContentLength(); + downloadPart.crc = clientCRC; } downloadCheckPoint.update(partIndex, true); if (downloadFileRequest.isEnableCheckpoint()) { diff --git a/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java b/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java index 0d57a81a..5a8cfab9 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java @@ -178,8 +178,8 @@ public int compare(PartETag p1, PartETag p2) { reponseHandlers); } + result.setClientCRC(calcObjectCRCFromParts(completeMultipartUploadRequest.getPartETags())); if (getInnerClient().getClientConfiguration().isCrcCheckEnabled()) { - result.setClientCRC(calcObjectCRCFromParts(completeMultipartUploadRequest.getPartETags())); OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId()); } diff --git a/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java b/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java index 925c038f..d6a9dbfe 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java @@ -205,6 +205,7 @@ public int hashCode() { result = prime * result + number; result = prime * result + (int) (offset ^ (offset >>> 32)); result = prime * result + (int) (size ^ (size >>> 32)); + result = prime * result + (int) (crc ^ (crc >>> 32)); return result; } @@ -212,6 +213,7 @@ public int hashCode() { public long offset; // the offset in the file public long size; // part size public boolean isCompleted; // upload completeness flag. + public long crc; //part crc } static class PartResult { @@ -222,6 +224,13 @@ public PartResult(int number, long offset, long length) { this.length = length; } + public PartResult(int number, long offset, long length, long partCRC) { + this.number = number; + this.offset = offset; + this.length = length; + this.partCRC = partCRC; + } + public int getNumber() { return number; } @@ -432,7 +441,7 @@ private ArrayList upload(UploadCheckPoint uploadCheckPoint, UploadFi multipartOperation, listener))); } else { taskResults.add(new PartResult(i + 1, uploadCheckPoint.uploadParts.get(i).offset, - uploadCheckPoint.uploadParts.get(i).size)); + uploadCheckPoint.uploadParts.get(i).size, uploadCheckPoint.uploadParts.get(i).crc)); } } service.shutdown(); @@ -501,6 +510,7 @@ public PartResult call() throws Exception { OSSUtils.checkChecksum(uploadPartResult.getClientCRC(), uploadPartResult.getServerCRC(), uploadPartResult.getRequestId()); tr.setPartCRC(uploadPartResult.getClientCRC()); tr.setLength(uploadPartResult.getPartSize()); + uploadPart.crc = uploadPartResult.getClientCRC(); } PartETag partETag = new PartETag(uploadPartResult.getPartNumber(), uploadPartResult.getETag()); uploadCheckPoint.update(partIndex, partETag, true); From 5008dd27489be9677dad9ea65406ca498b105024 Mon Sep 17 00:00:00 2001 From: shallwewu Date: Thu, 1 Nov 2018 11:13:57 +0800 Subject: [PATCH 11/18] fix ut --- .../oss/integrationtests/BucketProcesTest.java | 14 +++++++++++--- .../oss/integrationtests/PostPolicyTest.java | 3 ++- .../aliyun/oss/integrationtests/PutObjectTest.java | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/aliyun/oss/integrationtests/BucketProcesTest.java b/src/test/java/com/aliyun/oss/integrationtests/BucketProcesTest.java index ac9699d3..81b44d7f 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/BucketProcesTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/BucketProcesTest.java @@ -30,6 +30,8 @@ import com.aliyun.oss.model.ImageProcess; import com.aliyun.oss.model.SetBucketProcessRequest; +import static com.aliyun.oss.integrationtests.TestUtils.waitForCacheExpiration; + public class BucketProcesTest extends TestBase { @Test @@ -49,7 +51,9 @@ public void testBucketImageProcessConf() { ImageProcess imageProcess = new ImageProcess("Img", true, "jpg,png", "/,-"); SetBucketProcessRequest request = new SetBucketProcessRequest(bucketName, imageProcess); ossClient.setBucketProcess(request); - + + waitForCacheExpiration(2); + // get 1 bucketProcess = ossClient.getBucketProcess(new GenericRequest(bucketName)); Assert.assertEquals(bucketProcess.getImageProcess().getCompliedHost(), "Img"); @@ -64,7 +68,9 @@ public void testBucketImageProcessConf() { imageProcess = new ImageProcess("Both", false, "gif", "-"); request = new SetBucketProcessRequest(bucketName, imageProcess); ossClient.setBucketProcess(request); - + + waitForCacheExpiration(2); + // get 2 bucketProcess = ossClient.getBucketProcess(new GenericRequest(bucketName)); Assert.assertEquals(bucketProcess.getImageProcess().getCompliedHost(), "Both"); @@ -79,7 +85,9 @@ public void testBucketImageProcessConf() { imageProcess = new ImageProcess("Img", true, "*", "/", true); request = new SetBucketProcessRequest(bucketName, imageProcess); ossClient.setBucketProcess(request); - + + waitForCacheExpiration(2); + // get 3 bucketProcess = ossClient.getBucketProcess(new GenericRequest(bucketName)); Assert.assertEquals(bucketProcess.getImageProcess().getCompliedHost(), "Img"); diff --git a/src/test/java/com/aliyun/oss/integrationtests/PostPolicyTest.java b/src/test/java/com/aliyun/oss/integrationtests/PostPolicyTest.java index f605395d..7e85cb90 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/PostPolicyTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/PostPolicyTest.java @@ -65,7 +65,8 @@ public void testGenPostPolicy() { String actualPostSignature = ossClient.calculatePostSignature(actualPostPolicy); // It has something to do with the local time Assert.assertTrue((actualPostSignature.equals("88kD3wGu1W5isVAdWSG765DRPKY=") || - actualPostSignature.equals("KbUYorFeyyqxntffsNlrRcV50Ds="))); + actualPostSignature.equals("KbUYorFeyyqxntffsNlrRcV50Ds=") || + actualPostSignature.equals("oGVOEb+wFKpZMgMqI0NNfSldA6s="))); } catch (Exception e) { Assert.fail(e.getMessage()); } diff --git a/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java b/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java index c4776823..854bd25a 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java @@ -348,7 +348,7 @@ public void testPutObjectByUrlSignature() throws Exception { final String key = "put-object-by-urlsignature"; final String metaKey0 = "author"; final String metaValue0 = "aliy"; - final String expirationString = "Sun, 12 Apr 2018 12:00:00 GMT"; + final String expirationString = "Sun, 12 Apr 2020 12:00:00 GMT"; final long inputStreamLength = 128 * 1024; //128KB GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key, HttpMethod.PUT); From 1aebf332237469c6a310a660773164bb5ee9323c Mon Sep 17 00:00:00 2001 From: shallwewu Date: Wed, 21 Nov 2018 19:46:28 +0800 Subject: [PATCH 12/18] fix Chinese character encoding in sign v2 --- .../com/aliyun/oss/internal/SignV2Utils.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java index fb2357dd..599a131b 100644 --- a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java +++ b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java @@ -1,6 +1,7 @@ package com.aliyun.oss.internal; import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.ClientException; import com.aliyun.oss.HttpMethod; import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.HmacSHA256Signature; @@ -9,6 +10,7 @@ import com.aliyun.oss.common.utils.HttpUtil; import com.aliyun.oss.model.GeneratePresignedUrlRequest; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.util.*; @@ -229,7 +231,7 @@ private static String buildCanonicalizedResource(String resourcePath, Map= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') - || (c >= '0' && c <= '9') || c == '_' || c == '-' - || c == '~' || c == '.') { - result += c; - } else if (c == '/') { - result += "%2F"; - } else { - String temp = Integer.toHexString((int)c); - - if (temp.length() < 2) { - temp = "0" + temp; + try { + for (char c : uri.toCharArray()) { + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') + || (c >= '0' && c <= '9') || c == '_' || c == '-' + || c == '~' || c == '.') { + result += c; + } else if (c == '/') { + result += "%2F"; + } else { + byte[] b; + b = Character.toString(c).getBytes("utf-8"); + + for (int i = 0; i < b.length; i++) { + int k = b[i]; + + if (k < 0) { + k += 256; + } + result += "%" + Integer.toHexString(k).toUpperCase(); + } } - result += "%" + temp.toUpperCase(); } + } catch (UnsupportedEncodingException e) { + throw new ClientException(e); } - return result; } From 0cd38f7c2d8abe3e0c6dc09f58f70373e82b35c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Tue, 27 Nov 2018 10:44:07 +0800 Subject: [PATCH 13/18] refine --- src/main/java/com/aliyun/oss/OSSClient.java | 2 +- .../com/aliyun/oss/internal/SignUtils.java | 2 +- .../com/aliyun/oss/internal/SignV2Utils.java | 6 +++--- .../oss/integrationtests/CMKIDTest.java | 6 +++--- .../aliyun/oss/integrationtests/SignTest.java | 19 +++++++++++++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index a9b2be14..c0e6cef0 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -302,7 +302,7 @@ public void switchCredentials(Credentials creds) { @Override public void switchSignatureVersion(String signatureVersion) { - if (signatureVersion != SignParameters.AUTH_V1 && signatureVersion != SignParameters.AUTH_V2) { + if (!SignParameters.AUTH_V1.equals(signatureVersion) && !SignParameters.AUTH_V2.equals(signatureVersion)) { throw new IllegalArgumentException("unsupported signature version" + signatureVersion); } diff --git a/src/main/java/com/aliyun/oss/internal/SignUtils.java b/src/main/java/com/aliyun/oss/internal/SignUtils.java index 43852c8c..235fb1ca 100644 --- a/src/main/java/com/aliyun/oss/internal/SignUtils.java +++ b/src/main/java/com/aliyun/oss/internal/SignUtils.java @@ -52,7 +52,7 @@ public static String buildCanonicalString(String method, String resourcePath, Re String expires) { StringBuilder canonicalString = new StringBuilder(); - canonicalString.append(method + SignParameters.NEW_LINE); + canonicalString.append(method).append(SignParameters.NEW_LINE); Map headers = request.getHeaders(); TreeMap headersToSign = new TreeMap(); diff --git a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java index 599a131b..1a8d57f6 100644 --- a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java +++ b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java @@ -30,10 +30,10 @@ public static String composeRequestAuthorization(String accessKeyId, String sign Set ts = buildSortedAdditionalHeaderNames(request.getOriginalRequest().getHeaders().keySet(), request.getOriginalRequest().getAdditionalHeaderNames()); - sb.append(AUTHORIZATION_PREFIX_V2 + AUTHORIZATION_ACCESS_KEY_ID + ":" + accessKeyId + ", "); + sb.append(AUTHORIZATION_PREFIX_V2 + AUTHORIZATION_ACCESS_KEY_ID).append(":").append(accessKeyId).append(", "); if (ts != null && !ts.isEmpty()) { - sb.append(AUTHORIZATION_ADDITIONAL_HEADERS + ":"); + sb.append(AUTHORIZATION_ADDITIONAL_HEADERS).append(":"); String separator = ""; @@ -77,7 +77,7 @@ private static Set buildRawAdditionalHeaderNames(Set headerNames public static String buildCanonicalString(String method, String resourcePath, RequestMessage request, Set additionalHeaderNames) { StringBuilder canonicalString = new StringBuilder(); - canonicalString.append(method + SignParameters.NEW_LINE); + canonicalString.append(method).append(SignParameters.NEW_LINE); Map headers = request.getHeaders(); TreeMap fixedHeadersToSign = new TreeMap(); TreeMap canonicalizedOssHeadersToSign = new TreeMap(); diff --git a/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java index 4a56e4d5..2985078a 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/CMKIDTest.java @@ -276,9 +276,9 @@ public void testAppendObjectWithCMKID() { BufferedReader reader = new BufferedReader(new InputStreamReader(o.getObjectContent())); StringBuilder sb = new StringBuilder(); - while (true) { - String line = reader.readLine(); - if (line == null) break; + String line; + + while ((line = reader.readLine()) != null) { sb.append(line); } diff --git a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java index 03fb7ed2..8387a891 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java @@ -83,6 +83,10 @@ public void testGenerateSignedV2URL() { .append(key).append("?x-oss-"); Assert.assertTrue(url.toString().startsWith(expectedUrlPrefix.toString())); + Assert.assertTrue(url.toString().contains("x-oss-signature")); + Assert.assertTrue(url.toString().contains("x-oss-signature-version")); + Assert.assertTrue(url.toString().contains("x-oss-expires")); + Assert.assertTrue(url.toString().contains("x-oss-access-key-id")); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -111,4 +115,19 @@ public void testSwitchSignatureVersion() { Assert.fail(e.getMessage()); } } + + @Test + public void testSwitchUnsupportedSignatureVersion() { + String unsupportedSignatureVersion = "unsupported"; + ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); + conf.setSignatureVersion(SignParameters.AUTH_V2); + OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); + try { + ossClient.switchSignatureVersion(unsupportedSignatureVersion); + Assert.fail("switch unsupported signature version should not be successful"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().startsWith("unsupported signature version")); + } + + } } From 84c3f13e91d37d6bb73f72d8902a5d311b6fdcd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Tue, 27 Nov 2018 14:50:27 +0800 Subject: [PATCH 14/18] fix MD5 signature --- src/main/java/com/aliyun/oss/internal/SignUtils.java | 2 +- src/main/java/com/aliyun/oss/internal/SignV2Utils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/SignUtils.java b/src/main/java/com/aliyun/oss/internal/SignUtils.java index 235fb1ca..6dacb9e5 100644 --- a/src/main/java/com/aliyun/oss/internal/SignUtils.java +++ b/src/main/java/com/aliyun/oss/internal/SignUtils.java @@ -143,7 +143,7 @@ public static String buildSignedURL(GeneratePresignedUrlRequest request, Credent if (request.getContentType() != null && !request.getContentType().trim().equals("")) { requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType()); } - if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) { + if (request.getContentMD5() != null && !request.getContentMD5().trim().equals("")) { requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5()); } for (Map.Entry h : request.getUserMetadata().entrySet()) { diff --git a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java index 1a8d57f6..06a64e9a 100644 --- a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java +++ b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java @@ -167,7 +167,7 @@ public static String buildSignedURL(GeneratePresignedUrlRequest request, Credent if (request.getContentType() != null && !request.getContentType().trim().equals("")) { requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType()); } - if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) { + if (request.getContentMD5() != null && !request.getContentMD5().trim().equals("")) { requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5()); } for (Map.Entry h : request.getUserMetadata().entrySet()) { From c90f0dd7dce4ecff815b5231403e76b7b9dc2b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Mon, 3 Dec 2018 14:30:45 +0800 Subject: [PATCH 15/18] refine sample --- src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java | 1 - src/samples/ImageSample.java | 2 +- src/samples/MultipartUploadSample.java | 2 +- src/samples/UploadSample.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java b/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java index 9860d29a..b3e7a533 100644 --- a/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java +++ b/src/main/java/com/aliyun/oss/internal/LiveChannelOperation.java @@ -319,7 +319,6 @@ public OSSObject getVodPlaylist(GetVodPlaylistRequest getVodPlaylistRequest) thr RequestMessage request = new OSSRequestMessageBuilder(getInnerClient()).setEndpoint(getEndpoint()) .setMethod(HttpMethod.GET).setBucket(bucketName).setKey(liveChannelName).setParameters(parameters) - .setInputStream(new ByteArrayInputStream(new byte[0])).setInputSize(0) .setOriginalRequest(getVodPlaylistRequest).build(); return doOperation(request, new GetObjectResponseParser(bucketName, liveChannelName), bucketName, liveChannelName, true); diff --git a/src/samples/ImageSample.java b/src/samples/ImageSample.java index e61c4a49..54a9f93c 100644 --- a/src/samples/ImageSample.java +++ b/src/samples/ImageSample.java @@ -98,7 +98,7 @@ public static void main(String[] args) throws IOException { } catch (OSSException oe) { System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason."); - System.out.println("Error Message: " + oe.getErrorCode()); + System.out.println("Error Message: " + oe.getErrorMessage()); System.out.println("Error Code: " + oe.getErrorCode()); System.out.println("Request ID: " + oe.getRequestId()); System.out.println("Host ID: " + oe.getHostId()); diff --git a/src/samples/MultipartUploadSample.java b/src/samples/MultipartUploadSample.java index 2aa0d15a..19bc224d 100644 --- a/src/samples/MultipartUploadSample.java +++ b/src/samples/MultipartUploadSample.java @@ -150,7 +150,7 @@ public static void main(String[] args) throws IOException { } catch (OSSException oe) { System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason."); - System.out.println("Error Message: " + oe.getErrorCode()); + System.out.println("Error Message: " + oe.getErrorMessage()); System.out.println("Error Code: " + oe.getErrorCode()); System.out.println("Request ID: " + oe.getRequestId()); System.out.println("Host ID: " + oe.getHostId()); diff --git a/src/samples/UploadSample.java b/src/samples/UploadSample.java index a00bc223..cb74effa 100644 --- a/src/samples/UploadSample.java +++ b/src/samples/UploadSample.java @@ -66,7 +66,7 @@ public static void main(String[] args) throws IOException { } catch (OSSException oe) { System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason."); - System.out.println("Error Message: " + oe.getErrorCode()); + System.out.println("Error Message: " + oe.getErrorMessage()); System.out.println("Error Code: " + oe.getErrorCode()); System.out.println("Request ID: " + oe.getRequestId()); System.out.println("Host ID: " + oe.getHostId()); From c7a972ff0e21fe69b6f61227886f893b906c3a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Tue, 4 Dec 2018 10:05:24 +0800 Subject: [PATCH 16/18] support csv content type --- .../com/aliyun/oss/internal/Mimetypes.java | 2 ++ .../oss/integrationtests/PutObjectTest.java | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/aliyun/oss/internal/Mimetypes.java b/src/main/java/com/aliyun/oss/internal/Mimetypes.java index 2fea255a..864bba8f 100644 --- a/src/main/java/com/aliyun/oss/internal/Mimetypes.java +++ b/src/main/java/com/aliyun/oss/internal/Mimetypes.java @@ -90,6 +90,8 @@ public void loadMimetypes(InputStream is) throws IOException { } } } + + extensionToMimetypeMap.put("csv", "text/csv"); } public String getMimetype(String fileName) { diff --git a/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java b/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java index 854bd25a..de825a1f 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/PutObjectTest.java @@ -494,5 +494,25 @@ public void testIncorrentSignature() throws Exception { } } } - + + @Test + public void testPutCSVTypeFile() throws Exception { + final String key = "1.csv"; + final int instreamLength = 128 * 1024; + + InputStream instream = null; + try { + instream = genFixedLengthInputStream(instreamLength); + ossClient.putObject(bucketName, key, instream); + + OSSObject o = ossClient.getObject(bucketName, key); + Assert.assertEquals(o.getObjectMetadata().getContentType(), "text/csv"); + } catch (Exception e) { + Assert.fail(e.getMessage()); + + if (instream != null) { + instream.close(); + } + } + } } From 009bd46341a1693094f54b8504a2caa1433a41bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Wed, 12 Dec 2018 11:16:09 +0800 Subject: [PATCH 17/18] version --- pom.xml | 2 +- src/main/java/com/aliyun/oss/internal/Mimetypes.java | 2 -- src/main/resources/mime.types | 4 +++- src/main/resources/versioninfo.properties | 2 +- .../java/com/aliyun/oss/common/utils/VersionUtilTest.java | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 51f7dabc..c65cebcc 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ com.aliyun.oss aliyun-sdk-oss - 3.3.0-SNAPSHOT + 3.4.0 jar Aliyun OSS SDK for Java The Aliyun OSS SDK for Java used for accessing Aliyun Object Storage Service diff --git a/src/main/java/com/aliyun/oss/internal/Mimetypes.java b/src/main/java/com/aliyun/oss/internal/Mimetypes.java index 864bba8f..2fea255a 100644 --- a/src/main/java/com/aliyun/oss/internal/Mimetypes.java +++ b/src/main/java/com/aliyun/oss/internal/Mimetypes.java @@ -90,8 +90,6 @@ public void loadMimetypes(InputStream is) throws IOException { } } } - - extensionToMimetypeMap.put("csv", "text/csv"); } public String getMimetype(String fileName) { diff --git a/src/main/resources/mime.types b/src/main/resources/mime.types index 90e65da9..e7fb0c7e 100644 --- a/src/main/resources/mime.types +++ b/src/main/resources/mime.types @@ -225,4 +225,6 @@ xml application/xml xsl application/xml xslt application/xslt+xml xul application/vnd.mozilla.xul+xml -webp image/webp \ No newline at end of file +webp image/webp +csv text/csv +json application/json \ No newline at end of file diff --git a/src/main/resources/versioninfo.properties b/src/main/resources/versioninfo.properties index 14103c3f..e6640dcd 100644 --- a/src/main/resources/versioninfo.properties +++ b/src/main/resources/versioninfo.properties @@ -1 +1 @@ -version=3.3.0 +version=3.4.0 diff --git a/src/test/java/com/aliyun/oss/common/utils/VersionUtilTest.java b/src/test/java/com/aliyun/oss/common/utils/VersionUtilTest.java index de85df33..edac1f5b 100644 --- a/src/test/java/com/aliyun/oss/common/utils/VersionUtilTest.java +++ b/src/test/java/com/aliyun/oss/common/utils/VersionUtilTest.java @@ -28,7 +28,7 @@ public class VersionUtilTest { @Test public void testGetDefaultUserAgent() { String userAgent = VersionInfoUtils.getDefaultUserAgent(); - assertTrue(userAgent.startsWith("aliyun-sdk-java/3.3.0(")); + assertTrue(userAgent.startsWith("aliyun-sdk-java/3.4.0(")); assertEquals(userAgent.split("/").length, 4); assertEquals(userAgent.split(";").length, 2); assertEquals(userAgent.split("\\(").length, 2); From 485b7267abda04cfdd0e11f5e9b0519921670009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=A7=E8=B4=A4?= Date: Thu, 13 Dec 2018 16:12:37 +0800 Subject: [PATCH 18/18] fix v2 generate signed url --- .../com/aliyun/oss/ClientConfiguration.java | 10 +- src/main/java/com/aliyun/oss/OSS.java | 3 +- src/main/java/com/aliyun/oss/OSSClient.java | 13 +- .../aliyun/oss/common/comm/SignVersion.java | 6 + .../com/aliyun/oss/internal/OSSOperation.java | 15 +- .../aliyun/oss/internal/OSSRequestSigner.java | 7 +- .../aliyun/oss/internal/SignParameters.java | 4 - .../com/aliyun/oss/internal/SignV2Utils.java | 44 ++++-- .../oss/integrationtests/GetObjectTest.java | 2 +- .../aliyun/oss/integrationtests/SignTest.java | 133 ++++++++++++++++-- 10 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 src/main/java/com/aliyun/oss/common/comm/SignVersion.java diff --git a/src/main/java/com/aliyun/oss/ClientConfiguration.java b/src/main/java/com/aliyun/oss/ClientConfiguration.java index 513b6541..d4780b24 100644 --- a/src/main/java/com/aliyun/oss/ClientConfiguration.java +++ b/src/main/java/com/aliyun/oss/ClientConfiguration.java @@ -31,10 +31,10 @@ import com.aliyun.oss.common.auth.RequestSigner; import com.aliyun.oss.common.comm.IdleConnectionReaper; import com.aliyun.oss.common.comm.Protocol; +import com.aliyun.oss.common.comm.SignVersion; import com.aliyun.oss.common.utils.ResourceManager; import com.aliyun.oss.common.utils.VersionInfoUtils; import com.aliyun.oss.internal.OSSConstants; -import com.aliyun.oss.internal.SignParameters; /** * Client configurations for accessing to OSS services. @@ -60,7 +60,7 @@ public class ClientConfiguration { public static final String DEFAULT_CNAME_EXCLUDE_LIST = "aliyuncs.com,aliyun-inc.com,aliyun.com"; - public static final String DEFAULT_SIGNATURE_VERSION = SignParameters.AUTH_V1; + public static final SignVersion DEFAULT_SIGNATURE_VERSION = SignVersion.V1; protected String userAgent = DEFAULT_USER_AGENT; protected int maxErrorRetry = DEFAULT_MAX_RETRIES; @@ -97,7 +97,7 @@ public class ClientConfiguration { protected List signerHandlers = new LinkedList(); - protected String signatureVersion = DEFAULT_SIGNATURE_VERSION; + protected SignVersion signatureVersion = DEFAULT_SIGNATURE_VERSION; /** * Gets the user agent string. @@ -662,7 +662,7 @@ public void setSignerHandlers(List signerHandlers) { * * @return signature version */ - public String getSignatureVersion() { + public SignVersion getSignatureVersion() { return signatureVersion; } @@ -671,7 +671,7 @@ public String getSignatureVersion() { * * @param signatureVersion */ - public void setSignatureVersion(String signatureVersion) { + public void setSignatureVersion(SignVersion signatureVersion) { this.signatureVersion = signatureVersion; } } diff --git a/src/main/java/com/aliyun/oss/OSS.java b/src/main/java/com/aliyun/oss/OSS.java index 9cddb941..007c3e2d 100644 --- a/src/main/java/com/aliyun/oss/OSS.java +++ b/src/main/java/com/aliyun/oss/OSS.java @@ -28,6 +28,7 @@ import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.comm.ResponseMessage; +import com.aliyun.oss.common.comm.SignVersion; import com.aliyun.oss.model.*; import com.aliyun.oss.model.SetBucketCORSRequest.CORSRule; @@ -58,7 +59,7 @@ public interface OSS { * @param signatureVersion * the signature version to switch to。 */ - public void switchSignatureVersion(String signatureVersion); + public void switchSignatureVersion(SignVersion signatureVersion); /** * Shuts down the OSS instance (release all resources) The OSS instance is diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index c0e6cef0..f8a4c8a4 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -47,10 +47,7 @@ import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.common.auth.ServiceSignature; -import com.aliyun.oss.common.comm.DefaultServiceClient; -import com.aliyun.oss.common.comm.ResponseMessage; -import com.aliyun.oss.common.comm.ServiceClient; -import com.aliyun.oss.common.comm.TimeoutServiceClient; +import com.aliyun.oss.common.comm.*; import com.aliyun.oss.common.utils.BinaryUtil; import com.aliyun.oss.common.utils.DateUtil; import com.aliyun.oss.internal.*; @@ -301,9 +298,9 @@ public void switchCredentials(Credentials creds) { } @Override - public void switchSignatureVersion(String signatureVersion) { - if (!SignParameters.AUTH_V1.equals(signatureVersion) && !SignParameters.AUTH_V2.equals(signatureVersion)) { - throw new IllegalArgumentException("unsupported signature version" + signatureVersion); + public void switchSignatureVersion(SignVersion signatureVersion) { + if (signatureVersion == null) { + throw new IllegalArgumentException("signatureVersion should not be null."); } this.getClientConfiguration().setSignatureVersion(signatureVersion); @@ -700,7 +697,7 @@ public URL generatePresignedUrl(GeneratePresignedUrlRequest request) throws Clie } String url; - if (serviceClient.getClientConfiguration().getSignatureVersion() != null && serviceClient.getClientConfiguration().getSignatureVersion().equals(SignParameters.AUTH_V2)) { + if (serviceClient.getClientConfiguration().getSignatureVersion() != null && serviceClient.getClientConfiguration().getSignatureVersion() == SignVersion.V2) { url = SignV2Utils.buildSignedURL(request, credsProvider.getCredentials(), serviceClient.getClientConfiguration(), endpoint); } else { url = SignUtils.buildSignedURL(request, credsProvider.getCredentials(), serviceClient.getClientConfiguration(), endpoint); diff --git a/src/main/java/com/aliyun/oss/common/comm/SignVersion.java b/src/main/java/com/aliyun/oss/common/comm/SignVersion.java new file mode 100644 index 00000000..31d0b0f9 --- /dev/null +++ b/src/main/java/com/aliyun/oss/common/comm/SignVersion.java @@ -0,0 +1,6 @@ +package com.aliyun.oss.common.comm; + +public enum SignVersion { + V1, + V2 +} diff --git a/src/main/java/com/aliyun/oss/internal/OSSOperation.java b/src/main/java/com/aliyun/oss/internal/OSSOperation.java index 55c91307..939c6e14 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSOperation.java @@ -33,18 +33,7 @@ import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.RequestSigner; -import com.aliyun.oss.common.comm.ExecutionContext; -import com.aliyun.oss.common.comm.NoRetryStrategy; -import com.aliyun.oss.common.comm.RequestChecksumHanlder; -import com.aliyun.oss.common.comm.RequestHandler; -import com.aliyun.oss.common.comm.RequestMessage; -import com.aliyun.oss.common.comm.RequestProgressHanlder; -import com.aliyun.oss.common.comm.ResponseChecksumHandler; -import com.aliyun.oss.common.comm.ResponseHandler; -import com.aliyun.oss.common.comm.ResponseMessage; -import com.aliyun.oss.common.comm.ResponseProgressHandler; -import com.aliyun.oss.common.comm.RetryStrategy; -import com.aliyun.oss.common.comm.ServiceClient; +import com.aliyun.oss.common.comm.*; import com.aliyun.oss.common.parser.ResponseParseException; import com.aliyun.oss.common.parser.ResponseParser; import com.aliyun.oss.common.utils.ExceptionFactory; @@ -165,7 +154,7 @@ protected T doOperation(RequestMessage request, ResponseParser parser, St } } - private static RequestSigner createSigner(HttpMethod method, String bucketName, String key, Credentials creds, String signatureVersion) { + private static RequestSigner createSigner(HttpMethod method, String bucketName, String key, Credentials creds, SignVersion signatureVersion) { String resourcePath = "/" + ((bucketName != null) ? bucketName + "/" : "") + ((key != null ? key : "")); return new OSSRequestSigner(method.toString(), resourcePath, creds, signatureVersion); diff --git a/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java b/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java index 97411194..67c8be85 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java +++ b/src/main/java/com/aliyun/oss/internal/OSSRequestSigner.java @@ -23,6 +23,7 @@ import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.RequestSigner; import com.aliyun.oss.common.comm.RequestMessage; +import com.aliyun.oss.common.comm.SignVersion; public class OSSRequestSigner implements RequestSigner { @@ -33,9 +34,9 @@ public class OSSRequestSigner implements RequestSigner { private Credentials creds; - private String signatureVersion; + private SignVersion signatureVersion; - public OSSRequestSigner(String httpMethod, String resourcePath, Credentials creds, String signatureVersion) { + public OSSRequestSigner(String httpMethod, String resourcePath, Credentials creds, SignVersion signatureVersion) { this.httpMethod = httpMethod; this.resourcePath = resourcePath; this.creds = creds; @@ -50,7 +51,7 @@ public void sign(RequestMessage request) throws ClientException { if (accessKeyId.length() > 0 && secretAccessKey.length() > 0) { String signature; - if (SignParameters.AUTH_V2.equals(signatureVersion)) { + if (signatureVersion == SignVersion.V2) { signature = SignV2Utils.buildSignature(secretAccessKey, httpMethod, resourcePath, request); request.addHeader(OSSHeaders.AUTHORIZATION, SignV2Utils.composeRequestAuthorization(accessKeyId,signature, request)); } else { diff --git a/src/main/java/com/aliyun/oss/internal/SignParameters.java b/src/main/java/com/aliyun/oss/internal/SignParameters.java index 705b2935..30fbb321 100644 --- a/src/main/java/com/aliyun/oss/internal/SignParameters.java +++ b/src/main/java/com/aliyun/oss/internal/SignParameters.java @@ -14,10 +14,6 @@ public class SignParameters { - public static final String AUTH_V1 = "auth-v1"; - - public static final String AUTH_V2 = "auth-v2"; - public static final String AUTHORIZATION_PREFIX = "OSS "; public static final String AUTHORIZATION_PREFIX_V2 = "OSS2 "; diff --git a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java index 06a64e9a..bcf7fb4e 100644 --- a/src/main/java/com/aliyun/oss/internal/SignV2Utils.java +++ b/src/main/java/com/aliyun/oss/internal/SignV2Utils.java @@ -27,30 +27,34 @@ public class SignV2Utils { public static String composeRequestAuthorization(String accessKeyId, String signature, RequestMessage request) { StringBuilder sb = new StringBuilder(); - Set ts = buildSortedAdditionalHeaderNames(request.getOriginalRequest().getHeaders().keySet(), - request.getOriginalRequest().getAdditionalHeaderNames()); - sb.append(AUTHORIZATION_PREFIX_V2 + AUTHORIZATION_ACCESS_KEY_ID).append(":").append(accessKeyId).append(", "); - if (ts != null && !ts.isEmpty()) { - sb.append(AUTHORIZATION_ADDITIONAL_HEADERS).append(":"); - - String separator = ""; + String additionHeaderNameStr = buildSortedAdditionalHeaderNameStr(request.getOriginalRequest().getHeaders().keySet(), + request.getOriginalRequest().getAdditionalHeaderNames()); - for (String header : ts) { - sb.append(separator); - sb.append(header.toLowerCase()); - separator = ";"; - } - sb.append(", "); + if (!additionHeaderNameStr.isEmpty()) { + sb.append(AUTHORIZATION_ADDITIONAL_HEADERS).append(":").append(additionHeaderNameStr).append(", "); } sb.append(AUTHORIZATION_SIGNATURE).append(":").append(signature); return sb.toString(); } - private static TreeSet buildSortedAdditionalHeaderNames(Set headerNames, Set additionalHeaderNames) { - TreeSet ts = new TreeSet(); + private static String buildSortedAdditionalHeaderNameStr(Set headerNames, Set additionalHeaderNames) { + Set ts = buildSortedAdditionalHeaderNames(headerNames, additionalHeaderNames); + StringBuilder sb = new StringBuilder(); + String separator = ""; + + for (String header : ts) { + sb.append(separator); + sb.append(header); + separator = ";"; + } + return sb.toString(); + } + + private static Set buildSortedAdditionalHeaderNames(Set headerNames, Set additionalHeaderNames) { + Set ts = new TreeSet(); if (headerNames != null && additionalHeaderNames != null) { for (String additionalHeaderName : additionalHeaderNames) { @@ -197,11 +201,21 @@ public static String buildSignedURL(GeneratePresignedUrlRequest request, Credent requestMessage.addParameter(OSS_SIGNATURE_VERSION, SignParameters.AUTHORIZATION_V2); requestMessage.addParameter(OSS_EXPIRES, expires); requestMessage.addParameter(OSS_ACCESS_KEY_ID_PARAM, accessId); + String additionalHeaderNameStr = buildSortedAdditionalHeaderNameStr(requestMessage.getHeaders().keySet(), + request.getAdditionalHeaderNames()); + + if (!additionalHeaderNameStr.isEmpty()) { + requestMessage.addParameter(OSS_ADDITIONAL_HEADERS, additionalHeaderNameStr); + } Set rawAdditionalHeaderNames = buildRawAdditionalHeaderNames(request.getHeaders().keySet(), request.getAdditionalHeaderNames()); String canonicalString = buildCanonicalString(method.toString(), canonicalResource, requestMessage, rawAdditionalHeaderNames); String signature = new HmacSHA256Signature().computeSignature(accessKey, canonicalString); Map params = new LinkedHashMap(); + + if (!additionalHeaderNameStr.isEmpty()) { + params.put(OSS_ADDITIONAL_HEADERS, additionalHeaderNameStr); + } params.put(OSS_SIGNATURE, signature); params.putAll(requestMessage.getParameters()); diff --git a/src/test/java/com/aliyun/oss/integrationtests/GetObjectTest.java b/src/test/java/com/aliyun/oss/integrationtests/GetObjectTest.java index fad8bd61..0ba8f063 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/GetObjectTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/GetObjectTest.java @@ -529,7 +529,7 @@ public void testGetObjectWithSpecialChars() { @Test public void testGetObjectByUrlsignature() { final String key = "put-object-by-urlsignature"; - final String expirationString = "Sun, 12 Apr 2018 12:00:00 GMT"; + final String expirationString = "Sun, 12 Apr 2020 12:00:00 GMT"; final long inputStreamLength = 128 * 1024; //128KB final long firstByte= inputStreamLength / 2; final long lastByte = inputStreamLength - 1; diff --git a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java index 8387a891..732dba23 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/SignTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/SignTest.java @@ -1,21 +1,30 @@ package com.aliyun.oss.integrationtests; import com.aliyun.oss.ClientBuilderConfiguration; +import com.aliyun.oss.HttpMethod; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.internal.SignParameters; +import com.aliyun.oss.common.comm.SignVersion; +import com.aliyun.oss.common.utils.DateUtil; +import com.aliyun.oss.common.utils.HttpHeaders; +import com.aliyun.oss.common.utils.IOUtils; import com.aliyun.oss.model.GeneratePresignedUrlRequest; +import com.aliyun.oss.model.OSSObject; import com.aliyun.oss.model.PutObjectRequest; import junit.framework.Assert; import org.junit.Test; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URL; import java.util.*; import static com.aliyun.oss.integrationtests.TestUtils.genFixedLengthFile; +import static com.aliyun.oss.integrationtests.TestUtils.genFixedLengthInputStream; import static com.aliyun.oss.integrationtests.TestUtils.removeFile; +import static com.aliyun.oss.internal.OSSConstants.DEFAULT_OBJECT_CONTENT_TYPE; public class SignTest { @@ -23,7 +32,7 @@ public class SignTest { public void testSignV2() { String key = "test-sign-V2"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); - conf.setSignatureVersion(SignParameters.AUTH_V2); + conf.setSignatureVersion(SignVersion.V2); OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; @@ -33,10 +42,10 @@ public void testSignV2() { try { filePath = genFixedLengthFile(1 * 1024 * 1024); //1MB PutObjectRequest request = new PutObjectRequest(bucket, key, new File(filePath)); - request.addHeader("x-oss-head1", "test1"); - request.addHeader("abc", "4fdfsd"); - request.addHeader("ZAbc", "4fde324fsd"); - request.addHeader("XYZ", "4fde324fsd"); + request.addHeader("x-oss-head1", "value"); + request.addHeader("abc", "value"); + request.addHeader("ZAbc", "value"); + request.addHeader("XYZ", "value"); request.addAdditionalHeaderName("ZAbc"); request.addAdditionalHeaderName("x-oss-head1"); request.addAdditionalHeaderName("abc"); @@ -56,7 +65,7 @@ public void testSignV2() { public void testGenerateSignedV2URL() { String key = "test-sign-v2-url"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); - conf.setSignatureVersion(SignParameters.AUTH_V2); + conf.setSignatureVersion(SignVersion.V2); OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); Date expiration = new Date(new Date().getTime() + 1000 * 60 *10); long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); @@ -96,7 +105,7 @@ public void testGenerateSignedV2URL() { public void testSwitchSignatureVersion() { String key = "test-switch-signature-version"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); - conf.setSignatureVersion(SignParameters.AUTH_V2); + conf.setSignatureVersion(SignVersion.V2); OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; @@ -108,7 +117,7 @@ public void testSwitchSignatureVersion() { ossClient.putObject(bucket, key, new File(filePath)); - ossClient.switchSignatureVersion(SignParameters.AUTH_V1); + ossClient.switchSignatureVersion(SignVersion.V1); ossClient.putObject(bucket, key, new File(filePath)); } catch (Exception e) { @@ -117,17 +126,111 @@ public void testSwitchSignatureVersion() { } @Test - public void testSwitchUnsupportedSignatureVersion() { - String unsupportedSignatureVersion = "unsupported"; + public void testPutObjectByUrlSignature() { + final String key = "put-object-by-urlSignature"; ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); - conf.setSignatureVersion(SignParameters.AUTH_V2); + conf.setSignatureVersion(SignVersion.V2); OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); + long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); + String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; + ossClient.createBucket(bucket); + + final String expirationString = "Sun, 12 Apr 2020 12:00:00 GMT"; + final long inputStreamLength = 128 * 1024; //128KB + + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, key, HttpMethod.PUT); try { - ossClient.switchSignatureVersion(unsupportedSignatureVersion); - Assert.fail("switch unsupported signature version should not be successful"); + Date expiration = DateUtil.parseRfc822Date(expirationString); + request.setExpiration(expiration); + request.setContentType(DEFAULT_OBJECT_CONTENT_TYPE); + request.addHeader("x-oss-head1", "value"); + request.addHeader("abc", "value"); + request.addHeader("ZAbc", "value"); + request.addHeader("XYZ", "value"); + request.addAdditionalHeaderName("ZAbc"); + request.addAdditionalHeaderName("x-oss-head1"); + request.addAdditionalHeaderName("abc"); + request.addQueryParameter("param1", "value1"); + URL signedUrl = ossClient.generatePresignedUrl(request); + + Map requestHeaders = new HashMap(); + + requestHeaders.put("x-oss-head1", "value"); + requestHeaders.put("abc", "value"); + requestHeaders.put("ZAbc", "value"); + requestHeaders.put("XYZ", "value"); + requestHeaders.put(HttpHeaders.CONTENT_TYPE, DEFAULT_OBJECT_CONTENT_TYPE); + InputStream instream = genFixedLengthInputStream(inputStreamLength); + // Using url signature & chunked encoding to upload specified inputstream. + ossClient.putObject(signedUrl, instream, -1, requestHeaders, true); + OSSObject o = ossClient.getObject(bucket, key); + Assert.assertEquals(key, o.getKey()); + Assert.assertEquals(inputStreamLength, o.getObjectMetadata().getContentLength()); } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().startsWith("unsupported signature version")); + Assert.fail(ex.getMessage()); } + } + + @Test + public void testGetObjectByUrlsignature() { + final String key = "get-object-by-urlsignature"; + ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); + conf.setSignatureVersion(SignVersion.V2); + OSS ossClient = new OSSClientBuilder().build(TestConfig.OSS_TEST_ENDPOINT, TestConfig.OSS_TEST_ACCESS_KEY_ID, TestConfig.OSS_TEST_ACCESS_KEY_SECRET, conf); + long ticks = new Date().getTime() / 1000 + new Random().nextInt(5000); + String bucket = TestBase.BUCKET_NAME_PREFIX + ticks; + ossClient.createBucket(bucket); + + final String expirationString = "Sun, 12 Apr 2020 12:00:00 GMT"; + final long inputStreamLength = 128 * 1024; //128KB + final long firstByte= inputStreamLength / 2; + final long lastByte = inputStreamLength - 1; + try { + ossClient.putObject(bucket, key, genFixedLengthInputStream(inputStreamLength), null); + + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, key); + Date expiration = DateUtil.parseRfc822Date(expirationString); + request.setExpiration(expiration); + request.setContentType(DEFAULT_OBJECT_CONTENT_TYPE); + request.addHeader(HttpHeaders.RANGE, String.format("bytes=%d-%d", firstByte, lastByte)); + request.addHeader("x-oss-head1", "value"); + request.addHeader("abc", "value"); + request.addHeader("ZAbc", "value"); + request.addHeader("XYZ", "value"); + request.addAdditionalHeaderName("ZAbc"); + request.addAdditionalHeaderName("x-oss-head1"); + request.addAdditionalHeaderName("abc"); + request.addQueryParameter("param1", "value1"); + URL signedUrl = ossClient.generatePresignedUrl(request); + + Map requestHeaders = new HashMap(); + + requestHeaders.put("x-oss-head1", "value"); + requestHeaders.put("abc", "value"); + requestHeaders.put("ZAbc", "value"); + requestHeaders.put("XYZ", "value"); + requestHeaders.put(HttpHeaders.CONTENT_TYPE, DEFAULT_OBJECT_CONTENT_TYPE); + requestHeaders.put(HttpHeaders.RANGE, String.format("bytes=%d-%d", firstByte, lastByte)); + + OSSObject o = ossClient.getObject(signedUrl, requestHeaders); + + try { + int bytesRead = -1; + int totalBytes = 0; + byte[] buffer = new byte[4096]; + while ((bytesRead = o.getObjectContent().read(buffer)) != -1) { + totalBytes += bytesRead; + } + + Assert.assertEquals((lastByte - firstByte + 1), totalBytes); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } finally { + IOUtils.safeClose(o.getObjectContent()); + } + } catch (Exception ex) { + Assert.fail(ex.getMessage()); + } } }