From 5171fee314dd84a4e57fa4291d36b5221f85332e Mon Sep 17 00:00:00 2001 From: Dimuthu Wannipurage Date: Tue, 3 May 2022 17:43:00 -0400 Subject: [PATCH] Adding Swift connector --- agent/pom.xml | 21 ++- .../airavata/mft/core/ConnectorResolver.java | 6 + .../mft/core/MetadataCollectorResolver.java | 3 + pom.xml | 2 + scripts/build.sh | 2 +- transport/pom.xml | 1 + transport/scp-transport/pom.xml | 2 +- transport/swift-transport/pom.xml | 47 ++++++ .../swift/SwiftIncomingConnector.java | 148 +++++++++++++++++ .../swift/SwiftMetadataCollector.java | 156 ++++++++++++++++++ .../swift/SwiftOutgoingConnector.java | 149 +++++++++++++++++ 11 files changed, 527 insertions(+), 10 deletions(-) create mode 100644 transport/swift-transport/pom.xml create mode 100644 transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java create mode 100644 transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java create mode 100644 transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java diff --git a/agent/pom.xml b/agent/pom.xml index 7e3a9cd0..c0ab77d7 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -36,7 +36,7 @@ org.apache.airavata mft-scp-transport - 0.01-SNAPSHOT + ${project.version} org.apache.custos @@ -47,27 +47,27 @@ org.apache.airavata mft-local-transport - 0.01-SNAPSHOT + ${project.version} org.apache.airavata mft-s3-transport - 0.01-SNAPSHOT + ${project.version} org.apache.airavata mft-box-transport - 0.01-SNAPSHOT + ${project.version} org.apache.airavata mft-azure-transport - 0.01-SNAPSHOT + ${project.version} org.apache.airavata mft-gcp-transport - 0.01-SNAPSHOT + ${project.version} org.apache.airavata @@ -77,12 +77,17 @@ org.apache.airavata mft-ftp-transport - 0.01-SNAPSHOT + ${project.version} + + + org.apache.airavata + mft-swift-transport + ${project.version} org.apache.airavata mft-common-clients - 0.01-SNAPSHOT + ${project.version} com.orbitz.consul diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java index f061a31e..a3910a5c 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java @@ -70,6 +70,9 @@ public static Optional resolveIncomingChunkedConnector case "S3": className = "org.apache.airavata.mft.transport.s3.S3IncomingConnector"; break; + case "SWIFT": + className = "org.apache.airavata.mft.transport.swift.SwiftIncomingConnector"; + break; } if (className != null) { @@ -87,6 +90,9 @@ public static Optional resolveOutgoingChunkedConnector case "S3": className = "org.apache.airavata.mft.transport.s3.S3OutgoingConnector"; break; + case "SWIFT": + className = "org.apache.airavata.mft.transport.swift.SwiftOutgoingConnector"; + break; } if (className != null) { diff --git a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java index c9da7315..9ab820a4 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java @@ -51,6 +51,9 @@ public static Optional resolveMetadataCollector(String type) case "FTP": className = "org.apache.airavata.mft.transport.ftp.FTPMetadataCollector"; break; + case "SWIFT": + className = "org.apache.airavata.mft.transport.swift.SwiftMetadataCollector"; + break; } if (className != null) { diff --git a/pom.xml b/pom.xml index cb33e968..05735098 100755 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,8 @@ 0.27.0 2.5.1 1.1-SNAPSHOT + 2.5.0 + 2.6 diff --git a/scripts/build.sh b/scripts/build.sh index 42f30380..2d495f67 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -18,7 +18,7 @@ # under the License. cd ../ -#mvn clean install +mvn clean install rm -rf airavata-mft mkdir -p airavata-mft cp agent/target/MFT-Agent-0.01-bin.zip airavata-mft/ diff --git a/transport/pom.xml b/transport/pom.xml index d53cfcac..c537b24a 100755 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -41,6 +41,7 @@ gcp-transport ftp-transport dropbox-transport + swift-transport diff --git a/transport/scp-transport/pom.xml b/transport/scp-transport/pom.xml index 7680f543..eac4d435 100755 --- a/transport/scp-transport/pom.xml +++ b/transport/scp-transport/pom.xml @@ -51,7 +51,7 @@ commons-io commons-io - 2.6 + ${commons.io.version} diff --git a/transport/swift-transport/pom.xml b/transport/swift-transport/pom.xml new file mode 100644 index 00000000..b26faccc --- /dev/null +++ b/transport/swift-transport/pom.xml @@ -0,0 +1,47 @@ + + + + mft-transport + org.apache.airavata + 0.01-SNAPSHOT + + 4.0.0 + + mft-swift-transport + + + 11 + 11 + + + + + org.apache.airavata + mft-core + ${project.version} + + + org.apache.jclouds.api + openstack-swift + ${jclouds.version} + + + org.apache.jclouds.api + openstack-nova + ${jclouds.version} + + + org.apache.jclouds.api + openstack-keystone + ${jclouds.version} + + + commons-io + commons-io + ${commons.io.version} + + + + \ No newline at end of file diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java new file mode 100644 index 00000000..1e3a6bf9 --- /dev/null +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.swift; + +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.IncomingChunkedConnector; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.apache.commons.io.IOUtils; +import org.jclouds.ContextBuilder; +import org.jclouds.http.options.GetOptions; +import org.jclouds.openstack.keystone.auth.config.CredentialTypes; +import org.jclouds.openstack.keystone.config.KeystoneProperties; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.features.ObjectApi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.StandardCopyOption; +import java.util.Properties; + +public class SwiftIncomingConnector implements IncomingChunkedConnector { + + private static final Logger logger = LoggerFactory.getLogger(SwiftIncomingConnector.class); + + private GenericResource resource; + private SwiftApi swiftApi; + private ObjectApi objectApi; + + @Override + public void init(ConnectorConfig cc) throws Exception { + try (ResourceServiceClient resourceClient = ResourceServiceClientBuilder + .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) { + + resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setResourceId(cc.getResourceId()).build()); + } + + if (resource.getStorageCase() != GenericResource.StorageCase.SWIFTSTORAGE) { + logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), cc.getResourceId()); + throw new Exception("Invalid storage type specified for resource " + cc.getResourceId()); + } + + SwiftStorage swiftStorage = resource.getSwiftStorage(); + + SwiftSecret swiftSecret; + + try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient( + cc.getSecretServiceHost(), cc.getSecretServicePort())) { + + swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setSecretId(cc.getCredentialToken()).build()); + + String provider = "openstack-swift"; + + Properties overrides = new Properties(); + overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); + + String identity = null; + String credential = null; + switch (swiftSecret.getSecretCase()) { + case PASSWORDSECRET: + identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); + credential = swiftSecret.getPasswordSecret().getPassword(); + overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); + break; + case AUTHCREDENTIALSECRET: + identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); + credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); + break; + } + + swiftApi = ContextBuilder.newBuilder(provider) + .endpoint(swiftStorage.getEndpoint()) + .credentials(identity, credential) + .overrides(overrides) + .buildApi(SwiftApi.class); + + objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); + + } + } + + @Override + public void complete() throws Exception { + if (swiftApi != null) { + swiftApi.close(); + } + } + + @Override + public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception { + SwiftObject swiftObject = objectApi.get( + resource.getFile().getResourcePath(), + GetOptions.Builder.range(startByte, endByte)); + + InputStream inputStream = swiftObject.getPayload().openStream(); + + File targetFile = new File("/tmp/targetFile.tmp"); + + java.nio.file.Files.copy( + inputStream, + targetFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + + IOUtils.closeQuietly(inputStream); + } + + @Override + public InputStream downloadChunk(int chunkId, long startByte, long endByte) throws Exception { + + SwiftObject swiftObject = objectApi.get( + resource.getFile().getResourcePath(), + GetOptions.Builder.range(startByte, endByte)); + + return swiftObject.getPayload().openStream(); + } +} diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java new file mode 100644 index 00000000..2200b2b8 --- /dev/null +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.swift; + +import org.apache.airavata.mft.common.AuthToken; +import org.apache.airavata.mft.core.DirectoryResourceMetadata; +import org.apache.airavata.mft.core.FileResourceMetadata; +import org.apache.airavata.mft.core.api.MetadataCollector; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.jclouds.ContextBuilder; +import org.jclouds.openstack.keystone.auth.config.CredentialTypes; +import org.jclouds.openstack.keystone.config.KeystoneProperties; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.features.ObjectApi; + +import java.util.Properties; + +public class SwiftMetadataCollector implements MetadataCollector { + + private String resourceServiceHost; + private int resourceServicePort; + private String secretServiceHost; + private int secretServicePort; + boolean initialized = false; + + @Override + public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) { + this.resourceServiceHost = resourceServiceHost; + this.resourceServicePort = resourceServicePort; + this.secretServiceHost = secretServiceHost; + this.secretServicePort = secretServicePort; + this.initialized = true; + } + + private void checkInitialized() { + if (!initialized) { + throw new IllegalStateException("Swift Metadata Collector is not initialized"); + } + } + + private SwiftApi getSwiftApi(SwiftStorage swiftStorage, SwiftSecret swiftSecret) { + String provider = "openstack-swift"; + + Properties overrides = new Properties(); + overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); + + String identity = null; + String credential = null; + switch (swiftSecret.getSecretCase()) { + case PASSWORDSECRET: + identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); + credential = swiftSecret.getPasswordSecret().getPassword(); + overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); + break; + case AUTHCREDENTIALSECRET: + identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); + credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); + break; + } + + return ContextBuilder.newBuilder(provider) + .endpoint(swiftStorage.getEndpoint()) + .credentials(identity, credential) + .overrides(overrides) + .buildApi(SwiftApi.class); + } + + @Override + public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + checkInitialized(); + + ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); + GenericResource swiftResource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); + SwiftSecret swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + SwiftApi swiftApi = getSwiftApi(swiftResource.getSwiftStorage(), swiftSecret); + + ObjectApi objectApi = swiftApi.getObjectApi(swiftResource.getSwiftStorage().getRegion(), swiftResource.getSwiftStorage().getContainer()); + + SwiftObject swiftObject = objectApi.get(swiftResource.getFile().getResourcePath()); + + FileResourceMetadata metadata = new FileResourceMetadata(); + metadata.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength()); + metadata.setMd5sum(swiftObject.getETag()); + metadata.setUpdateTime(swiftObject.getLastModified().getTime()); + metadata.setCreatedTime(swiftObject.getLastModified().getTime()); + return metadata; + } + + @Override + public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("Method not implemented"); + } + + @Override + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + throw new UnsupportedOperationException("Method not implemented"); + } + + @Override + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("Method not implemented"); + } + + @Override + public Boolean isAvailable(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + checkInitialized(); + + ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); + GenericResource swiftResource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); + SwiftSecret swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + SwiftApi swiftApi = getSwiftApi(swiftResource.getSwiftStorage(), swiftSecret); + + ObjectApi objectApi = swiftApi.getObjectApi(swiftResource.getSwiftStorage().getRegion(), swiftResource.getSwiftStorage().getContainer()); + + SwiftObject swiftObject = objectApi.get(swiftResource.getFile().getResourcePath()); + + return swiftObject != null; + } + + @Override + public Boolean isAvailable(AuthToken authToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("Method not implemented"); + } +} diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java new file mode 100644 index 00000000..3734e8b5 --- /dev/null +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.swift; + +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.OutgoingChunkedConnector; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.jclouds.ContextBuilder; +import org.jclouds.io.payloads.InputStreamPayload; +import org.jclouds.openstack.keystone.auth.config.CredentialTypes; +import org.jclouds.openstack.keystone.config.KeystoneProperties; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.Segment; +import org.jclouds.openstack.swift.v1.features.ObjectApi; +import org.jclouds.openstack.swift.v1.features.StaticLargeObjectApi; +import org.jclouds.openstack.swift.v1.options.PutOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class SwiftOutgoingConnector implements OutgoingChunkedConnector { + + private static final Logger logger = LoggerFactory.getLogger(SwiftOutgoingConnector.class); + + private GenericResource resource; + private SwiftApi swiftApi; + private ObjectApi objectApi; + private StaticLargeObjectApi staticLargeObjectApi; + + private final Map segmentMap = new ConcurrentHashMap(); + + // Referring to https://www.mirantis.com/blog/large-objects-in-cloud-storages/ + + @Override + public void init(ConnectorConfig cc) throws Exception { + try (ResourceServiceClient resourceClient = ResourceServiceClientBuilder + .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) { + + resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setResourceId(cc.getResourceId()).build()); + } + + if (resource.getStorageCase() != GenericResource.StorageCase.SWIFTSTORAGE) { + logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), cc.getResourceId()); + throw new Exception("Invalid storage type specified for resource " + cc.getResourceId()); + } + + SwiftStorage swiftStorage = resource.getSwiftStorage(); + + SwiftSecret swiftSecret; + + try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient( + cc.getSecretServiceHost(), cc.getSecretServicePort())) { + + swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setSecretId(cc.getCredentialToken()).build()); + + String provider = "openstack-swift"; + + Properties overrides = new Properties(); + overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); + + String identity = null; + String credential = null; + switch (swiftSecret.getSecretCase()) { + case PASSWORDSECRET: + identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); + credential = swiftSecret.getPasswordSecret().getPassword(); + overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); + break; + case AUTHCREDENTIALSECRET: + identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); + credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); + overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); + break; + } + + swiftApi = ContextBuilder.newBuilder(provider) + .endpoint(swiftStorage.getEndpoint()) + .credentials(identity, credential) + .overrides(overrides) + .buildApi(SwiftApi.class); + + objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); + staticLargeObjectApi = swiftApi.getStaticLargeObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); + } + } + + @Override + public void complete() throws Exception { + + List segments = new ArrayList<>(); + for (int id = 0; id < segmentMap.size(); id ++) { + segments.add(segmentMap.get(id)); + } + + String etag = staticLargeObjectApi.replaceManifest(resource.getFile().getResourcePath(), + segments, new HashMap<>()); + + if (swiftApi != null) { + swiftApi.close(); + } + } + + @Override + public void uploadChunk(int chunkId, long startByte, long endByte, String uploadFile) throws Exception { + InputStream fis = new FileInputStream(uploadFile); + uploadChunk(chunkId, startByte, endByte, fis); + } + + @Override + public void uploadChunk(int chunkId, long startByte, long endByte, InputStream inputStream) throws Exception { + String etag = objectApi.put(resource.getFile().getResourcePath() + chunkId, new InputStreamPayload(inputStream)); + Segment segment = Segment.builder().etag(etag) + .path(resource.getFile().getResourcePath() + chunkId) + .sizeBytes(endByte - startByte).build(); + segmentMap.put(chunkId, segment); + } +}