Skip to content
Permalink
Browse files
Adding Swift connector
  • Loading branch information
DImuthuUpe committed May 3, 2022
1 parent 8838e55 commit 5171fee314dd84a4e57fa4291d36b5221f85332e
Showing 11 changed files with 527 additions and 10 deletions.
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-scp-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.custos</groupId>
@@ -47,27 +47,27 @@
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-local-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-s3-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-box-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-azure-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-gcp-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
@@ -77,12 +77,17 @@
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-ftp-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-swift-transport</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-common-clients</artifactId>
<version>0.01-SNAPSHOT</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.orbitz.consul</groupId>
@@ -70,6 +70,9 @@ public static Optional<IncomingChunkedConnector> resolveIncomingChunkedConnector
case "S3":
className = "org.apache.airavata.mft.transport.s3.S3IncomingConnector";
break;
case "SWIFT":
className = "org.apache.airavata.mft.transport.swift.SwiftIncomingConnector";
break;
}

if (className != null) {
@@ -87,6 +90,9 @@ public static Optional<OutgoingChunkedConnector> resolveOutgoingChunkedConnector
case "S3":
className = "org.apache.airavata.mft.transport.s3.S3OutgoingConnector";
break;
case "SWIFT":
className = "org.apache.airavata.mft.transport.swift.SwiftOutgoingConnector";
break;
}

if (className != null) {
@@ -51,6 +51,9 @@ public static Optional<MetadataCollector> resolveMetadataCollector(String type)
case "FTP":
className = "org.apache.airavata.mft.transport.ftp.FTPMetadataCollector";
break;
case "SWIFT":
className = "org.apache.airavata.mft.transport.swift.SwiftMetadataCollector";
break;
}

if (className != null) {
@@ -149,6 +149,8 @@
<sshj>0.27.0</sshj>
<mariadb.jdbc>2.5.1</mariadb.jdbc>
<custos.clients.version>1.1-SNAPSHOT</custos.clients.version>
<jclouds.version>2.5.0</jclouds.version>
<commons.io.version>2.6</commons.io.version>
</properties>

</project>
@@ -18,7 +18,7 @@
# under the License.

cd ../
#mvn clean install
mvn clean install
rm -rf airavata-mft
mkdir -p airavata-mft
cp agent/target/MFT-Agent-0.01-bin.zip airavata-mft/
@@ -41,6 +41,7 @@
<module>gcp-transport</module>
<module>ftp-transport</module>
<module>dropbox-transport</module>
<module>swift-transport</module>
</modules>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
<version>${commons.io.version}</version>
</dependency>
</dependencies>

@@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>mft-transport</artifactId>
<groupId>org.apache.airavata</groupId>
<version>0.01-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>mft-swift-transport</artifactId>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.api</groupId>
<artifactId>openstack-swift</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.api</groupId>
<artifactId>openstack-nova</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.api</groupId>
<artifactId>openstack-keystone</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons.io.version}</version>
</dependency>
</dependencies>

</project>
@@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.airavata.mft.transport.swift;

import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.IncomingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
import org.apache.airavata.mft.resource.client.ResourceServiceClient;
import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.common.GenericResource;
import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
import org.apache.airavata.mft.secret.client.SecretServiceClient;
import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.commons.io.IOUtils;
import org.jclouds.ContextBuilder;
import org.jclouds.http.options.GetOptions;
import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
import org.jclouds.openstack.keystone.config.KeystoneProperties;
import org.jclouds.openstack.swift.v1.SwiftApi;
import org.jclouds.openstack.swift.v1.domain.SwiftObject;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.InputStream;
import java.nio.file.StandardCopyOption;
import java.util.Properties;

public class SwiftIncomingConnector implements IncomingChunkedConnector {

private static final Logger logger = LoggerFactory.getLogger(SwiftIncomingConnector.class);

private GenericResource resource;
private SwiftApi swiftApi;
private ObjectApi objectApi;

@Override
public void init(ConnectorConfig cc) throws Exception {
try (ResourceServiceClient resourceClient = ResourceServiceClientBuilder
.buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) {

resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder()
.setAuthzToken(cc.getAuthToken())
.setResourceId(cc.getResourceId()).build());
}

if (resource.getStorageCase() != GenericResource.StorageCase.SWIFTSTORAGE) {
logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), cc.getResourceId());
throw new Exception("Invalid storage type specified for resource " + cc.getResourceId());
}

SwiftStorage swiftStorage = resource.getSwiftStorage();

SwiftSecret swiftSecret;

try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(
cc.getSecretServiceHost(), cc.getSecretServicePort())) {

swiftSecret = secretClient.swift().getSwiftSecret(SwiftSecretGetRequest.newBuilder()
.setAuthzToken(cc.getAuthToken())
.setSecretId(cc.getCredentialToken()).build());

String provider = "openstack-swift";

Properties overrides = new Properties();
overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + "");

String identity = null;
String credential = null;
switch (swiftSecret.getSecretCase()) {
case PASSWORDSECRET:
identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName();
credential = swiftSecret.getPasswordSecret().getPassword();
overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId());
overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
break;
case AUTHCREDENTIALSECRET:
identity = swiftSecret.getAuthCredentialSecret().getCredentialId();
credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret();
overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS);
break;
}

swiftApi = ContextBuilder.newBuilder(provider)
.endpoint(swiftStorage.getEndpoint())
.credentials(identity, credential)
.overrides(overrides)
.buildApi(SwiftApi.class);

objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());

}
}

@Override
public void complete() throws Exception {
if (swiftApi != null) {
swiftApi.close();
}
}

@Override
public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception {
SwiftObject swiftObject = objectApi.get(
resource.getFile().getResourcePath(),
GetOptions.Builder.range(startByte, endByte));

InputStream inputStream = swiftObject.getPayload().openStream();

File targetFile = new File("/tmp/targetFile.tmp");

java.nio.file.Files.copy(
inputStream,
targetFile.toPath(),
StandardCopyOption.REPLACE_EXISTING);

IOUtils.closeQuietly(inputStream);
}

@Override
public InputStream downloadChunk(int chunkId, long startByte, long endByte) throws Exception {

SwiftObject swiftObject = objectApi.get(
resource.getFile().getResourcePath(),
GetOptions.Builder.range(startByte, endByte));

return swiftObject.getPayload().openStream();
}
}

0 comments on commit 5171fee

Please sign in to comment.