From aff62367ee1439f7e2ee54540d4b060d2dc057b1 Mon Sep 17 00:00:00 2001 From: Dimuthu Wannipurage Date: Tue, 31 Dec 2019 03:58:59 -0500 Subject: [PATCH] Implementing a SQL backend for the Secret service and end to end workflow for scp transfers --- .../airavata/mft/agent/ConsulTester.java | 10 ++- .../apache/airavata/mft/agent/MFTAgent.java | 5 +- .../backend/sql/SQLResourceBackend.java | 13 +-- .../sql/entity/LocalResourceEntity.java | 1 + .../backend/sql/entity/SCPResourceEntity.java | 8 +- services/secret-service/server/pom.xml | 6 ++ .../airavata/mft/secret/server/AppConfig.java | 30 +++++++ .../server/SecretServiceApplication.java | 2 + .../secret/server/backend/SecretBackend.java | 29 +++++++ .../server/backend/sql/SQLSecretBackend.java | 59 +++++++++++++ .../backend/sql/entity/SCPSecretEntity.java | 87 +++++++++++++++++++ .../sql/repository/SecretRepository.java | 27 ++++++ .../server/handler/SecretServiceHandler.java | 46 +++++++--- .../stub/src/main/proto/SecretService.proto | 54 +++++++++++- transport/pom.xml | 12 +++ .../transport/scp/SCPMetadataCollector.java | 23 +++-- .../mft/transport/scp/SCPReceiver.java | 27 ++++-- .../airavata/mft/transport/scp/SCPSender.java | 28 ++++-- .../mft/transport/scp/SCPTransportUtil.java | 30 +++++++ 19 files changed, 450 insertions(+), 47 deletions(-) create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java index 39240def..0a0f1c8d 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java @@ -29,12 +29,16 @@ public static void main(String args[]) throws Exception { ObjectMapper mapper = new ObjectMapper(); TransferRequest request = new TransferRequest(); - request.setSourceId("1"); + request.setSourceId("40107348-c457-4b0d-b206-b923caa2ab8d"); request.setSourceType("SCP"); - request.setDestinationId("2"); + request.setSourceToken("866d421e-3624-434f-ae71-04a90d39e70c"); + + request.setDestinationId("24cf4870-ee50-4076-b97c-aeb481fec324"); request.setDestinationType("SCP"); + request.setDestinationToken("866d421e-3624-434f-ae71-04a90d39e70c"); + request.setAgentList(Collections.singletonList("agent0")); - request.setTransferId("transfer005"); + request.setTransferId("transfer010"); String asString = mapper.writeValueAsString(request); diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java index 4424e963..92e851b8 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java @@ -71,10 +71,11 @@ private void acceptRequests() { String transferId = mediator.transfer(inConnector, outConnector, metadata); System.out.println("Submitted transfer " + transferId); - System.out.println("Deleting key " + value.getKey()); - kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571 } catch (Exception e) { e.printStackTrace(); + } finally { + System.out.println("Deleting key " + value.getKey()); + kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571 } }); diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java index 0be7b6d2..1ffabe33 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java @@ -63,20 +63,23 @@ public boolean updateSCPStorage(SCPStorageUpdateRequest request) { @Override public boolean deleteSCPStorage(SCPStorageDeleteRequest request) { - scpStorageRepository.delete(request.getStorageId()); + //scpStorageRepository.delete(request.getStorageId()); return true; } @Override public Optional getSCPResource(SCPResourceGetRequest request) { Optional resourceEntity = scpResourceRepository.findByResourceId(request.getResourceId()); - return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass()).build()); + + return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass()) + .setScpStorage(mapper.map(scpResourceEntity.getScpStorage(), SCPStorage.newBuilder().getClass())).build()); + // Here we have to do nested mapping as the dozer -> protobuf conversion is not happening for inner objects } @Override public SCPResource createSCPResource(SCPResourceCreateRequest request) { SCPResourceEntity savedEntity = scpResourceRepository.save(mapper.map(request, SCPResourceEntity.class)); - return mapper.map(savedEntity, SCPResource.newBuilder().getClass()).build(); + return getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(savedEntity.getResourceId()).build()).get(); } @Override @@ -87,7 +90,7 @@ public boolean updateSCPResource(SCPResourceUpdateRequest request) { @Override public boolean deleteSCPResource(SCPResourceDeleteRequest request) { - scpResourceRepository.delete(request.getResourceId()); + scpResourceRepository.deleteById(request.getResourceId()); return true; } @@ -111,7 +114,7 @@ public boolean updateLocalResource(LocalResourceUpdateRequest request) { @Override public boolean deleteLocalResource(LocalResourceDeleteRequest request) { - localResourceRepository.delete(request.getResourceId()); + localResourceRepository.deleteById(request.getResourceId()); return true; } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java index 2d48b5f6..2a8ed81e 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java @@ -33,6 +33,7 @@ public class LocalResourceEntity { @GeneratedValue(generator = "uuid") @GenericGenerator(name = "uuid", strategy = "uuid2") private String resourceId; + @Column(name = "RESOURCE_PATH") private String resourcePath; diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java index cab753bb..63a6b322 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java @@ -29,13 +29,13 @@ public class SCPResourceEntity { @GenericGenerator(name = "uuid", strategy = "uuid2") private String resourceId; + @Column(name = "SCP_STORAGE_ID") + private String scpStorageId; + @ManyToOne(fetch = FetchType.EAGER) - @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, updatable = false) + @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, insertable = false, updatable = false) private SCPStorageEntity scpStorage; - @Column(name = "SCP_STORAGE_ID", insertable = false, updatable = false) - private String scpStorageId; - @Column(name = "RESOURCE_PATH") private String resourcePath; diff --git a/services/secret-service/server/pom.xml b/services/secret-service/server/pom.xml index bfa1c529..0f0a0427 100644 --- a/services/secret-service/server/pom.xml +++ b/services/secret-service/server/pom.xml @@ -38,5 +38,11 @@ mft-secret-service-stub 0.01-SNAPSHOT + + com.h2database + h2 + ${h2} + runtime + \ No newline at end of file diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java new file mode 100644 index 00000000..fb2d2d1e --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server; + +import org.apache.airavata.mft.secret.server.backend.SecretBackend; +import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AppConfig { + + @Bean + SecretBackend secretBackend() {return new SQLSecretBackend();} +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java index d7830ac8..c9e930d4 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java @@ -19,7 +19,9 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +@ComponentScan(basePackages = {"org.apache.airavata"}) @SpringBootApplication public class SecretServiceApplication { public static void main(String args[]) { diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java new file mode 100644 index 00000000..96487473 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend; + +import org.apache.airavata.mft.secret.service.*; + +import java.util.Optional; + +public interface SecretBackend { + public Optional getSCPSecret(SCPSecretGetRequest request); + public SCPSecret createSCPSecret(SCPSecretCreateRequest request); + public boolean updateSCPSecret(SCPSecretUpdateRequest request); + public boolean deleteSCPSecret(SCPSecretDeleteRequest request); +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java new file mode 100644 index 00000000..513bbba4 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql; + +import org.apache.airavata.mft.secret.server.backend.SecretBackend; +import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.repository.SecretRepository; +import org.apache.airavata.mft.secret.service.*; +import org.dozer.DozerBeanMapper; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Optional; + +public class SQLSecretBackend implements SecretBackend { + + @Autowired + private SecretRepository secretRepository; + + private DozerBeanMapper mapper = new DozerBeanMapper(); + + @Override + public Optional getSCPSecret(SCPSecretGetRequest request) { + Optional secretEty = secretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(scpSecretEntity -> mapper.map(scpSecretEntity, SCPSecret.newBuilder().getClass()).build()); + } + + @Override + public SCPSecret createSCPSecret(SCPSecretCreateRequest request) { + SCPSecretEntity savedEntity = secretRepository.save(mapper.map(request, SCPSecretEntity.class)); + return mapper.map(savedEntity, SCPSecret.newBuilder().getClass()).build(); + } + + @Override + public boolean updateSCPSecret(SCPSecretUpdateRequest request) { + secretRepository.save(mapper.map(request, SCPSecretEntity.class)); + return true; + } + + @Override + public boolean deleteSCPSecret(SCPSecretDeleteRequest request) { + secretRepository.deleteById(request.getSecretId()); + return true; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java new file mode 100644 index 00000000..3fd75d92 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.entity; + +import org.hibernate.annotations.GenericGenerator; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class SCPSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "PRIVATE_KEY") + private String privateKey; + + @Column(name = "PUBLIC_KEY") + private String publicKey; + + @Column(name = "PASSPHRASE") + private String passphrase; + + @Column(name = "USER_NAME") + private String user; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getPrivateKey() { + return privateKey; + } + + public void setPrivateKey(String privateKey) { + this.privateKey = privateKey; + } + + public String getPublicKey() { + return publicKey; + } + + public void setPublicKey(String publicKey) { + this.publicKey = publicKey; + } + + public String getPassphrase() { + return passphrase; + } + + public void setPassphrase(String passphrase) { + this.passphrase = passphrase; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java new file mode 100644 index 00000000..70bf6c96 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.repository; + +import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity; +import org.springframework.data.repository.CrudRepository; + +import java.util.Optional; + +public interface SecretRepository extends CrudRepository { + Optional findBySecretId(String resourceId); +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java index 26c741ac..26f20fb3 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java @@ -17,22 +17,48 @@ package org.apache.airavata.mft.secret.server.handler; +import com.google.protobuf.Empty; import io.grpc.stub.StreamObserver; -import org.apache.airavata.mft.secret.service.SCPSecret; -import org.apache.airavata.mft.secret.service.SCPSecretRequest; -import org.apache.airavata.mft.secret.service.SecretServiceGrpc; +import org.apache.airavata.mft.secret.server.backend.SecretBackend; +import org.apache.airavata.mft.secret.service.*; import org.lognet.springboot.grpc.GRpcService; +import org.springframework.beans.factory.annotation.Autowired; @GRpcService public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBase { + + @Autowired + private SecretBackend backend; + + @Override + public void getSCPSecret(SCPSecretGetRequest request, StreamObserver responseObserver) { + this.backend.getSCPSecret(request).ifPresentOrElse(secret -> { + responseObserver.onNext(secret); + responseObserver.onCompleted(); + }, () -> { + responseObserver.onError(new Exception("No SCP Secret with id " + request.getSecretId())); + }); + } + + @Override + public void createSCPSecret(SCPSecretCreateRequest request, StreamObserver responseObserver) { + responseObserver.onNext(this.backend.createSCPSecret(request)); + responseObserver.onCompleted(); + } + @Override - public void getSCPSecret(SCPSecretRequest request, StreamObserver responseObserver) { - SCPSecret.Builder builder = SCPSecret.newBuilder() - .setPrivateKey("private") - .setPrivateKey("pubKey") - .setPassphrase("pass") - .setSecretId("sec1"); - responseObserver.onNext(builder.build()); + public void updateSCPSecret(SCPSecretUpdateRequest request, StreamObserver responseObserver) { + this.backend.updateSCPSecret(request); responseObserver.onCompleted(); } + + @Override + public void deleteSCPSecret(SCPSecretDeleteRequest request, StreamObserver responseObserver) { + boolean res = this.backend.deleteSCPSecret(request); + if (res) { + responseObserver.onCompleted(); + } else { + responseObserver.onError(new Exception("Failed to delete SCP Secret with id " + request.getSecretId())); + } + } } diff --git a/services/secret-service/stub/src/main/proto/SecretService.proto b/services/secret-service/stub/src/main/proto/SecretService.proto index f2d0cc7e..9fa99b29 100644 --- a/services/secret-service/stub/src/main/proto/SecretService.proto +++ b/services/secret-service/stub/src/main/proto/SecretService.proto @@ -4,23 +4,69 @@ option java_multiple_files = true; package org.apache.airavata.mft.secret.service; import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +message AuthToken { + string token = 1; +} message SCPSecret { string secretId = 1; string privateKey = 2; string publicKey = 3; string passphrase = 4; + string user = 5; +} + +message SCPSecretGetRequest { + string secretId = 1; + AuthToken authzToken = 2; +} + +message SCPSecretCreateRequest { + string privateKey = 1; + string publicKey = 2; + string passphrase = 3; + string user = 4; + AuthToken authzToken = 5; +} + +message SCPSecretUpdateRequest { + string secretId = 1; + string privateKey = 2; + string publicKey = 3; + string passphrase = 4; + string user = 5; + AuthToken authzToken = 6; } -message SCPSecretRequest { - string resourceId = 1; - string authzToken = 2; +message SCPSecretDeleteRequest { + string secretId = 1; + AuthToken authzToken = 2; } service SecretService { - rpc getSCPSecret (SCPSecretRequest) returns (SCPSecret) { + rpc getSCPSecret (SCPSecretGetRequest) returns (SCPSecret) { option (google.api.http) = { get: "/v1.0/secret/scp" }; } + + rpc createSCPSecret (SCPSecretCreateRequest) returns (SCPSecret) { + option (google.api.http) = { + post: "/v1.0/secret/scp" + }; + } + + rpc updateSCPSecret (SCPSecretUpdateRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + put: "/v1.0/secret/scp" + }; + } + + rpc deleteSCPSecret (SCPSecretDeleteRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1.0/secret/scp" + }; + } } \ No newline at end of file diff --git a/transport/pom.xml b/transport/pom.xml index 64266c5a..27e3cba7 100755 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -36,6 +36,18 @@ scp-transport local-transport + + + org.apache.airavata + mft-resource-service-client + 0.01-SNAPSHOT + + + org.apache.airavata + mft-secret-service-client + 0.01-SNAPSHOT + + \ No newline at end of file diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java index abe53054..6f95895b 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java @@ -28,6 +28,14 @@ import net.schmizz.sshj.userauth.password.Resource; import org.apache.airavata.mft.core.ResourceMetadata; import org.apache.airavata.mft.core.api.MetadataCollector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.ResourceServiceGrpc; +import org.apache.airavata.mft.resource.service.SCPResource; +import org.apache.airavata.mft.resource.service.SCPResourceGetRequest; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.SCPSecret; +import org.apache.airavata.mft.secret.service.SCPSecretGetRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; import java.io.IOException; import java.util.ArrayList; @@ -38,11 +46,16 @@ public class SCPMetadataCollector implements MetadataCollector { public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws IOException { - SSHResourceIdentifier sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId); + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002); + SCPResource scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003); + SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + try (SSHClient sshClient = new SSHClient()) { sshClient.addHostKeyVerifier((h, p, key) -> true); - KeyProvider keyProvider = sshClient.loadKeys(sshResourceIdentifier.getKeyFile(), sshResourceIdentifier.getKeyPassphrase()); + KeyProvider keyProvider = sshClient.loadKeys(scpSecret.getPrivateKey(), scpSecret.getPassphrase()); final List am = new LinkedList<>(); am.add(new AuthPublickey(keyProvider)); am.add(new AuthKeyboardInteractive(new ChallengeResponseProvider() { @@ -65,11 +78,11 @@ public boolean shouldRetry() { } })); - sshClient.connect(sshResourceIdentifier.getHost(), sshResourceIdentifier.getPort()); - sshClient.auth(sshResourceIdentifier.getUser(), am); + sshClient.connect(scpResource.getScpStorage().getHost(), scpResource.getScpStorage().getPort()); + sshClient.auth(scpSecret.getUser(), am); try (SFTPClient sftpClient = sshClient.newSFTPClient()) { - FileAttributes lstat = sftpClient.lstat(sshResourceIdentifier.getRemotePath()); + FileAttributes lstat = sftpClient.lstat(scpResource.getResourcePath()); sftpClient.close(); ResourceMetadata metadata = new ResourceMetadata(); diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java index e56e9e0f..141395f1 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java @@ -22,6 +22,12 @@ import org.apache.airavata.mft.core.CircularStreamingBuffer; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.*; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.SCPSecret; +import org.apache.airavata.mft.secret.service.SCPSecretGetRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; import java.io.IOException; import java.io.InputStream; @@ -29,14 +35,21 @@ public class SCPReceiver implements Connector { private Session session; - private SSHResourceIdentifier sshResourceIdentifier; + private SCPResource scpResource; public void init(String resourceId, String credentialToken) throws Exception { - this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId); - this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(), - sshResourceIdentifier.getPort(), - sshResourceIdentifier.getKeyFile(), - sshResourceIdentifier.getKeyPassphrase()); + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002); + this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003); + SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + this.session = SCPTransportUtil.createSession( + scpSecret.getUser(), + scpResource.getScpStorage().getHost(), + scpResource.getScpStorage().getPort(), + scpSecret.getPrivateKey(), + scpSecret.getPassphrase()); } public void destroy() { @@ -53,7 +66,7 @@ public void startStream(ConnectorContext context) throws Exception { throw new Exception("Session can not be null. Make sure that SCP Receiver is properly initialized"); } - transferRemoteToStream(session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer()); + transferRemoteToStream(session, this.scpResource.getResourcePath(), context.getStreamBuffer()); } private void transferRemoteToStream(Session session, String from, CircularStreamingBuffer streamBuffer) throws Exception { diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java index 31bb1382..7f6b217c 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java @@ -23,20 +23,34 @@ import org.apache.airavata.mft.core.CircularStreamingBuffer; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.ResourceServiceGrpc; +import org.apache.airavata.mft.resource.service.SCPResource; +import org.apache.airavata.mft.resource.service.SCPResourceGetRequest; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.SCPSecret; +import org.apache.airavata.mft.secret.service.SCPSecretGetRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; public class SCPSender implements Connector { - private SSHResourceIdentifier sshResourceIdentifier; private Session session; + private SCPResource scpResource; public void init(String resourceId, String credentialToken) { - this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId); - this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(), - sshResourceIdentifier.getPort(), - sshResourceIdentifier.getKeyFile(), - sshResourceIdentifier.getKeyPassphrase()); + + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002); + this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003); + SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + this.session = SCPTransportUtil.createSession(scpSecret.getUser(), scpResource.getScpStorage().getHost(), + scpResource.getScpStorage().getPort(), + scpSecret.getPrivateKey(), + scpSecret.getPassphrase()); } public void destroy() { @@ -53,7 +67,7 @@ public void startStream(ConnectorContext context) throws Exception { System.out.println("Session can not be null. Make sure that SCP Sender is properly initialized"); throw new Exception("Session can not be null. Make sure that SCP Sender is properly initialized"); } - copyLocalToRemote(this.session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize()); + copyLocalToRemote(this.session, this.scpResource.getResourcePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize()); } private void copyLocalToRemote(Session session, String to, CircularStreamingBuffer streamBuffer, long fileSize) throws JSchException, IOException { diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java index d81e3dc5..bcb83427 100755 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java @@ -20,11 +20,41 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.*; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.SCPSecret; +import org.apache.airavata.mft.secret.service.SCPSecretCreateRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; import java.util.Properties; public class SCPTransportUtil { // TODO replace with an API call to the registry + + + public static void main(String args[]) { + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002); + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003); + + //resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId("82848d70-ef71-47d2-935a-4fd3e0184a42").build()); + SCPStorage scpStorage1 = resourceClient.createSCPStorage(SCPStorageCreateRequest.newBuilder().setPort(22).setHost("pgadev.scigap.org").build()); + SCPResource scpResource1 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId()) + .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/2mb.txt").build()); + SCPResource scpResource2 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId()) + .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/new-file.txt").build()); + + System.out.println("Resource 1 id " + scpResource1.getResourceId()); + System.out.println("Resource 2 id " + scpResource2.getResourceId()); + + SCPSecret scpSecret1 = secretClient.createSCPSecret(SCPSecretCreateRequest.newBuilder() + .setUser("pga") + .setPassphrase("shithappens@12") + .setPrivateKey("/Users/dimuthu/.ssh/id_rsa").build()); + + System.out.println("Secret id " + scpSecret1.getSecretId()); + } + public static SSHResourceIdentifier getSSHResourceIdentifier(String resourceId) { SSHResourceIdentifier identifier = new SSHResourceIdentifier();