Implementing a SQL backend for the Secret service and end to end work…

…flow for scp transfers

agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java

@@ -29,12 +29,16 @@ public static void main(String args[]) throws Exception {
         ObjectMapper mapper = new ObjectMapper();
 
         TransferRequest request = new TransferRequest();
-        request.setSourceId("1");
+        request.setSourceId("40107348-c457-4b0d-b206-b923caa2ab8d");
         request.setSourceType("SCP");
-        request.setDestinationId("2");
+        request.setSourceToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
+        request.setDestinationId("24cf4870-ee50-4076-b97c-aeb481fec324");
         request.setDestinationType("SCP");
+        request.setDestinationToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
         request.setAgentList(Collections.singletonList("agent0"));
-        request.setTransferId("transfer005");
+        request.setTransferId("transfer010");
 
         String asString = mapper.writeValueAsString(request);
 

agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java

@@ -71,10 +71,11 @@ private void acceptRequests() {
                         String transferId = mediator.transfer(inConnector, outConnector, metadata);
                         System.out.println("Submitted transfer " + transferId);
 
-                        System.out.println("Deleting key " + value.getKey());
-                        kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
                     } catch (Exception e) {
                         e.printStackTrace();
+                    } finally {
+                        System.out.println("Deleting key " + value.getKey());
+                        kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
                     }
                 });
 

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java

@@ -63,20 +63,23 @@ public boolean updateSCPStorage(SCPStorageUpdateRequest request) {
 
     @Override
     public boolean deleteSCPStorage(SCPStorageDeleteRequest request) {
-        scpStorageRepository.delete(request.getStorageId());
+        //scpStorageRepository.delete(request.getStorageId());
         return true;
     }
 
     @Override
     public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request) {
         Optional<SCPResourceEntity> resourceEntity = scpResourceRepository.findByResourceId(request.getResourceId());
-        return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass()).build());
+
+        return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass())
+                .setScpStorage(mapper.map(scpResourceEntity.getScpStorage(), SCPStorage.newBuilder().getClass())).build());
+        // Here we have to do nested mapping as the dozer -> protobuf conversion is not happening for inner objects
     }
 
     @Override
     public SCPResource createSCPResource(SCPResourceCreateRequest request) {
         SCPResourceEntity savedEntity = scpResourceRepository.save(mapper.map(request, SCPResourceEntity.class));
-        return mapper.map(savedEntity, SCPResource.newBuilder().getClass()).build();
+        return getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(savedEntity.getResourceId()).build()).get();
     }
 
     @Override
@@ -87,7 +90,7 @@ public boolean updateSCPResource(SCPResourceUpdateRequest request) {
 
     @Override
     public boolean deleteSCPResource(SCPResourceDeleteRequest request) {
-        scpResourceRepository.delete(request.getResourceId());
+        scpResourceRepository.deleteById(request.getResourceId());
         return true;
     }
 
@@ -111,7 +114,7 @@ public boolean updateLocalResource(LocalResourceUpdateRequest request) {
 
     @Override
     public boolean deleteLocalResource(LocalResourceDeleteRequest request) {
-        localResourceRepository.delete(request.getResourceId());
+        localResourceRepository.deleteById(request.getResourceId());
         return true;
     }
 }

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java

@@ -33,6 +33,7 @@ public class LocalResourceEntity {
     @GeneratedValue(generator = "uuid")
     @GenericGenerator(name = "uuid", strategy = "uuid2")
     private String resourceId;
+
     @Column(name = "RESOURCE_PATH")
     private String resourcePath;
 

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java

@@ -29,13 +29,13 @@ public class SCPResourceEntity {
     @GenericGenerator(name = "uuid", strategy = "uuid2")
     private String resourceId;
 
+    @Column(name = "SCP_STORAGE_ID")
+    private String scpStorageId;
+
     @ManyToOne(fetch = FetchType.EAGER)
-    @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, updatable = false)
+    @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, insertable = false, updatable = false)
     private SCPStorageEntity scpStorage;
 
-    @Column(name = "SCP_STORAGE_ID", insertable = false, updatable = false)
-    private String scpStorageId;
-
     @Column(name = "RESOURCE_PATH")
     private String resourcePath;
 

services/secret-service/server/pom.xml

@@ -38,5 +38,11 @@
             <artifactId>mft-secret-service-stub</artifactId>
             <version>0.01-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>${h2}</version>
+            <scope>runtime</scope>
+        </dependency>
     </dependencies>
 </project>

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server;
+
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class AppConfig {
+
+    @Bean
+    SecretBackend secretBackend() {return new SQLSecretBackend();}
+}

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java

@@ -19,7 +19,9 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
 
+@ComponentScan(basePackages = {"org.apache.airavata"})
 @SpringBootApplication
 public class SecretServiceApplication {
     public static void main(String args[]) {

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend;
+
+import org.apache.airavata.mft.secret.service.*;
+
+import java.util.Optional;
+
+public interface SecretBackend {
+    public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request);
+    public SCPSecret createSCPSecret(SCPSecretCreateRequest request);
+    public boolean updateSCPSecret(SCPSecretUpdateRequest request);
+    public boolean deleteSCPSecret(SCPSecretDeleteRequest request);
+}

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql;
+
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.repository.SecretRepository;
+import org.apache.airavata.mft.secret.service.*;
+import org.dozer.DozerBeanMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Optional;
+
+public class SQLSecretBackend implements SecretBackend {
+
+    @Autowired
+    private SecretRepository secretRepository;
+
+    private DozerBeanMapper mapper = new DozerBeanMapper();
+
+    @Override
+    public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) {
+        Optional<SCPSecretEntity> secretEty = secretRepository.findBySecretId(request.getSecretId());
+        return secretEty.map(scpSecretEntity -> mapper.map(scpSecretEntity, SCPSecret.newBuilder().getClass()).build());
+    }
+
+    @Override
+    public SCPSecret createSCPSecret(SCPSecretCreateRequest request) {
+        SCPSecretEntity savedEntity = secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+        return mapper.map(savedEntity, SCPSecret.newBuilder().getClass()).build();
+    }
+
+    @Override
+    public boolean updateSCPSecret(SCPSecretUpdateRequest request) {
+        secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+        return true;
+    }
+
+    @Override
+    public boolean deleteSCPSecret(SCPSecretDeleteRequest request) {
+        secretRepository.deleteById(request.getSecretId());
+        return true;
+    }
+}

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class SCPSecretEntity {
+
+    @Id
+    @Column(name = "SECRET_ID")
+    @GeneratedValue(generator = "uuid")
+    @GenericGenerator(name = "uuid", strategy = "uuid2")
+    private String secretId;
+
+    @Column(name = "PRIVATE_KEY")
+    private String privateKey;
+
+    @Column(name = "PUBLIC_KEY")
+    private String publicKey;
+
+    @Column(name = "PASSPHRASE")
+    private String passphrase;
+
+    @Column(name = "USER_NAME")
+    private String user;
+
+    public String getSecretId() {
+        return secretId;
+    }
+
+    public void setSecretId(String secretId) {
+        this.secretId = secretId;
+    }
+
+    public String getPrivateKey() {
+        return privateKey;
+    }
+
+    public void setPrivateKey(String privateKey) {
+        this.privateKey = privateKey;
+    }
+
+    public String getPublicKey() {
+        return publicKey;
+    }
+
+    public void setPublicKey(String publicKey) {
+        this.publicKey = publicKey;
+    }
+
+    public String getPassphrase() {
+        return passphrase;
+    }
+
+    public void setPassphrase(String passphrase) {
+        this.passphrase = passphrase;
+    }
+
+    public String getUser() {
+        return user;
+    }
+
+    public void setUser(String user) {
+        this.user = user;
+    }
+}

services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.repository;
+
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.Optional;
+
+public interface SecretRepository extends CrudRepository<SCPSecretEntity, String> {
+    Optional<SCPSecretEntity> findBySecretId(String resourceId);
+}