control-service: switch to Approle Vault authentication

projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl

@@ -184,7 +184,8 @@ Generate default Vault configuration.
 */}}
 {{- define "pipelines-control-service.vaultSecret" -}}
 URI: {{ default "http://localhost:8200" .Values.secrets.vault.uri | b64enc | quote }}
-TOKEN: {{ default "root" .Values.secrets.vault.token | b64enc | quote }}
+ROLEID: {{ default "root" .Values.secrets.vault.approle.roleid | b64enc | quote }}
+SECRETID: {{ default "root" .Values.secrets.vault.approle.secretid | b64enc | quote }}
 {{- end -}}
 
 {{/*

projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml

@@ -307,11 +307,16 @@ spec:
                 secretKeyRef:
                   name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
                   key: URI
-            - name: VDK_VAULT_TOKEN
+            - name: VDK_VAULT_APPROLE_ROLEID
               valueFrom:
                 secretKeyRef:
                   name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
-                  key: TOKEN
+                  key: ROLEID
+            - name: VDK_VAULT_APPROLE_SECRETID
+              valueFrom:
+                secretKeyRef:
+                  name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
+                  key: SECRETID
             - name: DATAJOBS_VAULT_SIZE_LIMIT_BYTES
               value: "{{ .Values.secrets.vault.sizeLimitBytes }}"
             {{- end }}

projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml

@@ -1093,10 +1093,13 @@ alertmanager:
 secrets:
     vault:
         enabled: false
-        ## Name of the secret which holds Vault URI and Token. The chart will not attempt to create this, but will use it as is.
-        ## The secret should contain keys: URI, TOKEN
+        ## Name of the secret which holds Vault URI and Approle RoleId and SecretId. The chart will not attempt to
+        ## create this, but will use it as is.
+        ## The secret should contain keys: URI, ROLEID, SECRETID
         externalSecretName: ""
-        ## Alternatively provide the uri and token here. externalSecretName takes precedence if both are set.
+        ## Alternatively provide the uri and Approle Settings here. externalSecretName takes precedence if both are set.
         uri: "http://localhost:8200"
-        token: "root"
+        approle:
+            roleid: foo
+            secretid: foo
         sizeLimitBytes: 1048576

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/secrets/service/vault/TestVaultJobSecretsServiceIT.java

@@ -10,17 +10,26 @@
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.exception.DataJobSecretsSizeLimitException;
 import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClients;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.vault.authentication.TokenAuthentication;
+import org.springframework.vault.authentication.AppRoleAuthentication;
+import org.springframework.vault.authentication.AppRoleAuthenticationOptions;
 import org.springframework.vault.client.VaultEndpoint;
 import org.springframework.vault.core.VaultTemplate;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.DefaultUriBuilderFactory;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import org.testcontainers.vault.VaultContainer;
 
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Collections;
@@ -37,19 +46,73 @@
 
   @Container
   private static final VaultContainer vaultContainer =
-      new VaultContainer<>("vault:1.0.2").withVaultToken("root");
+      new VaultContainer<>("vault:1.13.3").withVaultToken("root");
 
   private static VaultJobSecretsService vaultJobSecretService;
 
   @BeforeAll
-  public static void init() throws URISyntaxException {
+  public static void init() throws URISyntaxException, IOException, InterruptedException {
     String vaultUri = vaultContainer.getHttpHostAddress();
 
-    VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
-    TokenAuthentication clientAuthentication = new TokenAuthentication("root");
+    // Setup vault app roles authentication
+    // https://developer.hashicorp.com/vault/tutorials/auth-methods/approle
+    // enable AppRoles
+    vaultContainer.execInContainer("vault", "auth", "enable", "approle");
+
+    // Create a new test policy via rest as there's no good way to do it via the command mechanism
+    HttpClient httpClient = HttpClients.createDefault();
+    HttpPost httpPost = new HttpPost(vaultUri + "/v1/sys/policies/acl/testpolicy");
+    httpPost.setHeader("X-Vault-Token", "root");
+    StringEntity requestBody =
+        new StringEntity(
+            "{\n"
+                + "  \"policy\": \"path \\\"secret/*\\\" {\\n"
+                + "  capabilities = [ \\\"create\\\", \\\"read\\\",\\\"update\\\", \\\"patch\\\","
+                + " \\\"delete\\\",\\\"list\\\" ]\\n"
+                + "}\"\n"
+                + "}");
+    httpPost.setEntity(requestBody);
+    HttpResponse response = httpClient.execute(httpPost);
+
+    // create "test" role with the policy
+    vaultContainer.execInContainer(
+        "vault",
+        "write",
+        "auth/approle/role/test",
+        "token_policies=testpolicy",
+        "token_ttl=1h",
+        "token_max_ttl=4h");
+    // get the role id
+    org.testcontainers.containers.Container.ExecResult execResult =
+        vaultContainer.execInContainer(
+            "vault", "read", "auth/approle/role/test/role-id"); // read the role-id
+    String output = execResult.getStdout();
+    String roleId = output.substring(output.lastIndexOf(" ")).trim();
+
+    // get the role secret id
+    execResult =
+        vaultContainer.execInContainer(
+            "vault", "write", "-force", "auth/approle/role/test/secret-id"); // read the secret-id
+    output = execResult.getStdout();
+    String secretId =
+        output
+            .substring(output.indexOf("secret_id") + 9, output.indexOf("secret_id_accessor"))
+            .trim();
+    VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri + "/v1/"));
+
+    // create the authentication
+    AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
+        AppRoleAuthenticationOptions.builder()
+            .roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
+            .secretId(AppRoleAuthenticationOptions.SecretId.provided(secretId));
+
+    RestTemplate restTemplate = new RestTemplate();
+    restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri + "/v1/"));
+
+    AppRoleAuthentication clientAuthentication =
+        new AppRoleAuthentication(builder.build(), restTemplate);
 
     VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
-
     vaultJobSecretService = new VaultJobSecretsService(vaultTemplate);
   }
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultConfiguration.java

@@ -5,27 +5,69 @@
 
 package com.vmware.taurus.secrets.service.vault;
 
+import com.vmware.taurus.exception.SecretStorageNotConfiguredException;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.vault.authentication.TokenAuthentication;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.vault.authentication.AppRoleAuthentication;
+import org.springframework.vault.authentication.AppRoleAuthenticationOptions;
+import org.springframework.vault.authentication.ClientAuthentication;
+import org.springframework.vault.authentication.SessionManager;
 import org.springframework.vault.client.VaultEndpoint;
+import org.springframework.vault.config.AbstractVaultConfiguration;
 import org.springframework.vault.core.VaultOperations;
 import org.springframework.vault.core.VaultTemplate;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.DefaultUriBuilderFactory;
 
 import java.net.URI;
 import java.net.URISyntaxException;
 
 @Configuration
-public class VaultConfiguration {
+public class VaultConfiguration extends AbstractVaultConfiguration {
+
+  @Value("${vdk.vault.uri:}")
+  String vaultUri;
+
+  @Value("${vdk.vault.approle.roleid:}")
+  String roleId;
+
+  @Value("${vdk.vault.approle.secretid:}")
+  String secretId;
 
   @Bean
   public VaultOperations vaultOperations(
-      @Value("${vdk.vault.uri:}") String vaultUri, @Value("${vdk.vault.token:}") String vaultToken)
-      throws URISyntaxException {
-    VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
-    TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken);
+      VaultEndpoint vaultEndpoint, SessionManager sessionManager) {
+
+    SimpleClientHttpRequestFactory clientHttpRequestFactory = new SimpleClientHttpRequestFactory();
+    return new VaultTemplate(vaultEndpoint, clientHttpRequestFactory, sessionManager);
+  }
+
+  @NotNull
+  @Override
+  @Bean
+  public VaultEndpoint vaultEndpoint() {
+    try {
+      return VaultEndpoint.from(new URI(this.vaultUri));
+    } catch (URISyntaxException e) {
+      throw new SecretStorageNotConfiguredException();
+    }
+  }
+
+  @NotNull
+  @Override
+  @Bean
+  public ClientAuthentication clientAuthentication() {
+    AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
+        AppRoleAuthenticationOptions.builder()
+            .roleId(AppRoleAuthenticationOptions.RoleId.provided(this.roleId))
+            .secretId(AppRoleAuthenticationOptions.SecretId.provided(this.secretId));
+
+    RestTemplate restTemplate = new RestTemplate();
+    restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri));
 
-    return new VaultTemplate(vaultEndpoint, clientAuthentication);
+    return new AppRoleAuthentication(builder.build(), restTemplate);
   }
 }

projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties

@@ -259,5 +259,8 @@ datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATI
 # Hashicorp Vault Integration settings
 # When disabled/not configured the Secrets functionality won't work
 featureflag.vault.integration.enabled=false
-vdk.vault.uri=http://localhost:8200
-vdk.vault.token=root
+# If you get 404 errors related to vault operations, double-check the vault URI, it should usually end with "/v1/"
+vdk.vault.uri=https://localhost:8200/v1/
+vdk.vault.approle.roleid=feds259142-9b53-d163-1ae9-d6286d3dfe22
+vdk.vault.approle.secretid=ddsac36f9e9-1555-8c29-9a7f-283608d06fbd
+datajobs.vault.size.limit.bytes=1048576

specs/vep-1493-vault-integration/README.md

@@ -258,19 +258,19 @@ Changes to the properties cli command:
 We are going to enhance the VDK-CS configuration with an optional Spring Vault Configuration.
 
 ```yaml
-spring.cloud.vault:
-    host: localhost
-    port: 8200
-    scheme: https
-    uri: https://localhost:8200
-    connection-timeout: 5000
-    read-timeout: 15000
-    config:
-    token: 19aefa97-cccc-bbbb-aaaa-225940e63d76
+# Hashicorp Vault Integration settings
+# When disabled/not configured the Secrets functionality won't work
+featureflag.vault.integration.enabled=true
+vdk.vault.uri=https://localhost:8200/v1/
+vdk.vault.approle.roleid=z2d59142-9b53-d163-1ae9-d6286d3dfe22
+vdk.vault.approle.secretid=c336f9e9-1555-8c29-9a7f-283608d06fbd
+datajobs.vault.size.limit.bytes=1048576
 ```
-The configuration can be marked optional as outlined in the
-[documentation](https://docs.spring.io/spring-cloud-vault/docs/current/reference/html/#vault.configdata.location.optional)
-which allows users who are not interested in using a secret storage, to simply disable this feature.
+The feature flag allows users who are not interested in using a secret storage, to simply disable this feature.
+
+The VDK Control Service is going to authenticate using Vault's [AppRole mechanism](https://developer.hashicorp.com/vault/tutorials/auth-methods/approle).
+THe Service Operator should setup the App Role authentication based on their needs in Vault, and provide the Vault URI
+the App Role's Role ID and Secret ID.
 
 ### Secrets service