Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

control-service: switch to Approle Vault authentication #2435

Merged
merged 8 commits into from
Jul 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ Generate default Vault configuration.
*/}}
{{- define "pipelines-control-service.vaultSecret" -}}
URI: {{ default "http://localhost:8200" .Values.secrets.vault.uri | b64enc | quote }}
TOKEN: {{ default "root" .Values.secrets.vault.token | b64enc | quote }}
ROLEID: {{ default "root" .Values.secrets.vault.approle.roleid | b64enc | quote }}
SECRETID: {{ default "root" .Values.secrets.vault.approle.secretid | b64enc | quote }}
{{- end -}}

{{/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,11 +307,16 @@ spec:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: URI
- name: VDK_VAULT_TOKEN
- name: VDK_VAULT_APPROLE_ROLEID
valueFrom:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: TOKEN
key: ROLEID
- name: VDK_VAULT_APPROLE_SECRETID
valueFrom:
secretKeyRef:
name: { { .Values.secrets.vault.externalSecretName | default (include "pipelines-control-service.vaultSecretName" . ) } }
key: SECRETID
- name: DATAJOBS_VAULT_SIZE_LIMIT_BYTES
value: "{{ .Values.secrets.vault.sizeLimitBytes }}"
{{- end }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1093,10 +1093,13 @@ alertmanager:
secrets:
vault:
enabled: false
## Name of the secret which holds Vault URI and Token. The chart will not attempt to create this, but will use it as is.
## The secret should contain keys: URI, TOKEN
## Name of the secret which holds Vault URI and Approle RoleId and SecretId. The chart will not attempt to
## create this, but will use it as is.
## The secret should contain keys: URI, ROLEID, SECRETID
externalSecretName: ""
## Alternatively provide the uri and token here. externalSecretName takes precedence if both are set.
## Alternatively provide the uri and Approle Settings here. externalSecretName takes precedence if both are set.
uri: "http://localhost:8200"
token: "root"
approle:
roleid: foo
secretid: foo
sizeLimitBytes: 1048576
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,18 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.vault.authentication.TokenAuthentication;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.core.VaultTemplate;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.vault.VaultContainer;

import java.net.URI;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static com.vmware.taurus.secrets.service.vault.VaultTestSetup.setupVaultTemplate;
import static org.junit.jupiter.api.Assertions.assertThrows;

@SpringBootTest(
Expand All @@ -37,18 +36,18 @@ public class TestVaultJobSecretsServiceIT extends BaseIT {

@Container
private static final VaultContainer vaultContainer =
new VaultContainer<>("vault:1.0.2").withVaultToken("root");
new VaultContainer<>("vault:1.13.3").withVaultToken("root");

private static VaultJobSecretsService vaultJobSecretService;

@BeforeAll
public static void init() throws URISyntaxException {
public static void init() throws URISyntaxException, IOException, InterruptedException {
String vaultUri = vaultContainer.getHttpHostAddress();

VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
TokenAuthentication clientAuthentication = new TokenAuthentication("root");
// Setup vault app roles authentication
// https://developer.hashicorp.com/vault/tutorials/auth-methods/approle

VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
VaultTemplate vaultTemplate = setupVaultTemplate(vaultUri, vaultContainer);

vaultJobSecretService = new VaultJobSecretsService(vaultTemplate);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright 2021-2023 VMware, Inc.
* SPDX-License-Identifier: Apache-2.0
*/

package com.vmware.taurus.secrets.service.vault;

import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
import org.jetbrains.annotations.NotNull;
import org.springframework.vault.authentication.AppRoleAuthentication;
import org.springframework.vault.authentication.AppRoleAuthenticationOptions;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.DefaultUriBuilderFactory;
import org.testcontainers.containers.ContainerState;
import org.testcontainers.vault.VaultContainer;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class VaultTestSetup {

@NotNull
static VaultTemplate setupVaultTemplate(String vaultUri, VaultContainer vaultContainer)
throws IOException, InterruptedException, URISyntaxException {
setupAppRole(vaultUri, vaultContainer);
String roleId = getRoleId(vaultContainer);
String secretId = getSecretId(vaultContainer);
// create the authentication
AppRoleAuthentication clientAuthentication =
getAppRoleAuthentication(vaultUri, roleId, secretId);
VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri + "/v1/"));
VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
return vaultTemplate;
}

private static void setupAppRole(String vaultUri, ContainerState vaultContainer)
throws IOException, InterruptedException {
vaultContainer.execInContainer("vault", "auth", "enable", "approle");

// Create a new test policy via rest as there's no good way to do it via the command mechanism
HttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(vaultUri + "/v1/sys/policies/acl/testpolicy");
httpPost.setHeader("X-Vault-Token", "root");
StringEntity requestBody =
new StringEntity(
"{\n"
+ " \"policy\": \"path \\\"secret/*\\\" {\\n"
+ " capabilities = [ \\\"create\\\", \\\"read\\\",\\\"update\\\", \\\"patch\\\","
+ " \\\"delete\\\",\\\"list\\\" ]\\n"
+ "}\"\n"
+ "}");
httpPost.setEntity(requestBody);
httpClient.execute(httpPost);

// create "test" role with the policy
vaultContainer.execInContainer(
"vault",
"write",
"auth/approle/role/test",
"token_policies=testpolicy",
"token_ttl=1h",
"token_max_ttl=4h");
}

@NotNull
private static String getRoleId(ContainerState vaultContainer)
throws IOException, InterruptedException {
org.testcontainers.containers.Container.ExecResult execResult =
vaultContainer.execInContainer(
"vault", "read", "auth/approle/role/test/role-id"); // read the role-id
String output = execResult.getStdout();
String roleId = output.substring(output.lastIndexOf(" ")).trim();
return roleId;
}

@NotNull
private static String getSecretId(ContainerState vaultContainer)
throws IOException, InterruptedException {
org.testcontainers.containers.Container.ExecResult execResult =
vaultContainer.execInContainer(
"vault", "write", "-force", "auth/approle/role/test/secret-id"); // read the secret-id
String output = execResult.getStdout();
String secretId =
output
.substring(output.indexOf("secret_id") + 9, output.indexOf("secret_id_accessor"))
.trim();
return secretId;
}

@NotNull
private static AppRoleAuthentication getAppRoleAuthentication(
String vaultUri, String roleId, String secretId) {
AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.provided(secretId));

RestTemplate restTemplate = new RestTemplate();
restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri + "/v1/"));

AppRoleAuthentication clientAuthentication =
new AppRoleAuthentication(builder.build(), restTemplate);
return clientAuthentication;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,83 @@

package com.vmware.taurus.secrets.service.vault;

import com.vmware.taurus.exception.SecretStorageNotConfiguredException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.vault.authentication.TokenAuthentication;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.lang.Nullable;
import org.springframework.vault.authentication.*;
import org.springframework.vault.client.VaultEndpoint;
import org.springframework.vault.config.AbstractVaultConfiguration;
import org.springframework.vault.core.VaultOperations;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.DefaultUriBuilderFactory;

import java.net.URI;
import java.net.URISyntaxException;

@Slf4j
@Configuration
public class VaultConfiguration {
public class VaultConfiguration extends AbstractVaultConfiguration {

@Value("${vdk.vault.uri:}")
String vaultUri;

@Value("${vdk.vault.approle.roleid:}")
String roleId;

@Value("${vdk.vault.approle.secretid:}")
String secretId;

@Value("${vdk.vault.token:}")
@Nullable
String vaultToken;

@Bean
public VaultOperations vaultOperations(
@Value("${vdk.vault.uri:}") String vaultUri, @Value("${vdk.vault.token:}") String vaultToken)
throws URISyntaxException {
VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
TokenAuthentication clientAuthentication = new TokenAuthentication(vaultToken);
VaultEndpoint vaultEndpoint, SessionManager sessionManager) {

SimpleClientHttpRequestFactory clientHttpRequestFactory = new SimpleClientHttpRequestFactory();
return new VaultTemplate(vaultEndpoint, clientHttpRequestFactory, sessionManager);
}

@NotNull
@Override
@Bean
public VaultEndpoint vaultEndpoint() {
try {
return VaultEndpoint.from(new URI(this.vaultUri));
} catch (URISyntaxException e) {
throw new SecretStorageNotConfiguredException();
}
}

@NotNull
@Override
@Bean
public ClientAuthentication clientAuthentication() {
// Token authentication should only be used for development purposes. If the token expires, the
// secrets
// functionality will stop working
if (StringUtils.isNotBlank(vaultToken)) {
log.warn("Initializing vault integration with Token Authentication.");
return new TokenAuthentication(vaultToken);
dakodakov marked this conversation as resolved.
Show resolved Hide resolved
} else {
log.info("Initializing vault integration with AppRole Authentication.");
AppRoleAuthenticationOptions.AppRoleAuthenticationOptionsBuilder builder =
AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(this.roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.provided(this.secretId));

RestTemplate restTemplate = new RestTemplate();
restTemplate.setUriTemplateHandler(new DefaultUriBuilderFactory(vaultUri));

return new VaultTemplate(vaultEndpoint, clientAuthentication);
return new AppRoleAuthentication(builder.build(), restTemplate);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,18 @@ datajobs.aws.defaultSessionDurationSeconds=${DATAJOBS_AWS_DEFAULT_SESSION_DURATI

# Hashicorp Vault Integration settings
# When disabled/not configured the Secrets functionality won't work
# In production, we would only use AppRole Authentication which require setup in vault:
# https://developer.hashicorp.com/vault/docs/auth/approle
# https://developer.hashicorp.com/vault/tutorials/auth-methods/approle
# and should provide roleid and secretid to the service.
#
# For local development you can start a vault server with the following command:
# vault server -dev -dev-root-token-id="root"
# and configure only the uri and the token
featureflag.vault.integration.enabled=false
vdk.vault.uri=http://localhost:8200
# If you get 404 errors related to vault operations, double-check the vault URI, it should usually end with "/v1/"
vdk.vault.uri=http://localhost:8200/v1/
vdk.vault.approle.roleid=
vdk.vault.approle.secretid=
vdk.vault.token=root
datajobs.vault.size.limit.bytes=1048576
24 changes: 12 additions & 12 deletions specs/vep-1493-vault-integration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,19 +258,19 @@ Changes to the properties cli command:
We are going to enhance the VDK-CS configuration with an optional Spring Vault Configuration.

```yaml
spring.cloud.vault:
host: localhost
port: 8200
scheme: https
uri: https://localhost:8200
connection-timeout: 5000
read-timeout: 15000
config:
token: 19aefa97-cccc-bbbb-aaaa-225940e63d76
# Hashicorp Vault Integration settings
# When disabled/not configured the Secrets functionality won't work
featureflag.vault.integration.enabled=true
vdk.vault.uri=https://localhost:8200/v1/
vdk.vault.approle.roleid=z2d59142-9b53-d163-1ae9-d6286d3dfe22
vdk.vault.approle.secretid=c336f9e9-1555-8c29-9a7f-283608d06fbd
datajobs.vault.size.limit.bytes=1048576
```
The configuration can be marked optional as outlined in the
[documentation](https://docs.spring.io/spring-cloud-vault/docs/current/reference/html/#vault.configdata.location.optional)
which allows users who are not interested in using a secret storage, to simply disable this feature.
The feature flag allows users who are not interested in using a secret storage, to simply disable this feature.

The VDK Control Service is going to authenticate using Vault's [AppRole mechanism](https://developer.hashicorp.com/vault/tutorials/auth-methods/approle).
THe Service Operator should setup the App Role authentication based on their needs in Vault, and provide the Vault URI
dakodakov marked this conversation as resolved.
Show resolved Hide resolved
the App Role's Role ID and Secret ID.

### Secrets service

Expand Down