Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Issue RSS022-79 - Integration of HarshiCorp Vault
Added container for running a Vault server and one to set it up i.e.
'vault-administration'
Added other modification to the dockerisation

Added a Maven dependency: vault-java-driver

Change the Worker so it gets the secret key for encryption from the
Harshcorp Vault.
Hashicorp Vault information to retrieve the key are defined in the
datavault.properties file.
  • Loading branch information
WilliamPetit committed Jun 19, 2018
1 parent 7e8531a commit a5d2b7d
Show file tree
Hide file tree
Showing 18 changed files with 321 additions and 223 deletions.
3 changes: 1 addition & 2 deletions .travis.yml
Expand Up @@ -18,8 +18,7 @@ script:
# docker needs jar file to start
- mvn clean test package
- docker-compose build
- docker-compose up -d rabbitmq
- docker-compose up -d mysql
- docker-compose up -d rabbitmq mysql vault vault-administration
- sleep 60 # wait that containers are ready
- docker-compose logs rabbitmq
- docker-compose exec rabbitmq rabbitmqctl authenticate_user datavault datavault
Expand Down
3 changes: 2 additions & 1 deletion broker.Dockerfile
Expand Up @@ -41,6 +41,7 @@ COPY --from=0 /usr/src/datavault-assembly/target/datavault-assembly-1.0-SNAPSHOT
COPY --from=0 /usr/src/datavault-assembly/target/datavault-assembly-1.0-SNAPSHOT-assembly/datavault-home/webapps ${DATAVAULT_HOME}/webapps
COPY docker/config ${DATAVAULT_HOME}/config
COPY docker/scripts ${DATAVAULT_HOME}/scripts
COPY docker/pure/ /tmp/

RUN ln -s ${DATAVAULT_HOME}/webapps/datavault-broker ${CATALINA_HOME}/webapps/datavault-broker

Expand All @@ -51,5 +52,5 @@ RUN chown -R datavault:datavault ${CATALINA_HOME}
WORKDIR ${CATALINA_HOME}
EXPOSE 8080

ENTRYPOINT ["/docker_datavault-home/scripts/docker-entrypoint.sh"]
ENTRYPOINT ["/docker_datavault-home/scripts/docker-entrypoint.sh", "brocker"]
CMD ["/usr/local/tomcat/bin/catalina.sh", "run"]
Expand Up @@ -104,7 +104,7 @@ sftp.host = sg.datastore.ed.ac.uk
# Default SFTP port
sftp.port = 22222
# Default SFTP path
sftp.rootPath = /sg/datastore/edina/users/wpetit
sftp.rootPath = /sg/datastore/edina/users/<uun>
# Default SFTP key passphrase
sftp.passphrase = datavault

Expand Down
Expand Up @@ -16,18 +16,28 @@ public enum AESMode {GCM, CBC};
private Long chunkingByteSize;
private Boolean encryptionEnabled;
private AESMode encryptionMode;
private String vaultAddress;
private String vaultToken;
private String vaultKeyPath;
private String vaultKeyName;

public Context() {};
public Context(Path tempDir, Path metaDir, EventStream eventStream,
Boolean chunkingEnabled, Long chunkingByteSize,
Boolean encryptionEnabled, AESMode encryptionMode) {
Boolean encryptionEnabled, AESMode encryptionMode,
String vaultAddress, String vaultToken,
String vaultKeyPath, String vaultKeyName) {
this.tempDir = tempDir;
this.metaDir = metaDir;
this.eventStream = eventStream;
this.chunkingEnabled = chunkingEnabled;
this.chunkingByteSize = chunkingByteSize;
this.encryptionEnabled = encryptionEnabled;
this.encryptionMode = encryptionMode;
this.vaultAddress = vaultAddress;
this.vaultToken = vaultToken;
this.vaultKeyPath = vaultKeyPath;
this.vaultKeyName = vaultKeyName;
}

public Path getTempDir() {
Expand Down Expand Up @@ -81,4 +91,36 @@ public void setEncryptionMode(AESMode encryptionMode) {
}

public AESMode getEncryptionMode() { return encryptionMode; }

public String getVaultAddress() {
return vaultAddress;
}

public void setVaultAddress(String vaultAddress) {
this.vaultAddress = vaultAddress;
}

public String getVaultToken() {
return vaultToken;
}

public void setVaultToken(String vaultToken) {
this.vaultToken = vaultToken;
}

public String getVaultKeyPath() {
return vaultKeyPath;
}

public void setVaultKeyPath(String vaultKeyPath) {
this.vaultKeyPath = vaultKeyPath;
}

public String getVaultKeyName() {
return vaultKeyName;
}

public void setVaultKeyName(String vaultKeyName) {
this.vaultKeyName = vaultKeyName;
}
}
Expand Up @@ -79,7 +79,7 @@
</p>
<br>

<form id="create-vault" class="form" role="form" action="vaults/create" method="post" novalidate="novalidate" _lpchecked="1">
<form id="create-vault" class="form" role="form" action="${springMacroRequestContext.getContextPath()}/vaults/create" method="post" novalidate="novalidate" _lpchecked="1">
<p class="main">
Please select the Pure record describing the data that will be contained in this vault from the list below:
</p>
Expand Down
7 changes: 7 additions & 0 deletions datavault-worker/pom.xml
Expand Up @@ -130,6 +130,13 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>

<!-- HashiCorp Vault -->
<dependency>
<groupId>com.bettercloud</groupId>
<artifactId>vault-java-driver</artifactId>
<version>3.1.0</version>
</dependency>
</dependencies>

<build>
Expand Down
Expand Up @@ -12,6 +12,7 @@
import java.nio.file.StandardOpenOption;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Base64;
import java.util.EnumSet;

import javax.crypto.Cipher;
Expand All @@ -20,8 +21,23 @@
import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;

import org.apache.commons.io.FileUtils;
import org.datavaultplatform.common.task.Context;
import org.datavaultplatform.common.task.Context.AESMode;
import org.datavaultplatform.worker.tasks.Retrieve;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;

public class Encryption {

private static final Logger logger = LoggerFactory.getLogger(Retrieve.class);

public static int BUFFER_SIZE = 50 * 1024; // 50KB
public static int SMALL_BUFFER_SIZE = 1024; // 1KB
public static int AES_BLOCK_SIZE = 16; // 16 Bytes
Expand All @@ -34,6 +50,8 @@ public class Encryption {
public static String CBC_ALGO_TRANSFORMATION_STRING = "AES/CBC/PKCS5Padding";
public static String CTR_ALGO_TRANSFORMATION_STRING = "AES/CTR/PKCS5Padding";
public static String CCM_ALGO_TRANSFORMATION_STRING = "AES/CCM/NoPadding";

private static Vault vault = null;

/**
* Generate a secret key for AES encryption Need JCE Unlimited Strength to
Expand Down Expand Up @@ -84,6 +102,10 @@ public static byte[] generateIV(int size) {
return iv;
}

public static Cipher initGCMCipher(Context context, int opmode, byte[] iv) throws Exception{
return initGCMCipher(opmode, Encryption.getSecretKeyFromVault(context), iv, null);
}

public static Cipher initGCMCipher(int opmode, SecretKey aesKey, byte[] iv) throws Exception{
return initGCMCipher(opmode, aesKey, iv, null);
}
Expand Down Expand Up @@ -121,6 +143,10 @@ public static Cipher initGCMCipher(int opmode, SecretKey aesKey, byte[] iv, byte
return c;
}

public static Cipher initCBCCipher(Context context, int opmode, byte[] iv) throws Exception {
return initCBCCipher(opmode, Encryption.getSecretKeyFromVault(context), iv);
}

/**
* Initialise a AES-CBC Cipher
*
Expand Down Expand Up @@ -218,4 +244,96 @@ public static void doMappedBufferedCrypto(File inputFile, File outputFile, Ciphe
outputMappedByteBuffer.clear(); // do something with the data and clear/compact it.
outputFileChannel.close();
}

private static SecretKey getSecretKeyFromVault(Context context) throws Exception {
if(vault == null) {
setVault(context);
}

logger.debug("get secret key: "+context.getVaultKeyPath()+" "+context.getVaultKeyName());

String encodedKey = vault.logical().read(context.getVaultKeyPath()).getData().get(context.getVaultKeyName());

logger.debug("encodedKey received: "+encodedKey);

// decode the base64 encoded string
byte[] decodedKey = Base64.getDecoder().decode(encodedKey);
// rebuild key using SecretKeySpec
SecretKey secretKey = new SecretKeySpec(decodedKey, 0, decodedKey.length, "AES");

return secretKey;
}

private static void setVault(Context context) throws VaultException {
final VaultConfig vaultConfig = new VaultConfig()
.address(context.getVaultAddress())
.token(context.getVaultToken())
.build();
logger.debug("Vault address: "+vaultConfig.getAddress());
logger.debug("Vault token: "+vaultConfig.getToken());
logger.debug("Vault Max Retries: "+vaultConfig.getMaxRetries());
logger.debug("Vault Retry Interval: "+vaultConfig.getRetryIntervalMilliseconds());
logger.debug("Vault Retry Open Timeout: "+vaultConfig.getOpenTimeout());
logger.debug("Vault Retry Read Timeout: "+vaultConfig.getReadTimeout());

vault = new Vault(vaultConfig);
}

public static Vault getVault() {
return vault;
}

/**
* Perform encryption on file
*
* @param file - file to be encrypted
* @param aesKey - secret key
* @param aesMode - AES encryption mode
* @return generated IV
* @throws Exception
*/
public static byte[] encryptFile(Context context, File file) throws Exception {
return doCrypto(context, file, Cipher.ENCRYPT_MODE, null);
}

/**
* Perform decryption on file
*
* @param file - encrypted file
* @param aesKey - secret key
* @param aesMode - AES encryption mode
* @param iv - Initialisation Vector used for the encryption
* @throws Exception
*/
public static void decryptFile(Context context, File file, byte[] iv) throws Exception {
doCrypto(context, file, Cipher.DECRYPT_MODE, iv);
}

private static byte[] doCrypto(Context context, File file, int encryptMode, byte[] iv) throws Exception {

if(encryptMode == Cipher.ENCRYPT_MODE) {
// Generating IV
iv = Encryption.generateIV(Encryption.IV_SIZE);
}

Cipher cipher;
switch (context.getEncryptionMode()) {
case GCM:
cipher = Encryption.initGCMCipher(context, encryptMode, iv); break;
case CBC:
cipher = Encryption.initCBCCipher(context, encryptMode, iv); break;
default:
cipher = Encryption.initGCMCipher(context, encryptMode, iv); break;
}

File tempEncryptedFile = new File(file.getAbsoluteFile() + ".encrypted");

logger.debug("Encrypting chunk: " + file.getName());
Encryption.doByteBufferFileCrypto(file, tempEncryptedFile, cipher);

FileUtils.copyFile(tempEncryptedFile, file);
FileUtils.deleteQuietly(tempEncryptedFile);

return iv;
}
}
Expand Up @@ -33,6 +33,10 @@ public class Receiver {
private Boolean chunkingEnabled;
private Long chunkingByteSize;
private Boolean encryptionEnabled;
private String vaultAddress;
private String vaultToken;
private String vaultKeyPath;
private String vaultKeyName;
AESMode encryptionMode = AESMode.GCM;

/**
Expand Down Expand Up @@ -123,6 +127,50 @@ public void setEncryptionEnabled(Boolean encryptionEnabled) {
* @return true if encryption is enabled, false otherwise
*/
public Boolean isEncryptionEnabled() { return encryptionEnabled; }

/**
* @return HarshiCorp Vault server address
*/
public String getVaultAddress() {
return vaultAddress;
}

/**
* Define HarshiCorp Vault server address
*/
public void setVaultAddress(String vaultAddress) {
this.vaultAddress = vaultAddress;
}

/**
* @return HarshiCorp Vault Token ID
*/
public String getVaultToken() {
return vaultToken;
}

/**
* Define HarshiCorp Vault Token ID
*/
public void setVaultToken(String vaultToken) {
this.vaultToken = vaultToken;
}

public String getVaultKeyPath() {
return vaultKeyPath;
}

public void setVaultKeyPath(String vaultKeyPath) {
this.vaultKeyPath = vaultKeyPath;
}

public String getVaultKeyName() {
return vaultKeyName;
}

public void setVaultKeyName(String vaultKeyName) {
this.vaultKeyName = vaultKeyName;
}

/**
* Setup a connection to the queue then wait for messages to arrive. When we recieve a message delivery
Expand Down Expand Up @@ -187,7 +235,8 @@ public void receive(EventSender events) throws IOException, InterruptedException
Context context = new Context(
tempDirPath, metaDirPath, events,
chunkingEnabled, chunkingByteSize,
encryptionEnabled, encryptionMode);
encryptionEnabled, encryptionMode,
vaultAddress, vaultToken, vaultKeyPath, vaultKeyName);
concreteTask.performAction(context);

// Clean up the temporary directory
Expand Down

0 comments on commit a5d2b7d

Please sign in to comment.