Allow using NGINX keyval store for credential caching

Author: dekobon

common/etc/nginx/include/s3gateway.js

@@ -150,12 +150,60 @@ function _credentialsTempFile() {
 }
 
 /**
- * Read the contents of the credentials file into memory. If it is not
- * found, then return undefined.
+ * Write the instance profile credentials to a caching backend.
  *
- * @returns {undefined|object} AWS instance profile credentials or undefined
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
  */
-function readCredentials() {
+function writeCredentials(r, credentials) {
+    // Do not bother writing credentials if we are running in a mode where we
+    // do not need instance credentials.
+    if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
+        return;
+    }
+
+    if (!credentials) {
+        throw `Cannot write invalid credentials: ${JSON.stringify(credentials)}`;
+    }
+
+    if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) {
+        _writeCredentialsToKeyValStore(r, credentials);
+    } else {
+        _writeCredentialsToFile(credentials);
+    }
+}
+
+/**
+ * Write the instance profile credentials to the NGINX Keyval store.
+ *
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
+ * @private
+ */
+function _writeCredentialsToKeyValStore(r, credentials) {
+    r.variables.instance_credential_json = JSON.stringify(credentials);
+}
+
+/**
+ * Write the instance profile credentials to a file on the file system. This
+ * file will be quite small and should end up in the file cache relatively
+ * quickly if it is repeatedly read.
+ *
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
+ * @private
+ */
+function _writeCredentialsToFile(credentials) {
+    fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials));
+}
+
+/**
+ * Get the instance profile credentials needed to authenticated against S3 from
+ * a backend cache. If the credentials cannot be found, then return undefined.
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
+ */
+function readCredentials(r) {
     if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
         return {
             accessKeyId: process.env['S3_ACCESS_KEY_ID'],
@@ -165,6 +213,44 @@ function readCredentials() {
         };
     }
 
+    if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) {
+        return _readCredentialsFromKeyValStore(r);
+    } else {
+        return _readCredentialsFromFile();
+    }
+}
+
+/**
+ * Read credentials from the NGINX Keyval store. If it is not found, then
+ * return undefined.
+ *
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
+ * @private
+ */
+function _readCredentialsFromKeyValStore(r) {
+    var cached = r.variables.instance_credential_json;
+
+    if (!cached) {
+        return undefined;
+    }
+
+    try {
+        return JSON.parse(cached);
+    } catch (e) {
+        _debug_log(r, `Error parsing JSON value from r.variables.instance_credential_json: ${e}`);
+        return undefined;
+    }
+}
+
+/**
+ * Read the contents of the credentials file into memory. If it is not
+ * found, then return undefined.
+ *
+ * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
+ * @private
+ */
+function _readCredentialsFromFile() {
     var credsFilePath = _credentialsTempFile();
 
     try {
@@ -201,7 +287,7 @@ function s3auth(r) {
 
     var signature;
 
-    var credentials = readCredentials();
+    var credentials = readCredentials(r);
     if (sigver == '2') {
         signature = signatureV2(r, bucket, credentials);
     } else {
@@ -211,8 +297,14 @@ function s3auth(r) {
     return signature;
 }
 
-function s3SecurityToken() {
-    var credentials = readCredentials();
+/**
+ * Get the current session token from the instance profile credential cache.
+ *
+ * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
+ * @returns {string} current session token or empty string
+ */
+function s3SecurityToken(r) {
+    var credentials = readCredentials(r);
     if (credentials.sessionToken) {
         return credentials.sessionToken;
     }
@@ -782,7 +874,21 @@ var maxValidityOffsetMs = 4.5 * 60 * 100;
  * @returns {Promise<void>}
  */
 async function fetchCredentials(r) {
-    var current = readCredentials();
+    // If we are not using an AWS instance profile to set our credentials we
+    // exit quickly and don't write a credentials file.
+    if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
+        r.return(200);
+        return;
+    }
+
+    try {
+        var current = readCredentials(r);
+    } catch (e) {
+        _debug_log(r, `Could not read credentials: ${e}`);
+        r.return(500);
+        return;
+    }
+
     if (current) {
         var exp = new Date(current.expiration).getTime() - maxValidityOffsetMs;
         if (now.getTime() < exp) {
@@ -793,13 +899,6 @@ async function fetchCredentials(r) {
 
     var credentials;
 
-    // If we are not using an AWS instance profile to set our credentials we
-    // exit quickly and don't write a credentials file.
-    if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
-        r.return(200);
-        return;
-    }
-
     _debug_log(r, 'Cached credentials are expired or not present, requesting new ones');
 
     if (process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']) {
@@ -821,9 +920,9 @@ async function fetchCredentials(r) {
         }
     }
     try {
-        fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials));
+        writeCredentials(r, credentials);
     } catch (e) {
-        _debug_log(r, 'Could not write credentials file: ' + JSON.stringify(e));
+        _debug_log(r, `Could not write credentials: ${e}`);
         r.return(500);
         return;
     }
@@ -899,6 +998,7 @@ export default {
     awsHeaderDate,
     fetchCredentials,
     readCredentials,
+    writeCredentials,
     s3date,
     s3auth,
     s3SecurityToken,

oss/etc/nginx/conf.d/gateway/server_variables.conf

@@ -2,3 +2,8 @@
 # caching is turned off. This feature uses the keyval store, so it
 # is only enabled when using NGINX Plus.
 set $cache_signing_key_enabled 0;
+
+# Variable indicating to the s3gateway.js script that session token
+# caching is turned on. This feature uses the keyval store, so it
+# is only enabled when using NGINX Plus.
+set $cache_instance_credentials_enabled 0;

plus/etc/nginx/conf.d/gateway/server_variables.conf

@@ -2,3 +2,8 @@
 # caching is turned on. This feature uses the keyval store, so it
 # is only enabled when using NGINX Plus.
 set $cache_signing_key_enabled 1;
+
+# Variable indicating to the s3gateway.js script that session token
+# caching is turned on. This feature uses the keyval store, so it
+# is only enabled when using NGINX Plus.
+set $cache_instance_credentials_enabled 1;

plus/etc/nginx/conf.d/instance_credential_cache.conf

@@ -0,0 +1,4 @@
+# This key value zone allows us to cache a portion of the cryptographic
+# signatures used by AWS v4 signatures.
+keyval_zone zone=instance_credential_cache:32k type=string timeout=6h;
+keyval 'instance_credential' $instance_credential_json zone=instance_credential_cache;

test/unit/s3gateway_test.js

@@ -311,11 +311,12 @@ function testEscapeURIPathPreservesDoubleSlashes() {
 
 function testReadCredentialsWithAccessAndSecretKeySet() {
     printHeader('testReadCredentialsWithAccessAndSecretKeySet');
+    let r = {};
     process.env['S3_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY';
     process.env['S3_SECRET_KEY'] = 'SOME_SECRET_KEY';
 
     try {
-        var credentials = s3gateway.readCredentials();
+        var credentials = s3gateway.readCredentials(r);
         if (credentials.accessKeyId !== process.env['S3_ACCESS_KEY_ID']) {
             throw 'static credentials do not match returned value [accessKeyId]';
         }
@@ -335,8 +336,14 @@ function testReadCredentialsWithAccessAndSecretKeySet() {
     }
 }
 
-function testReadCredentials() {
-    printHeader('testReadCredentials');
+function testReadCredentialsFromFilePath() {
+    printHeader('testReadCredentialsFromFilePath');
+    let r = {
+        variables: {
+            cache_instance_credentials_enabled: 0
+        }
+    };
+
     var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE'];
     var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp');
     var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`;
@@ -347,7 +354,7 @@ function testReadCredentials() {
 
     try {
         process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile;
-        var credentials = s3gateway.readCredentials();
+        var credentials = s3gateway.readCredentials(r);
         var testDataAsJSON = JSON.parse(testData);
         if (credentials.accessKeyId !== testDataAsJSON.accessKeyId) {
             throw 'JSON test data does not match credentials [accessKeyId]';
@@ -373,14 +380,19 @@ function testReadCredentials() {
 
 function testReadCredentialsFromNonexistentPath() {
     printHeader('testReadCredentialsFromNonexistentPath');
+    let r = {
+        variables: {
+            cache_instance_credentials_enabled: 0
+        }
+    };
     var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE'];
     var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp');
     var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`;
     var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`;
 
     try {
         process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile;
-        var credentials = s3gateway.readCredentials();
+        var credentials = s3gateway.readCredentials(r);
         if (credentials !== undefined) {
             throw 'Credentials returned when no credentials file should be present';
         }
@@ -395,6 +407,43 @@ function testReadCredentialsFromNonexistentPath() {
     }
 }
 
+function testReadAndWriteCredentialsFromKeyValStore() {
+    printHeader('testReadAndWriteCredentialsFromKeyValStore');
+
+    let accessKeyId = process.env['S3_ACCESS_KEY_ID'];
+    let secretKey = process.env['S3_SECRET_KEY'];
+    delete process.env.S3_ACCESS_KEY_ID;
+    delete process.env.S3_SECRET_KEY;
+
+    try {
+        let r = {
+            variables: {
+                cache_instance_credentials_enabled: 1,
+                instance_credential_json: null
+            }
+        };
+        let expectedCredentials = {
+            AccessKeyId: 'AN_ACCESS_KEY_ID',
+            Expiration: '2017-05-17T15:09:54Z',
+            RoleArn: 'TASK_ROLE_ARN',
+            SecretAccessKey: 'A_SECRET_ACCESS_KEY',
+            Token: 'A_SECURITY_TOKEN',
+        };
+
+        s3gateway.writeCredentials(r, expectedCredentials);
+        let credentials = JSON.stringify(s3gateway.readCredentials(r));
+        let expectedJson = JSON.stringify(expectedCredentials);
+
+        if (credentials !== expectedJson) {
+            console.log(`EXPECTED:\n${expectedJson}\nACTUAL:\n${credentials}`);
+            throw 'Credentials do not match expected value';
+        }
+    } finally {
+        process.env['S3_ACCESS_KEY_ID'] = accessKeyId;
+        process.env['S3_SECRET_KEY'] = secretKey;
+    }
+}
+
 async function testEcsCredentialRetrieval() {
     printHeader('testEcsCredentialRetrieval');
     process.env['S3_ACCESS_KEY_ID'] = undefined;
@@ -531,8 +580,9 @@ async function test() {
     testEditAmzHeadersHeadDirectory();
     testEscapeURIPathPreservesDoubleSlashes();
     testReadCredentialsWithAccessAndSecretKeySet();
-    testReadCredentials();
+    testReadCredentialsFromFilePath();
     testReadCredentialsFromNonexistentPath();
+    testReadAndWriteCredentialsFromKeyValStore();
     await testEcsCredentialRetrieval();
     await testEc2CredentialRetrieval();
 }