Fixes nginx#118

The gateway will now use the S3_ACCESS_KEY_ID and S3_SECRET_KEY
environment variables when running in an EC2 instance.

Signed-off-by: Elijah Zupancic <e.zupancic@f5.com>

Author: dekobon

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -34,6 +34,9 @@ required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO"
 if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then
   echo "Running inside an ECS task, using container credentials"
 
+elif [[ -v S3_SESSION_TOKEN ]]; then
+  echo "S3 Session token specified - not using IMDS for credentials"
+
 # b) Using Instance Metadata Service (IMDS) credentials, if IMDS is present at http://169.254.169.254.
 #    See https://docs.aws.amazon.com/sdkref/latest/guide/feature-imds-credentials.html.
 #    Example: We are running inside an EC2 instance.
@@ -52,10 +55,6 @@ else
   required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY")
 fi
 
-if [[ -v S3_SESSION_TOKEN ]]; then
-  echo "S3 Session token present"
-fi
-
 for name in ${required[@]}; do
   if [[ ! -v name ]]; then
       >&2 echo "Required ${name} environment variable missing"

common/etc/nginx/include/s3gateway.js

@@ -19,13 +19,13 @@ import awssig2 from "./awssig2.js";
 import awssig4 from "./awssig4.js";
 import utils from "./utils.js";
 
-_require_env_var('S3_BUCKET_NAME');
-_require_env_var('S3_SERVER');
-_require_env_var('S3_SERVER_PROTO');
-_require_env_var('S3_SERVER_PORT');
-_require_env_var('S3_REGION');
-_require_env_var('AWS_SIGS_VERSION');
-_require_env_var('S3_STYLE');
+_requireEnvVars('S3_BUCKET_NAME');
+_requireEnvVars('S3_SERVER');
+_requireEnvVars('S3_SERVER_PROTO');
+_requireEnvVars('S3_SERVER_PORT');
+_requireEnvVars('S3_REGION');
+_requireEnvVars('AWS_SIGS_VERSION');
+_requireEnvVars('S3_STYLE');
 
 const fs = require('fs');
 
@@ -208,7 +208,7 @@ function _s3ReqParamsForSigV2(r, bucket) {
     if (PROVIDE_INDEX_PAGE && _isDirectory(r.variables.uri_path)){
         uri = r.variables.uri_path + INDEX_PAGE
     }
- 
+
     return {
         uri: '/' + bucket + uri,
         httpDate: s3date(r)
@@ -469,11 +469,11 @@ function _isDirectory(path) {
  * @param envVarName {string} environment variable to check for
  * @private
  */
-function _require_env_var(envVarName) {
+function _requireEnvVars(envVarName) {
     const isSet = envVarName in process.env;
 
     if (!isSet) {
-        throw('Required environment variable ' + envVarName + ' is missing');
+        throw(`Required environment variable ${envVarName} is missing`);
     }
 }
 
@@ -509,7 +509,7 @@ const maxValidityOffsetMs = 4.5 * 60 * 1000;
 async function fetchCredentials(r) {
     /* If we are not using an AWS instance profile to set our credentials we
        exit quickly and don't write a credentials file. */
-    if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
+    if (utils.areAllEnvVarsSet(['S3_ACCESS_KEY_ID', 'S3_SECRET_KEY'])) {
         r.return(200);
         return;
     }
@@ -539,8 +539,9 @@ async function fetchCredentials(r) {
 
     utils.debug_log(r, 'Cached credentials are expired or not present, requesting new ones');
 
-    if (process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']) {
-        const uri = ECS_CREDENTIAL_BASE_URI + process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'];
+    if (utils.areAllEnvVarsSet('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')) {
+        const relative_uri = process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || '';
+        const uri = ECS_CREDENTIAL_BASE_URI + relative_uri;
         try {
             credentials = await _fetchEcsRoleCredentials(uri);
         } catch (e) {
@@ -549,7 +550,7 @@ async function fetchCredentials(r) {
             return;
         }
     }
-    else if(process.env['AWS_WEB_IDENTITY_TOKEN_FILE']) {
+    else if (utils.areAllEnvVarsSet('AWS_WEB_IDENTITY_TOKEN_FILE')) {
         try {
             credentials = await _fetchWebIdentityCredentials(r)
         } catch(e) {

common/etc/nginx/include/utils.js

@@ -21,6 +21,27 @@
  */
 const DEBUG = parseBoolean(process.env['S3_DEBUG']);
 
+/**
+ * Checks to see if all of the elements of the passed array are present as keys
+ * in the running process' environment variables. Alternatively, if a single
+ * string is passed, it will check for the presence of that string.
+ * @param envVars {array[string]|string} array of expected keys or single expected key
+ * @returns {boolean} true if all keys are set as environment variables
+ */
+function areAllEnvVarsSet(envVars) {
+    if (envVars instanceof Array) {
+        const envVarsLen = envVars.length;
+        for (let i = 0; i < envVarsLen; i++) {
+            if (!process.env[envVars[i]]) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    return envVars in process.env;
+}
 
 /**
  * Parses a string delimited by semicolons into an array of values
@@ -133,5 +154,6 @@ export default {
     getEightDigitDate,
     padWithLeadingZeros,
     parseArray,
-    parseBoolean
+    parseBoolean,
+    areAllEnvVarsSet
 }

test/unit/s3gateway_test.js

@@ -204,9 +204,10 @@ function testEscapeURIPathPreservesDoubleSlashes() {
 
 async function testEcsCredentialRetrieval() {
     printHeader('testEcsCredentialRetrieval');
-    process.env['S3_ACCESS_KEY_ID'] = undefined;
+    delete process.env['S3_ACCESS_KEY_ID'];
     process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = '/example';
     globalThis.ngx.fetch = function (url) {
+        console.log('  fetching mock credentials');
         globalThis.recordedUrl = url;
 
         return Promise.resolve({
@@ -245,14 +246,14 @@ async function testEcsCredentialRetrieval() {
     await s3gateway.fetchCredentials(r);
 
     if (globalThis.recordedUrl !== 'http://169.254.170.2/example') {
-        throw 'No or wrong ECS credentials fetch URL recorded: ' + globalThis.recordedUrl;
+        throw `No or wrong ECS credentials fetch URL recorded: ${globalThis.recordedUrl}`;
     }
 }
 
 async function testEc2CredentialRetrieval() {
     printHeader('testEc2CredentialRetrieval');
-    process.env['S3_ACCESS_KEY_ID'] = undefined;
-    process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = undefined;
+    delete process.env['S3_ACCESS_KEY_ID'];
+    delete process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'];
     globalThis.ngx.fetch = function (url, options) {
         if (url === 'http://169.254.169.254/latest/api/token' && options && options.method === 'PUT') {
             return Promise.resolve({

test/unit/utils_test.js

@@ -122,13 +122,73 @@ function testPad() {
     }
 }
 
+function testAreAllEnvVarsSet() {
+    function testAreAllEnvVarsSetStringFound() {
+        console.log('  ## testAreAllEnvVarsSetStringFound');
+        const key = 'TEST_ENV_VAR_KEY';
+        process.env[key] = 'some value';
+        const actual = utils.areAllEnvVarsSet(key);
+        if (!actual) {
+            throw 'Environment variable that was set not indicated as present';
+        }
+    }
+
+    function testAreAllEnvVarsSetStringNotFound() {
+        console.log('  ## testAreAllEnvVarsSetStringNotFound');
+        const actual = utils.areAllEnvVarsSet('UNKNOWN_ENV_VAR_KEY');
+        if (actual) {
+            throw 'Unknown environment variable indicated as being present';
+        }
+    }
+
+    function testAreAllEnvVarsSetStringArrayFound() {
+        console.log('  ## testAreAllEnvVarsSetStringArrayFound');
+        const keys = ['TEST_ENV_VAR_KEY_1', 'TEST_ENV_VAR_KEY_2', 'TEST_ENV_VAR_KEY_3'];
+        for (let i = 0; i < keys.length; i++) {
+            process.env[keys[i]] = 'something';
+        }
+        const actual = utils.areAllEnvVarsSet(keys);
+        if (!actual) {
+            throw 'Environment variables that were set not indicated as present';
+        }
+    }
+
+    function testAreAllEnvVarsSetStringArrayNotFound() {
+        console.log('  ## testAreAllEnvVarsSetStringArrayNotFound');
+        const keys = ['UNKNOWN_ENV_VAR_KEY_1', 'UNKNOWN_ENV_VAR_KEY_2', 'UNKNOWN_ENV_VAR_KEY_3'];
+        const actual = utils.areAllEnvVarsSet(keys);
+        if (actual) {
+            throw 'Unknown environment variables that were not set indicated as present';
+        }
+    }
+
+    function testAreAllEnvVarsSetStringArrayWithSomeSet() {
+        console.log('  ## testAreAllEnvVarsSetStringArrayWithSomeSet');
+        const keys = ['TEST_ENV_VAR_KEY_1', 'UNKNOWN_ENV_VAR_KEY_2', 'UNKNOWN_ENV_VAR_KEY_3'];
+        process.env[keys[0]] = 'something';
+
+        const actual = utils.areAllEnvVarsSet(keys);
+        if (actual) {
+            throw 'Unknown environment variables that were not set indicated as present';
+        }
+    }
+
+    printHeader('testAreAllEnvVarsSet');
+    testAreAllEnvVarsSetStringFound();
+    testAreAllEnvVarsSetStringNotFound();
+    testAreAllEnvVarsSetStringArrayFound();
+    testAreAllEnvVarsSetStringArrayNotFound();
+    testAreAllEnvVarsSetStringArrayWithSomeSet();
+}
+
 async function test() {
     testAmzDatetime();
     testEightDigitDate();
     testPad();
     testParseArray();
+    testAreAllEnvVarsSet();
 }
-    
+
 function printHeader(testName) {
     console.log(`\n## ${testName}`);
 }