feat: payload hash func, default host name var, common var for AWS credentials and debug

- Move reusable functions of fectching credentials into awscredentials.js
- Replace name of env variables of AWS credentials and debug to support common AWS signature lib for multiple services in AWS
- Add a NJS var of default host name to be used for Role Session Name at multiple services in AWS
- Add a payload hash function to support multiple services in AWS such as S3 and Lambda, and it can be used for not only GET but also POST method.

Author: shawnhankim

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -34,7 +34,7 @@ required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO"
 if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then
   echo "Running inside an ECS task, using container credentials"
 
-elif [[ -v S3_SESSION_TOKEN ]]; then
+elif [[ -v AWS_SESSION_TOKEN ]]; then
   echo "S3 Session token specified - not using IMDS for credentials"
 
 # b) Using Instance Metadata Service (IMDS) credentials, if IMDS is present at http://169.254.169.254.
@@ -52,7 +52,7 @@ elif [[ -v AWS_WEB_IDENTITY_TOKEN_FILE ]]; then
 # If none of the options above is used, require static credentials.
 # See https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html.
 else
-  required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY")
+  required+=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY")
 fi
 
 for name in ${required[@]}; do
@@ -101,7 +101,7 @@ if [ $failed -gt 0 ]; then
 fi
 
 echo "S3 Backend Environment"
-echo "Access Key ID: ${S3_ACCESS_KEY_ID}"
+echo "Access Key ID: ${AWS_ACCESS_KEY_ID}"
 echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}"
 echo "Region: ${S3_REGION}"
 echo "Addressing Style: ${S3_STYLE}"

common/etc/nginx/include/awscredentials.js

@@ -18,6 +18,40 @@ import utils from "./utils.js";
 
 const fs = require('fs');
 
+/**
+ * The current moment as a timestamp. This timestamp will be used across
+ * functions in order for there to be no variations in signatures.
+ * @type {Date}
+ */
+const NOW = new Date();
+
+/**
+ * Constant base URI to fetch credentials together with the credentials relative URI, see
+ * https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html for more details.
+ * @type {string}
+ */
+const ECS_CREDENTIAL_BASE_URI = 'http://169.254.170.2';
+
+/**
+ * @type {string}
+ */
+const EC2_IMDS_TOKEN_ENDPOINT = 'http://169.254.169.254/latest/api/token';
+
+const EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/';
+
+/**
+ * Offset to the expiration of credentials, when they should be considered expired and refreshed. The maximum
+ * time here can be 5 minutes, the IMDS and ECS credentials endpoint will make sure that each returned set of credentials
+ * is valid for at least another 5 minutes.
+ *
+ * To make sure we always refresh the credentials instead of retrieving the same again, keep credentials until 4:30 minutes
+ * before they really expire.
+ *
+ * @type {number}
+ */
+const maxValidityOffsetMs = 4.5 * 60 * 1000;
+
+
 /**
  * Get the current session token from either the instance profile credential 
  * cache or environment variables.
@@ -34,24 +68,26 @@ function sessionToken(r) {
 }
 
 /**
- * Get the instance profile credentials needed to authenticated against S3 from
+ * Get the instance profile credentials needed to authenticate against S3 from
  * a backend cache. If the credentials cannot be found, then return undefined.
  * @param r {Request} HTTP request object (not used, but required for NGINX configuration)
  * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string|null), expiration: (string|null)}} AWS instance profile credentials or undefined
  */
 function readCredentials(r) {
     // TODO: Change the generic constants naming for multiple AWS services.
-    if ('S3_ACCESS_KEY_ID' in process.env && 'S3_SECRET_KEY' in process.env) {
-        const sessionToken = 'S3_SESSION_TOKEN' in process.env ?
-                              process.env['S3_SESSION_TOKEN'] : null;
+    if ('AWS_ACCESS_KEY_ID' in process.env && 'AWS_SECRET_ACCESS_KEY' in process.env) {
+        let sessionToken = 'AWS_SESSION_TOKEN' in process.env ?
+            process.env['AWS_SESSION_TOKEN'] : null;
+        if (sessionToken !== null && sessionToken.length === 0) {
+            sessionToken = null;
+        }
         return {
-            accessKeyId: process.env['S3_ACCESS_KEY_ID'],
-            secretAccessKey: process.env['S3_SECRET_KEY'],
+            accessKeyId: process.env['AWS_ACCESS_KEY_ID'],
+            secretAccessKey: process.env['AWS_SECRET_ACCESS_KEY'],
             sessionToken: sessionToken,
             expiration: null
         };
     }
-
     if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) {
         return _readCredentialsFromKeyValStore(r);
     } else {
@@ -113,8 +149,8 @@ function _readCredentialsFromFile() {
  * @private
  */
 function _credentialsTempFile() {
-    if (process.env['S3_CREDENTIALS_TEMP_FILE']) {
-        return process.env['S3_CREDENTIALS_TEMP_FILE'];
+    if (process.env['AWS_CREDENTIALS_TEMP_FILE']) {
+        return process.env['AWS_CREDENTIALS_TEMP_FILE'];
     }
     if (process.env['TMPDIR']) {
         return `${process.env['TMPDIR']}/credentials.json`
@@ -132,7 +168,7 @@ function _credentialsTempFile() {
 function writeCredentials(r, credentials) {
     /* Do not bother writing credentials if we are running in a mode where we
        do not need instance credentials. */
-    if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
+    if (process.env['AWS_ACCESS_KEY_ID'] && process.env['AWS_SECRET_ACCESS_KEY']) {
         return;
     }
 
@@ -171,7 +207,232 @@ function _writeCredentialsToFile(credentials) {
     fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials));
 }
 
+/**
+ * Get the credentials needed to create AWS signatures in order to authenticate
+ * to AWS service. If the gateway is being provided credentials via a instance 
+ * profile credential as provided over the metadata endpoint, this function will:
+ * 1. Try to read the credentials from cache
+ * 2. Determine if the credentials are stale
+ * 3. If the cached credentials are missing or stale, it gets new credentials
+ *    from the metadata endpoint.
+ * 4. If new credentials were pulled, it writes the credentials back to the
+ *    cache.
+ *
+ * If the gateway is not using instance profile credentials, then this function
+ * quickly exits.
+ *
+ * @param r {Request} HTTP request object
+ * @returns {Promise<void>}
+ */
+async function fetchCredentials(r) {
+    /* If we are not using an AWS instance profile to set our credentials we
+       exit quickly and don't write a credentials file. */
+    if (utils.areAllEnvVarsSet(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'])) {
+        r.return(200);
+        return;
+    }
+
+    let current;
+
+    try {
+        current = readCredentials(r);
+    } catch (e) {
+        utils.debug_log(r, `Could not read credentials: ${e}`);
+        r.return(500);
+        return;
+    }
+
+    if (current) {
+        // If AWS returns a Unix timestamp it will be in seconds, but in Date constructor we should provide timestamp in milliseconds
+        // In some situations (including EC2 and Fargate) current.expiration will be an RFC 3339 string - see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
+        const expireAt = typeof current.expiration == 'number' ? current.expiration * 1000 : current.expiration
+        const exp = new Date(expireAt).getTime() - maxValidityOffsetMs;
+        if (NOW.getTime() < exp) {
+            r.return(200);
+            return;
+        }
+    }
+
+    let credentials;
+
+    utils.debug_log(r, 'Cached credentials are expired or not present, requesting new ones');
+
+    if (utils.areAllEnvVarsSet('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')) {
+        const relative_uri = process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || '';
+        const uri = ECS_CREDENTIAL_BASE_URI + relative_uri;        
+        try {
+            credentials = await _fetchEcsRoleCredentials(uri);
+        } catch (e) {
+            utils.debug_log(r, 'Could not load ECS task role credentials: ' + JSON.stringify(e));
+            r.return(500);
+            return;
+        }
+    }
+    else if (utils.areAllEnvVarsSet('AWS_WEB_IDENTITY_TOKEN_FILE')) {
+        try {
+            credentials = await _fetchWebIdentityCredentials(r)
+        } catch (e) {
+            utils.debug_log(r, 'Could not assume role using web identity: ' + JSON.stringify(e));
+            r.return(500);
+            return;
+        }
+    } else {
+        try {
+            credentials = await _fetchEC2RoleCredentials();
+        } catch (e) {
+            utils.debug_log(r, 'Could not load EC2 task role credentials: ' + JSON.stringify(e));
+            r.return(500);
+            return;
+        }
+    }
+    try {
+        writeCredentials(r, credentials);
+    } catch (e) {
+        utils.debug_log(r, `Could not write credentials: ${e}`);
+        r.return(500);
+        return;
+    }
+    r.return(200);
+}
+
+/**
+ * Get the credentials needed to generate AWS signatures from the ECS
+ * (Elastic Container Service) metadata endpoint.
+ *
+ * @param credentialsUri {string} endpoint to get credentials from
+ * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>}
+ * @private
+ */
+async function _fetchEcsRoleCredentials(credentialsUri) {
+    const resp = await ngx.fetch(credentialsUri);
+    if (!resp.ok) {
+        throw 'Credentials endpoint response was not ok.';
+    }
+    const creds = await resp.json();
+
+    return {
+        accessKeyId: creds.AccessKeyId,
+        secretAccessKey: creds.SecretAccessKey,
+        sessionToken: creds.Token,
+        expiration: creds.Expiration,
+    };
+}
+
+/**
+ * Get the credentials needed to generate AWS signatures from the EC2
+ * metadata endpoint.
+ *
+ * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>}
+ * @private
+ */
+async function _fetchEC2RoleCredentials() {
+    const tokenResp = await ngx.fetch(EC2_IMDS_TOKEN_ENDPOINT, {
+        headers: {
+            'x-aws-ec2-metadata-token-ttl-seconds': '21600',
+        },
+        method: 'PUT',
+    });
+    const token = await tokenResp.text();
+    let resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT, {
+        headers: {
+            'x-aws-ec2-metadata-token': token,
+        },
+    });
+    /* This _might_ get multiple possible roles in other scenarios, however,
+       EC2 supports attaching one role only.It should therefore be safe to take
+       the whole output, even given IMDS _might_ (?) be able to return multiple
+       roles. */
+    const credName = await resp.text();
+    if (credName === "") {
+        throw 'No credentials available for EC2 instance';
+    }
+    resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT + credName, {
+        headers: {
+            'x-aws-ec2-metadata-token': token,
+        },
+    });
+    const creds = await resp.json();
+
+    return {
+        accessKeyId: creds.AccessKeyId,
+        secretAccessKey: creds.SecretAccessKey,
+        sessionToken: creds.Token,
+        expiration: creds.Expiration,
+    };
+}
+
+/**
+ * Get the credentials by assuming calling AssumeRoleWithWebIdentity with the environment variable
+ * values ROLE_ARN, AWS_WEB_IDENTITY_TOKEN_FILE and HOSTNAME
+ *
+ * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>}
+ * @private
+ */
+async function _fetchWebIdentityCredentials(r) {
+    const arn = process.env['AWS_ROLE_ARN'];
+    const name = process.env['HOSTNAME'] || r.variables.defaultHostName;
+
+    let sts_endpoint = process.env['STS_ENDPOINT'];
+    if (!sts_endpoint) {
+        /* On EKS, the ServiceAccount can be annotated with
+           'eks.amazonaws.com/sts-regional-endpoints' to control
+           the usage of regional endpoints. We are using the same standard
+           environment variable here as the AWS SDK. This is with the exception
+           of replacing the value `legacy` with `global` to match what EKS sets
+           the variable to.
+           See: https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html
+           See: https://docs.aws.amazon.com/eks/latest/userguide/configure-sts-endpoint.html */
+        const sts_regional = process.env['AWS_STS_REGIONAL_ENDPOINTS'] || 'global';
+        if (sts_regional === 'regional') {
+            /* STS regional endpoints can be derived from the region's name.
+               See: https://docs.aws.amazon.com/general/latest/gr/sts.html */
+            const region = process.env['AWS_REGION'];
+            if (region) {
+                sts_endpoint = `https://sts.${region}.amazonaws.com`;
+            } else {
+                throw 'Missing required AWS_REGION env variable';
+            }
+        } else {
+            // This is the default global endpoint
+            sts_endpoint = 'https://sts.amazonaws.com';
+        }
+    }
+
+    const token = fs.readFileSync(process.env['AWS_WEB_IDENTITY_TOKEN_FILE']);
+
+    const params = `Version=2011-06-15&Action=AssumeRoleWithWebIdentity&RoleArn=${arn}&RoleSessionName=${name}&WebIdentityToken=${token}`;
+
+    const response = await ngx.fetch(sts_endpoint + "?" + params, {
+        headers: {
+            "Accept": "application/json"
+        },
+        method: 'GET',
+    });
+
+    const resp = await response.json();
+    const creds = resp.AssumeRoleWithWebIdentityResponse.AssumeRoleWithWebIdentityResult.Credentials;
+
+    return {
+        accessKeyId: creds.AccessKeyId,
+        secretAccessKey: creds.SecretAccessKey,
+        sessionToken: creds.SessionToken,
+        expiration: creds.Expiration,
+    };
+}
+
+/**
+ * Get the current timestamp. This timestamp will be used across functions in 
+ * order for there to be no variations in signatures.
+ *
+ * @returns {Date} The current moment as a timestamp
+ */
+function Now() {
+    return NOW;
+}
+
 export default {
+    Now,
+    fetchCredentials,
     readCredentials,
     sessionToken,
     writeCredentials