fix: ACNA-4162 - add constructor options to set log level and logRetryAfterSeconds

README.md

@@ -91,7 +91,7 @@ with defaults set to max of 3 retries and initial Delay as 100ms</p>
 ## Functions
 
 <dl>
-<dt><a href="#createFetch">createFetch([proxyAuthOptions])</a> ⇒ <code>function</code></dt>
+<dt><a href="#createFetch">createFetch([proxyOptions])</a> ⇒ <code>function</code></dt>
 <dd><p>Return the appropriate Fetch function depending on proxy settings.</p>
 </dd>
 <dt><a href="#parseRetryAfterHeader">parseRetryAfterHeader(header)</a> ⇒ <code>number</code></dt>
@@ -106,7 +106,7 @@ Spec: <a href="https://tools.ietf.org/html/rfc7231#section-7.1.3">https://tools.
 <dt><a href="#RetryOptions">RetryOptions</a> : <code>object</code></dt>
 <dd><p>Fetch Retry Options</p>
 </dd>
-<dt><a href="#ProxyAuthOptions">ProxyAuthOptions</a> : <code>object</code></dt>
+<dt><a href="#ProxyOptions">ProxyOptions</a> : <code>object</code></dt>
 <dd><p>Proxy Auth Options</p>
 </dd>
 </dl>
@@ -119,6 +119,23 @@ The retries use exponential backoff strategy
 with defaults set to max of 3 retries and initial Delay as 100ms
 
 **Kind**: global class  
+
+* [HttpExponentialBackoff](#HttpExponentialBackoff)
+    * [new HttpExponentialBackoff([options])](#new_HttpExponentialBackoff_new)
+    * [.exponentialBackoff(url, requestOptions, [retryOptions], [retryOn], [retryDelay])](#HttpExponentialBackoff+exponentialBackoff) ⇒ <code>Promise.&lt;Response&gt;</code>
+
+<a name="new_HttpExponentialBackoff_new"></a>
+
+### new HttpExponentialBackoff([options])
+Creates an instance of HttpExponentialBackoff
+
+
+| Param | Type | Description |
+| --- | --- | --- |
+| [options] | <code>object</code> | configuration options |
+| [options.logLevel] | <code>string</code> | the log level to use (default: process.env.LOG_LEVEL or 'info') |
+| [options.logRetryAfterSeconds] | <code>number</code> | the number of seconds after which to log a warning if the Retry-After header is greater than the number of seconds. Set to 0 to disable. |
+
 <a name="HttpExponentialBackoff+exponentialBackoff"></a>
 
 ### httpExponentialBackoff.exponentialBackoff(url, requestOptions, [retryOptions], [retryOn], [retryDelay]) ⇒ <code>Promise.&lt;Response&gt;</code>
@@ -144,23 +161,23 @@ This provides a wrapper for fetch that facilitates proxy auth authorization.
 **Kind**: global class  
 
 * [ProxyFetch](#ProxyFetch)
-    * [new ProxyFetch(proxyAuthOptions)](#new_ProxyFetch_new)
+    * [new ProxyFetch(proxyOptions)](#new_ProxyFetch_new)
     * [.fetch(resource, options)](#ProxyFetch+fetch) ⇒ <code>Promise.&lt;Response&gt;</code>
 
 <a name="new_ProxyFetch_new"></a>
 
-### new ProxyFetch(proxyAuthOptions)
+### new ProxyFetch(proxyOptions)
 Initialize this class with Proxy auth options
 
 
 | Param | Type | Description |
 | --- | --- | --- |
-| proxyAuthOptions | [<code>ProxyAuthOptions</code>](#ProxyAuthOptions) | the auth options to connect with |
+| proxyOptions | [<code>ProxyOptions</code>](#ProxyOptions) | the auth options to connect with |
 
 <a name="ProxyFetch+fetch"></a>
 
 ### proxyFetch.fetch(resource, options) ⇒ <code>Promise.&lt;Response&gt;</code>
-Fetch function, using the configured NTLM Auth options.
+Fetch function, using the configured fetch options, and proxy options (set in the constructor).
 
 **Kind**: instance method of [<code>ProxyFetch</code>](#ProxyFetch)  
 **Returns**: <code>Promise.&lt;Response&gt;</code> - Promise object representing the http response  
@@ -172,15 +189,15 @@ Fetch function, using the configured NTLM Auth options.
 
 <a name="createFetch"></a>
 
-## createFetch([proxyAuthOptions]) ⇒ <code>function</code>
+## createFetch([proxyOptions]) ⇒ <code>function</code>
 Return the appropriate Fetch function depending on proxy settings.
 
 **Kind**: global function  
 **Returns**: <code>function</code> - the Fetch API function  
 
 | Param | Type | Description |
 | --- | --- | --- |
-| [proxyAuthOptions] | [<code>ProxyAuthOptions</code>](#ProxyAuthOptions) | the proxy auth options |
+| [proxyOptions] | [<code>ProxyOptions</code>](#ProxyOptions) | the proxy options |
 
 <a name="parseRetryAfterHeader"></a>
 
@@ -207,11 +224,11 @@ Fetch Retry Options
 | --- | --- | --- |
 | maxRetries | <code>number</code> | the maximum number of retries to try (default:3) |
 | initialDelayInMillis | <code>number</code> | the initial delay in milliseconds (default:100ms) |
-| proxy | [<code>ProxyAuthOptions</code>](#ProxyAuthOptions) | the (optional) proxy auth options |
+| proxy | [<code>ProxyOptions</code>](#ProxyOptions) | the (optional) proxy auth options |
 
-<a name="ProxyAuthOptions"></a>
+<a name="ProxyOptions"></a>
 
-## ProxyAuthOptions : <code>object</code>
+## ProxyOptions : <code>object</code>
 Proxy Auth Options
 
 **Kind**: global typedef  
@@ -223,6 +240,7 @@ Proxy Auth Options
 | [username] | <code>string</code> | the username for basic auth |
 | [password] | <code>string</code> | the password for basic auth |
 | rejectUnauthorized | <code>boolean</code> | set to false to not reject unauthorized server certs |
+| [logLevel] | <code>string</code> | the log level to use (default: process.env.LOG_LEVEL or 'info') |
 
 ### Debug Logs
 

src/HttpExponentialBackoff.js

@@ -12,18 +12,18 @@ governing permissions and limitations under the License.
 const DEFAULT_MAX_RETRIES = 3
 const DEFAULT_INITIAL_DELAY_MS = 100
 const loggerNamespace = '@adobe/aio-lib-core-networking:HttpExponentialBackoff'
-const logger = require('@adobe/aio-lib-core-logging')(loggerNamespace, { level: process.env.LOG_LEVEL })
+const coreLogging = require('@adobe/aio-lib-core-logging')
 const { createFetch, parseRetryAfterHeader } = require('./utils')
 
-/* global Request, Response, ProxyAuthOptions */ // for linter
+/* global Request, Response, ProxyOptions */ // for linter
 
 /**
  * Fetch Retry Options
  *
  * @typedef {object} RetryOptions
  * @property {number} maxRetries the maximum number of retries to try (default:3)
  * @property {number} initialDelayInMillis the initial delay in milliseconds (default:100ms)
- * @property {ProxyAuthOptions} proxy  the (optional) proxy auth options
+ * @property {ProxyOptions} proxy  the (optional) proxy auth options
  */
 
 /**
@@ -32,6 +32,19 @@ const { createFetch, parseRetryAfterHeader } = require('./utils')
  * with defaults set to max of 3 retries and initial Delay as 100ms
  */
 class HttpExponentialBackoff {
+  /**
+   * Creates an instance of HttpExponentialBackoff
+   *
+   * @param {object} [options] configuration options
+   * @param {string} [options.logLevel] the log level to use (default: process.env.LOG_LEVEL or 'info')
+   * @param {number} [options.logRetryAfterSeconds] the number of seconds after which to log a warning if the Retry-After header is greater than the number of seconds. Set to 0 to disable.
+   */
+  constructor (options = {}) {
+    this.logLevel = options.logLevel || process.env.LOG_LEVEL || 'info'
+    this.logger = coreLogging(loggerNamespace, { level: this.logLevel })
+    this.logRetryAfterSeconds = options.logRetryAfterSeconds
+  }
+
   /**
    * This function will retry connecting to a url end-point, with
    * exponential backoff. Returns a Promise.
@@ -102,10 +115,10 @@ class HttpExponentialBackoff {
     * @private
     */
   __getRetryOn (retries) {
-    return function (attempt, error, response) {
+    return (attempt, error, response) => {
       if (attempt < retries && (error !== null || (response.status > 499 && response.status < 600) || response.status === 429)) {
         const msg = `Retrying after attempt ${attempt + 1}. failed: ${error || response.statusText}`
-        logger.debug(msg)
+        this.logger.debug(msg)
         return true
       }
       return false
@@ -120,10 +133,10 @@ class HttpExponentialBackoff {
     * @private
     */
   __getRetryDelay (initialDelayInMillis) {
-    return function (attempt, error, response) {
+    return (attempt, _error, _response) => {
       const timeToWait = Math.pow(2, attempt) * initialDelayInMillis // 1000, 2000, 4000
       const msg = `Request will be retried after ${timeToWait} ms`
-      logger.debug(msg)
+      this.logger.debug(msg)
       return timeToWait
     }
   }
@@ -140,10 +153,15 @@ class HttpExponentialBackoff {
   __getRetryDelayWithRetryAfterHeader (initialDelayInMillis) {
     return (attempt, error, response) => {
       const retryAfter = response?.headers.get('Retry-After') || null
-      const timeToWait = parseRetryAfterHeader(retryAfter)
-      if (!isNaN(timeToWait)) {
-        logger.debug(`Request will be retried after ${timeToWait} ms`)
-        return timeToWait
+      const timeToWaitMs = parseRetryAfterHeader(retryAfter)
+      if (!isNaN(timeToWaitMs)) {
+        const message = `Request will be retried after ${timeToWaitMs} ms`
+        if (this.logRetryAfterSeconds && timeToWaitMs > (this.logRetryAfterSeconds * 1000)) {
+          this.logger.warn(message)
+        } else {
+          this.logger.debug(message)
+        }
+        return timeToWaitMs
       }
       return this.__getRetryDelay(initialDelayInMillis)(attempt, error, response)
     }

src/ProxyFetch.js

@@ -10,7 +10,7 @@ governing permissions and limitations under the License.
 */
 
 const loggerNamespace = '@adobe/aio-lib-core-networking:ProxyFetch'
-const logger = require('@adobe/aio-lib-core-logging')(loggerNamespace, { level: process.env.LOG_LEVEL })
+const coreLogging = require('@adobe/aio-lib-core-logging')
 const originalFetch = require('node-fetch')
 const { codes } = require('./SDKErrors')
 const { HttpProxyAgent } = require('http-proxy-agent')
@@ -42,42 +42,44 @@ class PatchedHttpsProxyAgent extends HttpsProxyAgent {
  * @private
  *
  * @param {string} resourceUrl an endpoint url for proxyAgent selection
- * @param {ProxyAuthOptions} proxyOptions an object which contains auth information
+ * @param {ProxyOptions} proxyOptions an object which contains proxy information
+ * @param {object} logger the logger instance
  * @returns {http.Agent} a http.Agent for basic auth proxy
  */
-function proxyAgent (resourceUrl, proxyAuthOptions) {
+function proxyAgent (resourceUrl, proxyOptions, logger) {
   if (typeof resourceUrl !== 'string') {
     throw new codes.ERROR_PROXY_FETCH_INITIALIZATION_TYPE({ sdkDetails: { resourceUrl }, messageValues: 'resourceUrl must be of type string' })
   }
 
-  const { proxyUrl, username, password, rejectUnauthorized = true } = proxyAuthOptions
-  const proxyOpts = urlToHttpOptions(proxyUrl)
+  const { proxyUrl, username, password, rejectUnauthorized = true } = proxyOptions
+  const proxyHttpOptions = urlToHttpOptions(proxyUrl)
 
-  if (!proxyOpts.auth && username && password) {
+  if (!proxyHttpOptions.auth && username && password) {
     logger.debug('username and password not set in proxy url, using credentials passed in the constructor.')
-    proxyOpts.auth = `${username}:${password}`
+    proxyHttpOptions.auth = `${username}:${password}`
   }
 
-  proxyOpts.rejectUnauthorized = rejectUnauthorized
+  proxyHttpOptions.rejectUnauthorized = rejectUnauthorized
   if (rejectUnauthorized === false) {
     logger.warn(`proxyAgent - rejectUnauthorized is set to ${rejectUnauthorized}`)
   }
 
   if (resourceUrl.startsWith('https')) {
-    return new PatchedHttpsProxyAgent(proxyUrl, proxyOpts)
+    return new PatchedHttpsProxyAgent(proxyUrl, proxyHttpOptions)
   } else {
-    return new HttpProxyAgent(proxyUrl, proxyOpts)
+    return new HttpProxyAgent(proxyUrl, proxyHttpOptions)
   }
 }
 
 /**
  * Proxy Auth Options
  *
- * @typedef {object} ProxyAuthOptions
+ * @typedef {object} ProxyOptions
  * @property {string} proxyUrl - the proxy's url
  * @property {string} [username] the username for basic auth
  * @property {string} [password] the password for basic auth
  * @property {boolean} rejectUnauthorized - set to false to not reject unauthorized server certs
+ * @property {string} [logLevel] the log level to use (default: process.env.LOG_LEVEL or 'info')
  */
 
 /**
@@ -87,11 +89,14 @@ class ProxyFetch {
   /**
    * Initialize this class with Proxy auth options
    *
-   * @param {ProxyAuthOptions} proxyAuthOptions the auth options to connect with
+   * @param {ProxyOptions} proxyOptions the auth options to connect with
    */
-  constructor (proxyAuthOptions = {}) {
-    logger.debug(`constructor - authOptions: ${JSON.stringify(proxyAuthOptions)}`)
-    const { proxyUrl } = proxyAuthOptions
+  constructor (proxyOptions = {}) {
+    this.logLevel = proxyOptions.logLevel || process.env.LOG_LEVEL || 'info'
+    this.logger = coreLogging(loggerNamespace, { level: this.logLevel })
+
+    this.logger.debug(`constructor - authOptions: ${JSON.stringify(proxyOptions)}`)
+    const { proxyUrl } = proxyOptions
     const { auth } = urlToHttpOptions(proxyUrl)
 
     if (!proxyUrl) {
@@ -100,15 +105,15 @@ class ProxyFetch {
     }
 
     if (!auth) {
-      logger.debug('constructor: username or password not set, proxy is anonymous.')
+      this.logger.debug('constructor: username or password not set, proxy is anonymous.')
     }
 
-    this.authOptions = proxyAuthOptions
+    this.proxyOptions = proxyOptions
     return this
   }
 
   /**
-   * Fetch function, using the configured NTLM Auth options.
+   * Fetch function, using the configured fetch options, and proxy options (set in the constructor).
    *
    * @param {string | Request} resource - the url or Request object to fetch from
    * @param {object} options - the fetch options
@@ -117,7 +122,7 @@ class ProxyFetch {
   async fetch (resource, options = {}) {
     return originalFetch(resource, {
       ...options,
-      agent: proxyAgent(resource, this.authOptions)
+      agent: proxyAgent(resource, this.proxyOptions, this.logger)
     })
   }
 }

src/utils.js

@@ -15,29 +15,29 @@ const { getProxyForUrl } = require('proxy-from-env')
 const loggerNamespace = '@adobe/aio-lib-core-networking/utils'
 const logger = require('@adobe/aio-lib-core-logging')(loggerNamespace, { level: process.env.LOG_LEVEL })
 
-/* global ProxyAuthOptions */
+/* global ProxyOptions */
 
 /**
  * Return the appropriate Fetch function depending on proxy settings.
  *
- * @param {ProxyAuthOptions} [proxyAuthOptions] the proxy auth options
+ * @param {ProxyOptions} [proxyOptions] the proxy options
  * @returns {Function} the Fetch API function
  */
-function createFetch (proxyAuthOptions) {
+function createFetch (proxyOptions) {
   const fn = async (resource, options) => {
-    // proxyAuthOptions as a parameter will override any proxy env vars
-    if (!proxyAuthOptions) {
+    // proxyOptions as a parameter will override any proxy env vars
+    if (!proxyOptions) {
       const proxyUrl = getProxyForUrl(resource)
       if (proxyUrl) {
-        proxyAuthOptions = { proxyUrl }
+        proxyOptions = { proxyUrl }
       }
     }
 
-    if (proxyAuthOptions) {
-      logger.debug(`createFetch: proxy url found ${proxyAuthOptions.proxyUrl} for url ${resource}`)
+    if (proxyOptions) {
+      logger.debug(`createFetch: proxy url found ${proxyOptions.proxyUrl} for url ${resource}`)
       // in this closure: for fetch-retry, if we don't require it dynamically here, ProxyFetch will be an empty object
       const ProxyFetch = require('./ProxyFetch')
-      const proxyFetch = new ProxyFetch(proxyAuthOptions)
+      const proxyFetch = new ProxyFetch(proxyOptions)
       return proxyFetch.fetch(resource, options)
     } else {
       logger.debug('createFetch: proxy url not found, using plain fetch')