📦 NEW: Support digest auth (#390)

closes #388

README.md

@@ -75,23 +75,24 @@ console.log('status: %s, body size: %d, headers: %j', res.statusCode, data.lengt
 
 #### Arguments
 
-- **url** String | Object - The URL to request, either a String or a Object that return by [url.parse](http://nodejs.org/api/url.html#url_url_parse_urlstr_parsequerystring_slashesdenotehost).
+- **url** String | Object - The URL to request, either a String or a Object that return by [url.parse](https://nodejs.org/api/url.html#url_url_parse_urlstr_parsequerystring_slashesdenotehost).
 - ***options*** Object - Optional
   - ***method*** String - Request method, defaults to `GET`. Could be `GET`, `POST`, `DELETE` or `PUT`. Alias 'type'.
   - ***data*** Object - Data to be sent. Will be stringify automatically.
-  - ***content*** String | [Buffer](http://nodejs.org/api/buffer.html) - Manually set the content of payload. If set, `data` will be ignored.
-  - ***stream*** [stream.Readable](http://nodejs.org/api/stream.html#stream_class_stream_readable) - Stream to be pipe to the remote. If set, `data` and `content` will be ignored.
-  - ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
+  - ***content*** String | [Buffer](https://nodejs.org/api/buffer.html) - Manually set the content of payload. If set, `data` will be ignored.
+  - ***stream*** [stream.Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable) - Stream to be pipe to the remote. If set, `data` and `content` will be ignored.
+  - ***writeStream*** [stream.Writable](https://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
   - ***files*** {Array<ReadStream|Buffer|String> | Object | ReadStream | Buffer | String - The files will send with `multipart/form-data` format, base on `formstream`. If `method` not set, will use `POST` method by default.
   - ***contentType*** String - Type of request data. Could be `json` (**Notes**: not use `application/json` here). If it's `json`, will auto set `Content-Type: application/json` header.
   - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
   - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.
   - ***headers*** Object - Request headers.
   - ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
   - ***auth*** String - `username:password` used in HTTP Basic Authorization.
-  - ***agent*** [http.Agent](http://nodejs.org/api/http.html#http_class_http_agent) - HTTP Agent object.
+  - ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](https://en.wikipedia.org/wiki/Digest_access_authentication).
+  - ***agent*** [http.Agent](https://nodejs.org/api/http.html#http_class_http_agent) - HTTP Agent object.
       Set `false` if you does not use agent.
-  - ***httpsAgent*** [https.Agent](http://nodejs.org/api/https.html#https_class_https_agent) - HTTPS Agent object.
+  - ***httpsAgent*** [https.Agent](https://nodejs.org/api/https.html#https_class_https_agent) - HTTPS Agent object.
       Set `false` if you does not use agent.
   - ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
   - ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.

src/HttpClient.ts

@@ -27,7 +27,7 @@ import pump from 'pump';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent';
 import { RequestURL, RequestOptions, HttpMethod } from './Request';
 import { HttpClientResponseMeta, HttpClientResponse, ReadableWithMeta } from './Response';
-import { parseJSON, sleep } from './utils';
+import { parseJSON, sleep, digestAuthHeader } from './utils';
 
 const FormData = FormDataNative ?? FormDataNode;
 // impl isReadable on Node.js 14
@@ -178,6 +178,7 @@ export class HttpClient extends EventEmitter {
       url: requestUrl.href,
       args,
       ctx: args.ctx,
+      retries: requestContext.retries,
     };
     // keep urllib createCallbackResponse style
     const resHeaders: IncomingHttpHeaders = {};
@@ -324,8 +325,7 @@ export class HttpClient extends EventEmitter {
           requestOptions.body = args.content;
           if (args.contentType) {
             headers['content-type'] = args.contentType;
-          }
-          if (typeof args.content === 'string' && !headers['content-type']) {
+          } else if (typeof args.content === 'string' && !headers['content-type']) {
             headers['content-type'] = 'text/plain;charset=UTF-8';
           }
         }
@@ -365,7 +365,24 @@ export class HttpClient extends EventEmitter {
         this.emit('request', reqMeta);
       }
 
-      const response = await undiciRequest(requestUrl, requestOptions);
+      let response = await undiciRequest(requestUrl, requestOptions);
+      // handle digest auth
+      if (response.statusCode === 401 && response.headers['www-authenticate'] &&
+          !requestOptions.headers.authorization && args.digestAuth) {
+        const authenticate = response.headers['www-authenticate'];
+        if (authenticate.startsWith('Digest ')) {
+          debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
+          requestOptions.headers.authorization = digestAuthHeader(requestOptions.method!,
+            requestUrl.pathname, authenticate, args.digestAuth);
+          debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
+          if (response.headers['set-cookie']) {
+            // FIXME: merge exists cookie header
+            requestOptions.headers.cookie = response.headers['set-cookie'].join(';');
+          }
+          response = await undiciRequest(requestUrl, requestOptions);
+        }
+      }
+
       opaque = response.opaque;
       if (args.timing) {
         res.timing.waiting = performanceTime(requestStartTime);

src/Request.ts

@@ -76,6 +76,10 @@ export type RequestOptions = {
    * Alias to `headers.authorization = xxx`
    **/
   auth?: string;
+  /**
+   * username:password used in HTTP Digest Authorization.
+   * */
+  digestAuth?: string;
   /** follow HTTP 3xx responses as redirects. defaults to true. */
   followRedirect?: boolean;
   /** The maximum number of redirects to follow, defaults to 10. */

src/utils.ts

@@ -1,3 +1,4 @@
+import { randomBytes, createHash } from 'crypto';
 import { FixJSONCtlChars } from './Request';
 
 const JSONCtlCharsMap = {
@@ -51,3 +52,77 @@ export function sleep(ms: number) {
     setTimeout(resolve, ms);
   });
 }
+
+function md5(s: string) {
+  const sum = createHash('md5');
+  sum.update(s, 'utf8');
+  return sum.digest('hex');
+}
+
+const AUTH_KEY_VALUE_RE = /(\w{1,100})=["']?([^'"]+)["']?/;
+let NC = 0;
+const NC_PAD = '00000000';
+
+export function digestAuthHeader(method: string, uri: string, wwwAuthenticate: string, userpass: string) {
+  // WWW-Authenticate: Digest realm="testrealm@host.com",
+  //                       qop="auth,auth-int",
+  //                       nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",
+  //                       opaque="5ccc069c403ebaf9f0171e9517f40e41"
+  // Authorization: Digest username="Mufasa",
+  //                    realm="testrealm@host.com",
+  //                    nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",
+  //                    uri="/dir/index.html",
+  //                    qop=auth,
+  //                    nc=00000001,
+  //                    cnonce="0a4f113b",
+  //                    response="6629fae49393a05397450978507c4ef1",
+  //                    opaque="5ccc069c403ebaf9f0171e9517f40e41"
+  // HA1 = MD5( "Mufasa:testrealm@host.com:Circle Of Life" )
+  //      = 939e7578ed9e3c518a452acee763bce9
+  //
+  //  HA2 = MD5( "GET:/dir/index.html" )
+  //      = 39aff3a2bab6126f332b942af96d3366
+  //
+  //  Response = MD5( "939e7578ed9e3c518a452acee763bce9:\
+  //                   dcd98b7102dd2f0e8b11d0f600bfb0c093:\
+  //                   00000001:0a4f113b:auth:\
+  //                   39aff3a2bab6126f332b942af96d3366" )
+  //           = 6629fae49393a05397450978507c4ef1
+  const parts = wwwAuthenticate.split(',');
+  const opts: Record<string, string> = {};
+  for (const part of parts) {
+    const m = part.match(AUTH_KEY_VALUE_RE);
+    if (m) {
+      opts[m[1]] = m[2].replace(/["']/g, '');
+    }
+  }
+
+  if (!opts.realm || !opts.nonce) {
+    return '';
+  }
+
+  let qop = opts.qop || '';
+  const [ user, pass ] = userpass.split(':');
+
+  let nc = String(++NC);
+  nc = `${NC_PAD.substring(nc.length)}${nc}`;
+  const cnonce = randomBytes(8).toString('hex');
+
+  const ha1 = md5(`${user}:${opts.realm}:${pass}`);
+  const ha2 = md5(`${method.toUpperCase()}:${uri}`);
+  let s = `${ha1}:${opts.nonce}`;
+  if (qop) {
+    qop = qop.split(',')[0];
+    s += `:${nc}:${cnonce}:${qop}`;
+  }
+  s += `:${ha2}`;
+  const response = md5(s);
+  let authstring = `Digest username="${user}", realm="${opts.realm}", nonce="${opts.nonce}", uri="${uri}", response="${response}"`;
+  if (opts.opaque) {
+    authstring += `, opaque="${opts.opaque}"`;
+  }
+  if (qop) {
+    authstring += `, qop=${qop}, nc=${nc}, cnonce="${cnonce}"`;
+  }
+  return authstring;
+}

test/fixtures/server.ts

@@ -71,6 +71,27 @@ export async function startServer(options?: {
       }));
     }
 
+    if (pathname === '/digestAuth') {
+      const authorization = req.headers.authorization;
+      if (!authorization) {
+        res.setHeader('www-authenticate', 'Digest realm="testrealm@urllib.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41"');
+        res.statusCode = 401;
+        return res.end(JSON.stringify({
+          error: 'authorization required',
+        }));
+      }
+      if (!authorization.includes('Digest username="user"')) {
+        res.setHeader('www-authenticate', 'Digest realm="testrealm@urllib.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41"');
+        res.statusCode = 401;
+        return res.end(JSON.stringify({
+          error: 'authorization invaild',
+        }));
+      }
+      return res.end(JSON.stringify({
+        authorization,
+      }));
+    }
+
     if (pathname === '/wrongjson') {
       res.setHeader('content-type', 'application/json');
       return res.end(Buffer.from('{"foo":""'));

test/options.digestAuth.test.ts

@@ -0,0 +1,77 @@
+import { strict as assert } from 'assert';
+import urllib from '../src';
+import { startServer } from './fixtures/server';
+
+describe('options.digestAuth.test.ts', () => {
+  let close: any;
+  let _url: string;
+  beforeAll(async () => {
+    const { closeServer, url } = await startServer();
+    close = closeServer;
+    _url = url;
+  });
+
+  afterAll(async () => {
+    await close();
+  });
+
+  it('should auth pass', async () => {
+    const response = await urllib.request(`${_url}digestAuth`, {
+      digestAuth: 'user:pwd',
+      dataType: 'json',
+    });
+    assert.equal(response.status, 200);
+    assert(response.data.authorization);
+    // console.log(response.data);
+    assert.match(response.data.authorization, /Digest username="user", realm="testrealm@urllib.com", nonce="/);
+  });
+
+  it('should auth fail', async () => {
+    const response = await urllib.request(`${_url}digestAuth`, {
+      digestAuth: 'invailduser:pwd',
+      dataType: 'json',
+    });
+    assert.equal(response.status, 401);
+    assert.deepEqual(response.data, {
+      error: 'authorization invaild',
+    });
+  });
+
+  it('should digest auth required', async () => {
+    const response = await urllib.request(`${_url}digestAuth?t=123123`, {
+      dataType: 'json',
+    });
+    assert.equal(response.status, 401);
+    assert.deepEqual(response.data, {
+      error: 'authorization required',
+    });
+  });
+
+  it.skip('should request with digest auth success in httpbin', async () => {
+    var url = 'https://httpbin.org/digest-auth/auth/user/passwd';
+    const response = await urllib.request(url, {
+      digestAuth: 'user:passwd',
+      dataType: 'json',
+      timeout: 10000,
+    });
+    console.log(response.headers);
+    assert.equal(response.status, 200);
+    assert.deepEqual(response.data, {
+      user: 'user',
+      authenticated: true,
+    });
+  });
+
+  it.skip('should request with digest auth fail in httpbin', async () => {
+    var url = 'https://httpbin.org/digest-auth/auth/user/passwd';
+    const response = await urllib.request(url, {
+      digestAuth: 'user:passwdfail',
+      dataType: 'json',
+      timeout: 10000,
+    });
+    // console.log(response.headers);
+    assert(response.headers['www-authenticate']);
+    assert.equal(response.status, 401);
+    assert.equal(response.data, null);
+  });
+});