implements slack token replacement as a hook

package.json

@@ -45,6 +45,7 @@
     "insight": "^0.10.1",
     "js-string-escape": "^1.0.1",
     "mkdirp": "^0.5.1",
+    "nonce-str": "^1.0.1",
     "normalize-port": "^1.0.0",
     "normalize-url": "^2.0.1",
     "preprocess-cli-tool": "^1.0.1",

src/cli.ts

@@ -168,8 +168,10 @@ export default async function main(): Promise<void> {
       }));
     }
     if (argv.slackReplaceTokens) {
+      // NOTE: what about responses that contain a token? currently, this is only the `oauth.access`
+      // and `oauth.token` Web API methods.
       loading = loading.then(loaded => import('./hooks/slack-replace-tokens').then((module) => {
-        loaded.push(module);
+        loaded.push(module.createHook(console.log));
         return loaded;
       }));
     }

src/controller.ts

@@ -85,7 +85,8 @@ export class Controller implements Service {
     this.outgoingPort = outgoingPort;
 
     this.recorderHooks = hooks.filter((hook) => {
-      return ['outgoingProxyRequestInfo', 'serializerRawRequestBody'].includes(hook.hookType);
+      // TODO: use the intersection of enums to describe the set of hooks
+      return ['outgoingProxyRequestInfo', 'serializerRawRequest'].includes(hook.hookType);
     });
 
     this.print = print;

src/hooks/slack-replace-tokens.ts

@@ -1,5 +1,90 @@
-export const hookType = 'serializerRawRequestBody';
+import { URLSearchParams, parse as urlParse, format as urlFormat } from 'url';
+import { RequestInfo, SerializerRawRequest } from '../steno';
+import { PrintFn } from '../util';
+import nonce from 'nonce-str'; // tslint:disable-line import-name
 
-export function processor(): void {
-  return;
+const authorizationHeaderPattern = /^Bearer (.*)$/;
+
+function createPlaceholderToken(length: number): string {
+  return `xoxf-${nonce(length - 5)}`;
+}
+
+export function createHook(print: PrintFn): SerializerRawRequest {
+  print('Starting with Slack token replacement enabled. Each time a token is encountered in an ' +
+        'interaction, it will be replaced with a placeholder. Make these substitutions in test ' +
+        'code. This way, when your test code runs against steno in replay mode, the interactions ' +
+        'will continue to match.\n\n\nWARNING: In this mode, sensitive data is logged to stdout. ' +
+        'Do not store these logs without access control.\n\n');
+
+  const tokenReplacements: Map<string, string> = new Map();
+
+  function replaceToken(token: string): string {
+    const replacement = tokenReplacements.get(token) || createPlaceholderToken(token.length);
+    tokenReplacements.set(token, replacement);
+    print(`Slack token replaced: TOKEN=${token} REPLACEMENT=${replacement}`);
+    return replacement;
+  }
+
+  return {
+    hookType: 'serializerRawRequest',
+    processor: (request: RequestInfo): RequestInfo => {
+      // TODO: RequestInfo is useful as an internal representation, we do not want to export this type
+
+      // Look for tokens in all the places we might see one in the Slack API
+
+      // 1. JSON write requests to the Web API include tokens in the Authorization header
+      let authorization;
+      let match = null;
+      if ((authorization = request.headers['authorization'] as string) !== undefined &&
+          (match = authorizationHeaderPattern.exec(authorization)) !== null) {
+        const token = match[1];
+        if (token !== undefined) {
+          const replacement = replaceToken(token);
+          request.headers['authorization'] = (request.headers['authorization'] as string).replace(token, replacement);
+        }
+      }
+
+      // 2. Some requests to the Web API include tokens in the URL-encoded body
+      if (request.headers['content-type'] === 'application/x-www-form-urlencoded' && request.body !== undefined) {
+        const bodyParams = new URLSearchParams(`?${request.body.toString()}`);
+        const token = bodyParams.get('token');
+        if (token !== null) {
+          const replacement = replaceToken(token);
+          bodyParams.set('token', replacement);
+          // NOTE: URLSearchParams has opinions about how to encode itself as a string, that may
+          // differ from how the original request was encoded. The main example is spaces, which
+          // can be encoded as "%20", but URLSearchParams chooses to encode them as "+". Both are
+          // technically correct.
+          request.body = Buffer.from(bodyParams.toString());
+        }
+      }
+
+      // 3. Other requests to the Web API include tokens in the URL-encoded query parameter
+      const parsedUrl = urlParse(request.url);
+      if (parsedUrl.search !== undefined) {
+        const queryParams = new URLSearchParams(parsedUrl.search);
+        const token = queryParams.get('token');
+        if (token !== null) {
+          const replacement = replaceToken(token);
+          queryParams.set('token', replacement);
+          delete parsedUrl.query;
+          // NOTE: URLSearchParams has opinions about how to encode itself as a string, that may
+          // differ from how the original request was encoded. The main example is spaces, which
+          // can be encoded as "%20", but URLSearchParams chooses to encode them as "+". Both are
+          // technically correct.
+          parsedUrl.search = queryParams.toString();
+          request.url = urlFormat(parsedUrl);
+        }
+      }
+
+      // incoming requests from slash commands, events api, and interactive messages have a
+      // verification token in them, not a client authentication token
+
+      // outgoing requests from incoming webhooks and response_urls (interactive message or slash
+      // command) don't have client authentication tokens in them, although the URL itself could
+      // be considered sentitive data
+
+      return request;
+    },
+  };
 }

src/record/http-serializer.ts

@@ -3,7 +3,7 @@ import { createWriteStream, open as fsOpen, WriteStream } from 'fs';
 import { IncomingHttpHeaders } from 'http';
 import mkdirp from 'mkdirp';
 import { join as pathJoin } from 'path';
-import { RequestInfo, ResponseInfo, responseBodyToString } from '../steno';
+import { RequestInfo, ResponseInfo, responseBodyToString, StenoHook, SerializerRawRequest } from '../steno';
 import { promisify } from 'util';
 
 const createDirectory = promisify(mkdirp);
@@ -83,12 +83,17 @@ export class HttpSerializer {
   public storagePath: string;
   /** a map of destinations for requests that are pending a response (keys are request IDs) */
   public pendingRequestDestinations: Map<string, Destination>;
+  private transformRawRequestBodyHook?: SerializerRawRequest;
 
   // NOTE: might want to implement a task queue in order to keep track of operations
-  constructor(storagePath: string) {
+  // TODO: more specific type for hooks
+  constructor(storagePath: string, hooks: StenoHook[]) {
     this.storagePath = storagePath;
-    log(`storage path: ${this.storagePath}`);
+    this.transformRawRequestBodyHook = hooks.find((hook) => {
+      return hook.hookType === 'serializerRawRequest';
+    }) as SerializerRawRequest;
     this.pendingRequestDestinations = new Map();
+    log(`storage path: ${this.storagePath}`);
   }
 
   /**
@@ -118,6 +123,11 @@ export class HttpSerializer {
   public onRequest(requestInfo: RequestInfo, prefix = ''): void {
     log('on request');
 
+    if (this.transformRawRequestBodyHook !== undefined) {
+      // tslint:disable-next-line no-parameter-reassignment
+      requestInfo = this.transformRawRequestBodyHook.processor(requestInfo);
+    }
+
     const data = this.generateRequestData(requestInfo);
     const baseFilename = this.generateFilename(requestInfo, prefix);
     createDestination(baseFilename)

src/record/recorder.ts

@@ -35,13 +35,17 @@ export class Recorder implements Service, Device {
     incomingTargetConfig: ProxyTargetConfig, incomingPort: string | number,
     outgoingTargetConfig: ProxyTargetConfig, outgoingPort: string | number,
     storagePath: string,
+    // TODO: more specific type
     hooks: StenoHook[],
     print: PrintFn = console.log,
   ) {
-    this.serializer = new HttpSerializer(storagePath);
+    // TODO: use an enum to describe the set of serializer hooks
+    const serializerHooks = hooks.filter(hook => ['serializerRawRequest'].includes(hook.hookType));
+    this.serializer = new HttpSerializer(storagePath, serializerHooks);
 
-    const outgoingHooks = hooks.filter(hook => ['outgoingProxyRequestInfo'].includes(hook.hookType));
-    this.outgoingProxy = createProxy(outgoingTargetConfig, outgoingHooks);
+    // TODO: use an enum to describe the set of outgoing proxy hooks
+    const outgoingProxyHooks = hooks.filter(hook => ['outgoingProxyRequestInfo'].includes(hook.hookType));
+    this.outgoingProxy = createProxy(outgoingTargetConfig, outgoingProxyHooks);
     this.outgoingPort = outgoingPort;
 
     this.incomingProxy = createProxy(incomingTargetConfig);

src/steno.ts

@@ -6,15 +6,16 @@ export interface Service {
   start(): Promise<void>;
 }
 
-export type StenoHook = OutgoingProxyRequestInfo | SerializerRawRequestBody;
+export type StenoHook = OutgoingProxyRequestInfo | SerializerRawRequest;
 
 export interface OutgoingProxyRequestInfo {
   hookType: 'outgoingProxyRequestInfo';
   processor: (originalReq: IncomingMessage, reqOptions: RequestOptions) => RequestOptions;
 }
 
-export interface SerializerRawRequestBody {
-  hookType: 'serializerRawRequestBody';
+export interface SerializerRawRequest {
+  hookType: 'serializerRawRequest';
+  processor: (request: RequestInfo) => RequestInfo;
 }
 
 // TODO: maybe split this into two types?

src/types/nonce-str.d.ts

@@ -0,0 +1,2 @@
+
+declare module 'nonce-str';

src/util.ts

@@ -30,6 +30,7 @@ export interface IncomingHttpTrailers {
 // base type. The index includes `undefined` as a value type, when strictly speaking this not
 // possible. The only reason it is added is so that other common headers can be named as optional
 // properties, so that intellisense has some awareness of those common headers.
+// TODO: use index types to create this from IncomngHttpHeaders
 export interface NotOptionalIncomingHttpHeaders {
   [header: string]: string | string[];
 }