Devtools Protocol (#174)

* Refine docs

* Decouple relay from dashboard.

Will be re-used by devtools protocol plugin.

* Just have a single manager for all eventing

* Ofcourse managers cant be shared across processes

* Remove unused

* Add DevtoolsProtocolPlugin

* Emit REQUEST_COMPLETE core event

* Emit only if --enable-events used

* Add event emitter for response cycle

* Fill up core events to devtools protocol expectations

* Serve static content with Cache-Control header and gzip compression

* Add PWA manifest.json and icons from sample PWA apps (replace later)

* Catch any exception and be ssl agnostic

* Add CSP headers and avoid inline scripts

* Re-enable iframe and deobfuscation

* Embed plugins within <section/> block

* Make tab switching agnostic of block name

* Add support for browser history on tab change

* Default hash to #home

* Switch to tab if hash is already set

* Expand canvas to fill screen even without content

* Remove inline css for embedded devtools

* Make dashboard backend websocket API pluggable

* doc

README.md

@@ -724,7 +724,7 @@ Embed proxy.py
 ## Blocking Mode
 
 Start `proxy.py` in embedded mode with default configuration
-by using the `main` method. Example:
+by using `proxy.main` method. Example:
 
 ```
 import proxy
@@ -782,10 +782,10 @@ Note that:
 1. `start` is a context manager.
    It will start `proxy.py` when called and will shut it down
    once scope ends.
-3. Just like `main`, startup flags with `start` method too
+3. Just like `main`, startup flags with `start` method
    can be customized by either passing flags as list of
-   input arguments `start(['--port', '8899'])` or by using passing
-   flags as kwargs `start(port=8899)`.
+   input arguments e.g. `start(['--port', '8899'])` or
+   by using passing flags as kwargs e.g. `start(port=8899)`.
 
 Unit testing with proxy.py
 ==========================
@@ -1131,13 +1131,14 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH]
              [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
              [--devtools-ws-path DEVTOOLS_WS_PATH]
              [--disable-headers DISABLE_HEADERS] [--disable-http-proxy]
-             [--enable-devtools] [--enable-events] [--enable-static-server]
-             [--enable-web-server] [--hostname HOSTNAME] [--key-file KEY_FILE]
+             [--enable-devtools] [--enable-events]
+             [--enable-static-server] [--enable-web-server]
+             [--hostname HOSTNAME] [--key-file KEY_FILE]
              [--log-level LOG_LEVEL] [--log-file LOG_FILE]
              [--log-format LOG_FORMAT] [--num-workers NUM_WORKERS]
              [--open-file-limit OPEN_FILE_LIMIT] [--pac-file PAC_FILE]
-             [--pac-file-url-path PAC_FILE_URL_PATH] [--pid-file PID_FILE]
-             [--plugins PLUGINS] [--port PORT]
+             [--pac-file-url-path PAC_FILE_URL_PATH]
+             [--pid-file PID_FILE] [--plugins PLUGINS] [--port PORT]
              [--server-recvbuf-size SERVER_RECVBUF_SIZE]
              [--static-server-dir STATIC_SERVER_DIR] [--threadless]
              [--timeout TIMEOUT] [--version]
@@ -1186,7 +1187,7 @@ optional arguments:
   --disable-http-proxy  Default: False. Whether to disable
                         proxy.HttpProxyPlugin.
   --enable-devtools     Default: False. Enables integration with Chrome
-                        Devtool Frontend.
+                        Devtool Frontend. Also see --devtools-ws-path.
   --enable-events       Default: False. Enables core to dispatch lifecycle
                         events. Plugins can be used to subscribe for core
                         events.

benchmark/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡Fast, Lightweight, Programmable, TLS interception capable
+    proxy server for Application debugging, testing and development.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""

dashboard/dashboard.py

@@ -9,35 +9,99 @@
 """
 import os
 import json
-import queue
 import logging
-import threading
-import multiprocessing
-import uuid
-from typing import List, Tuple, Optional, Any, Dict
+from abc import ABC, abstractmethod
+from typing import List, Tuple, Any, Dict
 
+from proxy.common.flags import Flags
+from proxy.core.event import EventSubscriber
 from proxy.http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes
 from proxy.http.parser import HttpParser
 from proxy.http.websocket import WebsocketFrame
 from proxy.http.codes import httpStatusCodes
 from proxy.common.utils import build_http_response, bytes_
-from proxy.common.types import DictQueueType
 from proxy.core.connection import TcpClientConnection
 
 logger = logging.getLogger(__name__)
 
 
-class ProxyDashboard(HttpWebServerBasePlugin):
+class ProxyDashboardWebsocketPlugin(ABC):
+    """Abstract class for plugins extending dashboard websocket API."""
+
+    def __init__(
+            self,
+            flags: Flags,
+            client: TcpClientConnection,
+            subscriber: EventSubscriber) -> None:
+        self.flags = flags
+        self.client = client
+        self.subscriber = subscriber
+
+    @abstractmethod
+    def methods(self) -> List[str]:
+        """Return list of methods that this plugin will handle."""
+        pass
+
+    @abstractmethod
+    def handle_message(self, message: Dict[str, Any]) -> None:
+        """Handle messages for registered methods."""
+        pass
+
+    def reply(self, data: Dict[str, Any]) -> None:
+        self.client.queue(
+            WebsocketFrame.text(
+                bytes_(
+                    json.dumps(data))))
+
+
+class InspectTrafficPlugin(ProxyDashboardWebsocketPlugin):
+    """Websocket API for inspect_traffic.ts frontend plugin."""
 
-    RELAY_MANAGER: multiprocessing.managers.SyncManager = multiprocessing.Manager()
+    def methods(self) -> List[str]:
+        return [
+            'enable_inspection',
+            'disable_inspection',
+        ]
+
+    def handle_message(self, message: Dict[str, Any]) -> None:
+        if message['method'] == 'enable_inspection':
+            # inspection can only be enabled if --enable-events is used
+            if not self.flags.enable_events:
+                self.client.queue(
+                    WebsocketFrame.text(
+                        bytes_(
+                            json.dumps(
+                                {'id': message['id'], 'response': 'not enabled'})
+                        )
+                    )
+                )
+            else:
+                self.subscriber.subscribe(
+                    lambda event: ProxyDashboard.callback(
+                        self.client, event))
+                self.reply(
+                    {'id': message['id'], 'response': 'inspection_enabled'})
+        elif message['method'] == 'disable_inspection':
+            self.subscriber.unsubscribe()
+            self.reply({'id': message['id'],
+                        'response': 'inspection_disabled'})
+        else:
+            raise NotImplementedError()
+
+
+class ProxyDashboard(HttpWebServerBasePlugin):
+    """Proxy Dashboard."""
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
-        self.inspection_enabled: bool = False
-        self.relay_thread: Optional[threading.Thread] = None
-        self.relay_shutdown: Optional[threading.Event] = None
-        self.relay_channel: Optional[DictQueueType] = None
-        self.relay_sub_id: Optional[str] = None
+        self.subscriber = EventSubscriber(self.event_queue)
+        # Initialize Websocket API plugins
+        self.plugins: Dict[str, ProxyDashboardWebsocketPlugin] = {}
+        plugins = [InspectTrafficPlugin]
+        for plugin in plugins:
+            p = plugin(self.flags, self.client, self.subscriber)
+            for method in p.methods():
+                self.plugins[method] = p
 
     def routes(self) -> List[Tuple[int, bytes]]:
         return [
@@ -83,65 +147,19 @@ def on_websocket_message(self, frame: WebsocketFrame) -> None:
             logger.info(frame.opcode)
             return
 
-        if message['method'] == 'ping':
+        method = message['method']
+        if method == 'ping':
             self.reply({'id': message['id'], 'response': 'pong'})
-        elif message['method'] == 'enable_inspection':
-            # inspection can only be enabled if --enable-events is used
-            if not self.flags.enable_events:
-                self.client.queue(
-                    WebsocketFrame.text(
-                        bytes_(
-                            json.dumps(
-                                {'id': message['id'], 'response': 'not enabled'})
-                        )
-                    )
-                )
-            else:
-                self.inspection_enabled = True
-
-                self.relay_shutdown = threading.Event()
-                self.relay_channel = ProxyDashboard.RELAY_MANAGER.Queue()
-                self.relay_thread = threading.Thread(
-                    target=self.relay_events,
-                    args=(self.relay_shutdown, self.relay_channel, self.client))
-                self.relay_thread.start()
-
-                self.relay_sub_id = uuid.uuid4().hex
-                self.event_queue.subscribe(
-                    self.relay_sub_id, self.relay_channel)
-
-                self.reply(
-                    {'id': message['id'], 'response': 'inspection_enabled'})
-        elif message['method'] == 'disable_inspection':
-            self.shutdown_relay()
-            self.inspection_enabled = False
-            self.reply({'id': message['id'],
-                        'response': 'inspection_disabled'})
+        elif method in self.plugins:
+            self.plugins[method].handle_message(message)
         else:
             logger.info(frame.data)
             logger.info(frame.opcode)
             self.reply({'id': message['id'], 'response': 'not_implemented'})
 
     def on_websocket_close(self) -> None:
         logger.info('app ws closed')
-        self.shutdown_relay()
-
-    def shutdown_relay(self) -> None:
-        if not self.inspection_enabled:
-            return
-
-        assert self.relay_shutdown
-        assert self.relay_thread
-        assert self.relay_sub_id
-
-        self.event_queue.unsubscribe(self.relay_sub_id)
-        self.relay_shutdown.set()
-        self.relay_thread.join()
-
-        self.relay_thread = None
-        self.relay_shutdown = None
-        self.relay_channel = None
-        self.relay_sub_id = None
+        # unsubscribe
 
     def reply(self, data: Dict[str, Any]) -> None:
         self.client.queue(
@@ -150,21 +168,9 @@ def reply(self, data: Dict[str, Any]) -> None:
                     json.dumps(data))))
 
     @staticmethod
-    def relay_events(
-            shutdown: threading.Event,
-            channel: DictQueueType,
-            client: TcpClientConnection) -> None:
-        while not shutdown.is_set():
-            try:
-                ev = channel.get(timeout=1)
-                ev['push'] = 'inspect_traffic'
-                client.queue(
-                    WebsocketFrame.text(
-                        bytes_(
-                            json.dumps(ev))))
-            except queue.Empty:
-                pass
-            except EOFError:
-                break
-            except KeyboardInterrupt:
-                break
+    def callback(client: TcpClientConnection, event: Dict[str, Any]) -> None:
+        event['push'] = 'inspect_traffic'
+        client.queue(
+            WebsocketFrame.text(
+                bytes_(
+                    json.dumps(event))))

dashboard/rollup.config.js

@@ -1,6 +1,6 @@
 import typescript from 'rollup-plugin-typescript';
 import copy from 'rollup-plugin-copy';
-// import obfuscatorPlugin from 'rollup-plugin-javascript-obfuscator';
+import obfuscatorPlugin from 'rollup-plugin-javascript-obfuscator';
 
 export const input = 'src/proxy.ts';
 export const output = {
@@ -21,6 +21,9 @@ export const plugins = [
         }, {
             src: 'src/proxy.css',
             dest: '../public/dashboard',
+        }, {
+            src: 'src/manifest.json',
+            dest: '../public/dashboard',
         }, {
             src: 'src/core/plugins/inspect_traffic.json',
             dest: '../public/dashboard/devtools'
@@ -32,7 +35,7 @@ export const plugins = [
             dest: '../public/dashboard/devtools'
         }],
     }),
-    /* obfuscatorPlugin({
+    obfuscatorPlugin({
         log: false,
         sourceMap: true,
         compact: true,
@@ -42,5 +45,5 @@ export const plugins = [
         stringArrayThreshold: 1,
         stringArrayEncoding: 'rc4',
         identifierNamesGenerator: 'mangled',
-    }) */
+    })
 ];

dashboard/src/core/plugin.ts

@@ -12,6 +12,7 @@ import { WebsocketApi } from './ws'
 export interface IDashboardPlugin {
   name: string
   title: string
+  tabId(): string
   initializeTab(): JQuery<HTMLElement>
   initializeHeader(): JQuery<HTMLElement>
   initializeBody(): JQuery<HTMLElement>
@@ -30,11 +31,16 @@ export abstract class DashboardPlugin implements IDashboardPlugin {
     this.websocketApi = websocketApi
   }
 
+  public tabId () : string {
+    return this.name + '_tab'
+  }
+
   public makeTab (name: string, icon: string) : JQuery<HTMLElement> {
     return $('<a/>')
       .attr({
-        href: '#',
-        plugin_name: this.name
+        href: '#' + this.name,
+        plugin_name: this.name,
+        id: this.tabId()
       })
       .addClass('nav-link')
       .text(name)

dashboard/src/core/plugins/home.ts

@@ -21,6 +21,23 @@ export class HomePlugin extends DashboardPlugin {
     return this.makeHeader(this.title)
   }
 
+  // Show following metrics on home page:
+  // 0. Uptime
+  // 1. Total number of requests served counter
+  //    - Separate numbers for proxy and in-built http server
+  // 2. Number of active requests counter
+  //    - Click to inspect via inspect traffic tab
+  //    - Will be hard here to focus on exact active request within embedded Devtools
+  // 3. Requests served per second / minute / hours chart
+  // 4. Active requests per second / minute / hours chart
+  // 5. List of all proxy.py processes
+  //    - Threads per process
+  //    - RAM / CPU per process over time charts
+  // 6. Bandwidth served
+  //    - Total incoming bytes
+  //    - Total outgoing bytes
+  //    - Ingress / Egress bytes per sec / min / hour
+  // 7. Active plugin list
   public initializeBody (): JQuery<HTMLElement> {
     return $('<div></div>')
   }

dashboard/src/core/plugins/inspect_traffic.ts

@@ -9,8 +9,6 @@
 */
 import { DashboardPlugin } from '../plugin'
 
-declare const Root: any
-
 export class InspectTrafficPlugin extends DashboardPlugin {
   public name: string = 'inspect_traffic'
   public title: string = 'Inspect Traffic'
@@ -28,12 +26,15 @@ export class InspectTrafficPlugin extends DashboardPlugin {
   }
 
   public activated (): void {
-    this.websocketApi.enableInspection(this.handleEvents.bind(this))
+    this.websocketApi.sendMessage(
+      { method: 'enable_inspection' },
+      this.handleEvents.bind(this))
     this.ensureIFrame()
   }
 
   public deactivated (): void {
-    this.websocketApi.disableInspection()
+    this.websocketApi.sendMessage(
+      { method: 'disable_inspection' })
   }
 
   public handleEvents (message: Record<string, any>): void {
@@ -57,7 +58,7 @@ export class InspectTrafficPlugin extends DashboardPlugin {
   private getDevtoolsIFrame (): JQuery<HTMLElement> {
     return $('<iframe></iframe>')
       .attr('id', this.getDevtoolsIFrameID())
-      .attr('height', '80%')
+      .attr('height', '100%')
       .attr('width', '100%')
       .attr('padding', '0')
       .attr('margin', '0')