Network monitoring and HAR export

dist/api.md

@@ -35,6 +35,42 @@ Interface for the OpenWPM WebExtension Experiment API.
 
 **Parameters**
 
+### `browser.openwpm.enableNetworkMonitorForTab( tabId )` 
+
+  Enables network monitoring for a specific tab.
+
+
+**Parameters**
+
+- `tabId`
+  - type: tabId
+  - $ref: 
+  - optional: false
+
+### `browser.openwpm.disableNetworkMonitorForTab( tabId )` 
+
+  Disables network monitoring for a specific tab.
+
+
+**Parameters**
+
+- `tabId`
+  - type: tabId
+  - $ref: 
+  - optional: false
+
+### `browser.openwpm.getHarForTab( tabId )` 
+
+  Returns the current HAR for a specific tab (requires enabled network monitoring for the given tab).
+
+
+**Parameters**
+
+- `tabId`
+  - type: tabId
+  - $ref: 
+  - optional: false
+
 ## Events
 
 ### `browser.openwpm.onStarted () ` Event

package.json

@@ -60,7 +60,7 @@
     "build": "yaml2json src/schema.yaml -p > dist/schema.json && npm run generate && cd src && webpack",
     "eslint": "eslint . --ext js --ext json",
     "eslint-fix": "npm run eslint -- --fix",
-    "format": "prettier '**/*.{css,js,jsm,json,md}' --trailing-comma=all --ignore-path=.eslintignore --write",
+    "format": "echo disabled due to issue with windows - prettier '**/*.{css,js,jsm,json,md}' --trailing-comma=all --ignore-path=.eslintignore --write",
     "generate": "npm-run-all -s -n generate:verifyWeeSchema generate:documentSchema generate:generateStubApi",
     "generate:documentSchema": "documentSchema dist/schema.json > dist/api.md",
     "generate:generateStubApi": "generateStubApi dist/schema.json > src/stubApi.js",

src/api.js/Monitor.js

@@ -0,0 +1,205 @@
+"use strict";
+
+import { NewTabObserver } from "./NewTabObserver";
+
+const { require } = ChromeUtils.import(
+  "resource://devtools/shared/Loader.jsm",
+  {},
+);
+const {
+  DevToolsShim,
+} = require("chrome://devtools-startup/content/DevToolsShim.jsm");
+const Services = require("Services");
+const { NetMonitorAPI } = require("devtools/client/netmonitor/src/api");
+const {
+  buildHarLog,
+} = require("devtools/client/netmonitor/src/har/har-builder-utils");
+
+export class Monitor {
+  constructor() {
+    this.tabSpecificMonitors = {};
+    this.newTabObserver = new NewTabObserver();
+  }
+
+  async startMonitoringNewTabs(tabManager) {
+    console.log("startMonitoringNewTabs - this.newTabObserver", this.newTabObserver);
+    this.newTabObserver.init(this, tabManager);
+  }
+
+  async stopMonitoringNewTabs() {
+    console.log("stopMonitoringNewTabs - this.newTabObserver", this.newTabObserver);
+    this.newTabObserver.uninit();
+  }
+
+  /**
+   * @param tabBase TabBase
+   * @returns {Promise<void>}
+   */
+  async enableMonitoringForTab(tabBase) {
+    const tabSpecificMonitor = this.getTabSpecificMonitorByTabBase(tabBase);
+    return tabSpecificMonitor.start();
+  }
+
+  /**
+   * @param tabBase TabBase
+   * @returns {Promise<void>}
+   */
+  async disableMonitoringForTab(tabBase) {
+    const existingTabSpecificMonitor = this.tabSpecificMonitors[tabBase.id];
+    if (existingTabSpecificMonitor) {
+      await existingTabSpecificMonitor.stop();
+      delete this.tabSpecificMonitors[tabBase.id];
+    }
+  }
+
+  async getHarForTab(tabBase) {
+    const tabSpecificMonitor = this.getTabSpecificMonitorByTabBase(tabBase);
+    return tabSpecificMonitor.getHAR();
+  }
+
+  getTabSpecificMonitorByTabBase(tabBase) {
+    const existingTabSpecificMonitor = this.tabSpecificMonitors[tabBase.id];
+    if (existingTabSpecificMonitor) {
+      return existingTabSpecificMonitor;
+    }
+    const tabSpecificMonitor = new TabSpecificMonitor(tabBase);
+    this.tabSpecificMonitors[tabBase.id] = tabSpecificMonitor;
+    return tabSpecificMonitor;
+  }
+}
+
+export class TabSpecificMonitor {
+  /**
+   * @param tabBase TabBase
+   */
+  constructor(tabBase) {
+    this.tabBase = tabBase;
+  }
+
+  /**
+   * Initialize NetMonitorAPI object and connect to the
+   * Firefox backend for a specific tab
+   * @returns {Promise<void>}
+   */
+  async start() {
+    const netMonitor = this.getNetMonitorAPI();
+    if (netMonitor.toolbox) {
+      console.log("NetMonitor already connected to a Mock toolbox");
+    }
+    const { tabBase } = this;
+    const { nativeTab } = tabBase;
+    const target = DevToolsShim.createTargetForTab(nativeTab);
+    const MockToolbox = {
+      target,
+      getPanel: () => {},
+    };
+    // Debug
+    target.on("close", async () => {
+      console.log("target close", arguments);
+      const har = await this.getHAR();
+      console.log("har in target:close", har);
+    });
+    target.on("will-navigate", async () => {
+      console.log("target will-navigate", arguments);
+      const har = await this.getHAR();
+      console.log("har in target:will-navigate", har);
+    });
+    target.on("navigate", () => {
+      console.log("target navigate", arguments);
+    });
+    target.on("tabNavigated", () => {
+      console.log("target tabNavigated", arguments);
+    });
+    target.on("tabDetached", () => {
+      console.log("target tabDetached", arguments);
+    });
+    console.log(
+      "netMonitor.toolbox before connect",
+      netMonitor.toolbox,
+      tabBase.id,
+    );
+    await target.makeRemote();
+    await target.activeConsole.startListeners(["NetworkActivity"]);
+    await netMonitor.connect(MockToolbox);
+  }
+
+  /**
+   * @returns {Promise<void>}
+   */
+  async stop() {
+    await this.getNetMonitorAPI().destroy();
+  }
+
+  /**
+   * The monitor is considered connected
+   * if its NetMonitorAPI instance has its
+   * connector, connector.connector and toolbox properties set
+   * @returns {boolean}
+   */
+  connected(netMonitor) {
+    return !!(
+      netMonitor.connector &&
+      netMonitor.connector.connector &&
+      netMonitor.toolbox
+    );
+  }
+
+  /**
+   * Return Netmonitor API object. This object offers Network monitor
+   * public API that can be consumed by other panels or WE API.
+   */
+  getNetMonitorAPI() {
+    if (this._netMonitorAPI) {
+      return this._netMonitorAPI;
+    }
+
+    // Create and initialize Network monitor API object.
+    // This object is only connected to the backend - not to the UI.
+    this._netMonitorAPI = new NetMonitorAPI();
+
+    return this._netMonitorAPI;
+  }
+
+  /**
+   * Returns data (HAR) collected by the Network monitor.
+   */
+  async getHAR() {
+    console.log("getHAR");
+    let har;
+
+    const netMonitor = this.getNetMonitorAPI();
+    if (!this.connected(netMonitor)) {
+      console.log(
+        "getHAR from NetMonitor skipped since not yet connected to a tab",
+      );
+    } else {
+      const state = netMonitor.store.getState();
+      console.log("netMonitor.store.getState() just before HAR export", state);
+      /*
+      In order to supply options.includeResponseBodies to HAR exporter,
+      we have imported and modified code from NetMonitor.getHar() here:
+      const {
+        HarExporter,
+      } = require("devtools/client/netmonitor/src/har/har-exporter");
+      const {
+        getSortedRequests,
+      } = require("devtools/client/netmonitor/src/selectors/index");
+      const options = {
+        connector: netMonitor.connector,
+        items: getSortedRequests(state),
+      };
+      har = await HarExporter.getHar(options);
+      */
+
+      har = await netMonitor.getHar();
+      console.log("getHAR har from NetMonitor", har);
+    }
+
+    // Return default empty HAR file if needed.
+    har = har || buildHarLog(Services.appinfo);
+
+    // Return the log directly to be compatible with
+    // Chrome WebExtension API.
+    return har.log;
+  }
+}

src/api.js/NewTabObserver.js

@@ -0,0 +1,129 @@
+"use strict";
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+import logger from "./logger";
+import WindowWatcher from "./WindowWatcher";
+
+export class NewTabObserver {
+
+  constructor() {
+    this.onTabChangeRef = this.onTabChange.bind(this);
+  }
+
+  async init(monitor, tabManager) {
+
+    this.monitor = monitor;
+    this.tabManager = tabManager;
+
+    // run once now on the most recent window.
+    // const win = this.getMostRecentWindow();
+    await this.startWatchingWindows();
+
+  }
+
+  async startWatchingWindows() {
+    // load content into existing windows and listen for new windows to load content in
+    WindowWatcher.start(this.loadIntoWindow.bind(this), this.unloadFromWindow.bind(this), this.onWindowError.bind(this));
+  }
+
+  loadIntoWindow(win) {
+    // Add listeners to all open windows to know when to update pageAction
+    this.addWindowEventListeners(win);
+    // Add listeners to all new windows to know when to update pageAction.
+    // Depending on which event happens (ex: onOpenWindow, onLocationChange),
+    // it will call that listener method that exists on "this"
+    Services.wm.addListener(this);
+  }
+
+  unloadFromWindow(win) {
+    this.removeWindowEventListeners(win);
+    Services.wm.removeListener(this);
+    // handle the case where the window closed, but intro or pageAction panel
+    // is still open.
+    this.handleWindowClosing(win);
+  }
+
+  onWindowError(msg) {
+    console.debug(msg);
+  }
+
+  /**
+   * Three cases of user looking at diff page:
+   *   - switched windows (onOpenWindow)
+   *   - loading new pages in the same tab (on page load in frame script)
+   *   - switching tabs but not switching windows (tabSelect)
+   * Each one needs its own separate handler, because each one is detected by its
+   * own separate event.
+   *
+   * @param {ChromeWindow} win NEEDS_DOC
+   * @returns {void}
+   */
+  addWindowEventListeners(win) {
+    if (win && win.gBrowser) {
+      win.gBrowser.addTabsProgressListener(this);
+      win.gBrowser.tabContainer.addEventListener(
+        "TabSelect",
+        this.onTabChangeRef,
+      );
+    }
+  }
+
+  removeWindowEventListeners(win) {
+    if (win && win.gBrowser) {
+      win.gBrowser.removeTabsProgressListener(this);
+      win.gBrowser.tabContainer.removeEventListener(
+        "TabSelect",
+        this.onTabChangeRef,
+      );
+    }
+  }
+
+  handleWindowClosing(win) {
+    console.log("handleWindowClosing - win:", win);
+  }
+
+  /**
+   * This method is called when opening a new tab among many other times.
+   * This is a listener for the addTabsProgressListener
+   * Not appropriate for modifying the page itself because the page hasn't
+   * finished loading yet. More info: https://tinyurl.com/lpzfbpj
+   *
+   * @param  {Object} browser  NEEDS_DOC
+   * @param  {Object} progress NEEDS_DOC
+   * @param  {Object} request  NEEDS_DOC
+   * @param  {Object} uri      NEEDS_DOC
+   * @param  {number} flags    NEEDS_DOC
+   * @returns {void}
+   */
+  async onLocationChange(browser, progress, request, uri, flags) {
+    console.log("onLocationChange - browser, progress, request, uri, flags:", browser, progress, request, uri, flags);
+    // ensure the location change event is occuring in the top frame (not an iframe for example)
+    if (!progress.isTopLevel) {
+      console.log("onLocationChange - bailing since we are not in the top frame");
+    }
+
+    // Setup a tab-specific monitor
+    const tabId = 1; // TODO: Actually get the correct tab id from the arguments to this event listener
+    const tabBase = this.tabManager.get(tabId);
+    await this.monitor.enableMonitoringForTab(tabBase);
+    logger.debug(
+      `Started tab monitoring for tab with id ${tabId}`,
+      this.monitor,
+    );
+  }
+
+  onTabChange(evt) {
+    console.log("onTabChange - evt:", evt);
+  }
+
+  handleEmbeddedBrowserLoad(evt) {
+    console.log("handleEmbeddedBrowserLoad", evt);
+  }
+
+  async uninit() {
+    // Remove all listeners from existing windows and stop listening for new windows
+    WindowWatcher.stop();
+  }
+
+}