Merge pull request #1261 from beakerbrowser/external-dat-process

Move dat into a 'daemon' process

app/background-process.js

@@ -22,9 +22,11 @@ import registerContextMenu from './background-process/ui/context-menu'
 import * as downloads from './background-process/ui/downloads'
 import * as permissions from './background-process/ui/permissions'
 import * as basicAuth from './background-process/ui/basic-auth'
+import * as hiddenWindows from './background-process/hidden-windows'
 
 import * as beakerProtocol from './background-process/protocols/beaker'
 import * as beakerFaviconProtocol from './background-process/protocols/beaker-favicon'
+import * as beakerHiddenWindowProtocol from './background-process/protocols/beaker-hidden-window'
 
 import * as testDriver from './background-process/test-driver'
 import * as openURL from './background-process/open-url'
@@ -57,7 +59,7 @@ if (beakerCore.getEnvVar('BEAKER_TEST_DRIVER')) {
 app.enableMixedSandbox()
 
 // configure the protocols
-protocol.registerStandardSchemes(['dat', 'beaker'], { secure: true })
+protocol.registerStandardSchemes(['dat', 'beaker', 'beaker-hidden-window'], { secure: true })
 
 // handle OS event to open URLs
 app.on('open-url', (e, url) => {
@@ -76,6 +78,10 @@ app.on('open-file', (e, filepath) => {
 })
 
 app.on('ready', async function () {
+  // start the daemon process
+  beakerHiddenWindowProtocol.setup()
+  var datDaemonWindow = await hiddenWindows.spawn('dat-daemon', './dat-daemon.js')
+
   // setup core
   await beakerCore.setup({
     // paths
@@ -92,7 +98,8 @@ app.on('ready', async function () {
     },
     rpcAPI: rpc,
     downloadsWebAPI: downloads.WEBAPI,
-    browserWebAPI: beakerBrowser.WEBAPI
+    browserWebAPI: beakerBrowser.WEBAPI,
+    datDaemonWc: datDaemonWindow.webContents
   })
 
   // base

app/background-process/hidden-windows.js

@@ -0,0 +1,38 @@
+import {app, BrowserWindow} from 'electron'
+
+// globals
+// =
+
+var hiddenWindows = {}
+
+// exported api
+// =
+
+export async function spawn (id, modulePath) {
+  var fullModulePath = require.resolve(modulePath)
+  var win = new BrowserWindow({show: false})
+  // win.webContents.toggleDevTools()
+  console.log('[', id, '] Starting...')
+  win.loadURL(`beaker-hidden-window://loader/?module=${encodeURIComponent(fullModulePath)}&userDataPath=${encodeURIComponent(app.getPath('userData'))}`)
+  win.webContents.on('console-message', (e, level, msg) => {
+    console.log('[', id, ']', msg)
+  })
+  await new Promise((resolve, reject) => {
+    win.webContents.on('did-finish-load', resolve)
+  })
+  hiddenWindows[id] = win
+  return win
+}
+
+export function close (id) {
+  hiddenWindows[id].close()
+  delete hiddenWindows[id]
+}
+
+export async function exec (id, script) {
+  return hiddenWindows[id].executeJavaScript(script)
+}
+
+export async function send (id, ...args) {
+  return hiddenWindows[id].webContents.send(...args)
+}

app/background-process/protocols/beaker-hidden-window.js

@@ -0,0 +1,10 @@
+import {protocol} from 'electron'
+import path from 'path'
+
+export function setup () {
+  protocol.registerFileProtocol('beaker-hidden-window', (request, cb) => {
+    cb({path: path.join(__dirname, 'hidden-window.html')})
+  }, e => {
+    if (e) { console.error('Failed to register beaker-hidden-window protocol', e) }
+  })
+}

app/background-process/test-driver.js

@@ -3,6 +3,8 @@ import {ipcMain} from 'electron'
 import * as beakerCore from '@beaker/core'
 import * as windows from './ui/windows'
 
+const LOG_MESSAGES = false
+
 var testPort = +beakerCore.getEnvVar('BEAKER_TEST_DRIVER')
 var sock
 
@@ -23,21 +25,31 @@ export function setup () {
   sock.on('listening', hit)
   ipcMain.once('shell-window:ready', hit)
   function hit () {
-    if (!(--todos)) send({isReady: true, port: sock.address().port})
+    if (!(--todos)) {
+      // HACK
+      // there's some kind of race which causes `executeJavaScript` to not run in the shell window during tests
+      // this timeout is intended to solve that
+      // -prf
+      setTimeout(() => {
+        send({isReady: true, port: sock.address().port})
+      }, 1e3)
+    }
   }
 }
 
 // internal methods
 // =
 
 function send (obj) {
+  if (LOG_MESSAGES) console.log('driverserver sent', JSON.stringify(obj))
   obj = Buffer.from(JSON.stringify(obj), 'utf8')
   sock.send(obj, 0, obj.length, testPort, '127.0.0.1', err => {
     if (err) console.log('Error communicating with the test driver', err)
   })
 }
 
 async function onMessage (message) {
+  if (LOG_MESSAGES) console.log('driverserver got', message.toString('utf8'))
   const {msgId, cmd, args} = JSON.parse(message.toString('utf8'))
   var method = METHODS[cmd]
   if (!method) method = () => new Error('Invalid method: ' + cmd)

app/background-process/ui/windows.js

@@ -39,7 +39,7 @@ export async function setup () {
     else app.on('ready', ensureOneWindowExists)
   })
   ipcMain.on('new-window', () => createShellWindow())
-  app.on('window-all-closed', () => {
+  app.on('custom-window-all-closed', () => {
     if (process.platform !== 'darwin') app.quit()
   })
 
@@ -218,8 +218,8 @@ export function createShellWindow (windowState) {
 export function getActiveWindow () {
   // try to pull the focused window; if there isnt one, fallback to the last created
   var win = BrowserWindow.getFocusedWindow()
-  if (!win) {
-    win = BrowserWindow.getAllWindows().pop()
+  if (!win || win.webContents.getURL() !== 'beaker://shell-window/') {
+    win = BrowserWindow.getAllWindows().filter(win => win.webContents.getURL() === 'beaker://shell-window/').pop()
   }
   return win
 }
@@ -317,6 +317,11 @@ function ensureVisibleOnSomeDisplay (windowState) {
 function onClosed (win) {
   return e => {
     numActiveWindows--
+    if (numActiveWindows === 0) {
+      // emit a custom 'window-all-closed'
+      // we need to do this because we have hidden windows running additional behaviors
+      app.emit('custom-window-all-closed')
+    }
 
     // deny any outstanding permission requests
     permissions.denyAllRequests(win)

app/dat-daemon.js

@@ -0,0 +1,8 @@
+const {join} = require('path')
+const rpcAPI = require('pauls-electron-rpc')
+const beakerCoreDatDaemon = require('@beaker/core/dat/daemon')
+
+beakerCoreDatDaemon.setup({
+  rpcAPI,
+  logfilePath: join(window.userDataPath, 'dat.log')
+})

app/hidden-window.html

@@ -0,0 +1,30 @@
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy" content="default-src 'self' 'unsafe-inline'">
+  </head>
+  <body>
+    <h1>Hidden Window</h1>
+    <script>
+      // patch console methods to properly give us full content to stdout
+      function patchConsole (name) {
+        var org = console[name]
+        console[name] = (...args) => {
+          org(args.map(v => {
+            if (typeof v === 'string') return v
+            return JSON.stringify(v)
+          }).join(' '))
+        }
+      }
+      patchConsole('log')
+      patchConsole('debug')
+      patchConsole('warn')
+      patchConsole('error')
+
+      // load module
+      var url = new URL(window.location)
+      console.log('Loading: ' + url.searchParams.get('module'))
+      window.userDataPath = url.searchParams.get('userDataPath')
+      require(url.searchParams.get('module'))
+    </script>
+  </body>
+</html>

app/package.json

@@ -8,7 +8,7 @@
   "copyright": "© 2018, Blue Link Labs",
   "main": "background-process.build.js",
   "dependencies": {
-    "@beaker/core": "2.19.0",
+    "@beaker/core": "3.0.0",
     "@beaker/dat-archive-file-diff": "^1.0.0",
     "@beaker/electron-localshortcut": "^1.0.0",
     "anymatch": "^1.3.2",
@@ -70,7 +70,7 @@
     "os-name": "^2.0.1",
     "parse-dat-url": "^3.0.1",
     "pauls-dat-api": "^8.0.1",
-    "pauls-electron-rpc": "^4.1.3",
+    "pauls-electron-rpc": "^4.3.0",
     "pauls-word-boundary": "^1.0.0",
     "pify": "^2.3.0",
     "pretty-bytes": "^3.0.1",

tests/dat-archive-synced-folder-test.js

@@ -474,6 +474,7 @@ test('previewMode=true with a local folder', async t => {
   var datUrl = res.url
   t.truthy(datUrl.startsWith('dat://'))
 
+
   const readArchiveFile = path => (
     mainTab.executeJavascript(`
       var archive = new DatArchive("${datUrl}")
@@ -487,13 +488,9 @@ test('previewMode=true with a local folder', async t => {
 
   // configure
   var res = await mainTab.executeJavascript(`
-    beaker.archives.setLocalSyncPath("${datUrl}", "${escapeWindowsSlashes(filePath)}")
+    beaker.archives.setLocalSyncPath("${datUrl}", "${escapeWindowsSlashes(filePath)}", {previewMode: true})
   `)
   t.falsy(res)
-  var res = await mainTab.executeJavascript(`
-    beaker.archives.setUserSettings("${datUrl}", {previewMode: true})
-  `)
-  t.deepEqual(res.previewMode, true)
 
   // write files locally
   await dir.write('local-file.txt', 'local')
@@ -504,13 +501,16 @@ test('previewMode=true with a local folder', async t => {
   // wait a sec
   await new Promise(resolve => setTimeout(resolve, 1e3))
 
+
   // make sure no sync has occurred
   var res = await mainTab.executeJavascript(`
     var archive = new DatArchive("${datUrl}")
     archive.readdir('/', {recursive: true})
   `)
   t.deepEqual(res.map(toUnixPath).sort(), ['.datignore', 'dat.json'].sort())
 
+
+
   // write to the archive
   var syncPromise = waitForSync(mainTab, datUrl, 'folder')
   var res = await mainTab.executeJavascript(`
@@ -521,6 +521,7 @@ test('previewMode=true with a local folder', async t => {
   // wait for sync to occur
   await syncPromise
 
+
   // check local folder
   t.deepEqual((await dir.listAsync()).sort(), ['.datignore', 'dat.json', 'local-file.txt', 'archive-file.txt', 'local-folder'].sort())
   t.deepEqual((await dir.listAsync('local-folder')).sort(), ['file1.txt', 'file2.txt'].sort())
@@ -552,6 +553,7 @@ test('previewMode=true with a local folder', async t => {
   await syncPromise2
   await mainTab.executeJavascript(`beaker.archives.ensureLocalSyncFinished("${datUrl}")`)
 
+
   // check local folder
   t.deepEqual((await dir.listAsync()).sort(), ['.datignore', 'dat.json', 'local-file.txt', 'archive-file.txt', 'local-folder'].sort())
   t.deepEqual((await dir.listAsync('local-folder')).sort(), ['file1.txt', 'file2.txt'].sort())

tests/dat-archive-web-api-test.js

@@ -856,6 +856,28 @@ test('versioned reads and writes', async t => {
     console.log('Weird history', history)
   }
   t.deepEqual(history.length, 5)
+  var history2 = await app.executeJavascript(`
+    var archive = new DatArchive("${newTestDatURL}")
+    archive.history({reverse: true})
+  `)
+  t.deepEqual(history2[0].version, history[4].version)
+  var history2 = await app.executeJavascript(`
+    var archive = new DatArchive("${newTestDatURL}")
+    archive.history({start: 0, end: 5, reverse: true})
+  `)
+  t.deepEqual(history2[0].version, history[4].version)
+  var history2 = await app.executeJavascript(`
+    var archive = new DatArchive("${newTestDatURL}")
+    archive.history({start: 2, end: 4})
+  `)
+  t.deepEqual(history2[0].version, history[1].version)
+  t.deepEqual(history2[1].version, history[2].version)
+  var history2 = await app.executeJavascript(`
+    var archive = new DatArchive("${newTestDatURL}")
+    archive.history({start: 2, end: 4, reverse: true})
+  `)
+  t.deepEqual(history2[0].version, history[2].version)
+  t.deepEqual(history2[1].version, history[1].version)
 
   // read back versions
   t.deepEqual((await readdir(newTestDatURL + '+1', '/')).length, 1)

tests/experimental-capture-page-web-api.js

@@ -71,7 +71,7 @@ test('experiment must be opted into', async t => {
 test('capturePage()', async t => {
   // capture https://example.com
   var png = mainTab.executeJavascript(`
-    experimental.capturePage('https://example.com').then(png => ({byteLength: png.byteLength, isArrayBuffer: png instanceof ArrayBuffer}))
+    experimental.capturePage('https://example.com').then(png => ({byteLength: png.byteLength}))
   `)
 
   // accept the permission prompt
@@ -80,6 +80,5 @@ test('capturePage()', async t => {
   png = await png
 
   // check results
-  t.truthy(png.isArrayBuffer)
   t.truthy(png.byteLength > 0)
 })

tests/experimental-dat-peers-web-api-test.js

@@ -80,7 +80,7 @@ test('experiment must be opted into', async t => {
   `)
 
   // make sure the change has made it to browser 2
-  // await new Promise(resolve => setTimeout(resolve, 1e3))
+  await new Promise(resolve => setTimeout(resolve, 1e3))
   var manifest = await app2.executeJavascript(`
     (async function () {
       var archive = new DatArchive("${createdDatUrl}")

tests/experimental-library-web-api-test.js

@@ -74,6 +74,9 @@ test('experiment must be opted into', async t => {
 })
 
 test('library.list()', async t => {
+  // give time for the mtime to be read
+  await new Promise(r => setTimeout(r, 1e3))
+
   // list (no options)
   var listingP = mainTab.executeJavascript(`
     experimental.library.list()

tests/lib/browser-driver.js

@@ -1,6 +1,8 @@
 const childProcess = require('child_process')
 const dgram = require('dgram')
 
+const LOG_MESSAGES = false
+
 exports.start = function (opts) {
   return new BrowserDriver(opts)
 }
@@ -36,6 +38,7 @@ class BrowserDriver {
       // handle rpc responses
       this.sock.on('message', (message) => {
         message = JSON.parse(message.toString('utf8'))
+        if (LOG_MESSAGES) console.log('driverclient got', message)
 
         // special handling for isReady message
         if (message.isReady) {
@@ -56,6 +59,7 @@ class BrowserDriver {
 
   async rpc (cmd, ...args) {
     // send rpc request
+    if (LOG_MESSAGES) console.log('driverclient sent', {msgId, cmd, args})
     var msgId = this.rpcCalls.length
     var msg = Buffer.from(JSON.stringify({msgId, cmd, args}), 'utf8')
     this.sock.send(msg, 0, msg.length, this.browserPort, '127.0.0.1', err => {