use zeekrunner scripts and zqd cli option (#732)

This is part of the brimsec/brim repo side work for #731.

Use the -zeekrunner cli option to zqd ( brimdata/super#718 ), defaulting to the value of environment variable BRIM_ZEEK_RUNNER, else to the zeekrunner included from the zdeps zeek artifact ( brimdata/zeek#23 ).

CONTRIBUTING.md

@@ -32,9 +32,11 @@ When developing features that need a non-released zqd instance, you can:
 
 ### zeek
 
-Brim uses [Zeek](https://www.zeek.org) to convert packet captures into Zeek logs. These logs are then combined and stored in [ZNG](https://github.com/brimsec/zq/blob/master/zng/docs/spec.md) format. As an npm postinstall step, a [zeek artifact](https://github.com/brimsec/zeek/releases) (an archive with a zeek binary and other configuration files) is downloaded and stored in the `./zdeps` directory.
+Brim, via zqd, uses [Zeek](https://www.zeek.org) to convert packet captures into Zeek logs. These logs are then combined and stored in [ZNG](https://github.com/brimsec/zq/blob/master/zng/docs/spec.md) format.
 
-zqd runs zeek as needed to ingest packet capture data. zqd expects that a `zeek` command is available in its PATH; Brim ensures this is true for the zeek artifact under `./zdeps`.
+As an npm postinstall step, a [zeek artifact](https://github.com/brimsec/zeek/releases) is downloaded and expanded into the `./zdeps/zeek` directory. This artifact contains a zeek binary and associated scripts, and a "zeek runner" script or command that is called by zqd. zqd is passed the full path to the zeek runner via the `-zeekrunner` command line option. When a pcap file is ingested, zqd runs the zeek runner with no arguments and its working directory set to an output directory for the zeek TSV logs, and then feeds the pcap data to the zeek runner via stdin. zqd then internally converts the zeek TSV logs into ZNG format.
+
+An alternate Zeek setup may be used by overriding the zeek runner location. This may be done either by launching Brim with the `BRIM_ZEEK_RUNNER` environment variable set to the absolute path of a zeek runner script or commmand, or by setting a preference in the Brim UI (pending [brimsec/brim#741](https://github.com/brimsec/brim/issues/741)).
 
 ## Tests
 

package.json

@@ -146,7 +146,7 @@
     "rimraf": "^3.0.2",
     "source-map-support": "^0.5.16",
     "valid-url": "^1.0.9",
-    "zq": "git+https://github.com/brimsec/zq.git#3de3c96be7a4efda0ab81486231c733c67f86448"
+    "zq": "git+https://github.com/brimsec/zq.git#5f91910a566a833f1591911144bad0538c77ccdf"
   },
   "optionalDependencies": {
     "electron-installer-debian": "^3.0.0",

scripts/download-zdeps/index.js

@@ -105,18 +105,8 @@ async function zeekDownload(version, zdepsPath) {
   const plat = platformDefs[process.platform]
   const zeekPath = path.join(zdepsPath, "zeek")
 
-  let artifactFile, artifactUrl
-
-  if (process.platform == "win32") {
-    // Special casing for zeek on windows as it's not yet created automatically
-    // like linux/mac.
-    artifactFile = "zeek.zip"
-    artifactUrl =
-      "https://storage.googleapis.com/brimsec/zeek-windows/zeek-20200403.zip"
-  } else {
-    artifactFile = `zeek-${version}.${plat.osarch}.zip`
-    artifactUrl = `https://github.com/brimsec/zeek/releases/download/${version}/${artifactFile}`
-  }
+  const artifactFile = `zeek-${version}.${plat.osarch}.zip`
+  const artifactUrl = `https://github.com/brimsec/zeek/releases/download/${version}/${artifactFile}`
 
   const tmpdir = tmp.dirSync({unsafeCleanup: true})
   try {
@@ -132,11 +122,7 @@ async function zeekDownload(version, zdepsPath) {
     tmpdir.removeCallback()
   }
 
-  if (process.platform == "win32") {
-    console.log("zeek windows artifact downloaded to " + zeekPath)
-  } else {
-    console.log("zeek " + version + " downloaded to " + zeekPath)
-  }
+  console.log("zeek " + version + " downloaded to " + zeekPath)
 }
 
 // Build the zqd binary inside the node_modules/zq directory via "make build".
@@ -157,7 +143,7 @@ async function main() {
   try {
     // We encode the zeek version here for now to avoid the unncessary
     // git clone if it were in package.json.
-    const zeekVersion = "v3.0.2-brim2"
+    const zeekVersion = "v3.0.2-brim3"
     await zeekDownload(zeekVersion, zdepsPath)
 
     // The zq dependency should be a git tag or commit. Any tag that

src/js/zqd/zqd.js

@@ -11,18 +11,20 @@ import * as cmd from "../stdlib/cmd"
 import electronIsDev from "../electron/isDev"
 
 // Paths for the zqd and zeek programs.
-const zqdPath = join(app.getAppPath(), "zdeps")
-const zqdZeekPath = join(zqdPath, "zeek")
+const zdepsDirectory = join(app.getAppPath(), "zdeps")
 
 const platformDefs = {
   darwin: {
-    zqdBin: "zqd"
+    zqdBin: "zqd",
+    zeekRunnerBin: "zeekrunner"
   },
   linux: {
-    zqdBin: "zqd"
+    zqdBin: "zqd",
+    zeekRunnerBin: "zeekrunner"
   },
   win32: {
-    zqdBin: "zqd.exe"
+    zqdBin: "zqd.exe",
+    zeekRunnerBin: "zeekrunner.exe"
   }
 }
 
@@ -69,13 +71,34 @@ function zqdCommand(): string {
     return plat.zqdBin
   }
 
-  const zqdBin = resolve(join(zqdPath, plat.zqdBin))
+  const zqdBin = resolve(join(zdepsDirectory, plat.zqdBin))
   if (!pathExistsSync(zqdBin)) {
     throw new Error("zqd binary not present at " + zqdBin)
   }
   return zqdBin
 }
 
+function zeekRunnerCommand(): string {
+  const plat = platformDefs[process.platform]
+  if (!plat) {
+    throw new Error("unsupported platform for zqd")
+  }
+
+  let zeekRunner = process.env.BRIM_ZEEK_RUNNER
+  if (!zeekRunner) {
+    // TODO: https://github.com/brimsec/brim/issues/741
+    // If the environment varible isn't set, allow the user to set
+    // a preferred zeekrunner location via preferences.
+    zeekRunner = resolve(join(zdepsDirectory, "zeek", plat.zeekRunnerBin))
+  }
+
+  if (!pathExistsSync(zeekRunner)) {
+    throw new Error("zeek runner not present at " + zeekRunner)
+  }
+
+  return zeekRunner
+}
+
 export class ZQD {
   zqd: ChildProcess
   root: string
@@ -87,14 +110,6 @@ export class ZQD {
   start() {
     mkdirpSync(this.root, {recursive: true, mode: 0o755})
 
-    // We saw errors on cmd.com vs powershell when we tried to clone
-    // process.env and then determine whether to use "PATH" or "Path".
-    // Windows environment variables are case-insensitive; see the
-    // process.env docs. Directly altering process.env is safe and
-    // less error prone.
-    const sep = process.platform == "win32" ? ";" : ":"
-    process.env["PATH"] = [zqdPath, zqdZeekPath, process.env["PATH"]].join(sep)
-
     const opts = {
       stdio: "inherit"
     }
@@ -108,7 +123,9 @@ export class ZQD {
       "-datadir",
       this.root,
       "-config",
-      confFile
+      confFile,
+      "-zeekrunner",
+      zeekRunnerCommand()
     ]
     log.info("spawning zqd:", zqdCommand(), args.join(" "))