A set of tweaks to simplify electron packaging (#421)

  * Replace `ormconfig.js` with a newer mechanism of configuring
    TypeORM that can be included in the source code properly.
    The path to `ormconfig.js` has always been awkward to handle,
    and eliminating the file makes building different Grist setups
    a bit simpler.
  * Remove `electron` package. It is barely used, just for some old
    remnants of an older attempt at electron packaging. It was used
    for two types, which I left at `any` for now. More code pruning is
    no doubt possible here, but I'd rather do it when Electron packaging
    has solidified.
  * Add a hook for replacing the login system, and for adding some
    extra middleware the login system may need.
  * Add support for some more possible locations of Python, which
    arise when a standalone version of it is included in the Electron
    package. This isn't very general purpose, just configurations
    that I found useful.
  * Support using grist-core within a yarn workspace - the only tweak
    needed was webpack related.
  * Allow an external ID to be optionally associated with documents.

Dockerfile

@@ -103,7 +103,6 @@ COPY --from=sandbox /runsc /usr/bin/runsc
 
 # Add files needed for running server.
 ADD package.json /grist/package.json
-ADD ormconfig.js /grist/ormconfig.js
 ADD bower_components /grist/bower_components
 ADD sandbox /grist/sandbox
 ADD plugins /grist/plugins

app/client/lib/SafeBrowser.ts

@@ -40,7 +40,6 @@ import { getOriginUrl } from 'app/common/urlUtils';
 import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI';
 import { RenderOptions, RenderTarget } from 'app/plugin/RenderOptions';
 import { checkers } from 'app/plugin/TypeCheckers';
-import { IpcMessageEvent } from 'electron';
 import { IMsgCustom, IMsgRpcCall, Rpc } from 'grain-rpc';
 import { Disposable } from './dispose';
 const G = getBrowserGlobals('document', 'window');
@@ -316,7 +315,7 @@ class WebviewProcess extends ViewProcess {
     // TODO: find a way for keyboard events to play nice when webviews are non-modal.
     Mousetrap.setPaused(true);
     this.autoDisposeCallback(() => Mousetrap.setPaused(false));
-    webview.addEventListener('ipc-message', (event: IpcMessageEvent) => {
+    webview.addEventListener('ipc-message', (event: any /* IpcMessageEvent */) => {
       // The event object passed to the listener is missing proper documentation. In the examples
       // listed in https://electronjs.org/docs/api/ipc-main the arguments should be passed to the
       // listener after the event object, but this is not happening here. Only we know it is a

app/client/lib/uploads.ts

@@ -14,7 +14,6 @@ import {GristLoadConfig} from 'app/common/gristUrls';
 import {byteString, safeJsonParse} from 'app/common/gutil';
 import {FetchUrlOptions, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
 import {docUrl} from 'app/common/urlUtils';
-import {OpenDialogOptions} from 'electron';
 import noop = require('lodash/noop');
 import trimStart = require('lodash/trimStart');
 import {basename} from 'path';      // made available by webpack using path-browserify module.
@@ -69,18 +68,18 @@ function getFileDialogOptions(options: SelectFileOptions): FileDialogOptions {
 }
 
 // Helper to convert SelectFileOptions to electron's OpenDialogOptions.
-function getElectronOptions(options: SelectFileOptions): OpenDialogOptions {
-  const resOptions: OpenDialogOptions = {
-    filters: [],
+function getElectronOptions(options: SelectFileOptions) /*: OpenDialogOptions */ {
+  const resOptions /*: OpenDialogOptions*/ = {
+    filters: [] as Array<{name: string, extensions: any}>,
     properties: ['openFile'],
   };
   if (options.extensions) {
     // Electron does not expect leading period.
     const extensions = options.extensions.map(e => trimStart(e, '.'));
-    resOptions.filters!.push({name: 'Select files', extensions});
+    resOptions.filters.push({name: 'Select files', extensions});
   }
   if (options.multiple) {
-    resOptions.properties!.push('multiSelections');
+    resOptions.properties.push('multiSelections');
   }
   return resOptions;
 }

app/common/UserAPI.ts

@@ -115,6 +115,8 @@ export interface DocumentOptions {
   description?: string|null;
   icon?: string|null;
   openMode?: OpenDocMode|null;
+  externalId?: string|null;  // A slot for storing an externally maintained id.
+                             // Not used in grist-core, but handy for Electron app.
 }
 
 export interface DocumentProperties extends CommonProperties {

app/gen-server/entity/Document.ts

@@ -99,6 +99,9 @@ export class Document extends Resource {
         if (props.options.icon !== undefined) {
           this.options.icon = sanitizeIcon(props.options.icon);
         }
+        if (props.options.externalId !== undefined) {
+          this.options.externalId = props.options.externalId;
+        }
         // Normalize so that null equates with absence.
         for (const key of Object.keys(this.options) as Array<keyof DocumentOptions>) {
           if (this.options[key] === null) {

app/server/lib/FlexServer.ts

@@ -36,7 +36,8 @@ import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
 import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg';
 import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
-import {DocTemplate, GristLoginMiddleware, GristServer, RequestWithGrist} from 'app/server/lib/GristServer';
+import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
+        RequestWithGrist} from 'app/server/lib/GristServer';
 import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
 import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
 import {IBilling} from 'app/server/lib/IBilling';
@@ -157,6 +158,7 @@ export class FlexServer implements GristServer {
   private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
   private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
   private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
+  private _getLoginSystem?: () => Promise<GristLoginSystem>;
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
@@ -233,6 +235,11 @@ export class FlexServer implements GristServer {
     });
   }
 
+  // Allow overridding the login system.
+  public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
+    this._getLoginSystem = loginSystem;
+  }
+
   public getHost(): string {
     return `${this.host}:${this.getOwnPort()}`;
   }
@@ -481,12 +488,19 @@ export class FlexServer implements GristServer {
     this.app.use(/^\/help\//, expressWrap(async (req, res) => {
       res.redirect('https://support.getgrist.com');
     }));
+    // If there is a directory called "static_ext", serve material from there
+    // as well. This isn't used in grist-core but is handy for extensions such
+    // as an Electron app.
+    const staticExtDir = getAppPathTo(this.appRoot, 'static') + '_ext';
+    const staticExtApp = fse.existsSync(staticExtDir) ?
+      express.static(staticExtDir, options) : null;
     const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options);
     const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options);
     if (process.env.GRIST_LOCALES_DIR) {
       const locales = express.static(process.env.GRIST_LOCALES_DIR, options);
       this.app.use("/locales", this.tagChecker.withTag(locales));
     }
+    if (staticExtApp) { this.app.use(this.tagChecker.withTag(staticExtApp)); }
     this.app.use(this.tagChecker.withTag(staticApp));
     this.app.use(this.tagChecker.withTag(bowerApp));
   }
@@ -700,7 +714,7 @@ export class FlexServer implements GristServer {
     this.addOrg();
 
     // Create the sessionStore and related objects.
-    const {sessions, sessionMiddleware, sessionStore} = initGristSessions(this.instanceRoot, this);
+    const {sessions, sessionMiddleware, sessionStore} = initGristSessions(getUnpackedAppRoot(this.instanceRoot), this);
     this.app.use(sessionMiddleware);
     this.app.use(signInStatusMiddleware);
 
@@ -901,11 +915,16 @@ export class FlexServer implements GristServer {
 
     // TODO: We could include a third mock provider of login/logout URLs for better tests. Or we
     // could create a mock SAML identity provider for testing this using the SAML flow.
-    const loginSystem = await (process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() : getLoginSystem());
+    const loginSystem = await (process.env.GRIST_TEST_LOGIN ? getTestLoginSystem() :
+      (this._getLoginSystem?.() || getLoginSystem()));
     this._loginMiddleware = await loginSystem.getMiddleware(this);
     this._getLoginRedirectUrl = tbind(this._loginMiddleware.getLoginRedirectUrl, this._loginMiddleware);
     this._getSignUpRedirectUrl = tbind(this._loginMiddleware.getSignUpRedirectUrl, this._loginMiddleware);
     this._getLogoutRedirectUrl = tbind(this._loginMiddleware.getLogoutRedirectUrl, this._loginMiddleware);
+    const wildcardMiddleware = this._loginMiddleware.getWildcardMiddleware?.();
+    if (wildcardMiddleware?.length) {
+        this.app.use(wildcardMiddleware);
+    }
   }
 
   public addComm() {

app/server/lib/GristServer.ts

@@ -62,6 +62,8 @@ export interface GristLoginMiddleware {
   getLoginOrSignUpMiddleware?(): express.RequestHandler[];
   // Optional middleware for the GET /logout route.
   getLogoutMiddleware?(): express.RequestHandler[];
+  // Optional middleware for all routes.
+  getWildcardMiddleware?(): express.RequestHandler[];
   // Returns arbitrary string for log.
   addEndpoints(app: express.Express): Promise<string>;
   // Optionally, extract profile from request. Result can be a profile,

app/server/lib/MinimalLogin.ts

@@ -40,7 +40,7 @@ export async function getMinimalLoginSystem(): Promise<GristLoginSystem> {
   };
 }
 
-function getDefaultProfile(): UserProfile {
+export function getDefaultProfile(): UserProfile {
   return {
     email: process.env.GRIST_DEFAULT_EMAIL || 'you@example.com',
     name: 'You',

app/server/lib/NSandbox.ts

@@ -5,6 +5,7 @@ import {arrayToString} from 'app/common/arrayToString';
 import * as marshal from 'app/common/marshal';
 import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox';
 import log from 'app/server/lib/log';
+import {getAppRoot, getAppRootFor, getUnpackedAppRoot} from 'app/server/lib/places';
 import {
   DirectProcessControl,
   ISandboxControl,
@@ -575,7 +576,7 @@ function gvisor(options: ISandboxOptions): SandboxProcess {
   // Check for local virtual environments created with core's
   // install:python2 or install:python3 targets. They'll need
   // some extra sharing to make available in the sandbox.
-  const venv = path.join(process.cwd(),
+  const venv = path.join(getAppRootFor(getAppRoot(), 'sandbox'),
                          pythonVersion === '2' ? 'venv' : 'sandbox_venv3');
   if (fs.existsSync(venv)) {
     wrapperArgs.addMount(venv);
@@ -869,19 +870,24 @@ function findPython(command: string|undefined, preferredVersion?: string) {
   // TODO: rationalize this, it is a product of haphazard growth.
   const prefs = preferredVersion === '2' ? ['venv', 'sandbox_venv3'] : ['sandbox_venv3', 'venv'];
   for (const venv of prefs) {
-    const pythonPath = path.join(process.cwd(), venv, 'bin', 'python');
-    if (fs.existsSync(pythonPath)) {
-      command = pythonPath;
-      break;
+    const base = getUnpackedAppRoot();
+    // Try a battery of possible python executable paths when python is installed
+    // in a standalone directory.
+    // This battery of possibilities comes from Electron packaging, where python
+    // is bundled with Grist. Not all the possibilities are needed (there are
+    // multiple popular python bundles per OS).
+    for (const possiblePath of [['bin', 'python'], ['bin', 'python3'],
+                                ['Scripts', 'python.exe'], ['python.exe']] as const) {
+      const pythonPath = path.join(base, venv, ...possiblePath);
+      if (fs.existsSync(pythonPath)) {
+        return pythonPath;
+      }
     }
   }
   // Fall back on system python.
-  if (!command) {
-    command = which.sync(preferredVersion === '2' ? 'python2' : 'python3', {nothrow: true})
-      || which.sync(preferredVersion === '2' ? 'python2.7' : 'python3.9', {nothrow: true})
-      || which.sync('python');
-  }
-  return command;
+  return which.sync(preferredVersion === '2' ? 'python2' : 'python3', {nothrow: true})
+    || which.sync(preferredVersion === '2' ? 'python2.7' : 'python3.9', {nothrow: true})
+    || which.sync('python');
 }
 
 /**

app/server/lib/dbUtils.ts

@@ -1,6 +1,7 @@
 import {synchronizeProducts} from 'app/gen-server/entity/Product';
+import {codeRoot} from 'app/server/lib/places';
 import {Mutex} from 'async-mutex';
-import {Connection, createConnection, getConnection} from 'typeorm';
+import {Connection, createConnection, DataSourceOptions, getConnection} from 'typeorm';
 
 // Summary of migrations found in database and in code.
 interface MigrationSummary {
@@ -61,7 +62,7 @@ export async function getOrCreateConnection(): Promise<Connection> {
       if (!String(e).match(/ConnectionNotFoundError/)) {
         throw e;
       }
-      const connection = await createConnection();
+      const connection = await createConnection(getTypeORMSettings());
       // When using Sqlite, set a busy timeout of 3s to tolerate a little
       // interference from connections made by tests. Logging doesn't show
       // any particularly slow queries, but bad luck is possible.
@@ -98,3 +99,49 @@ export async function undoLastMigration(connection: Connection) {
   });
   if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); }
 }
+
+// Replace the old janky ormconfig.js file, which was always a source of
+// pain to use since it wasn't properly integrated into the typescript
+// project.
+function getTypeORMSettings(): DataSourceOptions {
+  // If we have a redis server available, tell typeorm.  Then any queries built with
+  // .cache() called on them will be cached via redis.
+  // We use a separate environment variable for the moment so that we don't have to
+  // enable this until we really need it.
+  const redisUrl = process.env.TYPEORM_REDIS_URL ? new URL(process.env.TYPEORM_REDIS_URL) : undefined;
+  const cache = redisUrl ? {
+    cache: {
+      type: "redis",
+      options: {
+        host: redisUrl.hostname,
+        port: parseInt(redisUrl.port || "6379", 10)
+      }
+    } as const
+  } : undefined;
+
+  return {
+    "name": process.env.TYPEORM_NAME || "default",
+    "type": (process.env.TYPEORM_TYPE as any) || "sqlite",  // officially, TYPEORM_CONNECTION -
+                                                   // but if we use that, this file will never
+                                                   // be read, and we can't configure
+                                                   // caching otherwise.
+    "database": process.env.TYPEORM_DATABASE || "landing.db",
+    "username": process.env.TYPEORM_USERNAME || undefined,
+    "password": process.env.TYPEORM_PASSWORD || undefined,
+    "host": process.env.TYPEORM_HOST || undefined,
+    "port": process.env.TYPEORM_PORT ? parseInt(process.env.TYPEORM_PORT, 10) : undefined,
+    "synchronize": false,
+    "migrationsRun": false,
+    "logging": process.env.TYPEORM_LOGGING === "true",
+    "entities": [
+      `${codeRoot}/app/gen-server/entity/*.js`
+    ],
+    "migrations": [
+      `${codeRoot}/app/gen-server/migration/*.js`        // migration files don't actually get packaged.
+    ],
+    "subscribers": [
+      `${codeRoot}/app/gen-server/subscriber/*.js`
+    ],
+    ...cache,
+  };
+}

app/server/lib/places.ts

@@ -9,14 +9,25 @@ import * as path from 'path';
  */
 export const codeRoot = path.dirname(path.dirname(path.dirname(__dirname)));
 
+let _cachedAppRoot: string|undefined;
+
 /**
- * Returns the appRoot, i.e. the directory containing ./sandbox, ./node_modules, ./ormconfig.js,
+ * Returns the appRoot, i.e. the directory containing ./sandbox, ./node_modules,
  * etc.
  */
 export function getAppRoot(): string {
+  if (_cachedAppRoot) { return _cachedAppRoot; }
+  _cachedAppRoot = getAppRootWithoutCaching();
+  return _cachedAppRoot;
+}
+
+// Uncached version of getAppRoot()
+function getAppRootWithoutCaching(): string {
   if (process.env.APP_ROOT_PATH) { return process.env.APP_ROOT_PATH; }
-  if (codeRoot.endsWith('/_build/core')) { return path.dirname(path.dirname(codeRoot)); }
-  return codeRoot.endsWith('/_build') ? path.dirname(codeRoot) : codeRoot;
+  if (codeRoot.endsWith('/_build/core') || codeRoot.endsWith('\\_build\\core')) {
+    return path.dirname(path.dirname(codeRoot));
+  }
+  return (codeRoot.endsWith('/_build') || codeRoot.endsWith('\\_build')) ? path.dirname(codeRoot) : codeRoot;
 }
 
 /**
@@ -25,7 +36,14 @@ export function getAppRoot(): string {
  * which is that .asar file in packaged form, and returns a directory where
  * remaining files are available on the regular filesystem.
  */
-export function getUnpackedAppRoot(appRoot: string): string {
+export function getUnpackedAppRoot(appRoot: string = getAppRoot()): string {
+  if (path.basename(appRoot) == 'app.asar') {
+    return path.resolve(path.dirname(appRoot), 'app.asar.unpacked');
+  }
+  if (path.dirname(appRoot).endsWith('app.asar')) {
+    return path.resolve(path.dirname(path.dirname(appRoot)),
+                        'app.asar.unpacked', 'core');
+  }
   return path.resolve(path.dirname(appRoot), path.basename(appRoot, '.asar'));
 }
 

app/server/mergedServerMain.ts

@@ -6,6 +6,7 @@
  */
 
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
+import {GristLoginSystem} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
 
 // Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
@@ -36,20 +37,25 @@ interface ServerOptions extends FlexServerOptions {
                            //    logToConsole is set to true)
   externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
                            // variables, which get set in various ways in dev/test entry points)
+  loginSystem?: () => Promise<GristLoginSystem>;
 }
 
 /**
  * Start a server on the given port, including the functionality specified in serverTypes.
  */
 export async function main(port: number, serverTypes: ServerType[],
-                           options: ServerOptions = {logToConsole: true}) {
+                           options: ServerOptions = {}) {
   const includeHome = serverTypes.includes("home");
   const includeDocs = serverTypes.includes("docs");
   const includeStatic = serverTypes.includes("static");
   const includeApp = serverTypes.includes("app");
 
   const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
 
+  if (options.loginSystem) {
+    server.setLoginSystem(options.loginSystem);
+  }
+
   server.addCleanup();
   server.setDirectory();
 
@@ -58,7 +64,7 @@ export async function main(port: number, serverTypes: ServerType[],
     server.testAddRouter();
   }
 
-  if (options.logToConsole) { server.addLogging(); }
+  if (options.logToConsole !== false) { server.addLogging(); }
   if (options.externalStorage === false) { server.disableExternalStorage(); }
   await server.loadConfig();