(core) updates from grist-core

app/client/DefaultHooks.ts

@@ -0,0 +1,11 @@
+import { UrlTweaks } from 'app/common/gristUrls';
+
+export interface IHooks {
+  iframeAttributes?: Record<string, any>,
+  fetch?: typeof fetch,
+  baseURI?: string,
+  urlTweaks?: UrlTweaks,
+}
+
+export const defaultHooks: IHooks = {
+};

app/client/Hooks.ts

@@ -0,0 +1,3 @@
+import {defaultHooks} from 'app/client/DefaultHooks';
+
+export const hooks = defaultHooks;

app/client/components/GristWSConnection.ts

@@ -1,11 +1,12 @@
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {guessTimezone} from 'app/client/lib/guessTimezone';
 import {getSessionStorage} from 'app/client/lib/storage';
+import {newUserAPIImpl} from 'app/client/models/AppModel';
 import {getWorker} from 'app/client/models/gristConfigCache';
 import {CommResponseBase} from 'app/common/CommTypes';
 import * as gutil from 'app/common/gutil';
 import {addOrgToPath, docUrl, getGristConfig} from 'app/common/urlUtils';
-import {UserAPI, UserAPIImpl} from 'app/common/UserAPI';
+import {UserAPI} from 'app/common/UserAPI';
 import {Events as BackboneEvents} from 'backbone';
 import {Disposable} from 'grainjs';
 
@@ -25,7 +26,7 @@ async function getDocWorkerUrl(assignmentId: string|null): Promise<string|null>
   // never changes.
   if (assignmentId === null) { return docUrl(null); }
 
-  const api: UserAPI = new UserAPIImpl(getGristConfig().homeUrl!);
+  const api: UserAPI = newUserAPIImpl();
   return getWorker(api, assignmentId);
 }
 

app/client/components/WidgetFrame.ts

@@ -1,5 +1,6 @@
 import BaseView from 'app/client/components/BaseView';
 import {GristDoc} from 'app/client/components/GristDoc';
+import {hooks} from 'app/client/Hooks';
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel';
 import {AccessLevel, isSatisfied} from 'app/common/CustomWidget';
@@ -157,6 +158,7 @@ export class WidgetFrame extends DisposableWithEvents {
     return onElem(
       (this._iframe = dom('iframe', dom.cls('clipboard_focus'), dom.cls('custom_view'), {
         src: fullUrl,
+        ...hooks.iframeAttributes,
       }))
     );
   }

app/client/exposeModulesForTests.js

@@ -6,7 +6,7 @@ Object.assign(window.exposedModules, {
   grainjs: require('grainjs'),
   ko: require('knockout'),
   moment: require('moment-timezone'),
-  Comm: require('./components/Comm'),
+  Comm: require('app/client/components/Comm'),
   _loadScript: require('./lib/loadScript'),
   ConnectState: require('./models/ConnectState'),
 });

app/client/lib/localization.ts

@@ -1,3 +1,4 @@
+import {hooks} from 'app/client/Hooks';
 import {getGristConfig} from 'app/common/urlUtils';
 import {DomContents} from 'grainjs';
 import i18next from 'i18next';
@@ -34,7 +35,7 @@ export async function setupLocale() {
     // Detect what is resolved languages to load.
     const languages = i18next.languages;
     // Fetch all json files (all of which should be already preloaded);
-    const loadPath = `${document.baseURI}locales/{{lng}}.{{ns}}.json`;
+    const loadPath = `${hooks.baseURI || document.baseURI}locales/{{lng}}.{{ns}}.json`;
     const pathsToLoad: Promise<any>[] = [];
     async function load(lang: string, n: string) {
       const resourceUrl = loadPath.replace('{{lng}}', lang.replace("-", "_")).replace('{{ns}}', n);

app/client/models/AppModel.ts

@@ -1,4 +1,5 @@
 import {BehavioralPromptsManager} from 'app/client/components/BehavioralPromptsManager';
+import {hooks} from 'app/client/Hooks';
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {makeT} from 'app/client/lib/localization';
 import {sessionStorageObs} from 'app/client/lib/localStorageObs';
@@ -131,7 +132,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel {
 
   constructor(
     window: {gristConfig?: GristLoadConfig},
-    public readonly api: UserAPI = new UserAPIImpl(getHomeUrl()),
+    public readonly api: UserAPI = newUserAPIImpl(),
   ) {
     super();
     setErrorNotifier(this.notifier);
@@ -436,6 +437,12 @@ export function getHomeUrl(): string {
   return (gristConfig && gristConfig.homeUrl) || `${protocol}//${host}`;
 }
 
+export function newUserAPIImpl(): UserAPIImpl {
+  return new UserAPIImpl(getHomeUrl(), {
+    fetch: hooks.fetch,
+  });
+}
+
 export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) {
   if (!org) { return ''; }
   if (user && user.anonymous && org.owner && org.owner.id === user.id) {

app/client/models/gristUrlState.ts

@@ -23,6 +23,7 @@
  * Note that the form of URLs depends on the settings in window.gristConfig object.
  */
 import {unsavedChanges} from 'app/client/components/UnsavedChanges';
+import {hooks} from 'app/client/Hooks';
 import {UrlState} from 'app/client/lib/UrlState';
 import {decodeUrl, encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState,
         parseFirstUrlPart} from 'app/common/gristUrls';
@@ -134,15 +135,19 @@ export class UrlStateImpl {
    */
   public encodeUrl(state: IGristUrlState, baseLocation: Location | URL): string {
     const gristConfig = this._window.gristConfig || {};
-    return encodeUrl(gristConfig, state, baseLocation);
+    return encodeUrl(gristConfig, state, baseLocation, {
+      tweaks: hooks.urlTweaks,
+    });
   }
 
   /**
    * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation.
    */
   public decodeUrl(location: Location | URL): IGristUrlState {
     const gristConfig = this._window.gristConfig || {};
-    return decodeUrl(gristConfig, location);
+    return decodeUrl(gristConfig, location, {
+      tweaks: hooks.urlTweaks,
+    });
   }
 
   /**

app/client/ui/App.ts

@@ -84,7 +84,7 @@ export class App extends DisposableWithEvents {
 
     const isHelpPaneVisible = ko.observable(false);
 
-    G.document.querySelector('#grist-logo-wrapper').remove();
+    G.document.querySelector('#grist-logo-wrapper')?.remove();
 
     // Help pop-up pane
     const helpDiv = document.body.appendChild(

app/common/gristUrls.ts

@@ -198,7 +198,8 @@ export function encodeUrl(gristConfig: Partial<GristLoadConfig>,
                           options: {
                             // make an api url - warning: just barely works, and
                             // only for documents
-                            api?: boolean
+                            api?: boolean,
+                            tweaks?: UrlTweaks,
                           } = {}): string {
   const url = new URL(baseLocation.href);
   const parts = ['/'];
@@ -269,8 +270,10 @@ export function encodeUrl(gristConfig: Partial<GristLoadConfig>,
     }
   }
   const queryStr = encodeQueryParams(queryParams);
+
   url.pathname = parts.join('');
   url.search = queryStr;
+
   if (state.hash) {
     // Project tests use hashes, so only set hash if there is an anchor.
     url.hash = hashParts.join('.');
@@ -285,13 +288,23 @@ export function encodeUrl(gristConfig: Partial<GristLoadConfig>,
   } else {
     url.hash = '';
   }
+  options.tweaks?.postEncode?.({
+    url,
+    parts,
+    state,
+    baseLocation,
+  });
   return url.href;
 }
 
 /**
  * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation.
  */
-export function decodeUrl(gristConfig: Partial<GristLoadConfig>, location: Location | URL): IGristUrlState {
+export function decodeUrl(gristConfig: Partial<GristLoadConfig>, location: Location | URL, options?: {
+  tweaks?: UrlTweaks,
+}): IGristUrlState {
+  location = new URL(location.href);  // Make sure location is a URL.
+  options?.tweaks?.preDecode?.({ url: location });
   const parts = location.pathname.slice(1).split('/');
   const map = new Map<string, string>();
   for (let i = 0; i < parts.length; i += 2) {
@@ -871,3 +884,28 @@ export function getSlugIfNeeded(doc: {id: string, urlId: string|null, name: stri
   if (!shouldIncludeSlug(doc)) { return; }
   return nameToSlug(doc.name);
 }
+
+/**
+ * It is possible we want to remap Grist URLs in some way - specifically,
+ * grist-static does this. We allow for a hook that is called after
+ * encoding state as a URL, and a hook that is called before decoding
+ * state from a URL.
+ */
+export interface UrlTweaks {
+  /**
+   * Tweak an encoded URL. Operates on the URL directly, in place.
+   */
+  postEncode?(options: {
+    url: URL,
+    parts: string[],
+    state: IGristUrlState,
+    baseLocation: Location | URL,
+  }): void;
+
+  /**
+   * Tweak a URL prior to decoding it. Operates on the URL directly, in place.
+   */
+  preDecode?(options: {
+    url: URL,
+  }): void;
+}

app/common/marshal.ts

@@ -33,6 +33,13 @@ import * as util from 'util';
 export interface MarshalOptions {
   stringToBuffer?: boolean;
   version?: number;
+
+  // True if we want keys in dicts to be buffers.
+  // It is convenient to have some freedom here to simplify implementation
+  // of marshaling for some SQLite wrappers. This flag was initially
+  // introduced for a fork of Grist using better-sqlite3, and I don't
+  // remember exactly what the issues were.
+  keysAreBuffers?: boolean;
 }
 
 export interface UnmarshalOptions {
@@ -129,11 +136,13 @@ export class Marshaller {
   private _memBuf: MemBuffer;
   private readonly _floatCode: number;
   private readonly _stringCode: number;
+  private readonly _keysAreBuffers: boolean;
 
   constructor(options?: MarshalOptions) {
     this._memBuf = new MemBuffer(undefined);
     this._floatCode = options && options.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT;
     this._stringCode = options && options.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE;
+    this._keysAreBuffers = Boolean(options?.keysAreBuffers);
   }
 
   public dump(): Uint8Array {
@@ -261,7 +270,7 @@ export class Marshaller {
     const keys = Object.keys(obj);
     keys.sort();
     for (const key of keys) {
-      this.marshal(key);
+      this.marshal(this._keysAreBuffers ? Buffer.from(key) : key);
       this.marshal(obj[key]);
     }
     this._memBuf.writeUint8(marshalCodes.NULL);

app/gen-server/entity/AclRule.ts

@@ -13,7 +13,7 @@ export class AclRule extends BaseEntity {
   @PrimaryGeneratedColumn()
   public id: number;
 
-  @Column()
+  @Column({type: Number})
   public permissions: number;
 
   @OneToOne(type => Group, group => group.aclRule)

app/gen-server/entity/Alias.ts

@@ -5,13 +5,13 @@ import {Organization} from './Organization';
 
 @Entity({name: 'aliases'})
 export class Alias extends BaseEntity {
-  @PrimaryColumn({name: 'org_id'})
+  @PrimaryColumn({name: 'org_id', type: Number})
   public orgId: number;
 
-  @PrimaryColumn({name: 'url_id'})
+  @PrimaryColumn({name: 'url_id', type: String})
   public urlId: string;
 
-  @Column({name: 'doc_id'})
+  @Column({name: 'doc_id', type: String})
   public docId: string;
 
   @ManyToOne(type => Document)

app/gen-server/entity/BillingAccount.ts

@@ -34,14 +34,14 @@ export class BillingAccount extends BaseEntity {
   @JoinColumn({name: 'product_id'})
   public product: Product;
 
-  @Column()
+  @Column({type: Boolean})
   public individual: boolean;
 
   // A flag for when all is well with the user's subscription.
   // Probably shouldn't use this to drive whether service is provided or not.
   // Strip recommends updating an end-of-service datetime every time payment
   // is received, adding on a grace period of some days.
-  @Column({name: 'in_good_standing', default: nativeValues.trueValue})
+  @Column({name: 'in_good_standing', type: Boolean, default: nativeValues.trueValue})
   public inGoodStanding: boolean;
 
   @Column({type: nativeValues.jsonEntityType, nullable: true})

app/gen-server/entity/BillingAccountManager.ts

@@ -10,14 +10,14 @@ export class BillingAccountManager extends BaseEntity {
   @PrimaryGeneratedColumn()
   public id: number;
 
-  @Column({name: 'billing_account_id'})
+  @Column({name: 'billing_account_id', type: Number})
   public billingAccountId: number;
 
   @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' })
   @JoinColumn({name: 'billing_account_id'})
   public billingAccount: BillingAccount;
 
-  @Column({name: 'user_id'})
+  @Column({name: 'user_id', type: Number})
   public userId: number;
 
   @ManyToOne(type => User, { onDelete: 'CASCADE' })

app/gen-server/entity/Document.ts

@@ -24,7 +24,7 @@ function isValidUrlId(urlId: string) {
 @Entity({name: 'docs'})
 export class Document extends Resource {
 
-  @PrimaryColumn()
+  @PrimaryColumn({type: String})
   public id: string;
 
   @ManyToOne(type => Workspace)
@@ -35,7 +35,7 @@ export class Document extends Resource {
   public aclRules: AclRuleDoc[];
 
   // Indicates whether the doc is pinned to the org it lives in.
-  @Column({name: 'is_pinned', default: false})
+  @Column({name: 'is_pinned', type: Boolean, default: false})
   public isPinned: boolean;
 
   // Property that may be returned when the doc is fetched to indicate the access the

app/gen-server/entity/Group.ts

@@ -9,7 +9,7 @@ export class Group extends BaseEntity {
   @PrimaryGeneratedColumn()
   public id: number;
 
-  @Column()
+  @Column({type: String})
   public name: string;
 
   @ManyToMany(type => User)

app/gen-server/entity/Login.ts

@@ -5,18 +5,18 @@ import {User} from "./User";
 @Entity({name: 'logins'})
 export class Login extends BaseEntity {
 
-  @PrimaryColumn()
+  @PrimaryColumn({type: Number})
   public id: number;
 
   // This is the normalized email address we use for equality and indexing.
-  @Column()
+  @Column({type: String})
   public email: string;
 
   // This is how the user's email address should be displayed.
-  @Column({name: 'display_email'})
+  @Column({name: 'display_email', type: String})
   public displayEmail: string;
 
-  @Column({name: 'user_id'})
+  @Column({name: 'user_id', type: Number})
   public userId: number;
 
   @ManyToOne(type => User)

app/gen-server/entity/Organization.ts

@@ -29,6 +29,7 @@ export class Organization extends Resource {
   public id: number;
 
   @Column({
+    type: String,
     nullable: true
   })
   public domain: string;
@@ -46,7 +47,7 @@ export class Organization extends Resource {
   @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization)
   public aclRules: AclRuleOrg[];
 
-  @Column({name: 'billing_account_id'})
+  @Column({name: 'billing_account_id', type: Number})
   public billingAccountId: number;
 
   @ManyToOne(type => BillingAccount)

app/gen-server/entity/Pref.ts

@@ -11,10 +11,10 @@ export class Pref {
   // one, but we haven't marked them as so in the DB since the SQL standard frowns
   // on nullable primary keys (and Postgres doesn't support them).  We could add
   // another primary key, but we don't actually need one.
-  @PrimaryColumn({name: 'user_id'})
+  @PrimaryColumn({name: 'user_id', type: Number})
   public userId: number|null;
 
-  @PrimaryColumn({name: 'org_id'})
+  @PrimaryColumn({name: 'org_id', type: Number})
   public orgId: number|null;
 
   @ManyToOne(type => User)