Merge remote-tracking branch 'origin/master' into jupyter-explicit-ke…

…rnel

src/package.json

@@ -89,8 +89,8 @@
     "webpack-static": "cd $SALVUS_ROOT; scripts/update_color_scheme.coffee; scripts/update_react_static && CC_STATICPAGES=true NODE_ENV=development webpack --debug --output-pathinfo --progress --colors",
     "install-all": "scripts/smc-install-all",
     "make": "npm run install-all",
-    "delete-package-lock" : "git ls-files '../*/package-lock.json' | xargs rm -f",
-    "restore-package-lock" : "git ls-files '../*/package-lock.json' | xargs git checkout --",
+    "delete-package-lock": "git ls-files '../*/package-lock.json' | xargs rm -f",
+    "restore-package-lock": "git ls-files '../*/package-lock.json' | xargs git checkout --",
     "clean": "find $SMC_ROOT -type d -name node_modules | xargs rm -rf; rm -rf $SMC_ROOT/static"
   },
   "repository": {

src/smc-project/browser-websocket/api.ts

@@ -24,9 +24,10 @@ export function init_websocket_api(
   primus.on("connection", function(spark) {
     // Now handle the connection
     logger.debug(
-      "primus api",
+      "primus-api",
       `new connection from ${spark.address.ip} -- ${spark.id}`
     );
+
     spark.on("request", async function(data, done) {
       logger.debug("primus-api", "request", typeof data, JSON.stringify(data));
       try {
@@ -35,7 +36,7 @@ export function init_websocket_api(
         done(resp);
       } catch (err) {
         // put this in for debugging...
-        // It's normal to sometimes get errors, e.g., when a Jupyter kernel 
+        // It's normal to sometimes get errors, e.g., when a Jupyter kernel
         // isn't yet available.
         // console.trace(); logger.debug("primus-api error stacktrack", err.stack, err);
         done({ error: err.toString(), status: "error" });
@@ -45,6 +46,13 @@ export function init_websocket_api(
       logger.debug("primus-api", "data", typeof data, JSON.stringify(data));
     });*/
   });
+
+  primus.on("disconnection", function(spark) {
+    logger.debug(
+      "primus-api",
+      `end connection from ${spark.address.ip} -- ${spark.id}`
+    );
+  })
 }
 
 import { run_prettier, run_prettier_string } from "../formatters/prettier";

src/smc-project/browser-websocket/server.ts

@@ -35,6 +35,7 @@ export function init_websocket_server(
 
   init_websocket_api(primus, logger, client);
 
+  /*
   const eval_channel = primus.channel("eval");
   eval_channel.on("connection", function(spark) {
     // Now handle the connection
@@ -51,6 +52,7 @@ export function init_websocket_server(
       }
     });
   });
+  */
 
   const router = express.Router();
   const library: string = primus.library();

src/smc-project/package.json

@@ -34,7 +34,7 @@
     "posix": "^4.0.0",
     "prettier": "^1.16.0",
     "primus": "^7.3.2",
-    "primus-multiplex": "^3.2.1",
+    "primus-multiplex": "github:STRML/primus-multiplex",
     "primus-responder": "^1.0.4",
     "prom-client": "^11.1.3",
     "prometheus-gc-stats": "^0.6.0",

src/smc-project/sync/server.ts

@@ -177,6 +177,7 @@ class SyncTableChannel {
   private init_handlers(): void {
     this.log("init_handlers");
     this.channel.on("connection", this.new_connection.bind(this));
+    this.channel.on("disconnection", this.end_connection.bind(this));
   }
 
   private async init_synctable(): Promise<void> {
@@ -227,7 +228,10 @@ class SyncTableChannel {
     if (m === undefined) {
       return 0;
     }
-    return (this.connections_from_one_client[spark.conn.id] = Math.max(0,m - 1));
+    return (this.connections_from_one_client[spark.conn.id] = Math.max(
+      0,
+      m - 1
+    ));
   }
 
   private async new_connection(spark: Spark): Promise<void> {
@@ -296,17 +300,16 @@ class SyncTableChannel {
         this.log("error handling mesg -- ", err, err.stack);
       }
     });
+  }
 
-    spark.on("end", () => {
-      const m = this.decrement_connection_count(spark);
-      this.log(
-        `spark event -- end connection ${spark.address.ip} -- ${
-          spark.id
-        }  -- num_connections = ${this.num_connections()}  (from this client = ${m})`
-      );
-      this.check_if_should_close();
-    });
-
+  private async end_connection(spark: Spark): Promise<void> {
+    const m = this.decrement_connection_count(spark);
+    this.log(
+      `spark event -- end connection ${spark.address.ip} -- ${
+        spark.id
+      }  -- num_connections = ${this.num_connections()}  (from this client = ${m})`
+    );
+    this.check_if_should_close();
   }
 
   private send_synctable_to_browser(spark: Spark): void {

src/smc-project/x11/server.ts

@@ -6,7 +6,7 @@ TODO:
   - [ ] when stopping project, kill xpra's
 */
 
-import { spawn } from "child_process";
+import { spawn, SpawnOptions } from "child_process";
 import { callback } from "awaiting";
 const { abspath } = require("smc-util-node/misc_node");
 const { path_split } = require("smc-util/misc");
@@ -124,7 +124,7 @@ class X11Channel {
     const env = clone(process.env);
     env.DISPLAY = `:${this.display}`;
     const cwd = this.get_cwd();
-    const options = { cwd, env, detached: true, stdio: "ignore" };
+    const options: SpawnOptions = { cwd, env, detached: true, stdio: "ignore" };
     args = args != null ? args : [];
     try {
       const sub = spawn(command, args, options);

src/smc-util/smc-version.js

@@ -1,2 +1,2 @@
 /* autogenerated by the update_version script */
-exports.version=1550557037;
+exports.version=1551080928;

src/smc-util/sync/editor/db/doc.ts

@@ -267,8 +267,26 @@ export class DBDocument implements Document {
           const v = to[key];
           if (!isEqual(from[key], v)) {
             if (this.string_cols.has(key) && from[key] != null && v != null) {
-              // A string patch
-              obj[key] = string_make_patch(from[key], v);
+              if (typeof from[key] == "string" && typeof v == "string") {
+                // A valid string patch, converting one string to another.
+                obj[key] = string_make_patch(from[key], v);
+              } else {
+                /* This should be impossible, due to the type checking that
+                   I've added to set in this same commit.  However, it can't
+                   hurt to put in the above check on types, just in case.
+                     https://github.com/sagemathinc/cocalc/issues/3625
+                   A string col actually contains something that is not
+                   a string.  (Maybe it's due to loading something old?)
+                   In any case, it's better to "best effort" this, rather
+                   than to make the entire document be un-savable and
+                   un-usable to the user.
+                   We just give up and record no change in this case, so
+                   when doc is read in later (or by another user), there
+                   will be no weird corruption.
+                   This case will probably go away completely when all
+                   client code is written with proper typing.
+                */
+              }
             } else if (is_object(from[key]) && is_object(v)) {
               // Changing from one map to another, where they are not
               // equal -- can use a merge to make this more efficient.
@@ -394,23 +412,36 @@ export class DBDocument implements Document {
           // null = how to delete fields
           record = record.delete(field);
         } else {
-          if (this.string_cols.has(field) && is_array(value)) {
-            // special case: a string patch
-            record = record.set(
-              field,
-              string_apply_patch(value, before.get(field, ""))[0]
-            );
-          } else {
-            let new_val;
-            const cur = record.get(field);
-            const change = immutable.fromJS(value);
-            if (immutable.Map.isMap(cur) && immutable.Map.isMap(change)) {
-              new_val = merge_set(cur, change);
-            } else {
-              new_val = change;
+          if (this.string_cols.has(field)) {
+            if (is_array(value)) {
+              // special case: a string patch
+              record = record.set(
+                field,
+                string_apply_patch(value, before.get(field, ""))[0]
+              );
+              continue;
+            }
+            if (typeof value != "string") {
+              // Putting this guard in to address
+              // https://github.com/sagemathinc/cocalc/issues/3625
+              // which was caused by some client code setting a string_col
+              // to something that is not a string.  We'll next have
+              // to wait for this exception to be triggered someday...
+              throw Error(
+                `'${field}' must be a string, but tried to set to '${value}' of type ${typeof value}`
+              );
             }
-            record = record.set(field, new_val);
+            // falls through to actually set it below.
+          }
+          let new_val;
+          const cur = record.get(field);
+          const change = immutable.fromJS(value);
+          if (immutable.Map.isMap(cur) && immutable.Map.isMap(change)) {
+            new_val = merge_set(cur, change);
+          } else {
+            new_val = change;
           }
+          record = record.set(field, new_val);
         }
       }
 

src/smc-util/sync/editor/db/test/doc.test.ts

@@ -214,6 +214,22 @@ describe("test various types of patches", () => {
     const patch2 = doc.make_patch(doc2);
     expect(patch2).toEqual([1, [{ key: "cocalc", value: [1, "2", 5] }]]);
   });
+
+  it("tests that string column runtime type checking works", () => {
+    // This checks that https://github.com/sagemathinc/cocalc/issues/3625
+    // is fixed.
+    const doc = new DBDocument(
+      new Set(["key"]),
+      new Set(["value"]) /* so must always be a string */,
+      fromJS([{ key: "cocalc", value: "a string" }])
+    );
+
+    const doc2 = doc.set({ value: "foo" });
+    const x = doc2.get_one({})!;
+    expect(x.get("value")).toBe("foo");
+
+    expect(() => doc2.set({ value: 0 })).toThrow("must be a string");
+  });
 });
 
 describe("test conversion to and *from* strings", () => {

src/smc-webapp/jupyter/codemirror-static.tsx

@@ -116,7 +116,17 @@ export class CodeMirrorStatic extends Component<CodeMirrorStaticProps> {
       }
     };
 
-    CodeMirror.runMode(this.props.value, mode, append);
+    try {
+      CodeMirror.runMode(this.props.value, mode, append);
+    } catch(err) {
+      /* This does happen --
+            https://github.com/sagemathinc/cocalc/issues/3626
+         However, basically silently ignoring it (with a console.log)
+         is probably the best option for now (rather than figuring
+         out every possible bad input that could cause this), since
+         it completely crashes cocalc. */
+      console.log(`WARNING: CodeMirror.runMode failed -- ${err}`);
+    }
     line_numbers = false;
     append("\n"); // TODO: should this have 2 parameters?
 

src/smc-webapp/project/websocket/connect.ts

@@ -114,10 +114,10 @@ async function connection_to_project0(project_id: string): Promise<any> {
         //console.log("success!");
       }
     },
-    start_delay: 300,
-    max_delay: 3000,
-    factor: 1.2,
-    desc : 'connecting to project'
+    start_delay: 1000,
+    max_delay: 15000, // do not make too aggressive or it DDOS proxy server
+    factor: 1.3,
+    desc: "connecting to project"
     //log: (...x) => {
     //  console.log("retry primus:", ...x);
     //}
@@ -127,10 +127,10 @@ async function connection_to_project0(project_id: string): Promise<any> {
   // However, we don't want to overwrite the usual global window.Primus.
   const conn = (connections[project_id] = Primus.connect({
     reconnect: {
-      max: 5000,
-      min: 1000,
-      factor: 1.3,
-      retries: 1000
+      max: 30000,  // do not make too aggressive or it DDOS proxy server
+      min: 3000,
+      factor: 1.5,
+      retries: Infinity
     }
   }));
   conn.api = new API(conn);

src/smc-webapp/project/websocket/synctable.ts

@@ -166,7 +166,17 @@ class SyncTableChannel extends EventEmitter {
   private async clean_up_sockets(): Promise<void> {
     if (this.channel != null) {
       this.channel.removeAllListeners();
-      this.channel.end();
+      if (this.channel.conn.writable) {
+        /* The multiplex plugin should probably check that
+           conn is writable before trying to write, but it
+           doesn't.  So we only call end here if the conn
+           hasn't already closed itself.  Not doing this, causes a
+           stacktrace randomly, which isn't good.  (To reproduce,
+           open a file in a project, then close the project tab,
+           then re-open the project tab, and repeat a few times
+           until getting a stacktrace.) */
+        this.channel.end();
+      }
       delete this.channel;
     }
     if (this.websocket != null) {

src/smc-webapp/support.cjsx

@@ -394,19 +394,19 @@ SupportInfo = rclass
                     <a href="https://doc.cocalc.com" target="_blank" rel="noopener"><b>Looking for documentation and help?</b></a>
                 </li>
                 <li>
-                    <a target="_blank" rel="noopener" href="https://github.com/sagemathinc/cocalc/wiki/MySubscriptionDoesNotWork">Subscription does not work?</a>
+                    <a target="_blank" rel="noopener" href="https://doc.cocalc.com/subscriptions.html#what-if-your-subscription-does-not-seem-to-do-anything">Subscription does not work?</a>
                 </li>
                 <li>
-                    <a target="_blank" rel="noopener" href="https://github.com/sagemathinc/cocalc/wiki/DeleteProject">File or project seems gone?</a>
+                    <a target="_blank" rel="noopener" href="https://doc.cocalc.com/howto/missing-project.html">File or project seems gone?</a>
                 </li>
                 <li>
-                    <a target="_blank" rel="noopener" href="https://github.com/sagemathinc/cocalc/wiki/SageWorksheetWontRun">Sage worksheet or Jupyter notebook is slow or will not run?</a>
+                    <a target="_blank" rel="noopener" href="https://doc.cocalc.com/howto/slow-worksheet.html">Sage worksheet or Jupyter notebook is slow or will not run?</a>
                 </li>
                 <li>
-                    <a target="_blank" rel="noopener" href="https://github.com/sagemathinc/cocalc/wiki/KernelTerminated">Jupyter notebook keeps crashing with "Kernel terminated"?</a>
+                    <a target="_blank" rel="noopener" href="https://doc.cocalc.com/howto/jupyter-kernel-terminated.html">Jupyter notebook keeps crashing with "Kernel terminated"?</a>
                 </li>
                 <li>
-                    <a target="_blank" rel="noopener" href="https://github.com/sagemathinc/cocalc/wiki/SageQuestion">Questions about how to use Sage?</a>
+                    <a target="_blank" rel="noopener" href="https://doc.cocalc.com/howto/sage-question.html">Questions about how to use Sage?</a>
                 </li>
                 <li>
                     <b>Requesting that we install software?</b> Fill out the form below...