Merge remote-tracking branch 'origin/master' into jupyter-explicit-ke…

…rnel

src/scripts/zvol_storage.py

@@ -73,12 +73,12 @@
 
 SSH_ACCESS_PUBLIC_KEY = "/home/salvus/salvus/salvus/scripts/skel/.ssh/authorized_keys2"
 
-import argparse, hashlib, math, os, random, shutil, string, sys, time, uuid, json, signal
+import argparse, hashlib, math, os, shutil, sys, time, uuid, json, signal
 from subprocess import Popen, PIPE
 
 
 def print_json(s):
-    print json.dumps(s, separators=(',', ':'))
+    print(json.dumps(s, separators=(',', ':')))
 
 
 def uid(project_id):
@@ -227,8 +227,11 @@ def __repr__(self):
         return "Stream(%s): %s to %s stored in %s" % (
             self.project.project_id, self.start, self.end, self.path)
 
-    def __cmp__(self, other):
-        return cmp((self.end, self.start), (other.end, other.start))
+    def __eq__(self, other):
+        return (self.end, self.start) == (other.end, other.start)
+
+    def __ne__(self, other):
+        return not(self  == other)
 
     def size_mb(self):
         return int(os.path.getsize(self.path) / 1e6)
@@ -343,7 +346,6 @@ def create_user(self):
             ignore_errors=True)
 
     def delete_user(self):
-        u = self.uid
         cmd('sudo /usr/sbin/userdel %s; sudo /usr/sbin/groupdel %s' %
             (self.username, self.username),
             ignore_errors=True)

src/smc-hub/client.coffee

@@ -33,6 +33,8 @@ db_schema            = require('smc-util/db-schema')
 
 underscore = require('underscore')
 
+{callback} = require('awaiting')
+
 DEBUG2 = !!process.env.SMC_DEBUG2
 
 REQUIRE_ACCOUNT_TO_EXECUTE_CODE = false
@@ -2603,6 +2605,25 @@ class exports.Client extends EventEmitter
                 else
                     cb()
 
+    _check_syncdoc_access: (string_id, cb) =>
+        if not @account_id?
+            cb('you must be signed in to access syncdoc')
+            return
+        if not typeof string_id == 'string' and string_id.length == 40
+            cb('string_id must be specified and valid')
+            return
+        @database._query
+            query : "SELECT project_id FROM syncstrings"
+            where : {"string_id = $::CHAR(40)" : string_id}
+            cb    : (err, results) =>
+                if err
+                    cb(err)
+                else if results.rows.length != 1
+                    cb("no such syncdoc")
+                else
+                    project_id = results.rows[0].project_id
+                    @_check_project_access(project_id, cb)
+
     mesg_disconnect_from_project: (mesg) =>
         dbg = @dbg('mesg_disconnect_from_project')
         @_check_project_access mesg.project_id, (err) =>
@@ -2637,4 +2658,19 @@ class exports.Client extends EventEmitter
                 @error_to_client(id:mesg.id, error:"unable to touch project #{mesg.project_id} -- #{err}")
             else
                 @push_to_client(message.success(id:mesg.id))
-        )
+        )
+
+    mesg_get_syncdoc_history: (mesg) =>
+        dbg = @dbg('mesg_syncdoc_history')
+        try
+            dbg("checking conditions")
+            # this raises an error if user does not have access
+            await callback(@_check_syncdoc_access, mesg.string_id)
+            # get the history
+            history = await @database.syncdoc_history_async(mesg.string_id, mesg.patches)
+            dbg("success!")
+            @push_to_client(message.syncdoc_history(id:mesg.id, history:history))
+        catch err
+            dbg("failed -- #{err}")
+            @error_to_client(id:mesg.id, error:"unable to get syncdoc history for string_id #{mesg.string_id} -- #{err}")
+

src/smc-hub/local_hub_connection.coffee

@@ -341,6 +341,31 @@ class LocalHub # use the function "new_local_hub" above; do not construct this d
                     cb()
         async.map(misc.keys(v), f, (err) => cb?(err))
 
+    # async -- throws error if project doesn't have access to string with this id.
+    check_syncdoc_access: (string_id) =>
+        if not typeof string_id == 'string' and string_id.length == 40
+            throw Error('string_id must be specified and valid')
+            return
+        opts =
+            query : "SELECT project_id FROM syncstrings"
+            where : {"string_id = $::CHAR(40)" : string_id}
+        results = await callback2(@database._query, opts)
+        if results.rows.length != 1
+            throw Error("no such syncdoc")
+        if results.rows[0].project_id != @project_id
+            throw Error("project does NOT have access to this syncdoc")
+        return  # everything is fine.
+
+    mesg_get_syncdoc_history: (mesg, write_mesg) =>
+        try
+            # this raises an error if user does not have access
+            await @check_syncdoc_access(mesg.string_id)
+            # get the history
+            history = await @database.syncdoc_history_async(mesg.string_id, mesg.patches)
+            write_mesg(message.syncdoc_history(id:mesg.id, history:history))
+        catch err
+            write_mesg(message.error(id:mesg.id, error:"unable to get syncdoc history for string_id #{mesg.string_id} -- #{err}"))
+
     #
     # end project query support code
     #
@@ -421,6 +446,8 @@ class LocalHub # use the function "new_local_hub" above; do not construct this d
                         @mesg_query(mesg, write_mesg)
                     when 'query_cancel'
                         @mesg_query_cancel(mesg, write_mesg)
+                    when 'get_syncdoc_history'
+                        @mesg_get_syncdoc_history(mesg, write_mesg)
                     when 'file_written_to_project'
                         # ignore -- don't care; this is going away
                         return

src/smc-hub/postgres-server-queries.coffee

@@ -26,6 +26,8 @@ PROJECT_GROUPS = misc.PROJECT_GROUPS
 
 {PROJECT_COLUMNS, one_result, all_results, count_result, expire_time} = require('./postgres-base')
 
+{syncdoc_history} = require('./postgres/syncdoc-history')
+
 exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
     # write an event to the central_log table
     log: (opts) =>
@@ -2737,3 +2739,16 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                         where : locals.where
                         cb    : cb
         ], opts.cb)
+
+    syncdoc_history: (opts) =>
+        opts = defaults opts,
+            string_id : required
+            patches   : false      # if true, include actual patches
+            cb        : required
+        try
+            opts.cb(undefined, await syncdoc_history(@, opts.string_id, opts.patches))
+        catch err
+            opts.cb(err)
+
+    syncdoc_history_async : (string_id, patches) =>
+        return await syncdoc_history(@, string_id, patches)

src/smc-hub/postgres/syncdoc-history.ts

@@ -0,0 +1,94 @@
+import { callback2 } from "smc-util/async-utils";
+
+import { trunc } from "smc-util/misc2";
+
+import { PostgreSQL } from "./types";
+
+export interface Patch {
+  time_utc: Date;
+  patch_length?: number;
+  patch?: string;
+  user?: string;
+  account_id?: string;
+  format?: number;
+  snapshot?: string;
+}
+
+type User = { account_id: string; user: string };
+
+async function get_users(db: PostgreSQL, where): Promise<User[]> {
+  const query = "SELECT project_id, users FROM syncstrings";
+  // get the user_id --> account_id map
+  const results = await callback2(db._query, { query, where });
+  if (results.rows.length != 1) {
+    throw Error("no such syncstring");
+  }
+  const account_ids: string[] = results.rows[0].users
+    ? results.rows[0].users
+    : [];   // syncdoc exists, but not used yet.
+  const project_id: string = results.rows[0].project_id;
+  const project_title: string = trunc(
+    (await callback2(db.get_project, {
+      columns: ["title"],
+      project_id
+    })).title,
+    80
+  );
+
+  // get the names of the users
+  const names = await callback2(db.account_ids_to_usernames, { account_ids });
+  const users: User[] = [];
+  for (let account_id of account_ids) {
+    if (account_id == project_id) {
+      users.push({ account_id, user: `Project: ${project_title}` });
+      continue;
+    }
+    const name = names[account_id];
+    if (name == null) continue;
+    const user = trunc(`${name.first_name} ${name.last_name}`, 80);
+    users.push({ account_id, user });
+  }
+  return users;
+}
+
+export async function syncdoc_history(
+  db: PostgreSQL,
+  string_id: string,
+  include_patches: boolean = false
+): Promise<Patch[]> {
+  const where = { "string_id = $::CHAR(40)": string_id };
+  const users: User[] = await get_users(db, where);
+
+  const order_by = "time";
+  let query: string;
+  if (include_patches) {
+    query = "SELECT time, user_id, format, patch, snapshot FROM patches";
+  } else {
+    query =
+      "SELECT time, user_id, format, length(patch) as patch_length FROM patches";
+  }
+  const results = await callback2(db._query, { query, where, order_by });
+  const patches: Patch[] = [];
+  function format_patch(row): Patch {
+    const patch: Patch = { time_utc: row.time, format: row.format };
+    const u = users[row.user_id];
+    if (u != null) {
+      for (let k in u) {
+        patch[k] = u[k];
+      }
+    }
+    if (include_patches) {
+      patch.patch = row.patch;
+      if (row.snapshot != null) {
+        patch.snapshot = row.snapshot;
+      }
+    } else {
+      patch.patch_length = row.patch_length;
+    }
+    return patch;
+  }
+  for (let row of results.rows) {
+    patches.push(format_patch(row));
+  }
+  return patches;
+}

src/smc-hub/postgres/types.ts

@@ -43,4 +43,6 @@ export interface PostgreSQL extends EventEmitter {
     cb: Function
   ): void;
   changefeed(opts: ChangefeedOptions): Changes;
+  account_ids_to_usernames(opts: { account_ids: string[]; cb: Function });
+  get_project(opts: { project_id: string; columns?: string[]; cb: Function });
 }

src/smc-project/client.coffee

@@ -24,7 +24,7 @@ fs     = require('fs')
 
 {EventEmitter} = require('events')
 
-{once} = require("smc-util/async-utils");
+{callback2, once} = require("smc-util/async-utils");
 
 async   = require('async')
 winston = require('winston')
@@ -583,4 +583,13 @@ class exports.Client extends EventEmitter
 
     # no-op
     touch_project: (opts) =>
-        opts.cb?()
+        opts.cb?()
+
+    # async
+    get_syncdoc_history: (string_id, patches=false) =>
+        dbg = @dbg("get_syncdoc_history")
+        dbg(string_id, patches)
+        mesg = message.get_syncdoc_history
+            string_id : string_id
+            patches   : patches
+        return await callback2(@call, {message:mesg})