Merge branch 'rethinkdb' of https://github.com/sagemathinc/smc into r…

…ethinkdb

INSTALL.md

@@ -26,17 +26,7 @@ you do this and expose it to other users, see the *CRITICAL* db remark below.
 Configure a clean minimal Ubuntu 15.04 install (db0, db1, ...) with an assumed account "salvus" to run Rethinkdb as follows:
 
 	sudo su
-	apt-get update && apt-get upgrade && apt-get install libprotobuf9 python-pip
-    sudo pip install rethinkdb   # the python driver
-
-
-    # See https://github.com/rethinkdb/rethinkdb/releases for downloads
-
-    # For **testing** the auto-fail-over beta -- data format not compatible with stable use:
-	cd /tmp; wget http://download.rethinkdb.com/dev/2.1.0-0BETA2/rethinkdb_2.1.0%2b0BETA2~0vivid_amd64.deb && dpkg -i rethinkdb_2.1.0+0BETA2~0vivid_amd64.deb
-
-    # For stable use:
-    source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list && wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | apt-key add - && apt-get update && apt-get install rethinkdb
+	apt-get update && apt-get upgrade && apt-get install libprotobuf9 python-pip dstat iotop && pip install rethinkdb && source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list && wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | apt-key add - && apt-get update && apt-get install rethinkdb
 
     # Configure rethinkdb
     cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/default.conf

salvus/cassandra.coffee

@@ -3213,25 +3213,31 @@ class exports.Salvus extends exports.Cassandra
     r_central_log: (cb) =>
         require('rethink').rethinkdb cb:(err, db)=>
             table = db.table('central_log')
-            @dump_table
-                table   : 'central_log'
-                columns : ['time','event','value']
-                each    : (row, cb) ->
-                    row.value = misc.from_json(row.value)
-                    row.time = new Date(row.time)
-                    table.insert(row, conflict:"replace").run(cb)
-                cb      : cb
+            table.delete().run (err) =>  # drop table first, since r_file_access_log not idempotent
+                if err
+                    cb(err); return
+                @dump_table
+                    table   : 'central_log'
+                    columns : ['time','event','value']
+                    each    : (row, cb) ->
+                        row.value = misc.from_json(row.value)
+                        row.time = new Date(row.time)
+                        table.insert(row, conflict:"replace").run(cb)
+                    cb      : cb
 
     r_client_error_log: (cb) =>
         require('rethink').rethinkdb cb:(err, db)=>
             table = db.table('client_error_log')
-            @dump_table
-                table   : 'client_error_log'
-                columns : ['timestamp', 'account_id', 'type', 'error']
-                json    : ['error']
-                each    : (row, cb) ->
-                    table.insert({timestamp:new Date(row.timestamp), account_id:row.account_id, event:row.type, error:row.error}, conflict:"replace").run(cb)
-                cb      : cb
+            table.delete().run (err) =>  # drop table first, since r_file_access_log not idempotent
+                if err
+                    cb(err); return
+                @dump_table
+                    table   : 'client_error_log'
+                    columns : ['timestamp', 'account_id', 'type', 'error']
+                    json    : ['error']
+                    each    : (row, cb) ->
+                        table.insert({timestamp:new Date(row.timestamp), account_id:row.account_id, event:row.type, error:row.error}, conflict:"replace").run(cb)
+                    cb      : cb
 
     r_compute_servers: (cb) =>
         require('rethink').rethinkdb cb:(err, db)=>

salvus/compute.coffee

@@ -24,14 +24,9 @@ EXPERIMENTAL = false
 
 ###
 
-Development testing:
+require('compute').compute_server(db_hosts:['db0'], cb:(e,s)->console.log(e);global.s=s)
 
-id='e7a8a705-1c40-4397-836a-b60e259e1137'; x={};require('compute').compute_server(keyspace:'devel',cb:(e,s)->console.log(e);x.s=s;x.s.project(project_id:id,cb:(e,p)->console.log(e);x.p=p))
-
-Live use
-
-
-id='e7a8a705-1c40-4397-836a-b60e259e1137';  x={};require('compute').compute_server(db_hosts:['smc0-us-central1-c'],cb:(e,s)->console.log(e);x.s=s;x.s.project(project_id:id,cb:(e,p)->console.log(e);x.p=p))
+s.project(project_id:'eb5c61ae-b37c-411f-9509-10adb51eb90b',cb:(e,p)->global.p=p;console.log(e))
 
 ###
 
@@ -678,13 +673,32 @@ class ProjectClient extends EventEmitter
                         else
                             if x
                                 {host, assigned} = x
-                            if host?
+                            if host   # important: DO NOT just do "host?", since host='' is in the database for older projects!
                                 dbg("got host='#{host}' that was assigned #{assigned}")
                             else
                                 dbg("no host assigned")
                             cb()
             (cb) =>
-                if host?
+                if host
+                    # The host might no longer be defined at all, so we should check this here.
+                    dbg("make sure the host still exists")
+                    @compute_server.database.get_compute_server
+                        host : host
+                        cb   : (err, x) =>
+                            if err
+                                cb(err)
+                            else
+                                if not x
+                                    # The compute server doesn't exist anymore.  Forget our useless host
+                                    # assignment and get a new host below.
+                                    host = undefined
+                                cb()
+                else
+                    cb()
+
+
+            (cb) =>
+                if host
                     cb()
                 else
                     dbg("assigning some host")
@@ -705,10 +719,10 @@ class ProjectClient extends EventEmitter
             if not err
                 @_set_host(host)
                 @assigned = assigned  # when host was assigned
-                dbg("henceforth using host=#{@host} that was assigned #{@assigned}")
+                dbg("henceforth using host='#{@host}' that was assigned #{@assigned}")
                 if host != previous_host
                     @clear_state()
-                    dbg("HOST CHANGE: #{previous_host} --> #{host}")
+                    dbg("HOST CHANGE: '#{previous_host}' --> '#{host}'")
             dbg("time=#{misc.mswalltime(t)}ms")
             opts.cb?(err, host)
         )
@@ -1159,7 +1173,7 @@ class ProjectClient extends EventEmitter
                 ], cb)
             (cb) =>
                 dbg("update database with new project location")
-                @set_project_host
+                @compute_server.database.set_project_host
                     project_id : @project_id
                     host       : opts.target
                     cb         : (err, assigned) =>

salvus/conf/deploy_smc/hosts

@@ -18,17 +18,14 @@ compute2-us us-central1-c gce compute2-us    compute
 compute3-us us-central1-c gce compute3-us    compute
 compute4-us us-central1-c gce compute4-us    compute
 
-# SMC nodes in the US:
+# web server nodes in the US:
 
-smc0-us-central1-c dc0 us-central1-c cassandra hub gce nginx edge smc0-us-central1-c
-smc1-us-central1-c dc0 us-central1-c cassandra hub gce nginx edge smc1-us-central1-c
-smc2-us-central1-c dc0 us-central1-c cassandra hub gce nginx edge smc2-us-central1-c
-
-
-# SMC nodes in Europe:
-#smc0-europe-west1-d dc1 europe-west1-d cassandra hub gce nginx edge smc0-europe-west1-d
-#smc1-europe-west1-d dc1 europe-west1-d cassandra hub gce nginx edge smc1-europe-west1-d
-#smc2-europe-west1-d dc1 europe-west1-d cassandra hub gce nginx edge smc2-europe-west1-d
+web0 gce hub nginx haproxy web
+web1 gce hub nginx haproxy web
+#web2 gce hub  web
+#web3 gce hub web
 
+# database nodes
 
+db-stable database
 

salvus/conf/deploy_smc/services

@@ -1,32 +1 @@
 # services
-
-# Google Compute Engine
-[vmgce]
-
-# Cassandra nodes
-[cassandra] {'cluster_name':'salvus', 'path':'/mnt/cassandra',  'thrift_framed_transport_size_in_mb':'99',  'authenticator':'org.apache.cassandra.auth.PasswordAuthenticator', 'authorizer':'org.apache.cassandra.auth.CassandraAuthorizer', 'memtable_flush_writers':2, 'memtable_allocation_type':'offheap_objects', 'concurrent_compactors':'1'}
-
-# us-central1-c
-smc0-us-central1-c {'seed':True,   'topology':'us-central1:c', 'num_tokens':256}
-smc1-us-central1-c {'seed':False,  'topology':'us-central1:c', 'num_tokens':256}
-smc2-us-central1-c {'seed':True,   'topology':'us-central1:c', 'num_tokens':256}
-
-# us-central1-c
-#smc0-europe-west1-d {'seed':True,   'topology':'europe-west1:d', 'num_tokens':256} #,  'auto_bootstrap': False}
-#smc1-europe-west1-d {'seed':False,  'topology':'europe-west1:d', 'num_tokens':256} #,  'auto_bootstrap': False}
-#smc2-europe-west1-d {'seed':True,   'topology':'europe-west1:d', 'num_tokens':256} #,  'auto_bootstrap': False}
-
-[stunnel] {'sudo':True}
-edge
-
-[haproxy] {'sudo':True, 'insecure_redirect_port':80, 'sitename':'cloud.sagemath.com'}
-edge
-
-[nginx]
-nginx
-
-[hub]
-hub {'id':'0', 'port':'5000', 'proxy_port':'5001'}
-#hub {'id':'1', 'port':'5002', 'proxy_port':'5003'}
-#hub {'id':'2', 'port':'5004', 'proxy_port':'5005'}
-#hub {'id':'3', 'port':'5006', 'proxy_port':'5007'}

salvus/page/editor.coffee

@@ -1074,7 +1074,7 @@ class exports.Editor
         else
             @project_page.init_sortable_file_list()
             n = x.length
-            width = Math.min(250, parseInt((x[0].parent().width() - 25) / n + 2)) # floor to prevent rounding problems
+            width = Math.min(250, parseInt((x[0].parent().width() - 40) / n + 2)) # floor to prevent rounding problems
             if width < 0
                 width = 0
 

salvus/page/project.coffee

@@ -376,20 +376,31 @@ class ProjectPage
         @container.find(".project-editor").append(@editor.element)
 
     display_tab: (name) =>
+        if @_last_display_tab_name == name
+            # tab already displayed
+            return
         @container.find(".project-pages").children().removeClass('active')
         @container.find(".file-pages").children().removeClass('active')
         @container.css(position: 'static')
+
+        # hide the currently open tab
+        for tab in @tabs
+            if tab.name == @_last_display_tab_name
+                tab.onblur?()
+                tab.target.hide()
+                break
+        @_last_display_tab_name = name
+        # show the tab we are opening
         for tab in @tabs
             if tab.name == name
                 @current_tab = tab
                 tab.target.show()
                 tab.label.addClass('active')
                 tab.onshow?()
                 @focus()
-            else if tab.name == @_last_display_tab_name
-                tab.onblur?()
-                tab.target.hide()
-        @_last_display_tab_name = name
+                break
+        # fix the size of the tabs at the top
+        @editor?.resize_open_file_tabs()
 
         if name == 'project-new-file'
             @actions.set_next_default_filename(require('account').default_filename())
@@ -399,9 +410,6 @@ class ProjectPage
             sort_by_time = @store.state.sort_by_time ? true
             show_hidden = @store.state.show_hidden ? false
             @actions.set_directory_files(@store.state.current_path, sort_by_time, show_hidden)
-        if name != 'project-editor'
-            @editor?.hide()
-            @editor?.resize_open_file_tabs()
 
     show_editor_chat_window: (path) =>
         @editor?.show_chat_window(path)

salvus/rethink.coffee

@@ -54,9 +54,7 @@ To automate sharding/replication of all the tables (depends on deployment).  E.g
 if there are 3 nodes, do this to reconfigure *all* tables:
 
     db = require('rethink').rethinkdb(hosts:['db0'])
-	db.db.reconfigure(replicas:3, shards:1).run(console.log)
-
-# CRITICAL: Right now do not shard since that breaks raft autofailover completely.
+	db.db.reconfigure(replicas:3, shards:3).run(console.log)
 
 ###
 
@@ -253,8 +251,8 @@ class RethinkDB
                     num_nodes = x; cb(err)
             (cb) =>
                 if num_nodes > 1
-                    dbg("ensuring there are #{num_nodes} replicas and 1 shard of every table")
-                    @db.reconfigure(replicas:num_nodes, shards:1).run(cb)
+                    dbg("ensuring there are #{num_nodes} replicas and #{num_nodes} shard of every table")
+                    @db.reconfigure(replicas:num_nodes, shards:num_nodes).run(cb)
                 else
                     dbg("single-node server, so not changing replicas")
                     cb()

salvus/scripts/check_hub

@@ -1,8 +1,8 @@
 echo "loadavg          `cat /proc/loadavg | awk '{ print $1, $2, $3 }'`"
 
-export P=`pidof node`
+export P=`pidof nodejs`
 echo "etime            `ps -o etime --pid=$P | tail -1 | tr -d ' '`"
-echo "sign_in_timeouts `tail -n 2000 ~/salvus/salvus/data/logs/hub*.log |grep \"There was an error signing you in (Timeout after 30 seconds).\" |wc -l`"
+echo "sign_in_timeouts `tail -n 2000 ~/salvus/salvus/data/logs/hub*.log |grep \"There was an error signing you\" |wc -l`"
 
 check_hub_logs  5000
 

salvus/scripts/gce/gce.py

@@ -219,6 +219,8 @@ def create_boot_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
             '--zone', zone], system=True)
 
     def create_all_boot_snapshots(self):
+        log("snapshotting db-stable server boot image")
+        self.create_boot_snapshot(node='', prefix='db-stable', zone='us-central1-c', devel=False)
         log("snapshotting dev boot images")
         for name in self.dev_instances():
             i = name.rfind('-')
@@ -230,8 +232,6 @@ def create_all_boot_snapshots(self):
         self.create_boot_snapshot(node=0, prefix='backup', zone='us-central1-c', devel=False)
         log("snapshotting admin boot image")
         self.create_boot_snapshot(node='',prefix='admin', zone='us-central1-c', devel=False)
-        log("snapshotting SMC server boot image")
-        self.create_boot_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
         log("snapshotting compute machine boot image")
         self.create_boot_snapshot(node=0, prefix='compute', zone='us-central1-c', devel=False)
 

salvus/scripts/move_project

@@ -1,2 +1,2 @@
 cd $SALVUS_ROOT
-echo "id='$1';  x={};require('compute').compute_server(db_hosts:['smc0-us-central1-c'],cb:(e,s)->console.log(e);x.s=s;x.s.project(project_id:id,cb:(e,p)->console.log(e);x.p=p;x.p.move(target:'$2',cb:(e)->console.log('DONE',e))))" | coffee
+echo "id='$1';  require('compute').compute_server(db_hosts:['db0'],cb:(e,s)->console.log(e);s.project(project_id:id,cb:(e,p)->console.log(e);p.move(target:'$2',cb:(e)->console.log('DONE',e);process.exit())))" | coffee