Merge branch 'starwisp_lib' of github.com:nebogeo/mongoose-web into s…

…tarwisp_lib

eavdb/eavdb.ss

@@ -33,38 +33,44 @@
 
 (provide (all-defined-out))
 
-(msg "hello from eavdb.ss")
-
 
 (define (upgrade-table db name)
-  (db-exec db (string-append "alter table " name " add version integer")))
-
+  (db-exec db (string-append "alter table " name " add version integer"))
+  (db-exec db (string-append "alter table " name " add sent integer default 0")))
 
 ;; create eav tables (add types as required)
+;; aggregating version updates - should clean all this up
+;;
+;; remember that this is run on both pi and android - on the tablets we don't really know
+;; how old the initial versions of the databases are, they might go right back to the beginning
+;; so need to be super conservative and alter tables to add new stuff
 (define (setup db table)
   (msg "db setup")
-  (db-exec db (string-append "create table " table "_entity ( entity_id integer primary key autoincrement, entity_type varchar(256), unique_id varchar(256), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_entity ( entity_id integer primary key autoincrement, entity_type varchar(256), unique_id varchar(256), dirty integer, version integer, sent integer default 0)"))
+  ;; throws benign errors if it exists - we catch and ignore em...
+  (db-exec db (string-append "alter table " table "_entity add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_entity on " table "_entity (unique_id)"))
 
 
-  (db-exec db (string-append "create table " table "_attribute ( id integer primary key autoincrement, attribute_id varchar(256), entity_type varchar(256), attribute_type varchar(256))"))
+  (db-exec db (string-append "create table " table "_attribute ( id integer primary key autoincrement, attribute_id varchar(256), entity_type varchar(256), attribute_type varchar(256), sent integer default 0)"))
+  (db-exec db (string-append "alter table " table "_attribute add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_attribute on " table "_attribute (entity_type)"))
 
 
 
-  (db-exec db (string-append "create table " table "_value_varchar ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_varchar ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_varchar"))
   (db-exec db (string-append "create index if not exists index_" table "_value_varchar on " table "_value_varchar (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_int ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value integer, dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_int ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value integer, dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_int"))
   (db-exec db (string-append "create index if not exists index_" table "_value_int on " table "_value_int (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_real ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value real, dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_real ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value real, dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_real"))
   (db-exec db (string-append "create index if not exists index_" table "_value_real on " table "_value_real (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_file ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_file ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_file"))
   (db-exec db (string-append "create index if not exists index_" table "_value_file on " table "_value_file (entity_id,attribute_id)"))
 
@@ -115,3 +121,30 @@
    (lambda (i)
      (get-entity-only db table i kt-list))
    (filter-entities-inc-deleted db table type filter)))
+
+(define (run-unit-tests)
+  (msg "running eavdb tests...")
+  (define last-test-db (open (string->path "unit-test.db")))
+  ;; clear out last test
+  (db-exec last-test-db "drop table sync_entity;")
+  (db-exec last-test-db "drop table sync_attribute;")
+  (db-exec last-test-db "drop table sync_value_varchar;")
+  (db-exec last-test-db "drop table sync_value_int;")
+  (db-exec last-test-db "drop table sync_value_real;")
+  (db-exec last-test-db "drop table sync_value_file;")
+
+  (db-exec last-test-db "drop table stream_entity;")
+  (db-exec last-test-db "drop table stream_attribute;")
+  (db-exec last-test-db "drop table stream_value_varchar;")
+  (db-exec last-test-db "drop table stream_value_int;")
+  (db-exec last-test-db "drop table stream_value_real;")
+  (db-exec last-test-db "drop table stream_value_file;")
+  ;; reopen to run setup
+  (define test-db (db-open "unit-test.db" setup))
+  (ktv-test)
+  (entity-update-test test-db "sync")
+  (entity-sync-test test-db "sync")
+  (msg "finished running eavdb tests..."))
+
+(run-unit-tests)
+

eavdb/entity-insert.ss

@@ -54,7 +54,7 @@
   (db-exec db "begin transaction")
   (let ((id (db-insert
              db (string-append
-                 "insert into " table "_entity values (null, ?, ?, ?, ?)")
+                 "insert into " table "_entity (entity_id, entity_type, unique_id, dirty, version) values (null, ?, ?, ?, ?)")
              entity-type unique-id dirty version)))
 
     ;; create the attributes if they are new, and validate them if they exist
@@ -78,7 +78,7 @@
   (db-exec db "begin transaction")
   (let ((id (db-insert
              db (string-append
-                 "insert into " table "_entity values (?, ?, ?, ?, ?)")
+                 "insert into " table "_entity (entity_id, entity_type, unique_id, dirty, version) values (?, ?, ?, ?, ?)")
              id entity-type unique-id dirty version)))
 
     ;; create the attributes if they are new, and validate them if they exist

eavdb/entity-update.ss

@@ -28,16 +28,17 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; updating data
 
+;; setting sent=0 to update rabbitmq - probably not needed?
 (define (update-entity-changed db table entity-id)
   (db-exec
    db (string-append
-       "update " table "_entity set dirty=?, version=version+1 where entity_id = ?")
+       "update " table "_entity set dirty=?, sent=0, version=version+1 where entity_id = ?")
    1 entity-id))
 
 (define (update-entity-version db table entity-id version)
   (db-exec
    db (string-append
-       "update " table "_entity set dirty=0, version=? where entity_id = ?")
+       "update " table "_entity set dirty=0, sent=0, version=? where entity_id = ?")
    version entity-id))
 
 ;; update an entire entity (version incl), via a (possibly partial) list of key/value pairs

eavdb/entity-values.ss

@@ -41,7 +41,7 @@
       ((null? t)
        (msg "adding new attribute for" entity-type " called " key " of type " type)
        (db-insert
-        db (string-append "insert into " table "_attribute values (null, ?, ?, ?)")
+        db (string-append "insert into " table "_attribute (id, attribute_id, entity_type, attribute_type) values (null, ?, ?, ?)")
         key entity-type type)
        type)
       (else
@@ -58,11 +58,12 @@
 (define (insert-value db table entity-id ktv dirty)
   ;; use type to dispatch insert to correct value table
   (db-insert db (string-append "insert into " table "_value_" (ktv-type ktv)
-                               " values (null, ?, ?, ?, ?, 0)")
+                               "(id, entity_id, attribute_id, value, dirty, version) values (null, ?, ?, ?, ?, 0)")
              entity-id (ktv-key ktv) (ktv-value ktv) (if dirty 1 0)))
 
 ;; update the value given an entity type, a attribute type and it's key (= attriute_id)
 ;; creates the value if it doesn't already exist, updates it otherwise if it's different
+;; setting sent=0 to work with the rabbitmq updater here - probably not needed
 (define (update-value db table entity-id ktv)
   (let ((s (select-first
             db (string-append
@@ -74,11 +75,12 @@
         (if (not (ktv-eq? ktv (list (ktv-key ktv) (ktv-type ktv) s)))
             (db-exec
              db (string-append "update " table "_value_" (ktv-type ktv)
-                               " set value=?, dirty=1  where entity_id = ? and attribute_id = ?")
+                               " set value=?, dirty=1, sent=0  where entity_id = ? and attribute_id = ?")
              (ktv-value ktv) entity-id (ktv-key ktv))
             '())))) ;;(msg "values for" (ktv-key ktv) "are the same (" (ktv-value ktv) "==" s ")")))))
 
 ;; don't make dirty or update version here
+;; setting sent=0 to work with the rabbitmq updater here
 (define (update-value-from-sync db table entity-id ktv)
   (let ((s (select-first
             db (string-append
@@ -91,7 +93,7 @@
         (insert-value db table entity-id ktv #t) ;; <- don't make dirty!?
         (db-exec
          db (string-append "update " table "_value_" (ktv-type ktv)
-                           " set value=?, dirty=0 where entity_id = ? and attribute_id = ?")
+                           " set value=?, dirty=0, sent=0 where entity_id = ? and attribute_id = ?")
          (ktv-value ktv) entity-id (ktv-key ktv)))))
 
 ;; get all the (current) attributes for an entity type

pi-admin/battery_detect.py

@@ -0,0 +1,12 @@
+import smbus
+import os
+
+bus = smbus.SMBus(1)
+
+# if we are on battery power
+if bus.read_byte_data(0x69,0)==2:
+    # "unconditional file safe shutdown and power OFF when battery powered"
+    bus.write_byte_data(0x6b,0x00,0xcc)
+
+    # shut down nicely
+    #os.system("sudo halt")

pi-admin/create_tunnel.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+create_ssh_tunnel() {
+    /usr/bin/ssh -o "ExitOnForwardFailure yes" -N -R 2222:localhost:22 mongoose@be.fo.am &
+    if [[ $? -eq 0 ]]; then
+	echo SSH tunnel created successfully
+    else
+	echo An error occurred creating SSH tunnel. RC was $?
+    fi
+}
+create_rabbit_tunnel() {
+    /usr/bin/ssh -o "ExitOnForwardFailure yes" -N -R 5672:localhost:5672 mongoose@be.fo.am &
+    if [[ $? -eq 0 ]]; then
+	echo Rabbit tunnel created successfully
+    else
+	echo An error occurred creating a RabbitMQ tunnel. RC was $?
+    fi
+}
+
+/bin/pidof ssh
+if [[ $? -ne 0 ]]; then
+    echo Creating new ssh tunnel connection
+    create_ssh_tunnel
+    echo Creating new RabbitMQ tunnel connection
+    create_rabbit_tunnel
+fi

pi-admin/crontab

@@ -0,0 +1,34 @@
+# Edit this file to introduce tasks to be run by cron.
+# 
+# Each task to run has to be defined through a single line
+# indicating with different fields when the task will be run
+# and what command to run for the task
+# 
+# To define the time you can provide concrete values for
+# minute (m), hour (h), day of month (dom), month (mon),
+# and day of week (dow) or use '*' in these fields (for 'any').# 
+# Notice that tasks will be started based on the cron's system
+# daemon's notion of time and timezones.
+# 
+# Output of the crontab jobs (including errors) is sent through
+# email to the user the crontab file belongs to (unless redirected).
+# 
+# For example, you can run a backup of all your user accounts
+# at 5 a.m every week with:
+# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
+# 
+# For more information see the manual pages of crontab(5) and cron(8)
+# 
+# m h  dom mon dow   command
+
+@reboot supervise /home/pi/mongoose-2000/web
+
+# adding path so we can find dotnet etc
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+@reboot ~/datapipe.sh
+*/1 * * * * ~/create_tunnel.sh > tunnel.log 2>&1
+*/1 * * * * python ~/battery_detect.py
+
+@reboot sleep 15 && ~/start_adhoc.sh >> hostapd.log 2>&1
+@reboot sleep 30 && ~/start_adhoc.sh >> hostapd.log 2>&1

pi-admin/datapipe.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd mongoose-2000/dotnet/datpipe/
+dotnet ./DataPipe.Main.dll

pi-admin/interfaces

@@ -0,0 +1,23 @@
+# interfaces(5) file used by ifup(8) and ifdown(8)
+
+# Please note that this file is written to be used with dhcpcd
+# For static IP, consult /etc/dhcpcd.conf and 'man dhcpcd.conf'
+
+# Include files from /etc/network/interfaces.d:
+source-directory /etc/network/interfaces.d
+
+auto lo
+iface lo inet loopback
+
+iface eth0 inet manual
+
+allow-hotplug wlan0
+iface wlan0 inet static
+	address 192.168.2.1
+	netmask 255.255.255.0
+	network 192.168.2.0
+	broadcast 192.168.1.255
+
+allow-hotplug wlan1
+iface wlan1 inet manual
+    wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf

pi-admin/start_adhoc.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+sudo ifconfig wlan0 up
+sleep 5
+sudo /usr/sbin/hostapd /etc/hostapd/hostapd.conf &
+sleep 5
+sudo ifconfig wlan0 up
+sleep 5
+sudo /usr/sbin/hostapd /etc/hostapd/hostapd.conf &
+

web/run

@@ -1,4 +1,4 @@
 #!/bin/bash
-./server.scm 8888 >>client/htdocs/log.txt 2>&1
+./server.scm 8888 >client/htdocs/log.txt 2>&1
 
 

web/scripts/sql.ss

@@ -34,12 +34,13 @@
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; fix this mess
 (define (db-open db-name setup-fn)
   (cond
     ((file-exists? (string->path db-name))
      (display "open existing db")(newline)
      (let ((db (open (string->path db-name))))
-       ;; upgrade...
+       ;; upgrade
        (setup-fn db "sync")
        (setup-fn db "stream")
        db))

web/server.scm

@@ -198,7 +198,7 @@
  start
  ;; port number is read from command line as argument
  ;; ie: ./server.scm 8080
-;; #:listen-ip "192.168.2.1"
+ #:listen-ip "192.168.2.1"
  #:port (string->number (command-line #:args (port) port))
  #:command-line? #t
  #:servlet-path "/mongoose"