From 2632a0cab494a7ccce327084154ace73a6864282 Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Tue, 5 Sep 2017 14:47:04 +0000
Subject: [PATCH 1/6] changes for new column

---
 eavdb/eavdb.ss         | 5 +++--
 eavdb/entity-insert.ss | 4 ++--
 eavdb/entity-values.ss | 4 ++--
 web/run                | 2 +-
 web/scripts/sql.ss     | 3 ++-
 web/server.scm         | 2 +-
 6 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/eavdb/eavdb.ss b/eavdb/eavdb.ss
index 2defa02..72a2131 100644
--- a/eavdb/eavdb.ss
+++ b/eavdb/eavdb.ss
@@ -37,8 +37,9 @@
 
 
 (define (upgrade-table db name)
-  (db-exec db (string-append "alter table " name " add version integer")))
-
+  (db-exec db (string-append "alter table " name " add version integer"))
+  (db-exec db (string-append "alter table " name " add sent integer default 0"))
+  )
 
 ;; create eav tables (add types as required)
 (define (setup db table)
diff --git a/eavdb/entity-insert.ss b/eavdb/entity-insert.ss
index 0a508a7..5affaf0 100644
--- a/eavdb/entity-insert.ss
+++ b/eavdb/entity-insert.ss
@@ -54,7 +54,7 @@
   (db-exec db "begin transaction")
   (let ((id (db-insert
              db (string-append
-                 "insert into " table "_entity values (null, ?, ?, ?, ?)")
+                 "insert into " table "_entity (entity_id, entity_type, unique_id, dirty, version) values (null, ?, ?, ?, ?)")
              entity-type unique-id dirty version)))
 
     ;; create the attributes if they are new, and validate them if they exist
@@ -78,7 +78,7 @@
   (db-exec db "begin transaction")
   (let ((id (db-insert
              db (string-append
-                 "insert into " table "_entity values (?, ?, ?, ?, ?)")
+                 "insert into " table "_entity (entity_id, entity_type, unique_id, dirty, version) values (?, ?, ?, ?, ?)")
              id entity-type unique-id dirty version)))
 
     ;; create the attributes if they are new, and validate them if they exist
diff --git a/eavdb/entity-values.ss b/eavdb/entity-values.ss
index d06ecaa..5d5a22b 100644
--- a/eavdb/entity-values.ss
+++ b/eavdb/entity-values.ss
@@ -41,7 +41,7 @@
       ((null? t)
        (msg "adding new attribute for" entity-type " called " key " of type " type)
        (db-insert
-        db (string-append "insert into " table "_attribute values (null, ?, ?, ?)")
+        db (string-append "insert into " table "_attribute (id, attribute_id, entity_type, attribute_type) values (null, ?, ?, ?)")
         key entity-type type)
        type)
       (else
@@ -58,7 +58,7 @@
 (define (insert-value db table entity-id ktv dirty)
   ;; use type to dispatch insert to correct value table
   (db-insert db (string-append "insert into " table "_value_" (ktv-type ktv)
-                               " values (null, ?, ?, ?, ?, 0)")
+                               "(id, entity_id, attribute_id, value, dirty, version) values (null, ?, ?, ?, ?, 0)")
              entity-id (ktv-key ktv) (ktv-value ktv) (if dirty 1 0)))
 
 ;; update the value given an entity type, a attribute type and it's key (= attriute_id)
diff --git a/web/run b/web/run
index ea3c015..008552c 100755
--- a/web/run
+++ b/web/run
@@ -1,4 +1,4 @@
 #!/bin/bash
-./server.scm 8888 >>client/htdocs/log.txt 2>&1
+./server.scm 8888 >client/htdocs/log.txt 2>&1
 
 
diff --git a/web/scripts/sql.ss b/web/scripts/sql.ss
index 1ce8c38..1ecc6ff 100644
--- a/web/scripts/sql.ss
+++ b/web/scripts/sql.ss
@@ -34,12 +34,13 @@
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; fix this mess
 (define (db-open db-name setup-fn)
   (cond
     ((file-exists? (string->path db-name))
      (display "open existing db")(newline)
      (let ((db (open (string->path db-name))))
-       ;; upgrade...
+       ;; upgrade
        (setup-fn db "sync")
        (setup-fn db "stream")
        db))
diff --git a/web/server.scm b/web/server.scm
index 5e224f0..ffe24c0 100755
--- a/web/server.scm
+++ b/web/server.scm
@@ -198,7 +198,7 @@
  start
  ;; port number is read from command line as argument
  ;; ie: ./server.scm 8080
-;; #:listen-ip "192.168.2.1"
+ #:listen-ip "192.168.2.1"
  #:port (string->number (command-line #:args (port) port))
  #:command-line? #t
  #:servlet-path "/mongoose"

From 76ffe9c701adc84b6904faf73c30e5c712768469 Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Tue, 12 Sep 2017 09:56:32 +0000
Subject: [PATCH 2/6] added sent columns to the rest of the tables

---
 eavdb/eavdb.ss | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/eavdb/eavdb.ss b/eavdb/eavdb.ss
index 72a2131..ab6db1f 100644
--- a/eavdb/eavdb.ss
+++ b/eavdb/eavdb.ss
@@ -42,13 +42,16 @@
   )
 
 ;; create eav tables (add types as required)
+;; aggregating version updates - should clean all this up
 (define (setup db table)
   (msg "db setup")
   (db-exec db (string-append "create table " table "_entity ( entity_id integer primary key autoincrement, entity_type varchar(256), unique_id varchar(256), dirty integer, version integer)"))
+  (db-exec db (string-append "alter table " table "_entity add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_entity on " table "_entity (unique_id)"))
 
 
   (db-exec db (string-append "create table " table "_attribute ( id integer primary key autoincrement, attribute_id varchar(256), entity_type varchar(256), attribute_type varchar(256))"))
+  (db-exec db (string-append "alter table " table "_attribute add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_attribute on " table "_attribute (entity_type)"))
 
 

From 3da2c97a572619dfefc2a2ba8b2f5085d3d5add1 Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Tue, 12 Sep 2017 17:25:37 +0000
Subject: [PATCH 3/6] added admin scripts for tunnelling, datapipe startup and
 battery detect shutdown

---
 pi-admin/battery_detect.py |  9 +++++++++
 pi-admin/create_tunnel.sh  | 25 +++++++++++++++++++++++++
 pi-admin/datapipe.sh       |  3 +++
 pi-admin/interfaces        | 23 +++++++++++++++++++++++
 4 files changed, 60 insertions(+)
 create mode 100644 pi-admin/battery_detect.py
 create mode 100755 pi-admin/create_tunnel.sh
 create mode 100755 pi-admin/datapipe.sh
 create mode 100644 pi-admin/interfaces

diff --git a/pi-admin/battery_detect.py b/pi-admin/battery_detect.py
new file mode 100644
index 0000000..4afed50
--- /dev/null
+++ b/pi-admin/battery_detect.py
@@ -0,0 +1,9 @@
+import smbus
+import os
+
+bus = smbus.SMBus(1)
+
+# if we are on battery power
+if bus.read_byte_data(0x69,0)==2:
+    # shut down nicely
+    os.system("sudo halt")
diff --git a/pi-admin/create_tunnel.sh b/pi-admin/create_tunnel.sh
new file mode 100755
index 0000000..2edf615
--- /dev/null
+++ b/pi-admin/create_tunnel.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+create_ssh_tunnel() {
+    /usr/bin/ssh -o "ExitOnForwardFailure yes" -N -R 2222:localhost:22 mongoose@be.fo.am &
+    if [[ $? -eq 0 ]]; then
+	echo SSH tunnel created successfully
+    else
+	echo An error occurred creating SSH tunnel. RC was $?
+    fi
+}
+create_rabbit_tunnel() {
+    /usr/bin/ssh -o "ExitOnForwardFailure yes" -N -R 5672:localhost:5672 mongoose@be.fo.am &
+    if [[ $? -eq 0 ]]; then
+	echo Rabbit tunnel created successfully
+    else
+	echo An error occurred creating a RabbitMQ tunnel. RC was $?
+    fi
+}
+
+/bin/pidof ssh
+if [[ $? -ne 0 ]]; then
+    echo Creating new ssh tunnel connection
+    create_ssh_tunnel
+    echo Creating new RabbitMQ tunnel connection
+    create_rabbit_tunnel
+fi
diff --git a/pi-admin/datapipe.sh b/pi-admin/datapipe.sh
new file mode 100755
index 0000000..c417bae
--- /dev/null
+++ b/pi-admin/datapipe.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+cd mongoose-2000/dotnet/datpipe/
+dotnet ./DataPipe.Main.dll
diff --git a/pi-admin/interfaces b/pi-admin/interfaces
new file mode 100644
index 0000000..0645dbc
--- /dev/null
+++ b/pi-admin/interfaces
@@ -0,0 +1,23 @@
+# interfaces(5) file used by ifup(8) and ifdown(8)
+
+# Please note that this file is written to be used with dhcpcd
+# For static IP, consult /etc/dhcpcd.conf and 'man dhcpcd.conf'
+
+# Include files from /etc/network/interfaces.d:
+source-directory /etc/network/interfaces.d
+
+auto lo
+iface lo inet loopback
+
+iface eth0 inet manual
+
+allow-hotplug wlan0
+iface wlan0 inet static
+	address 192.168.2.1
+	netmask 255.255.255.0
+	network 192.168.2.0
+	broadcast 192.168.1.255
+
+allow-hotplug wlan1
+iface wlan1 inet manual
+    wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf

From 0e2810e2cc1ea11d0e55a4286e6198e595071bea Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Wed, 13 Sep 2017 15:48:06 +0000
Subject: [PATCH 4/6] now shuts down UPS and Pi properly when on battery power

---
 pi-admin/battery_detect.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pi-admin/battery_detect.py b/pi-admin/battery_detect.py
index 4afed50..dea4aed 100644
--- a/pi-admin/battery_detect.py
+++ b/pi-admin/battery_detect.py
@@ -5,5 +5,8 @@
 
 # if we are on battery power
 if bus.read_byte_data(0x69,0)==2:
+    # "unconditional file safe shutdown and power OFF when battery powered"
+    bus.write_byte_data(0x6b,0x00,0xcc)
+    
     # shut down nicely
-    os.system("sudo halt")
+    #os.system("sudo halt")

From 3df3261c7b1ca4792bbe62e56bf6a0f980e3fce9 Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Sun, 17 Sep 2017 17:23:18 +0000
Subject: [PATCH 5/6] pi stuff

---
 pi-admin/crontab        | 34 ++++++++++++++++++++++++++++++++++
 pi-admin/start_adhoc.sh |  9 +++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 pi-admin/crontab
 create mode 100755 pi-admin/start_adhoc.sh

diff --git a/pi-admin/crontab b/pi-admin/crontab
new file mode 100644
index 0000000..b170886
--- /dev/null
+++ b/pi-admin/crontab
@@ -0,0 +1,34 @@
+# Edit this file to introduce tasks to be run by cron.
+# 
+# Each task to run has to be defined through a single line
+# indicating with different fields when the task will be run
+# and what command to run for the task
+# 
+# To define the time you can provide concrete values for
+# minute (m), hour (h), day of month (dom), month (mon),
+# and day of week (dow) or use '*' in these fields (for 'any').# 
+# Notice that tasks will be started based on the cron's system
+# daemon's notion of time and timezones.
+# 
+# Output of the crontab jobs (including errors) is sent through
+# email to the user the crontab file belongs to (unless redirected).
+# 
+# For example, you can run a backup of all your user accounts
+# at 5 a.m every week with:
+# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
+# 
+# For more information see the manual pages of crontab(5) and cron(8)
+# 
+# m h  dom mon dow   command
+
+@reboot supervise /home/pi/mongoose-2000/web
+
+# adding path so we can find dotnet etc
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+@reboot ~/datapipe.sh
+*/1 * * * * ~/create_tunnel.sh > tunnel.log 2>&1
+*/1 * * * * python ~/battery_detect.py
+
+@reboot sleep 15 && ~/start_adhoc.sh >> hostapd.log 2>&1
+@reboot sleep 30 && ~/start_adhoc.sh >> hostapd.log 2>&1
diff --git a/pi-admin/start_adhoc.sh b/pi-admin/start_adhoc.sh
new file mode 100755
index 0000000..3ee4ad0
--- /dev/null
+++ b/pi-admin/start_adhoc.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+sudo ifconfig wlan0 up
+sleep 5
+sudo /usr/sbin/hostapd /etc/hostapd/hostapd.conf &
+sleep 5
+sudo ifconfig wlan0 up
+sleep 5
+sudo /usr/sbin/hostapd /etc/hostapd/hostapd.conf &
+

From 6bad5dab784c5ccb9e55ed0ad0a8443ba1d9e96d Mon Sep 17 00:00:00 2001
From: dave griffiths <dave@fo.am>
Date: Tue, 19 Sep 2017 10:08:01 +0000
Subject: [PATCH 6/6] setting sent to zero when updating values, added sent to
 schema properly - but kept alter for tablets, now running unit tests all the
 time too

---
 eavdb/eavdb.ss         | 49 +++++++++++++++++++++++++++++++++---------
 eavdb/entity-update.ss |  5 +++--
 eavdb/entity-values.ss |  6 ++++--
 3 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/eavdb/eavdb.ss b/eavdb/eavdb.ss
index ab6db1f..765de22 100644
--- a/eavdb/eavdb.ss
+++ b/eavdb/eavdb.ss
@@ -33,42 +33,44 @@
 
 (provide (all-defined-out))
 
-(msg "hello from eavdb.ss")
-
 
 (define (upgrade-table db name)
   (db-exec db (string-append "alter table " name " add version integer"))
-  (db-exec db (string-append "alter table " name " add sent integer default 0"))
-  )
+  (db-exec db (string-append "alter table " name " add sent integer default 0")))
 
 ;; create eav tables (add types as required)
 ;; aggregating version updates - should clean all this up
+;;
+;; remember that this is run on both pi and android - on the tablets we don't really know
+;; how old the initial versions of the databases are, they might go right back to the beginning
+;; so need to be super conservative and alter tables to add new stuff
 (define (setup db table)
   (msg "db setup")
-  (db-exec db (string-append "create table " table "_entity ( entity_id integer primary key autoincrement, entity_type varchar(256), unique_id varchar(256), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_entity ( entity_id integer primary key autoincrement, entity_type varchar(256), unique_id varchar(256), dirty integer, version integer, sent integer default 0)"))
+  ;; throws benign errors if it exists - we catch and ignore em...
   (db-exec db (string-append "alter table " table "_entity add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_entity on " table "_entity (unique_id)"))
 
 
-  (db-exec db (string-append "create table " table "_attribute ( id integer primary key autoincrement, attribute_id varchar(256), entity_type varchar(256), attribute_type varchar(256))"))
+  (db-exec db (string-append "create table " table "_attribute ( id integer primary key autoincrement, attribute_id varchar(256), entity_type varchar(256), attribute_type varchar(256), sent integer default 0)"))
   (db-exec db (string-append "alter table " table "_attribute add sent integer default 0"))
   (db-exec db (string-append "create index if not exists index_" table "_attribute on " table "_attribute (entity_type)"))
 
 
 
-  (db-exec db (string-append "create table " table "_value_varchar ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_varchar ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_varchar"))
   (db-exec db (string-append "create index if not exists index_" table "_value_varchar on " table "_value_varchar (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_int ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value integer, dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_int ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value integer, dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_int"))
   (db-exec db (string-append "create index if not exists index_" table "_value_int on " table "_value_int (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_real ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value real, dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_real ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value real, dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_real"))
   (db-exec db (string-append "create index if not exists index_" table "_value_real on " table "_value_real (entity_id,attribute_id)"))
 
-  (db-exec db (string-append "create table " table "_value_file ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer)"))
+  (db-exec db (string-append "create table " table "_value_file ( id integer primary key autoincrement, entity_id integer, attribute_id varchar(255), value varchar(4096), dirty integer, version integer, sent integer default 0)"))
   (upgrade-table db (string-append table "_value_file"))
   (db-exec db (string-append "create index if not exists index_" table "_value_file on " table "_value_file (entity_id,attribute_id)"))
 
@@ -119,3 +121,30 @@
    (lambda (i)
      (get-entity-only db table i kt-list))
    (filter-entities-inc-deleted db table type filter)))
+
+(define (run-unit-tests)
+  (msg "running eavdb tests...")
+  (define last-test-db (open (string->path "unit-test.db")))
+  ;; clear out last test
+  (db-exec last-test-db "drop table sync_entity;")
+  (db-exec last-test-db "drop table sync_attribute;")
+  (db-exec last-test-db "drop table sync_value_varchar;")
+  (db-exec last-test-db "drop table sync_value_int;")
+  (db-exec last-test-db "drop table sync_value_real;")
+  (db-exec last-test-db "drop table sync_value_file;")
+
+  (db-exec last-test-db "drop table stream_entity;")
+  (db-exec last-test-db "drop table stream_attribute;")
+  (db-exec last-test-db "drop table stream_value_varchar;")
+  (db-exec last-test-db "drop table stream_value_int;")
+  (db-exec last-test-db "drop table stream_value_real;")
+  (db-exec last-test-db "drop table stream_value_file;")
+  ;; reopen to run setup
+  (define test-db (db-open "unit-test.db" setup))
+  (ktv-test)
+  (entity-update-test test-db "sync")
+  (entity-sync-test test-db "sync")
+  (msg "finished running eavdb tests..."))
+
+(run-unit-tests)
+  
diff --git a/eavdb/entity-update.ss b/eavdb/entity-update.ss
index 79c52d3..d7e9bca 100644
--- a/eavdb/entity-update.ss
+++ b/eavdb/entity-update.ss
@@ -28,16 +28,17 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; updating data
 
+;; setting sent=0 to update rabbitmq - probably not needed?
 (define (update-entity-changed db table entity-id)
   (db-exec
    db (string-append
-       "update " table "_entity set dirty=?, version=version+1 where entity_id = ?")
+       "update " table "_entity set dirty=?, sent=0, version=version+1 where entity_id = ?")
    1 entity-id))
 
 (define (update-entity-version db table entity-id version)
   (db-exec
    db (string-append
-       "update " table "_entity set dirty=0, version=? where entity_id = ?")
+       "update " table "_entity set dirty=0, sent=0, version=? where entity_id = ?")
    version entity-id))
 
 ;; update an entire entity (version incl), via a (possibly partial) list of key/value pairs
diff --git a/eavdb/entity-values.ss b/eavdb/entity-values.ss
index 5d5a22b..9de9ffc 100644
--- a/eavdb/entity-values.ss
+++ b/eavdb/entity-values.ss
@@ -63,6 +63,7 @@
 
 ;; update the value given an entity type, a attribute type and it's key (= attriute_id)
 ;; creates the value if it doesn't already exist, updates it otherwise if it's different
+;; setting sent=0 to work with the rabbitmq updater here - probably not needed
 (define (update-value db table entity-id ktv)
   (let ((s (select-first
             db (string-append
@@ -74,11 +75,12 @@
         (if (not (ktv-eq? ktv (list (ktv-key ktv) (ktv-type ktv) s)))
             (db-exec
              db (string-append "update " table "_value_" (ktv-type ktv)
-                               " set value=?, dirty=1  where entity_id = ? and attribute_id = ?")
+                               " set value=?, dirty=1, sent=0  where entity_id = ? and attribute_id = ?")
              (ktv-value ktv) entity-id (ktv-key ktv))
             '())))) ;;(msg "values for" (ktv-key ktv) "are the same (" (ktv-value ktv) "==" s ")")))))
 
 ;; don't make dirty or update version here
+;; setting sent=0 to work with the rabbitmq updater here
 (define (update-value-from-sync db table entity-id ktv)
   (let ((s (select-first
             db (string-append
@@ -91,7 +93,7 @@
         (insert-value db table entity-id ktv #t) ;; <- don't make dirty!?
         (db-exec
          db (string-append "update " table "_value_" (ktv-type ktv)
-                           " set value=?, dirty=0 where entity_id = ? and attribute_id = ?")
+                           " set value=?, dirty=0, sent=0 where entity_id = ? and attribute_id = ?")
          (ktv-value ktv) entity-id (ktv-key ktv)))))
 
 ;; get all the (current) attributes for an entity type