shared-node (#1361)

* Upload NS status when possible. Before this checkin, ns-loop created the status data, and uploaded it to nightscout (if possible). But in case of failure, the file will get overwritten. This means that in the case of no internet, OAPS decisions get lost. This checkin fixes it by adding timestamps to the files. When internet is available, the files will be uploaded according to the correct order. Files older than a day will be deleted. Testing: I have prevented uploading of files for a few hours, and when it was enabled again, data was shown correctly in nightscout site. Decreased timeout, and saw the files being deleted. Signed-off-by: Tzachi Dar <tzachi.dar@gmail.com> * In the case that a status file does not contain iob delete it instead of uploading it. Signed-off-by: Tzachi Dar <tzachi.dar@gmail.com> * Add code to run js code on a shared server. This saves initilaztion time. Worked for a day, which means testing only started. * Fix a typo. * Code to start shared node. * Add code that verifies that shared node is answering to requests and starting it if needed. * Add code to store program input in order to test it. * fix dashes to be underscores * Add json file to shared code. * Add code to capture data of json commands. * New json funciontality, and changes from tests. * Trival changes to white spaces handeling. * Minor refactoring to the code. Signed-off-by: Tzachi Dar <tzachi.dar@gmail.com> * install socat on all platforms for shared-node * remove non-json debug output breaking json parsing * Merge branch 'tzachi-share-node-ns-status' of https://github.com/tzachi-dar/oref0 into tzachi-dar-tzachi-share-node-ns-status Conflicts: bin/oref0-ns-loop.sh * install socat on all platforms for shared-node * remove non-json debug output breaking json parsing * Remove temp files from the shared node. Signed-off-by: Tzachi Dar <tzachi.dar@gmail.com> * Stop copying data to test_data. Signed-off-by: Tzachi Dar <tzachi.dar@gmail.com> * Ad an update script and call it every 15 minutes. Co-authored-by: Tzachi Dar <tzachi.dar@gmail.com>
openaps · Mar 17, 2020 · 504a478 · 504a478
1 parent c37b8c4
commit 504a478
Show file tree

Hide file tree

Showing 14 changed files with 439 additions and 34 deletions.
diff --git a/bin/mm-format-ns-treatments.sh b/bin/mm-format-ns-treatments.sh
@@ -18,7 +18,8 @@ EOT
 # | json -e "this.type = 'mm://openaps/$self'" \
 model=$(jq -r . $MODEL)
 
-oref0-normalize-temps $HISTORY \
+
+run_remote_command "oref0-normalize-temps $HISTORY" \
   | jq '[ .[]
     | .medtronic = ( [ "mm://openaps/'$self'/", ( . | if ._type then ._type else .eventType end ) ] | join("") )
     | .created_at = if .created_at then .created_at else .timestamp end

diff --git a/bin/ns-status.js b/bin/ns-status.js
@@ -2,10 +2,11 @@
 'use strict';
 
 var os = require("os");
+var fs = require('fs');
 
 var requireUtils = require('../lib/require-utils');
-var safeRequire = requireUtils.safeRequire;
 var requireWithTimestamp = requireUtils.requireWithTimestamp;
+var safeLoadFile = requireUtils.safeLoadFile;
 
 /*
   Prepare Status info to for upload to Nightscout
@@ -23,7 +24,7 @@ var requireWithTimestamp = requireUtils.requireWithTimestamp;
 
 */
 
-function mmtuneStatus (status) {
+function mmtuneStatus (status, cwd, mmtune_input) {
     var mmtune = requireWithTimestamp(cwd + mmtune_input);
     if (mmtune) {
         if (mmtune.scanDetails && mmtune.scanDetails.length) {
@@ -35,7 +36,7 @@ function mmtuneStatus (status) {
     }
 }
 
-function preferencesStatus (status) {
+function preferencesStatus (status, cwd ,preferences_input) {
     var preferences = requireWithTimestamp(cwd + preferences_input);
     if (preferences) {
       status.preferences = preferences;
@@ -47,8 +48,8 @@ function preferencesStatus (status) {
     }
 }
 
-function uploaderStatus (status) {
-    var uploader = require(cwd + uploader_input);
+function uploaderStatus (status, cwd, uploader_input) {
+    var uploader = JSON.parse(fs.readFileSync(cwd + uploader_input, 'utf8'));
     if (uploader) {
         if (typeof uploader === 'number') {
             status.uploader = {
@@ -60,9 +61,12 @@ function uploaderStatus (status) {
     }
 }
 
-if (!module.parent) {
 
-    var argv = require('yargs')
+
+
+var ns_status = function ns_status(argv_params) {
+
+    var argv = require('yargs')(argv_params)
         .usage("$0 <clock.json> <iob.json> <suggested.json> <enacted.json> <battery.json> <reservoir.json> <status.json> [--uploader uploader.json] [mmtune.json] [--preferences preferences.json]")
         .option('preferences', {
             alias: 'p',
@@ -77,10 +81,16 @@ if (!module.parent) {
             default: false
         })
         .strict(true)
+        .fail(function (msg, err, yargs) {
+            if (err) {
+                return console.error('Error found', err);
+            }
+            return console.error('Parsing of command arguments failed', msg)
+            })
         .help('help');
-
     var params = argv.argv;
     var inputs = params._;
+
     var clock_input = inputs[0];
     var iob_input = inputs[1];
     var suggested_input = inputs[2];
@@ -94,9 +104,11 @@ if (!module.parent) {
 
     if (inputs.length < 7 || inputs.length > 8) {
         argv.showHelp();
-        process.exit(1);
+        return;
     }
 
+    // TODO: For some reason the following line does not work (../package.json ia not found).
+    //var pjson = JSON.parse(fs.readFileSync('../package.json', 'utf8'));
     var pjson = require('../package.json');
 
     var cwd = process.cwd() + '/';
@@ -138,28 +150,41 @@ if (!module.parent) {
                 version: pjson.version
             },
             pump: {
-                clock: safeRequire(cwd + clock_input),
-                battery: safeRequire(cwd + battery_input),
-                reservoir: safeRequire(cwd + reservoir_input),
+                clock: safeLoadFile(cwd + clock_input),
+                battery: safeLoadFile(cwd + battery_input),
+                reservoir: safeLoadFile(cwd + reservoir_input),
                 status: requireWithTimestamp(cwd + status_input)
             },
             created_at: new Date()
         };
 
         if (mmtune_input) {
-            mmtuneStatus(status);
+            mmtuneStatus(status, cwd, mmtune_input);
         }
 
         if (preferences_input) {
-            preferencesStatus(status);
+            preferencesStatus(status, cwd ,preferences_input);
         }
 
         if (uploader_input) {
-            uploaderStatus(status);
+            uploaderStatus(status, cwd, uploader_input);
         }
 
-        console.log(JSON.stringify(status));
+        return JSON.stringify(status);
     } catch (e) {
         return console.error("Could not parse input data: ", e);
     }
 }
+
+if (!module.parent) {
+    // remove the first parameter.
+    var command = process.argv;
+    command.shift();
+    command.shift();
+    var result = ns_status(command);
+    if(result !== undefined) {
+        console.log(result);
+    }
+}
+
+exports = module.exports = ns_status
diff --git a/bin/oref0-bash-common-functions.sh b/bin/oref0-bash-common-functions.sh
@@ -8,6 +8,45 @@ self=$(basename $0)
 
 PREFERENCES_FILE="preferences.json"
 
+function run_remote_command () {
+    set -o pipefail
+    out_file=$( mktemp /tmp/shared_node.XXXXXXXXXXXX)
+    #echo $out_file
+    echo -n $1 |socat -t90 - UNIX-CONNECT:/tmp/oaps_shared_node > $out_file || return 1
+    #cat $out_file
+    jq -j .err $out_file >&2
+    jq -j .stdout $out_file 
+    return_val=$( jq -r .return_val $out_file)
+    rm $out_file
+    return $(( return_val ))
+}
+
+function start_share_node_if_needed() {
+    # First check if node is alive
+    output="$(echo ping |socat -t90 - UNIX-CONNECT:/tmp/oaps_shared_node)"
+    echo $output
+    if [ "$output" = '{"err":"","stdout":"pong","return_val":0}' ]; then
+        echo shared node is alive
+        return 0
+    fi
+    echo 'killing node so it will restart later'
+        node_pid="$(ps -ef | grep node | grep oref0-shared-node.js | grep -v grep | awk '{print $2 }')"
+    echo $node_pid
+    kill -9 $node_pid
+    # Node should start automaticly by oref0-shared-node-loop
+    # Waiting 90 seconds for it to start
+    for i in {1..90}
+    do
+       sleep 1
+       output="$(echo ping |socat -t90 - UNIX-CONNECT:/tmp/oaps_shared_node)"
+       echo $output
+       if [ "$output" = '{"err":"","stdout":"pong","return_val":0}' ]; then
+           echo shared node is alive
+           return 0
+       fi
+    done
+    die Waiting for shared node failed 
+}
 
 function overtemp {
     # check for CPU temperature above 85°C

diff --git a/bin/oref0-cron-every-15min.sh b/bin/oref0-cron-every-15min.sh
@@ -28,3 +28,4 @@ fi
 ) &
 
 oref0-version --check-for-updates > /tmp/oref0-updates.txt &
+/root/src/oref0/bin/oref0-upgrade.sh
diff --git a/bin/oref0-cron-every-minute.sh b/bin/oref0-cron-every-minute.sh
@@ -112,6 +112,10 @@ if ! is_bash_process_running_named oref0-pump-loop; then
     oref0-pump-loop 2>&1 | tee -a /var/log/openaps/pump-loop.log | adddate openaps.pump-loop | uncolor |tee -a /var/log/openaps/openaps-date.log &
 fi
 
+if ! is_bash_process_running_named oref0-shared-node-loop; then
+    oref0-shared-node-loop 2>&1 | tee -a /var/log/openaps/shared-node.log | adddate openaps.shared-node | uncolor |tee -a /var/log/openaps/openaps-date.log &
+fi
+
 if [[ ! -z "$BT_PEB" ]]; then
     if ! is_process_running_named "peb-urchin-status $BT_PEB"; then
         peb-urchin-status $BT_PEB 2>&1 | tee -a /var/log/openaps/urchin-loop.log | adddate openaps.urchin-loop | uncolor |tee -a /var/log/openaps/openaps-date.log &
@@ -138,6 +142,7 @@ done | while read file; do
     # attempt a logrotate
     logrotate /etc/logrotate.conf -f
 done
+start_share_node_if_needed
 
 # check if 5 minutes have passed, and if yes, turn of the screen to save power
 ttyport="$(get_pref_string .ttyport)"

diff --git a/bin/oref0-get-ns-entries.js b/bin/oref0-get-ns-entries.js
@@ -49,7 +49,6 @@ if (!module.parent) {
   }
 
   var nsurl = params._.slice(1, 2).pop();
-  if (nsurl && nsurl.charAt(nsurl.length - 1) == "/") nsurl = nsurl.substr(0, nsurl.length - 1); // remove trailing slash if it exists
 
   var apisecret = params._.slice(2, 3).pop();
   var hours = Number(params._.slice(3, 4).pop());
@@ -63,6 +62,8 @@ if (!module.parent) {
     usage();
     process.exit(1);
   }
+  // remove trailing slash if it exists
+  if (nsurl && nsurl.charAt(nsurl.length - 1) == "/") nsurl = nsurl.substr(0, nsurl.length - 1);
 
   if (apisecret != null && !apisecret.startsWith("token=") && apisecret.length != 40) {
     var shasum = crypto.createHash('sha1');

diff --git a/bin/oref0-normalize-temps.js b/bin/oref0-normalize-temps.js
@@ -17,9 +17,12 @@
 var find_insulin = require('../lib/temps');
 var find_bolus = require('../lib/bolus');
 var describe_pump = require('../lib/pump');
+var fs = require('fs');
 
-if (!module.parent) {
-  var argv = require('yargs')
+
+
+var oref0_normalize_temps = function oref0_normalize_temps(argv_params) {  
+  var argv = require('yargs')(argv_params)
     .usage('$0 <pumphistory.json>')
     .demand(1)
     // error and show help if some other args given
@@ -31,13 +34,12 @@ if (!module.parent) {
 
   if (params._.length > 1) {
     argv.showHelp();
-    console.error('Too many arguments');
-    process.exit(1);
+    return console.error('Too many arguments');
   }
 
   var cwd = process.cwd()
   try {
-    var all_data = require(cwd + '/' + iob_input);
+    var all_data = JSON.parse(fs.readFileSync(cwd + '/' + iob_input));
   } catch (e) {
     return console.error("Could not parse pumphistory: ", e);
   }
@@ -50,6 +52,18 @@ if (!module.parent) {
   // treatments.sort(function (a, b) { return a.date > b.date });
 
 
-  console.log(JSON.stringify(treatments));
+  return JSON.stringify(treatments);
+}
+
+if (!module.parent) {
+   // remove the first parameter.
+   var command = process.argv;
+   command.shift();
+   command.shift();
+   var result = oref0_normalize_temps(command)
+   if(result !== undefined) {
+       console.log(result);
+   }
 }
 
+exports = module.exports = oref0_normalize_temps
diff --git a/bin/oref0-ns-loop.sh b/bin/oref0-ns-loop.sh
@@ -92,8 +92,7 @@ function glucose_fresh {
 }
 
 function find_valid_ns_glucose {
-    # TODO: use jq for this if possible
-    cat cgm/ns-glucose.json | json -c "minAgo=(new Date()-new Date(this.dateString))/60/1000; return minAgo < 10 && minAgo > -5 && this.glucose > 38"
+    run_remote_command 'json -f cgm/ns-glucose.json -c "minAgo=(new Date()-new Date(this.dateString))/60/1000; return minAgo < 10 && minAgo > -5 && this.glucose > 38"'
 }
 
 function ns_temptargets {
@@ -202,17 +201,18 @@ function upload_ns_status {
 # first parameter - ns_status file name
 function format_ns_status {
     if [ -s monitor/edison-battery.json ]; then
-        ns-status monitor/clock-zoned.json monitor/iob.json enact/suggested.json enact/enacted.json monitor/battery.json monitor/reservoir.json monitor/status.json --preferences preferences.json --uploader monitor/edison-battery.json > upload/$1
+        run_remote_command 'ns-status monitor/clock-zoned.json monitor/iob.json enact/suggested.json enact/enacted.json monitor/battery.json monitor/reservoir.json monitor/status.json --preferences preferences.json --uploader monitor/edison-battery.json' > upload/$1
     else
-        ns-status monitor/clock-zoned.json monitor/iob.json enact/suggested.json enact/enacted.json monitor/battery.json monitor/reservoir.json monitor/status.json --preferences preferences.json > upload/$1
+        run_remote_command 'ns-status monitor/clock-zoned.json monitor/iob.json enact/suggested.json enact/enacted.json monitor/battery.json monitor/reservoir.json monitor/status.json --preferences preferences.json' > upload/$1
     fi
 }
 
 #openaps format-latest-nightscout-treatments && test $(json -f upload/latest-treatments.json -a created_at eventType | wc -l ) -gt 0 && (ns-upload $NIGHTSCOUT_HOST $API_SECRET treatments.json upload/latest-treatments.json ) || echo \\\"No recent treatments to upload\\\"
 function upload_recent_treatments {
     #echo Uploading treatments
     format_latest_nightscout_treatments || die "Couldn't format latest NS treatments"
-    if test $(json -f upload/latest-treatments.json -a created_at eventType | wc -l ) -gt 0; then
+
+    if test $(jq -r '.[] |.created_at + " " + .eventType' upload/latest-treatments.json | wc -l ) -gt 0; then
         ns-upload $NIGHTSCOUT_HOST $API_SECRET treatments.json upload/latest-treatments.json | colorize_json || die "Couldn't upload latest treatments to NS"
     else
         echo "No new treatments to upload"

diff --git a/bin/oref0-setup.sh b/bin/oref0-setup.sh
@@ -1062,6 +1062,9 @@ if prompt_yn "" N; then
 
     #Moved this out of the conditional, so that x12 models will work with smb loops
     sudo apt-get -y install bc ntpdate bash-completion || die "Couldn't install bc etc."
+    # now required on all platforms for shared-node
+    echo "Installing socat and ntp..."
+    apt-get install -y socat ntp
     cd $directory || die "Can't cd $directory"
     do_openaps_import $HOME/src/oref0/lib/oref0-setup/supermicrobolus.json
 
@@ -1125,8 +1128,6 @@ if prompt_yn "" N; then
         sed -i.bak -e "s/#dtparam=i2c_arm=on/dtparam=i2c_arm=on/" /boot/config.txt
         egrep "^dtparam=i2c1=on" /boot/config.txt || echo "dtparam=i2c1=on,i2c1_baudrate=400000" >> /boot/config.txt
         echo "i2c-dev" > /etc/modules-load.d/i2c.conf
-        echo "Installing socat and ntp..."
-        apt-get install -y socat ntp
         echo "Installing pi-buttons..."
         systemctl stop pi-buttons
         cd $HOME/src && git clone git://github.com/bnielsen1965/pi-buttons.git

diff --git a/bin/oref0-shared-node-loop.sh b/bin/oref0-shared-node-loop.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source $(dirname $0)/oref0-bash-common-functions.sh || (echo "ERROR: Failed to run oref0-bash-common-functions.sh. Is oref0 correctly installed?"; exit 1)
+
+# Shared node loop.
+main() {
+    echo
+    echo Starting Shared-Node-loop at $(date):
+    while true; do
+
+        node ../src/oref0/bin/oref0-shared-node.js
+        echo Tough luck, shared node crashed. Starting it againg at  $(date)
+    done
+}
+
+usage "$@" <<EOT
+Usage: $self
+Sync data with Nightscout. Typically runs from crontab.
+EOT
+
+main "$@"