Merge remote-tracking branch 'origin/master' into ignore-glucose-null

.travis.yml

@@ -8,7 +8,7 @@ python-steps: &python-steps
   install: pip install flake8
   script:
     # stop the build if there are Python syntax errors or undefined names
-    - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+    - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
@@ -22,6 +22,5 @@ matrix:
       <<: *node_js-steps
     - python: "2.7"
       <<: *python-steps
-    - python: "3.7"
-      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
+    - python: "3.8"
       <<: *python-steps

bin/all-autosens-history.sh

@@ -35,7 +35,7 @@ main() {
                     echo $line >> parts/entries-$year-$month.json
                 fi
             done | uniq
-            for year in 2018 2017 2016; do
+            for year in 2019 2018 2017 2016; do
                 for month in 12 11 10 09 08 07 06 05 04 03 02 01; do
                     if [ -f parts/entries-$year-$month.json ]; then
                         cat parts/entries-$year-$month.json | jq -s . > $year-$month-entries.json

bin/killall-g.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Usage: killall-g command [seconds]
+# Kill the named bash script and its entire process group, like "killall -g command"
+# if a second argument is provided, works much like killall -g --older-than,
+# and looks for processes older than the specified number of seconds
+# works with bash scripts called via "bash script_name", which killall doesn't support
+# (it only lets you killall based on the process name, which is just "bash"
+# for non-bash scripts, you can specify a third argument to match against fname
+
+script=$1
+if [[ -z "$2" ]]; then
+    older_than=0
+else
+    older_than=$2
+fi
+if [[ -z "$3" ]]; then
+    process="bash"
+else
+    process=$3
+fi
+ps x -A -o pid,fname,etimes,pgid,args | egrep -v "grep|killall" | awk '$NF ~ /'$script'/' | while read pid fname etimes pgid args; do
+    #echo pid $pid, args $args, fname $fname, pgid $pgid, etimes $etimes
+    if [[ $fname == $process ]] && [ $etimes -ge $older_than ] && ps -p $pid > /dev/null; then
+        pstree -a $pid
+        echo killing $args pid $pid pgid $pgid running for $etimes seconds
+        kill -- -$pgid
+    fi;
+done;
+#ps x -O pgid,etimes | egrep -v "grep|killall" | grep $1 | tail -1 | awk '{if ($3 >= '$older_than') print $2}'  | while read pgid; do kill -- -$pgid; done

bin/ns-status.js

@@ -35,6 +35,18 @@ function mmtuneStatus (status) {
     }
 }
 
+function preferencesStatus (status) {
+    var preferences = requireWithTimestamp(cwd + preferences_input);
+    if (preferences) {
+      status.preferences = preferences;
+      if (preferences.nightscout_host) { status.preferences.nightscout_host = "redacted"; }
+      if (preferences.bt_mac) { status.preferences.bt_mac = "redacted"; }
+      if (preferences.pushover_token) { status.preferences.pushover_token = "redacted"; }
+      if (preferences.pushover_user) { status.preferences.pushover_user = "redacted"; }
+      if (preferences.pump_serial) { status.preferences.pump_serial = "redacted"; }
+    }
+}
+
 function uploaderStatus (status) {
     var uploader = require(cwd + uploader_input);
     if (uploader) {
@@ -51,7 +63,13 @@ function uploaderStatus (status) {
 if (!module.parent) {
 
     var argv = require('yargs')
-        .usage("$0 <clock.json> <iob.json> <suggested.json> <enacted.json> <battery.json> <reservoir.json> <status.json> [--uploader uploader.json] [mmtune.json]")
+        .usage("$0 <clock.json> <iob.json> <suggested.json> <enacted.json> <battery.json> <reservoir.json> <status.json> [--uploader uploader.json] [mmtune.json] [--preferences preferences.json]")
+        .option('preferences', {
+            alias: 'p',
+            nargs: 1,
+            describe: "OpenAPS preferences file",
+            default: false
+        })
         .option('uploader', {
             alias: 'u',
             nargs: 1,
@@ -71,13 +89,16 @@ if (!module.parent) {
     var reservoir_input = inputs[5];
     var status_input = inputs[6];
     var mmtune_input = inputs[7];
+    var preferences_input = params.preferences;
     var uploader_input = params.uploader;
 
     if (inputs.length < 7 || inputs.length > 8) {
         argv.showHelp();
         process.exit(1);
     }
 
+    var pjson = require('../package.json');
+
     var cwd = process.cwd() + '/';
 
     var hostname = 'unknown';
@@ -113,7 +134,8 @@ if (!module.parent) {
             openaps: {
                 iob: iob,
                 suggested: suggested,
-                enacted: enacted
+                enacted: enacted,
+                version: pjson.version
             },
             pump: {
                 clock: safeRequire(cwd + clock_input),
@@ -127,6 +149,10 @@ if (!module.parent) {
             mmtuneStatus(status);
         }
 
+        if (preferences_input) {
+            preferencesStatus(status);
+        }
+
         if (uploader_input) {
             uploaderStatus(status);
         }

bin/openaps-bootstrap.sh

@@ -33,6 +33,6 @@ sleep 5
 echo "Press Enter to continue installing the current release (master) of oref0,"
 read -p "or enter the oref0 branch name to install." -r
 BRANCH=${REPLY:-master}
-curl https://raw.githubusercontent.com/openaps/oref0/$BRANCH/bin/openaps-install.sh > /tmp/openaps-install.sh $BRANCH
-bash /tmp/openaps-install.sh
+curl https://raw.githubusercontent.com/openaps/oref0/$BRANCH/bin/openaps-install.sh > /tmp/openaps-install.sh
+bash /tmp/openaps-install.sh $BRANCH
 )

bin/openaps-install.sh

@@ -9,16 +9,28 @@ sed -r -i"" "s/localhost( jubilinux)?$/localhost $myrighostname/" /etc/hosts
 sed -r -i"" "s/127.0.1.1.*$/127.0.1.1       $myrighostname/" /etc/hosts
 
 # if passwords are old, force them to be changed at next login
-passwd -S edison 2>/dev/null | grep 20[01][0-6] && passwd -e root
+passwd -S root 2>/dev/null | grep 20[01][0-6] && passwd -e root
 # automatically expire edison account if its password is not changed in 3 days
 passwd -S edison 2>/dev/null | grep 20[01][0-6] && passwd -e edison -i 3
 
-if [ -e /run/sshwarn ] ; then
-    echo Please select a secure password for ssh logins to your rig:
-    echo 'For the "root" account:'
-    passwd root
-    echo 'And for the "pi" account (same password is fine):'
-    passwd pi
+# Password checking for Raspbian
+if test -f /etc/os-release && grep -q Raspbian /etc/os-release && test -f /boot/issue.txt ; then
+    if [[ "$(awk -F'[ -]' '/Raspberry/ {print $5"/"$6"/"$4}' /boot/issue.txt)" == "$(sudo passwd -S root|awk '{print $3}')" ]]; then 
+        # Password of 'root' user has the same date as the reference build date. Change it.
+        passwdPrompt=1
+        echo "Please select a secure password for ssh logins to your rig (same password for multiple accounts is fine):"
+        echo 'For the "root" account:'
+        sudo passwd root 
+    fi
+    if [[ "$(awk -F'[ -]' '/Raspberry/ {print $5"/"$6"/"$4}' /boot/issue.txt)" == "$(sudo passwd -S pi|awk '{print $3}')" ]]; then 
+        # Password of 'pi' user has the same date as the reference build date. Change it.
+        # If we haven't already prompted with the following text, display it.
+        test ${passwdPrompt:-0} -ne 1 && 
+            echo "Please select a secure password for ssh logins to your rig (same password for multiple accounts is fine):"
+        echo 'For the "pi" account:'
+        sudo passwd pi 
+    fi
+    unset passwdPrompt
 fi
 
 # set timezone
@@ -58,10 +70,8 @@ fi
 sed -i "s/daily/hourly/g" /etc/logrotate.conf
 sed -i "s/#compress/compress/g" /etc/logrotate.conf
 
-# Change the openaps-packages.sh curl command to the following before merging dev to master:
-#curl -s https://raw.githubusercontent.com/openaps/oref0/$BRANCH/bin/openaps-packages.sh | bash -
-curl -s https://raw.githubusercontent.com/openaps/oref0/dev/bin/openaps-packages.sh | bash -
-mkdir -p ~/src; cd ~/src && git clone git://github.com/openaps/oref0.git ; (cd oref0 && git checkout $BRANCH && git pull)
+curl -s https://raw.githubusercontent.com/openaps/oref0/$BRANCH/bin/openaps-packages.sh | bash -
+mkdir -p ~/src; cd ~/src && ls -d oref0 && (cd oref0 && git checkout $BRANCH && git pull) || git clone git://github.com/openaps/oref0.git
 echo "Press Enter to run oref0-setup with the current release ($BRANCH branch) of oref0,"
 read -p "or press ctrl-c to cancel. " -r
 cd && ~/src/oref0/bin/oref0-setup.sh

bin/openaps-packages.sh

@@ -24,7 +24,7 @@ if ! nodejs --version | grep -e 'v8\.' -e 'v1[02468]\.' &> /dev/null ; then
         # nodesource doesn't support armv6
         if ! arch | grep -e 'armv6' &> /dev/null ; then
             sudo bash -c "curl -sL https://deb.nodesource.com/setup_8.x | bash -" || die "Couldn't setup node 8"
-            sudo apt-get install -y nodejs || die "Couldn't install nodejs"
+            sudo apt-get install -y nodejs=8.* || die "Couldn't install nodejs"
         else
             sudo apt-get install -y nodejs npm || die "Couldn't install nodejs and npm"
             npm install npm@latest -g || die "Couldn't update npm"

bin/oref0-autotune.py

@@ -156,7 +156,7 @@ def get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory):
     date_list = [start_date + datetime.timedelta(days=x) for x in range(0, (end_date - start_date).days)]
 
     for date in date_list:
-        url="{0}/api/v1/entries/sgv.json?find\[date\]\[\$gte\]={1}&find\[date\]\[\$lte\]={1}`&count=1000"
+        url="{0}/api/v1/entries/sgv.json?find\[date\]\[\$gte\]={1}&find\[date\]\[\$lte\]={1}`&count=1500"
         url = url.format(nightscout_host, date)
         #TODO: Add ability to use API secret for Nightscout.
         res = requests.get(url)

bin/oref0-autotune.sh

@@ -203,7 +203,7 @@ echo "Grabbing NIGHTSCOUT treatments.json and entries/sgv.json for date range...
 for i in "${date_list[@]}"
 do 
     # pull CGM data from 4am-4am
-    query="find%5Bdate%5D%5B%24gte%5D=$(to_epochtime "$i +4 hours" |nonl; echo 000)&find%5Bdate%5D%5B%24lte%5D=$(to_epochtime "$i +28 hours" |nonl; echo 000)&count=1000"
+    query="find%5Bdate%5D%5B%24gte%5D=$(to_epochtime "$i +4 hours" |nonl; echo 000)&find%5Bdate%5D%5B%24lte%5D=$(to_epochtime "$i +28 hours" |nonl; echo 000)&count=1500"
     echo Query: $NIGHTSCOUT_HOST entries/sgv.json $query
     ns-get host $NIGHTSCOUT_HOST entries/sgv.json $query > ns-entries.$i.json || die "Couldn't download ns-entries.$i.json"
     ls -la ns-entries.$i.json || die "No ns-entries.$i.json downloaded"

bin/oref0-bluetoothup.sh

@@ -11,6 +11,28 @@ adapter=$(get_pref_string .bt_hci 2>/dev/null) || adapter=0
 
 DAEMON_PATHS=(/usr/local/bin/bluetoothd /usr/libexec/bluetooth/bluetoothd /usr/sbin/bluetoothd)
 
+# wait up to 3 seconds for hci name to be set
+function wait_for_hci_name {
+   max_wait_seconds=3
+   elapsed_seconds=0
+   while (( elapsed_seconds < max_wait_seconds )) && ! ( hciconfig -a hci${adapter} | grep -q "$HOSTNAME" )
+   do
+      sleep 1
+      elapsed_seconds=$((elapsed_seconds + 1))
+   done
+   echo "$(date) Waited $elapsed_seconds second(s) for hci name to be set"
+}
+
+function stop_bluetooth {
+   echo "$(date) Stopping bluetoothd..."
+   if is_debian_jessie ; then
+      sudo killall bluetoothd
+   else
+      sudo systemctl stop bluetooth
+   fi
+   echo "$(date) Stopped bluetoothd"
+}
+
 for EXEC_PATH in ${DAEMON_PATHS[@]}; do
   if [ -x $EXEC_PATH ]; then
     EXECUTABLE=$EXEC_PATH
@@ -23,24 +45,43 @@ if [ "$DEBUG" != "" ]; then
 fi
 
 # start bluetoothd if bluetoothd is not running
+# Added a bunch of if is_debian_jessie checks as stretch seems to behave better here.
+
 if ! ( ps -fC bluetoothd >/dev/null ) ; then
-   echo bluetoothd not running! Starting bluetoothd...
-   sudo $EXECUTABLE 2>&1 | tee -a /var/log/openaps/bluetoothd.log &
+   if is_debian_jessie ; then
+      echo bluetoothd not running! Starting bluetoothd.
+      sudo $EXECUTABLE 2>&1 | tee -a /var/log/openaps/bluetoothd.log &
+   else
+      echo bluetoothd not running! Starting bluetoothd via systemctl.
+      sudo systemctl start bluetooth
+   fi
 fi
 
 if is_edison && ! ( hciconfig -a hci${adapter} | grep -q "PSCAN" ) ; then
-   echo Bluetooth PSCAN not enabled! Restarting bluetoothd...
-   sudo killall bluetoothd
-   sudo $EXECUTABLE 2>&1 | tee -a /var/log/openaps/bluetoothd.log &
+   if is_debian_jessie ; then
+      echo Bluetooth PSCAN not enabled! Restarting bluetoothd...
+      sudo killall bluetoothd 
+      sudo $EXECUTABLE 2>&1 | tee -a /var/log/openaps/bluetoothd.log &
+   else
+      echo Bluetooth PSCAN not enabled! Restarting bluetoothd via systemctl...
+      sudo systemctl restart bluetooth
+   fi
 fi
 
 if ( hciconfig -a hci${adapter} | grep -q "DOWN" ) ; then
+   # Not sure that this is needed on Stretch, add an is_debian_jessie check here if something different required.
    echo Bluetooth hci DOWN! Bringing it to UP.
    sudo hciconfig hci${adapter} up
-   sudo $EXECUTABLE 2>&1 | tee -a /var/log/openaps/bluetoothd.log &
-fi
+ fi
 
 if !( hciconfig -a hci${adapter} | grep -q $HOSTNAME ) ; then
+   # Not sure that this is needed on Stretch, add an is_debian_jessie check here if something different required.
    echo Bluetooth hci name does not match hostname: $HOSTNAME. Setting bluetooth hci name.
    sudo hciconfig hci${adapter} name $HOSTNAME
+   wait_for_hci_name
+   if ! ( hciconfig -a hci${adapter} | grep -q "$HOSTNAME" ) ; then
+      hciconfig -a hci${adapter}
+      echo "$(date) Failed to set bluetooth hci name. Stop bluetoothd and allow next cycle to handle restart."
+      stop_bluetooth
+   fi
 fi

bin/oref0-cron-every-minute.sh

@@ -42,20 +42,20 @@ if ! is_bash_process_running_named "oref0-online $BT_MAC"; then
     oref0-online "$BT_MAC" 2>&1 >> /var/log/openaps/network.log &
 fi
 
-sudo wpa_cli scan &
+sudo wpa_cli -i wlan0 scan &
 
 (
     killall -g --older-than 30m openaps
-    killall -g --older-than 30m oref0-pump-loop
+    killall-g oref0-pump-loop 1800
     killall -g --older-than 30m openaps-report
-    killall -g --older-than 10m oref0-g4-loop
+    killall-g oref0-g4-loop 600
 ) &
 
 # kill pump-loop after 5 minutes of not writing to pump-loop.log
 find /var/log/openaps/pump-loop.log -mmin +5 | grep pump && (
     echo No updates to pump-loop.log in 5m - killing processes
     killall -g --older-than 5m openaps
-    killall -g --older-than 5m oref0-pump-loop
+    killall-g oref0-pump-loop 300
     killall -g --older-than 5m openaps-report
 ) | tee -a /var/log/openaps/pump-loop.log | adddate openaps.pump-loop | uncolor |tee -a /var/log/openaps/openaps-date.log &
 

bin/oref0-cron-nightly.sh

@@ -5,8 +5,9 @@ source $(dirname $0)/oref0-bash-common-functions.sh || (echo "ERROR: Failed to r
 usage "$@" <<EOT
 Usage: $self
 Do all the things that oref0 does once per night, based on config files.
-Currently this just means autotune. This should run from cron, in the
-myopenaps directory. By default, this happens at 4:05am every night.
+Currently this just means autotune and uploading profile to Nightscout. 
+This should run from cron, in the myopenaps directory. By default, this 
+happens at 4:05am every night.
 EOT
 
 assert_cwd_contains_ini
@@ -18,5 +19,9 @@ directory="$PWD"
 if [[ $ENABLE =~ autotune ]]; then
     # autotune nightly at 4:05am using data from NS
     echo "=== Running Autotune at `date` === " | tee -a /var/log/openaps/autotune.log
-    (oref0-autotune -d=$directory -n=$NIGHTSCOUT_HOST && cat $directory/autotune/profile.json | jq . | grep -q start && cp $directory/autotune/profile.json $directory/settings/autotune.json) 2>&1 | tee -a /var/log/openaps/autotune.log &
+    (oref0-autotune -d=$directory -n=$NIGHTSCOUT_HOST && \
+        cat $directory/autotune/profile.json | jq . | grep -q start && \
+        cp $directory/autotune/profile.json $directory/settings/autotune.json && \
+        echo Sleeping 60s to let oref0-ns-loop generate a new profile.json to upload && sleep 60 && \
+        oref0-upload-profile settings/profile.json $NIGHTSCOUT_HOST $API_SECRET) 2>&1 | tee -a /var/log/openaps/autotune.log &
 fi