Use Python virtual environment (#203)

CHANGELOG.md

@@ -17,6 +17,7 @@
 
 ### Miscellaneous
 
+  - Use Python virtual environment
   - Refactor the relay/sensor conditional management system
   - User names are no longer case-sensitive
   - Switch to using Flask-Login

install/mycodo.service

@@ -4,8 +4,8 @@ After=syslog.target network.target influxd.service
 
 [Service]
 Type=forking
-ExecStart=/var/www/mycodo/mycodo/mycodo_daemon.py
-ExecStop=/var/www/mycodo/mycodo/mycodo_client.py -t
+ExecStart=/var/www/mycodo/env/bin/python /var/www/mycodo/mycodo/mycodo_daemon.py
+ExecStop=/var/www/mycodo/env/bin/python /var/www/mycodo/mycodo/mycodo_client.py -t
 ExecReload=/bin/kill -HUP $MAINPID
 
 [Install]

install/setup.sh

@@ -45,7 +45,10 @@ apt-get update
 apt-get purge -y python-pip
 
 /bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh update-packages
-pip install -U pip
+
+pip install --upgrade pip
+
+/bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh setup-virtualenv
 
 printf "#### Installing gpiod\n"
 cd ${INSTALL_DIRECTORY}/install
@@ -65,9 +68,7 @@ cd ${INSTALL_DIRECTORY}/install/wiringPi
 cd ${INSTALL_DIRECTORY}/install
 rm -rf ./wiringPi
 
-printf "#### Installing pip requirements from requirements.txt\n"
-cd ${INSTALL_DIRECTORY}/install
-pip install -r requirements.txt --upgrade
+/bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh upgrade-pip-packages
 
 /bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh update-influxdb
 service influxdb start

mycodo/mycodo_client.py

@@ -80,6 +80,9 @@ def daemon_status(self):
     def flash_lcd(self, lcd_id, state):
         return self.rpyc_client.root.flash_lcd(lcd_id, state)
 
+    def is_in_virtualenv(self):
+        return self.rpyc_client.root.is_in_virtualenv()
+
     def pid_hold(self, pid_id):
         return self.rpyc_client.root.pid_hold(pid_id)
 
@@ -172,7 +175,7 @@ def parseargs(parser):
         logger.info(
             "[Remote command] Check Daemon: {msg}".format(msg=return_msg))
 
-    if args.ramuse:
+    elif args.ramuse:
         return_msg = daemon_control.ram_use()
         logger.info(
             "[Remote command] Daemon Ram in Use: {msg} MB".format(msg=return_msg))

mycodo/mycodo_daemon.py

@@ -138,6 +138,13 @@ def exposed_daemon_status():
             """
             return 'alive'
 
+        @staticmethod
+        def exposed_is_in_virtualenv():
+            """Returns True if this script is running in a virtualenv"""
+            if hasattr(sys, 'real_prefix'):
+                return True
+            return False
+
         @staticmethod
         def exposed_pid_hold(pid_id):
             """Hold PID Controller operation"""

mycodo/mycodo_flask/general_routes.py

@@ -243,8 +243,8 @@ def last_data(sensor_measure, sensor_id, sensor_period):
                                           sensor_period)).raw
         number = len(raw_data['series'][0]['values'])
         time_raw = raw_data['series'][0]['values'][number - 1][0]
-        value = '{:.3f}'.format(
-            float(raw_data['series'][0]['values'][number - 1][1]))
+        value = raw_data['series'][0]['values'][number - 1][1]
+        value = '{:.3f}'.format(float(value))
         # Convert date-time to epoch (potential bottleneck for data)
         dt = date_parse(time_raw)
         timestamp = calendar.timegm(dt.timetuple()) * 1000

mycodo/mycodo_flask/page_routes.py

@@ -1,11 +1,12 @@
 # coding=utf-8
 """ collection of Page endpoints """
-import logging
-import os
 import datetime
 import flask_login
 import glob
+import logging
+import os
 import subprocess
+import sys
 import time
 from collections import OrderedDict
 
@@ -425,9 +426,10 @@ def page_info():
     ifconfig.wait()
 
     daemon_pid = subprocess.Popen(
-        "pgrep mycodo_daemon", stdout=subprocess.PIPE, shell=True)
+        "pgrep -f '/var/www/mycodo/env/bin/python /var/www/mycodo/mycodo/mycodo_daemon.py'", stdout=subprocess.PIPE, shell=True)
     (daemon_pid_output, _) = daemon_pid.communicate()
     daemon_pid.wait()
+    daemon_pid_output = daemon_pid_output.split('\n')[0]
 
     pstree = subprocess.Popen(
         "pstree -p {pid}".format(pid=daemon_pid_output), stdout=subprocess.PIPE, shell=True)
@@ -443,9 +445,15 @@ def page_info():
     for each_ver in AlembicVersion.query.all():
         database_version.append(each_ver.version_num)
 
+    virtualenv_flask = False
+    if hasattr(sys, 'real_prefix'):
+        virtualenv_flask = True
+
+    virtualenv_daemon = False
     if daemon_pid_output:
         control = DaemonControl()
         ram_use = control.ram_use()
+        virtualenv_daemon = control.is_in_virtualenv()
     else:
         ram_use = 0
 
@@ -460,7 +468,9 @@ def page_info():
                            ram_use=ram_use,
                            top=top_output,
                            uname=uname_output,
-                           uptime=uptime_output)
+                           uptime=uptime_output,
+                           virtualenv_daemon=virtualenv_daemon,
+                           virtualenv_flask=virtualenv_flask)
 
 
 @blueprint.route('/lcd', methods=('GET', 'POST'))

mycodo/mycodo_flask/templates/tools/info.html

@@ -21,11 +21,13 @@
         <br>{{_('Daemon Status:')}}
         {%- if daemon_status == "alive" %}
           <span style="color: #4E9258; font-weight: bold;">{{_('Running')}}</span>
-          <br>{{_('Daemon PID:')}} <span style="color: #4E9258; font-weight: bold;">{{daemon_pid}}</span>
+          <br>{{_('Daemon Process ID:')}} <span style="color: #4E9258; font-weight: bold;">{{daemon_pid}}</span>
           <br>{{_('Daemon RAM usage:')}} <span style="color: #4E9258; font-weight: bold;">{{ram_use}} MB</span>
+          <br>{{_('Daemon Virtualenv:')}} <span style="color: #4E9258; font-weight: bold;">{{virtualenv_daemon}}</span>
         {%- else %}
           <span style="color: #F70D1A; font-weight: bold;">{{_('Not Running')}}</span>
         {% endif %}
+        <br>{{_('Frontend Virtualenv:')}} <span style="color: #4E9258; font-weight: bold;">{{virtualenv_flask}}</span>
       </div>
     </div>
 

mycodo/scripts/upgrade_commands.sh

@@ -51,7 +51,7 @@ case "${1:-''}" in
         rm -f certificate.csr certificate.key
     ;;
     'initialize')
-        printf "\n#### Initialize: Create proper users, directories, and permissions\n"
+        printf "\n#### Creating proper users, directories, and permissions\n"
         useradd -M mycodo
         adduser mycodo gpio
         adduser mycodo adm
@@ -81,13 +81,20 @@ case "${1:-''}" in
         chown root:mycodo ${INSTALL_DIRECTORY}/Mycodo/mycodo/scripts/mycodo_wrapper
         chmod 4770 ${INSTALL_DIRECTORY}/Mycodo/mycodo/scripts/mycodo_wrapper
     ;;
+    'setup-virtualenv')
+        if [ ! -d ${INSTALL_DIRECTORY}/Mycodo/env ]; then
+            virtualenv --system-site-packages ${INSTALL_DIRECTORY}/Mycodo/env
+        else
+            printf "## Virtualenv already exists, skipping creation\n"
+        fi
+    ;;
     'update-cron')
         printf "#### Updating crontab entry\n"
         /bin/bash ${INSTALL_DIRECTORY}/Mycodo/install/crontab.sh mycodo --remove
         /bin/bash ${INSTALL_DIRECTORY}/Mycodo/install/crontab.sh mycodo
     ;;
     'update-influxdb')
-        printf "\n#### Ensure compatible version of influxdb is installed ####\n"
+        printf "\n#### Ensuring compatible version of influxdb is installed ####\n"
         INSTALL_ADDRESS="https://dl.influxdata.com/influxdb/releases/"
         INSTALL_FILE="influxdb_1.2.1_armhf.deb"
         CORRECT_VERSION="1.2.1-1"
@@ -101,13 +108,24 @@ case "${1:-''}" in
         fi
     ;;
     'update-packages')
-        printf "\n#### Installing prerequisite apt packages.\n"
+        printf "\n#### Installing prerequisite apt packages and update pip\n"
         apt-get update -y
         apt-get install -y apache2 gawk git libav-tools libffi-dev libi2c-dev python-dev python-numpy python-opencv python-setuptools python-smbus sqlite3
         easy_install pip
+        pip install pip --upgrade
+    ;;
+    'upgrade-pip-packages')
+        printf "\n#### Installing pip requirements from requirements.txt\n"
+        if [ ! -d ${INSTALL_DIRECTORY}/Mycodo/env ]; then
+            printf "\n## Error: Virtualenv doesn't exist. Create with $0 setup-virtualenv\n"
+        else
+            source ${INSTALL_DIRECTORY}/Mycodo/env/bin/activate
+            ${INSTALL_DIRECTORY}/Mycodo/env/bin/pip install pip --upgrade
+            ${INSTALL_DIRECTORY}/Mycodo/env/bin/pip install -r ${INSTALL_DIRECTORY}/Mycodo/install/requirements.txt --upgrade
+        fi
     ;;
     'update-mycodo-startup-script')
-        printf "\n#### Enable mycodo startup script\n"
+        printf "\n#### Enabling mycodo startup script\n"
         systemctl disable mycodo.service
         rm -rf /etc/systemd/system/mycodo.service
         systemctl enable ${INSTALL_DIRECTORY}/Mycodo/install/mycodo.service

mycodo/scripts/upgrade_mycodo_release.sh

@@ -100,6 +100,15 @@ runSelfUpgrade() {
   fi
   printf "Done.\n"
 
+  if [ -d ${INSTALL_DIRECTORY}/Mycodo/env ] ; then
+    printf "Moving env directory..."
+    if ! mv ${INSTALL_DIRECTORY}/Mycodo/env ${MYCODO_NEW_TMP_DIR} ; then
+      printf "Failed: Error while trying to move env directory.\n"
+      error_found
+    fi
+    printf "Done.\n"
+  fi
+
   printf "Moving databases from ${INSTALL_DIRECTORY}/Mycodo/databases/ to ${MYCODO_NEW_TMP_DIR}/databases..."
   if ! cp ${INSTALL_DIRECTORY}/Mycodo/databases/*.db ${MYCODO_NEW_TMP_DIR}/databases ; then
     printf "Failed: Error while trying to copy databases."

mycodo/scripts/upgrade_post.sh

@@ -19,6 +19,8 @@ ln -sf ${INSTALL_DIRECTORY}/install/mycodo_flask_apache.conf /etc/apache2/sites-
 
 /bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh update-packages
 
+/bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh upgrade-pip-packages
+
 /bin/bash ${INSTALL_DIRECTORY}/mycodo/scripts/upgrade_commands.sh update-influxdb
 
 printf "\n#### Checking if python modules are up-to-date\n"

mycodo_flask.wsgi

@@ -1,5 +1,8 @@
 import sys
 
+activate_this = '/var/www/mycodo/env/bin/activate_this.py'
+execfile(activate_this, dict(__file__=activate_this))
+
 sys.path.append('/var/www/mycodo/mycodo')
 
 from mycodo.start_flask_ui import app as application