RMQ version update, Mongo version update, Bug fixes

.gitlab-ci.yml

@@ -1,46 +1,95 @@
-# This file is a template, and might need editing before it works on your project.
-# You can copy and paste this template into a new `.gitlab-ci.yml` file.
-# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
-#
-# To contribute improvements to CI/CD templates, please follow the Development guide at:
-# https://docs.gitlab.com/ee/development/cicd/templates.html
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
-
-# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options
-
-# you can delete this line if you're not using Docker
-# image: busybox:latest
-
-22.04 job:
-    tags:
-        - ubuntu2204
-
-    before_script:
-    - python3 bootstrap.py --all
-
-    after_script:
-    - rm -rf dist 
-    - rm -rf /tmp/tmp* 
-    - rm -rf env
-
-    script:
-        - env/bin/activate
-        - pytest volttrontesting
-
-20.04 job:
-    tags:
-        - ubuntu2004
-
-    before_script:
-    - python3 bootstrap.py --all
-
-    after_script:
-    - rm -rf dist 
-    - rm -rf /tmp/tmp* 
-    - rm -rf env
-
-    script:
-        - env/bin/activate
-        - pytest volttrontesting
-
+# This file is a template, and might need editing before it works on your project.
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
+
+# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options
+
+# you can delete this line if you're not using Docker
+# image: busybox:latest
+
+stages:
+    - build
+    - test
+
+.parallel-tests:
+    parallel:
+        matrix:
+            - TEST:
+                - services/core/ActuatorAgent/tests
+                - services/core/DataMover/tests/
+                - services/core/DNP3OutstationAgent/tests
+                - services/core/OpenADRVenAgent/tests
+                - services/core/PlatformDriverAgent/tests
+                - services/core/SQLHistorian/tests
+                - services/core/VolttronCentral/tests
+                - services/core/VolttronCentralPlatform/tests
+                - services/core/WeatherDotGov/tests
+                - services/ops
+                - volttrontesting/gevent/yield_test.py
+                - volttrontesting/platform/auth_tests
+                - volttrontesting/platform/control_tests
+                - volttrontesting/platform/dbutils
+                - volttrontesting/platform/web
+                - volttrontesting/platform/test_basehistorian.py
+                - volttrontesting/platform/test_connection.py
+                - volttrontesting/platform/test_core_agent.py
+                - volttrontesting/platform/test_instance_setup.py
+                - volttrontesting/platform/test_keystore.py
+                - volttrontesting/platform/test_packaging.py
+                - volttrontesting/platform/test_platform_init.py
+                - volttrontesting/platform/test_platform_rmq.py
+                - volttrontesting/platform/test_platform_web.py
+                - volttrontesting/platform/test_rmq_platform_shutdown.py
+                - volttrontesting/platform/test_sqlite3_fix.py
+                - volttrontesting/services/historian
+                - volttrontesting/services/aggregate_historian
+                - volttrontesting/services/tagging
+                - volttrontesting/services/weather
+                - volttrontesting/services/test_pubsub_service.py
+                - volttrontesting/subsystems
+                - volttrontesting/testutils
+                - volttrontesting/zmq
+
+build 20.04:
+    stage: build
+    tags:
+        - ubuntu2004
+
+    before_script:
+        #- killall -9 volttron beam.smp python
+        - rm -rf dist ~/.volttron ~/.volttron_instances
+        - rm -rf /tmp/tmp*
+        - rm -rf ~/rabbitmq_server
+
+    script:
+        - python3 bootstrap.py --all
+        - source env/bin/activate
+        - python3 bootstrap.py --rabbitmq
+        - echo "BUILD_DIR_20_04=`pwd`" >> build.env
+        - echo "$BUILD_DIR_20_04"
+        - echo `pwd`
+
+    artifacts:
+        reports:
+            dotenv: build.env
+
+test 20.04:
+    stage: test
+    needs: [build 20.04]
+    variables:
+            GIT_CHECKOUT: "false"
+    tags:
+        - ubuntu2004
+    extends: .parallel-tests
+    script:
+            - cd $BUILD_DIR_20_04
+            - echo `pwd`
+            - source env/bin/activate
+            - pytest $TEST
+
+

README.md

@@ -113,36 +113,40 @@ You can deactivate the environment at any time by running `deactivate`.
 
 #### Steps for RabbitMQ
 
-##### 1. Install Erlang version 24 packages
+##### 1. Install Erlang version 25 packages
 
-For RabbitMQ based VOLTTRON, some RabbitMQ specific software packages must be installed.
-
-###### On Debian based systems and CentOS 6/7
-
-If you are running an Debian or CentOS system, you can install the RabbitMQ dependencies by running the rabbit 
-  dependencies script, passing in the OS name and appropriate distribution as parameters. The following are supported:
-
-- `debian focal` (for Ubuntu 20.04)
+###### Install Erlang pre-requisites
+```shell
+sudo apt-get update
+sudo apt-get install -y gnupg apt-transport-https libsctp1 
+```
 
-- `debian bionic` (for Ubuntu 18.04)
+###### Purge previous versions of Erlang
+```shell
+sudo apt-get purge -yf erlang-base
+```
 
-- `debian stretch` (for Debian Stretch)
+###### Install Erlang
 
-- `debian buster` (for Debian Buster)
+Download and install ErlangOTP from [Erlang Solutions](https://www.erlang-solutions.com/downloads/).
+RMQ uses components - ssl, public_key, asn1, and crypto. These are by default included in the OTP
+RabbitMQ 3.9.29 is compatible with Erlang versions 24.3.4.2 to 25.2. VOLTTRON was tested with Erlang version 25.2-1
 
-- `raspbian buster` (for Raspbian/Raspberry Pi OS buster)
+Example:
 
-Example command:
+On Ubuntu 22.04:
 
-```sh
-./scripts/rabbit_dependencies.sh debian xenial
+```shell
+wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~jammy_amd64.deb
+sudo dpkg -i esl-erlang_25.2-1~ubuntu~jammy_amd64.deb
 ```
 
-###### Alternatively
+On Ubuntu 20.04:
+```shell
+wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~focal_amd64.deb
+sudo dpkg -i esl-erlang_25.2-1~ubuntu~focal_amd64.deb
+```
 
-You can download and install Erlang from [Erlang Solutions](https://www.erlang-solutions.com/resources/download.html).
-Please include OTP/components - ssl, public_key, asn1, and crypto.
-Also lock your version of Erlang using the [yum-plugin-versionlock](https://access.redhat.com/solutions/98873)
 
 ##### 2. Configure hostname
 
@@ -155,9 +159,13 @@ connect to empd (port 4369) on <hostname>." Note: RabbitMQ startup error would s
 and not in RabbitMQ logs (/var/log/rabbitmq/rabbitmq@hostname.log)
 
 ##### 3. Bootstrap
+Remove older version of rabbitmq_server directory. Defaults to <user_home>/rabbitmq_server/rabbitmq_server-3.9.7
+
+Run the rabbitmq boostrap command within an activated VOLTTRON environment
 
 ```sh
 cd volttron
+source env/bin/activate
 python3 bootstrap.py --rabbitmq [optional install directory. defaults to
 <user_home>/rabbitmq_server]
 ```
@@ -175,7 +183,7 @@ it needs to be set to the RabbitMQ installation directory (default path is
 `<user_home>/rabbitmq_server/rabbitmq_server-<rmq-version>`)
 
 ```sh
-echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.9.7'|sudo tee --append ~/.bashrc
+echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.9.29'|sudo tee --append ~/.bashrc
 source ~/.bashrc
 
 $RABBITMQ_HOME/sbin/rabbitmqctl status
@@ -232,7 +240,7 @@ Your VOLTTRON_HOME currently set to: /home/vdev/new_vhome2
 
 Is this the volttron you are attempting to setup?  [Y]:
 Creating rmq config yml
-RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.9.7]:
+RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.9.29]:
 Fully qualified domain name of the system: [cs_cbox.pnl.gov]:
 
 Enable SSL Authentication: [Y]:
@@ -252,7 +260,7 @@ AMQPS (SSL) port RabbitMQ address: [5671]:
 https port for the RabbitMQ management plugin: [15671]:
 INFO:rmq_setup.pyc:Starting rabbitmq server
 Warning: PID file not written; -detached was passed.
-INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.7
+INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29
 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost
 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost
 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost
@@ -266,7 +274,7 @@ INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1):
 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost
 INFO:rmq_setup.pyc:**Stopped rmq server
 Warning: PID file not written; -detached was passed.
-INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.7
+INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29
 INFO:rmq_setup.pyc:
 
 #######################

bootstrap.py

@@ -86,7 +86,7 @@
 
 _WINDOWS = sys.platform.startswith('win')
 default_rmq_dir = os.path.join(os.path.expanduser("~"), "rabbitmq_server")
-rmq_version = "3.9.7"
+rmq_version = "3.9.29"
 rabbitmq_server = f"rabbitmq_server-{rmq_version}"
 
 

ci-integration/run-tests.sh

@@ -46,12 +46,12 @@ echo "bootstrapping RABBITMQ"
 python bootstrap.py --rabbitmq --market
 
 echo "rabbitmq status"
-"$HOME/rabbitmq_server/rabbitmq_server-3.9.7/sbin/rabbitmqctl" status
+"$HOME/rabbitmq_server/rabbitmq_server-3.9.29/sbin/rabbitmqctl" status
 
 echo "TestDirs"
 for dir in $testdirs; do
   echo "*********TESTDIR: $dir"
-  py.test -s -v "$dir"
+  pytest -s -v "$dir"
 
   tmp_code=$?
   exit_code=$tmp_code
@@ -79,7 +79,7 @@ for dir in $splitdirs; do
 
         if [ -d "${D}" ]; then
             echo "*********SPLITDIR: $D"
-            py.test -s -v "${D}"
+            pytest -s -v "${D}"
             tmp_code=$?
             if [ $tmp_code -ne 0 ]; then
               if [ $tmp_code -ne 5 ]; then
@@ -100,7 +100,7 @@ for dir in $filedirs; do
   for testfile in "$dir"/*.py; do
     echo "Using testfile: $testfile"
     if [ "$testfile" != "volttrontesting/platform/packaging-tests.py" ]; then
-       py.test -s -v "$testfile"
+       pytest -s -v "$testfile"
 
        tmp_code=$?
        exit_code=$tmp_code

ci-integration/virtualization/Dockerfile

@@ -31,8 +31,8 @@ RUN chmod +x /startup/entrypoint.sh && \
 USER $VOLTTRON_USER
 RUN mkdir $RMQ_ROOT
 RUN set -eux \
-    && wget -P $VOLTTRON_USER_HOME https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.9.7/rabbitmq-server-generic-unix-3.9.7.tar.xz \
-    && tar -xf $VOLTTRON_USER_HOME/rabbitmq-server-generic-unix-3.9.7.tar.xz --directory $RMQ_ROOT \
+    && wget -P $VOLTTRON_USER_HOME https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.9.29/rabbitmq-server-generic-unix-3.9.29.tar.xz \
+    && tar -xf $VOLTTRON_USER_HOME/rabbitmq-server-generic-unix-3.9.29.tar.xz --directory $RMQ_ROOT \
     && $RMQ_HOME/sbin/rabbitmq-plugins enable rabbitmq_management rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management rabbitmq_auth_mechanism_ssl rabbitmq_trust_store
 RUN python3 -m pip install gevent-pika --user
 ############################################

docs/source/deploying-volttron/multi-platform/multi-platform-multi-bus.rst

@@ -83,7 +83,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an
     Is this the volttron you are attempting to setup? [Y]:
     What type of message bus (rmq/zmq)? [zmq]: rmq
     Name of this volttron instance: [volttron1]: central
-    RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.7]:
+    RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.29]:
     Fully qualified domain name of the system: [central]:
     Would you like to create a new self signed root CAcertificate for this instance: [Y]:
 
@@ -95,7 +95,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an
         Organization Unit: volttron
     Do you want to use default values for RabbitMQ home, ports, and virtual host: [Y]:
     2020-04-13 13:29:36,347 rmq_setup.py INFO: Starting RabbitMQ server
-    2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.7 is running at
+    2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at
     2020-04-13 13:29:46,554 volttron.utils.rmq_mgmt DEBUG: Creating new VIRTUAL HOST: volttron
     2020-04-13 13:29:46,582 volttron.utils.rmq_mgmt DEBUG: Create READ, WRITE and CONFIGURE permissions for the user: central-admin
     Create new exchange: volttron, {'durable': True, 'type': 'topic', 'arguments': {'alternate-exchange': 'undeliverable'}}
@@ -108,7 +108,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an
     2020-04-13 13:29:46,601 rmq_setup.py INFO: Creating root ca with the following info: {'C': 'US', 'ST': 'WA', 'L': 'Richland', 'O': 'PNNL', 'OU': 'VOLTTRON', 'CN': 'central-root-ca'}
     Created CA cert
     2020-04-13 13:29:49,668 rmq_setup.py INFO: **Stopped rmq server
-    2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.7 is running at
+    2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at
     2020-04-13 13:30:00,557 rmq_setup.py INFO:
 
     #######################
@@ -443,7 +443,7 @@ name is set to "collector2".
     Is this the volttron you are attempting to setup? [Y]:
     What type of message bus (rmq/zmq)? [zmq]: rmq
     Name of this volttron instance: [volttron1]: collector2
-    RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.7]:
+    RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.29]:
     Fully qualified domain name of the system: [node-rmq]:
     Would you like to create a new self signed root CA certificate for this instance: [Y]:
 
@@ -455,7 +455,7 @@ name is set to "collector2".
         Organization Unit: volttron
     Do you want to use default values for RabbitMQ home, ports, and virtual host: [Y]:
     2020-04-13 13:29:36,347 rmq_setup.py INFO: Starting RabbitMQ server
-    2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.7 is running at
+    2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at
     2020-04-13 13:29:46,554 volttron.utils.rmq_mgmt DEBUG: Creating new VIRTUAL HOST: volttron
     2020-04-13 13:29:46,582 volttron.utils.rmq_mgmt DEBUG: Create READ, WRITE and CONFIGURE permissions for the user: collector2-admin
     Create new exchange: volttron, {'durable': True, 'type': 'topic', 'arguments': {'alternate-exchange': 'undeliverable'}}
@@ -468,7 +468,7 @@ name is set to "collector2".
     2020-04-13 13:29:46,601 rmq_setup.py INFO: Creating root ca with the following info: {'C': 'US', 'ST': 'WA', 'L': 'Richland', 'O': 'PNNL', 'OU': 'VOLTTRON', 'CN': 'collector2-root-ca'}
     Created CA cert
     2020-04-13 13:29:49,668 rmq_setup.py INFO: **Stopped rmq server
-    2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.7 is running at
+    2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at
     2020-04-13 13:30:00,557 rmq_setup.py INFO:
 
     #######################

docs/source/developing-volttron/developing-agents/agent-development.rst

@@ -918,7 +918,7 @@ VOLTTRON uses *pytest* as a framework for executing tests.  All unit tests shoul
 For instructions on writing unit and integration tests with *pytest*, refer to the
 :ref:`Writing Agent Tests <Writing-Agent-Tests>` documentation.
 
-*pytest* is not installed with the distribution by default. To install py.test and it's dependencies execute the
+*pytest* is not installed with the distribution by default. To install pytest and it's dependencies execute the
 following:
 
 .. code-block:: bash

docs/source/developing-volttron/development-environment/pycharm/index.rst

@@ -6,7 +6,7 @@ Pycharm Development Environment
 
 Pycharm is an IDE dedicated to developing python projects. It provides coding
 assistance and easy access to debugging tools as well as integration with
-py.test. It is a popular tool for working with VOLTTRON.
+pytest. It is a popular tool for working with VOLTTRON.
 Jetbrains provides a free community version that can be downloaded from
 https://www.jetbrains.com/pycharm/
 
@@ -98,8 +98,8 @@ top level source directory; git will ignore changes to these files.
 Testing an Agent
 ----------------
 
-Agent tests written in py.test can be run simply by right-clicking the tests
-directory and selecting `Run 'py.test in tests`, so long as the root directory
+Agent tests written in pytest can be run simply by right-clicking the tests
+directory and selecting `Run 'pytest in tests`, so long as the root directory
 is set as the VOLTTRON source root.
 
 |Run Tests|